[bug-fix] Make StatsReporter thread-safe

com.unity.ml-agents/CHANGELOG.md

@@ -36,6 +36,8 @@ empty string). (#4155)
 - Fixed an error when setting `initialize_from` in the trainer confiiguration YAML to
 `null`. (#4175)
 - Fixed issue with FoodCollector, Soccer, and WallJump when playing with keyboard. (#4147, #4174)
+- Fixed a crash in StatsReporter when using threaded trainers with very frequent summary writes
+(#4201)
 
 ## [1.1.0-preview] - 2020-06-10
 ### Major Changes

ml-agents/mlagents/trainers/stats.py

@@ -6,6 +6,7 @@
 import csv
 import os
 import time
+from threading import RLock
 
 from mlagents_envs.logging_util import get_logger
 from mlagents_envs.timers import set_gauge
@@ -290,6 +291,7 @@ def _get_filepath(self, category: str) -> str:
 class StatsReporter:
     writers: List[StatsWriter] = []
     stats_dict: Dict[str, Dict[str, List]] = defaultdict(lambda: defaultdict(list))
+    lock = RLock()
 
     def __init__(self, category: str):
         """
@@ -302,7 +304,8 @@ def __init__(self, category: str):
 
     @staticmethod
     def add_writer(writer: StatsWriter) -> None:
-        StatsReporter.writers.append(writer)
+        with StatsReporter.lock:
+            StatsReporter.writers.append(writer)
 
     def add_property(self, property_type: StatsPropertyType, value: Any) -> None:
         """
@@ -313,16 +316,18 @@ def add_property(self, property_type: StatsPropertyType, value: Any) -> None:
         :param key: The type of property.
         :param value: The property itself.
         """
-        for writer in StatsReporter.writers:
-            writer.add_property(self.category, property_type, value)
+        with StatsReporter.lock:
+            for writer in StatsReporter.writers:
+                writer.add_property(self.category, property_type, value)
 
     def add_stat(self, key: str, value: float) -> None:
         """
         Add a float value stat to the StatsReporter.
         :param key: The type of statistic, e.g. Environment/Reward.
         :param value: the value of the statistic.
         """
-        StatsReporter.stats_dict[self.category][key].append(value)
+        with StatsReporter.lock:
+            StatsReporter.stats_dict[self.category][key].append(value)
 
     def set_stat(self, key: str, value: float) -> None:
         """
@@ -331,7 +336,8 @@ def set_stat(self, key: str, value: float) -> None:
         :param key: The type of statistic, e.g. Environment/Reward.
         :param value: the value of the statistic.
         """
-        StatsReporter.stats_dict[self.category][key] = [value]
+        with StatsReporter.lock:
+            StatsReporter.stats_dict[self.category][key] = [value]
 
     def write_stats(self, step: int) -> None:
         """
@@ -340,14 +346,15 @@ def write_stats(self, step: int) -> None:
         and the buffer cleared.
         :param step: Training step which to write these stats as.
         """
-        values: Dict[str, StatsSummary] = {}
-        for key in StatsReporter.stats_dict[self.category]:
-            if len(StatsReporter.stats_dict[self.category][key]) > 0:
-                stat_summary = self.get_stats_summaries(key)
-                values[key] = stat_summary
-        for writer in StatsReporter.writers:
-            writer.write_stats(self.category, values, step)
-        del StatsReporter.stats_dict[self.category]
+        with StatsReporter.lock:
+            values: Dict[str, StatsSummary] = {}
+            for key in StatsReporter.stats_dict[self.category]:
+                if len(StatsReporter.stats_dict[self.category][key]) > 0:
+                    stat_summary = self.get_stats_summaries(key)
+                    values[key] = stat_summary
+            for writer in StatsReporter.writers:
+                writer.write_stats(self.category, values, step)
+            del StatsReporter.stats_dict[self.category]
 
     def get_stats_summaries(self, key: str) -> StatsSummary:
         """