Swallowing ValueError in add_histogram

Summary: add_histogram() throws ValueError exception when histogram cannot be created, e.g., when all the values are the same. Swallowing the exception so that the training can continue.

Reviewed By: czxttkl

Differential Revision: D17758609

fbshipit-source-id: a58275e1e2f24f01c982dc28948ab29185b4a0ce

ml/rl/tensorboardX.py

@@ -74,6 +74,13 @@ def _reset_globals(cls):
     def increase_global_step(cls):
         cls._global_step += 1
 
+    @classmethod
+    def add_histogram(cls, key, val, *args, **kwargs):
+        try:
+            return cls.__getattr__("add_histogram")(key, val, *args, **kwargs)
+        except ValueError:
+            logger.warning(f"Cannot create histogram for {key}, got values: {val}")
+
     @classmethod
     def add_custom_scalars(cls, writer):
         """

ml/rl/test/base/test_tensorboardX.py

@@ -63,6 +63,12 @@ def test_not_swallowing_exception(self):
             ), summary_writer_context(writer):
                 SummaryWriterContext.add_scalar("test", torch.ones(1))
 
+    def test_swallowing_histogram_value_error(self):
+        with TemporaryDirectory() as tmp_dir:
+            writer = SummaryWriter(tmp_dir)
+            with summary_writer_context(writer):
+                SummaryWriterContext.add_histogram("bad_histogram", torch.ones(100, 1))
+
     def test_global_step(self):
         with TemporaryDirectory() as tmp_dir:
             writer = SummaryWriter(tmp_dir)