Added support for logging dicts/iterables (#3369)

vfdev-5 · web-flow · commit e205e61f0d9c · 2025-03-29T02:13:59.000+01:00
Fixes #3294
diff --git a/ignite/handlers/base_logger.py b/ignite/handlers/base_logger.py
@@ -1,5 +1,6 @@
 """Base logger and its helper handlers."""
 
+import collections.abc as collections
 import numbers
 import warnings
 from abc import ABCMeta, abstractmethod
@@ -145,30 +146,64 @@ def _setup_output_metrics_state_attrs(
 
         metrics_state_attrs_dict: Dict[Any, Union[str, float, numbers.Number]] = OrderedDict()
 
-        def key_tuple_tf(tag: str, name: str, *args: str) -> Tuple[str, ...]:
-            return (tag, name) + args
+        def key_tuple_fn(parent_key: Union[str, Tuple[str, ...]], *args: str) -> Tuple[str, ...]:
+            if parent_key is None or isinstance(parent_key, str):
+                return (parent_key,) + args
+            return parent_key + args
 
-        def key_str_tf(tag: str, name: str, *args: str) -> str:
-            return "/".join((tag, name) + args)
+        def key_str_fn(parent_key: str, *args: str) -> str:
+            args_str = "/".join(args)
+            return f"{parent_key}/{args_str}"
 
-        key_tf = key_tuple_tf if key_tuple else key_str_tf
+        key_fn = key_tuple_fn if key_tuple else key_str_fn
 
-        for name, value in metrics_state_attrs.items():
+        def handle_value_fn(
+            value: Union[str, int, float, numbers.Number, torch.Tensor]
+        ) -> Union[None, str, float, numbers.Number]:
             if isinstance(value, numbers.Number):
-                metrics_state_attrs_dict[key_tf(self.tag, name)] = value
+                return value
             elif isinstance(value, torch.Tensor) and value.ndimension() == 0:
-                metrics_state_attrs_dict[key_tf(self.tag, name)] = value.item()
-            elif isinstance(value, torch.Tensor) and value.ndimension() == 1:
-                for i, v in enumerate(value):
-                    metrics_state_attrs_dict[key_tf(self.tag, name, str(i))] = v.item()
+                return value.item()
             else:
                 if isinstance(value, str) and log_text:
-                    metrics_state_attrs_dict[key_tf(self.tag, name)] = value
+                    return value
                 else:
                     warnings.warn(f"Logger output_handler can not log metrics value type {type(value)}")
+            return None
+
+        metrics_state_attrs_dict = _flatten_dict(metrics_state_attrs, key_fn, handle_value_fn, parent_key=self.tag)
         return metrics_state_attrs_dict
 
 
+def _flatten_dict(
+    in_dict: collections.Mapping,
+    key_fn: Callable,
+    value_fn: Callable,
+    parent_key: Optional[Union[str, Tuple[str, ...]]] = None,
+) -> Dict:
+    items = {}
+    for key, value in in_dict.items():
+        new_key = key_fn(parent_key, key)
+        if isinstance(value, collections.Mapping):
+            items.update(_flatten_dict(value, key_fn, value_fn, new_key))
+        elif any(
+            [
+                isinstance(value, tuple) and hasattr(value, "_fields"),  # namedtuple
+                not isinstance(value, str) and isinstance(value, collections.Sequence),
+            ]
+        ):
+            for i, item in enumerate(value):
+                items.update(_flatten_dict({str(i): item}, key_fn, value_fn, new_key))
+        elif isinstance(value, torch.Tensor) and value.ndimension() == 1:
+            for i, item in enumerate(value):
+                items.update(_flatten_dict({str(i): item.item()}, key_fn, value_fn, new_key))
+        else:
+            new_value = value_fn(value)
+            if new_value is not None:
+                items[new_key] = new_value
+    return items
+
+
 class BaseWeightsScalarHandler(BaseWeightsHandler):
     """
     Helper handler to log model's weights or gradients as scalars.
diff --git a/ignite/handlers/clearml_logger.py b/ignite/handlers/clearml_logger.py
@@ -109,6 +109,13 @@ class ClearMLLogger(BaseLogger):
                 log_handler=WeightsScalarHandler(model)
             )
 
+    Note:
+        :class:`~ignite.handlers.clearml_logger.OutputHandler` can handle
+        metrics, state attributes and engine output values of the following format:
+        - scalar values (i.e. int, float)
+        - 0d and 1d pytorch tensors
+        - dicts and list/tuples of previous types
+
     """
 
     def __init__(self, **kwargs: Any):
@@ -342,9 +349,10 @@ def __call__(self, engine: Engine, logger: ClearMLLogger, event_name: Union[str,
         for key, value in metrics.items():
             if len(key) == 2:
                 logger.clearml_logger.report_scalar(title=key[0], series=key[1], iteration=global_step, value=value)
-            elif len(key) == 3:
+            elif len(key) >= 3:
+                series = "/".join(key[2:])
                 logger.clearml_logger.report_scalar(
-                    title=f"{key[0]}/{key[1]}", series=key[2], iteration=global_step, value=value
+                    title=f"{key[0]}/{key[1]}", series=series, iteration=global_step, value=value
                 )
 
 
diff --git a/ignite/handlers/mlflow_logger.py b/ignite/handlers/mlflow_logger.py
@@ -84,6 +84,49 @@ class MLflowLogger(BaseLogger):
                 optimizer=optimizer,
                 param_name='lr'  # optional
             )
+
+    Note:
+        :class:`~ignite.handlers.mlflow_logger.OutputHandler` can handle
+        metrics, state attributes and engine output values of the following format:
+        - scalar values (i.e. int, float)
+        - 0d and 1d pytorch tensors
+        - dicts and list/tuples of previous types
+
+        .. code-block:: python
+
+            # !!! This is not a runnable code !!!
+            evalutator.state.metrics = {
+                "a": 0,
+                "dict_value": {
+                    "a": 111,
+                    "c": {"d": 23, "e": [123, 234]},
+                },
+                "list_value": [12, 13, {"aa": 33, "bb": 44}],
+                "tuple_value": (112, 113, {"aaa": 33, "bbb": 44}),
+            }
+
+            handler = OutputHandler(
+                tag="tag",
+                metric_names="all",
+            )
+
+            handler(evaluator, mlflow_logger, event_name=Events.EPOCH_COMPLETED)
+            # Behind it would call `mlflow_logger.log_metrics` on
+            # {
+            #     "tag/a": 0,
+            #     "tag/dict_value/a": 111,
+            #     "tag/dict_value/c/d": 23,
+            #     "tag/dict_value/c/e/0": 123,
+            #     "tag/dict_value/c/e/1": 234,
+            #     "tag/list_value/0": 12,
+            #     "tag/list_value/1": 13,
+            #     "tag/list_value/2/aa": 33,
+            #     "tag/list_value/2/bb": 44,
+            #     "tag/tuple_value/0": 112,
+            #     "tag/tuple_value/1": 113,
+            #     "tag/tuple_value/2/aaa": 33,
+            #     "tag/tuple_value/2/bbb": 44,
+            # }
     """
 
     def __init__(self, tracking_uri: Optional[str] = None):
diff --git a/ignite/handlers/neptune_logger.py b/ignite/handlers/neptune_logger.py
@@ -153,6 +153,13 @@ def score_function(engine):
                     output_transform=lambda loss: {"loss": loss},
                 )
 
+    Note:
+        :class:`~ignite.handlers.neptune_logger.OutputHandler` can handle
+        metrics, state attributes and engine output values of the following format:
+        - scalar values (i.e. int, float)
+        - 0d and 1d pytorch tensors
+        - dicts and list/tuples of previous types
+
     """
 
     def __getattr__(self, attr: Any) -> Any:
diff --git a/ignite/handlers/polyaxon_logger.py b/ignite/handlers/polyaxon_logger.py
@@ -92,6 +92,13 @@ class PolyaxonLogger(BaseLogger):
             )
             # to manually end a run
             plx_logger.close()
+
+    Note:
+        :class:`~ignite.handlers.polyaxon_logger.OutputHandler` can handle
+        metrics, state attributes and engine output values of the following format:
+        - scalar values (i.e. int, float)
+        - 0d and 1d pytorch tensors
+        - dicts and list/tuples of previous types
     """
 
     def __init__(self, *args: Any, **kwargs: Any):
diff --git a/ignite/handlers/tensorboard_logger.py b/ignite/handlers/tensorboard_logger.py
@@ -145,6 +145,49 @@ class TensorboardLogger(BaseLogger):
                     output_transform=lambda loss: {"loss": loss}
                 )
 
+    Note:
+        :class:`~ignite.handlers.tensorboard_logger.OutputHandler` can handle
+        metrics, state attributes and engine output values of the following format:
+        - scalar values (i.e. int, float)
+        - 0d and 1d pytorch tensors
+        - dicts and list/tuples of previous types
+
+        .. code-block:: python
+
+            # !!! This is not a runnable code !!!
+            evalutator.state.metrics = {
+                "a": 0,
+                "dict_value": {
+                    "a": 111,
+                    "c": {"d": 23, "e": [123, 234]},
+                },
+                "list_value": [12, 13, {"aa": 33, "bb": 44}],
+                "tuple_value": (112, 113, {"aaa": 33, "bbb": 44}),
+            }
+
+            handler = OutputHandler(
+                tag="tag",
+                metric_names="all",
+            )
+
+            handler(evaluator, tb_logger, event_name=Events.EPOCH_COMPLETED)
+            # Behind it would call `tb_logger.writer.add_scalar` on
+            # {
+            #     "tag/a": 0,
+            #     "tag/dict_value/a": 111,
+            #     "tag/dict_value/c/d": 23,
+            #     "tag/dict_value/c/e/0": 123,
+            #     "tag/dict_value/c/e/1": 234,
+            #     "tag/list_value/0": 12,
+            #     "tag/list_value/1": 13,
+            #     "tag/list_value/2/aa": 33,
+            #     "tag/list_value/2/bb": 44,
+            #     "tag/tuple_value/0": 112,
+            #     "tag/tuple_value/1": 113,
+            #     "tag/tuple_value/2/aaa": 33,
+            #     "tag/tuple_value/2/bbb": 44,
+            # }
+
     """
 
     def __init__(self, *args: Any, **kwargs: Any):
diff --git a/ignite/handlers/tqdm_logger.py b/ignite/handlers/tqdm_logger.py
@@ -298,8 +298,7 @@ def __call__(self, engine: Engine, logger: ProgressBar, event_name: Union[str, E
         rendered_metrics = self._setup_output_metrics_state_attrs(engine, log_text=True)
         metrics = OrderedDict()
         for key, value in rendered_metrics.items():
-            key = "_".join(key[1:])  # tqdm has tag as description
-
+            key = "_".join(key[1:])  # skip tag as tqdm has tag as description
             metrics[key] = value
 
         if metrics:
diff --git a/ignite/handlers/visdom_logger.py b/ignite/handlers/visdom_logger.py
@@ -137,6 +137,13 @@ class VisdomLogger(BaseLogger):
                     output_transform=lambda loss: {"loss": loss}
                 )
 
+    Note:
+        :class:`~ignite.handlers.visdom_logger.OutputHandler` can handle
+        metrics, state attributes and engine output values of the following format:
+        - scalar values (i.e. int, float)
+        - 0d and 1d pytorch tensors
+        - dicts and list/tuples of previous types
+
     .. versionchanged:: 0.4.7
         accepts an optional list of `state_attributes`
     """
diff --git a/ignite/handlers/wandb_logger.py b/ignite/handlers/wandb_logger.py
@@ -120,6 +120,12 @@ def score_function(engine):
             )
             evaluator.add_event_handler(Events.COMPLETED, model_checkpoint, {'model': model})
 
+    Note:
+        :class:`~ignite.handlers.wandb_logger.OutputHandler` can handle
+        metrics, state attributes and engine output values of the following format:
+        - scalar values (i.e. int, float)
+        - 0d and 1d pytorch tensors
+        - dicts and list/tuples of previous types
 
     """
 
diff --git a/tests/ignite/handlers/test_base_logger.py b/tests/ignite/handlers/test_base_logger.py
@@ -103,6 +103,44 @@ def test_base_output_handler_setup_output_metrics():
     metrics = handler._setup_output_metrics_state_attrs(engine=engine, key_tuple=False)
     assert metrics == {"tag/a": 0, "tag/b": 1}
 
+    # metrics with mappings, iterables
+    true_metrics = {
+        "a": 0,
+        "b": "1",
+        "dict_value": {
+            "a": 111,
+            "b": "222",
+            "c": {"d": 23, "e": [123, 234]},
+            "f": [{"g": 11, "h": (321, 432)}, 778],
+        },
+        "list_value": [12, "13", {"aa": 33, "bb": 44}],
+        "tuple_value": (112, "113", {"aaa": 33, "bbb": 44}),
+    }
+    engine.state = State(metrics=true_metrics)
+    handler = DummyOutputHandler("tag", metric_names="all", output_transform=None)
+    metrics = handler._setup_output_metrics_state_attrs(engine=engine, key_tuple=False, log_text=True)
+    assert metrics == {
+        "tag/a": 0,
+        "tag/b": "1",
+        "tag/dict_value/a": 111,
+        "tag/dict_value/b": "222",
+        "tag/dict_value/c/d": 23,
+        "tag/dict_value/c/e/0": 123,
+        "tag/dict_value/c/e/1": 234,
+        "tag/dict_value/f/0/g": 11,
+        "tag/dict_value/f/0/h/0": 321,
+        "tag/dict_value/f/0/h/1": 432,
+        "tag/dict_value/f/1": 778,
+        "tag/list_value/0": 12,
+        "tag/list_value/1": "13",
+        "tag/list_value/2/aa": 33,
+        "tag/list_value/2/bb": 44,
+        "tag/tuple_value/0": 112,
+        "tag/tuple_value/1": "113",
+        "tag/tuple_value/2/aaa": 33,
+        "tag/tuple_value/2/bbb": 44,
+    }
+
 
 def test_base_output_handler_setup_output_state_attrs():
     engine = Engine(lambda engine, batch: None)
diff --git a/tests/ignite/handlers/test_clearml_logger.py b/tests/ignite/handlers/test_clearml_logger.py
@@ -209,6 +209,36 @@ def test_output_handler_metric_names(dirname):
         any_order=True,
     )
 
+    wrapper = OutputHandler("tag", metric_names="all")
+    mock_logger = MagicMock(spec=ClearMLLogger)
+    mock_logger.clearml_logger = MagicMock()
+
+    mock_engine = MagicMock()
+    mock_engine.state = State(
+        metrics={
+            "a": 123,
+            "b": {"c": [2.34, {"d": 1}]},
+            "c": (22, [33, -5.5], {"e": 32.1}),
+        }
+    )
+    mock_engine.state.iteration = 5
+
+    wrapper(mock_engine, mock_logger, Events.ITERATION_STARTED)
+
+    assert mock_logger.clearml_logger.report_scalar.call_count == 7
+    mock_logger.clearml_logger.report_scalar.assert_has_calls(
+        [
+            call(title="tag", series="a", iteration=5, value=123),
+            call(title="tag/b", series="c/0", iteration=5, value=2.34),
+            call(title="tag/b", series="c/1/d", iteration=5, value=1),
+            call(title="tag/c", series="0", iteration=5, value=22),
+            call(title="tag/c", series="1/0", iteration=5, value=33),
+            call(title="tag/c", series="1/1", iteration=5, value=-5.5),
+            call(title="tag/c", series="2/e", iteration=5, value=32.1),
+        ],
+        any_order=True,
+    )
+
 
 def test_output_handler_both(dirname):
     wrapper = OutputHandler("tag", metric_names=["a", "b"], output_transform=lambda x: {"loss": x})
diff --git a/tests/ignite/handlers/test_mlflow_logger.py b/tests/ignite/handlers/test_mlflow_logger.py
@@ -85,6 +85,36 @@ def test_output_handler_metric_names():
     assert mock_logger.log_metrics.call_count == 1
     mock_logger.log_metrics.assert_has_calls([call({"tag a": 55.56}, step=7)], any_order=True)
 
+    wrapper = OutputHandler("tag", metric_names="all")
+    mock_logger = MagicMock(spec=MLflowLogger)
+    mock_logger.log_metrics = MagicMock()
+
+    mock_engine = MagicMock()
+    mock_engine.state = State(
+        metrics={
+            "a": 123,
+            "b": {"c": [2.34, {"d": 1}]},
+            "c": (22, [33, -5.5], {"e": 32.1}),
+        }
+    )
+    mock_engine.state.iteration = 5
+
+    wrapper(mock_engine, mock_logger, Events.ITERATION_STARTED)
+
+    assert mock_logger.log_metrics.call_count == 1
+    mock_logger.log_metrics.assert_called_once_with(
+        {
+            "tag a": 123,
+            "tag b c 0": 2.34,
+            "tag b c 1 d": 1,
+            "tag c 0": 22,
+            "tag c 1 0": 33,
+            "tag c 1 1": -5.5,
+            "tag c 2 e": 32.1,
+        },
+        step=5,
+    )
+
 
 def test_output_handler_both():
     wrapper = OutputHandler("tag", metric_names=["a", "b"], output_transform=lambda x: {"loss": x})
diff --git a/tests/ignite/handlers/test_neptune_logger.py b/tests/ignite/handlers/test_neptune_logger.py
diff --git a/tests/ignite/handlers/test_polyaxon_logger.py b/tests/ignite/handlers/test_polyaxon_logger.py
diff --git a/tests/ignite/handlers/test_tensorboard_logger.py b/tests/ignite/handlers/test_tensorboard_logger.py
diff --git a/tests/ignite/handlers/test_visdom_logger.py b/tests/ignite/handlers/test_visdom_logger.py
diff --git a/tests/ignite/handlers/test_wandb_logger.py b/tests/ignite/handlers/test_wandb_logger.py
