docs: add experimental metrics and SIL Kit Dashboard (#257)

Add documentation for the `Experimental: Metrics` configuration and mention the  usage of the Vector SIL Kit Dashboard.
The Default value for Experimental>>Metrics>>CollectFromRemote is true for the sil-kit-registry executable.

---------

Co-authored-by: Daniel Edwards <daniel.edwards@vector.com>
Co-authored-by: Nadege Griesser <nadege.griesser@vector.com>

Author: 3 people

SilKit/source/config/ParticipantConfiguration.hpp

@@ -260,7 +260,7 @@ struct MetricsSink
 struct Metrics
 {
     std::vector<MetricsSink> sinks;
-    bool collectFromRemote{false};
+    std::optional<bool> collectFromRemote;
     std::chrono::seconds updateInterval{1};
 };
 

SilKit/source/config/ParticipantConfigurationFromXImpl.cpp

@@ -352,8 +352,11 @@ void Cache(const TimeSynchronization& root, TimeSynchronizationCache& cache)
 void Cache(const Metrics& root, MetricsCache& cache)
 {
     static const Metrics defaultObject;
-    CacheNonDefault(defaultObject.collectFromRemote, root.collectFromRemote, "Metrics.CollectFromRemote",
-                    cache.collectFromRemote);
+    if(root.collectFromRemote.has_value())
+    {
+        CacheNonDefault(false, root.collectFromRemote.value(), "Metrics.CollectFromRemote",
+                        cache.collectFromRemote);
+    }
 
     for (const auto& sink : root.sinks)
     {
@@ -521,10 +524,13 @@ void MergeTimeSynchronizationCache(const TimeSynchronizationCache& cache, TimeSy
 
 void MergeMetricsCache(const MetricsCache& cache, Metrics& metrics)
 {
-    MergeCacheField(cache.collectFromRemote, metrics.collectFromRemote);
+    if(metrics.collectFromRemote.has_value())
+    {
+        MergeCacheField(cache.collectFromRemote, metrics.collectFromRemote.value());
+    }
     MergeCacheSet(cache.jsonFileSinks, metrics.sinks);
 
-    if (cache.remoteSink.has_value() && metrics.collectFromRemote)
+    if (cache.remoteSink.has_value() && cache.collectFromRemote.value_or(false))
     {
         throw SilKit::ConfigurationError{
             "Cannot have 'Remote' metrics sink together with 'CollectFromRemote' being true"};

SilKit/source/config/YamlReader.cpp

@@ -242,18 +242,6 @@ void YamlReader::Read(SilKit::Config::Metrics& obj)
 {
     OptionalRead(obj.sinks, "Sinks");
     OptionalRead(obj.collectFromRemote, "CollectFromRemote");
-
-    if (obj.collectFromRemote)
-    {
-        for (auto&& sink : obj.sinks)
-        {
-            if (sink.type == SilKit::Config::MetricsSink::Type::Remote)
-            {
-                throw SilKit::ConfigurationError{
-                    "Metrics collectFromRemote is enabled while having a Remote MetricsSink active"};
-            }
-        }
-    }
 }
 
 void YamlReader::Read(SilKit::Config::MdfChannel& obj)

SilKit/source/config/YamlWriter.cpp

@@ -282,9 +282,9 @@ void YamlWriter::Write(const SilKit::Config::Metrics& obj)
 {
     MakeMap();
     OptionalWrite(obj.sinks, "Sinks");
-    if (obj.collectFromRemote)
+    if (obj.collectFromRemote.has_value())
     {
-        WriteKeyValue("CollectFromRemote", obj.collectFromRemote);
+        WriteKeyValue("CollectFromRemote", obj.collectFromRemote.value());
     }
 }
 

SilKit/source/core/participant/Participant_impl.hpp

@@ -222,7 +222,7 @@ void Participant<SilKitConnectionT>::SetupMetrics()
     (void)GetMetricsManager();
 
     // NB: Create the metrics receiver if enabled in the configuration
-    if (_participantConfig.experimental.metrics.collectFromRemote)
+    if (_participantConfig.experimental.metrics.collectFromRemote.value_or(false))
     {
         Core::SupplementalData supplementalData;
         supplementalData[SilKit::Core::Discovery::controllerType] =

SilKit/source/core/vasio/VAsioRegistry.cpp

@@ -157,7 +157,7 @@ auto VAsioRegistry::StartListening(const std::string& listenUri) -> std::string
     _connection.StartIoWorker();
 
     _connection.RegisterSilKitService(this);
-    if (_vasioConfig->experimental.metrics.collectFromRemote)
+    if (_vasioConfig->experimental.metrics.collectFromRemote.value_or(false))
     {
         _connection.RegisterSilKitService(_metricsReceiver.get());
     }
@@ -323,7 +323,7 @@ void VAsioRegistry::SetupMetrics()
         processor.SetSinks(std::move(sinks));
     }
 
-    if (_vasioConfig->experimental.metrics.collectFromRemote)
+    if (_vasioConfig->experimental.metrics.collectFromRemote.value_or(false))
     {
         auto metricsReceiver = std::make_unique<VSilKit::MetricsReceiver>(nullptr, *this);
 

Utilities/SilKitRegistry/Registry.cpp

@@ -129,6 +129,13 @@ void OverrideFromRegistryConfiguration(std::shared_ptr<SilKit::Config::IParticip
     }
 
     config->experimental.metrics = registryConfiguration.experimental.metrics;
+
+    if(!registryConfiguration.experimental.metrics.collectFromRemote.has_value())
+    {
+        // implicitly enable collectFromRemote for the registry, if the user did not set it
+        config->experimental.metrics.collectFromRemote = true;
+    }
+
 }
 
 void OverrideRegistryUri(std::shared_ptr<SilKit::Config::IParticipantConfiguration> configuration,

docs/configuration/configuration.rst

@@ -202,7 +202,7 @@ The Registry Configuration File
 
 An instance of the |ProductName| Registry (``sil-kit-registry(.exe)``) can be configured via a YAML file.
 The configuration file is optional and overrides the settings specified on the command line.
-It also allows extended configuration, beyond what the command line allows, particularly for logging.
+It also allows extended configuration, beyond what the command line allows, particularly for logging and data export to a `Vector SIL Kit Dashboard <https://vector.com/sil-kit-dashboard>`_ instance.
 
 The outline of a registry configuration file is as follows:
 
@@ -266,15 +266,19 @@ Overview
 
        **NOTE** The default URI to use for a local |ProductName| dashboard setup is http://localhost:8082.
 
+   * - :ref:`Experimental<sec:cfg-registry-experimental>`
+     - Experimental configuration options for the |ProductName|.
+
 .. _subsec:registry-config-options:
 
 Configuration Options
 ~~~~~~~~~~~~~~~~~~~~~
 
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
 
    logging-configuration
+   experimental-configuration
 
 
 API and Data Type Reference

docs/configuration/experimental-configuration.rst

@@ -54,4 +54,58 @@ TimeSynchronization
        .. note::
          Option *Auto* can be chosen without any concerns. 
          In the case of option *On*, however, it is necessary to verify that the transmission of messages within a time step does not depend on incoming messages from other participants.
-         In this case, the time step will not be terminated and the communication will block.
+         In this case, the time step will not be terminated and the communication will block.
+
+Metrics for participants
+------------------------
+Each participant supports collecting static attributes of a simulation and
+runtime performance metrics.
+They are collected and distributed to the configured sinks at the given update interval.
+
+.. code-block:: yaml
+
+    Experimental:
+        Metrics:
+          Sinks:
+            - Name: SomeSink1
+              Type: Remote
+
+
+.. list-table:: Participant Metrics Configuration
+   :widths: 15 85
+   :header-rows: 1
+
+   * - Property Name
+     - Description
+
+   * - Sinks
+     - A list of named metric sinks. They can be of type ``JsonFile`` or ``Remote``.
+   * - updateInterval
+     - The time between sending batches of metrics to the registry in seconds.
+
+.. _sec:cfg-registry-experimental:
+
+Metrics for the registry
+------------------------
+
+The registry configuration supports collecting metrics from remote participants
+and forwarding the collected data to the SIL Kit Dashboard for further
+analysis and visualization.
+Refer to the documentation of the `Vector SIL Kit Dashboard <https://vector.com/sil-kit-dashboard>`_ for further instructions.
+
+.. code-block:: yaml
+
+    Experimental:
+        Metrics:
+          CollectFromRemote: true
+
+
+.. list-table:: Registry Metrics Configuration
+   :widths: 15 85
+   :header-rows: 1
+
+   * - Property Name
+     - Description
+
+   * - CollectFromRemote
+     - Collect metrics from all connected participants. Defaults to ``true``.

docs/troubleshooting/advanced.rst

@@ -0,0 +1,28 @@
+Advanced 
+========
+
+This section shows how to use advanced tools like the `Vector SIL Kit Dashboard <https://vector.com/sil-kit-dashboard>`_ to visualize simulations, in order to identitfy participant misconfigurations, detect system misbehavior or analyze participant performance.
+
+Getting Started with the dashboard
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+The dashboard is a freeware application that collects and visualizes SIL Kit simulation data.
+It can be used to show static attributes of participants and also show their simulation properties like active peer connections and simulated, active networks.
+In addition, the collection of performance metrics can be enabled in the configuration files of the SIL Kit participants.
+
+This section shows how to enable the data collection of simulations for the Vector SIL Kit Dashboard.
+Please refer to the SIL Kit Dashboard documentation for updated instructions.
+
+#. The SIL Kit registry will forward simulation data to a running Vector SIL Kit Dashboard instance, if configured properly.
+   The ``sil-kit-registry`` executable supports the ``--dashboard-uri https://1.2.3.4`` command line flag and the registry's configuration supports the ``DashboardUri: https://1.2.3.4`` declaration.
+   Ensure that the registry is able to connect to the Dashboard when started, e.g. by verifying the log output.
+   The registry's configuration of ``CollectFromRemotes: true`` defaults to true -- it can be disabled explicitly.
+#. Each participant that should be included in the dashboard visualization, must add a remot metric sink to its conifugration file:
+   ```
+   Experimental:
+     Metrics:
+       Sinks:
+        - Name: RemoteSink1
+          Type: Remote 
+   ```
+    This will enable the data updates from the participant to the registry, which will forward it to the dashboard.
+#. Start a simulation run, the dashboard will show the simulation's state, including the participant states, attributes and simulated network details. If the metrics sinks are enabled, detailed performance counters will be available for analysis.