Feature/introspection

.gitignore

@@ -66,3 +66,10 @@ Notes.txt
 tmp/
 
 compile_commands.json
+
+# Doxygen
+.doxygen/
+
+# Introspection Data
+config_introspection_results/
+config_utilities/introspection_viewer/data.json

README.md

@@ -1,125 +1,138 @@
 ![CMake: Build](https://github.com/MIT-SPARK/config_utilities/actions/workflows/cmake_build.yml/badge.svg)
 
 # config_utilities
-`config_utilities` is a minimal but powerful C++ library, providing tools for automated modular software configuration. Parse, verify, and print C++ config structs and run-time configuration of object-oriented modular systems.
+
+`config_utilities` is a minimal but powerful C++ library, providing tools for automated modular software configuration. Parse, verify, modify, and print C++ config structs to build modular object-oriented software systems at run-time.
 
 ## Table of contents
+
 - [Credits](#credits)
 - [Why `config_utilities`?](#why-config_utilities)
 - [Installation](#installation)
 - [How to `config_utilities`?](#how-to-config_utilities)
 - [Example Projects using `config_utilities`](#example-projects-using-config_utilities)
 
 ## Credits
+
 This library was developed by [Lukas Schmid](https://schmluk.github.io/) and [Nathan Hughes](http://mit.edu/sparklab/people.html) at the [MIT-SPARK Lab](http://mit.edu/sparklab), based on functionalities in [ethz-asl/config_utilities](https://github.com/ethz-asl/config_utilities) and [Hydra](https://github.com/MIT-SPARK/Hydra), and is released under a [BSD-3-Clause License](LICENSE)! Additional contributions welcome! This work was supported in part by the Swiss National Science Foundation and Amazon.
 
 ## Why `config_utilities`?
+
 Among many other, the key features of `config_utilities` include:
+
 - **Minimal dependencies**: Only C++17 standard library and [yaml-cpp](https://github.com/jbeder/yaml-cpp).
 - Declare **any struct a config**, also from external projects:
-    ```c++
-    namespace external_project {
-        void declare_config(ExternalObject& config); // that's all!
-    }   // namespace external_project
-    ```
+  ```c++
+  namespace external_project {
+      void declare_config(ExternalObject& config); // that's all!
+  }   // namespace external_project
+  ```
 - **Minimal** and **clear, human readable interfaces** for config definitions:
-    ```c++
-    void declare_config(MyConfig& config) {
-        using namespace config;
-        name("MyConfig");                             // Name for printing.
-        field(config.distance, "distance", "m");      // Field definition.
-        check(config.distance, GT, 0.0, "distance");  // Ensure 'distance > 0'.
-    }
-    ```
+  ```c++
+  void declare_config(MyConfig& config) {
+      using namespace config;
+      name("MyConfig");                             // Name for printing.
+      field(config.distance, "distance", "m");      // Field definition.
+      check(config.distance, GT, 0.0, "distance");  // Ensure 'distance > 0'.
+  }
+  ```
 - **Everything** related to a config is defined in a **single place**.
   ```c++
   void declare_config(MyConfig& config) { /* ALL the information about a config is here */ }
   ```
 - Parse any declared config from **various data sources**, **without pulling in dependencies** on data sources into core libraries:
-    ```c++
-    core.cpp {
-        struct MyConfig { ... };
-        void declare_config(MyConfig& config) { ... };
-    }  // Depends only on C++ and yaml-cpp.
-
-    application.cpp {
-        MyConfig config1 = fromYamlFile<MyConfig>(file_path);
-        MyConfig config2 = fromRos<MyConfig>(node_handle);
-    }  // Pull in dependencies as needed.
-    ```
+
+  ```c++
+  core.cpp {
+      struct MyConfig { ... };
+      void declare_config(MyConfig& config) { ... };
+  }  // Depends only on C++ and yaml-cpp.
+
+  application.cpp {
+      MyConfig config1 = fromYamlFile<MyConfig>(file_path);
+      MyConfig config2 = fromRos<MyConfig>(node_handle);
+  }  // Pull in dependencies as needed.
+  ```
+
 - **Automatically implements** frequently used functionalities, such as **checking** for valid parameters and **formatting** to string:
-    ```c++
-    MyObject(const MyConfig& config) : config_(checkValid(config)) {
-        // Do magic with config_, which has valid parameters only.
-        std::cout << config_ << std::endl;
-    }
-    ```
+  ```c++
+  MyObject(const MyConfig& config) : config_(checkValid(config)) {
+      // Do magic with config_, which has valid parameters only.
+      std::cout << config_ << std::endl;
+  }
+  ```
 - Automatic **run-time module creation** based on specified configurations:
-    ```cpp
-    static auto registration = Registration<Base, MyDerived>("my_derived");
 
-    const std::string module_type = "my_derived";
-    std::unique_ptr<Base> module = create(module_type);
-    ```
+  ```cpp
+  static auto registration = Registration<Base, MyDerived>("my_derived");
+
+  const std::string module_type = "my_derived";
+  std::unique_ptr<Base> module = create(module_type);
+  ```
+
 - **Informative, clear**, and **human readable** printing of potentially complicated config structs:
-    ```
-    =================================== MyConfig ===================================
-    distance [m]:                 -0.9876
-    A ridiculously long field name that will not be wrapped: Short Value (default)
-    A ridiculously long field name that will also not be wrapped:
-                                  A really really really ridiculously long string th
-                                  at will be wrapped. (default)
-    A really really really really really really ridiculously long field name that wi
-    ll be wrapped:                A really really really ridiculously long string th
-                                  at will also be wrapped. (default)
-    vec:                          [5, 4, 3, 2, 1]
-    map:                          {b: 2, c: 3, d: 4, e: 5}
-    mat:                          [[1, 2, 3],
-                                   [4, 5, 6],
-                                   [7, 8, 9]]
-    my_enum:                      B
-    sub_config [SubConfig] (default):
-       f:                         0.123 (default)
-       s:                         test (default)
-       sub_sub_config [SubSubConfig] (default):
-          color:                  [255, 127, 0] (default)
-          size:                   5 (default)
-    ================================================================================
-    ```
+
+  ```
+  =================================== MyConfig ===================================
+  distance [m]:                 -0.9876
+  A ridiculously long field name that will not be wrapped: Short Value (default)
+  A ridiculously long field name that will also not be wrapped:
+                                A really really really ridiculously long string th
+                                at will be wrapped. (default)
+  A really really really really really really ridiculously long field name that wi
+  ll be wrapped:                A really really really ridiculously long string th
+                                at will also be wrapped. (default)
+  vec:                          [5, 4, 3, 2, 1]
+  map:                          {b: 2, c: 3, d: 4, e: 5}
+  mat:                          [[1, 2, 3],
+                                 [4, 5, 6],
+                                 [7, 8, 9]]
+  my_enum:                      B
+  sub_config [SubConfig] (default):
+     f:                         0.123 (default)
+     s:                         test (default)
+     sub_sub_config [SubSubConfig] (default):
+        color:                  [255, 127, 0] (default)
+        size:                   5 (default)
+  ================================================================================
+  ```
 
 - **Verbose warnings and errors** if desired for clear and easy development and use:
-    ```
-    =================================== MyConfig ===================================
-    Warning: Check failed for 'distance': param > 0 (is: '-1').
-    Warning: Check failed for 'subconfig.f': param within [0, 100) (is: '123').
-    Warning: Failed to parse param 'vec': Data is not a sequence.
-    Warning: Failed to parse param 'mat': Incompatible Matrix dimensions: Requested
-            3x3 but got 3x4.
-    Warning: Failed to parse param 'my_enum': Name 'D' is out of bounds for enum wit
-            h names ['A', 'B', 'C'].
-    Warning: Failed to parse param 'uint': Value '-1' underflows storage min of '0'.
-    ================================================================================
-
-    [ERROR] No module of type 'NotRegistered' registered to the factory for BaseT='d
-    emo::Base' and ConstructorArguments={'int'}. Registered are: 'DerivedB', 'Derive
-    dA'.
-
-    [ERROR] Cannot create a module of type 'DerivedA': No modules registered to the
-    factory for BaseT='demo::Base' and ConstructorArguments={'int', 'float'}. Regist
-    er modules using a static config::Registration<BaseT, DerivedT, ConstructorArgum
-    ents...> struct.
-    ```
+
+  ```
+  =================================== MyConfig ===================================
+  Warning: Check failed for 'distance': param > 0 (is: '-1').
+  Warning: Check failed for 'subconfig.f': param within [0, 100) (is: '123').
+  Warning: Failed to parse param 'vec': Data is not a sequence.
+  Warning: Failed to parse param 'mat': Incompatible Matrix dimensions: Requested
+          3x3 but got 3x4.
+  Warning: Failed to parse param 'my_enum': Name 'D' is out of bounds for enum wit
+          h names ['A', 'B', 'C'].
+  Warning: Failed to parse param 'uint': Value '-1' underflows storage min of '0'.
+  ================================================================================
+
+  [ERROR] No module of type 'NotRegistered' registered to the factory for BaseT='d
+  emo::Base' and ConstructorArguments={'int'}. Registered are: 'DerivedB', 'Derive
+  dA'.
+
+  [ERROR] Cannot create a module of type 'DerivedA': No modules registered to the
+  factory for BaseT='demo::Base' and ConstructorArguments={'int', 'float'}. Regist
+  er modules using a static config::Registration<BaseT, DerivedT, ConstructorArgum
+  ents...> struct.
+  ```
 
 ## Installation
 
 This package is compatible with `ROS2`/`colcon`. Just clone it into your workspace and you should be all set!
+
 ```bash
 cd ~/my_ws/src
 git clone git@github.com:MIT-SPARK/config_utilities.git
 colcon build config_utilities_ros
 ```
 
 If you want to build and install without colcon/ROS, that is easy, too! Just clone this repository and build via CMake:
+
 ```bash
 git clone git@github.com:MIT-SPARK/config_utilities.git
 cd config_utilities/config_utilities
@@ -133,9 +146,11 @@ sudo make install
 ```
 
 ## How to `config_utilities`
+
 We provide detailed introductions about everything you need to know about `config_utilities` in the following [tutorials](docs/README.md#tutorials) and some verbose example [demos](docs/README.md#demos) that you can run.
 
 The (non-ros) demos can be run via the `run_demo.py` utility in the scripts directory. If you are building this library via catkin, you can run one of the following to see the results of one of the corresponding demo files:
+
 ```
 python3 scripts/run_demo.py config
 python3 scripts/run_demo.py inheritance
@@ -146,20 +161,24 @@ python3 scripts/run_demo.py factory
 > If you're building via cmake, you can point `run_demo.py` to the build directory with `-b/--build_path`.
 
 The ros demo for dynamic config updates can be run via:
+
 ```
 ros2 launch config_utilities_ros demo_ros_dynamic_config.yaml
 ```
 
 If you are looking for a specific use case that is not in the tutorials or demos, chances are you can find a good example in the `tests/` directory!
 
-
 # Example Projects using `config_utilities`
+
 Many cool projects are already using `config_utilities`! If you are using `config_utilities` and would like to be featured, please open a pull request or reach out!
 
 ### Academic Open-source Projects
+
 These projects are openly available, check them out to see what `config_utilities` can do!
-* [MIT-SPARK/Khronos](https://github.com/MIT-SPARK/Khronos)
-* [MIT-SPARK/Hydra](https://github.com/MIT-SPARK/Hydra)
+
+- [MIT-SPARK/Khronos](https://github.com/MIT-SPARK/Khronos)
+- [MIT-SPARK/Hydra](https://github.com/MIT-SPARK/Hydra)
 
 ### Use in Industry
-Several leading companies and start-ups in robotics have let us know that they use `config_utilities`, but we cannot currently list them. 
+
+Several leading companies and start-ups in robotics have let us know that they use `config_utilities`, but we cannot currently list them.

config_utilities/CMakeLists.txt

@@ -16,6 +16,7 @@ find_package(yaml-cpp REQUIRED)
 find_package(Boost CONFIG REQUIRED COMPONENTS filesystem system)
 find_package(Eigen3 QUIET)
 find_package(glog QUIET)
+find_package(nlohmann_json QUIET)
 
 add_library(
   ${PROJECT_NAME}
@@ -29,6 +30,7 @@ add_library(
   src/factory.cpp
   src/formatter.cpp
   src/field_input_info.cpp
+  src/introspection.cpp
   src/log_to_stdout.cpp
   src/log_to_glog.cpp
   src/logger.cpp
@@ -62,6 +64,10 @@ if(glog_FOUND)
   target_link_libraries(${PROJECT_NAME} PUBLIC glog::glog)
   target_compile_definitions(${PROJECT_NAME} PRIVATE CONFIG_UTILS_ENABLE_GLOG_LOGGING)
 endif()
+if(nlohmann_json_FOUND)
+  target_link_libraries(${PROJECT_NAME} PRIVATE nlohmann_json::nlohmann_json)
+  target_compile_definitions(${PROJECT_NAME} PRIVATE CONFIG_UTILS_ENABLE_JSON)
+endif()
 
 add_executable(composite-configs app/composite_configs.cpp)
 target_link_libraries(composite-configs ${PROJECT_NAME})
@@ -84,6 +90,10 @@ install(
   LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
   ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
   RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
+install(DIRECTORY introspection_viewer/ DESTINATION ${CMAKE_INSTALL_BINDIR})
+install(PROGRAMS introspection_viewer/introspection_viewer.py
+        RENAME config-utilities-viewer
+        DESTINATION ${CMAKE_INSTALL_BINDIR})
 install(DIRECTORY include/ DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 install(
   EXPORT config_utilities-targets
@@ -107,6 +117,6 @@ install(FILES ${CMAKE_CURRENT_BINARY_DIR}/config_utilitiesConfig.cmake
 
 # Exposes this package to ament for ros2 run and other utilities to work
 find_package(ament_cmake_core QUIET)
-if (${ament_cmake_core_FOUND})
+if (${ament_cmake_core_FOUND}) 
   ament_package()
 endif()

config_utilities/app/composite_configs.cpp

@@ -22,6 +22,9 @@ Invalid YAML or missing files get dropped during compositing.
   -d/--disable-substitutions: Turn off substitutions resolution
   --no-disable-substitutions: Turn substitution resolution on (currently on by default)
   --force-block-style: Force emitted YAML to only use block style
+  -i/--config-utilities-introspect [OUTPUT_DIR]: Enable introspection and output results to OUTPUT_DIR.
+                                                 If OUTPUT_DIR is not specified, defaults to 'introspection_results'.
+                                                 Note that if the directory already exists, it will be overwritten.
 
 Example:
 > echo "{a: 42, bar: hello}" > /tmp/test_in.yaml

config_utilities/include/config_utilities/factory.h

@@ -41,6 +41,7 @@
 #include <string>
 #include <vector>
 
+#include "config_utilities/internal/introspection.h"
 #include "config_utilities/internal/logger.h"
 #include "config_utilities/internal/string_utils.h"
 #include "config_utilities/internal/visitor.h"
@@ -257,7 +258,6 @@ class ModuleRegistry {
 
   template <typename BaseT, typename ConfigT>
   static void registerConfig(const std::string& type) {
-    // NOTE(lschmid): This is not forbidden behavior, but is not recommended so for now simply warn the user.
     const auto key = ConfigPair::fromTypes<BaseT, ConfigT>();
     auto& registry = instance().config_registry;
     auto iter = registry.find(key);
@@ -266,6 +266,7 @@ class ModuleRegistry {
       return;
     }
 
+    // NOTE(lschmid): This is not forbidden behavior, but is not recommended so for now simply warn the user.
     if (iter != registry.end() && iter->second != type) {
       Logger::logInfo("Overwriting type name for config '" + typeName<ConfigT>() + "' for base module '" +
                       typeName<BaseT>() + "' from '" + iter->second + "' to '" + type +
@@ -427,9 +428,21 @@ struct ObjectWithConfigFactory {
   // Add entries.
   template <typename DerivedT, typename DerivedConfigT>
   static void addEntry(const std::string& type) {
-    const Constructor method = [](const YAML::Node& data, Args... args) -> BaseT* {
+    const Constructor method = [type](const YAML::Node& data, Args... args) -> BaseT* {
       DerivedConfigT config;
-      Visitor::setValues(config, data);
+      auto set_data = Visitor::setValues(config, data);
+      if (Settings::instance().introspection.enabled()) {
+        auto get_info = Visitor::getInfo(config);
+        // Also log the type parameter used for creation.
+        auto& set_field = set_data.field_infos.emplace_back();
+        set_field.name = Settings::instance().factory.type_param_name;
+        set_field.value = type;
+        set_field.is_meta_field = true;
+        auto& get_field = get_info.field_infos.emplace_back();
+        get_field = set_field;
+        set_field.was_parsed = true;
+        Introspection::logSetValue(set_data, get_info);
+      }
       return new DerivedT(config, std::move(args)...);
     };
 
@@ -443,6 +456,12 @@ struct ObjectWithConfigFactory {
   static std::unique_ptr<BaseT> create(const YAML::Node& data, Args... args) {
     std::string type;
     if (!getType(data, type)) {
+      if (Settings::instance().introspection.enabled()) {
+        // Log that the type could not be determined.
+        Introspection::logSingleEvent(
+            Introspection::Event(Introspection::Event::Type::GetError, Introspection::By::config("Factory::create()")),
+            Settings::instance().factory.type_param_name);
+      }
       return nullptr;
     }