MINIFICPP-2723 Add log and properties.d dir property files to debug bundle

Signed-off-by: Ferenc Gerlits <fgerlits@gmail.com>

Closes #2124

Author: lordgamez

libminifi/include/properties/Configure.h

@@ -49,6 +49,9 @@ class ConfigureImpl : public ConfigurationImpl, public virtual core::AgentIdenti
   std::string getAgentIdentifier() const override;
   void setFallbackAgentIdentifier(const std::string& id) override;
 
+  std::optional<std::filesystem::path> logPropertiesFilePath() const override;
+  std::optional<std::filesystem::path> extraLogPropertiesFilesDirName() const override;
+
   using Configuration::set;
   void set(const std::string& key, const std::string& value, PropertyChangeLifetime lifetime) override;
   bool commitChanges() override;

libminifi/include/properties/Properties.h

@@ -124,8 +124,9 @@ class PropertiesImpl : public virtual Properties {
 
   std::map<std::string, std::string> getProperties() const override;
 
+  std::filesystem::path extraPropertiesFilesDirName() const override;
+
  private:
-  std::filesystem::path extraPropertiesFilesDirName() const;
   void setPropertiesFromFile(const std::filesystem::path& properties_file, std::string_view prefix);
 
   std::map<std::string, PropertyValue> properties_;

libminifi/src/Configure.cpp

@@ -110,6 +110,20 @@ void ConfigureImpl::setFallbackAgentIdentifier(const std::string& id) {
   fallback_identifier_ = id;
 }
 
+std::optional<std::filesystem::path> ConfigureImpl::logPropertiesFilePath() const {
+  if (logger_properties_) {
+    return logger_properties_->getFilePath();
+  }
+  return std::nullopt;
+}
+
+std::optional<std::filesystem::path> ConfigureImpl::extraLogPropertiesFilesDirName() const {
+  if (logger_properties_) {
+    return logger_properties_->extraPropertiesFilesDirName();
+  }
+  return std::nullopt;
+}
+
 void ConfigureImpl::set(const std::string& key, const std::string& value, PropertyChangeLifetime lifetime) {
   const std::string_view log_prefix = "nifi.log.";
   if (utils::string::startsWith(key, log_prefix)) {

libminifi/src/FlowController.cpp

@@ -482,12 +482,29 @@ std::map<std::string, std::unique_ptr<io::InputStream>> FlowController::getDebug
     std::string index_str = i == logs.size() - 1 ? "" : "." + std::to_string(logs.size() - 1 - i);
     debug_info["minifi.log" + index_str + ".gz"] = std::move(logs[i]);
   }
+
   if (auto opt_flow_path = flow_configuration_->getConfigurationPath()) {
     if (utils::file::exists(opt_flow_path.value())) {
       debug_info["config.yml"] = std::make_unique<io::FileStream>(opt_flow_path.value(), 0, false);
     }
   }
+
+  const auto addDirectoryContents = [this, &debug_info](const std::filesystem::path& dir) {
+    if (!utils::file::exists(dir)) { return; }
+    for (const auto& [parent, file] : utils::file::list_dir_all(dir, logger_, false)) {
+      debug_info[parent.filename().string() + "/" + file.string()] = std::make_unique<io::FileStream>(dir / file, 0, false);
+    }
+  };
+
   debug_info["minifi.properties"] = std::make_unique<io::FileStream>(configuration_->getFilePath(), 0, false);
+  addDirectoryContents(configuration_->extraPropertiesFilesDirName());
+
+  if (auto log_properties_path = configuration_->logPropertiesFilePath()) {
+    debug_info["minifi-log.properties"] = std::make_unique<io::FileStream>(*log_properties_path, 0, false);
+  }
+  if (auto log_properties_dir = configuration_->extraLogPropertiesFilesDirName()) {
+    addDirectoryContents(*log_properties_dir);
+  }
 
   return debug_info;
 }

libminifi/test/integration/C2DebugBundleTest.cpp

@@ -138,6 +138,9 @@ class C2HeartbeatHandler : public ServerAwareHandler {
 };
 
 static std::string properties_file = "some.dummy.content = here\n";
+static std::string c2_properties_file = "c2.dummy.content = content\n";
+static std::string log_properties_file = "log.dummy.content = log\n";
+static std::string extra_log_properties_file = "extra.log.dummy.content = extra\n";
 static std::string flow_config_file = empty_flow;
 
 TEST_CASE("C2DebugBundleTest", "[c2test]") {
@@ -149,7 +152,12 @@ TEST_CASE("C2DebugBundleTest", "[c2test]") {
 
   std::filesystem::path home_dir = controller.createTempDirectory();
   minifi::utils::file::PathUtils::create_dir(home_dir / "conf");
+  minifi::utils::file::PathUtils::create_dir(home_dir / "conf" / "minifi.properties.d");
+  minifi::utils::file::PathUtils::create_dir(home_dir / "conf" / "minifi-log.properties.d");
   std::ofstream{home_dir / "conf" / "minifi.properties", std::ios::binary} << properties_file;
+  std::ofstream{home_dir / "conf" / "minifi.properties.d" / "90_c2.properties", std::ios::binary} << c2_properties_file;
+  std::ofstream{home_dir / "conf" / "minifi-log.properties", std::ios::binary} << log_properties_file;
+  std::ofstream{home_dir / "conf" / "minifi-log.properties.d" / "99_c2.properties", std::ios::binary} << extra_log_properties_file;
   std::ofstream{home_dir / "conf" / "config.yml", std::ios::binary} << flow_config_file;
 
   VerifyDebugInfo harness(home_dir / "conf" / "config.yml", [&]() -> bool {
@@ -174,8 +182,11 @@ TEST_CASE("C2DebugBundleTest", "[c2test]") {
               file_content.length());
       archive_content[info->filename] = std::move(file_content);
     }
-    REQUIRE(archive_content["minifi.properties"] == properties_file);
-    REQUIRE(archive_content["config.yml"] == flow_config_file);
+    CHECK(archive_content["minifi.properties"] == properties_file);
+    CHECK(archive_content["minifi.properties.d/90_c2.properties"] == c2_properties_file);
+    CHECK(archive_content["minifi-log.properties"] == log_properties_file);
+    CHECK(archive_content["minifi-log.properties.d/99_c2.properties"] == extra_log_properties_file);
+    CHECK(archive_content["config.yml"] == flow_config_file);
     auto log_gz = archive_content["minifi.log.gz"];
     auto log_stream = std::make_shared<minifi::io::BufferStream>();
     {
@@ -185,11 +196,12 @@ TEST_CASE("C2DebugBundleTest", "[c2test]") {
     std::string log_text;
     log_text.resize(log_stream->size());
     log_stream->read(as_writable_bytes(std::span(log_text)));
-    REQUIRE(log_text.find("Tis but a scratch") != std::string::npos);
-    REQUIRE(archive_content["manifest.json"].find("minifi-archive-extensions") != std::string::npos);
+    CHECK(log_text.find("Tis but a scratch") != std::string::npos);
+    CHECK(archive_content["manifest.json"].find("minifi-archive-extensions") != std::string::npos);
     return true;
   });
 
+  harness.getLoggerProperties()->loadConfigureFile(home_dir / "conf" / "minifi-log.properties");
   harness.getConfiguration()->loadConfigureFile(home_dir / "conf" / "minifi.properties");
   harness.setUrl("http://localhost:0/heartbeat", &heartbeat_handler);
   harness.setUrl("http://localhost:0/acknowledge", &ack_handler);
@@ -213,6 +225,7 @@ TEST_CASE("Test that the debug bundle operation works when config.yml does not e
   std::filesystem::path home_dir = controller.createTempDirectory();
   minifi::utils::file::PathUtils::create_dir(home_dir / "conf");
   std::ofstream{home_dir / "conf" / "minifi.properties", std::ios::binary} << properties_file;
+  std::ofstream{home_dir / "conf" / "minifi-log.properties", std::ios::binary} << log_properties_file;
 
   VerifyDebugInfo harness(home_dir / "conf" / "config.yml", [&]() -> bool {
     if (!ack_handler.isAcknowledged("79")) {
@@ -224,6 +237,7 @@ TEST_CASE("Test that the debug bundle operation works when config.yml does not e
   });
 
   harness.getConfiguration()->loadConfigureFile(home_dir / "conf" / "minifi.properties");
+  harness.getLoggerProperties()->loadConfigureFile(home_dir / "conf" / "minifi-log.properties");
   harness.setUrl("http://localhost:0/heartbeat", &heartbeat_handler);
   harness.setUrl("http://localhost:0/acknowledge", &ack_handler);
   harness.setUrl("http://localhost:0/debug_bundle", &bundle_handler);

libminifi/test/libtest/integration/IntegrationBase.cpp

@@ -31,7 +31,8 @@
 namespace org::apache::nifi::minifi::test {
 
 IntegrationBase::IntegrationBase(const std::optional<std::filesystem::path>& test_file_location, const std::optional<std::filesystem::path>& home_path, std::chrono::milliseconds waitTime)
-    : configuration(std::make_shared<minifi::ConfigureImpl>()),
+    : logger_properties_(std::make_shared<core::logging::LoggerProperties>(std::filesystem::temp_directory_path())),
+      configuration(std::make_shared<minifi::ConfigureImpl>(std::nullopt, logger_properties_)),
       wait_time_(waitTime),
       home_path_(home_path) {
   flow_config_path_.config_path = test_file_location;

libminifi/test/libtest/integration/IntegrationBase.h

@@ -50,7 +50,8 @@ class IntegrationBase {
       std::chrono::milliseconds waitTime = std::chrono::milliseconds(DEFAULT_WAITTIME_MSECS));
   IntegrationBase(const IntegrationBase&) = delete;
   IntegrationBase(IntegrationBase&& other) noexcept
-      : configuration{std::move(other.configuration)},
+      : logger_properties_{std::move(other.logger_properties_)},
+        configuration{std::move(other.configuration)},
         flowController_{std::move(other.flowController_)},
         wait_time_{other.wait_time_},
         port{std::move(other.port)},
@@ -64,6 +65,7 @@ class IntegrationBase {
   IntegrationBase& operator=(const IntegrationBase&) = delete;
   IntegrationBase& operator=(IntegrationBase&& other) noexcept {
     if (&other == this) return *this;
+    logger_properties_ = std::move(other.logger_properties_);
     configuration = std::move(other.configuration);
     flowController_ = std::move(other.flowController_);
     wait_time_ = other.wait_time_;
@@ -94,6 +96,10 @@ class IntegrationBase {
     return configuration;
   }
 
+  const std::shared_ptr<core::logging::LoggerProperties>& getLoggerProperties() const {
+    return logger_properties_;
+  }
+
   void setConfiguration(std::shared_ptr<minifi::Configure> configuration) {
     this->configuration = std::move(configuration);
   }
@@ -124,6 +130,7 @@ class IntegrationBase {
   }
 
   void configureSecurity();
+  std::shared_ptr<core::logging::LoggerProperties> logger_properties_;
   std::shared_ptr<minifi::Configure> configuration;
   std::unique_ptr<minifi::utils::file::AssetManager> asset_manager_;
   std::unique_ptr<core::BulletinStore> bulletin_store_;

minifi-api/include/minifi-cpp/properties/Configure.h

@@ -41,6 +41,9 @@ class Configure : public virtual Configuration, public virtual core::AgentIdenti
 
   virtual void setFallbackAgentIdentifier(const std::string& id) = 0;
 
+  virtual std::optional<std::filesystem::path> logPropertiesFilePath() const = 0;
+  virtual std::optional<std::filesystem::path> extraLogPropertiesFilesDirName() const = 0;
+
   using Configuration::set;
   void set(const std::string& key, const std::string& value, PropertyChangeLifetime lifetime) override = 0;
   bool commitChanges() override = 0;

minifi-api/include/minifi-cpp/properties/Properties.h

@@ -60,6 +60,7 @@ class Properties {
   virtual utils::ChecksumCalculator& getChecksumCalculator() = 0;
   virtual std::filesystem::path getFilePath() const = 0;
   virtual std::map<std::string, std::string> getProperties() const = 0;
+  virtual std::filesystem::path extraPropertiesFilesDirName() const = 0;
 
   static std::shared_ptr<Properties> create();
 };