feat core: allow reading files from YamlConfig in ComponentsManager

Fixes #548

Tests: проотестировано CI
259bb37f1a639d7f8d6d3ca04e107aff05582e82

Author: apolukhin

core/src/components/manager_config.cpp

@@ -65,9 +65,9 @@ ManagerConfig ParseFromAny(
     config_vars = builder.ExtractValue();
   }
 
-  auto config =
-      yaml_config::YamlConfig(config_yaml, std::move(config_vars),
-                              yaml_config::YamlConfig::Mode::kEnvAllowed);
+  auto config = yaml_config::YamlConfig(
+      config_yaml, std::move(config_vars),
+      yaml_config::YamlConfig::Mode::kEnvAndFileAllowed);
   auto result = config[kManagerConfigField].As<ManagerConfig>();
   result.enabled_experiments =
       config[kUserverExperimentsField].As<utils::impl::UserverExperimentSet>(

universal/include/userver/yaml_config/yaml_config.hpp

@@ -34,29 +34,37 @@ using ParseException = formats::yaml::ParseException;
 /// @snippet universal/src/yaml_config/yaml_config_test.cpp  sample vars
 /// Then the result of `yaml["some_element"]["some"].As<int>()` is `42`.
 ///
-/// If YAML key ends on '#env' and the mode is YamlConfig::Mode::kEnvAllowed,
+/// If YAML key ends on '#env' and the mode is YamlConfig::Mode::kEnvAllowed
+/// or YamlConfig::Mode::kEnvAndFileAllowed,
 /// then the value of the key is searched in
 /// environment variables of the process and returned as a value. For example:
 /// @snippet universal/src/yaml_config/yaml_config_test.cpp  sample env
 ///
+/// If YAML key ends on '#file' and the mode is
+/// YamlConfig::Mode::kEnvAndFileAllowed, then the value of the key is the
+/// content of specified YAML parsed file. For example:
+/// @snippet universal/src/yaml_config/yaml_config_test.cpp  sample read_file
+///
 /// If YAML key ends on '#fallback', then the value of the key is used as a
-/// fallback for environment and `$` variables. For example for the following
-/// YAML with YamlConfig::Mode::kEnvAllowed:
+/// fallback for environment, file and `$` variables. For example for the
+/// following YAML with YamlConfig::Mode::kEnvAndFileAllowed:
 /// @snippet universal/src/yaml_config/yaml_config_test.cpp  sample multiple
 /// The result of `yaml["some_element"]["some"].As<int>()` is the value of
 /// `variable` from `config_vars` if it exists; otherwise the value is the
 /// contents of the environment variable `SOME_ENV_VARIABLE` if it exists;
-/// otherwise the value if `100500`, from the fallback.
+/// otherwise the value is the content of the file with name `file.yaml`;
+/// otherwise the value is `100500`, from the fallback.
 ///
 /// Another example:
 /// @snippet universal/src/yaml_config/yaml_config_test.cpp  sample env fallback
 /// With YamlConfig::Mode::kEnvAllowed the result of
 /// `yaml["some_element"]["value"].As<int>()` is the value of `ENV_NAME`
 /// environment variable if it exists; otherwise it is `5`.
 ///
-/// @warning YamlConfig::Mode::kEnvAllowed should be used only on configs that
+/// @warning YamlConfig::Mode::kEnvAllowed or
+/// YamlConfig::Mode::kEnvAndFileAllowed should be used only on configs that
 /// come from trusted environments. Otherwise, an attacker could create a
-/// config with `#env` and read any of your environment variables, including
+/// config and read any of your environment variables of files, including
 /// variables that contain passwords and other sensitive data.
 class YamlConfig {
  public:
@@ -68,8 +76,10 @@ class YamlConfig {
   struct DefaultConstructed {};
 
   enum class Mode {
-    kSecure,      /// < secure mode, without reading environment variables
-    kEnvAllowed,  /// < allows reading of environment variables
+    kSecure,  /// < secure mode, without reading environment variables or files
+    kEnvAllowed,         /// < allows reading of environment variables
+    kEnvAndFileAllowed,  /// < allows reading of environment variables and
+                         /// files
   };
 
   using const_iterator = Iterator<IterTraits>;

universal/src/yaml_config/impl/validate_static_config.cpp

@@ -16,18 +16,14 @@ namespace yaml_config::impl {
 
 namespace {
 
-constexpr std::string_view kFallbackSuffix = "#fallback";
-constexpr std::string_view kEnvSuffix = "#env";
-
+// TODO: remove in TAXICOMMON-8973
 std::string RemoveFallbackAndEnvSuffix(std::string_view option) {
-  if (utils::text::EndsWith(option, kFallbackSuffix)) {
-    return std::string(
-        option.substr(0, option.length() - kFallbackSuffix.length()));
-  }
-  if (utils::text::EndsWith(option, kEnvSuffix)) {
-    return std::string(option.substr(0, option.length() - kEnvSuffix.length()));
+  for (const std::string_view suffix : {"#env", "#file", "#fallback"}) {
+    if (utils::text::EndsWith(option, suffix)) {
+      return std::string{option.substr(0, option.size() - suffix.size())};
+    }
   }
-  return std::string(option);
+  return std::string{option};
 }
 
 bool IsTypeValid(FieldType type, const formats::yaml::Value& value) {

universal/src/yaml_config/yaml_config.cpp

@@ -1,14 +1,15 @@
 #include <userver/yaml_config/yaml_config.hpp>
 
 #include <fmt/format.h>
-#include <boost/algorithm/string/predicate.hpp>
+#include <boost/filesystem/operations.hpp>
 
 #include <userver/formats/common/conversion_stack.hpp>
 #include <userver/formats/json/value.hpp>
 #include <userver/formats/json/value_builder.hpp>
 #include <userver/formats/yaml/serialize.hpp>
 #include <userver/logging/log.hpp>
 #include <userver/utils/string_to_duration.hpp>
+#include <userver/utils/text_light.hpp>
 
 USERVER_NAMESPACE_BEGIN
 
@@ -26,52 +27,72 @@ std::string GetSubstitutionVarName(const formats::yaml::Value& value) {
   return value.As<std::string>().substr(1);
 }
 
-std::string GetFallbackName(std::string_view str) {
-  return std::string{str} + "#fallback";
-}
-
 std::string GetEnvName(std::string_view str) {
   return std::string{str} + "#env";
 }
 
+std::string GetFileName(std::string_view str) {
+  return std::string{str} + "#file";
+}
+
+std::string GetFallbackName(std::string_view str) {
+  return std::string{str} + "#fallback";
+}
+
 template <typename Field>
 YamlConfig MakeMissingConfig(const YamlConfig& config, Field field) {
   const auto path = formats::common::MakeChildPath(config.GetPath(), field);
   return {formats::yaml::Value()[path], {}};
 }
 
 void AssertEnvMode(YamlConfig::Mode mode) {
-  if (mode != YamlConfig::Mode::kEnvAllowed) {
+  if (mode == YamlConfig::Mode::kSecure) {
     throw std::runtime_error(
-        "YamlConfig was not constructed with Mode::kEnvAllowed but an attempt "
+        "YamlConfig was not constructed with Mode::kEnvAllowed or "
+        "Mode::kEnvAndFileAllowed but an attempt "
         "to read an environment variable was made");
   }
 }
 
-std::optional<formats::yaml::Value> GetFromEnvByKey(
-    std::string_view key, const formats::yaml::Value& yaml,
-    YamlConfig::Mode mode) {
-  const auto env_name = yaml[GetEnvName(key)];
-  if (!env_name.IsMissing()) {
-    AssertEnvMode(mode);
-
-    // NOLINTNEXTLINE(concurrency-mt-unsafe)
-    const auto* env_value = std::getenv(env_name.As<std::string>().c_str());
-    if (env_value) {
-      LOG_INFO() << "using env value for '" << key << '\'';
-      return formats::yaml::FromString(env_value);
-    }
+void AssertFileMode(YamlConfig::Mode mode) {
+  if (mode != YamlConfig::Mode::kEnvAndFileAllowed) {
+    throw std::runtime_error(
+        "YamlConfig was not constructed with Mode::kEnvAndFileAllowed "
+        "but an attempt to read a file was made");
+  }
+}
 
-    const auto fallback_name = GetFallbackName(key);
-    if (yaml.HasMember(fallback_name)) {
-      LOG_INFO() << "using fallback value for '" << key << '\'';
-      return yaml[fallback_name];
-    }
+std::optional<formats::yaml::Value> GetFromEnvImpl(
+    const formats::yaml::Value& env_name, YamlConfig::Mode mode) {
+  if (env_name.IsMissing()) {
+    return {};
+  }
+
+  AssertEnvMode(mode);
+
+  // NOLINTNEXTLINE(concurrency-mt-unsafe)
+  const auto* env_value = std::getenv(env_name.As<std::string>().c_str());
+  if (env_value) {
+    return formats::yaml::FromString(env_value);
   }
 
   return {};
 }
 
+std::optional<formats::yaml::Value> GetFromFileImpl(
+    const formats::yaml::Value& file_name, YamlConfig::Mode mode) {
+  if (file_name.IsMissing()) {
+    return {};
+  }
+
+  AssertFileMode(mode);
+  const auto str_filename = file_name.As<std::string>();
+  if (!boost::filesystem::exists(str_filename)) {
+    return {};
+  }
+  return formats::yaml::blocking::FromFile(str_filename);
+}
+
 }  // namespace
 
 YamlConfig::YamlConfig(formats::yaml::Value yaml,
@@ -83,54 +104,54 @@ YamlConfig::YamlConfig(formats::yaml::Value yaml,
 const formats::yaml::Value& YamlConfig::Yaml() const { return yaml_; }
 
 YamlConfig YamlConfig::operator[](std::string_view key) const {
-  if (boost::algorithm::ends_with(key, "#env")) {
-    auto env_value = GetFromEnvByKey(key, yaml_, mode_);
-    if (env_value) {
-      // Strip substitutions off to disallow nested substitutions
-      return YamlConfig{std::move(*env_value), {}, Mode::kSecure};
-    }
-
-    // Avoid parsing #env as a string
+  // TODO: fix the iterators and assert this case in TAXICOMMON-8973
+  if (utils::text::EndsWith(key, "#env") ||
+      utils::text::EndsWith(key, "#file") ||
+      utils::text::EndsWith(key, "#fallback")) {
     return MakeMissingConfig(*this, key);
   }
 
   auto value = yaml_[key];
 
-  if (IsSubstitution(value)) {
+  const bool is_substitution = IsSubstitution(value);
+  if (is_substitution) {
     const auto var_name = GetSubstitutionVarName(value);
 
     auto var_data = config_vars_[var_name];
     if (!var_data.IsMissing()) {
       // Strip substitutions off to disallow nested substitutions
       return YamlConfig{std::move(var_data), {}, Mode::kSecure};
     }
+  }
 
-    auto env_value = GetFromEnvByKey(key, yaml_, mode_);
-    if (env_value) {
-      // Strip substitutions off to disallow nested substitutions
-      return YamlConfig{std::move(*env_value), {}, Mode::kSecure};
-    }
+  if (!value.IsMissing() && !is_substitution) {
+    return YamlConfig{std::move(value), config_vars_, mode_};
+  }
 
+  const auto env_name = yaml_[GetEnvName(key)];
+  auto env_value = GetFromEnvImpl(env_name, mode_);
+  if (env_value) {
+    // Strip substitutions off to disallow nested substitutions
+    return YamlConfig{std::move(*env_value), {}, Mode::kSecure};
+  }
+
+  const auto file_name = yaml_[GetFileName(key)];
+  auto file_value = GetFromFileImpl(file_name, mode_);
+  if (file_value) {
+    // Strip substitutions off to disallow nested substitutions
+    return YamlConfig{std::move(*file_value), {}, Mode::kSecure};
+  }
+
+  if (is_substitution || !env_name.IsMissing() || !file_name.IsMissing()) {
     const auto fallback_name = GetFallbackName(key);
     if (yaml_.HasMember(fallback_name)) {
       LOG_INFO() << "using fallback value for '" << key << '\'';
       // Strip substitutions off to disallow nested substitutions
       return YamlConfig{yaml_[fallback_name], {}, Mode::kSecure};
     }
-
-    // Avoid parsing $substitution as a string
-    return MakeMissingConfig(*this, key);
-  }
-
-  if (value.IsMissing()) {
-    auto env_value = GetFromEnvByKey(key, yaml_, mode_);
-    if (env_value) {
-      // Strip substitutions off to disallow nested substitutions
-      return YamlConfig{std::move(*env_value), {}, Mode::kSecure};
-    }
   }
 
-  return YamlConfig{std::move(value), config_vars_, mode_};
+  return MakeMissingConfig(*this, key);
 }
 
 YamlConfig YamlConfig::operator[](size_t index) const {

universal/src/yaml_config/yaml_config_test.cpp

@@ -5,6 +5,8 @@
 #include <formats/common/value_test.hpp>
 #include <userver/formats/yaml/serialize.hpp>
 #include <userver/formats/yaml/value_builder.hpp>
+#include <userver/fs/blocking/temp_directory.hpp>
+#include <userver/fs/blocking/write.hpp>
 #include <userver/utest/assert_macros.hpp>
 
 USERVER_NAMESPACE_BEGIN
@@ -75,6 +77,7 @@ TEST(YamlConfig, SampleMultiple) {
 # yaml
 some_element:
     some: $variable
+    some#file: /some/path/to/the/file.yaml
     some#env: SOME_ENV_VARIABLE
     some#fallback: 100500
 # /// [sample multiple]
@@ -91,6 +94,10 @@ TEST(YamlConfig, SampleMultiple) {
                                  yaml_config::YamlConfig::Mode::kEnvAllowed);
   EXPECT_EQ(yaml["some_element"]["some"].As<int>(), 42);
 
+  yaml = yaml_config::YamlConfig(
+      node, vars, yaml_config::YamlConfig::Mode::kEnvAndFileAllowed);
+  EXPECT_EQ(yaml["some_element"]["some"].As<int>(), 42);
+
   yaml = yaml_config::YamlConfig(node, {},
                                  yaml_config::YamlConfig::Mode::kEnvAllowed);
   EXPECT_EQ(yaml["some_element"]["some"].As<int>(), 100);
@@ -103,6 +110,10 @@ TEST(YamlConfig, SampleMultiple) {
 
   yaml = yaml_config::YamlConfig(node, {},
                                  yaml_config::YamlConfig::Mode::kEnvAllowed);
+  UEXPECT_THROW(yaml["some_element"]["some"].As<int>(), std::exception);
+
+  yaml = yaml_config::YamlConfig(
+      node, {}, yaml_config::YamlConfig::Mode::kEnvAndFileAllowed);
   EXPECT_EQ(yaml["some_element"]["some"].As<int>(), 100500);
 }
 
@@ -121,6 +132,48 @@ TEST(YamlConfig, SampleEnvFallback) {
   EXPECT_EQ(yaml["some_element"]["some"].As<int>(), 5);
 }
 
+TEST(YamlConfig, SampleFile) {
+  const auto dir = fs::blocking::TempDirectory::Create();
+  const auto path_to_file = dir.GetPath() + "/read_file_sample";
+
+  /// [sample read_file]
+  fs::blocking::RewriteFileContents(path_to_file, R"(some_key: ['a', 'b'])");
+  const auto yaml_content = fmt::format("some#file: {}", path_to_file);
+  auto node = formats::yaml::FromString(yaml_content);
+
+  yaml_config::YamlConfig yaml(
+      std::move(node), {}, yaml_config::YamlConfig::Mode::kEnvAndFileAllowed);
+  EXPECT_EQ(yaml["some"]["some_key"][0].As<std::string>(), "a");
+  /// [sample read_file]
+}
+
+TEST(YamlConfig, FileFallback) {
+  auto node = formats::yaml::FromString(R"(
+some_element:
+    some#file: /some/path/to/the/file.yaml
+    some#fallback: 5
+  )");
+
+  yaml_config::YamlConfig yaml(
+      std::move(node), {}, yaml_config::YamlConfig::Mode::kEnvAndFileAllowed);
+  EXPECT_EQ(yaml["some_element"]["some"].As<int>(), 5);
+}
+
+TEST(YamlConfig, FileFallbackUnallowed) {
+  auto node = formats::yaml::FromString(R"(
+some_element:
+    some#file: /some/path/to/the/file.yaml
+    some#fallback: 5
+  )");
+
+  yaml_config::YamlConfig yaml(node, {},
+                               yaml_config::YamlConfig::Mode::kEnvAllowed);
+  UEXPECT_THROW(yaml["some_element"]["some"].As<int>(), std::exception);
+
+  yaml = yaml_config::YamlConfig{node, {}, {}};
+  UEXPECT_THROW(yaml["some_element"]["some"].As<int>(), std::exception);
+}
+
 TEST(YamlConfig, Basic) {
   auto node = formats::yaml::FromString(R"(
     string: hello