Skip to content

Commit 4216b1e

Browse files
junhaoliaogibber9809haiqi96kirkrodrigues
authored
feat(core)!: Replace YAML config with CLI args and env vars for metadata DB (resolves #1146). (#1148)
Co-authored-by: Devin Gibson <[email protected]> Co-authored-by: haiqi96 <[email protected]> Co-authored-by: kirkrodrigues <[email protected]>
1 parent 58fd2c1 commit 4216b1e

27 files changed

+751
-328
lines changed

‎components/core/CMakeLists.txt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -558,6 +558,8 @@ set(SOURCE_FILES_unitTest
558558
src/clp/FileReader.hpp
559559
src/clp/FileWriter.cpp
560560
src/clp/FileWriter.hpp
561+
src/clp/global_metadata_db_utils.hpp
562+
src/clp/global_metadata_db_utils.cpp
561563
src/clp/GlobalMetadataDB.hpp
562564
src/clp/GlobalMetadataDBConfig.cpp
563565
src/clp/GlobalMetadataDBConfig.hpp
@@ -708,6 +710,7 @@ set(SOURCE_FILES_unitTest
708710
tests/test-ffi_KeyValuePairLogEvent.cpp
709711
tests/test-ffi_SchemaTree.cpp
710712
tests/test-FileDescriptorReader.cpp
713+
tests/test-GlobalMetadataDBConfig.cpp
711714
tests/test-GrepCore.cpp
712715
tests/test-hash_utils.cpp
713716
tests/test-ir_encoding_methods.cpp
@@ -766,7 +769,6 @@ if(CLP_BUILD_TESTING)
766769
${STD_FS_LIBS}
767770
clp::regex_utils
768771
clp::string_utils
769-
yaml-cpp
770772
ystdlib::containers
771773
ystdlib::error_handling
772774
${LIBLZMA_LIBRARIES}

‎components/core/cmake/Options/options.cmake‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,6 @@ function(set_clp_tests_dependencies)
180180
CLP_NEED_SIMDJSON
181181
CLP_NEED_SPDLOG
182182
CLP_NEED_SQLITE
183-
CLP_NEED_YAMLCPP
184183
CLP_NEED_YSTDLIB
185184
CLP_NEED_ZSTD
186185
)

‎components/core/config/metadata-db.yml‎

Lines changed: 0 additions & 10 deletions
This file was deleted.

‎components/core/src/clp/GlobalMetadataDBConfig.cpp‎

Lines changed: 162 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -1,110 +1,185 @@
11
#include "GlobalMetadataDBConfig.hpp"
22

3+
#include <cstdlib>
4+
#include <istream>
5+
#include <stdexcept>
6+
#include <string>
7+
#include <string_view>
8+
9+
#include <boost/program_options/options_description.hpp>
10+
#include <boost/program_options/value_semantic.hpp>
311
#include <fmt/core.h>
4-
#include <yaml-cpp/yaml.h>
512

6-
using std::exception;
13+
#include "GlobalMySQLMetadataDB.hpp"
14+
#include "type_utils.hpp"
15+
716
using std::invalid_argument;
817
using std::string;
918

10-
static exception get_yaml_missing_key_exception(string const& key_name) {
11-
throw invalid_argument(fmt::format("Missing key '{}'", key_name));
19+
namespace {
20+
// Constants
21+
constexpr clp::GlobalMetadataDBConfig::MetadataDBType cDefaultMetadataDbType{
22+
clp::GlobalMetadataDBConfig::MetadataDBType::SQLite
23+
};
24+
constexpr std::string_view cDefaultMetadataDbHost{"127.0.0.1"};
25+
constexpr int cDefaultMetadataDbPort{3306};
26+
constexpr std::string_view cDefaultMetadataDbName{"clp-db"};
27+
constexpr std::string_view cDefaultMetadataTablePrefix{"clp_"};
28+
constexpr int cMinPort{1};
29+
constexpr int cMaxPort{65'535};
30+
} // namespace
31+
32+
namespace clp {
33+
auto operator>>(std::istream& in, GlobalMetadataDBConfig::MetadataDBType& metadata_db_type)
34+
-> std::istream& {
35+
string db_type_string;
36+
in >> db_type_string;
37+
38+
for (size_t i = 0; i < GlobalMetadataDBConfig::cMetadataDBTypeNames.size(); ++i) {
39+
if (GlobalMetadataDBConfig::cMetadataDBTypeNames.at(i) == db_type_string) {
40+
metadata_db_type = static_cast<GlobalMetadataDBConfig::MetadataDBType>(i);
41+
return in;
42+
}
43+
}
44+
45+
throw invalid_argument(fmt::format("Unknown database type: {}", db_type_string));
1246
}
1347

14-
static exception
15-
get_yaml_unconvertable_value_exception(string const& key_name, string const& destination_type) {
16-
throw invalid_argument(
17-
fmt::format("'{}' could not be converted to type '{}'", key_name, destination_type)
18-
);
48+
GlobalMetadataDBConfig::GlobalMetadataDBConfig(
49+
boost::program_options::options_description& options_description
50+
) {
51+
std::string_view const cMetadataDbTypeMysqlOptDescPrefix{fmt::format(
52+
"(--db-type={} only)",
53+
cMetadataDBTypeNames[enum_to_underlying_type(MetadataDBType::MySQL)]
54+
)};
55+
56+
// clang-format off
57+
options_description.add_options()
58+
(
59+
"db-type",
60+
boost::program_options::value<MetadataDBType>(&m_metadata_db_type)
61+
->default_value(
62+
cDefaultMetadataDbType,
63+
string(
64+
cMetadataDBTypeNames[
65+
enum_to_underlying_type(cDefaultMetadataDbType)
66+
]
67+
)
68+
),
69+
fmt::format(
70+
"Database type [{} | {}]",
71+
cMetadataDBTypeNames[enum_to_underlying_type(MetadataDBType::SQLite)],
72+
cMetadataDBTypeNames[enum_to_underlying_type(MetadataDBType::MySQL)]
73+
).c_str()
74+
)
75+
(
76+
"db-host",
77+
boost::program_options::value<string>(&m_metadata_db_host)
78+
->default_value(string(cDefaultMetadataDbHost)),
79+
fmt::format(
80+
"{} Database host",
81+
cMetadataDbTypeMysqlOptDescPrefix
82+
).c_str()
83+
)
84+
(
85+
"db-port",
86+
boost::program_options::value<int>(&m_metadata_db_port)
87+
->default_value(cDefaultMetadataDbPort),
88+
fmt::format(
89+
"{} Database port",
90+
cMetadataDbTypeMysqlOptDescPrefix
91+
).c_str()
92+
)
93+
(
94+
"db-name",
95+
boost::program_options::value<string>(&m_metadata_db_name)
96+
->default_value(string(cDefaultMetadataDbName)),
97+
fmt::format(
98+
"{} Database name",
99+
cMetadataDbTypeMysqlOptDescPrefix
100+
).c_str()
101+
)
102+
(
103+
"db-table-prefix",
104+
boost::program_options::value<string>(&m_metadata_table_prefix)
105+
->default_value(string(cDefaultMetadataTablePrefix)),
106+
fmt::format(
107+
"{} Database table prefix",
108+
cMetadataDbTypeMysqlOptDescPrefix
109+
).c_str()
110+
);
111+
// clang-format on
19112
}
20113

21-
namespace clp {
22-
void GlobalMetadataDBConfig::parse_config_file(string const& config_file_path) {
23-
YAML::Node config = YAML::LoadFile(config_file_path);
114+
auto GlobalMetadataDBConfig::read_credentials_from_env_if_needed() -> void {
115+
if (MetadataDBType::SQLite == m_metadata_db_type) {
116+
// SQLite doesn't require extra parameters.
117+
return;
118+
}
24119

25-
if (!config["type"]) {
26-
throw get_yaml_missing_key_exception("type");
120+
// Silence the check since this class won't be used in a multithreaded context.
121+
// NOLINTNEXTLINE(concurrency-mt-unsafe)
122+
if (auto const* db_username{std::getenv("CLP_DB_USER")}; nullptr != db_username) {
123+
m_metadata_db_username.emplace(db_username);
27124
}
28125

29-
auto db_type_string = config["type"].as<string>();
30-
if ("sqlite" == db_type_string) {
31-
m_metadata_db_type = MetadataDBType::SQLite;
32-
} else if ("mysql" == db_type_string) {
33-
m_metadata_db_type = MetadataDBType::MySQL;
126+
// Silence the check since this class won't be used in a multithreaded context.
127+
// NOLINTNEXTLINE(concurrency-mt-unsafe)
128+
if (auto const* db_password{std::getenv("CLP_DB_PASS")}; nullptr != db_password) {
129+
m_metadata_db_password.emplace(db_password);
130+
}
131+
}
34132

35-
if (!config["host"]) {
36-
throw get_yaml_missing_key_exception("host");
37-
}
38-
try {
39-
m_metadata_db_host = config["host"].as<string>();
40-
} catch (YAML::BadConversion& e) {
41-
throw get_yaml_unconvertable_value_exception("host", "string");
42-
}
43-
if (m_metadata_db_host.empty()) {
44-
throw invalid_argument("Database 'host' not specified or empty.");
45-
}
133+
auto GlobalMetadataDBConfig::validate() const -> void {
134+
if (m_metadata_db_type == MetadataDBType::SQLite) {
135+
// SQLite doesn't require extra parameters.
136+
if (cDefaultMetadataDbHost != m_metadata_db_host
137+
|| cDefaultMetadataDbPort != m_metadata_db_port
138+
|| cDefaultMetadataDbName != m_metadata_db_name
139+
|| cDefaultMetadataTablePrefix != m_metadata_table_prefix
140+
|| m_metadata_db_username.has_value() || m_metadata_db_password.has_value())
141+
{
142+
throw invalid_argument(
143+
fmt::format(
144+
"MySQL-specific parameters cannot be used with --db-type={}."
145+
" Please remove them or set '--db-type={}'.",
146+
cMetadataDBTypeNames[enum_to_underlying_type(m_metadata_db_type)],
147+
cMetadataDBTypeNames[enum_to_underlying_type(MetadataDBType::MySQL)]
148+
)
149+
);
150+
}
151+
return;
152+
}
46153

47-
if (!config["port"]) {
48-
throw get_yaml_missing_key_exception("port");
49-
}
50-
try {
51-
m_metadata_db_port = config["port"].as<int>();
52-
} catch (YAML::BadConversion& e) {
53-
throw get_yaml_unconvertable_value_exception("port", "int");
54-
}
55-
if (m_metadata_db_port < 0) {
56-
throw invalid_argument("Database 'port' cannot be negative.");
57-
}
154+
if (m_metadata_db_host.empty()) {
155+
throw invalid_argument("Database '--db-host' is empty.");
156+
}
58157

59-
if (!config["name"]) {
60-
throw get_yaml_missing_key_exception("name");
61-
}
62-
try {
63-
m_metadata_db_name = config["name"].as<string>();
64-
} catch (YAML::BadConversion& e) {
65-
throw get_yaml_unconvertable_value_exception("name", "string");
66-
}
67-
if (m_metadata_db_name.empty()) {
68-
throw invalid_argument("Database 'name' not specified or empty.");
69-
}
158+
if (cMinPort > m_metadata_db_port || cMaxPort < m_metadata_db_port) {
159+
throw invalid_argument(
160+
fmt::format(
161+
"Database '--db-port' is out of range [{}, {}]: {}",
162+
cMinPort,
163+
cMaxPort,
164+
m_metadata_db_port
165+
)
166+
);
167+
}
70168

71-
if (!config["username"]) {
72-
throw get_yaml_missing_key_exception("username");
73-
}
74-
try {
75-
m_metadata_db_username = config["username"].as<string>();
76-
} catch (YAML::BadConversion& e) {
77-
throw get_yaml_unconvertable_value_exception("username", "string");
78-
}
79-
if (m_metadata_db_username.empty()) {
80-
throw invalid_argument("Database 'username' not specified or empty.");
81-
}
169+
if (m_metadata_db_name.empty()) {
170+
throw invalid_argument("Database '--db-name' is empty.");
171+
}
82172

83-
if (!config["password"]) {
84-
throw get_yaml_missing_key_exception("password");
85-
}
86-
try {
87-
m_metadata_db_password = config["password"].as<string>();
88-
} catch (YAML::BadConversion& e) {
89-
throw get_yaml_unconvertable_value_exception("password", "string");
90-
}
91-
if (m_metadata_db_password.empty()) {
92-
throw invalid_argument("Database 'password' not specified or empty.");
93-
}
173+
if (m_metadata_table_prefix.empty()) {
174+
throw invalid_argument("Database '--db-table_prefix' is empty.");
175+
}
94176

95-
if (!config["table_prefix"]) {
96-
throw get_yaml_missing_key_exception("table_prefix");
97-
}
98-
try {
99-
m_metadata_table_prefix = config["table_prefix"].as<string>();
100-
} catch (YAML::BadConversion& e) {
101-
throw get_yaml_unconvertable_value_exception("table_prefix", "string");
102-
}
103-
if (m_metadata_table_prefix.empty()) {
104-
throw invalid_argument("Database 'table_prefix' not specified or empty.");
105-
}
106-
} else {
107-
throw invalid_argument("Unknown type");
177+
if (false == m_metadata_db_username.has_value()) {
178+
throw invalid_argument("Environment variable 'CLP_DB_USER' not set.");
179+
}
180+
181+
if (false == m_metadata_db_password.has_value()) {
182+
throw invalid_argument("Environment variable 'CLP_DB_PASS' not set.");
108183
}
109184
}
110185
} // namespace clp

0 commit comments

Comments
 (0)