Skip to content

Commit ed328b5

Browse files
ktfktf
committed
DPL: read metadata from parent files
If the metadata is not found in the main file and if there is a list of parent files, try those as well.
1 parent 2d015ad commit ed328b5

File tree

5 files changed

+96
-38
lines changed

5 files changed

+96
-38
lines changed

‎Framework/AnalysisSupport/src/AODJAlienReaderHelpers.cxx‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -120,10 +120,17 @@ static inline auto extractOriginalsTuple(framework::pack<Os...>, ProcessingConte
120120

121121
AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const& config)
122122
{
123-
auto callback = AlgorithmSpec{adaptStateful([](ConfigParamRegistry const& options,
124-
DeviceSpec const& spec,
125-
Monitoring& monitoring,
126-
DataProcessingStats& stats) {
123+
// aod-parent-base-path-replacement is now a workflow option, so it needs to be
124+
// retrieved from the ConfigContext. This is because we do not allow workflow options
125+
// to change over start-stop-start because they can affect the topology generation.
126+
std::string parentFileReplacement;
127+
if (config.options().isSet("aod-parent-base-path-replacement")) {
128+
parentFileReplacement = config.options().get<std::string>("aod-parent-base-path-replacement");
129+
}
130+
auto callback = AlgorithmSpec{adaptStateful([parentFileReplacement](ConfigParamRegistry const& options,
131+
DeviceSpec const& spec,
132+
Monitoring& monitoring,
133+
DataProcessingStats& stats) {
127134
// FIXME: not actually needed, since data processing stats can specify that we should
128135
// send the initial value.
129136
stats.updateStats({static_cast<short>(ProcessingStatsId::ARROW_BYTES_CREATED), DataProcessingStats::Op::Set, 0});
@@ -141,11 +148,6 @@ AlgorithmSpec AODJAlienReaderHelpers::rootFileReaderCallback(ConfigContext const
141148

142149
auto maxRate = options.get<float>("aod-max-io-rate");
143150

144-
std::string parentFileReplacement;
145-
if (options.isSet("aod-parent-base-path-replacement")) {
146-
parentFileReplacement = options.get<std::string>("aod-parent-base-path-replacement");
147-
}
148-
149151
int parentAccessLevel = 0;
150152
if (options.isSet("aod-parent-access-level")) {
151153
parentAccessLevel = options.get<int>("aod-parent-access-level");

‎Framework/AnalysisSupport/src/Plugin.cxx‎

Lines changed: 76 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include <TObjString.h>
2323
#include <TString.h>
2424
#include <fmt/format.h>
25+
#include <memory>
2526

2627
O2_DECLARE_DYNAMIC_LOG(analysis_support);
2728

@@ -65,7 +66,7 @@ struct RunSummary : o2::framework::ServicePlugin {
6566
}
6667
};
6768

68-
std::vector<std::string> getListOfTables(TFile* f)
69+
std::vector<std::string> getListOfTables(std::unique_ptr<TFile>& f)
6970
{
7071
std::vector<std::string> r;
7172
TList* keyList = f->GetListOfKeys();
@@ -83,6 +84,32 @@ std::vector<std::string> getListOfTables(TFile* f)
8384
}
8485
return r;
8586
}
87+
auto readMetadata(std::unique_ptr<TFile>& currentFile) -> std::vector<ConfigParamSpec>
88+
{
89+
// Get the metadata, if any
90+
auto m = (TMap*)currentFile->Get("metaData");
91+
if (!m) {
92+
return {};
93+
}
94+
std::vector<ConfigParamSpec> results;
95+
auto it = m->MakeIterator();
96+
97+
// Serialise metadata into a ; separated string with : separating key and value
98+
bool first = true;
99+
while (auto obj = it->Next()) {
100+
if (first) {
101+
LOGP(info, "Metadata for file \"{}\":", currentFile->GetName());
102+
first = false;
103+
}
104+
auto objString = (TObjString*)m->GetValue(obj);
105+
LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
106+
std::string key = "aod-metadata-" + std::string(obj->GetName());
107+
char const* value = strdup(objString->String());
108+
results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
109+
}
110+
111+
return results;
112+
}
86113

87114
struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
88115
ConfigDiscovery* create() override
@@ -94,8 +121,6 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
94121
if (filename.empty()) {
95122
return {};
96123
}
97-
std::vector<ConfigParamSpec> results;
98-
TFile* currentFile = nullptr;
99124
if (filename.at(0) == '@') {
100125
filename.erase(0, 1);
101126
// read the text file and set filename to the contents of the first line
@@ -110,39 +135,64 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
110135
TGrid::Connect("alien://");
111136
}
112137
LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
113-
currentFile = TFile::Open(filename.c_str());
114-
if (!currentFile) {
138+
std::unique_ptr<TFile> currentFile{TFile::Open(filename.c_str())};
139+
if (currentFile.get() == nullptr) {
115140
LOGP(fatal, "Couldn't open file \"{}\"!", filename);
116141
}
142+
std::vector<ConfigParamSpec> results = readMetadata(currentFile);
143+
// Found metadata already in the main file.
144+
if (!results.empty()) {
145+
auto tables = getListOfTables(currentFile);
146+
if (tables.empty() == false) {
147+
results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
148+
}
149+
results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
150+
return results;
151+
}
117152

118-
// Get the metadata, if any
119-
auto m = (TMap*)currentFile->Get("metaData");
120-
if (!m) {
153+
// Lets try in parent files
154+
auto parentFiles = (TMap*)currentFile->Get("parentFiles");
155+
if (!parentFiles) {
121156
LOGP(info, "No metadata found in file \"{}\"", filename);
122157
results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
123158
return results;
124159
}
125-
auto it = m->MakeIterator();
126-
127-
// Serialise metadata into a ; separated string with : separating key and value
128-
bool first = true;
129-
while (auto obj = it->Next()) {
130-
if (first) {
131-
LOGP(info, "Metadata for file \"{}\":", filename);
132-
first = false;
160+
for (auto* p : *parentFiles) {
161+
std::string parentFilename = ((TPair*)p)->Value()->GetName();
162+
// Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be
163+
// a workflow option (because the metadata itself is potentially changing the topology).
164+
if (registry.isSet("aod-parent-base-path-replacement")) {
165+
auto parentFileReplacement = registry.get<std::string>("aod-parent-base-path-replacement");
166+
auto pos = parentFileReplacement.find(';');
167+
if (pos == std::string::npos) {
168+
throw std::runtime_error(fmt::format("Invalid syntax in aod-parent-base-path-replacement: \"{}\"", parentFileReplacement.c_str()));
169+
}
170+
auto from = parentFileReplacement.substr(0, pos);
171+
auto to = parentFileReplacement.substr(pos + 1);
172+
pos = parentFilename.find(from);
173+
if (pos != std::string::npos) {
174+
parentFilename.replace(pos, from.length(), to);
175+
}
133176
}
134-
auto objString = (TObjString*)m->GetValue(obj);
135-
LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
136-
std::string key = "aod-metadata-" + std::string(obj->GetName());
137-
char const* value = strdup(objString->String());
138-
results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
139-
}
140177

141-
auto tables = getListOfTables(currentFile);
142-
if (tables.empty() == false) {
143-
results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
178+
std::unique_ptr<TFile> parentFile{TFile::Open(parentFilename.c_str())};
179+
if (parentFile.get() == nullptr) {
180+
LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename);
181+
}
182+
results = readMetadata(parentFile);
183+
// Found metadata already in the main file.
184+
if (!results.empty()) {
185+
auto tables = getListOfTables(parentFile);
186+
if (tables.empty() == false) {
187+
results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
188+
}
189+
results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
190+
return results;
191+
}
192+
LOGP(info, "No metadata found in file \"{}\" nor in its parent file \"{}\"", filename, parentFilename);
193+
break;
144194
}
145-
currentFile->Close();
195+
results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
146196
return results;
147197
}};
148198
}

‎Framework/Core/src/ConfigParamDiscovery.cxx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@ namespace o2::framework
2525
std::vector<ConfigParamSpec> ConfigParamDiscovery::discover(ConfigParamRegistry& registry, int argc, char** argv)
2626
{
2727
std::vector<char const*> capabilitiesSpecs = {
28+
"O2Framework:DiscoverAODOptionsInCommandLineCapability",
2829
"O2Framework:DiscoverMetadataInAODCapability",
2930
"O2Framework:DiscoverMetadataInCommandLineCapability",
30-
"O2Framework:DiscoverAODOptionsInCommandLineCapability",
3131
};
3232

3333
// Load all the requested plugins and discover what we can do.

‎Framework/Core/src/Plugin.cxx‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,10 @@ auto lookForCommandLineAODOptions = [](ConfigParamRegistry& registry, int argc,
5757
O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverAODOptionsInCommandLineCapability", "AOD options found in arguments. Populating from them.");
5858
return true;
5959
}
60+
if (arg.starts_with("--aod-parent-base-path-replacement")) {
61+
O2_SIGNPOST_EVENT_EMIT(capabilities, sid, "DiscoverAODOptionsInCommandLineCapability", "AOD options found in arguments. Populating from them.");
62+
return true;
63+
}
6064
}
6165
return false;
6266
};
@@ -137,7 +141,7 @@ struct DiscoverAODOptionsInCommandLine : o2::framework::ConfigDiscoveryPlugin {
137141
bool injectOption = true;
138142
for (size_t i = 0; i < argc; i++) {
139143
std::string_view arg = argv[i];
140-
if (!arg.starts_with("--aod-writer-")) {
144+
if (!arg.starts_with("--aod-writer-") && arg != "--aod-parent-base-path-replacement") {
141145
continue;
142146
}
143147
std::string key = arg.data() + 2;
@@ -149,6 +153,9 @@ struct DiscoverAODOptionsInCommandLine : o2::framework::ConfigDiscoveryPlugin {
149153
results.push_back(ConfigParamSpec{"aod-writer-compression", VariantType::Int, numericValue, {"AOD Compression options"}});
150154
injectOption = false;
151155
}
156+
if (key == "aod-parent-base-path-replacement") {
157+
results.push_back(ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, value, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}});
158+
}
152159
}
153160
if (injectOption) {
154161
results.push_back(ConfigParamSpec{"aod-writer-compression", VariantType::Int, 505, {"AOD Compression options"}});

‎Framework/Core/src/WorkflowHelpers.cxx‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -217,7 +217,6 @@ void WorkflowHelpers::injectServiceDevices(WorkflowSpec& workflow, ConfigContext
217217
ConfigParamSpec{"aod-max-io-rate", VariantType::Float, 0.f, {"Maximum I/O rate in MB/s"}},
218218
ConfigParamSpec{"aod-reader-json", VariantType::String, {"json configuration file"}},
219219
ConfigParamSpec{"aod-parent-access-level", VariantType::String, {"Allow parent file access up to specified level. Default: no (0)"}},
220-
ConfigParamSpec{"aod-parent-base-path-replacement", VariantType::String, {R"(Replace base path of parent files. Syntax: FROM;TO. E.g. "alien:///path/in/alien;/local/path". Enclose in "" on the command line.)"}},
221220
ConfigParamSpec{"time-limit", VariantType::Int64, 0ll, {"Maximum run time limit in seconds"}},
222221
ConfigParamSpec{"orbit-offset-enumeration", VariantType::Int64, 0ll, {"initial value for the orbit"}},
223222
ConfigParamSpec{"orbit-multiplier-enumeration", VariantType::Int64, 0ll, {"multiplier to get the orbit from the counter"}},

0 commit comments

Comments
 (0)