DPL: read metadata from parent files

If the metadata is not found in the main file and if there is a
list of parent files, try those as well.

Author: ktf

Framework/AnalysisSupport/src/Plugin.cxx

@@ -22,6 +22,7 @@
 #include <TObjString.h>
 #include <TString.h>
 #include <fmt/format.h>
+#include <memory>
 
 O2_DECLARE_DYNAMIC_LOG(analysis_support);
 
@@ -65,7 +66,7 @@ struct RunSummary : o2::framework::ServicePlugin {
   }
 };
 
-std::vector<std::string> getListOfTables(TFile* f)
+std::vector<std::string> getListOfTables(std::unique_ptr<TFile>& f)
 {
   std::vector<std::string> r;
   TList* keyList = f->GetListOfKeys();
@@ -83,6 +84,32 @@ std::vector<std::string> getListOfTables(TFile* f)
   }
   return r;
 }
+auto readMetadata(std::unique_ptr<TFile>& currentFile) -> std::vector<ConfigParamSpec>
+{
+  // Get the metadata, if any
+  auto m = (TMap*)currentFile->Get("metaData");
+  if (!m) {
+    return {};
+  }
+  std::vector<ConfigParamSpec> results;
+  auto it = m->MakeIterator();
+
+  // Serialise metadata into a ; separated string with : separating key and value
+  bool first = true;
+  while (auto obj = it->Next()) {
+    if (first) {
+      LOGP(info, "Metadata for file \"{}\":", currentFile->GetName());
+      first = false;
+    }
+    auto objString = (TObjString*)m->GetValue(obj);
+    LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
+    std::string key = "aod-metadata-" + std::string(obj->GetName());
+    char const* value = strdup(objString->String());
+    results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
+  }
+
+  return results;
+}
 
 struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
   ConfigDiscovery* create() override
@@ -94,8 +121,6 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
         if (filename.empty()) {
           return {};
         }
-        std::vector<ConfigParamSpec> results;
-        TFile* currentFile = nullptr;
         if (filename.at(0) == '@') {
           filename.erase(0, 1);
           // read the text file and set filename to the contents of the first line
@@ -110,39 +135,64 @@ struct DiscoverMetadataInAOD : o2::framework::ConfigDiscoveryPlugin {
           TGrid::Connect("alien://");
         }
         LOGP(info, "Loading metadata from file {} in PID {}", filename, getpid());
-        currentFile = TFile::Open(filename.c_str());
-        if (!currentFile) {
+        std::unique_ptr<TFile> currentFile{TFile::Open(filename.c_str())};
+        if (currentFile.get() == nullptr) {
           LOGP(fatal, "Couldn't open file \"{}\"!", filename);
         }
+        std::vector<ConfigParamSpec> results = readMetadata(currentFile);
+        // Found metadata already in the main file.
+        if (!results.empty()) {
+          auto tables = getListOfTables(currentFile);
+          if (tables.empty() == false) {
+            results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
+          }
+          results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
+          return results;
+        }
 
-        // Get the metadata, if any
-        auto m = (TMap*)currentFile->Get("metaData");
-        if (!m) {
+        // Lets try in parent files
+        auto parentFiles = (TMap*)currentFile->Get("parentFiles");
+        if (!parentFiles) {
           LOGP(info, "No metadata found in file \"{}\"", filename);
           results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
           return results;
         }
-        auto it = m->MakeIterator();
-
-        // Serialise metadata into a ; separated string with : separating key and value
-        bool first = true;
-        while (auto obj = it->Next()) {
-          if (first) {
-            LOGP(info, "Metadata for file \"{}\":", filename);
-            first = false;
+        for (auto* p : *parentFiles) {
+          std::string parentFilename = ((TPair*)p)->Value()->GetName();
+          // Do the replacement. Notice this will require changing aod-parent-base-path-replacement to be
+          // a workflow option (because the metadata itself is potentially changing the topology).
+          if (registry.isSet("aod-parent-base-path-replacement")) {
+            auto parentFileReplacement = registry.get<std::string>("aod-parent-base-path-replacement");
+            auto pos = parentFileReplacement.find(';');
+            if (pos == std::string::npos) {
+              throw std::runtime_error(fmt::format("Invalid syntax in aod-parent-base-path-replacement: \"{}\"", parentFileReplacement.c_str()));
+            }
+            auto from = parentFileReplacement.substr(0, pos);
+            auto to = parentFileReplacement.substr(pos + 1);
+            pos = parentFilename.find(from);
+            if (pos != std::string::npos) {
+              parentFilename.replace(pos, from.length(), to);
+            }
           }
-          auto objString = (TObjString*)m->GetValue(obj);
-          LOGP(info, "- {}: {}", obj->GetName(), objString->String().Data());
-          std::string key = "aod-metadata-" + std::string(obj->GetName());
-          char const* value = strdup(objString->String());
-          results.push_back(ConfigParamSpec{key, VariantType::String, value, {"Metadata in AOD"}});
-        }
 
-        auto tables = getListOfTables(currentFile);
-        if (tables.empty() == false) {
-          results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
+          std::unique_ptr<TFile> parentFile{TFile::Open(parentFilename.c_str())};
+          if (parentFile.get() == nullptr) {
+            LOGP(fatal, "Couldn't open derived file \"{}\"!", parentFilename);
+          }
+          results = readMetadata(parentFile);
+          // Found metadata already in the main file.
+          if (!results.empty()) {
+            auto tables = getListOfTables(parentFile);
+            if (tables.empty() == false) {
+              results.push_back(ConfigParamSpec{"aod-metadata-tables", VariantType::ArrayString, tables, {"Tables in first AOD"}});
+            }
+            results.push_back(ConfigParamSpec{"aod-metadata-source", VariantType::String, filename, {"File from which the metadata was extracted."}});
+            return results;
+          }
+          LOGP(info, "No metadata found in file \"{}\" nor in its parent file \"{}\"", filename, parentFilename);
+          break;
         }
-        currentFile->Close();
+        results.push_back(ConfigParamSpec{"aod-metadata-disable", VariantType::String, "1", {"Metadata not found in AOD"}});
         return results;
       }};
   }