AOD: File Inspector

Author: f3sch

Framework/AODMerger/CMakeLists.txt

@@ -12,14 +12,23 @@
 o2_add_executable(merger
                   COMPONENT_NAME aod
                   SOURCES src/aodMerger.cxx
-                  PUBLIC_LINK_LIBRARIES  ROOT::Core ROOT::Net)
+                  PUBLIC_LINK_LIBRARIES ROOT::Core ROOT::Net)
 
 o2_add_executable(thinner
                   COMPONENT_NAME aod
                   SOURCES src/aodThinner.cxx
-                  PUBLIC_LINK_LIBRARIES  ROOT::Core ROOT::Net)
+                  PUBLIC_LINK_LIBRARIES ROOT::Core ROOT::Net)
 
 o2_add_executable(strainer
                   COMPONENT_NAME aod
                   SOURCES src/aodStrainer.cxx
-                  PUBLIC_LINK_LIBRARIES  ROOT::Core ROOT::Net)
+                  PUBLIC_LINK_LIBRARIES ROOT::Core ROOT::Net)
+
+o2_add_executable(inspector
+                  COMPONENT_NAME aod
+                  SOURCES src/aodInspector.cxx
+                  PUBLIC_LINK_LIBRARIES ROOT::Core ROOT::Net
+                                        Boost::program_options
+                                        O2::Version
+                                        O2::CommonUtils
+                                        O2::FrameworkLogger)

Framework/AODMerger/src/aodInspector.cxx

@@ -0,0 +1,282 @@
+#include <RtypesCore.h>
+#include <TKey.h>
+#include "O2Version.h"
+#include "Framework/Logger.h"
+#include "CommonUtils/StringUtils.h"
+
+#include "TFile.h"
+#include "TBranch.h"
+#include "TTree.h"
+#include "TGrid.h"
+#include "TMap.h"
+#include "TObjString.h"
+#include "TStopwatch.h"
+
+#include <boost/program_options.hpp>
+
+#include <filesystem>
+#include <memory>
+#include <vector>
+#include <string>
+#include <fstream>
+#include <unordered_map>
+
+namespace bpo = boost::program_options;
+namespace fs = std::filesystem;
+
+using FilePtr = std::unique_ptr<TFile>;
+
+// Description of possible exit codes
+enum ExitCodes : int {
+  Success = 0,         // there is only one success code
+  Fail = 1,            // non-descriptive general fail
+  ArgParse = 2,        // failed cli arg parse
+  NonExistantFile = 3, // the input file does not exists
+  InputList = 3,       // input file list not opened
+};
+
+// Collected info for a specific table
+struct TableInfo {
+  Long64_t uncBytes{0};
+  Long64_t compBytes{0};
+  Long64_t seen{0};
+};
+using TableInfoMap = std::unordered_map<std::string, TableInfo>;
+void printTableInfoMap(const TableInfoMap& info);
+void printMetaData(const TMap* meta);
+bool checkFileExists(const std::string& fileName);
+FilePtr openFile(const std::string& fileName);
+void testFile(const std::string& file, TableInfoMap& info);
+void testDataFrame(TDirectory* dir, TableInfoMap& info);
+void testTree(TTree* tree, TableInfo& table);
+void testBranch(TBranch* bra, TableInfoMap& info);
+
+bool initOptionsAndParse(bpo::options_description& options, int argc, char* argv[], bpo::variables_map& vm)
+{
+  options.add_options()(
+    "help,h", "Program description")(
+    "input,i", bpo::value<std::string>()->default_value("AO2D.root"), "Single AO2D file or text file with path to files per line");
+  try {
+    bpo::store(bpo::parse_command_line(argc, argv, options), vm);
+    bpo::notify(vm);
+  } catch (const bpo::error& e) {
+    LOGP(error, "Exception during command-line argument parsing: '{}'", e.what());
+    LOG(info) << options;
+    return false;
+  }
+  return true;
+}
+
+int main(int argc, char* argv[])
+{
+  LOGP(info, "AO2D Inspector (Version '{}'; git '{}'; Info '{}')", o2::fullVersion(), o2::gitRevision(), o2::getBuildInfo());
+  bpo::options_description options("Allowed options");
+  bpo::variables_map vm;
+  if (!initOptionsAndParse(options, argc, argv, vm)) {
+    return ExitCodes::ArgParse;
+  }
+
+  if (vm.count("help") != 0u) { // first thing check if help requested
+    LOG(info) << options;
+    return ExitCodes::Success;
+  }
+
+  auto inputFile = vm["input"].as<std::string>();
+  if (!checkFileExists(inputFile)) {
+    return ExitCodes::NonExistantFile;
+  }
+
+  TStopwatch watch;
+  watch.Start();
+
+  // built list of files to open eventually
+  std::vector<std::string> fileNames;
+  if (inputFile.ends_with(".root")) {
+    fileNames.emplace_back(inputFile);
+  } else { // treat as input list
+    std::ifstream file(inputFile);
+    if (!file.is_open()) {
+      LOGP(error, "Cannot open input list '{}'", inputFile);
+      return ExitCodes::InputList;
+    }
+    std::string line;
+    while (std::getline(file, line)) {
+      o2::utils::Str::trim(line);
+      if (line.empty() || line.starts_with('\n')) {
+	continue;
+      }
+      fileNames.emplace_back(line);
+    }
+  }
+  LOGP(info, "Prepared {} files for parsing ({:.2f} s)", fileNames.size(), watch.RealTime());
+
+  watch.Start(true);
+  TableInfoMap info;
+  for (const auto& file : fileNames) {
+    testFile(file, info);
+  }
+  LOGP(info, "Testing all files finished ({:.2f} s)", watch.RealTime());
+  printTableInfoMap(info);
+
+  return ExitCodes::Success;
+}
+
+bool checkFileExists(const std::string& fileName)
+{
+  auto b = o2::utils::Str::pathExists(fileName);
+  if (!b) {
+    LOGP(error, "File '{}' does not exist!", fileName);
+  }
+  return b;
+}
+
+FilePtr openFile(const std::string& fileName)
+{
+  if (!checkFileExists(fileName)) {
+    return nullptr;
+  }
+
+  std::string realFile;
+  if (fileName.compare(0, 8, "alien://") == 0) {
+    if (!gGrid && !TGrid::Connect("alien://")) {
+      LOGP(error, "Failed to initialize alien for {}", fileName);
+      return nullptr;
+    }
+    realFile = fileName;
+  } else {
+    realFile = o2::utils::Str::getFullPath(fileName);
+  }
+
+  FilePtr file = std::unique_ptr<TFile>(TFile::Open(realFile.c_str(), "READ"));
+  if (file == nullptr || !file->IsOpen() || file->IsZombie()) {
+    LOGP(error, "Failed to open file '{}' (from '{}')", realFile, fileName);
+    return nullptr;
+  }
+  return file;
+}
+
+void testFile(const std::string& fileName, TableInfoMap& info)
+{
+  auto file = openFile(fileName);
+  if (!file) {
+    return;
+  }
+  LOGP(info, "Size of {} is {}", fileName, file->GetSize());
+  auto keys = file->GetListOfKeys();
+  keys->Sort();
+  for (auto okey : *keys) {
+    auto key = dynamic_cast<TKey*>(okey);
+    if (key == nullptr) {
+      continue;
+    }
+    TString name = key->GetName();
+    TString keyClassName = key->GetClassName();
+
+    if (keyClassName == "TMap" && name == "metaData") {
+      auto meta = file->Get<TMap>("metaData");
+      if (meta != nullptr) {
+	printMetaData(meta);
+      }
+    }
+
+    if (keyClassName != "TDirectoryFile") {
+      continue;
+    }
+
+    if (!name.BeginsWith("DF_")) {
+      continue;
+    }
+
+    auto df = file->Get<TDirectoryFile>(name);
+    if (df != nullptr) {
+      testDataFrame(df, info);
+    }
+  }
+}
+
+void testDataFrame(TDirectory* dir, TableInfoMap& info)
+{
+  auto keys = dir->GetListOfKeys();
+  keys->Sort();
+  for (auto okey : *keys) {
+    auto key = dynamic_cast<TKey*>(okey);
+    if (key == nullptr) {
+      continue;
+    }
+
+    if (std::string(key->GetClassName()) != "TTree") {
+      continue;
+    }
+
+    const auto name = key->GetName();
+    if (name[0] != 'O' || name[1] != '2') {
+      continue;
+    }
+    if (info.find(name) == info.end()) {
+      info.emplace(name, TableInfo());
+    }
+    TableInfo& table = info[name];
+    auto tree = dir->Get<TTree>(name);
+    if (tree != nullptr) {
+      testTree(tree, table);
+    }
+  }
+}
+
+void testTree(TTree* tree, TableInfo& table)
+{
+  table.seen++;
+  table.compBytes += tree->GetZipBytes();
+  table.uncBytes += tree->GetTotBytes();
+}
+
+void testBranch(TBranch* bra, TableInfoMap& info) {}
+
+void printTableInfoMap(const TableInfoMap& tableInfoMap)
+{
+  constexpr int width = 20;
+  constexpr double mb = 1. / (1024. * 1024.);
+
+  std::array<std::string, 4> header{"Name", "UncBytes", "CompBytes", "Seen"};
+  std::string sep;
+  for (int i{0}; i < header.size(); ++i) {
+    sep += "+" + std::string(width + 2, '-');
+  }
+  sep += "+";
+
+  LOGP(info, "{}", sep);
+  LOGP(info, "| {:<{}} | {:>{}} | {:>{}} | {:>{}} |", header[0], width, header[1], width, header[2], width, header[3], width);
+  LOGP(info, "{}", sep);
+
+  Long64_t totUncBytes{0}, totCompBytes{0}, totSeen{0};
+
+  for (const auto& [name, table] : tableInfoMap) {
+    LOGP(info, "| {:<{}} | {:>{}} | {:>{}} | {:>{}} |", name, width, table.uncBytes * mb, width, table.compBytes * mb, width, table.seen, width);
+
+    totUncBytes += table.uncBytes;
+    totCompBytes += table.compBytes;
+    totSeen += table.seen;
+  }
+  LOGP(info, "{}", sep);
+  LOGP(info, "| {:<{}} | {:>{}} | {:>{}} | {:>{}} |", "Total", width, totUncBytes * mb, width, totCompBytes * mb, width, totSeen, width);
+  LOGP(info, "{}", sep);
+}
+
+void printMetaData(const TMap* meta)
+{
+  LOGP(info, "Found MetaData with:");
+  auto iter = meta->MakeIterator();
+  TObject* key{nullptr};
+  while ((key = iter->Next())) {
+    auto value = meta->GetValue(key);
+    auto keyStr = dynamic_cast<TObjString*>(key);
+    auto valueStr = dynamic_cast<TObjString*>(value);
+    if (keyStr && valueStr) {
+      LOGP(info, " - {}: {}", keyStr->GetName(), valueStr->GetName());
+    } else {
+      LOGP(info, " - {}: {}", keyStr->GetName(), (value ? value->GetName() : "null"));
+    }
+  }
+
+  delete iter;
+}