tp: classic-to-subcommand translation layer

python/generators/diff_tests/test_executor.py

@@ -318,10 +318,11 @@ def run(self, test: TestCase, trace_path: str) -> TestResult:
       try:
         cmd = [
             self.trace_processor_path,
+            'query',
             '--analyze-trace-proto-content',
             '--crop-track-events',
             '--extra-checks',
-            '--structured-query-spec',
+            '--summary-spec',
             spec_file_path,
             '--structured-query-id',
             test.blueprint.query.query_id,

src/trace_processor/shell/BUILD.gn

@@ -51,10 +51,12 @@ source_set("shell") {
     "../../../protos/perfetto/trace_processor:zero",
     "../../base",
     "../../base:version",
+    "../../protozero/text_to_proto:text_to_proto",
     "../metrics",
     "../rpc",
     "../rpc:stdiod",
     "../trace_summary",
+    "../trace_summary:gen_cc_trace_summary_descriptor",
     "../util:stdlib",
     "../util/deobfuscation:deobfuscator",
     "../util/symbolizer",

src/trace_processor/shell/metrics_subcommand.cc

@@ -20,6 +20,7 @@
 #include <vector>
 
 #include "perfetto/base/status.h"
+#include "perfetto/base/time.h"
 #include "perfetto/ext/base/status_macros.h"
 #include "src/trace_processor/shell/common_flags.h"
 #include "src/trace_processor/shell/interactive.h"
@@ -65,6 +66,8 @@ std::vector<FlagSpec> MetricsSubcommand::GetFlags() {
        [this](const char* v) { raw_extensions_.emplace_back(v); }},
       StringFlag("post-query", '\0', "FILE", "SQL file after metrics.",
                  &post_query_path_),
+      StringFlag("perf-file", '\0', "FILE", "Write perf timing data to FILE.",
+                 &perf_file_),
       BoolFlag("interactive", 'i', "Start interactive shell after metrics.",
                &interactive_),
   };
@@ -97,7 +100,8 @@ base::Status MetricsSubcommand::Run(const SubcommandContext& ctx) {
     RETURN_IF_ERROR(LoadMetricExtension(tp.get(), extension, pool));
   }
 
-  RETURN_IF_ERROR(LoadTraceFile(tp.get(), ctx.platform, trace_file).status());
+  ASSIGN_OR_RETURN(auto t_load,
+                   LoadTraceFile(tp.get(), ctx.platform, trace_file));
 
   // Pre-metrics query.
   if (!pre_path_.empty()) {
@@ -115,12 +119,18 @@ base::Status MetricsSubcommand::Run(const SubcommandContext& ctx) {
     format = MetricV1OutputFormat::kJson;
   }
 
+  base::TimeNanos t_query_start = base::GetWallTimeNs();
   RETURN_IF_ERROR(RunMetrics(tp.get(), metrics, format));
 
   // Post-query.
   if (!post_query_path_.empty()) {
     RETURN_IF_ERROR(RunQueriesFromFile(tp.get(), post_query_path_, true));
   }
+  base::TimeNanos t_query = base::GetWallTimeNs() - t_query_start;
+
+  if (!perf_file_.empty()) {
+    RETURN_IF_ERROR(PrintPerfFile(perf_file_, t_load, t_query));
+  }
 
   if (interactive_) {
     RETURN_IF_ERROR(StartInteractiveShell(

src/trace_processor/shell/metrics_subcommand.h

@@ -40,6 +40,7 @@ class MetricsSubcommand : public Subcommand {
   std::string metric_output_;
   std::vector<std::string> raw_extensions_;
   std::string post_query_path_;
+  std::string perf_file_;
   bool interactive_ = false;
 };
 

src/trace_processor/shell/query_subcommand.cc

@@ -16,7 +16,11 @@
 
 #include "src/trace_processor/shell/query_subcommand.h"
 
+#include <algorithm>
+#include <cstdint>
+#include <memory>
 #include <string>
+#include <string_view>
 #include <vector>
 
 #include "perfetto/base/build_config.h"
@@ -25,10 +29,15 @@
 #include "perfetto/base/time.h"
 #include "perfetto/ext/base/file_utils.h"
 #include "perfetto/ext/base/status_macros.h"
+#include "perfetto/ext/base/string_utils.h"
+#include "perfetto/trace_processor/summarizer.h"
+#include "src/protozero/text_to_proto/text_to_proto.h"
 #include "src/trace_processor/shell/common_flags.h"
+#include "src/trace_processor/shell/interactive.h"
 #include "src/trace_processor/shell/metatrace.h"
 #include "src/trace_processor/shell/query.h"
 #include "src/trace_processor/shell/subcommand.h"
+#include "src/trace_processor/trace_summary/trace_summary.descriptor.h"
 
 #if PERFETTO_BUILDFLAG(PERFETTO_OS_WIN)
 #include <io.h>
@@ -41,6 +50,63 @@
 #endif
 
 namespace perfetto::trace_processor::shell {
+namespace {
+
+// Returns true if the file at |path| with |content| should be treated as
+// textproto rather than binary proto. Uses the same heuristic as the
+// classic codepath: .pb → binary, .textproto → text, otherwise
+// content-sniff the first 128 bytes (all printable/whitespace → text).
+bool IsTextproto(const std::string& path, const std::string& content) {
+  if (base::EndsWith(path, ".pb")) {
+    return false;
+  }
+  if (base::EndsWith(path, ".textproto")) {
+    return true;
+  }
+  std::string_view prefix(content.c_str(),
+                          std::min<size_t>(content.size(), 128));
+  return std::all_of(prefix.begin(), prefix.end(),
+                     [](char c) { return std::isspace(c) || std::isprint(c); });
+}
+
+// Reads a TraceSummarySpec file and passes it to |summarizer|, converting
+// textproto to binary proto when necessary.
+base::Status LoadSpecIntoSummarizer(Summarizer* summarizer,
+                                    const std::string& path) {
+  std::string content;
+  if (!base::ReadFile(path, &content)) {
+    return base::ErrStatus("Unable to read spec file %s", path.c_str());
+  }
+
+  const uint8_t* spec_data;
+  size_t spec_size;
+  std::vector<uint8_t> binary_proto;
+  if (IsTextproto(path, content)) {
+    ASSIGN_OR_RETURN(binary_proto,
+                     protozero::TextToProto(kTraceSummaryDescriptor.data(),
+                                            kTraceSummaryDescriptor.size(),
+                                            ".perfetto.protos.TraceSummarySpec",
+                                            "-", std::string_view(content)));
+    spec_data = binary_proto.data();
+    spec_size = binary_proto.size();
+  } else {
+    spec_data = reinterpret_cast<const uint8_t*>(content.data());
+    spec_size = content.size();
+  }
+
+  SummarizerUpdateSpecResult update_result;
+  RETURN_IF_ERROR(summarizer->UpdateSpec(spec_data, spec_size, &update_result));
+  for (const auto& q : update_result.queries) {
+    if (q.error.has_value()) {
+      return base::ErrStatus("Error in query '%s' from spec '%s': %s",
+                             q.query_id.c_str(), path.c_str(),
+                             q.error->c_str());
+    }
+  }
+  return base::OkStatus();
+}
+
+}  // namespace
 
 const char* QuerySubcommand::name() const {
   return "query";
@@ -63,17 +129,29 @@ SQL can be provided in three ways:
   3. From stdin:           cat q.sql | tp query trace.pb
 
 Multiple semicolon-separated statements are supported. Use -i to drop into
-an interactive shell after the queries complete.)";
+an interactive shell after the queries complete.
+
+Advanced (for debugging/testing structured queries):
+  --structured-query-id ID --summary-spec FILE [...]
+  Executes a single structured query by ID from the given summary spec
+  files. The spec files replace -f/stdin/positional SQL. Output is the
+  query result table.)";
 }
 
 std::vector<FlagSpec> QuerySubcommand::GetFlags() {
   return {
       StringFlag("query-file", 'f', "FILE",
                  "Read SQL from FILE (use '-' for stdin).", &query_file_),
+      StringFlag("structured-query-id", '\0', "ID",
+                 "[Advanced] Run a single structured query by ID.",
+                 &structured_query_id_),
+      {"summary-spec", '\0', true, "FILE",
+       "[Advanced] Summary spec file for structured queries (repeatable).",
+       [this](const char* v) { structured_query_specs_.emplace_back(v); }},
       BoolFlag("interactive", 'i', "Start interactive shell after query.",
                &interactive_),
       BoolFlag("wide", 'W', "Double column width for output.", &wide_),
-      StringFlag("perf-file", 'p', "FILE", "Write perf timing data to FILE.",
+      StringFlag("perf-file", '\0', "FILE", "Write perf timing data to FILE.",
                  &perf_file_),
   };
 }
@@ -84,6 +162,11 @@ base::Status QuerySubcommand::Run(const SubcommandContext& ctx) {
   }
   std::string trace_file = ctx.positional_args[0];
 
+  // Advanced: structured query mode.
+  if (!structured_query_id_.empty()) {
+    return RunStructuredQuery(ctx, trace_file);
+  }
+
   // Determine SQL source:
   //   1. Positional:  query trace.pb "SELECT ..."
   //   2. File:        query -f file.sql trace.pb
@@ -112,8 +195,6 @@ base::Status QuerySubcommand::Run(const SubcommandContext& ctx) {
   ASSIGN_OR_RETURN(auto t_load,
                    LoadTraceFile(tp.get(), ctx.platform, trace_file));
 
-  // If we have a file, read it into sql. After this point, sql always has
-  // the SQL to execute.
   if (!query_file_.empty()) {
     if (!base::ReadFile(query_file_, &sql)) {
       return base::ErrStatus("query: unable to read file '%s'",
@@ -128,12 +209,59 @@ base::Status QuerySubcommand::Run(const SubcommandContext& ctx) {
     MaybeWriteMetatrace(tp.get(), ctx.global->metatrace_path);
     return status;
   }
-
   base::TimeNanos t_query = base::GetWallTimeNs() - t_query_start;
 
-  if (!perf_file_.empty())
+  if (!perf_file_.empty()) {
     RETURN_IF_ERROR(PrintPerfFile(perf_file_, t_load, t_query));
+  }
+
+  if (interactive_) {
+    RETURN_IF_ERROR(StartInteractiveShell(
+        tp.get(),
+        InteractiveOptions{
+            wide_ ? 40u : 20u, MetricV1OutputFormat::kNone, {}, {}, nullptr}));
+  }
+
+  RETURN_IF_ERROR(MaybeWriteMetatrace(tp.get(), ctx.global->metatrace_path));
+  return base::OkStatus();
+}
+
+base::Status QuerySubcommand::RunStructuredQuery(
+    const SubcommandContext& ctx,
+    const std::string& trace_file) {
+  if (structured_query_specs_.empty()) {
+    return base::ErrStatus(
+        "query: --structured-query-id requires at least one --summary-spec");
+  }
+
+  auto config = BuildConfig(*ctx.global, ctx.platform);
+  ASSIGN_OR_RETURN(auto tp,
+                   SetupTraceProcessor(*ctx.global, config, ctx.platform));
+  ASSIGN_OR_RETURN(auto t_load,
+                   LoadTraceFile(tp.get(), ctx.platform, trace_file));
+
+  std::unique_ptr<Summarizer> summarizer;
+  RETURN_IF_ERROR(tp->CreateSummarizer(&summarizer));
+  for (const auto& path : structured_query_specs_) {
+    RETURN_IF_ERROR(LoadSpecIntoSummarizer(summarizer.get(), path));
+  }
 
+  base::TimeNanos t_query_start = base::GetWallTimeNs();
+  SummarizerQueryResult query_result;
+  RETURN_IF_ERROR(summarizer->Query(structured_query_id_, &query_result));
+  if (!query_result.exists) {
+    return base::ErrStatus(
+        "Structured query ID '%s' not found in the provided spec files",
+        structured_query_id_.c_str());
+  }
+
+  RETURN_IF_ERROR(
+      RunQueries(tp.get(), "SELECT * FROM " + query_result.table_name, true));
+  base::TimeNanos t_query = base::GetWallTimeNs() - t_query_start;
+
+  if (!perf_file_.empty()) {
+    RETURN_IF_ERROR(PrintPerfFile(perf_file_, t_load, t_query));
+  }
   RETURN_IF_ERROR(MaybeWriteMetatrace(tp.get(), ctx.global->metatrace_path));
   return base::OkStatus();
 }

src/trace_processor/shell/query_subcommand.h

@@ -35,7 +35,12 @@ class QuerySubcommand : public Subcommand {
   base::Status Run(const SubcommandContext& ctx) override;
 
  private:
+  base::Status RunStructuredQuery(const SubcommandContext& ctx,
+                                  const std::string& trace_file);
+
   std::string query_file_;
+  std::string structured_query_id_;
+  std::vector<std::string> structured_query_specs_;
   bool interactive_ = false;
   bool wide_ = false;
   std::string perf_file_;

src/trace_processor/shell/summarize_subcommand.cc

@@ -16,13 +16,15 @@
 
 #include "src/trace_processor/shell/summarize_subcommand.h"
 
+#include <algorithm>
 #include <cstdint>
 #include <cstdio>
 #include <optional>
 #include <string>
 #include <vector>
 
 #include "perfetto/base/status.h"
+#include "perfetto/base/time.h"
 #include "perfetto/ext/base/file_utils.h"
 #include "perfetto/ext/base/status_macros.h"
 #include "perfetto/ext/base/string_utils.h"
@@ -34,6 +36,29 @@
 #include "src/trace_processor/trace_summary/summary.h"
 
 namespace perfetto::trace_processor::shell {
+namespace {
+
+TraceSummarySpecBytes::Format GuessSummarySpecFormat(
+    const std::string& path,
+    const std::string& content) {
+  if (base::EndsWith(path, ".pb")) {
+    return TraceSummarySpecBytes::Format::kBinaryProto;
+  }
+  if (base::EndsWith(path, ".textproto")) {
+    return TraceSummarySpecBytes::Format::kTextProto;
+  }
+  // Content-sniffing fallback: if the first 128 bytes are all printable or
+  // whitespace, assume text proto; otherwise binary.
+  std::string_view prefix(content.c_str(),
+                          std::min<size_t>(content.size(), 128));
+  if (std::all_of(prefix.begin(), prefix.end(),
+                  [](char c) { return std::isspace(c) || std::isprint(c); })) {
+    return TraceSummarySpecBytes::Format::kTextProto;
+  }
+  return TraceSummarySpecBytes::Format::kBinaryProto;
+}
+
+}  // namespace
 
 const char* SummarizeSubcommand::name() const {
   return "summarize";
@@ -70,6 +95,8 @@ std::vector<FlagSpec> SummarizeSubcommand::GetFlags() {
                  &output_format_),
       StringFlag("post-query", '\0', "FILE",
                  "SQL file to run after summarization.", &post_query_path_),
+      StringFlag("perf-file", '\0', "FILE", "Write perf timing data to FILE.",
+                 &perf_file_),
       BoolFlag("interactive", 'i',
                "Start interactive shell after summarization.", &interactive_),
   };
@@ -90,7 +117,8 @@ base::Status SummarizeSubcommand::Run(const SubcommandContext& ctx) {
   auto config = BuildConfig(*ctx.global, ctx.platform);
   ASSIGN_OR_RETURN(auto tp,
                    SetupTraceProcessor(*ctx.global, config, ctx.platform));
-  RETURN_IF_ERROR(LoadTraceFile(tp.get(), ctx.platform, trace_file).status());
+  ASSIGN_OR_RETURN(auto t_load,
+                   LoadTraceFile(tp.get(), ctx.platform, trace_file));
 
   // Load spec files.
   std::vector<std::string> spec_content;
@@ -105,10 +133,7 @@ base::Status SummarizeSubcommand::Run(const SubcommandContext& ctx) {
   std::vector<TraceSummarySpecBytes> specs;
   specs.reserve(spec_paths.size());
   for (uint32_t i = 0; i < spec_paths.size(); ++i) {
-    auto format = TraceSummarySpecBytes::Format::kTextProto;
-    if (base::EndsWith(spec_paths[i], ".pb")) {
-      format = TraceSummarySpecBytes::Format::kBinaryProto;
-    }
+    auto format = GuessSummarySpecFormat(spec_paths[i], spec_content[i]);
     specs.emplace_back(TraceSummarySpecBytes{
         reinterpret_cast<const uint8_t*>(spec_content[i].data()),
         spec_content[i].size(),
@@ -135,6 +160,7 @@ base::Status SummarizeSubcommand::Run(const SubcommandContext& ctx) {
     output_spec.format = TraceSummaryOutputSpec::Format::kTextProto;
   }
 
+  base::TimeNanos t_query_start = base::GetWallTimeNs();
   std::vector<uint8_t> output;
   RETURN_IF_ERROR(
       tp->Summarize(computation_config, specs, &output, output_spec));
@@ -146,6 +172,11 @@ base::Status SummarizeSubcommand::Run(const SubcommandContext& ctx) {
   if (!post_query_path_.empty()) {
     RETURN_IF_ERROR(RunQueriesFromFile(tp.get(), post_query_path_, true));
   }
+  base::TimeNanos t_query = base::GetWallTimeNs() - t_query_start;
+
+  if (!perf_file_.empty()) {
+    RETURN_IF_ERROR(PrintPerfFile(perf_file_, t_load, t_query));
+  }
 
   if (interactive_) {
     RETURN_IF_ERROR(StartInteractiveShell(