Add compile data capture to ats.

* Pull out common capture logic.
* Move "extra models" to generators

LiteRT-PiperOrigin-RevId: 819975154

Author: LukeBoyer

litert/ats/BUILD

@@ -44,10 +44,8 @@ cc_test(
         ":register",
         "//litert/c:litert_logging",
         "//litert/c:litert_op_code",
-        "//litert/cc:litert_buffer_ref",
         "//litert/cc:litert_c_types_printing",
         "//litert/cc:litert_detail",
-        "//litert/compiler/plugin:compiler_plugin",
         "//litert/test/generators",
         "//litert/test/generators:common",
         "//tflite/schema:schema_fbs",
@@ -106,18 +104,11 @@ cc_library(
     deps = [
         ":common",
         ":configure",
-        ":inference_capture",
-        "//litert/c:litert_common",
         "//litert/c:litert_logging",
         "//litert/cc:litert_detail",
         "//litert/cc:litert_expected",
-        "//litert/cc:litert_macros",
-        "//litert/cc:litert_model",
         "//litert/cc:litert_rng",
-        "//litert/core/model:model_load",
-        "//litert/test:simple_buffer",
         "//litert/test/generators",
-        "@com_google_absl//absl/strings:string_view",
     ],
 )
 
@@ -158,6 +149,7 @@ cc_library(
         "//litert/test:simple_buffer",
         "//litert/test/generators:common",
         "@com_google_absl//absl/strings:str_format",
+        "@com_google_absl//absl/strings:string_view",
         "@com_google_googletest//:gtest",
     ],
 )
@@ -167,6 +159,7 @@ cc_library(
     testonly = True,
     hdrs = ["compile_fixture.h"],
     deps = [
+        ":capture_common",
         ":common",
         ":compile_capture",
         ":configure",
@@ -177,6 +170,7 @@ cc_library(
         "//litert/core/model",
         "//litert/test:matchers",
         "//litert/test/generators:common",
+        "@com_google_absl//absl/strings:string_view",
         "@com_google_googletest//:gtest",
     ],
 )
@@ -217,6 +211,7 @@ cc_test(
     ],
     deps = [
         ":common",
+        ":compile_capture",
         ":compile_fixture",
         ":configure",
         ":executor",
@@ -248,6 +243,7 @@ cc_library(
     testonly = True,
     hdrs = ["inference_capture.h"],
     deps = [
+        ":capture_common",
         ":common",
         ":print",
         "//litert/cc:litert_detail",
@@ -301,6 +297,8 @@ cc_library(
     testonly = True,
     hdrs = ["compile_capture.h"],
     deps = [
+        ":capture_common",
+        ":common",
         ":print",
         "@com_google_absl//absl/strings:string_view",
     ],
@@ -310,12 +308,25 @@ cc_test(
     name = "compile_capture_test",
     srcs = ["compile_capture_test.cc"],
     deps = [
+        ":common",
         ":compile_capture",
-        "@com_google_absl//absl/strings",
         "@com_google_googletest//:gtest_main",
     ],
 )
 
+cc_library(
+    name = "capture_common",
+    testonly = True,
+    hdrs = ["capture_common.h"],
+    deps = [
+        ":common",
+        ":configure",
+        ":print",
+        "//litert/core/model",
+        "@com_google_absl//absl/strings:string_view",
+    ],
+)
+
 # PRE-CONFIGURED CTS SUITES ########################################################################
 
 litert_define_ats(

litert/ats/ats.cc

@@ -106,10 +106,14 @@ int Ats() {
 
   const auto res = RUN_ALL_TESTS();
 
-  options->Csv(i_cap);
-  options->Print(i_cap);
-  options->Csv(c_cap);
-  options->Print(c_cap);
+  // Final report.
+  if (options->CompileMode()) {
+    options->Csv(c_cap);
+    options->Print(c_cap);
+  } else {
+    options->Csv(i_cap);
+    options->Print(i_cap);
+  }
 
   return res;
 }

litert/ats/capture_common.h

@@ -0,0 +1,104 @@
+// Copyright 2025 Google LLC.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef THIRD_PARTY_ODML_LITERT_LITERT_ATS_CAPTURE_COMMON_H_
+#define THIRD_PARTY_ODML_LITERT_LITERT_ATS_CAPTURE_COMMON_H_
+
+#include <string>
+
+#include "absl/strings/string_view.h"  // from @com_google_absl
+#include "litert/ats/common.h"
+#include "litert/ats/configure.h"
+#include "litert/ats/print.h"
+#include "litert/core/model/model.h"
+
+namespace litert::testing {
+
+// Information about the input model.
+struct ModelDetail : public Printable<std::string, std::string, bool> {
+  // File name, if in memeory only graph, an identifier of the graph.
+  std::string name = "";
+  // Optional description or representation of the model.
+  std::string desc = "";
+  // Was the input model precompiled offline?
+  bool precompiled = false;
+
+  void SetFields(const TestNames& names, const LiteRtModelT& model) {
+    name = names.report_id;
+    desc = names.desc;
+    precompiled = GetBuildStamp(model).has_value();
+  }
+
+  ModelDetail() : Printable("ModelDetail", "name", "desc", "precompiled") {}
+
+ private:
+  Fields GetFields() const override { return Fields{name, desc, precompiled}; }
+};
+
+// Information about the accelerator used if any.
+struct AcceleratorDetail
+    : public Printable<ExecutionBackend, std::string, std::string> {
+  // The type of accelerator used.
+  ExecutionBackend a_type = ExecutionBackend::kCpu;
+
+  // Only applicable in the NPU case.
+  std::string soc_man = "n/a";
+  std::string soc_model = "n/a";
+
+  void SetFields(const AtsConf& conf) {
+    a_type = conf.Backend();
+    if (conf.IsNpu()) {
+      soc_man = conf.SocManufacturer();
+      soc_model = conf.SocModel();
+    }
+  }
+
+  AcceleratorDetail()
+      : Printable("AcceleratorDetail", "backend", "soc_man", "soc_model") {}
+
+ private:
+  Fields GetFields() const override {
+    return Fields{a_type, soc_man, soc_model};
+  }
+};
+
+// Information about any compilation that was done.
+struct CompilationDetail : public Printable<CompilationStatus> {
+  // The status of the compilation.
+  CompilationStatus status = CompilationStatus::kNotRequested;
+
+  CompilationDetail() : Printable("CompilationDetail", "status") {}
+
+  void SetFields(const AtsConf& conf, const LiteRtModelT& model, bool error) {
+    if (!conf.IsNpu()) {
+      return;
+    }
+    if (error) {
+      status = CompilationStatus::kError;
+    } else if (!internal::HasAnyCompiled(model)) {
+      status = CompilationStatus::kNoOpsCompiled;
+    } else if (!internal::IsFullyCompiled(model)) {
+      status = CompilationStatus::kPartiallyCompiled;
+    } else {
+      status = CompilationStatus::kFullyCompiled;
+    }
+  }
+
+ private:
+  Fields GetFields() const override { return Fields{status}; }
+};
+
+}  // namespace litert::testing
+
+#endif  // THIRD_PARTY_ODML_LITERT_LITERT_ATS_CAPTURE_COMMON_H_

litert/ats/check_ats.cc

@@ -24,6 +24,7 @@
 #include "absl/flags/reflection.h"  // from @com_google_absl
 #include "absl/strings/string_view.h"  // from @com_google_absl
 #include "litert/ats/common.h"
+#include "litert/ats/compile_capture.h"
 #include "litert/ats/compile_fixture.h"
 #include "litert/ats/configure.h"
 #include "litert/ats/executor.h"
@@ -114,15 +115,28 @@ Expected<void> CheckAts() {
   LITERT_ENSURE(!RUN_ALL_TESTS(), Error(kLiteRtStatusErrorRuntimeFailure),
                 "Failed to run all tests.");
 
-  const auto cap_ok = std::all_of(i_cap.Rows().begin(), i_cap.Rows().end(),
-                                  [](const InferenceCaptureEntry& row) {
-                                    return row.run.status != RunStatus::kError;
-                                  });
-  LITERT_ENSURE(cap_ok && i_cap.Rows().size() == test_id - 1,
+  const auto i_cap_ok =
+      std::all_of(i_cap.Rows().begin(), i_cap.Rows().end(),
+                  [](const InferenceCaptureEntry& row) {
+                    return row.run.status != RunStatus::kError;
+                  });
+  LITERT_ENSURE(i_cap_ok && i_cap.Rows().size() == test_id - 1,
                 Error(kLiteRtStatusErrorRuntimeFailure),
                 "Status capture contains errors.");
+
+  const auto c_cap_ok = std::all_of(c_cap.Rows().begin(), c_cap.Rows().end(),
+                                    [](const CompileCaptureEntry& row) {
+                                      return row.compilation_detail.status !=
+                                             CompilationStatus::kError;
+                                    });
+  LITERT_ENSURE(c_cap_ok && c_cap.Rows().size() == 1,
+                Error(kLiteRtStatusErrorRuntimeFailure),
+                "Status capture contains errors.");
+
   i_cap.Print(std::cerr);
   i_cap.Csv(std::cerr);
+  c_cap.Print(std::cerr);
+  c_cap.Csv(std::cerr);
 
   // Check side effects.
 

litert/ats/common.h

@@ -39,8 +39,8 @@ struct TestNames {
 
   // Create using repr of ops as desc. Only use if the model has 1-ish ops.
   static TestNames Create(size_t test_id, absl::string_view family,
-                          const LiteRtModelT& graph) {
-    auto suite = MakeSuite(test_id, family);
+                          absl::string_view logic, const LiteRtModelT& graph) {
+    auto suite = MakeSuite(test_id, family, logic);
     auto test = absl::StrFormat("%v", graph.Subgraph(0).Ops());
     auto desc = test;
     auto report_id = suite;
@@ -49,14 +49,16 @@ struct TestNames {
 
   // Create with an explicit desc.
   static TestNames Create(size_t test_id, absl::string_view family,
-                          absl::string_view test, absl::string_view desc = "") {
-    auto suite = MakeSuite(test_id, family);
-    return {suite, std::string(test), std::string(desc), std::string(test)};
+                          absl::string_view logic, absl::string_view test,
+                          absl::string_view desc = "") {
+    auto suite = MakeSuite(test_id, family, logic);
+    return {suite, std::string(logic), std::string(desc), std::string(test)};
   }
 
  private:
-  static std::string MakeSuite(size_t test_id, absl::string_view family) {
-    return absl::StrFormat("ats_%lu_%s", test_id, family);
+  static std::string MakeSuite(size_t test_id, absl::string_view family,
+                               absl::string_view logic) {
+    return absl::StrFormat("ats_%lu_%s_%s", test_id, family, logic);
   }
 };
 
@@ -80,6 +82,19 @@ enum class RunStatus {
   kTimeout,
 };
 
+enum class CompilationStatus {
+  // End never recorded.
+  kNotRequested,
+  // The compilation failed due to an error.
+  kError,
+  // Compilation succeeded, but no ops were compiled.
+  kNoOpsCompiled,
+  // Compilation succeeded, not all ops were compiled.
+  kPartiallyCompiled,
+  // Compilation succeeded, all ops were compiled.
+  kFullyCompiled,
+};
+
 // Timing related types.
 using Clock = std::chrono::steady_clock;
 using TimePoint = Clock::time_point;
@@ -138,6 +153,27 @@ void AbslStringify(Sink& sink, const RunStatus& status) {
   }
 }
 
+template <typename Sink>
+void AbslStringify(Sink& sink, const CompilationStatus& status) {
+  switch (status) {
+    case CompilationStatus::kNotRequested:
+      sink.Append("not_requested");
+      break;
+    case CompilationStatus::kError:
+      sink.Append("error");
+      break;
+    case CompilationStatus::kNoOpsCompiled:
+      sink.Append("no_ops_compiled");
+      break;
+    case CompilationStatus::kPartiallyCompiled:
+      sink.Append("partially_compiled");
+      break;
+    case CompilationStatus::kFullyCompiled:
+      sink.Append("fully_compiled");
+      break;
+  }
+}
+
 template <typename Sink>
 void AbslStringify(Sink& sink, const Nanoseconds& ns) {
   absl::Format(&sink, "%e", ns);