Shard output of cpp transpiler.

Large designs can lead to huge, slow-to-build c++ sources. This change produces one file per type.

PiperOrigin-RevId: 870023245

Author: grebe

docs_src/bazel_rules_macros.md

@@ -825,8 +825,8 @@ xls_dslx_cpp_type_library(
 
 ```
 
-will generate b_cpp_types_generate.cc and b_cpp_types_generate.h, and package them into a
-cc_library under the namespace "::xls::b".
+will generate (potentially many) c++ sources in _gen_b_cpp_types_generate_srcs/ as well as
+b_cpp_types_generate.h, and package them into a cc_library under the namespace "::xls::b".
 
 If another DSLX library `a.x` depends on types in `:b_dslx`, the `xls_dslx_cpp_type_library` for
  `a` should explicitly depend on the C++ library above and use the same parent namespace:

xls/build_rules/xls_dslx_rules.bzl

@@ -724,7 +724,7 @@ def _xls_dslx_generate_cpp_type_files_impl(ctx):
     dslx_path = ":${PWD}:" + ctx.genfiles_dir.path + ":" + ctx.bin_dir.path + dslx_srcs_wsroot_path + wsroot_dslx_path
 
     # Make arguments for the cpp_transpiler tool.
-    cc_file = ctx.outputs.source_file
+    cc_source_dir = ctx.actions.declare_directory(ctx.label.name + "_srcs")
     h_file = ctx.outputs.header_file
 
     include_header_files = []
@@ -740,7 +740,7 @@ def _xls_dslx_generate_cpp_type_files_impl(ctx):
 
     my_args.add(src.path)
 
-    my_args.add("--output_source_path={}".format(cc_file.path))
+    my_args.add("--output_source_path_prefix={}/{}".format(cc_source_dir.path, h_file.basename.removesuffix(_H_FILE_EXTENSION)))
     my_args.add("--output_header_path={}".format(h_file.path))
     my_args.add("--dslx_path=={}".format(dslx_path))
     my_args.add_joined("--include_headers", include_header_files, join_with = ",")
@@ -749,7 +749,7 @@ def _xls_dslx_generate_cpp_type_files_impl(ctx):
         my_args.add("--namespaces={}".format(ctx.attr.namespace))
 
     ctx.actions.run(
-        outputs = [cc_file, h_file],
+        outputs = [cc_source_dir, h_file],
         tools = [cpp_transpiler_tool],
         # The files required for converting the DSLX source file.
         inputs = runfiles.files,
@@ -763,7 +763,7 @@ def _xls_dslx_generate_cpp_type_files_impl(ctx):
     return [
         DefaultInfo(
             files = depset(
-                direct = [cc_file, h_file],
+                direct = [cc_source_dir, h_file],
                 transitive = get_transitive_built_files_for_xls(ctx),
             ),
             runfiles = runfiles,
@@ -775,11 +775,6 @@ xls_dslx_generate_cpp_type_files_attrs = {
         doc = "Direct cc_library dependencies that should be included in the generated C++ files.",
     ),
     "namespace": attr.string(doc = "The C++ namespace to generate the code in (e.g., `foo::bar`)."),
-    "source_file": attr.output(
-        doc = "The filename of the generated source file. The filename must " +
-              "have a '" + _CC_FILE_EXTENSION + "' extension.",
-        mandatory = True,
-    ),
     "header_file": attr.output(
         doc = "The filename of the generated header file. The filename must " +
               "have a '" + _H_FILE_EXTENSION + "' extension.",
@@ -798,7 +793,6 @@ Example:
         name = "b_cpp_types_generate",
         srcs = ["b.x"],
         deps = [":a_dslx"],
-        source_file = "b_cpp_types.cc",
         header_file = "b_cpp_types.h",
         namespace = "xls",
     )

xls/build_rules/xls_macros.bzl

@@ -401,8 +401,8 @@ def xls_dslx_cpp_type_library(
 
     ```
 
-    will generate b_cpp_types_generate.cc and b_cpp_types_generate.h, and package them into a
-    cc_library under the namespace "::xls::b".
+    will generate (potentially many) c++ sources in _gen_b_cpp_types_generate_srcs/ as well as
+    b_cpp_types_generate.h, and package them into a cc_library under the namespace "::xls::b".
 
     If another DSLX library `a.x` depends on types in `:b_dslx`, the `xls_dslx_cpp_type_library` for
      `a` should explicitly depend on the C++ library above and use the same parent namespace:
@@ -426,19 +426,21 @@ def xls_dslx_cpp_type_library(
         strongly encouraged to avoid polluting top-level namespaces per
         https://google.github.io/styleguide/cppguide.html#Namespace_Names.
     """
+    generated_sources_name = "_gen_" + name
     xls_dslx_generate_cpp_type_files(
-        name = name + "_generate_sources",
+        name = generated_sources_name,
         srcs = [src],
-        source_file = name + ".cc",
         header_file = name + ".h",
-        deps = deps,
         cpp_deps = cpp_deps,
+        deps = deps,
         namespace = namespace,
     )
     cc_library(
         name = name,
-        srcs = [":" + name + ".cc"],
+        srcs = [":" + generated_sources_name],
         hdrs = [":" + name + ".h"],
+        # TODO: bazelbuild/bazel#14843 - Remove this once no longer needed.
+        linkstatic = True,
         deps = [
             "@com_google_absl//absl/base:core_headers",
             "@com_google_absl//absl/status:status",

xls/dslx/cpp_transpiler/BUILD

@@ -113,7 +113,6 @@ cc_test(
     ]),
     deps = [
         ":cpp_transpiler",
-        ":cpp_type_generator",
         "//xls/common:golden_files",
         "//xls/common:source_location",
         "//xls/common:xls_gunit_main",
@@ -124,6 +123,7 @@ cc_test(
         "@com_google_absl//absl/status",
         "@com_google_absl//absl/status:status_matchers",
         "@com_google_absl//absl/strings:str_format",
+        "@com_google_absl//absl/types:span",
         "@googletest//:gtest",
     ],
 )
@@ -134,10 +134,10 @@ cc_binary(
     visibility = ["//xls:xls_public"],
     deps = [
         ":cpp_transpiler",
-        ":cpp_type_generator",
         "//xls/common:exit_status",
         "//xls/common:init_xls",
         "//xls/common/file:filesystem",
+        "//xls/common/status:ret_check",
         "//xls/common/status:status_macros",
         "//xls/dslx:create_import_data",
         "//xls/dslx:default_dslx_stdlib_path",

xls/dslx/cpp_transpiler/cpp_transpiler.cc

@@ -18,6 +18,7 @@
 #include <memory>
 #include <string>
 #include <string_view>
+#include <utility>
 
 #include "absl/status/statusor.h"
 #include "absl/strings/ascii.h"
@@ -35,7 +36,7 @@
 
 namespace xls::dslx {
 
-absl::StatusOr<CppSource> TranspileToCpp(
+absl::StatusOr<CppSourceGroup> TranspileToCpp(
     Module* module, ImportData* import_data,
     absl::Span<const std::string> additional_include_headers,
     std::string_view output_header_path, std::string_view namespaces) {
@@ -76,7 +77,9 @@ absl::StatusOr<CppSource> TranspileToCpp(
 #include "xls/public/status_macros.h"
 #include "xls/public/value.h"
 
-[[maybe_unused]] static bool FitsInNBitsSigned(int64_t value, int64_t n) {
+namespace {
+
+[[maybe_unused]] bool FitsInNBitsSigned(int64_t value, int64_t n) {
   // All bits from [n - 1, 64) must be all zero or all ones.
   if (n >= 64) {
     return true;
@@ -88,17 +91,19 @@ absl::StatusOr<CppSource> TranspileToCpp(
        (mask & value_as_unsigned) == mask;
 }
 
-[[maybe_unused]] static bool FitsInNBitsUnsigned(uint64_t value, int64_t n) {
+[[maybe_unused]] bool FitsInNBitsUnsigned(uint64_t value, int64_t n) {
   if (n >= 64) {
     return true;
   }
   return value < (uint64_t{1} << n);
 }
 
-[[maybe_unused]] static std::string __indent(int64_t amount) {
+[[maybe_unused]] std::string __indent(int64_t amount) {
   return std::string(amount * 2, ' ');
 }
 
+}  // namespace
+
 %s%s%s
 )";
   XLS_ASSIGN_OR_RETURN(TypeInfo * type_info,
@@ -139,13 +144,17 @@ absl::StatusOr<CppSource> TranspileToCpp(
     absl::StrAppend(&include_headers, "#include \"", header, "\"\n");
   }
 
-  return CppSource{
-      absl::Substitute(kHeaderTemplate, header_guard,
-                       absl::StrJoin(header, "\n\n"), namespace_begin,
-                       namespace_end, include_headers),
-      absl::StrFormat(kSourceTemplate, output_header_path, include_headers,
-                      namespace_begin, absl::StrJoin(source, "\n\n"),
-                      namespace_end)};
+  for (std::string& src : source) {
+    src = absl::StrFormat(kSourceTemplate, output_header_path, include_headers,
+                          namespace_begin, src, namespace_end);
+  }
+  std::string header_text = absl::Substitute(
+      kHeaderTemplate, header_guard, absl::StrJoin(header, "\n\n"),
+      namespace_begin, namespace_end, include_headers);
+  return CppSourceGroup{
+      .header = std::move(header_text),
+      .source = std::move(source),
+  };
 }
 
 }  // namespace xls::dslx

xls/dslx/cpp_transpiler/cpp_transpiler.h

@@ -16,15 +16,21 @@
 
 #include <string>
 #include <string_view>
+#include <vector>
 
 #include "absl/status/statusor.h"
 #include "absl/types/span.h"
-#include "xls/dslx/cpp_transpiler/cpp_type_generator.h"
 #include "xls/dslx/frontend/ast.h"
 #include "xls/dslx/import_data.h"
 
 namespace xls::dslx {
 
+// A group of C++ sources which all share the same header.
+struct CppSourceGroup {
+  std::string header;
+  std::vector<std::string> source;
+};
+
 // Converts DSLX types contained inside of "module" into their C++ equivalents.
 // For enums and constant declarations, this simply dumps the elements into the
 // returned text, using the smallest containing type for the latter, e.g., a
@@ -40,7 +46,7 @@ namespace xls::dslx {
 // should be infrequent, so users should feel comfortable using these
 // interfaces, but should also be aware of the potential for change in the
 // future.
-absl::StatusOr<CppSource> TranspileToCpp(
+absl::StatusOr<CppSourceGroup> TranspileToCpp(
     Module* module, ImportData* import_data,
     absl::Span<const std::string> additional_include_headers,
     std::string_view output_header_path, std::string_view namespaces = "");

xls/dslx/cpp_transpiler/cpp_transpiler_main.cc

@@ -22,14 +22,15 @@
 #include "absl/log/check.h"
 #include "absl/log/log.h"
 #include "absl/status/status.h"
+#include "absl/strings/str_cat.h"
 #include "absl/strings/str_split.h"
 #include "absl/types/span.h"
 #include "xls/common/exit_status.h"
 #include "xls/common/file/filesystem.h"
 #include "xls/common/init_xls.h"
+#include "xls/common/status/ret_check.h"
 #include "xls/common/status/status_macros.h"
 #include "xls/dslx/cpp_transpiler/cpp_transpiler.h"
-#include "xls/dslx/cpp_transpiler/cpp_type_generator.h"
 #include "xls/dslx/create_import_data.h"
 #include "xls/dslx/default_dslx_stdlib_path.h"
 #include "xls/dslx/import_data.h"
@@ -40,8 +41,9 @@
 ABSL_FLAG(std::vector<std::string>, include_headers, {}, "Include headers.");
 ABSL_FLAG(std::string, output_header_path, "",
           "Path at which to write the generated header.");
-ABSL_FLAG(std::string, output_source_path, "",
-          "Path at which to write the generated source.");
+ABSL_FLAG(std::string, output_source_path_prefix, "",
+          "Path at which to write the generated sources (suffix of .<index>.cc "
+          "will be added).");
 ABSL_FLAG(std::string, namespaces, "",
           "Double-colon-delimited namespaces with which to wrap the "
           "generated code, e.g., \"my::namespace\" (note: no leading `::`).");
@@ -69,7 +71,7 @@ absl::Status RealMain(const std::filesystem::path& module_path,
                       absl::Span<const std::filesystem::path> dslx_paths,
                       absl::Span<const std::string> include_headers,
                       std::string_view output_header_path,
-                      std::string_view output_source_path,
+                      std::string_view output_source_path_prefix,
                       std::string_view namespaces) {
   XLS_ASSIGN_OR_RETURN(std::string module_text, GetFileContents(module_path));
 
@@ -81,12 +83,16 @@ absl::Status RealMain(const std::filesystem::path& module_path,
       ParseAndTypecheck(module_text, std::string(module_path),
                         module_path.stem().string(), &import_data));
   XLS_ASSIGN_OR_RETURN(
-      CppSource sources,
+      CppSourceGroup sources,
       TranspileToCpp(module.module, &import_data, include_headers,
                      output_header_path, std::string(namespaces)));
+  XLS_RET_CHECK_GT(sources.source.size(), 0);
 
   XLS_RETURN_IF_ERROR(SetFileContents(output_header_path, sources.header));
-  XLS_RETURN_IF_ERROR(SetFileContents(output_source_path, sources.source));
+  for (int i = 0; i < sources.source.size(); ++i) {
+    XLS_RETURN_IF_ERROR(SetFileContents(
+        absl::StrCat(output_source_path_prefix, i, ".cc"), sources.source[i]));
+  }
   return absl::OkStatus();
 }
 
@@ -104,9 +110,10 @@ int main(int argc, char* argv[]) {
   std::string output_header_path = absl::GetFlag(FLAGS_output_header_path);
   QCHECK(!output_header_path.empty())
       << "--output_header_path must be specified.";
-  std::string output_source_path = absl::GetFlag(FLAGS_output_source_path);
-  QCHECK(!output_source_path.empty())
-      << "--output_source_path must be specified.";
+  std::string output_source_path_prefix =
+      absl::GetFlag(FLAGS_output_source_path_prefix);
+  QCHECK(!output_source_path_prefix.empty())
+      << "--output_source_path_prefix must be specified.";
 
   std::string dslx_path = absl::GetFlag(FLAGS_dslx_path);
   std::vector<std::string> dslx_path_strs = absl::StrSplit(dslx_path, ':');
@@ -120,7 +127,7 @@ int main(int argc, char* argv[]) {
   return xls::ExitStatus(xls::dslx::RealMain(
       args[0], absl::GetFlag(FLAGS_dslx_stdlib_path), dslx_paths,
       absl::GetFlag(FLAGS_include_headers), output_header_path,
-      output_source_path, absl::GetFlag(FLAGS_namespaces)));
+      output_source_path_prefix, absl::GetFlag(FLAGS_namespaces)));
 
   return 0;
 }