[TableGen] Split *GenRegisterInfo.inc. (#167700)

Reduces memory usage compiling backend sources, most notably for
AMDGPU by ~98 MB per source on average.

AMDGPUGenRegisterInfo.inc is tens of megabytes in size now, and
is even larger downstream. At the same time, it is included in
nearly all backend sources, typically just for a small portion of
its content, resulting in compilation being unnecessarily
memory-hungry, which in turn stresses buildbots and wastes their
resources.

Splitting .inc files also helps avoiding extra ccache misses
where changes in .td files don't cause changes in all parts of
what previously was a single .inc file.

It is thought that rather than building on top of the current
single-output-file design of TableGen, e.g., using `split-file`,
it would be more preferable to recognise the need for multi-file
outputs and give it a proper first-class support directly in
TableGen.

Author: kosarev

llvm/docs/TableGen/index.rst

@@ -55,6 +55,11 @@ TableGen runs just like any other LLVM tool.  The first (optional) argument
 specifies the file to read.  If a filename is not specified, ``llvm-tblgen``
 reads from standard input.
 
+The ``-o`` option specifies the output file or ``-`` to output to
+stdout. Where TableGen produces multiple output files, the option
+specifies the name of the main output file, which also works as the
+name prefix for other output files.
+
 To be useful, one of the `backends`_ must be used.  These backends are
 selectable on the command line (type '``llvm-tblgen -help``' for a list).  For
 example, to get a list of all of the definitions that subclass a particular type

llvm/include/llvm/TableGen/Main.h

@@ -15,19 +15,34 @@
 
 #include "llvm/Support/CommandLine.h"
 #include <functional>
+#include <map>
 
 namespace llvm {
 
 class raw_ostream;
 class RecordKeeper;
 
-/// Perform the action using Records, and write output to OS.
+struct TableGenOutputFiles {
+  std::string MainFile;
+
+  // Translates additional output file names to their contents.
+  std::map<StringRef, std::string> AdditionalFiles;
+};
+
 /// Returns true on error, false otherwise.
 using TableGenMainFn = bool(raw_ostream &OS, const RecordKeeper &Records);
 
+/// Perform the action using Records, and store output in OutFiles.
+/// Returns true on error, false otherwise.
+using MultiFileTableGenMainFn = bool(TableGenOutputFiles &OutFiles,
+                                     const RecordKeeper &Records);
+
 int TableGenMain(const char *argv0,
                  std::function<TableGenMainFn> MainFn = nullptr);
 
+int TableGenMain(const char *argv0,
+                 std::function<MultiFileTableGenMainFn> MainFn = nullptr);
+
 /// Controls emitting large character arrays as strings or character arrays.
 /// Typically set to false when building with MSVC.
 extern cl::opt<bool> EmitLongStrLiterals;

llvm/include/llvm/TableGen/TableGenBackend.h

@@ -15,6 +15,7 @@
 
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/TableGen/Main.h"
 #include "llvm/TableGen/Record.h"
 
 namespace llvm {
@@ -23,7 +24,27 @@ class RecordKeeper;
 class raw_ostream;
 
 namespace TableGen::Emitter {
-using FnT = function_ref<void(const RecordKeeper &Records, raw_ostream &OS)>;
+
+/// Represents the emitting function. Can produce a single or multple output
+/// files.
+struct FnT {
+  using SingleFileGeneratorType = void(const RecordKeeper &Records,
+                                       raw_ostream &OS);
+  using MultiFileGeneratorType = TableGenOutputFiles(
+      StringRef FilenamePrefix, const RecordKeeper &Records);
+
+  SingleFileGeneratorType *SingleFileGenerator = nullptr;
+  MultiFileGeneratorType *MultiFileGenerator = nullptr;
+
+  FnT() = default;
+  FnT(SingleFileGeneratorType *Gen) : SingleFileGenerator(Gen) {}
+  FnT(MultiFileGeneratorType *Gen) : MultiFileGenerator(Gen) {}
+
+  bool operator==(const FnT &Other) const {
+    return SingleFileGenerator == Other.SingleFileGenerator &&
+           MultiFileGenerator == Other.MultiFileGenerator;
+  }
+};
 
 /// Creating an `Opt` object registers the command line option \p Name with
 /// TableGen backend and associates the callback \p CB with that option. If
@@ -36,17 +57,33 @@ struct Opt {
 /// Convienence wrapper around `Opt` that registers `EmitterClass::run` as the
 /// callback.
 template <class EmitterC> class OptClass : Opt {
-  static void run(const RecordKeeper &RK, raw_ostream &OS) {
+  static TableGenOutputFiles run(StringRef /*FilenamePrefix*/,
+                                 const RecordKeeper &RK) {
+    std::string S;
+    raw_string_ostream OS(S);
     EmitterC(RK).run(OS);
+    return {S, {}};
   }
 
 public:
   OptClass(StringRef Name, StringRef Desc) : Opt(Name, run, Desc) {}
 };
 
+/// A version of the wrapper for backends emitting multiple files.
+template <class EmitterC> class MultiFileOptClass : Opt {
+  static TableGenOutputFiles run(StringRef FilenamePrefix,
+                                 const RecordKeeper &RK) {
+    return EmitterC(RK).run(FilenamePrefix);
+  }
+
+public:
+  MultiFileOptClass(StringRef Name, StringRef Desc) : Opt(Name, run, Desc) {}
+};
+
 /// Apply callback for any command line option registered above. Returns false
 /// is no callback was applied.
-bool ApplyCallback(const RecordKeeper &Records, raw_ostream &OS);
+bool ApplyCallback(const RecordKeeper &Records, TableGenOutputFiles &OutFiles,
+                   StringRef FilenamePrefix);
 
 } // namespace TableGen::Emitter
 

llvm/lib/TableGen/Main.cpp

@@ -23,6 +23,7 @@
 #include "llvm/Support/ErrorOr.h"
 #include "llvm/Support/FileSystem.h"
 #include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/Path.h"
 #include "llvm/Support/SMLoc.h"
 #include "llvm/Support/SourceMgr.h"
 #include "llvm/Support/ToolOutputFile.h"
@@ -104,8 +105,30 @@ static int createDependencyFile(const TGParser &Parser, const char *argv0) {
   return 0;
 }
 
+static int WriteOutput(const TGParser &Parser, const char *argv0,
+                       StringRef Filename, StringRef Content) {
+  if (WriteIfChanged) {
+    // Only updates the real output file if there are any differences.
+    // This prevents recompilation of all the files depending on it if there
+    // aren't any.
+    if (auto ExistingOrErr = MemoryBuffer::getFile(Filename, /*IsText=*/true))
+      if (std::move(ExistingOrErr.get())->getBuffer() == Content)
+        return 0;
+  }
+  std::error_code EC;
+  ToolOutputFile OutFile(Filename, EC, sys::fs::OF_Text);
+  if (EC)
+    return reportError(argv0, "error opening " + Filename + ": " +
+                                  EC.message() + "\n");
+  OutFile.os() << Content;
+  if (ErrorsPrinted == 0)
+    OutFile.keep();
+
+  return 0;
+}
+
 int llvm::TableGenMain(const char *argv0,
-                       std::function<TableGenMainFn> MainFn) {
+                       std::function<MultiFileTableGenMainFn> MainFn) {
   RecordKeeper Records;
   TGTimer &Timer = Records.getTimer();
 
@@ -144,13 +167,14 @@ int llvm::TableGenMain(const char *argv0,
 
   // Write output to memory.
   Timer.startBackendTimer("Backend overall");
-  std::string OutString;
-  raw_string_ostream Out(OutString);
+  SmallString<128> FilenamePrefix(OutputFilename);
+  sys::path::replace_extension(FilenamePrefix, "");
+  TableGenOutputFiles OutFiles;
   unsigned status = 0;
   // ApplyCallback will return true if it did not apply any callback. In that
   // case, attempt to apply the MainFn.
-  if (TableGen::Emitter::ApplyCallback(Records, Out))
-    status = MainFn ? MainFn(Out, Records) : 1;
+  if (TableGen::Emitter::ApplyCallback(Records, OutFiles, FilenamePrefix))
+    status = MainFn ? MainFn(OutFiles, Records) : 1;
   Timer.stopBackendTimer();
   if (status)
     return 1;
@@ -165,25 +189,17 @@ int llvm::TableGenMain(const char *argv0,
   }
 
   Timer.startTimer("Write output");
-  bool WriteFile = true;
-  if (WriteIfChanged) {
-    // Only updates the real output file if there are any differences.
-    // This prevents recompilation of all the files depending on it if there
-    // aren't any.
-    if (auto ExistingOrErr =
-            MemoryBuffer::getFile(OutputFilename, /*IsText=*/true))
-      if (std::move(ExistingOrErr.get())->getBuffer() == OutString)
-        WriteFile = false;
-  }
-  if (WriteFile) {
-    std::error_code EC;
-    ToolOutputFile OutFile(OutputFilename, EC, sys::fs::OF_Text);
-    if (EC)
-      return reportError(argv0, "error opening " + OutputFilename + ": " +
-                                    EC.message() + "\n");
-    OutFile.os() << OutString;
-    if (ErrorsPrinted == 0)
-      OutFile.keep();
+  if (int Ret = WriteOutput(Parser, argv0, OutputFilename, OutFiles.MainFile))
+    return Ret;
+  for (auto [Suffix, Content] : OutFiles.AdditionalFiles) {
+    SmallString<128> Filename(OutputFilename);
+    // TODO: Format using the split-file convention when writing to stdout?
+    if (Filename != "-") {
+      Filename = FilenamePrefix;
+      Filename.append(Suffix);
+    }
+    if (int Ret = WriteOutput(Parser, argv0, Filename, Content))
+      return Ret;
   }
 
   Timer.stopTimer();
@@ -193,3 +209,15 @@ int llvm::TableGenMain(const char *argv0,
     return reportError(argv0, Twine(ErrorsPrinted) + " errors.\n");
   return 0;
 }
+
+int llvm::TableGenMain(const char *argv0,
+                       std::function<TableGenMainFn> MainFn) {
+  return TableGenMain(argv0, [&MainFn](TableGenOutputFiles &OutFiles,
+                                       const RecordKeeper &Records) {
+    std::string S;
+    raw_string_ostream OS(S);
+    int Res = MainFn(OS, Records);
+    OutFiles = {S, {}};
+    return Res;
+  });
+}

llvm/lib/TableGen/TableGenBackend.cpp

@@ -61,12 +61,21 @@ Opt::Opt(StringRef Name, FnT CB, StringRef Desc, bool ByDefault) {
 /// Apply callback specified on the command line. Returns true if no callback
 /// was applied.
 bool llvm::TableGen::Emitter::ApplyCallback(const RecordKeeper &Records,
-                                            raw_ostream &OS) {
+                                            TableGenOutputFiles &OutFiles,
+                                            StringRef FilenamePrefix) {
   FnT Fn = CallbackFunction->getValue();
-  if (!Fn)
-    return true;
-  Fn(Records, OS);
-  return false;
+  if (Fn.SingleFileGenerator) {
+    std::string S;
+    raw_string_ostream OS(S);
+    Fn.SingleFileGenerator(Records, OS);
+    OutFiles = {S, {}};
+    return false;
+  }
+  if (Fn.MultiFileGenerator) {
+    OutFiles = Fn.MultiFileGenerator(FilenamePrefix, Records);
+    return false;
+  }
+  return true;
 }
 
 static void printLine(raw_ostream &OS, const Twine &Prefix, char Fill,

llvm/lib/Target/AMDGPU/SIFoldOperands.cpp

@@ -1358,7 +1358,7 @@ void SIFoldOperandsImpl::foldOperand(
       // Remove this if 16-bit SGPRs (i.e. SGPR_LO16) are added to the
       // VS_16RegClass
       //
-      // Excerpt from AMDGPUGenRegisterInfo.inc
+      // Excerpt from AMDGPUGenRegisterInfoEnums.inc
       // NoSubRegister, //0
       // hi16, // 1
       // lo16, // 2

llvm/test/TableGen/ArtificialSubregs.td

@@ -1,4 +1,4 @@
-// RUN: llvm-tblgen -gen-register-info -register-info-debug -I %p/../../include %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK
+// RUN: llvm-tblgen -gen-register-info -register-info-debug -I %p/../../include %s -o - 2>&1 >/dev/null | FileCheck %s --check-prefix=CHECK
 include "llvm/Target/Target.td"
 
 // This file tests that when using `isArtificial` for subregisters in

llvm/test/TableGen/ConcatenatedSubregs.td

@@ -1,4 +1,4 @@
-// RUN: llvm-tblgen -gen-register-info -register-info-debug -I %p/../../include %s -o /dev/null 2>&1 | FileCheck %s
+// RUN: llvm-tblgen -gen-register-info -register-info-debug -I %p/../../include %s -o - 2>&1 >/dev/null | FileCheck %s
 // Checks that tablegen correctly and completely infers subregister relations.
 include "llvm/Target/Target.td"
 

llvm/test/TableGen/HwModeBitSet.td

@@ -1,5 +1,5 @@
 // This is to test the scenario where different HwMode attributes coexist.
-// RUN: llvm-tblgen -gen-register-info -register-info-debug -I %p/../../include %s -o /dev/null 2>&1 | FileCheck %s --check-prefix=CHECK-REG
+// RUN: llvm-tblgen -gen-register-info -register-info-debug -I %p/../../include %s -o - 2>&1 >/dev/null | FileCheck %s --check-prefix=CHECK-REG
 // RUN: llvm-tblgen -gen-subtarget -I %p/../../include %s -o - 2>&1 | FileCheck %s --check-prefix=CHECK-SUBTARGET
 
 

llvm/test/TableGen/HwModeSubRegs.td

@@ -1,4 +1,4 @@
-// RUN: llvm-tblgen -gen-register-info -register-info-debug -I %p/../../include %s -o /dev/null 2>&1 | FileCheck %s
+// RUN: llvm-tblgen -gen-register-info -register-info-debug -I %p/../../include %s -o - 2>&1 >/dev/null | FileCheck %s
 include "llvm/Target/Target.td"
 
 def HasFeat : Predicate<"Subtarget->hasFeat()">;