Support: Add vfs::OutputBackend and OutputFile to virtualize compiler outputs (#113363)

Add OutputBackend and OutputFile to the `llvm::vfs` namespace for
virtualizing compiler outputs. This is intended for use in Clang,

The headers are:

- llvm/Support/VirtualOutputConfig.h
- llvm/Support/VirtualOutputError.h
- llvm/Support/VirtualOutputFile.h
- llvm/Support/VirtualOutputBackend.h

OutputFile is moveable and owns an OutputFileImpl, which is provided by
the derived OutputBackend.

- OutputFileImpl::keep() and OutputFileImpl::discard() should keep or
  discard the output.  OutputFile guarantees that exactly one of these
  will be called before destruction.
- OutputFile::keep() and OutputFile::discard() wrap OutputFileImpl
  and catch usage errors such as double-close.
- OutputFile::discardOnDestroy() installs an error handler for the
  destructor to use if the file is still open. The handler will be
  called if discard() fails.
- OutputFile::~OutputFile() calls report_fatal_error() if none of
  keep(), discard(), or discardOnDestroy() has been called. It still
  calls OutputFileImpl::discard().
- getOS() returns the wrapped raw_pwrite_stream. For convenience,
  OutputFile has an implicit conversion to `raw_ostream` and
  `raw_ostream &operator<<(OutputFile&, T&&)`.

OutputBackend can be stored in IntrusiveRefCntPtr.

- Most operations are thread-safe.
- clone() returns a backend that targets the same destination.
  All operations are thread-safe when done on different clones.
- createFile() takes a path and an OutputConfig (see below) and returns
  an OutputFile. Backends implement createFileImpl().

OutputConfig has flags to configure the output. Backends may ignore or
override flags that aren't relevant or implementable.

- The initial flags are:
    - AtomicWrite: whether the output should appear atomically (e.g., by
      using a temporary file and renaming it).
    - CrashCleanup: whether the output should be cleaned up if there's a
      crash (e.g., with RemoveFileOnSignal).
    - ImplyCreateDirectories: whether to implicitly create missing
      directories in the path to the file.
    - Text: matches sys::fs::OF_Text.
    - CRLF: matches sys::fs::OF_CRLF.
    - Append: matches sys::fs::OF_Append and can use with AtomicWrite
      for atomic append.
- OnlyIfDifferent: skip writting the output file if the existing file
      at the output path is identical to the content to be written.
- Each "Flag" has `setFlag(bool)` and `bool getFlag()` and shortcuts
  `setFlag()` and `setNoFlag()`. The setters are `constexpr` and return
  `OutputConfig&` to make it easy to declare a default value for a filed
  in a class or struct.
- Setters and getters for Binary and TextWithCRLF are derived from Text
  and CRLF. For convenience, sys::fs::OpenFlags can be passed
  directly to setOpenFlags().

This patch intentionally lacks a number of important features that have
been left for follow-ups:

- Set a (virtual) current working directory.
- Create a directory.
- Create a file or directory with a unique name (avoiding collisions
  with existing filenames).

Patch originally by dexonsmith

Author: cachemeifyoucan

llvm/include/llvm/Support/HashingOutputBackend.h

@@ -0,0 +1,126 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the declarations of the HashingOutputBackend class, which
+/// is the VirtualOutputBackend that only produces the hashes for the output
+/// files. This is useful for checking if the outputs are deterministic without
+/// storing output files in memory or on disk.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H
+#define LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H
+
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Support/HashBuilder.h"
+#include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/Support/VirtualOutputConfig.h"
+#include "llvm/Support/raw_ostream.h"
+#include <mutex>
+
+namespace llvm::vfs {
+
+/// raw_pwrite_stream that writes to a hasher.
+template <typename HasherT>
+class HashingStream : public llvm::raw_pwrite_stream {
+private:
+  SmallVector<char> Buffer;
+  raw_svector_ostream OS;
+
+  using HashBuilderT = HashBuilder<HasherT, endianness::native>;
+  HashBuilderT Builder;
+
+  void write_impl(const char *Ptr, size_t Size) override {
+    OS.write(Ptr, Size);
+  }
+
+  void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override {
+    OS.pwrite(Ptr, Size, Offset);
+  }
+
+  uint64_t current_pos() const override { return OS.str().size(); }
+
+public:
+  HashingStream() : OS(Buffer) { SetUnbuffered(); }
+
+  auto final() {
+    Builder.update(OS.str());
+    return Builder.final();
+  }
+};
+
+template <typename HasherT> class HashingOutputFile;
+
+/// An output backend that only generates the hash for outputs.
+template <typename HasherT> class HashingOutputBackend : public OutputBackend {
+private:
+  friend class HashingOutputFile<HasherT>;
+  void addOutputFile(StringRef Path, StringRef Hash) {
+    std::lock_guard<std::mutex> Lock(OutputHashLock);
+    OutputHashes[Path] = Hash.str();
+  }
+
+protected:
+  IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
+    return const_cast<HashingOutputBackend<HasherT> *>(this);
+  }
+
+  Expected<std::unique_ptr<OutputFileImpl>>
+  createFileImpl(StringRef Path, std::optional<OutputConfig> Config) override {
+    return std::make_unique<HashingOutputFile<HasherT>>(Path, *this);
+  }
+
+public:
+  /// Iterator for all the output file names.
+  ///
+  /// Not thread safe. Should be queried after all outputs are written.
+  auto outputFiles() const { return OutputHashes.keys(); }
+
+  /// Get hash value for the output files in hex representation.
+  /// Return None if the requested path is not generated.
+  ///
+  /// Not thread safe. Should be queried after all outputs are written.
+  std::optional<std::string> getHashValueForFile(StringRef Path) {
+    auto F = OutputHashes.find(Path);
+    if (F == OutputHashes.end())
+      return std::nullopt;
+    return toHex(F->second);
+  }
+
+private:
+  std::mutex OutputHashLock;
+  StringMap<std::string> OutputHashes;
+};
+
+/// HashingOutputFile.
+template <typename HasherT>
+class HashingOutputFile final : public OutputFileImpl {
+public:
+  Error keep() override {
+    auto Result = OS.final();
+    Backend.addOutputFile(OutputPath, toStringRef(Result));
+    return Error::success();
+  }
+  Error discard() override { return Error::success(); }
+  raw_pwrite_stream &getOS() override { return OS; }
+
+  HashingOutputFile(StringRef OutputPath,
+                    HashingOutputBackend<HasherT> &Backend)
+      : OutputPath(OutputPath.str()), Backend(Backend) {}
+
+private:
+  const std::string OutputPath;
+  HashingStream<HasherT> OS;
+  HashingOutputBackend<HasherT> &Backend;
+};
+
+} // namespace llvm::vfs
+
+#endif // LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H

llvm/include/llvm/Support/VirtualOutputBackend.h

@@ -0,0 +1,73 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the declarations of the VirtualOutputBackend class, which
+/// can be used to virtualized output files from LLVM tools.
+/// VirtualOutputBackend provides an unified interface to write outputs and a
+/// configurable interface for tools to operate on those outputs.
+/// VirtualOutputBackend contains basic implementations like writing to disk
+/// with different configurations, or advanced logics like output filtering
+/// and duplicating.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_VIRTUALOUTPUTBACKEND_H
+#define LLVM_SUPPORT_VIRTUALOUTPUTBACKEND_H
+
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/VirtualOutputConfig.h"
+#include "llvm/Support/VirtualOutputFile.h"
+
+namespace llvm::vfs {
+
+/// Interface for virtualized outputs.
+///
+/// If virtual functions are added here, also add them to \a
+/// ProxyOutputBackend.
+class OutputBackend : public RefCountedBase<OutputBackend> {
+  virtual void anchor();
+
+public:
+  /// Get a backend that points to the same destination as this one but that
+  /// has independent settings.
+  ///
+  /// Not thread-safe, but all operations are thread-safe when performed on
+  /// separate clones of the same backend.
+  IntrusiveRefCntPtr<OutputBackend> clone() const { return cloneImpl(); }
+
+  /// Create a file. If \p Config is \c std::nullopt, uses the backend's default
+  /// OutputConfig (may match \a OutputConfig::OutputConfig(), or may
+  /// have been customized).
+  ///
+  /// Thread-safe.
+  Expected<OutputFile>
+  createFile(const Twine &Path,
+             std::optional<OutputConfig> Config = std::nullopt);
+
+protected:
+  /// Must be thread-safe. Virtual function has a different name than \a
+  /// clone() so that implementations can override the return value.
+  virtual IntrusiveRefCntPtr<OutputBackend> cloneImpl() const = 0;
+
+  /// Create a file for \p Path. Must be thread-safe.
+  ///
+  /// \pre \p Config is valid or std::nullopt.
+  virtual Expected<std::unique_ptr<OutputFileImpl>>
+  createFileImpl(StringRef Path, std::optional<OutputConfig> Config) = 0;
+
+  OutputBackend() = default;
+
+public:
+  virtual ~OutputBackend() = default;
+};
+
+} // namespace llvm::vfs
+
+#endif // LLVM_SUPPORT_VIRTUALOUTPUTBACKEND_H

llvm/include/llvm/Support/VirtualOutputBackends.h

@@ -0,0 +1,124 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the declarations of the concrete VirtualOutputBackend
+/// classes, which are the implementation for different output style and
+/// functions. This file contains:
+/// * NullOutputBackend: discard all outputs written.
+/// * OnDiskOutputBackend: write output to disk, with support for common output
+/// types, like append, or atomic update.
+/// * FilteringOutputBackend: filer some output paths to underlying output
+/// backend.
+/// * MirrorOutputBackend: mirror output to two output backends.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_VIRTUALOUTPUTBACKENDS_H
+#define LLVM_SUPPORT_VIRTUALOUTPUTBACKENDS_H
+
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/Support/VirtualOutputConfig.h"
+
+namespace llvm::vfs {
+
+/// Create a backend that ignores all output.
+IntrusiveRefCntPtr<OutputBackend> makeNullOutputBackend();
+
+/// Make a backend where \a OutputBackend::createFile() forwards to
+/// \p UnderlyingBackend when \p Filter is true, and otherwise returns a
+/// \a NullOutput.
+IntrusiveRefCntPtr<OutputBackend> makeFilteringOutputBackend(
+    IntrusiveRefCntPtr<OutputBackend> UnderlyingBackend,
+    std::function<bool(StringRef, std::optional<OutputConfig>)> Filter);
+
+/// Create a backend that forwards \a OutputBackend::createFile() to both \p
+/// Backend1 and \p Backend2. Writing to such backend will create identical
+/// outputs using two different backends.
+IntrusiveRefCntPtr<OutputBackend>
+makeMirroringOutputBackend(IntrusiveRefCntPtr<OutputBackend> Backend1,
+                           IntrusiveRefCntPtr<OutputBackend> Backend2);
+
+/// A helper class for proxying another backend, with the default
+/// implementation to forward to the underlying backend.
+class ProxyOutputBackend : public OutputBackend {
+  void anchor() override;
+
+protected:
+  // Require subclass to implement cloneImpl().
+  //
+  // IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override;
+
+  Expected<std::unique_ptr<OutputFileImpl>>
+  createFileImpl(StringRef Path, std::optional<OutputConfig> Config) override {
+    OutputFile File;
+    if (Error E = UnderlyingBackend->createFile(Path, Config).moveInto(File))
+      return std::move(E);
+    return File.takeImpl();
+  }
+
+  OutputBackend &getUnderlyingBackend() const { return *UnderlyingBackend; }
+
+public:
+  ProxyOutputBackend(IntrusiveRefCntPtr<OutputBackend> UnderlyingBackend)
+      : UnderlyingBackend(std::move(UnderlyingBackend)) {
+    assert(this->UnderlyingBackend && "Expected non-null backend");
+  }
+
+private:
+  IntrusiveRefCntPtr<OutputBackend> UnderlyingBackend;
+};
+
+/// An output backend that creates files on disk, wrapping APIs in sys::fs.
+class OnDiskOutputBackend : public OutputBackend {
+  void anchor() override;
+
+protected:
+  IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
+    return clone();
+  }
+
+  Expected<std::unique_ptr<OutputFileImpl>>
+  createFileImpl(StringRef Path, std::optional<OutputConfig> Config) override;
+
+public:
+  /// Resolve an absolute path.
+  Error makeAbsolute(SmallVectorImpl<char> &Path) const;
+
+  /// On disk output settings.
+  struct OutputSettings {
+    /// Register output files to be deleted if a signal is received. Also
+    /// enabled for outputs with \a OutputConfig::getDiscardOnSignal().
+    bool RemoveOnSignal = true;
+
+    /// Use temporary files. Also enabled for outputs with \a
+    /// OutputConfig::getAtomicWrite().
+    bool UseTemporaries = true;
+
+    // Default configuration for this backend.
+    OutputConfig DefaultConfig;
+  };
+
+  IntrusiveRefCntPtr<OnDiskOutputBackend> clone() const {
+    auto Clone = makeIntrusiveRefCnt<OnDiskOutputBackend>();
+    Clone->Settings = Settings;
+    return Clone;
+  }
+
+  OnDiskOutputBackend() = default;
+
+  /// Settings for this backend.
+  ///
+  /// Access is not thread-safe.
+  OutputSettings Settings;
+};
+
+} // namespace llvm::vfs
+
+#endif // LLVM_SUPPORT_VIRTUALOUTPUTBACKENDS_H

llvm/include/llvm/Support/VirtualOutputConfig.def

@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file This file defines the virtual output configurations.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef HANDLE_OUTPUT_CONFIG_FLAG
+#error "Missing macro definition of HANDLE_OUTPUT_CONFIG_FLAG"
+#endif
+
+// Define HANDLE_OUTPUT_CONFIG_FLAG before including.
+//
+// #define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT)
+
+HANDLE_OUTPUT_CONFIG_FLAG(Text, false) // OF_Text.
+HANDLE_OUTPUT_CONFIG_FLAG(CRLF, false) // OF_CRLF.
+HANDLE_OUTPUT_CONFIG_FLAG(Append, false) // OF_Append.
+HANDLE_OUTPUT_CONFIG_FLAG(DiscardOnSignal, true) // E.g., RemoveFileOnSignal.
+HANDLE_OUTPUT_CONFIG_FLAG(AtomicWrite, true) // E.g., use temporaries.
+HANDLE_OUTPUT_CONFIG_FLAG(ImplyCreateDirectories, true)
+// Skip atomic write if existing file content is the same
+HANDLE_OUTPUT_CONFIG_FLAG(OnlyIfDifferent, false)
+
+#undef HANDLE_OUTPUT_CONFIG_FLAG