Support: Add vfs::OutputBackend and OutputFile to virtualize compiler outputs

llvm/include/llvm/Support/HashingOutputBackend.h

@@ -0,0 +1,125 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the declarations of the HashingOutputBackend class, which
+/// is the VirtualOutputBackend that is only producing the hash for the output
+/// files.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H
+#define LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H
+
+#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/Support/HashBuilder.h"
+#include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/Support/VirtualOutputConfig.h"
+#include "llvm/Support/raw_ostream.h"
+#include <mutex>
+
+namespace llvm::vfs {
+
+/// raw_pwrite_stream that writes to a hasher.
+template <typename HasherT>
+class HashingStream : public llvm::raw_pwrite_stream {
+private:
+  SmallVector<char> Buffer;
+  raw_svector_ostream OS;
+
+  using HashBuilderT = HashBuilder<HasherT, endianness::native>;
+  HashBuilderT Builder;
+
+  void write_impl(const char *Ptr, size_t Size) override {
+    OS.write(Ptr, Size);
+  }
+
+  void pwrite_impl(const char *Ptr, size_t Size, uint64_t Offset) override {
+    OS.pwrite(Ptr, Size, Offset);
+  }
+
+  uint64_t current_pos() const override { return OS.str().size(); }
+
+public:
+  HashingStream() : OS(Buffer) { SetUnbuffered(); }
+
+  auto final() {
+    Builder.update(OS.str());
+    return Builder.final();
+  }
+};
+
+template <typename HasherT> class HashingOutputFile;
+
+/// An output backend that only generates the hash for outputs.
+template <typename HasherT> class HashingOutputBackend : public OutputBackend {
+private:
+  friend class HashingOutputFile<HasherT>;
+  void addOutputFile(StringRef Path, StringRef Hash) {
+    std::lock_guard<std::mutex> Lock(OutputHashLock);
+    OutputHashes[Path] = std::string(Hash);
+  }
+
+protected:
+  IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
+    return const_cast<HashingOutputBackend<HasherT> *>(this);
+  }
+
+  Expected<std::unique_ptr<OutputFileImpl>>
+  createFileImpl(StringRef Path, std::optional<OutputConfig> Config) override {
+    return std::make_unique<HashingOutputFile<HasherT>>(Path, *this);
+  }
+
+public:
+  /// Iterator for all the output file names.
+  ///
+  /// Not thread safe. Should be queried after all outputs are written.
+  auto outputFiles() const { return OutputHashes.keys(); }
+
+  /// Get hash value for the output files in hex representation.
+  /// Return None if the requested path is not generated.
+  ///
+  /// Not thread safe. Should be queried after all outputs are written.
+  std::optional<std::string> getHashValueForFile(StringRef Path) {
+    auto F = OutputHashes.find(Path);
+    if (F == OutputHashes.end())
+      return std::nullopt;
+    return toHex(F->second);
+  }
+
+private:
+  std::mutex OutputHashLock;
+  StringMap<std::string> OutputHashes;
+};
+
+/// HashingOutputFile.
+template <typename HasherT>
+class HashingOutputFile final : public OutputFileImpl {
+public:
+  Error keep() override {
+    auto Result = OS.final();
+    Backend.addOutputFile(OutputPath, toStringRef(Result));
+    return Error::success();
+  }
+  Error discard() override { return Error::success(); }
+  raw_pwrite_stream &getOS() override { return OS; }
+
+  HashingOutputFile(StringRef OutputPath,
+                    HashingOutputBackend<HasherT> &Backend)
+      : OutputPath(OutputPath.str()), Backend(Backend) {}
+
+private:
+  const std::string OutputPath;
+  HashingStream<HasherT> OS;
+  HashingOutputBackend<HasherT> &Backend;
+};
+
+} // namespace llvm::vfs
+
+#endif // LLVM_SUPPORT_HASHINGOUTPUTBACKEND_H

llvm/include/llvm/Support/VirtualOutputBackend.h

@@ -0,0 +1,67 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the declarations of the VirtualOutputBackend class.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_VIRTUALOUTPUTBACKEND_H
+#define LLVM_SUPPORT_VIRTUALOUTPUTBACKEND_H
+
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/VirtualOutputConfig.h"
+#include "llvm/Support/VirtualOutputFile.h"
+
+namespace llvm::vfs {
+
+/// Interface for virtualized outputs.
+///
+/// If virtual functions are added here, also add them to \a
+/// ProxyOutputBackend.
+class OutputBackend : public RefCountedBase<OutputBackend> {
+  virtual void anchor();
+
+public:
+  /// Get a backend that points to the same destination as this one but that
+  /// has independent settings.
+  ///
+  /// Not thread-safe, but all operations are thread-safe when performed on
+  /// separate clones of the same backend.
+  IntrusiveRefCntPtr<OutputBackend> clone() const { return cloneImpl(); }
+
+  /// Create a file. If \p Config is \c std::nullopt, uses the backend's default
+  /// OutputConfig (may match \a OutputConfig::OutputConfig(), or may
+  /// have been customized).
+  ///
+  /// Thread-safe.
+  Expected<OutputFile>
+  createFile(const Twine &Path,
+             std::optional<OutputConfig> Config = std::nullopt);
+
+protected:
+  /// Must be thread-safe. Virtual function has a different name than \a
+  /// clone() so that implementations can override the return value.
+  virtual IntrusiveRefCntPtr<OutputBackend> cloneImpl() const = 0;
+
+  /// Create a file for \p Path. Must be thread-safe.
+  ///
+  /// \pre \p Config is valid or std::nullopt.
+  virtual Expected<std::unique_ptr<OutputFileImpl>>
+  createFileImpl(StringRef Path, std::optional<OutputConfig> Config) = 0;
+
+  OutputBackend() = default;
+
+public:
+  virtual ~OutputBackend() = default;
+};
+
+} // namespace llvm::vfs
+
+#endif // LLVM_SUPPORT_VIRTUALOUTPUTBACKEND_H

llvm/include/llvm/Support/VirtualOutputBackends.h

@@ -0,0 +1,116 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file contains the declarations of the VirtualOutputBackend classes,
+/// which can be used to virtual outputs from LLVM tools.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_VIRTUALOUTPUTBACKENDS_H
+#define LLVM_SUPPORT_VIRTUALOUTPUTBACKENDS_H
+
+#include "llvm/ADT/IntrusiveRefCntPtr.h"
+#include "llvm/Support/VirtualOutputBackend.h"
+#include "llvm/Support/VirtualOutputConfig.h"
+
+namespace llvm::vfs {
+
+/// Create a backend that ignores all output.
+IntrusiveRefCntPtr<OutputBackend> makeNullOutputBackend();
+
+/// Make a backend where \a OutputBackend::createFile() forwards to
+/// \p UnderlyingBackend when \p Filter is true, and otherwise returns a
+/// \a NullOutput.
+IntrusiveRefCntPtr<OutputBackend> makeFilteringOutputBackend(
+    IntrusiveRefCntPtr<OutputBackend> UnderlyingBackend,
+    std::function<bool(StringRef, std::optional<OutputConfig>)> Filter);
+
+/// Create a backend that forwards \a OutputBackend::createFile() to both \p
+/// Backend1 and \p Backend2 and sends content to both places.
+IntrusiveRefCntPtr<OutputBackend>
+makeMirroringOutputBackend(IntrusiveRefCntPtr<OutputBackend> Backend1,
+                           IntrusiveRefCntPtr<OutputBackend> Backend2);
+
+/// A helper class for proxying another backend, with the default
+/// implementation to forward to the underlying backend.
+class ProxyOutputBackend : public OutputBackend {
+  void anchor() override;
+
+protected:
+  // Require subclass to implement cloneImpl().
+  //
+  // IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override;
+
+  Expected<std::unique_ptr<OutputFileImpl>>
+  createFileImpl(StringRef Path, std::optional<OutputConfig> Config) override {
+    OutputFile File;
+    if (Error E = UnderlyingBackend->createFile(Path, Config).moveInto(File))
+      return std::move(E);
+    return File.takeImpl();
+  }
+
+  OutputBackend &getUnderlyingBackend() const { return *UnderlyingBackend; }
+
+public:
+  ProxyOutputBackend(IntrusiveRefCntPtr<OutputBackend> UnderlyingBackend)
+      : UnderlyingBackend(std::move(UnderlyingBackend)) {
+    assert(this->UnderlyingBackend && "Expected non-null backend");
+  }
+
+private:
+  IntrusiveRefCntPtr<OutputBackend> UnderlyingBackend;
+};
+
+/// An output backend that creates files on disk, wrapping APIs in sys::fs.
+class OnDiskOutputBackend : public OutputBackend {
+  void anchor() override;
+
+protected:
+  IntrusiveRefCntPtr<OutputBackend> cloneImpl() const override {
+    return clone();
+  }
+
+  Expected<std::unique_ptr<OutputFileImpl>>
+  createFileImpl(StringRef Path, std::optional<OutputConfig> Config) override;
+
+public:
+  /// Resolve an absolute path.
+  Error makeAbsolute(SmallVectorImpl<char> &Path) const;
+
+  /// On disk output settings.
+  struct OutputSettings {
+    /// Register output files to be deleted if a signal is received. Also
+    /// disabled for outputs with \a OutputConfig::getNoDiscardOnSignal().
+    bool DisableRemoveOnSignal = false;
+
+    /// Disable temporary files. Also disabled for outputs with \a
+    /// OutputConfig::getNoAtomicWrite().
+    bool DisableTemporaries = false;
+
+    // Default configuration for this backend.
+    OutputConfig DefaultConfig;
+  };
+
+  IntrusiveRefCntPtr<OnDiskOutputBackend> clone() const {
+    auto Clone = makeIntrusiveRefCnt<OnDiskOutputBackend>();
+    Clone->Settings = Settings;
+    return Clone;
+  }
+
+  OnDiskOutputBackend() = default;
+
+  /// Settings for this backend.
+  ///
+  /// Access is not thread-safe.
+  OutputSettings Settings;
+};
+
+} // namespace llvm::vfs
+
+#endif // LLVM_SUPPORT_VIRTUALOUTPUTBACKENDS_H

llvm/include/llvm/Support/VirtualOutputConfig.def

@@ -0,0 +1,30 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file This file defines the virtual output configurations.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef HANDLE_OUTPUT_CONFIG_FLAG
+#error "Missing macro definition of HANDLE_OUTPUT_CONFIG_FLAG"
+#endif
+
+// Define HANDLE_OUTPUT_CONFIG_FLAG before including.
+//
+// #define HANDLE_OUTPUT_CONFIG_FLAG(NAME, DEFAULT)
+
+HANDLE_OUTPUT_CONFIG_FLAG(Text, false) // OF_Text.
+HANDLE_OUTPUT_CONFIG_FLAG(CRLF, false) // OF_CRLF.
+HANDLE_OUTPUT_CONFIG_FLAG(Append, false) // OF_Append.
+HANDLE_OUTPUT_CONFIG_FLAG(DiscardOnSignal, true) // E.g., RemoveFileOnSignal.
+HANDLE_OUTPUT_CONFIG_FLAG(AtomicWrite, true) // E.g., use temporaries.
+HANDLE_OUTPUT_CONFIG_FLAG(ImplyCreateDirectories, true)
+// Skip atomic write if existing file content is the same
+HANDLE_OUTPUT_CONFIG_FLAG(OnlyIfDifferent, false)
+
+#undef HANDLE_OUTPUT_CONFIG_FLAG