[CAS] Add OnDiskTrieRawHashMap

llvm/include/llvm/CAS/FileOffset.h

@@ -0,0 +1,39 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+/// \file
+/// This file declares interface for FileOffset that represent stored data at an
+/// offset from the beginning of a file.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CAS_FILEOFFSET_H
+#define LLVM_CAS_FILEOFFSET_H
+
+#include <cstdlib>
+
+namespace llvm::cas {
+
+/// FileOffset is a wrapper around `uint64_t` to represent the offset of data
+/// from the beginning of the file.
+class FileOffset {
+public:
+  uint64_t get() const { return Offset; }
+
+  explicit operator bool() const { return Offset; }
+
+  FileOffset() = default;
+  explicit FileOffset(uint64_t Offset) : Offset(Offset) {}
+
+private:
+  uint64_t Offset = 0;
+};
+
+} // namespace llvm::cas
+
+#endif // LLVM_CAS_FILEOFFSET_H

llvm/include/llvm/CAS/OnDiskTrieRawHashMap.h

@@ -13,13 +13,14 @@
 ///
 //===----------------------------------------------------------------------===//
 
-#ifndef LLVM_CAS_ONDISKHASHMAPPEDTRIE_H
-#define LLVM_CAS_ONDISKHASHMAPPEDTRIE_H
+#ifndef LLVM_CAS_ONDISKTRIERAWHASHMAP_H
+#define LLVM_CAS_ONDISKTRIERAWHASHMAP_H
 
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/ADT/STLFunctionalExtras.h"
 #include "llvm/ADT/StringRef.h"
+#include "llvm/CAS/FileOffset.h"
 #include "llvm/Support/Error.h"
 #include <optional>
 
@@ -29,21 +30,6 @@ class raw_ostream;
 
 namespace cas {
 
-/// FileOffset is a wrapper around `uint64_t` to represent the offset of data
-/// from the beginning of the file.
-class FileOffset {
-public:
-  uint64_t get() const { return Offset; }
-
-  explicit operator bool() const { return Offset; }
-
-  FileOffset() = default;
-  explicit FileOffset(uint64_t Offset) : Offset(Offset) {}
-
-private:
-  uint64_t Offset = 0;
-};
-
 /// OnDiskTrieRawHashMap is a persistent trie data structure used as hash maps.
 /// The keys are fixed length, and are expected to be binary hashes with a
 /// normal distribution.
@@ -244,73 +230,7 @@ class OnDiskTrieRawHashMap {
   std::unique_ptr<ImplType> Impl;
 };
 
-/// Sink for data. Stores variable length data with 8-byte alignment. Does not
-/// track size of data, which is assumed to known from context, or embedded.
-/// Uses 0-padding but does not guarantee 0-termination.
-class OnDiskDataAllocator {
-public:
-  using ValueProxy = MutableArrayRef<char>;
-
-  /// An iterator-like return value for data insertion. Maybe it should be
-  /// called \c iterator, but it has no increment.
-  class pointer {
-  public:
-    FileOffset getOffset() const { return Offset; }
-    explicit operator bool() const { return bool(getOffset()); }
-    const ValueProxy &operator*() const {
-      assert(Offset && "Null dereference");
-      return Value;
-    }
-    const ValueProxy *operator->() const {
-      assert(Offset && "Null dereference");
-      return &Value;
-    }
-
-    pointer() = default;
-
-  private:
-    friend class OnDiskDataAllocator;
-    pointer(FileOffset Offset, ValueProxy Value)
-        : Offset(Offset), Value(Value) {}
-    FileOffset Offset;
-    ValueProxy Value;
-  };
-
-  /// Look up the data stored at the given offset.
-  const char *beginData(FileOffset Offset) const;
-
-  /// Allocate at least \p Size with 8-byte alignment.
-  Expected<pointer> allocate(size_t Size);
-
-  /// \returns the buffer that was allocated at \p create time, with size
-  /// \p UserHeaderSize.
-  MutableArrayRef<uint8_t> getUserHeader();
-
-  size_t size() const;
-  size_t capacity() const;
-
-  static Expected<OnDiskDataAllocator>
-  create(const Twine &Path, const Twine &TableName, uint64_t MaxFileSize,
-         std::optional<uint64_t> NewFileInitialSize,
-         uint32_t UserHeaderSize = 0,
-         function_ref<void(void *)> UserHeaderInit = nullptr);
-
-  OnDiskDataAllocator(OnDiskDataAllocator &&RHS);
-  OnDiskDataAllocator &operator=(OnDiskDataAllocator &&RHS);
-
-  // No copy. Just call \a create() again.
-  OnDiskDataAllocator(const OnDiskDataAllocator &) = delete;
-  OnDiskDataAllocator &operator=(const OnDiskDataAllocator &) = delete;
-
-  ~OnDiskDataAllocator();
-
-private:
-  struct ImplType;
-  explicit OnDiskDataAllocator(std::unique_ptr<ImplType> Impl);
-  std::unique_ptr<ImplType> Impl;
-};
-
 } // namespace cas
 } // namespace llvm
 
-#endif // LLVM_CAS_ONDISKHASHMAPPEDTRIE_H
+#endif // LLVM_CAS_ONDISKTRIERAWHASHMAP_H

llvm/lib/CAS/CMakeLists.txt

@@ -2,6 +2,7 @@ add_llvm_component_library(LLVMCAS
   ActionCache.cpp
   ActionCaches.cpp
   BuiltinCAS.cpp
+  DatabaseFile.cpp
   InMemoryCAS.cpp
   MappedFileRegionArena.cpp
   ObjectStore.cpp

llvm/lib/CAS/DatabaseFile.cpp

@@ -0,0 +1,123 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file This file implements the common abstractions for CAS database file.
+///
+//===----------------------------------------------------------------------===//
+
+#include "DatabaseFile.h"
+
+using namespace llvm;
+using namespace llvm::cas;
+using namespace llvm::cas::ondisk;
+
+Error ondisk::createTableConfigError(std::errc ErrC, StringRef Path,
+                                     StringRef TableName, const Twine &Msg) {
+  return createStringError(make_error_code(ErrC),
+                           Path + "[" + TableName + "]: " + Msg);
+}
+
+Error ondisk::checkTable(StringRef Label, size_t Expected, size_t Observed,
+                         StringRef Path, StringRef TrieName) {
+  if (Expected == Observed)
+    return Error::success();
+  return createTableConfigError(std::errc::invalid_argument, Path, TrieName,
+                                "mismatched " + Label +
+                                    " (expected: " + Twine(Expected) +
+                                    ", observed: " + Twine(Observed) + ")");
+}
+
+Expected<DatabaseFile>
+DatabaseFile::create(const Twine &Path, uint64_t Capacity,
+                     function_ref<Error(DatabaseFile &)> NewDBConstructor) {
+  // Constructor for if the file doesn't exist.
+  auto NewFileConstructor = [&](MappedFileRegionArena &Alloc) -> Error {
+    if (Alloc.capacity() <
+        sizeof(Header) + sizeof(MappedFileRegionArena::Header))
+      return createTableConfigError(std::errc::argument_out_of_domain,
+                                    Path.str(), "datafile",
+                                    "Allocator too small for header");
+    (void)new (Alloc.data()) Header{getMagic(), getVersion(), {0}};
+    DatabaseFile DB(Alloc);
+    return NewDBConstructor(DB);
+  };
+
+  // Get or create the file.
+  MappedFileRegionArena Alloc;
+  if (Error E = MappedFileRegionArena::create(Path, Capacity, sizeof(Header),
+                                              NewFileConstructor)
+                    .moveInto(Alloc))
+    return std::move(E);
+
+  return DatabaseFile::get(
+      std::make_unique<MappedFileRegionArena>(std::move(Alloc)));
+}
+
+Error DatabaseFile::addTable(TableHandle Table) {
+  assert(Table);
+  assert(&Table.getRegion() == &getRegion());
+  int64_t ExistingRootOffset = 0;
+  const int64_t NewOffset =
+      reinterpret_cast<const char *>(&Table.getHeader()) - getRegion().data();
+  if (H->RootTableOffset.compare_exchange_strong(ExistingRootOffset, NewOffset))
+    return Error::success();
+
+  // Silently ignore attempts to set the root to itself.
+  if (ExistingRootOffset == NewOffset)
+    return Error::success();
+
+  // Return an proper error message.
+  TableHandle Root(getRegion(), ExistingRootOffset);
+  if (Root.getName() == Table.getName())
+    return createStringError(
+        make_error_code(std::errc::not_supported),
+        "collision with existing table of the same name '" + Table.getName() +
+            "'");
+
+  return createStringError(make_error_code(std::errc::not_supported),
+                           "cannot add new table '" + Table.getName() +
+                               "'"
+                               " to existing root '" +
+                               Root.getName() + "'");
+}
+
+std::optional<TableHandle> DatabaseFile::findTable(StringRef Name) {
+  int64_t RootTableOffset = H->RootTableOffset.load();
+  if (!RootTableOffset)
+    return std::nullopt;
+
+  TableHandle Root(getRegion(), RootTableOffset);
+  if (Root.getName() == Name)
+    return Root;
+
+  return std::nullopt;
+}
+
+Error DatabaseFile::validate(MappedFileRegion &Region) {
+  if (Region.size() < sizeof(Header))
+    return createStringError(std::errc::invalid_argument,
+                             "database: missing header");
+
+  // Check the magic and version.
+  auto *H = reinterpret_cast<Header *>(Region.data());
+  if (H->Magic != getMagic())
+    return createStringError(std::errc::invalid_argument,
+                             "database: bad magic");
+  if (H->Version != getVersion())
+    return createStringError(std::errc::invalid_argument,
+                             "database: wrong version");
+
+  auto *MFH = reinterpret_cast<MappedFileRegionArena::Header *>(Region.data() +
+                                                                sizeof(Header));
+  // Check the bump-ptr, which should point past the header.
+  if (MFH->BumpPtr.load() < (int64_t)sizeof(Header))
+    return createStringError(std::errc::invalid_argument,
+                             "database: corrupt bump-ptr");
+
+  return Error::success();
+}