[CAS] Add llvm-cas tools to inspect on-disk LLVMCAS

llvm/test/CMakeLists.txt

@@ -28,6 +28,7 @@ llvm_canonicalize_cmake_booleans(
   LLVM_INCLUDE_SPIRV_TOOLS_TESTS
   LLVM_APPEND_VC_REV
   LLVM_HAS_LOGF128
+  LLVM_ENABLE_ONDISK_CAS
   )
 
 configure_lit_site_cfg(
@@ -73,6 +74,7 @@ set(LLVM_TEST_DEPENDS
           llvm-bcanalyzer
           llvm-bitcode-strip
           llvm-c-test
+          llvm-cas
           llvm-cat
           llvm-cfi-verify
           llvm-cgdata

llvm/test/lit.site.cfg.py.in

@@ -64,6 +64,7 @@ config.spirv_tools_tests = @LLVM_INCLUDE_SPIRV_TOOLS_TESTS@
 config.have_vc_rev = @LLVM_APPEND_VC_REV@
 config.force_vc_rev = "@LLVM_FORCE_VC_REVISION@"
 config.has_logf128 = @LLVM_HAS_LOGF128@
+config.have_ondisk_cas = @LLVM_ENABLE_ONDISK_CAS@
 
 import lit.llvm
 lit.llvm.initialize(lit_config, config)

llvm/test/tools/llvm-cas/Inputs/oneline

@@ -0,0 +1 @@
+content

llvm/test/tools/llvm-cas/Inputs/oneline-nonewline

@@ -0,0 +1 @@
+content

llvm/test/tools/llvm-cas/cache.test

@@ -0,0 +1,14 @@
+RUN: rm -rf %t %t.cas
+RUN: mkdir %t
+
+RUN: llvm-cas --cas %t.cas --make-blob \
+RUN:   --data /dev/null > %t/empty.casid
+RUN: echo "abc" | \
+RUN:   llvm-cas --cas %t.cas --make-blob \
+RUN:   --data - >%t/abc.casid
+
+RUN: llvm-cas --cas %t/cas --put-cache-key @%t/abc.casid @%t/empty.casid
+RUN: llvm-cas --cas %t/cas --get-cache-result @%t/abc.casid > %t/empty2.casid
+RUN: diff %t/empty.casid %t/empty2.casid
+
+RUN: not llvm-cas --cas %t/cas --get-cache-result @%t/empty.casid

llvm/test/tools/llvm-cas/lit.local.cfg

@@ -0,0 +1,2 @@
+if not config.have_ondisk_cas:
+    config.unsupported = True

llvm/test/tools/llvm-cas/make-blob.test

@@ -0,0 +1,37 @@
+RUN: rm -rf %t %t.cas
+RUN: mkdir %t
+
+RUN: llvm-cas --cas %t.cas --make-blob \
+RUN:   --data - </dev/null >%t/empty.casid
+RUN: sed -e 's,^.,CHECK: ,' <%t/empty.casid >%t/empty.check
+RUN: llvm-cas --cas %t.cas --make-blob \
+RUN:   --data /dev/null | FileCheck %t/empty.check
+RUN: echo "abc" | \
+RUN:   llvm-cas --cas %t.cas --make-blob \
+RUN:   --data - >%t/abc.casid
+RUN: llvm-cas --cas %t.cas --make-blob \
+RUN:   --data %S/Inputs/oneline >%t/oneline.casid
+RUN: llvm-cas --cas %t.cas --make-blob \
+RUN:   --data %S/Inputs/oneline-nonewline >%t/oneline-nonewline.casid
+
+RUN: llvm-cas --cas %t.cas --cat-data @%t/empty.casid |\
+RUN:   FileCheck %s -check-prefix CHECK-EMPTY -allow-empty
+CHECK-EMPTY-NOT: {{.}}
+
+RUN: llvm-cas --cas %t.cas --cat-data @%t/abc.casid |\
+RUN:   FileCheck %s -check-prefix CHECK-ABC
+CHECK-ABC: abc
+
+RUN: llvm-cas --cas %t.cas --cat-data @%t/oneline-nonewline.casid |\
+RUN:   FileCheck %s -check-prefix CHECK-ONELINE
+RUN: llvm-cas --cas %t.cas --cat-data @%t/oneline.casid |\
+RUN:   FileCheck %s -check-prefix CHECK-ONELINE
+CHECK-ONELINE: content
+
+# Double-check newlines.
+RUN: llvm-cas --cas %t.cas --cat-data @%t/oneline-nonewline.casid \
+RUN:   >%t/oneline-nonewline
+RUN: diff %S/Inputs/oneline-nonewline %t/oneline-nonewline
+RUN: llvm-cas --cas %t.cas --cat-data @%t/oneline.casid \
+RUN:   >%t/oneline
+RUN: diff %S/Inputs/oneline %t/oneline

llvm/test/tools/llvm-cas/make-node.test

@@ -0,0 +1,30 @@
+RUN: rm -rf %t
+RUN: mkdir %t
+
+# Make some empty objects.
+RUN: llvm-cas --cas %t/cas --make-node \
+RUN:   --data - </dev/null >%t/empty.casid
+
+RUN: llvm-cas --cas %t/cas --cat-data @%t/empty.casid |\
+RUN:   FileCheck %s -check-prefix CHECK-EMPTY -allow-empty
+RUN: llvm-cas --cas %t/cas --ls-node-refs @%t/empty.casid |\
+RUN:   FileCheck %s -check-prefix CHECK-EMPTY -allow-empty
+CHECK-EMPTY-NOT: {{.}}
+
+# Make a complex object, which references existing ones. Reference a blob and
+# other objects, and reference one of them twice to be sure they don't get
+# deduped.
+RUN: llvm-cas --cas %t/cas --make-blob --data /dev/null \
+RUN:   >%t/empty-blob.casid
+RUN: cat %t/empty.casid %t/empty.casid %t/empty-blob.casid \
+RUN:   >%t/complex.refs
+RUN: cat %t/complex.refs | sed -e 's,^.,CHECK: ,' > %t/complex.check
+RUN: llvm-cas --cas %t/cas --make-node \
+RUN:   --data %S/Inputs/oneline @%t/complex.refs \
+RUN:   >%t/complex.casid
+RUN: llvm-cas --cas %t/cas --cat-data \
+RUN:   @%t/complex.casid | FileCheck %s -check-prefix COMPLEX-DATA
+RUN: llvm-cas --cas %t/cas --ls-node-refs @%t/complex.casid |\
+RUN:   FileCheck %t/complex.check
+COMPLEX-DATA: content
+

llvm/test/tools/llvm-cas/print-id.test

@@ -0,0 +1,14 @@
+RUN: rm -rf %t
+RUN: mkdir %t
+
+RUN: echo "test" > %t/file
+RUN: llvm-cas --cas %t/cas --make-blob --data=%t/file > %t/id
+
+# Confirm that the ID has the right prefix, is well-formed, and that there's
+# nothing else on the line.
+RUN: FileCheck %s --match-full-lines --strict-whitespace <%t/id
+CHECK:llvmcas://{{[a-z0-9]+}}
+
+# Confirm that there's a newline after.
+RUN: wc -l <%t/id | FileCheck %s -check-prefix=NEWLINE
+NEWLINE: 1

llvm/tools/llvm-cas/CMakeLists.txt

@@ -0,0 +1,8 @@
+set(LLVM_LINK_COMPONENTS
+  Support
+  CAS
+  )
+
+add_llvm_tool(llvm-cas
+  llvm-cas.cpp
+  )

llvm/tools/llvm-cas/llvm-cas.cpp

@@ -0,0 +1,234 @@
+//===- llvm-cas.cpp - CAS tool --------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/CAS/ActionCache.h"
+#include "llvm/CAS/ObjectStore.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
+#include <memory>
+
+using namespace llvm;
+using namespace llvm::cas;
+
+static cl::list<std::string> Inputs(cl::Positional, cl::desc("Input object"));
+
+static int dump(ObjectStore &CAS);
+static int listObjectReferences(ObjectStore &CAS, const CASID &ID);
+static int catNodeData(ObjectStore &CAS, const CASID &ID);
+static int makeBlob(ObjectStore &CAS, StringRef DataPath);
+static int makeNode(ObjectStore &CAS, ArrayRef<std::string> References,
+                    StringRef DataPath);
+static int putCacheKey(ObjectStore &CAS, ActionCache &AC,
+                       ArrayRef<std::string> Objects);
+static int getCacheResult(ObjectStore &CAS, ActionCache &AC, const CASID &ID);
+static int validateObject(ObjectStore &CAS, const CASID &ID);
+
+int main(int Argc, char **Argv) {
+  InitLLVM X(Argc, Argv);
+
+  cl::opt<std::string> CASPath("cas", cl::desc("Path to CAS on disk."),
+                               cl::value_desc("path"));
+  cl::opt<std::string> UpstreamCASPath(
+      "upstream-cas", cl::desc("Path to another CAS."), cl::value_desc("path"));
+  cl::opt<std::string> DataPath("data",
+                                cl::desc("Path to data or '-' for stdin."),
+                                cl::value_desc("path"));
+
+  enum CommandKind {
+    Invalid,
+    Dump,
+    CatNodeData,
+    DiffGraphs,
+    TraverseGraph,
+    MakeBlob,
+    MakeNode,
+    ListObjectReferences,
+    MergeTrees,
+    Import,
+    PutCacheKey,
+    GetCacheResult,
+    Validate,
+  };
+  cl::opt<CommandKind> Command(
+      cl::desc("choose command action:"),
+      cl::values(
+          clEnumValN(Dump, "dump", "dump internal contents"),
+          clEnumValN(CatNodeData, "cat-data", "cat node data"),
+          clEnumValN(DiffGraphs, "diff-graphs", "diff graphs"),
+          clEnumValN(TraverseGraph, "traverse-graph", "traverse graph"),
+          clEnumValN(MakeBlob, "make-blob", "make blob"),
+          clEnumValN(MakeNode, "make-node", "make node"),
+          clEnumValN(ListObjectReferences, "ls-node-refs", "list node refs"),
+          clEnumValN(MergeTrees, "merge", "merge paths/cas-ids"),
+          clEnumValN(Import, "import", "import objects from another CAS"),
+          clEnumValN(PutCacheKey, "put-cache-key",
+                     "set a value for a cache key"),
+          clEnumValN(GetCacheResult, "get-cache-result",
+                     "get the result value from a cache key"),
+          clEnumValN(Validate, "validate", "validate the object for CASID")),
+      cl::init(CommandKind::Invalid));
+
+  cl::ParseCommandLineOptions(Argc, Argv, "llvm-cas CAS tool\n");
+  ExitOnError ExitOnErr("llvm-cas: ");
+
+  if (Command == CommandKind::Invalid)
+    ExitOnErr(createStringError(inconvertibleErrorCode(),
+                                "no command action is specified"));
+
+  // FIXME: Consider creating an in-memory CAS.
+  if (CASPath.empty())
+    ExitOnErr(
+        createStringError(inconvertibleErrorCode(), "missing --cas=<path>"));
+
+  std::shared_ptr<ObjectStore> CAS = ExitOnErr(createOnDiskCAS(CASPath));
+  std::shared_ptr<ActionCache> AC = ExitOnErr(createOnDiskActionCache(CASPath));
+  assert(CAS && "missng CAS instance");
+
+  if (Command == Dump)
+    return dump(*CAS);
+
+  if (Command == MakeBlob)
+    return makeBlob(*CAS, DataPath);
+
+  if (Command == MakeNode)
+    return makeNode(*CAS, Inputs, DataPath);
+
+  if (Inputs.empty())
+    ExitOnErr(createStringError(inconvertibleErrorCode(),
+                                "missing <object> to operate on"));
+
+  if (Command == PutCacheKey || Command == GetCacheResult) {
+    if (!AC)
+      ExitOnErr(createStringError(inconvertibleErrorCode(),
+                                  "no action-cache available"));
+  }
+
+  if (Command == PutCacheKey)
+    return putCacheKey(*CAS, *AC, Inputs);
+
+  // Remaining commands need exactly one CAS object.
+  if (Inputs.size() > 1)
+    ExitOnErr(createStringError(inconvertibleErrorCode(),
+                                "too many <object>s, expected 1"));
+  CASID ID = ExitOnErr(CAS->parseID(Inputs.front()));
+
+  if (Command == GetCacheResult)
+    return getCacheResult(*CAS, *AC, ID);
+
+  if (Command == ListObjectReferences)
+    return listObjectReferences(*CAS, ID);
+
+  if (Command == CatNodeData)
+    return catNodeData(*CAS, ID);
+
+  assert(Command == Validate);
+  return validateObject(*CAS, ID);
+}
+
+static Expected<std::unique_ptr<MemoryBuffer>>
+openBuffer(StringRef DataPath) {
+  if (DataPath.empty())
+    return createStringError(inconvertibleErrorCode(), "--data missing");
+  return errorOrToExpected(
+      DataPath == "-" ? llvm::MemoryBuffer::getSTDIN()
+                      : llvm::MemoryBuffer::getFile(DataPath));
+}
+
+int dump(ObjectStore &CAS) {
+  ExitOnError ExitOnErr("llvm-cas: dump: ");
+  CAS.print(llvm::outs());
+  return 0;
+}
+
+int makeBlob(ObjectStore &CAS, StringRef DataPath) {
+  ExitOnError ExitOnErr("llvm-cas: make-blob: ");
+  std::unique_ptr<MemoryBuffer> Buffer = ExitOnErr(openBuffer(DataPath));
+
+  ObjectProxy Blob =
+      ExitOnErr(CAS.createProxy(std::nullopt, Buffer->getBuffer()));
+  llvm::outs() << Blob.getID() << "\n";
+  return 0;
+}
+
+int catNodeData(ObjectStore &CAS, const CASID &ID) {
+  ExitOnError ExitOnErr("llvm-cas: cat-node-data: ");
+  llvm::outs() << ExitOnErr(CAS.getProxy(ID)).getData();
+  return 0;
+}
+
+int listObjectReferences(ObjectStore &CAS, const CASID &ID) {
+  ExitOnError ExitOnErr("llvm-cas: ls-node-refs: ");
+
+  ObjectProxy Object = ExitOnErr(CAS.getProxy(ID));
+  ExitOnErr(Object.forEachReference([&](ObjectRef Ref) -> Error {
+    llvm::outs() << CAS.getID(Ref) << "\n";
+    return Error::success();
+  }));
+
+  return 0;
+}
+
+static int makeNode(ObjectStore &CAS, ArrayRef<std::string> Objects,
+                    StringRef DataPath) {
+  std::unique_ptr<MemoryBuffer> Data =
+      ExitOnError("llvm-cas: make-node: data: ")(openBuffer(DataPath));
+
+  SmallVector<ObjectRef> IDs;
+  for (StringRef Object : Objects) {
+    ExitOnError ObjectErr("llvm-cas: make-node: ref: ");
+    std::optional<ObjectRef> ID =
+        CAS.getReference(ObjectErr(CAS.parseID(Object)));
+    if (!ID)
+      ObjectErr(createStringError(inconvertibleErrorCode(),
+                                  "unknown object '" + Object + "'"));
+    IDs.push_back(*ID);
+  }
+
+  ExitOnError ExitOnErr("llvm-cas: make-node: ");
+  ObjectProxy Object = ExitOnErr(CAS.createProxy(IDs, Data->getBuffer()));
+  llvm::outs() << Object.getID() << "\n";
+  return 0;
+}
+
+static int putCacheKey(ObjectStore &CAS, ActionCache &AC,
+                       ArrayRef<std::string> Objects) {
+  ExitOnError ExitOnErr("llvm-cas: put-cache-key: ");
+
+  if (Objects.size() % 2 != 0)
+    ExitOnErr(createStringError(inconvertibleErrorCode(),
+                                "expected pairs of inputs"));
+  while (!Objects.empty()) {
+    CASID Key = ExitOnErr(CAS.parseID(Objects[0]));
+    CASID Result = ExitOnErr(CAS.parseID(Objects[1]));
+    Objects = Objects.drop_front(2);
+    ExitOnErr(AC.put(Key, Result));
+  }
+  return 0;
+}
+
+static int getCacheResult(ObjectStore &CAS, ActionCache &AC, const CASID &ID) {
+  ExitOnError ExitOnErr("llvm-cas: get-cache-result: ");
+
+  auto Result = ExitOnErr(AC.get(ID));
+  if (!Result) {
+    outs() << "result not found\n";
+    return 1;
+  }
+  outs() << *Result << "\n";
+  return 0;
+}
+
+int validateObject(ObjectStore &CAS, const CASID &ID) {
+  ExitOnError ExitOnErr("llvm-cas: validate: ");
+  ExitOnErr(CAS.validate(ID));
+  outs() << ID << ": validated successfully\n";
+  return 0;
+}