Skip to content

[CAS] Add llvm-cas tools to inspect on-disk LLVMCAS #166481

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Nov 7, 2025
Merged

[CAS] Add llvm-cas tools to inspect on-disk LLVMCAS #166481

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions llvm/test/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ llvm_canonicalize_cmake_booleans(
LLVM_INCLUDE_SPIRV_TOOLS_TESTS
LLVM_APPEND_VC_REV
LLVM_HAS_LOGF128
LLVM_ENABLE_ONDISK_CAS
)

configure_lit_site_cfg(
Expand Down Expand Up @@ -81,6 +82,7 @@ set(LLVM_TEST_DEPENDS
llvm-bcanalyzer
llvm-bitcode-strip
llvm-c-test
llvm-cas
llvm-cat
llvm-cfi-verify
llvm-cgdata
Expand Down
4 changes: 4 additions & 0 deletions llvm/test/lit.cfg.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,6 +229,7 @@ def get_asan_rtlib():
"llvm-addr2line",
"llvm-bcanalyzer",
"llvm-bitcode-strip",
"llvm-cas",
"llvm-cgdata",
"llvm-config",
"llvm-cov",
Expand Down Expand Up @@ -792,6 +793,9 @@ def host_unwind_supports_jit():
if config.expensive_checks:
config.available_features.add("expensive_checks")

if config.have_ondisk_cas:
config.available_features.add("ondisk_cas")

if "MemoryWithOrigins" in config.llvm_use_sanitizer:
config.available_features.add("use_msan_with_origins")

Expand Down
1 change: 1 addition & 0 deletions llvm/test/lit.site.cfg.py.in
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@ config.spirv_tools_tests = @LLVM_INCLUDE_SPIRV_TOOLS_TESTS@
config.have_vc_rev = @LLVM_APPEND_VC_REV@
config.force_vc_rev = "@LLVM_FORCE_VC_REVISION@"
config.has_logf128 = @LLVM_HAS_LOGF128@
config.have_ondisk_cas = @LLVM_ENABLE_ONDISK_CAS@

import lit.llvm
lit.llvm.initialize(lit_config, config)
Expand Down
1 change: 1 addition & 0 deletions llvm/test/tools/llvm-cas/Inputs/oneline
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
content
1 change: 1 addition & 0 deletions llvm/test/tools/llvm-cas/Inputs/oneline-nonewline
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
content
14 changes: 14 additions & 0 deletions llvm/test/tools/llvm-cas/action-cache.test
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
RUN: rm -rf %t %t.cas
RUN: mkdir %t

RUN: llvm-cas --cas %t.cas --make-blob \
RUN: --data %S/Inputs/oneline >%t/oneline.casid
RUN: llvm-cas --cas %t.cas --make-blob \
RUN: --data %S/Inputs/oneline-nonewline >%t/oneline-nonewline.casid

RUN: llvm-cas --cas %t.cas --put-cache-key @%t/oneline.casid @%t/oneline-nonewline.casid
RUN: llvm-cas --cas %t.cas --get-cache-result @%t/oneline.casid > %t/result.casid
RUN: diff %t/oneline-nonewline.casid %t/result.casid

RUN: not llvm-cas --cas %t.cas --get-cache-result @%t/oneline-nonewline.casid 2>&1 | FileCheck %s
CHECK: result not found
14 changes: 14 additions & 0 deletions llvm/test/tools/llvm-cas/cache.test
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
RUN: rm -rf %t %t.cas
RUN: mkdir %t

RUN: llvm-cas --cas %t.cas --make-blob \
RUN: --data /dev/null > %t/empty.casid
RUN: echo "abc" | \
RUN: llvm-cas --cas %t.cas --make-blob \
RUN: --data - >%t/abc.casid

RUN: llvm-cas --cas %t/cas --put-cache-key @%t/abc.casid @%t/empty.casid
RUN: llvm-cas --cas %t/cas --get-cache-result @%t/abc.casid > %t/empty2.casid
RUN: diff %t/empty.casid %t/empty2.casid

RUN: not llvm-cas --cas %t/cas --get-cache-result @%t/empty.casid
27 changes: 27 additions & 0 deletions llvm/test/tools/llvm-cas/dump.test
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
RUN: rm -rf %t
RUN: mkdir %t

RUN: llvm-cas --cas %t/cas --make-blob \
RUN: --data - </dev/null

RUN: llvm-cas --cas %t/cas --make-blob \
RUN: --data %s

RUN: llvm-cas --cas %t/cas --dump | FileCheck %s

// check the dump format.
CHECK: index:
CHECK-NEXT: hash-num-bits=
CHECK-NEXT: root addr=
// it should has at least one index
CHECK-NEXT: - index=

// two records
CHECK: record
CHECK-NEXT: - addr=
CHECK-NEXT: - addr=

// both should be small enough to be in data pool
CHECK: pool:
CHECK-NEXT: - addr=
CHECK-NEXT: - addr=
2 changes: 2 additions & 0 deletions llvm/test/tools/llvm-cas/lit.local.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
if not config.have_ondisk_cas:
config.unsupported = True
41 changes: 41 additions & 0 deletions llvm/test/tools/llvm-cas/make-blob.test
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
RUN: rm -rf %t %t.cas
RUN: mkdir %t

RUN: llvm-cas --cas %t.cas --make-blob \
RUN: --data - </dev/null >%t/empty.casid
RUN: sed -e 's,^.,CHECK: ,' <%t/empty.casid >%t/empty.check
RUN: llvm-cas --cas %t.cas --make-blob \
RUN: --data /dev/null | FileCheck %t/empty.check
RUN: echo "abc" | \
RUN: llvm-cas --cas %t.cas --make-blob \
RUN: --data - >%t/abc.casid
RUN: llvm-cas --cas %t.cas --make-blob \
RUN: --data %S/Inputs/oneline >%t/oneline.casid
RUN: llvm-cas --cas %t.cas --make-blob \
RUN: --data %S/Inputs/oneline-nonewline >%t/oneline-nonewline.casid

RUN: llvm-cas --cas %t.cas --cat-node-data @%t/empty.casid |\
RUN: FileCheck %s -check-prefix CHECK-EMPTY -allow-empty
CHECK-EMPTY-NOT: {{.}}

RUN: llvm-cas --cas %t.cas --cat-node-data @%t/abc.casid |\
RUN: FileCheck %s -check-prefix CHECK-ABC
CHECK-ABC: abc

RUN: llvm-cas --cas %t.cas --cat-node-data @%t/oneline-nonewline.casid |\
RUN: FileCheck %s -check-prefix CHECK-ONELINE
RUN: llvm-cas --cas %t.cas --cat-node-data @%t/oneline.casid |\
RUN: FileCheck %s -check-prefix CHECK-ONELINE
CHECK-ONELINE: content

# Double-check newlines.
RUN: llvm-cas --cas %t.cas --cat-node-data @%t/oneline-nonewline.casid \
RUN: >%t/oneline-nonewline
RUN: diff %S/Inputs/oneline-nonewline %t/oneline-nonewline
RUN: llvm-cas --cas %t.cas --cat-node-data @%t/oneline.casid \
RUN: >%t/oneline
RUN: diff %S/Inputs/oneline %t/oneline

# Validate
RUN: llvm-cas --cas %t.cas --validate-object @%t/oneline-nonewline.casid
RUN: llvm-cas --cas %t.cas --validate-object @%t/oneline.casid
37 changes: 37 additions & 0 deletions llvm/test/tools/llvm-cas/make-node.test
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
RUN: rm -rf %t
RUN: mkdir %t

# Make some empty objects.
RUN: llvm-cas --cas %t/cas --make-node \
RUN: --data - </dev/null >%t/empty.casid

RUN: llvm-cas --cas %t/cas --cat-node-data @%t/empty.casid |\
RUN: FileCheck %s -check-prefix CHECK-EMPTY -allow-empty
RUN: llvm-cas --cas %t/cas --ls-node-refs @%t/empty.casid |\
RUN: FileCheck %s -check-prefix CHECK-EMPTY -allow-empty
CHECK-EMPTY-NOT: {{.}}

# Make a complex object, which references existing ones. Reference a blob and
# other objects, and reference one of them twice to be sure they don't get
# deduped.
RUN: llvm-cas --cas %t/cas --make-blob --data /dev/null \
RUN: >%t/empty-blob.casid
RUN: cat %t/empty.casid %t/empty.casid %t/empty-blob.casid \
RUN: >%t/complex.refs
RUN: cat %t/complex.refs | sed -e 's,^.,CHECK: ,' > %t/complex.check
RUN: llvm-cas --cas %t/cas --make-node \
RUN: --data %S/Inputs/oneline @%t/complex.refs \
RUN: >%t/complex.casid
RUN: llvm-cas --cas %t/cas --cat-node-data \
RUN: @%t/complex.casid | FileCheck %s -check-prefix COMPLEX-DATA
RUN: llvm-cas --cas %t/cas --ls-node-refs @%t/complex.casid |\
RUN: FileCheck %t/complex.check
COMPLEX-DATA: content

RUN: llvm-cas --cas %t/cas --validate-object @%t/complex.casid

# Import from a new CAS.
RUN: llvm-cas --cas %t/new-cas --upstream-cas %t/cas --import @%t/complex.casid
RUN: llvm-cas --cas %t/new-cas --cat-node-data \
RUN: @%t/complex.casid | FileCheck %s -check-prefix COMPLEX-DATA
RUN: llvm-cas --cas %t/new-cas --validate
13 changes: 13 additions & 0 deletions llvm/test/tools/llvm-cas/print-id.test
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
RUN: rm -rf %t
RUN: mkdir %t

RUN: llvm-cas --cas %t/cas --make-blob --data %S/Inputs/oneline > %t/id

# Confirm that the ID has the right prefix, is well-formed, and that there's
# nothing else on the line.
RUN: FileCheck %s --match-full-lines --strict-whitespace <%t/id
CHECK:llvmcas://{{[a-z0-9]+}}

# Confirm that there's a newline after.
RUN: wc -l <%t/id | FileCheck %s -check-prefix=NEWLINE
NEWLINE: 1
31 changes: 31 additions & 0 deletions llvm/test/tools/llvm-cas/validation.test
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
RUN: rm -rf %t
RUN: mkdir %t

# Ingest a blob which just fits inside the CAS data pool to make sure the validate passes.
RUN: truncate -s 7 %t/file
RUN: cat %t/file | \
RUN: llvm-cas --cas %t/cas --make-blob \
RUN: --data -
RUN: llvm-cas --cas %t/cas --validate --check-hash

RUN: llvm-cas --cas %t/cas --validate
RUN: llvm-cas --cas %t/cas --validate --check-hash

RUN: rm %t/cas/v1.1/data.v1
RUN: not llvm-cas --cas %t/cas --validate
RUN: not llvm-cas --cas %t/cas --validate --check-hash

RUN: mkdir %t/ac

RUN: llvm-cas --cas %t/ac --make-blob \
RUN: --data /dev/null > %t/empty.casid
RUN: echo "abc" | \
RUN: llvm-cas --cas %t/ac --make-blob \
RUN: --data - >%t/abc.casid

RUN: llvm-cas --cas %t/ac --put-cache-key @%t/abc.casid @%t/empty.casid
RUN: llvm-cas --cas %t/ac --validate
# Note: records are 40 bytes (32 hash bytes + 8 byte value), so trim the last
# allocated record, leaving it invalid.
RUN: truncate -s -40 %t/ac/v1.1/actions.v1
RUN: not llvm-cas --cas %t/ac --validate
17 changes: 17 additions & 0 deletions llvm/tools/llvm-cas/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
set(LLVM_TARGET_DEFINITIONS Options.td)
tablegen(LLVM Options.inc -gen-opt-parser-defs)
add_public_tablegen_target(LLVMCASToolTableGen)

set(LLVM_LINK_COMPONENTS
Support
CAS
Option
)

add_llvm_tool(llvm-cas
llvm-cas.cpp

DEPENDS
${tablegen_deps}
LLVMCASToolTableGen
)
63 changes: 63 additions & 0 deletions llvm/tools/llvm-cas/Options.td
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
include "llvm/Option/OptParser.td"

class F<string name> : Flag<["--", "-"], name>;

def grp_action : OptionGroup<"Actions">, HelpText<"llvm-cas actions">;
Copy link
Member

@petrhosek petrhosek Nov 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Have you considered using subcommands for actions? The subcommand support was introduced in #155026.

Copy link
Collaborator Author

@cachemeifyoucan cachemeifyoucan Nov 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I am aware. But we ship this tool already. I would like to prevent breaking current command-line interface if possible.

Copy link
Contributor

@ilovepi ilovepi Nov 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it possible to migrate them over the course of a release or two?

Copy link
Collaborator Author

@cachemeifyoucan cachemeifyoucan Nov 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe. I would like to do it incrementally since we need to teach build system about the new command before we can break the old one.

Copy link
Member

@petrhosek petrhosek Nov 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should be able to support both syntaxes to avoid breaking existing uses and mark the flag-based versions as deprecated so new users are encouraged to use the subcommand syntax. I'm fine doing it in a follow up change if you prefer.

Copy link
Collaborator Author

@cachemeifyoucan cachemeifyoucan Nov 6, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@petrhosek I think subcommand is not usable with positional arguments. That is a deal breaker for now to use subcommand parsing.

It would also be nice if a flag can be alias to a subcommand but that is not strictly needed.

Copy link
Collaborator Author

@cachemeifyoucan cachemeifyoucan Nov 6, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

file issue: #166830

Copy link
Contributor

@Prabhuk Prabhuk Nov 6, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@cachemeifyoucan -- I responded about positional argument handling in the bug. PTAL.


def help : F<"help">, HelpText<"Prints this help output">;
def : Flag<["-"], "h">, Alias<help>, HelpText<"Alias for --help">;

// Tool actions

def cas_dump : F<"dump">, HelpText<"Dump internal contents">, Group<grp_action>;
def cat_node_data : F<"cat-node-data">,
HelpText<"Cat node data">,
Group<grp_action>;
def make_blob : F<"make-blob">, HelpText<"Make blob">, Group<grp_action>;
def make_node : F<"make-node">, HelpText<"Make node">, Group<grp_action>;
def ls_node_refs : F<"ls-node-refs">,
HelpText<"List node refs">,
Group<grp_action>;
def import : F<"import">,
HelpText<"Import objects from another CAS">,
Group<grp_action>;
def put_cache_key : F<"put-cache-key">,
HelpText<"Set a value for a cache key">,
Group<grp_action>;
def get_cache_result : F<"get-cache-result">,
HelpText<"Get the result value from a cache key">,
Group<grp_action>;
def validate : F<"validate">,
HelpText<"Validate ObjectStore">,
Group<grp_action>;
def validate_object : F<"validate-object">,
HelpText<"Validate the object for CASID">,
Group<grp_action>;
def validate_if_needed : F<"validate-if-needed">,
HelpText<"Validate cas contents if needed">,
Group<grp_action>;
def prune : F<"prune">, HelpText<"Prune local cas storage">, Group<grp_action>;

// Tool options

def cas_path : Separate<["-", "--"], "cas">,
MetaVarName<"<path>">,
HelpText<"Path to CAS on disk">;

def upstream_cas : Separate<["-", "--"], "upstream-cas">,
MetaVarName<"<path>">,
HelpText<"Path to another upstream CAS">;

def data : Separate<["-", "--"], "data">,
MetaVarName<"<path>">,
HelpText<"Path to data or '-' for stdin">;

def check_hash : F<"check-hash">,
HelpText<"Check all hashes during validation">;

def allow_recovery : F<"allow-recovery">,
HelpText<"Allow recovery of CAS data">;

def force : F<"force">, HelpText<"Force validation even if unnecessary">;

def in_process : F<"in-process">, HelpText<"Validation in-process">;
Loading