Skip to content

Commit 81e936c

Browse files
akyrtzibenlangmuir
akyrtzi
authored and
benlangmuir
committed
[llvm][cas] Add "plugin://" CAS identifier scheme
Allows configuring to use the plugin for all the tools that use `createCASFromIdentifier`. (cherry picked from commit 1fbcf60)
1 parent 3e8689b commit 81e936c

File tree

12 files changed

+74
-30
lines changed

12 files changed

+74
-30
lines changed

‎llvm/include/llvm/CAS/ObjectStore.h

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -394,20 +394,21 @@ std::string getDefaultOnDiskCASStableID();
394394
/// schemes:
395395
/// * InMemory CAS: mem://
396396
/// * OnDisk CAS: file://${PATH_TO_ONDISK_CAS}
397-
/// * PlugIn CAS: plugin://${PATH_TO_PLUGIN}
397+
/// * PlugIn CAS: plugin://${PATH_TO_PLUGIN}?${OPT1}=${VAL1}&${OPT2}=${VAL2}..
398398
/// If no URL scheme is used, it defaults to following (but might change in
399399
/// future)
400400
/// * empty string: Error!
401401
/// * "auto": default OnDiskCAS location
402402
/// * Other: path to OnDiskCAS.
403-
/// FIXME: Need to implement proper URL encoding scheme that allows "%". Also
404-
/// considering encode plugin CAS arguments as following:
405-
/// "plugin://${PATH_TO_PLUGIN}?${ARG1},${ARG2}..."
406-
Expected<std::unique_ptr<ObjectStore>> createCASFromIdentifier(StringRef Path);
403+
/// For the plugin scheme, use argument "ondisk-path=${PATH}" to choose the
404+
/// on-disk directory that the plugin should use, otherwise the default
405+
/// OnDiskCAS location will be used.
406+
/// FIXME: Need to implement proper URL encoding scheme that allows "%".
407+
Expected<std::shared_ptr<ObjectStore>> createCASFromIdentifier(StringRef Path);
407408

408409
/// Register a URL scheme to CAS Identifier.
409410
using ObjectStoreCreateFuncTy =
410-
Expected<std::unique_ptr<ObjectStore>>(const Twine &);
411+
Expected<std::shared_ptr<ObjectStore>>(const Twine &);
411412
void registerCASURLScheme(StringRef Prefix, ObjectStoreCreateFuncTy *Func);
412413

413414
class ActionCache;

‎llvm/include/llvm/RemoteCachingService/RemoteCachingService.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
namespace llvm::cas {
1616
/// Create GRPC ObjectStore from a path.
17-
Expected<std::unique_ptr<ObjectStore>> createGRPCRelayCAS(const Twine &Path);
17+
Expected<std::shared_ptr<ObjectStore>> createGRPCRelayCAS(const Twine &Path);
1818

1919
/// Create GRPC ActionCache from a path.
2020
Expected<std::unique_ptr<ActionCache>> createGRPCActionCache(StringRef Path);

‎llvm/lib/CAS/ObjectStore.cpp

Lines changed: 41 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -198,22 +198,60 @@ ObjectProxy::getMemoryBuffer(StringRef Name,
198198
return CAS->getMemoryBuffer(H, Name, RequiresNullTerminator);
199199
}
200200

201-
static Expected<std::unique_ptr<ObjectStore>>
201+
static Expected<std::shared_ptr<ObjectStore>>
202+
createOnDiskCASImpl(const Twine &Path) {
203+
return createOnDiskCAS(Path);
204+
}
205+
206+
static Expected<std::shared_ptr<ObjectStore>>
202207
createInMemoryCASImpl(const Twine &) {
203208
return createInMemoryCAS();
204209
}
205210

211+
static Expected<std::shared_ptr<ObjectStore>>
212+
createPluginCASImpl(const Twine &URL) {
213+
// Format used is
214+
// plugin://${PATH_TO_PLUGIN}?${OPT1}=${VAL1}&${OPT2}=${VAL2}..
215+
// "ondisk-path" as option is treated specially, the rest of options are
216+
// passed to the plugin verbatim.
217+
SmallString<256> PathBuf;
218+
auto [PluginPath, Options] = URL.toStringRef(PathBuf).split('?');
219+
std::string OnDiskPath;
220+
SmallVector<std::pair<std::string, std::string>> PluginArgs;
221+
while (!Options.empty()) {
222+
StringRef Opt;
223+
std::tie(Opt, Options) = Options.split('&');
224+
auto [Name, Value] = Opt.split('=');
225+
if (Name == "ondisk-path") {
226+
OnDiskPath = Value;
227+
} else {
228+
PluginArgs.push_back({std::string(Name), std::string(Value)});
229+
}
230+
}
231+
232+
if (OnDiskPath.empty())
233+
OnDiskPath = getDefaultOnDiskCASPath();
234+
235+
std::pair<std::shared_ptr<ObjectStore>, std::shared_ptr<ActionCache>> CASDBs;
236+
if (Error E = createPluginCASDatabases(PluginPath, OnDiskPath, PluginArgs)
237+
.moveInto(CASDBs))
238+
return std::move(E);
239+
240+
return std::move(CASDBs.first);
241+
}
242+
206243
static ManagedStatic<StringMap<ObjectStoreCreateFuncTy *>> RegisteredScheme;
207244

208245
static StringMap<ObjectStoreCreateFuncTy *> &getRegisteredScheme() {
209246
if (!RegisteredScheme.isConstructed()) {
210247
RegisteredScheme->insert({"mem://", &createInMemoryCASImpl});
211-
RegisteredScheme->insert({"file://", &createOnDiskCAS});
248+
RegisteredScheme->insert({"file://", &createOnDiskCASImpl});
249+
RegisteredScheme->insert({"plugin://", &createPluginCASImpl});
212250
}
213251
return *RegisteredScheme;
214252
}
215253

216-
Expected<std::unique_ptr<ObjectStore>>
254+
Expected<std::shared_ptr<ObjectStore>>
217255
cas::createCASFromIdentifier(StringRef Path) {
218256
for (auto &Scheme : getRegisteredScheme()) {
219257
if (Path.consume_front(Scheme.getKey()))

‎llvm/lib/RemoteCachingService/CAS/GRPCRelayCAS.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -426,10 +426,10 @@ Error GRPCActionCache::putImpl(ArrayRef<uint8_t> ResolvedKey,
426426
return Error::success();
427427
}
428428

429-
Expected<std::unique_ptr<ObjectStore>>
429+
Expected<std::shared_ptr<ObjectStore>>
430430
cas::createGRPCRelayCAS(const Twine &Path) {
431431
Error Err = Error::success();
432-
auto CAS = std::make_unique<GRPCRelayCAS>(Path.str(), Err);
432+
auto CAS = std::make_shared<GRPCRelayCAS>(Path.str(), Err);
433433
if (Err)
434434
return std::move(Err);
435435

‎llvm/lib/RemoteCachingService/NullService/NullService.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ static Error unsupportedError() {
3232
"RemoteCachingService (LLVM_CAS_ENABLE_REMOTE_CACHE) is disabled");
3333
}
3434

35-
Expected<std::unique_ptr<ObjectStore>>
35+
Expected<std::shared_ptr<ObjectStore>>
3636
cas::createGRPCRelayCAS(const Twine &Path) {
3737
return unsupportedError();
3838
}

‎llvm/test/CMakeLists.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ configure_lit_site_cfg(
5757
# NOTE: Sync the substitutions in test/lit.cfg when adding to this list.
5858
set(LLVM_TEST_DEPENDS
5959
BugpointPasses
60+
CASPluginTest
6061
FileCheck
6162
LLVMHello
6263
LLVMWindowsDriver

‎llvm/test/tools/llvm-cas/ingest.test

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@ RUN: mkdir %t
44
RUN: llvm-cas --cas %t/cas --ingest --data %S/Inputs > %t/cas.id
55
RUN: llvm-cas --cas %t/cas --ls-tree-recursive @%t/cas.id | FileCheck %s
66

7+
// Using the plugin.
8+
RUN: llvm-cas --cas plugin://%llvmshlibdir/libCASPluginTest%pluginext?ondisk-path=%t/cas-plugin --ingest --data %S/Inputs > %t/cas-plugin.id
9+
RUN: llvm-cas --cas plugin://%llvmshlibdir/libCASPluginTest%pluginext?ondisk-path=%t/cas-plugin --ls-tree-recursive @%t/cas-plugin.id | FileCheck %s
10+
711
CHECK: syml
812
CHECK-SAME: broken_symlink -> missing
913
CHECK: file

‎llvm/tools/llvm-cas/llvm-cas.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -132,7 +132,7 @@ int main(int Argc, char **Argv) {
132132
ExitOnErr(
133133
createStringError(inconvertibleErrorCode(), "missing --cas=<path>"));
134134

135-
std::unique_ptr<ObjectStore> CAS;
135+
std::shared_ptr<ObjectStore> CAS;
136136
std::unique_ptr<ActionCache> AC;
137137
std::optional<StringRef> CASFilePath;
138138
if (sys::path::is_absolute(CASPath)) {
@@ -143,7 +143,7 @@ int main(int Argc, char **Argv) {
143143
}
144144
assert(CAS);
145145

146-
std::unique_ptr<ObjectStore> UpstreamCAS;
146+
std::shared_ptr<ObjectStore> UpstreamCAS;
147147
if (!UpstreamCASPath.empty())
148148
UpstreamCAS = ExitOnErr(createCASFromIdentifier(UpstreamCASPath));
149149

‎llvm/unittests/CAS/ActionCacheTest.cpp

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ using namespace llvm;
1818
using namespace llvm::cas;
1919

2020
TEST_P(CASTest, ActionCacheHit) {
21-
std::unique_ptr<ObjectStore> CAS = createObjectStore();
21+
std::shared_ptr<ObjectStore> CAS = createObjectStore();
2222
std::unique_ptr<ActionCache> Cache = createActionCache();
2323

2424
Optional<ObjectProxy> ID;
@@ -33,7 +33,7 @@ TEST_P(CASTest, ActionCacheHit) {
3333
}
3434

3535
TEST_P(CASTest, ActionCacheMiss) {
36-
std::unique_ptr<ObjectStore> CAS = createObjectStore();
36+
std::shared_ptr<ObjectStore> CAS = createObjectStore();
3737
std::unique_ptr<ActionCache> Cache = createActionCache();
3838

3939
Optional<ObjectProxy> ID1, ID2;
@@ -56,7 +56,7 @@ TEST_P(CASTest, ActionCacheMiss) {
5656
}
5757

5858
TEST_P(CASTest, ActionCacheRewrite) {
59-
std::unique_ptr<ObjectStore> CAS = createObjectStore();
59+
std::shared_ptr<ObjectStore> CAS = createObjectStore();
6060
std::unique_ptr<ActionCache> Cache = createActionCache();
6161

6262
Optional<ObjectProxy> ID1, ID2;

‎llvm/unittests/CAS/CASTestConfig.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ std::unique_ptr<unittest::cas::MockEnv> createGRPCEnv(StringRef Socket,
6262
StringRef TempDir);
6363

6464
static TestingAndDir createGRPCCAS(int I) {
65-
std::unique_ptr<ObjectStore> CAS;
65+
std::shared_ptr<ObjectStore> CAS;
6666
unittest::TempDir Temp("daemon", /*Unique=*/true);
6767
SmallString<100> DaemonPath(Temp.path());
6868
sys::path::append(DaemonPath, "grpc");

0 commit comments

Comments
 (0)