Skip to content

Commit 1d816b6

Browse files
dmbrysondmbryson
authored
Merge pull request #977 from swiftlang/t/remact
[llbuild3] Implement remote action support
2 parents 9e90fa8 + 9c25382 commit 1d816b6

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

55 files changed

+5641
-3097
lines changed

‎llbuild.xcodeproj/project.pbxproj

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -371,6 +371,15 @@
371371
40D884B22D5591E800E3518E /* CASTree.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 40D884B12D5591E300E3518E /* CASTree.cpp */; };
372372
40D884B42D5DC52E00E3518E /* LocalExecutor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40D884B32D5DC52700E3518E /* LocalExecutor.swift */; };
373373
40D884B62D5EE62400E3518E /* CASTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40D884B52D5EE62100E3518E /* CASTests.swift */; };
374+
40D884DC2D78130F00E3518E /* RemoteExecutor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40D884DB2D78130B00E3518E /* RemoteExecutor.swift */; };
375+
40D884DE2D7B94B700E3518E /* ActionExecutor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40D884DD2D7B94B300E3518E /* ActionExecutor.swift */; };
376+
40D884F12D7BC51700E3518E /* time_zone_format.cc in Sources */ = {isa = PBXBuildFile; fileRef = 40C1F98A2C79076200757481 /* time_zone_format.cc */; };
377+
40D884F32D7BCBBB00E3518E /* Logging.h in Headers */ = {isa = PBXBuildFile; fileRef = 40D884F22D7BCBB200E3518E /* Logging.h */; };
378+
40D884F52D7BCBCE00E3518E /* Logging.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 40D884F42D7BCBC700E3518E /* Logging.cpp */; };
379+
40D884FB2D7BD4E100E3518E /* Common.pb.h in Headers */ = {isa = PBXBuildFile; fileRef = 40D884F82D7BD4E100E3518E /* Common.pb.h */; };
380+
40D884FC2D7BD4E100E3518E /* Common.pb.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40D884FA2D7BD4E100E3518E /* Common.pb.swift */; };
381+
40D884FD2D7BD4E100E3518E /* Common.pb.cc in Sources */ = {isa = PBXBuildFile; fileRef = 40D884F92D7BD4E100E3518E /* Common.pb.cc */; };
382+
40D884FF2D7BDA3B00E3518E /* Logging.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40D884FE2D7BDA3400E3518E /* Logging.swift */; };
374383
40E331942D4DD7240011ABFA /* Common.h in Headers */ = {isa = PBXBuildFile; fileRef = 40E331932D4DD7120011ABFA /* Common.h */; };
375384
40E7C53C2C61CD8100FC0383 /* Types.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40E7C53B2C61CD7D00FC0383 /* Types.swift */; };
376385
40E7C53E2C61CF6800FC0383 /* Engine.swift in Sources */ = {isa = PBXBuildFile; fileRef = 40E7C53D2C61CF6300FC0383 /* Engine.swift */; };
@@ -3820,6 +3829,15 @@
38203829
40D884B12D5591E300E3518E /* CASTree.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = CASTree.cpp; sourceTree = "<group>"; };
38213830
40D884B32D5DC52700E3518E /* LocalExecutor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LocalExecutor.swift; sourceTree = "<group>"; };
38223831
40D884B52D5EE62100E3518E /* CASTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = CASTests.swift; sourceTree = "<group>"; };
3832+
40D884DB2D78130B00E3518E /* RemoteExecutor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RemoteExecutor.swift; sourceTree = "<group>"; };
3833+
40D884DD2D7B94B300E3518E /* ActionExecutor.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ActionExecutor.swift; sourceTree = "<group>"; };
3834+
40D884F22D7BCBB200E3518E /* Logging.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Logging.h; sourceTree = "<group>"; };
3835+
40D884F42D7BCBC700E3518E /* Logging.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = Logging.cpp; sourceTree = "<group>"; };
3836+
40D884F62D7BD41600E3518E /* Common.proto */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.protobuf; path = Common.proto; sourceTree = "<group>"; };
3837+
40D884F82D7BD4E100E3518E /* Common.pb.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Common.pb.h; sourceTree = "<group>"; };
3838+
40D884F92D7BD4E100E3518E /* Common.pb.cc */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = Common.pb.cc; sourceTree = "<group>"; };
3839+
40D884FA2D7BD4E100E3518E /* Common.pb.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Common.pb.swift; sourceTree = "<group>"; };
3840+
40D884FE2D7BDA3400E3518E /* Logging.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Logging.swift; sourceTree = "<group>"; };
38233841
40E331932D4DD7120011ABFA /* Common.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = Common.h; sourceTree = "<group>"; };
38243842
40E7B4252C61CA1F00FC0383 /* ProtocolBuffers_iOS.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; path = ProtocolBuffers_iOS.xcodeproj; sourceTree = "<group>"; };
38253843
40E7B4262C61CA1F00FC0383 /* ProtocolBuffers_OSX.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; path = ProtocolBuffers_OSX.xcodeproj; sourceTree = "<group>"; };
@@ -5073,6 +5091,7 @@
50735091
4072B4BB2C51995100B68221 /* ActionCache.pb.swift */,
50745092
4035F6C62D2D86FC00337926 /* ActionExecutor.h */,
50755093
4035F6C82D2D870F00337926 /* ActionExecutor.cpp */,
5094+
40D884DD2D7B94B300E3518E /* ActionExecutor.swift */,
50765095
4072B4BC2C51995100B68221 /* Artifact.proto */,
50775096
4072B4BD2C51995100B68221 /* Artifact.pb.h */,
50785097
4072B4BE2C51995100B68221 /* Artifact.pb.cc */,
@@ -5093,6 +5112,10 @@
50935112
4035F6B12D28761000337926 /* CASTree.pb.cc */,
50945113
4035F6B22D28761000337926 /* CASTree.pb.swift */,
50955114
40E331932D4DD7120011ABFA /* Common.h */,
5115+
40D884F62D7BD41600E3518E /* Common.proto */,
5116+
40D884F82D7BD4E100E3518E /* Common.pb.h */,
5117+
40D884F92D7BD4E100E3518E /* Common.pb.cc */,
5118+
40D884FA2D7BD4E100E3518E /* Common.pb.swift */,
50965119
4072B4C42C51995100B68221 /* Engine.h */,
50975120
4072B4C52C51995100B68221 /* Engine.cpp */,
50985121
4072B4CC2C51995100B68221 /* Engine.proto */,
@@ -5117,8 +5140,12 @@
51175140
4035F6B92D28CC4C00337926 /* LocalExecutor.h */,
51185141
4035F6BB2D28CC5D00337926 /* LocalExecutor.cpp */,
51195142
40D884B32D5DC52700E3518E /* LocalExecutor.swift */,
5143+
40D884F22D7BCBB200E3518E /* Logging.h */,
5144+
40D884F42D7BCBC700E3518E /* Logging.cpp */,
5145+
40D884FE2D7BDA3400E3518E /* Logging.swift */,
51205146
4035F6BD2D28CC6E00337926 /* RemoteExecutor.h */,
51215147
4035F6BF2D28CC7C00337926 /* RemoteExecutor.cpp */,
5148+
40D884DB2D78130B00E3518E /* RemoteExecutor.swift */,
51225149
4035F6CC2D46FAD100337926 /* Subtask.h */,
51235150
4072B4D42C51995100B68221 /* support */,
51245151
4072B4D02C51995100B68221 /* SwiftAdaptors.cpp */,
@@ -9060,7 +9087,9 @@
90609087
4035F6C72D2D870600337926 /* ActionExecutor.h in Headers */,
90619088
4035F6CB2D2E36FF00337926 /* LabelTrie.h in Headers */,
90629089
4035F6E52D47F54700337926 /* Engine.pb.h in Headers */,
9090+
40D884FB2D7BD4E100E3518E /* Common.pb.h in Headers */,
90639091
40E331942D4DD7240011ABFA /* Common.h in Headers */,
9092+
40D884F32D7BCBBB00E3518E /* Logging.h in Headers */,
90649093
4035F6B32D28761000337926 /* Action.pb.h in Headers */,
90659094
4035F6BA2D28CC5400337926 /* LocalExecutor.h in Headers */,
90669095
4035F6B42D28761000337926 /* CASTree.pb.h in Headers */,
@@ -10782,6 +10811,7 @@
1078210811
40C10C3B2C790CC400757481 /* utf8.cc in Sources */,
1078310812
40C10C3C2C790CC400757481 /* ascii.cc in Sources */,
1078410813
40C10C3D2C790CC400757481 /* charconv.cc in Sources */,
10814+
40D884F12D7BC51700E3518E /* time_zone_format.cc in Sources */,
1078510815
40C10C3E2C790CC400757481 /* cord.cc in Sources */,
1078610816
40C10C3F2C790CC400757481 /* cord_analysis.cc in Sources */,
1078710817
40C10C402C790CC400757481 /* cord_buffer.cc in Sources */,
@@ -10865,15 +10895,21 @@
1086510895
4072B5F52C51BF1700B68221 /* ActionCache.cpp in Sources */,
1086610896
4035F6C92D2D871500337926 /* ActionExecutor.cpp in Sources */,
1086710897
40E837EA2CCC64430022B026 /* CAS.cpp in Sources */,
10898+
40D884FC2D7BD4E100E3518E /* Common.pb.swift in Sources */,
10899+
40D884FD2D7BD4E100E3518E /* Common.pb.cc in Sources */,
1086810900
4035F64B2CF140B000337926 /* CAS.swift in Sources */,
1086910901
4072B5F62C51BF1700B68221 /* Error.pb.swift in Sources */,
1087010902
4072B5F72C51BF1700B68221 /* ActionCache.pb.swift in Sources */,
1087110903
4072B5F82C51BF1700B68221 /* Label.cpp in Sources */,
1087210904
4035F6B52D28761000337926 /* CASTree.pb.swift in Sources */,
1087310905
4035F6B62D28761000337926 /* CASTree.pb.cc in Sources */,
10906+
40D884DC2D78130F00E3518E /* RemoteExecutor.swift in Sources */,
10907+
40D884FF2D7BDA3B00E3518E /* Logging.swift in Sources */,
10908+
40D884F52D7BCBCE00E3518E /* Logging.cpp in Sources */,
1087410909
4035F6B72D28761000337926 /* Action.pb.swift in Sources */,
1087510910
4035F6B82D28761000337926 /* Action.pb.cc in Sources */,
1087610911
4035F6BC2D28CC6600337926 /* LocalExecutor.cpp in Sources */,
10912+
40D884DE2D7B94B700E3518E /* ActionExecutor.swift in Sources */,
1087710913
40D884B42D5DC52E00E3518E /* LocalExecutor.swift in Sources */,
1087810914
4072B5FC2C51BF1700B68221 /* SwiftAdaptors.cpp in Sources */,
1087910915
);

‎products/llbuild3/Errors.hpp

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,16 +64,20 @@ enum class ExecutorError: uint64_t {
6464
WaitFailed = 103,
6565
ControlProtocolError = 104,
6666
ProcessStatsError = 105,
67+
ProcessFailed = 106,
68+
UnexpectedOutput = 107,
6769

6870
// 200 - Client Implementation Errors
6971
BadRequest = 200,
7072
DuplicateProvider = 201,
7173
BadProviderPrefix = 202,
7274
NoProvider = 203,
75+
NoRemoteExecutor = 204,
7376

7477
// 1000 - Executor Internal Errors
7578
Unimplemented = 1000,
7679
InternalInconsistency = 1001,
80+
InternalProtobufSerialization = 1002,
7781

7882
Unknown = 0
7983
};

‎products/llbuild3/SwiftAdaptors.hpp

Lines changed: 72 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,20 @@ struct EngineConfig;
3333

3434
// Serialized Protobuf Objects
3535
typedef std::string ActionPB;
36+
typedef std::string ActionResultPB;
3637
typedef std::string ArtifactPB;
3738
typedef std::string CacheKeyPB;
3839
typedef std::string CacheValuePB;
3940
typedef std::string CASIDBytes;
4041
typedef std::string CASObjectPB;
42+
typedef std::string EngineIDBytes;
4143
typedef std::string ErrorPB;
4244
typedef std::string FileObjectPB;
4345
typedef std::string LabelPB;
46+
typedef std::string PlatformPB;
47+
typedef std::string RemoteActionIDBytes;
4448
typedef std::string SignaturePB;
49+
typedef std::string StatPB;
4550
typedef std::string TaskContextPB;
4651
typedef std::string TaskInputsPB;
4752
typedef std::string TaskNextStatePB;
@@ -53,6 +58,30 @@ typedef std::vector<std::pair<uint64_t, result<void*, ErrorPB>>> SubtaskResultMa
5358

5459
// External Adaptor Objects
5560

61+
class ClientContext;
62+
class Logger;
63+
typedef std::shared_ptr<ClientContext> ClientContextRef;
64+
typedef std::shared_ptr<Logger> LoggerRef;
65+
66+
LLBUILD3_EXPORT ClientContextRef makeExtClientContext(void* ctx,
67+
void (*)(void* ctx));
68+
69+
struct ExtLoggerContext {
70+
void* ctx = nullptr;
71+
EngineIDBytes engineID;
72+
};
73+
74+
struct ExtLogger {
75+
void* ctx;
76+
77+
void (*releaseFn)(void* ctx);
78+
void (*errorFn)(void* ctx, ExtLoggerContext, ErrorPB);
79+
void (*eventFn)(void* ctx, ExtLoggerContext, std::vector<StatPB>*);
80+
};
81+
82+
LLBUILD3_EXPORT LoggerRef makeExtLogger(ExtLogger);
83+
84+
5685
class CASDatabase;
5786
typedef std::shared_ptr<CASDatabase> CASDatabaseRef;
5887

@@ -176,17 +205,38 @@ LLBUILD3_EXPORT ActionCacheRef makeExtActionCache(ExtActionCache extCache);
176205
LLBUILD3_EXPORT ActionCacheRef makeInMemoryActionCache();
177206

178207

208+
struct ExtActionDescriptor {
209+
LabelPB name;
210+
PlatformPB platform;
211+
std::string executable;
212+
};
213+
214+
struct ExtActionProvider {
215+
void* ctx;
216+
217+
void (*releaseFn)(void* ctx);
218+
219+
void (*prefixesFn)(void*, std::vector<LabelPB>*);
220+
LabelPB (*resolveFn)(void*, LabelPB, ErrorPB*);
221+
ExtActionDescriptor (*descriptorFn)(void*, LabelPB, ErrorPB*);
222+
};
223+
179224
struct ExtLocalSandbox {
180225
void* ctx;
181226

182227
void (*releaseFn)(void* ctx);
183228

184229
void (*dirFn)(void* ctx, std::string*);
230+
void (*envFn)(void* ctx, std::vector<std::pair<std::string, std::string>>*);
185231
void (*prepareInputFn)(void* ctx, std::string* path, int type, CASIDBytes* id, ErrorPB*);
186232
void (*collectOutputsFn)(void*, std::vector<std::string>, std::vector<FileObjectPB>*, ErrorPB*);
187233
void (*releaseSandboxFn)(void* ctx);
188234
};
189235

236+
inline std::pair<std::string, std::string> makeStringPair(std::string l, std::string r) {
237+
return std::make_pair(l, r);
238+
}
239+
190240
struct ExtLocalSandboxProvider {
191241
void* ctx;
192242

@@ -195,18 +245,36 @@ struct ExtLocalSandboxProvider {
195245
ExtLocalSandbox (*createFn)(void* ctx, uint64_t handle, ErrorPB*);
196246
};
197247

248+
struct ExtRemoteExecutor {
249+
void* ctx;
250+
251+
void (*releaseFn)(void* ctx);
252+
253+
std::string builtinExecutable;
254+
255+
void (*prepareFn)(void* ctx, std::string path,
256+
std::function<void(CASIDBytes, ErrorPB)>);
257+
void (*executeFn)(void* ctx, CASIDBytes id, ActionPB,
258+
std::function<void(RemoteActionIDBytes, ErrorPB)>,
259+
std::function<void(ActionResultPB, ErrorPB)>);
260+
};
261+
198262
class ActionExecutor;
263+
class ActionProvider;
199264
class LocalExecutor;
200265
class LocalSandboxProvider;
201266
class RemoteExecutor;
202267
typedef std::shared_ptr<ActionExecutor> ActionExecutorRef;
268+
typedef std::shared_ptr<ActionProvider> ActionProviderRef;
203269
typedef std::shared_ptr<LocalExecutor> LocalExecutorRef;
204270
typedef std::shared_ptr<LocalSandboxProvider> LocalSandboxProviderRef;
205271
typedef std::shared_ptr<RemoteExecutor> RemoteExecutorRef;
206-
LLBUILD3_EXPORT ActionExecutorRef makeActionExecutor(CASDatabaseRef, ActionCacheRef, LocalExecutorRef, RemoteExecutorRef);
272+
LLBUILD3_EXPORT ActionExecutorRef makeActionExecutor(CASDatabaseRef, ActionCacheRef, LocalExecutorRef, RemoteExecutorRef, LoggerRef);
273+
LLBUILD3_EXPORT ActionProviderRef makeExtActionProvider(ExtActionProvider);
274+
LLBUILD3_EXPORT ErrorPB registerProviderWithExecutor(ActionExecutorRef, ActionProviderRef);
207275
LLBUILD3_EXPORT LocalSandboxProviderRef makeExtLocalSandboxProvider(ExtLocalSandboxProvider);
208276
LLBUILD3_EXPORT LocalExecutorRef makeLocalExecutor(LocalSandboxProviderRef);
209-
LLBUILD3_EXPORT RemoteExecutorRef makeRemoteExecutor();
277+
LLBUILD3_EXPORT RemoteExecutorRef makeRemoteExecutor(ExtRemoteExecutor);
210278

211279
struct ExtEngineConfig {
212280
std::optional<LabelPB> initRule;
@@ -228,6 +296,8 @@ LLBUILD3_EXPORT EngineRef makeEngine(ExtEngineConfig config,
228296
CASDatabaseRef casdb,
229297
ActionCacheRef cache,
230298
ActionExecutorRef executor,
299+
LoggerRef logger,
300+
ClientContextRef clientContext,
231301
const ExtRuleProvider provider);
232302

233303
}

0 commit comments

Comments
 (0)