Skip to content

Commit d319fda

Browse files
palfreypalfrey
authored
Initial Remote Asset support (#1646)
1 parent 0ff64b1 commit d319fda

File tree

30 files changed

+782
-8
lines changed

30 files changed

+782
-8
lines changed

‎.github/workflows/nix.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,3 +91,19 @@ jobs:
9191
- name: Test nix run
9292
run: |
9393
nix run -L .#nativelink-is-executable-test
94+
95+
buildstream:
96+
name: buildstream
97+
runs-on: ubuntu-24.04
98+
timeout-minutes: 45
99+
steps:
100+
- name: Checkout
101+
uses: >- # v4.2.2
102+
actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
103+
104+
- name: Prepare Worker
105+
uses: ./.github/actions/prepare-nix
106+
107+
- name: Test buildstream run
108+
run: |
109+
nix run -L .#buildstream-with-nativelink-test

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,3 +23,4 @@ nixos.bazelrc
2323
rust-project.json
2424
darwin.bazelrc
2525
nativelink.bazelrc
26+
integration_tests/buildstream/buildstream.log

‎deployment-examples/docker-compose/local-storage-cas.json5‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,9 @@
5656
"cas_stores": {
5757
"": "CAS_MAIN_STORE"
5858
}
59-
}
59+
},
60+
"fetch": {},
61+
"push": {},
6062
}
6163
}, {
6264
"listener": {

‎flake.nix‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,14 @@
342342
nativelink-worker-toolchain-buck2 = createWorker toolchain-buck2;
343343
nativelink-worker-buck2-toolchain = buck2-toolchain;
344344
image = nativelink-image;
345+
pyroaring = pkgs.callPackage ./tools/buildstream/pyroaring.nix {pythonPkgs = pkgs.python312Packages;};
346+
bst = pkgs.callPackage ./tools/buildstream/bst.nix {
347+
inherit pkgs pyroaring;
348+
pythonPkgs = pkgs.python312Packages;
349+
};
350+
buildstream-with-nativelink-test = pkgs.callPackage integration_tests/buildstream/buildstream-with-nativelink-test.nix {
351+
inherit nativelink bst;
352+
};
345353
}
346354
// (
347355
# It's not possible to crosscompile to darwin, not even between

‎integration_tests/buildstream/buildstream-with-nativelink-test.nix‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
nativelink,
3+
bst,
4+
writeShellScriptBin,
5+
}:
6+
writeShellScriptBin "buildstream-with-nativelink-test" ''
7+
set -xuo pipefail
8+
9+
cleanup() {
10+
local pids=$(jobs -pr)
11+
[ -n "$pids" ] && kill $pids
12+
}
13+
trap "cleanup" INT QUIT TERM EXIT
14+
15+
${nativelink}/bin/nativelink -- integration_tests/buildstream/buildstream_cas.json5 &
16+
17+
bst_output="$(cd integration_tests/buildstream && ${bst}/bin/bst -c buildstream.conf build hello.bst 2>&1 | tee -i buildstream.log)"
18+
19+
case $bst_output in
20+
*"SUCCESS Build"* )
21+
echo "Saw a successful buildstream build"
22+
;;
23+
*)
24+
echo 'Failed buildstream build:'
25+
echo $bst_output
26+
exit 1
27+
;;
28+
esac
29+
''

‎integration_tests/buildstream/buildstream.conf‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
artifacts:
2+
servers:
3+
- url: http://localhost:50051
4+
push: true
5+
remote-execution:
6+
execution-service:
7+
url: http://localhost:50051
8+
action-cache-service:
9+
url: http://localhost:50051
10+
storage-service:
11+
url: http://localhost:50051

‎integration_tests/buildstream/buildstream_cas.json5‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
{
2+
"stores": [
3+
{
4+
"name": "AC_MAIN_STORE",
5+
"filesystem": {
6+
"content_path": "/tmp/nativelink/data-worker-test/content_path-ac",
7+
"temp_path": "/tmp/nativelink/data-worker-test/tmp_path-ac",
8+
"eviction_policy": {
9+
// 1gb.
10+
"max_bytes": 1000000000
11+
}
12+
}
13+
}, {
14+
"name": "WORKER_FAST_SLOW_STORE",
15+
"fast_slow": {
16+
// "fast" must be a "filesystem" store because the worker uses it to make
17+
// hardlinks on disk to a directory where the jobs are running.
18+
"fast": {
19+
"filesystem": {
20+
"content_path": "/tmp/nativelink/data-worker-test/content_path-cas",
21+
"temp_path": "/tmp/nativelink/data-worker-test/tmp_path-cas",
22+
"eviction_policy": {
23+
// 10gb.
24+
"max_bytes": 10000000000
25+
}
26+
}
27+
},
28+
"slow": {
29+
/// Discard data.
30+
/// This example usage has the CAS and the Worker live in the same place,
31+
/// so they share the same underlying CAS. Since workers require a fast_slow
32+
/// store, we use the fast store as our primary data store, and the slow store
33+
/// is just a noop, since there's no shared storage in this config.
34+
"noop": {}
35+
}
36+
}
37+
}
38+
],
39+
"schedulers": [
40+
{
41+
"name": "MAIN_SCHEDULER",
42+
"simple": {
43+
"supported_platform_properties": {
44+
"cpu_count": "minimum",
45+
"memory_kb": "minimum",
46+
"network_kbps": "minimum",
47+
"disk_read_iops": "minimum",
48+
"disk_read_bps": "minimum",
49+
"disk_write_iops": "minimum",
50+
"disk_write_bps": "minimum",
51+
"shm_size": "minimum",
52+
"gpu_count": "minimum",
53+
"gpu_model": "exact",
54+
"cpu_vendor": "exact",
55+
"cpu_arch": "exact",
56+
"cpu_model": "exact",
57+
"kernel_version": "exact",
58+
"OSFamily": "priority",
59+
"container-image": "priority",
60+
"lre-rs": "priority",
61+
"ISA": "exact",
62+
}
63+
}
64+
}
65+
],
66+
"workers": [{
67+
"local": {
68+
"worker_api_endpoint": {
69+
"uri": "grpc://127.0.0.1:50061"
70+
},
71+
"cas_fast_slow_store": "WORKER_FAST_SLOW_STORE",
72+
"upload_action_result": {
73+
"ac_store": "AC_MAIN_STORE"
74+
},
75+
"work_directory": "/tmp/nativelink/work",
76+
"platform_properties": {
77+
"cpu_count": {
78+
"values": ["16"]
79+
},
80+
"memory_kb": {
81+
"values": ["500000"]
82+
},
83+
"network_kbps": {
84+
"values": ["100000"]
85+
},
86+
"cpu_arch": {
87+
"values": ["x86_64"]
88+
},
89+
"OSFamily": {
90+
"values": [""]
91+
},
92+
"container-image": {
93+
"values": [""]
94+
},
95+
"lre-rs": {
96+
"values": [""]
97+
},
98+
"ISA": {
99+
"values": ["x86-64"]
100+
},
101+
}
102+
}
103+
}],
104+
"servers": [{
105+
"name": "public",
106+
"listener": {
107+
"http": {
108+
"socket_address": "0.0.0.0:50051"
109+
}
110+
},
111+
"services": {
112+
"cas": {
113+
"": {
114+
"cas_store": "WORKER_FAST_SLOW_STORE"
115+
}
116+
},
117+
"ac": {
118+
"": {
119+
"ac_store": "AC_MAIN_STORE"
120+
}
121+
},
122+
"execution": {
123+
"": {
124+
"cas_store": "WORKER_FAST_SLOW_STORE",
125+
"scheduler": "MAIN_SCHEDULER"
126+
}
127+
},
128+
"fetch": {
129+
},
130+
"push": {
131+
},
132+
"capabilities": {
133+
"": {
134+
"remote_execution": {
135+
"scheduler": "MAIN_SCHEDULER"
136+
}
137+
}
138+
},
139+
"bytestream": {
140+
"cas_stores": {
141+
"": "WORKER_FAST_SLOW_STORE"
142+
}
143+
}
144+
}
145+
}, {
146+
"name": "private_workers_servers",
147+
"listener": {
148+
"http": {
149+
"socket_address": "0.0.0.0:50061"
150+
}
151+
},
152+
"services": {
153+
// Note: This should be served on a different port, because it has
154+
// a different permission set than the other services.
155+
// In other words, this service is a backend api. The ones above
156+
// are a frontend api.
157+
"worker_api": {
158+
"scheduler": "MAIN_SCHEDULER"
159+
},
160+
"admin": {},
161+
"health": {}
162+
}
163+
}],
164+
"global": {
165+
"max_open_files": 24576
166+
}
167+
}

‎integration_tests/buildstream/elements/base.bst‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
kind: stack
2+
description: Base stack
3+
4+
depends:
5+
- base/alpine.bst

‎integration_tests/buildstream/elements/base/alpine.bst‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
kind: import
2+
description: |
3+
4+
Alpine Linux base runtime
5+
6+
sources:
7+
- kind: tar
8+
9+
# This is a post doctored, trimmed down system image
10+
# of the Alpine linux distribution.
11+
#
12+
url: alpine:integration-tests-base.v1.x86_64.tar.xz
13+
ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639

‎integration_tests/buildstream/elements/hello.bst‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
kind: manual
2+
description: |
3+
4+
Building manually
5+
6+
# Depend on the base system
7+
depends:
8+
- base.bst
9+
10+
# Stage the files/src directory for building
11+
sources:
12+
- kind: local
13+
path: files/src
14+
15+
# Now configure the commands to run
16+
config:
17+
18+
build-commands:
19+
- make hello

0 commit comments

Comments
 (0)