Set up CI to run buck2 samples (#357)

nlopezgi · web-flow · commit d6f3530baec7 · 2024-12-04T12:30:10.000-05:00
- Creating PR to run Buck2 samples in CI - Uses github auth to authenticate buck2 client with EngFlow cluster - https://docs.engflow.com/re/config/options.html#experimental_github_auth_container - Uses latest version of Buck2, if we start having issues with this approach will migrate to use of dotslash for configs. - Runs the cpp, golang and python Buck2 samples, as more are added they need to be include in the test script - DONE: use pre-installed zstd once I release images with the package
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -102,3 +102,29 @@ jobs:
           OS: linux
           REMOTE_EXECUTION: ${{ matrix.remote_execution }}
         run: python3 infra/test-all.py
+
+  buck2-test:
+    runs-on:
+      - self-hosted
+      - "os=linux"
+      - "arch=x64"
+      - "engflow-container-image=docker://645088952840.dkr.ecr.eu-west-1.amazonaws.com/engflow-ci/debian11-dind-x64@sha256:a0f386162b6498b9220a33f33f158fd175d2873ddd6ed30ca67aa33915850e1e"
+      - "engflow-pool=ci_sysbox_x64"
+      - "engflow-runtime=sysbox-runc"
+      - "engflow-runner-id=${{ github.repository_id }}_buck2-test_${{ github.run_id }}_${{ github.run_number }}_${{ github.run_attempt }}"
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Buck2 binary
+        shell: bash
+        run: infra/setup-buck2.sh
+
+      - name: Set up authentication
+        shell: bash
+        run: infra/auth-buck2.sh
+
+      - name: Build and test
+        shell: bash
+        run: infra/test-buck2.sh
diff --git a/buck2/cpp/.buckconfig b/buck2/cpp/.buckconfig
@@ -41,7 +41,7 @@ digest_algorithms = SHA256
 engine_address       = <CLUSTER_NAME>.cluster.engflow.com
 action_cache_address = <CLUSTER_NAME>.cluster.engflow.com
 cas_address          = <CLUSTER_NAME>.cluster.engflow.com
-http_headers         = x-engflow-auth-method:jwt-v0,x-engflow-auth-token:LONG_JWT_STRING
+http_headers         = <AUTH_HTTP_HEADERS>
 
 [build]
   execution_platforms = root//platforms:remote_platform
diff --git a/buck2/cpp/README.md b/buck2/cpp/README.md
@@ -35,10 +35,12 @@ following:
 engine_address       = <CLUSTER_NAME>.cluster.engflow.com
 action_cache_address = <CLUSTER_NAME>.cluster.engflow.com
 cas_address          = <CLUSTER_NAME>.cluster.engflow.com
-http_headers         = x-engflow-auth-method:jwt-v0,x-engflow-auth-token:LONG_JWT_STRING
+http_headers         = <AUTH_HTTP_HEADERS>
  ```
 
- To obtain the value of `LONG_JWT_STRING`, log into https://<CLUSTER_NAME>.cluster.engflow.com/gettingstarted and use the value of `x-engflow-auth-token` in section `Method 2: JWT`.
+ To obtain the value of `<AUTH_HTTP_HEADERS>`, log into https://<CLUSTER_NAME>.cluster.engflow.com/gettingstarted and obtain the value of `x-engflow-auth-token` in section `Method 2: JWT`, take note of this value. Then set `AUTH_HTTP_HEADERS` with the value `x-engflow-auth-method:jwt-v0,x-engflow-auth-token:<JWT_TOKEN_FROM_GETTINGSTARTED_PAGE>.
+
+ Note for CI runs, the auth method used is [Github Tokens](https://docs.engflow.com/re/config/authentication.html#github-tokens).
 
 ### Usage instructions
 
diff --git a/buck2/golang/.buckconfig b/buck2/golang/.buckconfig
@@ -23,7 +23,7 @@ digest_algorithms = SHA256
 engine_address       = <CLUSTER_NAME>.cluster.engflow.com
 action_cache_address = <CLUSTER_NAME>.cluster.engflow.com
 cas_address          = <CLUSTER_NAME>.cluster.engflow.com
-http_headers         = x-engflow-auth-method:jwt-v0,x-engflow-auth-token:LONG_JWT_STRING
+http_headers         = <AUTH_HTTP_HEADERS>
 
 [build]
   execution_platforms = root//platforms:remote_platform
diff --git a/buck2/golang/README.md b/buck2/golang/README.md
@@ -46,7 +46,9 @@ following:
 engine_address       = <CLUSTER_NAME>.cluster.engflow.com
 action_cache_address = <CLUSTER_NAME>.cluster.engflow.com
 cas_address          = <CLUSTER_NAME>.cluster.engflow.com
-http_headers         = x-engflow-auth-method:jwt-v0,x-engflow-auth-token:LONG_JWT_STRING
+http_headers         = <AUTH_HTTP_HEADERS>
  ```
 
-To obtain the value of `LONG_JWT_STRING`, log into https://<CLUSTER_NAME>.cluster.engflow.com/gettingstarted and use the value of `x-engflow-auth-token` in section `Method 2: JWT`.
+To obtain the value of `<AUTH_HTTP_HEADERS>`, log into https://<CLUSTER_NAME>.cluster.engflow.com/gettingstarted and obtain the value of `x-engflow-auth-token` in section `Method 2: JWT`, take note of this value. Then set `AUTH_HTTP_HEADERS` with the value `x-engflow-auth-method:jwt-v0,x-engflow-auth-token:<JWT_TOKEN_FROM_GETTINGSTARTED_PAGE>.
+
+Note for CI runs, the auth method used is [Github Tokens](https://docs.engflow.com/re/config/authentication.html#github-tokens).
diff --git a/buck2/python/.buckconfig b/buck2/python/.buckconfig
@@ -23,7 +23,7 @@ digest_algorithms = SHA256
 engine_address       = <CLUSTER_NAME>.cluster.engflow.com
 action_cache_address = <CLUSTER_NAME>.cluster.engflow.com
 cas_address          = <CLUSTER_NAME>.cluster.engflow.com
-http_headers         = x-engflow-auth-method:jwt-v0,x-engflow-auth-token:LONG_JWT_STRING
+http_headers         = <AUTH_HTTP_HEADERS>
 
 [build]
   execution_platforms = root//platforms:remote_platform
diff --git a/buck2/python/README.md b/buck2/python/README.md
@@ -49,7 +49,9 @@ following:
 engine_address       = <CLUSTER_NAME>.cluster.engflow.com
 action_cache_address = <CLUSTER_NAME>.cluster.engflow.com
 cas_address          = <CLUSTER_NAME>.cluster.engflow.com
-http_headers         = x-engflow-auth-method:jwt-v0,x-engflow-auth-token:LONG_JWT_STRING
+http_headers         = <AUTH_HTTP_HEADERS>
  ```
 
-To obtain the value of `LONG_JWT_STRING`, log into https://<CLUSTER_NAME>.cluster.engflow.com/gettingstarted and use the value of `x-engflow-auth-token` in section `Method 2: JWT`.
+To obtain the value of `<AUTH_HTTP_HEADERS>`, log into https://<CLUSTER_NAME>.cluster.engflow.com/gettingstarted and obtain the value of `x-engflow-auth-token` in section `Method 2: JWT`, take note of this value. Then set `AUTH_HTTP_HEADERS` with the value `x-engflow-auth-method:jwt-v0,x-engflow-auth-token:<JWT_TOKEN_FROM_GETTINGSTARTED_PAGE>.
+
+Note for CI runs, the auth method used is [Github Tokens](https://docs.engflow.com/re/config/authentication.html#github-tokens).
diff --git a/infra/auth-buck2.sh b/infra/auth-buck2.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -xe
+
+# Modify the buck2/**/.buckconfig files to set the test cluster to opal.
+find ./ -type f -name '.buckconfig' -exec sed -i 's/<CLUSTER_NAME>/opal/g' {} \;
+
+# Modify the buck2/**/.buckconfig files to set the "Authorization=Bearer $GITHUB_TOKEN" in the http_headers.
+find ./ -type f -name '.buckconfig' -exec sed -i 's/<AUTH_HTTP_HEADERS>/Authorization:Bearer $GITHUB_TOKEN/g' {} \;
+
diff --git a/infra/setup-buck2.sh b/infra/setup-buck2.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -xe
+
+# Get the Buck2 binary.
+curl -L -O https://github.com/facebook/buck2/releases/download/latest/buck2-x86_64-unknown-linux-musl.zst
+
+# Unpack the binary.
+unzstd buck2-x86_64-unknown-linux-musl.zst
+
+# Change its name and make it executable.
+mv buck2-x86_64-unknown-linux-musl buck2-exe
+chmod +x buck2-exe
diff --git a/infra/test-buck2.sh b/infra/test-buck2.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+set -xe
+shopt -s expand_aliases
+
+alias buck2="$(realpath buck2-exe)"
+
+# Run cpp example
+cd buck2/cpp
+buck2 build //:cpp_lib
+buck2 test //:cpp_test
+cd ..
+
+# Run python example
+cd python
+buck2 build //main:check_main
+buck2 test //hello:hello_unittest_test
+cd ..
+
+# Run go example
+cd golang
+buck2 build //go:hello
+buck2 test //go/greeting:greeting_test
+cd ..
