From 78c3b76f92d7eedf1647abf8dc285186ea15251f Mon Sep 17 00:00:00 2001 From: David Luna Date: Sun, 7 Dec 2025 23:12:49 +0100 Subject: [PATCH 1/7] chore: add a cache server for builds --- .github/workflows/test.yml | 44 ++++++++----- .gitignore | 1 + scripts/nx-cache-server.mjs | 127 ++++++++++++++++++++++++++++++++++++ 3 files changed, 157 insertions(+), 15 deletions(-) create mode 100644 scripts/nx-cache-server.mjs diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b02978b16f..2e122e9d6b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,22 +28,36 @@ jobs: - name: Install run: npm ci --ignore-scripts timeout-minutes: 10 - # Note: when pushing to main we want to test only what changed in - # last commit. Otherwise we want to test all changes from origin/main. - # So we set the right values for base and head commits depending - # on the type of event (PR or push) - # ref: https://nx.dev/ci/features/affected#configure-affected-on-ci - - name: Set base and head commits + + # Get the compilation cache. Decide later what to do + - name: Compile Cache Lookup + # if: ${{ github.event_name == 'pull_request' }} + uses: actions/cache/restore@v4 + with: + key: compile-cache-main + path: .nx-server-cache + + - name: Compile run: | - if [ "${{github.event_name}}" == "push" ]; then - echo "NX_BASE=origin/main~1" >> "$GITHUB_ENV" - echo "NX_HEAD=origin/main" >> "$GITHUB_ENV" - else - echo "NX_BASE=origin/main" >> "$GITHUB_ENV" - echo "NX_HEAD=HEAD" >> "$GITHUB_ENV" - fi - - name: Compile (Delta) - run: npm run compile:ci:affected + node ./scripts/nx-cache-server.mjs & + npm run compile + env: + NX_CACHE_SERVER_PATH: .nx-server-cache + NX_CACHE_SERVER_PORT: 3333 + NX_SELF_HOSTED_REMOTE_CACHE_SERVER: http://localhost:3333 + + # For main branch delete cache and save new compilation result + - name: Delete Cache (Push) + # if: ${{ github.event_name == 'push' }} + run: gh cache delete compile-cache-main + - name: Update Cache (Push) + # if: ${{ github.event_name == 'push' }} + uses: actions/cache/save@v4 + with: + key: compile-cache-main + path: .nx-server-cache + + # Uplad the file cache for the follwing tasks - name: Upload Build Artifacts uses: actions/upload-artifact@v4 with: diff --git a/.gitignore b/.gitignore index bd652c0ece..c78ac07959 100644 --- a/.gitignore +++ b/.gitignore @@ -70,6 +70,7 @@ docs #lerna .changelog .nx +.nx-server-cache # OS generated files .DS_Store diff --git a/scripts/nx-cache-server.mjs b/scripts/nx-cache-server.mjs new file mode 100644 index 0000000000..571b0d79dd --- /dev/null +++ b/scripts/nx-cache-server.mjs @@ -0,0 +1,127 @@ +#!/usr/bin/env node +/* + * Copyright The OpenTelemetry Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Starts a custom server to provide caching to `nx` in CI + * + * See: https://nx.dev/docs/guides/tasks--caching/self-hosted-caching#build-your-own-caching-server + * + * Usage: + * NX_CACHE_SERVER_PORT=3000 NX_CACHE_SERVER_PATH=my-cache node scripts/nx-cache-server.mjs + */ + +import { createReadStream, createWriteStream, existsSync, mkdirSync } from 'fs'; +import { createServer } from 'http'; + +const port = process.env.NX_CACHE_SERVER_PORT; +const cachePath = process.env.NX_CACHE_SERVER_PATH; + +const server = createServer((req, res) => { + const url = new URL(`http://localhost}${req.url}`); + const [version, resource, hash] = url.pathname.split('/').filter(s => s); + const verb = req.method.toUpperCase(); + + // Validation + // Auth token? + + // Path is correct and contains hash + if (version !== 'v1' || resource !== 'cache' || typeof hash !== 'string') { + res.statusCode = 404; + res.end('Not found'); + return; + } + + if (verb === 'PUT') { + return setHash(req, res, hash); + } else if (verb === 'GET') { + return getHash(req, res, hash); + } + + res.writeHead(400, {'Content-Type': 'text/plain'}); + res.end('Invalid method.'); +}); + +/** + * @param {http.IncomingMessage} req + * @param {http.ServerResponse} res + * @param {string} hash + */ +function setHash(req, res, hash) { + const contentType = req.headers['content-type']; + const bodyLength = Number(req.headers['content-length']); + const invalidLength = isNaN(bodyLength); + const invalidType = contentType !== 'application/octet-stream'; + if (invalidLength || invalidType) { + const errMsgs = []; + if (invalidLength) { + errMsgs.push('content-lenght must be a number') + } + if (invalidType) { + errMsgs.push('content-type must be application/octet-stream'); + } + res.writeHead(404, {'Content-Type': 'text/plain'}); + res.end(`Invalid params: ${errMsgs.join(', ')}.`); + return; + } + + // Ensure cache + if (!existsSync(cachePath)) { + mkdirSync(cachePath, { recursive: true }); + } + + // Do not override existing hashes?? -- yes for now + const entryPath = `${cachePath}/${hash}` + if (existsSync(entryPath)) { + res.statusCode = 409; + res.end('Cannot override an existing record.'); + return; + } + + // Pipe stream to a file + const writeStream = createWriteStream(entryPath); + req.pipe(writeStream); + req.on('end', () => { + req.statusCode = 200; + res.end(); + }); +} + +/** + * @param {http.IncomingMessage} req + * @param {http.ServerResponse} res + * @param {string} hash + */ +function getHash(req, res, hash) { + // no cache or no entry -> nothing to return + const entryPath = `${cachePath}/${hash}` + if (!existsSync(cachePath) || !existsSync(entryPath)) { + res.writeHead(404, {'Content-Type': 'text/plain'}); + res.end('Not found.'); + return; + } + + // Pipe stream to a file + const readStream = createReadStream(entryPath); + readStream.pipe(res); + readStream.on('error', (err) => { + res.writeHead(500, {'Content-Type': 'text/plain'}); + res.end(`Unkonw error::: ${err}`); + }) +} + +console.log(`server listening to port ${port}`); +server.listen(port); \ No newline at end of file From 1cae5686eff4979303ae4a07111c4c0874eed727 Mon Sep 17 00:00:00 2001 From: David Luna Date: Sun, 7 Dec 2025 23:25:58 +0100 Subject: [PATCH 2/7] fix: add GH token --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2e122e9d6b..c42c91ae56 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -50,6 +50,8 @@ jobs: - name: Delete Cache (Push) # if: ${{ github.event_name == 'push' }} run: gh cache delete compile-cache-main + env: + GH_TOKEN: ${{ github.token }} - name: Update Cache (Push) # if: ${{ github.event_name == 'push' }} uses: actions/cache/save@v4 From c06e92e07bba5ea4faf066c769ee442d24a57e9f Mon Sep 17 00:00:00 2001 From: David Luna Date: Sun, 7 Dec 2025 23:43:50 +0100 Subject: [PATCH 3/7] fix: update permissions --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c42c91ae56..8b70a5a2f2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,6 +7,7 @@ on: permissions: contents: read + actions: write # this permission is needed to delete cache jobs: compile: From 04caed66f8cb1babc16526f80c0efce555c15198 Mon Sep 17 00:00:00 2001 From: David Luna Date: Mon, 8 Dec 2025 00:00:47 +0100 Subject: [PATCH 4/7] chore: fix env var name --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8b70a5a2f2..b1582a5a05 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,7 +52,7 @@ jobs: # if: ${{ github.event_name == 'push' }} run: gh cache delete compile-cache-main env: - GH_TOKEN: ${{ github.token }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Update Cache (Push) # if: ${{ github.event_name == 'push' }} uses: actions/cache/save@v4 From d3808514f46f7b787dea49eeb8e993298a167bd7 Mon Sep 17 00:00:00 2001 From: David Luna Date: Mon, 8 Dec 2025 00:18:16 +0100 Subject: [PATCH 5/7] chore: test with a different command --- .github/workflows/test.yml | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b1582a5a05..fbb3b2d9f6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,10 +7,11 @@ on: permissions: contents: read - actions: write # this permission is needed to delete cache jobs: compile: + permissions: + actions: write # this permission is needed to delete cache strategy: fail-fast: false runs-on: ubuntu-latest @@ -50,9 +51,15 @@ jobs: # For main branch delete cache and save new compilation result - name: Delete Cache (Push) # if: ${{ github.event_name == 'push' }} - run: gh cache delete compile-cache-main + # run: gh cache delete compile-cache-main + run: | + gh api \ + --method DELETE \ + -H "Accept: application/vnd.github+json" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + /repos/open-telemetry/opentelemetry-js-contrib/actions/caches/compile-cache-main env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Update Cache (Push) # if: ${{ github.event_name == 'push' }} uses: actions/cache/save@v4 From 65e4f48883aa834041b24d65ae05218fa9d7cb1b Mon Sep 17 00:00:00 2001 From: David Luna Date: Mon, 8 Dec 2025 00:30:08 +0100 Subject: [PATCH 6/7] chore: try with write-all permission --- .github/workflows/test.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fbb3b2d9f6..7e1eaea8b7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,8 +10,9 @@ permissions: jobs: compile: - permissions: - actions: write # this permission is needed to delete cache + # permissions: + # actions: write # this permission is needed to delete cache + permissions: write-all strategy: fail-fast: false runs-on: ubuntu-latest From 10294dc1c48e395dd7ec7f8628eb7c30d682191b Mon Sep 17 00:00:00 2001 From: David Luna Date: Mon, 8 Dec 2025 01:20:39 +0100 Subject: [PATCH 7/7] chore: use env var for cache key --- .github/workflows/test.yml | 40 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7e1eaea8b7..c7f3733a4d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,9 +10,6 @@ permissions: jobs: compile: - # permissions: - # actions: write # this permission is needed to delete cache - permissions: write-all strategy: fail-fast: false runs-on: ubuntu-latest @@ -32,43 +29,26 @@ jobs: run: npm ci --ignore-scripts timeout-minutes: 10 - # Get the compilation cache. Decide later what to do + # Setting a global cache which key changes every day + - name: Set Cache Key + run: echo "CACHE_KEY=compile-cache-main-$(date +%Y-%m-%d)" >> "$GITHUB_ENV" - name: Compile Cache Lookup # if: ${{ github.event_name == 'pull_request' }} uses: actions/cache/restore@v4 with: - key: compile-cache-main - path: .nx-server-cache - + key: ${{ env.CACHE_KEY }} + path: .nx - name: Compile - run: | - node ./scripts/nx-cache-server.mjs & - npm run compile - env: - NX_CACHE_SERVER_PATH: .nx-server-cache - NX_CACHE_SERVER_PORT: 3333 - NX_SELF_HOSTED_REMOTE_CACHE_SERVER: http://localhost:3333 - - # For main branch delete cache and save new compilation result - - name: Delete Cache (Push) - # if: ${{ github.event_name == 'push' }} - # run: gh cache delete compile-cache-main - run: | - gh api \ - --method DELETE \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - /repos/open-telemetry/opentelemetry-js-contrib/actions/caches/compile-cache-main - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: npm run compile - name: Update Cache (Push) # if: ${{ github.event_name == 'push' }} uses: actions/cache/save@v4 with: - key: compile-cache-main - path: .nx-server-cache + key: ${{ env.CACHE_KEY }} + path: .nx - # Uplad the file cache for the follwing tasks + # Upload the compilation results as an artifact thta will be available + # only for this runa dn will be deleted in one day. - name: Upload Build Artifacts uses: actions/upload-artifact@v4 with: