diff --git a/.github/config/golang-dev.jsonc b/.github/config/golang-dev.jsonc new file mode 100644 index 0000000000..9f9e4d00eb --- /dev/null +++ b/.github/config/golang-dev.jsonc @@ -0,0 +1,57 @@ +/* + Copyright 2025 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +{ + "package-file": [ "go.mod" ], + "ci-setup-filename": "ci-setup.json", + + // If these change, please update the .github/config/README.md too! + "ci-setup-defaults": { + "env": { }, + "secrets": { }, + "go-version-earliest": 1.23, + "go-version-latest": 1.24, + "timeout-minutes": 10 + }, + + "ignore": [ + ".github/blunderbuss.yaml", + ".github/CODEOWNERS", + // TODO: do not ignore .github/config once everything is in prod + ".github/config/", // prevent changes to exclusions from running all tests + ".github/flakybot.yaml", + ".github/header-checker-lint.yaml", + ".github/ISSUE_TEMPLATE/", + ".github/PULL_REQUEST_TEMPLATE.md", + ".github/renovate.json", + ".github/scripts/", + ".github/snippet-bot.yml", + ".gitignore", + "badfiles_test.go", + "cloud-samples-tools", // checked out by GH action in ci-*.yml + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "Makefile", + "README.md", + "regiontag_test.go", + "SECURITY.md", + "Taskfile.yaml" + ], + + "exclude-packages": [ + ] +} diff --git a/.github/custard-setup/action.yaml b/.github/custard-setup/action.yaml new file mode 100644 index 0000000000..d5643b3490 --- /dev/null +++ b/.github/custard-setup/action.yaml @@ -0,0 +1,65 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Custard setup +description: Sets up the testing environment. + +inputs: + path: + description: Path of the package to run. + required: true + ci-setup: + description: The CI setup for the package path. + required: true + project_id: + description: The Google Cloud project ID. + required: true + workload_identity_provider: + description: The Google Cloud workload identity provider. + required: true + service_account: + description: The Google Cloud service account to use for credentials. + required: true + +runs: + using: composite + steps: + - uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2 + id: auth + with: + project_id: ${{ inputs.project_id }} + workload_identity_provider: ${{ inputs.workload_identity_provider }} + service_account: ${{ inputs.service_account }} + access_token_lifetime: 600s # 10 minutes + token_format: id_token + id_token_audience: https://action.test/ # service must have this custom audience + id_token_include_email: true + - name: Export environment variables + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 + id: vars + with: + script: | + const { default: setupVars } = await import('${{ github.workspace }}/.github/scripts/setup-vars.js'); + return await setupVars({ + core, + projectId: '${{ inputs.project_id }}', + setup: ${{ inputs.ci-setup }}, + serviceAccount: '${{ inputs.service_account }}', + idToken: '${{ steps.auth.outputs.id_token }}', + }) + - uses: google-github-actions/get-secretmanager-secrets@e5bb06c2ca53b244f978d33348d18317a7f263ce # v2 + if: ${{ fromJson(steps.vars.outputs.result).secrets }} + with: + secrets: ${{ fromJson(steps.vars.outputs.result).secrets }} + export_to_environment: true diff --git a/.github/scripts/cmd/vars.js b/.github/scripts/cmd/vars.js new file mode 100644 index 0000000000..28bdf2bbd4 --- /dev/null +++ b/.github/scripts/cmd/vars.js @@ -0,0 +1,41 @@ +/* + Copyright 2025 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +import fs from "node:fs"; +import path from "node:path"; +import setupVars from "../setup-vars.js"; + +const project_id = process.env.PROJECT_ID; +if (!project_id) { + console.error( + "Please set the PROJECT_ID environment variable to your Google Cloud project." + ); + process.exit(1); +} + +const core = { + exportVariable: (_key, _value) => null, +}; + +const setupFile = process.argv[2]; +if (!setupFile) { + console.error("Please provide the path to a setup file."); + process.exit(1); +} +const data = fs.readFileSync(path.join("..", "..", setupFile), "utf8"); +const setup = JSON.parse(data); + +setupVars({ project_id, core, setup }); diff --git a/.github/scripts/package.json b/.github/scripts/package.json new file mode 100644 index 0000000000..b90f45fda5 --- /dev/null +++ b/.github/scripts/package.json @@ -0,0 +1,14 @@ +{ + "name": "custard", + "version": "1.0.0", + "type": "module", + "license": "Apache-2.0", + "private": true, + "scripts": { + "vars": "node cmd/vars.js", + "test": "mocha -p -j 2 **/*.test.js" + }, + "devDependencies": { + "mocha": "^11.1.0" + } +} diff --git a/.github/scripts/setup-vars.js b/.github/scripts/setup-vars.js new file mode 100644 index 0000000000..faa06e0e98 --- /dev/null +++ b/.github/scripts/setup-vars.js @@ -0,0 +1,84 @@ +/* + Copyright 2025 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +export default function setupVars( + { projectId, core, setup, serviceAccount, idToken }, + runId = null +) { + // Define automatic variables plus custom variables. + const vars = { + PROJECT_ID: projectId, + RUN_ID: runId || uniqueId(), + SERVICE_ACCOUNT: serviceAccount, + ...(setup.env || {}), + }; + + // Apply variable interpolation. + const env = Object.fromEntries( + Object.keys(vars).map((key) => [key, substituteVars(vars[key], vars)]) + ); + + // Export environment variables. + console.log("env:"); + for (const key in env) { + const value = env[key]; + console.log(` ${key}: ${value}`); + core.exportVariable(key, value); + } + + // Show exported secrets, for logging purposes. + // TODO: We might want to fetch the secrets here and export them directly. + // https://cloud.google.com/secret-manager/docs/create-secret-quickstart#secretmanager-quickstart-nodejs + console.log("secrets:"); + for (const key in setup.secrets || {}) { + // This is the Google Cloud Secret Manager secret ID. + // NOT the secret value, so it's ok to show. + console.log(` ${key}: ${setup.secrets[key]}`); + } + + // Set global secret for the Service Account identity token + // Use in place of 'gcloud auth print-identity-token' or auth.getIdTokenClient + // usage: curl -H 'Bearer: $ID_TOKEN' https:// + core.exportVariable("ID_TOKEN", idToken); + core.setSecret(idToken); + // For logging, show the source of the ID_TOKEN + console.log(` ID_TOKEN: steps.auth.outputs.id_token (from GitHub Action)`); + + // Return env and secrets to use for further steps. + return { + env: env, + // Transform secrets into the format needed for the GHA secret manager step. + secrets: Object.keys(setup.secrets || {}) + .map((key) => `${key}:${setup.secrets[key]}`) + .join("\n"), + }; +} + +export function substituteVars(value, env) { + for (const key in env) { + let re = new RegExp(`\\$(${key}\\b|\\{\\s*${key}\\s*\\})`, "g"); + value = value.replaceAll(re, env[key]); + } + return value; +} + +export function uniqueId(length = 6) { + const min = 2 ** 32; + const max = 2 ** 64; + return Math.floor(Math.random() * max + min) + .toString(36) + .slice(0, length); +} diff --git a/.github/scripts/setup-vars.test.js b/.github/scripts/setup-vars.test.js new file mode 100644 index 0000000000..e3417d5005 --- /dev/null +++ b/.github/scripts/setup-vars.test.js @@ -0,0 +1,202 @@ +/* + Copyright 2025 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */ + +import { deepStrictEqual } from "assert"; +import setupVars from "./setup-vars.js"; +import { substituteVars, uniqueId } from "./setup-vars.js"; + +const projectId = "my-test-project"; +const serviceAccount = "my-sa@my-project.iam.gserviceaccount.com"; +const core = { + exportVariable: (_key, _value) => null, + setSecret: (_key) => null, +}; + +const autovars = { + PROJECT_ID: projectId, + RUN_ID: "run-id", + SERVICE_ACCOUNT: serviceAccount, +}; + +describe("setupVars", () => { + describe("env", () => { + it("empty", () => { + const setup = {}; + const vars = setupVars( + { projectId, core, setup, serviceAccount }, + "run-id" + ); + const expected = autovars; + deepStrictEqual(vars.env, expected); + }); + + it("zero vars", () => { + const setup = { env: {} }; + const vars = setupVars( + { projectId, core, setup, serviceAccount }, + "run-id" + ); + const expected = autovars; + deepStrictEqual(vars.env, expected); + }); + + it("one var", () => { + const setup = { env: { A: "x" } }; + const vars = setupVars( + { projectId, core, setup, serviceAccount }, + "run-id" + ); + const expected = { ...autovars, A: "x" }; + deepStrictEqual(vars.env, expected); + }); + + it("three vars", () => { + const setup = { env: { A: "x", B: "y", C: "z" } }; + const vars = setupVars( + { projectId, core, setup, serviceAccount }, + "run-id" + ); + const expected = { ...autovars, A: "x", B: "y", C: "z" }; + deepStrictEqual(vars.env, expected); + }); + + it("should override automatic variables", () => { + const setup = { + env: { PROJECT_ID: "custom-value", SERVICE_ACCOUNT: "baz@foo.com" }, + }; + const vars = setupVars( + { projectId, core, setup, serviceAccount }, + "run-id" + ); + const expected = { + PROJECT_ID: "custom-value", + RUN_ID: "run-id", + SERVICE_ACCOUNT: "baz@foo.com", + }; + deepStrictEqual(vars.env, expected); + }); + + it("should interpolate variables", () => { + const setup = { env: { A: "x", B: "y", C: "$A/${B}" } }; + const vars = setupVars( + { projectId, core, setup, serviceAccount }, + "run-id" + ); + const expected = { ...autovars, A: "x", B: "y", C: "x/y" }; + deepStrictEqual(vars.env, expected); + }); + + it("should not interpolate secrets", () => { + const setup = { + env: { C: "$x/$y" }, + secrets: { A: "x", B: "y" }, + }; + const vars = setupVars( + { projectId, core, setup, serviceAccount }, + "run-id" + ); + const expected = { ...autovars, C: "$x/$y" }; + deepStrictEqual(vars.env, expected); + }); + }); + + describe("secrets", () => { + it("zero secrets", () => { + const setup = { secrets: {} }; + const vars = setupVars( + { projectId, core, setup, serviceAccount }, + "run-id" + ); + deepStrictEqual(vars.secrets, ""); + }); + + it("one secret", () => { + const setup = { secrets: { A: "x" } }; + const vars = setupVars( + { projectId, core, setup, serviceAccount }, + "run-id" + ); + const expected = "A:x"; + deepStrictEqual(vars.secrets, expected); + }); + + it("three secrets", () => { + const setup = { secrets: { A: "x", B: "y", C: "z" } }; + const vars = setupVars( + { projectId, core, setup, serviceAccount }, + "run-id" + ); + const expected = "A:x\nB:y\nC:z"; + deepStrictEqual(vars.secrets, expected); + }); + + it("should not interpolate variables", () => { + const setup = { + env: { A: "x", B: "y" }, + secrets: { C: "$A/$B" }, + }; + const vars = setupVars( + { projectId, core, setup, serviceAccount }, + "run-id" + ); + const expected = "C:$A/$B"; + deepStrictEqual(vars.secrets, expected); + }); + + it("should not interpolate secrets", () => { + const setup = { secrets: { A: "x", B: "y", C: "$A/$B" } }; + const vars = setupVars( + { projectId, core, setup, serviceAccount }, + "run-id" + ); + const expected = "A:x\nB:y\nC:$A/$B"; + deepStrictEqual(vars.secrets, expected); + }); + }); +}); + +describe("substituteVars", () => { + it("should interpolate $VAR", () => { + const got = substituteVars("$A-$B", { A: "x", B: "y" }); + const expected = "x-y"; + deepStrictEqual(got, expected); + }); + + it("should interpolate ${VAR}", () => { + const got = substituteVars("${A}-${B}", { A: "x", B: "y" }); + const expected = "x-y"; + deepStrictEqual(got, expected); + }); + + it("should interpolate ${ VAR }", () => { + const got = substituteVars("${ A }-${ \tB\t }", { A: "x", B: "y" }); + const expected = "x-y"; + deepStrictEqual(got, expected); + }); + + it("should not interpolate on non-word boundary", () => { + const got = substituteVars("$Ab", { A: "x" }); + const expected = "$Ab"; + deepStrictEqual(got, expected); + }); +}); + +describe("uniqueId", () => { + it("should match length", () => { + const n = 6; + deepStrictEqual(uniqueId(n).length, n); + }); +}); diff --git a/.github/workflows/custard-affected.yaml b/.github/workflows/custard-affected.yaml new file mode 100644 index 0000000000..e1c898e806 --- /dev/null +++ b/.github/workflows/custard-affected.yaml @@ -0,0 +1,77 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +on: + workflow_call: + inputs: + config-file: + description: Path to the 🍮 custard config file. + required: true + type: string + go-version: + description: Go version to use to build the custard tools. + default: ^1.22.0 + type: string + timeout-minutes: + description: Timeout to find affected packages. + default: 2 + type: number + + outputs: + paths: + description: The affected paths as a JSON list. + value: ${{ jobs.affected.outputs.paths }} + ci-setups: + description: The CI setup configurations for the affected packages. + value: ${{ jobs.affected.outputs.ci-setups }} + +jobs: + affected: + name: Finding affected tests + runs-on: ubuntu-latest + timeout-minutes: ${{ inputs.timeout-minutes }} + outputs: + paths: ${{ steps.custard.outputs.paths }} + ci-setups: ${{ steps.custard.outputs.ci-setups }} + steps: + # Install the Custard tools. + - name: Setup Go + uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34 # v5 + with: + go-version: ${{ inputs.go-version }} + - name: Fetch Custard from cloud-samples-tools + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + repository: GoogleCloudPlatform/cloud-samples-tools + ref: v0.2.1 + path: cloud-samples-tools + - name: Install Custard + run: go install ./cmd/custard + working-directory: cloud-samples-tools/custard + - name: Clean up the workspace + run: rm -rf cloud-samples-tools + + # Find the affected packages. + - name: Checkout the commit history + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + fetch-depth: 0 # fetch the entire branch history to find diffs + - name: Get diffs from main and the PR + run: git --no-pager diff --name-only HEAD origin/main | tee diffs.txt + - name: Find affected packages + id: custard + run: | + echo "paths=$(custard affected ${{ inputs.config-file }} diffs.txt paths.txt)" >> $GITHUB_OUTPUT + cat paths.txt + echo "ci-setups=$(custard setup-files ${{ inputs.config-file }} paths.txt)" >> $GITHUB_OUTPUT diff --git a/.github/workflows/custard-ci-dev.yaml b/.github/workflows/custard-ci-dev.yaml new file mode 100644 index 0000000000..a8d6b722f9 --- /dev/null +++ b/.github/workflows/custard-ci-dev.yaml @@ -0,0 +1,60 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Custard CI (dev) +on: + push: + branches: + - main + pull_request: + +jobs: + affected: + uses: ./.github/workflows/custard-affected.yaml + with: + config-file: .github/config/golang-dev.jsonc + + test-latest: + name: Test latest + needs: affected + runs-on: ubuntu-latest + timeout-minutes: 120 # 2 hours hard limit + permissions: + id-token: write # needed for google-github-actions/auth + strategy: + fail-fast: false + matrix: + path: ${{ fromJson(needs.affected.outputs.paths) }} + steps: + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Setup Custard testing environment + id: custard + uses: ./.github/custard-setup + with: + path: ${{ matrix.path }} + ci-setup: ${{ toJson(fromJson(needs.affected.outputs.ci-setups)[matrix.path]) }} + project_id: golang-samples-tests + workload_identity_provider: TODO + service_account: kokoro-golang-samples-tests@golang-samples-tests.iam.gserviceaccount.com + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ fromJson(needs.affected.outputs.ci-setups)[matrix.path].go-version-latest }} + - name: Run tests for ${{ matrix.path }} + run: | + timeout ${{ fromJson(needs.affected.outputs.ci-setups)[matrix.path].timeout-minutes }}m \ + make test dir=${{ matrix.path }}