From b90797d31fa7217d5a8d87488f0e6f590ae70dbf Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 25 Feb 2025 12:55:18 +1100 Subject: [PATCH 01/44] ci(idp-sql): run idp-sql tests in testing isolation --- .github/config/nodejs-prod.jsonc | 3 +-- run/idp-sql/app.js | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/config/nodejs-prod.jsonc b/.github/config/nodejs-prod.jsonc index 5f908446a0..9e0bc57522 100644 --- a/.github/config/nodejs-prod.jsonc +++ b/.github/config/nodejs-prod.jsonc @@ -42,7 +42,7 @@ ".kokoro/", ".prettierignore", ".prettierrc.js", - "cloud-samples-tools", // checked out by GH action in ci-*.yml + "cloud-samples-tools", // checked out by GH action in ci-*.yml "CODEOWNERS", "CODE_OF_CONDUCT.md", "CONTRIBUTING.md", @@ -94,7 +94,6 @@ "healthcare/fhir", // Error: Cannot find module 'whatwg-url' "iam/deny", // PERMISSION_DENIED: Permission iam.googleapis.com/denypolicies.create denied on resource cloudresourcemanager.googleapis.com/projects/long-door-651 "recaptcha_enterprise/snippets", // Cannot use import statement outside a module - "run/idp-sql", // Error: Invalid contents in the credentials file "run/markdown-preview/editor", // Error: could not create an identity token: Cannot fetch ID token in this environment, use GCE or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to a service account credentials JSON file "run/system-package", // Error: ENOENT: no such file or directory, access '/usr/bin/dot' "scheduler", // SyntaxError: Cannot use import statement outside a module diff --git a/run/idp-sql/app.js b/run/idp-sql/app.js index 5543a7aba2..5f974b8473 100644 --- a/run/idp-sql/app.js +++ b/run/idp-sql/app.js @@ -22,7 +22,7 @@ const {authenticateJWT, requestLogger} = require('./middleware'); const app = express(); app.use(express.static(__dirname + '/static')); -// Automatically parse request body as form data. +// Automatically parse request body as form data app.use(express.urlencoded({extended: false})); app.use(express.json()); From 1312379cd93bed735552b60cb471d776dc668a22 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 25 Feb 2025 14:13:25 +1100 Subject: [PATCH 02/44] add secret --- run/idp-sql/ci-setup.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 run/idp-sql/ci-setup.json diff --git a/run/idp-sql/ci-setup.json b/run/idp-sql/ci-setup.json new file mode 100644 index 0000000000..4438f36e58 --- /dev/null +++ b/run/idp-sql/ci-setup.json @@ -0,0 +1,5 @@ +{ + "secrets": { + "IDP_KEY": "nodejs-docs-samples-tests/nodejs-docs-samples-idp-key" + } + } From ae8a90c8658f82158df63a145b4cff5d3c429803 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 25 Feb 2025 14:19:27 +1100 Subject: [PATCH 03/44] update tests run, copied from eventarc/audit-storage --- run/idp-sql/package.json | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/run/idp-sql/package.json b/run/idp-sql/package.json index 5f7970a3de..7fc26bdee3 100644 --- a/run/idp-sql/package.json +++ b/run/idp-sql/package.json @@ -14,8 +14,10 @@ }, "scripts": { "start": "node index.js", - "test": "c8 mocha -p -j 2 test/app.test.js --timeout=120000 --exit", - "system-test": "c8 mocha -p -j 2 test/system.test.js --timeout=1800000 --exit" + "unit-test": "c8 mocha -p -j 2 test/app.test.js --timeout=120000 --exit", + "system-test": "c8 mocha -p -j 2 test/system.test.js --timeout=1800000 --exit", + "all-test": "npm run unit-test && npm run system-test", + "test": "npm -- run all-test" }, "dependencies": { "express": "^4.16.2", From 00e8114b1357366ec2fe6fb617fbecc932f1bb8c Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 25 Feb 2025 14:26:28 +1100 Subject: [PATCH 04/44] debug: swap order --- run/idp-sql/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/idp-sql/package.json b/run/idp-sql/package.json index 7fc26bdee3..67fd4e3704 100644 --- a/run/idp-sql/package.json +++ b/run/idp-sql/package.json @@ -16,7 +16,7 @@ "start": "node index.js", "unit-test": "c8 mocha -p -j 2 test/app.test.js --timeout=120000 --exit", "system-test": "c8 mocha -p -j 2 test/system.test.js --timeout=1800000 --exit", - "all-test": "npm run unit-test && npm run system-test", + "all-test": "npm run system-test && npm run unit-test", "test": "npm -- run all-test" }, "dependencies": { From 7e5ae5ffa38ba4997f2002a9dab050425206df0f Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 25 Feb 2025 14:34:20 +1100 Subject: [PATCH 05/44] Revert "debug: swap order" This reverts commit 00e8114b1357366ec2fe6fb617fbecc932f1bb8c. --- run/idp-sql/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/idp-sql/package.json b/run/idp-sql/package.json index 67fd4e3704..7fc26bdee3 100644 --- a/run/idp-sql/package.json +++ b/run/idp-sql/package.json @@ -16,7 +16,7 @@ "start": "node index.js", "unit-test": "c8 mocha -p -j 2 test/app.test.js --timeout=120000 --exit", "system-test": "c8 mocha -p -j 2 test/system.test.js --timeout=1800000 --exit", - "all-test": "npm run system-test && npm run unit-test", + "all-test": "npm run unit-test && npm run system-test", "test": "npm -- run all-test" }, "dependencies": { From f573a2ae4d21711a0b469fa322119b6bc28d9417 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 25 Feb 2025 14:45:20 +1100 Subject: [PATCH 06/44] debug: update firebase-admin version --- run/idp-sql/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/idp-sql/package.json b/run/idp-sql/package.json index 7fc26bdee3..1febe62959 100644 --- a/run/idp-sql/package.json +++ b/run/idp-sql/package.json @@ -21,7 +21,7 @@ }, "dependencies": { "express": "^4.16.2", - "firebase-admin": "^12.0.0", + "firebase-admin": "^13.0.0", "gcp-metadata": "^6.0.0", "google-auth-library": "^9.0.0", "handlebars": "^4.7.6", From 21b96051b6d376d1273085c2ec14a73919441425 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 25 Feb 2025 15:12:56 +1100 Subject: [PATCH 07/44] add env, secrets --- run/idp-sql/ci-setup.json | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/run/idp-sql/ci-setup.json b/run/idp-sql/ci-setup.json index 4438f36e58..7d2895a165 100644 --- a/run/idp-sql/ci-setup.json +++ b/run/idp-sql/ci-setup.json @@ -1,5 +1,12 @@ { - "secrets": { - "IDP_KEY": "nodejs-docs-samples-tests/nodejs-docs-samples-idp-key" - } + "env": { + "SERVICE_NAME": "idp-sql-ci", + "CLOUD_SQL_CONNECTION_NAME": "nodejs-docs-samples-tests:us-central1:postgres-ci", + "DB_NAME": "kokoro_ci", + "DB_USER": "kokoro_ci" + }, + "secrets": { + "IDP_KEY": "nodejs-docs-samples-tests/nodejs-docs-samples-idp-key", + "DB_PASSWORD": "nodejs-docs-samples-tests/nodejs-docs-samples-sql-password" } +} From 1a96dace5798a79cd9dd01e7ccc811c532483b96 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 25 Feb 2025 15:21:51 +1100 Subject: [PATCH 08/44] correct exit code on retry --- run/idp-sql/test/retry.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/idp-sql/test/retry.sh b/run/idp-sql/test/retry.sh index 78385733f0..0f36c2075a 100755 --- a/run/idp-sql/test/retry.sh +++ b/run/idp-sql/test/retry.sh @@ -59,7 +59,7 @@ do if ((attempt_num==max_attempts)) then echo "Attempt $attempt_num / $max_attempts failed! No more retries left!" - exit + exit 1 else echo "Attempt $attempt_num / $max_attempts failed!" sleep $((attempt_num++)) From b21482e58e5910526f8931e22b110f224576c51c Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 26 Feb 2025 09:56:03 +1100 Subject: [PATCH 09/44] dynamic service name --- run/idp-sql/ci-setup.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/idp-sql/ci-setup.json b/run/idp-sql/ci-setup.json index 7d2895a165..baa28fc736 100644 --- a/run/idp-sql/ci-setup.json +++ b/run/idp-sql/ci-setup.json @@ -1,6 +1,6 @@ { "env": { - "SERVICE_NAME": "idp-sql-ci", + "SERVICE_NAME": "idp-sql-${RUN_ID}", "CLOUD_SQL_CONNECTION_NAME": "nodejs-docs-samples-tests:us-central1:postgres-ci", "DB_NAME": "kokoro_ci", "DB_USER": "kokoro_ci" From dad045b4f12212d0df0b1c7489402e22604f0ad5 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 26 Feb 2025 09:56:24 +1100 Subject: [PATCH 10/44] wip: pass service account as env, to test --- .github/workflows/ci-prod.yaml | 3 ++- run/idp-sql/test/e2e_test_cleanup.yaml | 3 +++ run/idp-sql/test/e2e_test_setup.yaml | 4 ++++ run/idp-sql/test/system.test.js | 3 +++ 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-prod.yaml b/.github/workflows/ci-prod.yaml index 77208eeff4..9e1e95c6a5 100644 --- a/.github/workflows/ci-prod.yaml +++ b/.github/workflows/ci-prod.yaml @@ -85,6 +85,7 @@ jobs: path: ${{ fromJson(github.event_name == 'pull_request' && needs.affected.outputs.nodejs-paths || '[]') }} env: GOOGLE_SAMPLES_PROJECT: long-door-651 + GOOGLE_SERVICE_ACCOUNT: kokoro-system-test@long-door-651.iam.gserviceaccount.com CI_SETUP: ${{ toJson(fromJson(needs.affected.outputs.nodejs-setups)[matrix.path])}} steps: - name: CI Setup @@ -99,7 +100,7 @@ jobs: with: project_id: ${{ env.GOOGLE_SAMPLES_PROJECT }} workload_identity_provider: projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider - service_account: kokoro-system-test@long-door-651.iam.gserviceaccount.com + service_account: ${{ env.GOOGLE_SERVICE_ACCOUNT }} access_token_lifetime: 600s # 10 minutes - name: Export environment variables uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7 diff --git a/run/idp-sql/test/e2e_test_cleanup.yaml b/run/idp-sql/test/e2e_test_cleanup.yaml index 7521b55ee7..adf0c2482f 100644 --- a/run/idp-sql/test/e2e_test_cleanup.yaml +++ b/run/idp-sql/test/e2e_test_cleanup.yaml @@ -20,3 +20,6 @@ substitutions: _VERSION: manual _REGION: us-central1 _PLATFORM: managed + _SERVICE_ACCOUNT: ${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com + +serviceAccount: 'projects/${PROJECT_ID}/serviceAccounts/${_SERVICE_ACCOUNT}' diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/test/e2e_test_setup.yaml index 3053d19812..f1dc964eb6 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/test/e2e_test_setup.yaml @@ -57,3 +57,7 @@ substitutions: _DB_NAME: postgres _DB_USER: postgres _DB_PASSWORD: password1234 + _SERVICE_ACCOUNT: ${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com + +serviceAccount: 'projects/${PROJECT_ID}/serviceAccounts/${_SERVICE_ACCOUNT}' + diff --git a/run/idp-sql/test/system.test.js b/run/idp-sql/test/system.test.js index 343718467c..664baab4e0 100644 --- a/run/idp-sql/test/system.test.js +++ b/run/idp-sql/test/system.test.js @@ -32,6 +32,8 @@ describe('System Tests', () => { console.log('"SERVICE_NAME" env var not found. Defaulting to "idp-sql"'); SERVICE_NAME = 'idp-sql'; } + + const {GOOGLE_SERVICE_ACCOUNT} = process.env; const {SAMPLE_VERSION} = process.env; const PLATFORM = 'managed'; const REGION = 'us-central1'; @@ -60,6 +62,7 @@ describe('System Tests', () => { '--config ./test/e2e_test_setup.yaml ' + `--substitutions _SERVICE=${SERVICE_NAME},_PLATFORM=${PLATFORM},_REGION=${REGION}` + `,_DB_PASSWORD=${DB_PASSWORD},_CLOUD_SQL_CONNECTION_NAME=${CLOUD_SQL_CONNECTION_NAME}`; + if (GOOGLE_SERVICE_ACCOUNT) buildCmd += `,_SERVICE_ACCOUNT=${GOOGLE_SERVICE_ACCOUNT}`; if (SAMPLE_VERSION) buildCmd += `,_VERSION=${SAMPLE_VERSION}`; console.log('Starting Cloud Build...'); From bc3e82ede941311c394d2e6e3fee7948adde464e Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 26 Feb 2025 10:02:35 +1100 Subject: [PATCH 11/44] logging: CLOUD_LOGGING_ONLY --- run/idp-sql/test/e2e_test_cleanup.yaml | 1 + run/idp-sql/test/e2e_test_setup.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/run/idp-sql/test/e2e_test_cleanup.yaml b/run/idp-sql/test/e2e_test_cleanup.yaml index adf0c2482f..b8955703b7 100644 --- a/run/idp-sql/test/e2e_test_cleanup.yaml +++ b/run/idp-sql/test/e2e_test_cleanup.yaml @@ -23,3 +23,4 @@ substitutions: _SERVICE_ACCOUNT: ${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com serviceAccount: 'projects/${PROJECT_ID}/serviceAccounts/${_SERVICE_ACCOUNT}' +logging: CLOUD_LOGGING_ONLY diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/test/e2e_test_setup.yaml index f1dc964eb6..ce2ecbd0a2 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/test/e2e_test_setup.yaml @@ -60,4 +60,4 @@ substitutions: _SERVICE_ACCOUNT: ${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com serviceAccount: 'projects/${PROJECT_ID}/serviceAccounts/${_SERVICE_ACCOUNT}' - +logging: CLOUD_LOGGING_ONLY From 10e72a8ba30bfddb90ff3d654c8d32e8fcc4c11f Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 26 Feb 2025 12:12:56 +1100 Subject: [PATCH 12/44] correct YAML is useful --- run/idp-sql/test/e2e_test_cleanup.yaml | 3 ++- run/idp-sql/test/e2e_test_setup.yaml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/run/idp-sql/test/e2e_test_cleanup.yaml b/run/idp-sql/test/e2e_test_cleanup.yaml index b8955703b7..dffd8c2d3c 100644 --- a/run/idp-sql/test/e2e_test_cleanup.yaml +++ b/run/idp-sql/test/e2e_test_cleanup.yaml @@ -23,4 +23,5 @@ substitutions: _SERVICE_ACCOUNT: ${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com serviceAccount: 'projects/${PROJECT_ID}/serviceAccounts/${_SERVICE_ACCOUNT}' -logging: CLOUD_LOGGING_ONLY +options: + logging: CLOUD_LOGGING_ONLY diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/test/e2e_test_setup.yaml index ce2ecbd0a2..e22b407841 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/test/e2e_test_setup.yaml @@ -60,4 +60,5 @@ substitutions: _SERVICE_ACCOUNT: ${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com serviceAccount: 'projects/${PROJECT_ID}/serviceAccounts/${_SERVICE_ACCOUNT}' -logging: CLOUD_LOGGING_ONLY +options: + logging: CLOUD_LOGGING_ONLY From e95da9d4d8a050bf03872f2ea422ac3d089d008c Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 26 Feb 2025 12:24:41 +1100 Subject: [PATCH 13/44] dynamicsubtitutions --- run/idp-sql/test/e2e_test_cleanup.yaml | 1 + run/idp-sql/test/e2e_test_setup.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/run/idp-sql/test/e2e_test_cleanup.yaml b/run/idp-sql/test/e2e_test_cleanup.yaml index dffd8c2d3c..41ede5730a 100644 --- a/run/idp-sql/test/e2e_test_cleanup.yaml +++ b/run/idp-sql/test/e2e_test_cleanup.yaml @@ -25,3 +25,4 @@ substitutions: serviceAccount: 'projects/${PROJECT_ID}/serviceAccounts/${_SERVICE_ACCOUNT}' options: logging: CLOUD_LOGGING_ONLY + dynamicSubstitutions: true diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/test/e2e_test_setup.yaml index e22b407841..dc1c319bfc 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/test/e2e_test_setup.yaml @@ -62,3 +62,4 @@ substitutions: serviceAccount: 'projects/${PROJECT_ID}/serviceAccounts/${_SERVICE_ACCOUNT}' options: logging: CLOUD_LOGGING_ONLY + dynamicSubstitutions: true From df80739a918f1c1fed6dd27f5f5f2321a69dfb4e Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 28 Feb 2025 12:09:57 +1100 Subject: [PATCH 14/44] rename token as not to confuse with new default envvar --- run/idp-sql/test/system.test.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/run/idp-sql/test/system.test.js b/run/idp-sql/test/system.test.js index 664baab4e0..16365d74b8 100644 --- a/run/idp-sql/test/system.test.js +++ b/run/idp-sql/test/system.test.js @@ -54,7 +54,7 @@ describe('System Tests', () => { throw Error('"IDP_KEY" env var not found.'); } - let BASE_URL, ID_TOKEN; + let BASE_URL, CUSTOM_TOKEN; before(async () => { // Deploy service using Cloud Build let buildCmd = @@ -103,8 +103,8 @@ describe('System Tests', () => { ); const tokens = JSON.parse(response.body); - ID_TOKEN = tokens.idToken; - if (!ID_TOKEN) throw Error('Unable to acquire an ID token.'); + CUSTOM_TOKEN = tokens.idToken; + if (!CUSTOM_TOKEN) throw Error('Unable to acquire an IDP token.'); }); after(() => { @@ -131,14 +131,14 @@ describe('System Tests', () => { }); it('Can make a POST request with token', async () => { - assert(ID_TOKEN && ID_TOKEN.length > 0); + assert(CUSTOM_TOKEN && CUSTOM_TOKEN.length > 0); const options = { prefixUrl: BASE_URL.trim(), method: 'POST', form: {team: 'DOGS'}, headers: { - Authorization: `Bearer ${ID_TOKEN.trim()}`, + Authorization: `Bearer ${CUSTOM_TOKEN.trim()}`, }, retry: { limit: 5, From 8c2b87943061cb270281043c7f52f6482c8f34c5 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 28 Feb 2025 12:14:13 +1100 Subject: [PATCH 15/44] use new envvar --- run/idp-sql/app.js | 2 +- run/idp-sql/test/system.test.js | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/run/idp-sql/app.js b/run/idp-sql/app.js index 5f974b8473..5543a7aba2 100644 --- a/run/idp-sql/app.js +++ b/run/idp-sql/app.js @@ -22,7 +22,7 @@ const {authenticateJWT, requestLogger} = require('./middleware'); const app = express(); app.use(express.static(__dirname + '/static')); -// Automatically parse request body as form data +// Automatically parse request body as form data. app.use(express.urlencoded({extended: false})); app.use(express.json()); diff --git a/run/idp-sql/test/system.test.js b/run/idp-sql/test/system.test.js index 16365d74b8..c5f96810b8 100644 --- a/run/idp-sql/test/system.test.js +++ b/run/idp-sql/test/system.test.js @@ -33,7 +33,7 @@ describe('System Tests', () => { SERVICE_NAME = 'idp-sql'; } - const {GOOGLE_SERVICE_ACCOUNT} = process.env; + const {SERVICE_ACCOUNT} = process.env; const {SAMPLE_VERSION} = process.env; const PLATFORM = 'managed'; const REGION = 'us-central1'; @@ -62,7 +62,8 @@ describe('System Tests', () => { '--config ./test/e2e_test_setup.yaml ' + `--substitutions _SERVICE=${SERVICE_NAME},_PLATFORM=${PLATFORM},_REGION=${REGION}` + `,_DB_PASSWORD=${DB_PASSWORD},_CLOUD_SQL_CONNECTION_NAME=${CLOUD_SQL_CONNECTION_NAME}`; - if (GOOGLE_SERVICE_ACCOUNT) buildCmd += `,_SERVICE_ACCOUNT=${GOOGLE_SERVICE_ACCOUNT}`; + + if (SERVICE_ACCOUNT) buildCmd += `,_SERVICE_ACCOUNT=${SERVICE_ACCOUNT}`; if (SAMPLE_VERSION) buildCmd += `,_VERSION=${SAMPLE_VERSION}`; console.log('Starting Cloud Build...'); @@ -113,6 +114,7 @@ describe('System Tests', () => { '--config ./test/e2e_test_cleanup.yaml ' + `--substitutions _SERVICE=${SERVICE_NAME},_PLATFORM=${PLATFORM},_REGION=${REGION}`; if (SAMPLE_VERSION) cleanUpCmd += `,_VERSION=${SAMPLE_VERSION}`; + if (SERVICE_ACCOUNT) cleanUpCmd += `,_SERVICE_ACCOUNT=${SERVICE_ACCOUNT}`; execSync(cleanUpCmd, {shell: true}); }); From 1dfbefea2ed6e7068c922cb490838fd28a7a018c Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 28 Feb 2025 12:26:45 +1100 Subject: [PATCH 16/44] update SA usage --- run/idp-sql/test/e2e_test_setup.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/test/e2e_test_setup.yaml index dc1c319bfc..8adec6ac12 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/test/e2e_test_setup.yaml @@ -15,6 +15,11 @@ steps: --replication-policy="automatic" \ --data-file=postgres-secrets.json" + ./test/retry.sh "gcloud secrets add-iam-policy-binding ${_SERVICE}-secrets \ + --member="${_SERVICE_ACCOUNT}" \ + --role="roles/secretmanager.secretAccessor" + + - id: 'Build Container Image' name: 'gcr.io/cloud-builders/docker' entrypoint: '/bin/bash' @@ -37,11 +42,11 @@ steps: args: - '-c' - | - ./test/retry.sh "gcloud beta run deploy ${_SERVICE} \ + ./test/retry.sh "gcloud run deploy ${_SERVICE} \ --image gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} \ --allow-unauthenticated \ --region ${_REGION} \ - --platform ${_PLATFORM} \ + --service-account ${_SERVICE_ACCOUNT} \ --add-cloudsql-instances ${_CLOUD_SQL_CONNECTION_NAME} \ --update-secrets CLOUD_SQL_CREDENTIALS_SECRET=${_SERVICE}-secrets:latest" From 7c255fe1cd626df33d9af6113b653027308b2bbe Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 28 Feb 2025 12:35:18 +1100 Subject: [PATCH 17/44] add custom audience --- run/idp-sql/test/e2e_test_setup.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/test/e2e_test_setup.yaml index 8adec6ac12..26959949b6 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/test/e2e_test_setup.yaml @@ -48,6 +48,7 @@ steps: --region ${_REGION} \ --service-account ${_SERVICE_ACCOUNT} \ --add-cloudsql-instances ${_CLOUD_SQL_CONNECTION_NAME} \ + --add-custom-audiences="https://actions.test/" \ --update-secrets CLOUD_SQL_CREDENTIALS_SECRET=${_SERVICE}-secrets:latest" images: From 096ebe271d87775f1bd6c3d8dce14f44f5333a4d Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 28 Feb 2025 13:06:56 +1100 Subject: [PATCH 18/44] cleanup --- run/idp-sql/test/e2e_test_cleanup.yaml | 15 +++++++++++++++ run/idp-sql/test/e2e_test_setup.yaml | 17 +++++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/run/idp-sql/test/e2e_test_cleanup.yaml b/run/idp-sql/test/e2e_test_cleanup.yaml index 41ede5730a..eb2dfa5005 100644 --- a/run/idp-sql/test/e2e_test_cleanup.yaml +++ b/run/idp-sql/test/e2e_test_cleanup.yaml @@ -1,3 +1,18 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + steps: - id: 'Delete resources' diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/test/e2e_test_setup.yaml index 26959949b6..83d243baff 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/test/e2e_test_setup.yaml @@ -1,3 +1,18 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + steps: - id: 'Add a Secret to Secret Manager' @@ -19,7 +34,6 @@ steps: --member="${_SERVICE_ACCOUNT}" \ --role="roles/secretmanager.secretAccessor" - - id: 'Build Container Image' name: 'gcr.io/cloud-builders/docker' entrypoint: '/bin/bash' @@ -58,7 +72,6 @@ substitutions: _SERVICE: idp-sql _VERSION: manual _REGION: us-central1 - _PLATFORM: managed _CLOUD_SQL_CONNECTION_NAME: $PROJECT_ID:us-central1:idp-sql-instance _DB_NAME: postgres _DB_USER: postgres From 189f17c98ff90132afead1c9c412683c8f7423da Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 28 Feb 2025 13:12:54 +1100 Subject: [PATCH 19/44] remove platform, extra SA var --- run/idp-sql/test/e2e_test_cleanup.yaml | 6 ++---- run/idp-sql/test/system.test.js | 4 ++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/run/idp-sql/test/e2e_test_cleanup.yaml b/run/idp-sql/test/e2e_test_cleanup.yaml index eb2dfa5005..af15741cb9 100644 --- a/run/idp-sql/test/e2e_test_cleanup.yaml +++ b/run/idp-sql/test/e2e_test_cleanup.yaml @@ -27,15 +27,13 @@ steps: ./test/retry.sh "gcloud container images describe gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" \ "gcloud container images delete gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} --quiet" - ./test/retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM}" \ - "gcloud run services delete ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM} --quiet" + ./test/retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION} \ + "gcloud run services delete ${_SERVICE} --region ${_REGION} --quiet" substitutions: _SERVICE: idp-sql _VERSION: manual _REGION: us-central1 - _PLATFORM: managed - _SERVICE_ACCOUNT: ${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com serviceAccount: 'projects/${PROJECT_ID}/serviceAccounts/${_SERVICE_ACCOUNT}' options: diff --git a/run/idp-sql/test/system.test.js b/run/idp-sql/test/system.test.js index c5f96810b8..e035c63f9c 100644 --- a/run/idp-sql/test/system.test.js +++ b/run/idp-sql/test/system.test.js @@ -60,7 +60,7 @@ describe('System Tests', () => { let buildCmd = `gcloud builds submit --project ${GOOGLE_CLOUD_PROJECT} ` + '--config ./test/e2e_test_setup.yaml ' + - `--substitutions _SERVICE=${SERVICE_NAME},_PLATFORM=${PLATFORM},_REGION=${REGION}` + + `--substitutions _SERVICE=${SERVICE_NAME},_REGION=${REGION}` + `,_DB_PASSWORD=${DB_PASSWORD},_CLOUD_SQL_CONNECTION_NAME=${CLOUD_SQL_CONNECTION_NAME}`; if (SERVICE_ACCOUNT) buildCmd += `,_SERVICE_ACCOUNT=${SERVICE_ACCOUNT}`; @@ -112,7 +112,7 @@ describe('System Tests', () => { let cleanUpCmd = `gcloud builds submit --project ${GOOGLE_CLOUD_PROJECT} ` + '--config ./test/e2e_test_cleanup.yaml ' + - `--substitutions _SERVICE=${SERVICE_NAME},_PLATFORM=${PLATFORM},_REGION=${REGION}`; + `--substitutions _SERVICE=${SERVICE_NAME},_REGION=${REGION}`; if (SAMPLE_VERSION) cleanUpCmd += `,_VERSION=${SAMPLE_VERSION}`; if (SERVICE_ACCOUNT) cleanUpCmd += `,_SERVICE_ACCOUNT=${SERVICE_ACCOUNT}`; From d4cfd2d91d59bab93652badc43b84c7b49fdf81f Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 28 Feb 2025 13:16:57 +1100 Subject: [PATCH 20/44] lint --- run/idp-sql/test/e2e_test_setup.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/test/e2e_test_setup.yaml index 83d243baff..544b50c694 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/test/e2e_test_setup.yaml @@ -27,11 +27,11 @@ steps: sed -i "s/\"DB_USER\": \"postgres\"/\"DB_USER\": \"${_DB_USER}\"/" postgres-secrets.json ./test/retry.sh "gcloud secrets create ${_SERVICE}-secrets \ - --replication-policy="automatic" \ + --replication-policy=automatic \ --data-file=postgres-secrets.json" ./test/retry.sh "gcloud secrets add-iam-policy-binding ${_SERVICE}-secrets \ - --member="${_SERVICE_ACCOUNT}" \ + --member=${_SERVICE_ACCOUNT} \ --role="roles/secretmanager.secretAccessor" - id: 'Build Container Image' From 29a8f08ddf00beb236fcbca5679e2091c2379ec4 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 28 Feb 2025 13:22:24 +1100 Subject: [PATCH 21/44] more lint --- run/idp-sql/test/e2e_test_setup.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/test/e2e_test_setup.yaml index 544b50c694..4030b46e73 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/test/e2e_test_setup.yaml @@ -32,7 +32,7 @@ steps: ./test/retry.sh "gcloud secrets add-iam-policy-binding ${_SERVICE}-secrets \ --member=${_SERVICE_ACCOUNT} \ - --role="roles/secretmanager.secretAccessor" + --role=roles/secretmanager.secretAccessor" - id: 'Build Container Image' name: 'gcr.io/cloud-builders/docker' From cc70a1d93e46fc620586796dd3daeea64033cae8 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 28 Feb 2025 13:26:58 +1100 Subject: [PATCH 22/44] format --- run/idp-sql/test/e2e_test_setup.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/test/e2e_test_setup.yaml index 4030b46e73..a4d88b8315 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/test/e2e_test_setup.yaml @@ -31,7 +31,7 @@ steps: --data-file=postgres-secrets.json" ./test/retry.sh "gcloud secrets add-iam-policy-binding ${_SERVICE}-secrets \ - --member=${_SERVICE_ACCOUNT} \ + --member=serviceAccount:${_SERVICE_ACCOUNT} \ --role=roles/secretmanager.secretAccessor" - id: 'Build Container Image' From 429edee2eab979e65e6f93521311e4bc62900035 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 28 Feb 2025 13:34:03 +1100 Subject: [PATCH 23/44] add sample version value --- run/idp-sql/ci-setup.json | 1 + 1 file changed, 1 insertion(+) diff --git a/run/idp-sql/ci-setup.json b/run/idp-sql/ci-setup.json index baa28fc736..ebdccb4bf8 100644 --- a/run/idp-sql/ci-setup.json +++ b/run/idp-sql/ci-setup.json @@ -1,6 +1,7 @@ { "env": { "SERVICE_NAME": "idp-sql-${RUN_ID}", + "SAMPLE_VERSION": "${RUN_ID}", "CLOUD_SQL_CONNECTION_NAME": "nodejs-docs-samples-tests:us-central1:postgres-ci", "DB_NAME": "kokoro_ci", "DB_USER": "kokoro_ci" From 1792e7015cc7116452b0e99edc3c306087a2b486 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 28 Feb 2025 13:34:11 +1100 Subject: [PATCH 24/44] correct audience --- run/idp-sql/test/e2e_test_setup.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/test/e2e_test_setup.yaml index a4d88b8315..946d64e2c7 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/test/e2e_test_setup.yaml @@ -62,7 +62,7 @@ steps: --region ${_REGION} \ --service-account ${_SERVICE_ACCOUNT} \ --add-cloudsql-instances ${_CLOUD_SQL_CONNECTION_NAME} \ - --add-custom-audiences="https://actions.test/" \ + --add-custom-audiences="https://action.test/" \ --update-secrets CLOUD_SQL_CREDENTIALS_SECRET=${_SERVICE}-secrets:latest" images: From 0fc96bc5b067714415d3815056ae2e401e9252c9 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 28 Feb 2025 13:56:43 +1100 Subject: [PATCH 25/44] lint --- run/idp-sql/test/e2e_test_cleanup.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/idp-sql/test/e2e_test_cleanup.yaml b/run/idp-sql/test/e2e_test_cleanup.yaml index af15741cb9..e2c497f6cd 100644 --- a/run/idp-sql/test/e2e_test_cleanup.yaml +++ b/run/idp-sql/test/e2e_test_cleanup.yaml @@ -27,7 +27,7 @@ steps: ./test/retry.sh "gcloud container images describe gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" \ "gcloud container images delete gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} --quiet" - ./test/retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION} \ + ./test/retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION}" \ "gcloud run services delete ${_SERVICE} --region ${_REGION} --quiet" substitutions: From b00cc9bcb48c0c753ccd7f42b53943504d29c519 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 28 Feb 2025 15:24:54 +1100 Subject: [PATCH 26/44] disable cleanup for debugging --- run/idp-sql/test/system.test.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/run/idp-sql/test/system.test.js b/run/idp-sql/test/system.test.js index e035c63f9c..2265cd2c5e 100644 --- a/run/idp-sql/test/system.test.js +++ b/run/idp-sql/test/system.test.js @@ -116,7 +116,8 @@ describe('System Tests', () => { if (SAMPLE_VERSION) cleanUpCmd += `,_VERSION=${SAMPLE_VERSION}`; if (SERVICE_ACCOUNT) cleanUpCmd += `,_SERVICE_ACCOUNT=${SERVICE_ACCOUNT}`; - execSync(cleanUpCmd, {shell: true}); + //TODO(glasnt): re-enable cleanup + //execSync(cleanUpCmd, {shell: true}); }); it('Can successfully make a request', async () => { From 3867b4406b9a645050662c04db1b0db370d7034c Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 4 Mar 2025 10:57:41 +1100 Subject: [PATCH 27/44] add test to dev list --- .github/config/nodejs-dev.jsonc | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/config/nodejs-dev.jsonc b/.github/config/nodejs-dev.jsonc index 8766848d7f..2fd6f2f221 100644 --- a/.github/config/nodejs-dev.jsonc +++ b/.github/config/nodejs-dev.jsonc @@ -196,6 +196,7 @@ "run/image-processing", "run/jobs", "run/logging-manual", + "run/idp-sql", "run/markdown-preview/renderer", "run/pubsub", "run/system-package", From 7eb82c7c5b26689b66dbf2026d5b7519437e0499 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 4 Mar 2025 16:17:39 +1100 Subject: [PATCH 28/44] attempt proxy powered --- run/idp-sql/ci-setup.json | 9 ++++----- run/idp-sql/package.json | 6 ++++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/run/idp-sql/ci-setup.json b/run/idp-sql/ci-setup.json index ebdccb4bf8..8cc71f10d8 100644 --- a/run/idp-sql/ci-setup.json +++ b/run/idp-sql/ci-setup.json @@ -2,12 +2,11 @@ "env": { "SERVICE_NAME": "idp-sql-${RUN_ID}", "SAMPLE_VERSION": "${RUN_ID}", - "CLOUD_SQL_CONNECTION_NAME": "nodejs-docs-samples-tests:us-central1:postgres-ci", - "DB_NAME": "kokoro_ci", - "DB_USER": "kokoro_ci" + "CLOUD_SQL_CONNECTION_NAME": "nodejs-docs-samples-tests:us-central1:test-postgres-instance", + "DB_NAME": "ci-database", + "DB_USER": "ci-user" }, "secrets": { "IDP_KEY": "nodejs-docs-samples-tests/nodejs-docs-samples-idp-key", - "DB_PASSWORD": "nodejs-docs-samples-tests/nodejs-docs-samples-sql-password" - } + "DB_PASSWORD": "nodejs-docs-samples-tests/nodejs-docs-samples-test-postgres-instance-ci-user-password" } } diff --git a/run/idp-sql/package.json b/run/idp-sql/package.json index 1febe62959..2104e879d7 100644 --- a/run/idp-sql/package.json +++ b/run/idp-sql/package.json @@ -15,9 +15,11 @@ "scripts": { "start": "node index.js", "unit-test": "c8 mocha -p -j 2 test/app.test.js --timeout=120000 --exit", - "system-test": "c8 mocha -p -j 2 test/system.test.js --timeout=1800000 --exit", + "system-test": "npm -- run start-proxy && c8 mocha -p -j 2 test/system.test.js --timeout=1800000 --exit", "all-test": "npm run unit-test && npm run system-test", - "test": "npm -- run all-test" + "test": "npm -- run all-test", + "start-proxy": "! pgrep cloud_sql_proxy > /dev/null && cloud_sql_proxy -dir=/cloudsql -instances=$INSTANCE_CONNECTION_NAME &", + }, "dependencies": { "express": "^4.16.2", From 95896c040f2c63970be19b8dcc5452a1b2ebe4d5 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 4 Mar 2025 16:24:45 +1100 Subject: [PATCH 29/44] lint --- run/idp-sql/package.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/run/idp-sql/package.json b/run/idp-sql/package.json index 2104e879d7..c2b017ede0 100644 --- a/run/idp-sql/package.json +++ b/run/idp-sql/package.json @@ -18,8 +18,7 @@ "system-test": "npm -- run start-proxy && c8 mocha -p -j 2 test/system.test.js --timeout=1800000 --exit", "all-test": "npm run unit-test && npm run system-test", "test": "npm -- run all-test", - "start-proxy": "! pgrep cloud_sql_proxy > /dev/null && cloud_sql_proxy -dir=/cloudsql -instances=$INSTANCE_CONNECTION_NAME &", - + "start-proxy": "! pgrep cloud_sql_proxy > /dev/null && cloud_sql_proxy -dir=/cloudsql -instances=$INSTANCE_CONNECTION_NAME &" }, "dependencies": { "express": "^4.16.2", From b096ca4c2bc5115db71b9ce6c3566d86f144eb2f Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 4 Mar 2025 16:47:12 +1100 Subject: [PATCH 30/44] attempt: proxy shell --- run/idp-sql/package.json | 3 +-- run/idp-sql/test/proxy-setup.sh | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) create mode 100755 run/idp-sql/test/proxy-setup.sh diff --git a/run/idp-sql/package.json b/run/idp-sql/package.json index c2b017ede0..3ddcad8b3b 100644 --- a/run/idp-sql/package.json +++ b/run/idp-sql/package.json @@ -15,10 +15,9 @@ "scripts": { "start": "node index.js", "unit-test": "c8 mocha -p -j 2 test/app.test.js --timeout=120000 --exit", - "system-test": "npm -- run start-proxy && c8 mocha -p -j 2 test/system.test.js --timeout=1800000 --exit", + "system-test": "test/proxy-setup.sh && c8 mocha -p -j 2 test/system.test.js --timeout=1800000 --exit", "all-test": "npm run unit-test && npm run system-test", "test": "npm -- run all-test", - "start-proxy": "! pgrep cloud_sql_proxy > /dev/null && cloud_sql_proxy -dir=/cloudsql -instances=$INSTANCE_CONNECTION_NAME &" }, "dependencies": { "express": "^4.16.2", diff --git a/run/idp-sql/test/proxy-setup.sh b/run/idp-sql/test/proxy-setup.sh new file mode 100755 index 0000000000..78d7197634 --- /dev/null +++ b/run/idp-sql/test/proxy-setup.sh @@ -0,0 +1,3 @@ +curl -o cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.15.1/cloud-sql-proxy.linux.amd64 +chmod +x cloud-sql-proxy +cloud_sql_proxy -dir=/cloudsql -instances=$INSTANCE_CONNECTION_NAME & From 0ee57e24835032dffcbdc728d780f193053bab1a Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 4 Mar 2025 16:49:56 +1100 Subject: [PATCH 31/44] lint --- run/idp-sql/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/idp-sql/package.json b/run/idp-sql/package.json index 3ddcad8b3b..7bdfd66704 100644 --- a/run/idp-sql/package.json +++ b/run/idp-sql/package.json @@ -17,7 +17,7 @@ "unit-test": "c8 mocha -p -j 2 test/app.test.js --timeout=120000 --exit", "system-test": "test/proxy-setup.sh && c8 mocha -p -j 2 test/system.test.js --timeout=1800000 --exit", "all-test": "npm run unit-test && npm run system-test", - "test": "npm -- run all-test", + "test": "npm -- run all-test" }, "dependencies": { "express": "^4.16.2", From 2faec3f5b49c7db8cdd7dd31cc7e393366c7d04a Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 4 Mar 2025 16:52:12 +1100 Subject: [PATCH 32/44] format --- run/idp-sql/test/proxy-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run/idp-sql/test/proxy-setup.sh b/run/idp-sql/test/proxy-setup.sh index 78d7197634..3ec47667f2 100755 --- a/run/idp-sql/test/proxy-setup.sh +++ b/run/idp-sql/test/proxy-setup.sh @@ -1,3 +1,3 @@ curl -o cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.15.1/cloud-sql-proxy.linux.amd64 chmod +x cloud-sql-proxy -cloud_sql_proxy -dir=/cloudsql -instances=$INSTANCE_CONNECTION_NAME & +cloud-sql-proxy -dir=/cloudsql -instances=$INSTANCE_CONNECTION_NAME & From 414ebff7ba1e1fb6fbc0169eab011bb1fff2ccd9 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 4 Mar 2025 17:00:40 +1100 Subject: [PATCH 33/44] try using x-headers --- run/idp-sql/test/e2e_test_setup.yaml | 2 +- run/idp-sql/test/system.test.js | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/test/e2e_test_setup.yaml index 946d64e2c7..48eefe3ac3 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/test/e2e_test_setup.yaml @@ -58,7 +58,7 @@ steps: - | ./test/retry.sh "gcloud run deploy ${_SERVICE} \ --image gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} \ - --allow-unauthenticated \ + --no-allow-unauthenticated \ --region ${_REGION} \ --service-account ${_SERVICE_ACCOUNT} \ --add-cloudsql-instances ${_CLOUD_SQL_CONNECTION_NAME} \ diff --git a/run/idp-sql/test/system.test.js b/run/idp-sql/test/system.test.js index 2265cd2c5e..55d913f16e 100644 --- a/run/idp-sql/test/system.test.js +++ b/run/idp-sql/test/system.test.js @@ -54,6 +54,12 @@ describe('System Tests', () => { throw Error('"IDP_KEY" env var not found.'); } + // Get ID token for the authentication part + const {ID_TOKEN} = process.env; + if (!ID_TOKEN) { + throw Error('"ID_TOKEN" env var not found.'); + } + let BASE_URL, CUSTOM_TOKEN; before(async () => { // Deploy service using Cloud Build @@ -142,6 +148,7 @@ describe('System Tests', () => { form: {team: 'DOGS'}, headers: { Authorization: `Bearer ${CUSTOM_TOKEN.trim()}`, + "X-Serverless-Authorization": `Bearer ${ID_TOKEN}` }, retry: { limit: 5, @@ -167,6 +174,7 @@ describe('System Tests', () => { form: {team: 'DOGS'}, headers: { Authorization: 'Bearer iam-a-token', + "X-Serverless-Authorization": `Bearer ${ID_TOKEN}` }, retry: { limit: 5, From a2e7320bb3e0c424985ce04b12d94d1db2eca98b Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 18 Mar 2025 08:23:54 +1100 Subject: [PATCH 34/44] fix: headers --- run/idp-sql/test/proxy-setup.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/run/idp-sql/test/proxy-setup.sh b/run/idp-sql/test/proxy-setup.sh index 3ec47667f2..c1006a8e9a 100755 --- a/run/idp-sql/test/proxy-setup.sh +++ b/run/idp-sql/test/proxy-setup.sh @@ -1,3 +1,19 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Proof of concept: setting up proxy + curl -o cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.15.1/cloud-sql-proxy.linux.amd64 chmod +x cloud-sql-proxy cloud-sql-proxy -dir=/cloudsql -instances=$INSTANCE_CONNECTION_NAME & From 4e64db2822abbb2b3315c3966458b1879073d8dc Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 24 Mar 2025 10:19:34 +1100 Subject: [PATCH 35/44] pass db user/name if supplied --- run/idp-sql/test/system.test.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/run/idp-sql/test/system.test.js b/run/idp-sql/test/system.test.js index 55d913f16e..f5fa42911f 100644 --- a/run/idp-sql/test/system.test.js +++ b/run/idp-sql/test/system.test.js @@ -43,6 +43,9 @@ describe('System Tests', () => { if (!CLOUD_SQL_CONNECTION_NAME) { throw Error('"CLOUD_SQL_CONNECTION_NAME" env var not found.'); } + + const {DB_NAME} = process.env; + const {DB_USER} = process.env; const {DB_PASSWORD} = process.env; if (!DB_PASSWORD) { throw Error('"DB_PASSWORD" env var not found.'); @@ -71,6 +74,8 @@ describe('System Tests', () => { if (SERVICE_ACCOUNT) buildCmd += `,_SERVICE_ACCOUNT=${SERVICE_ACCOUNT}`; if (SAMPLE_VERSION) buildCmd += `,_VERSION=${SAMPLE_VERSION}`; + if (DB_USER) buildCmd += `,_DB_USER=${DB_USER}`; + if (DB_NAME) buildCmd += `,_DB_NAME=${DB_NAME}`; console.log('Starting Cloud Build...'); execSync(buildCmd, {timeout: 240000, shell: true}); // timeout at 4 mins From c05beef9c42c0b9c013f91ff179f4b7323b75e1e Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 24 Mar 2025 10:36:05 +1100 Subject: [PATCH 36/44] revert "try using x-headers" --- run/idp-sql/test/e2e_test_setup.yaml | 2 +- run/idp-sql/test/system.test.js | 12 ++---------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/test/e2e_test_setup.yaml index 48eefe3ac3..946d64e2c7 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/test/e2e_test_setup.yaml @@ -58,7 +58,7 @@ steps: - | ./test/retry.sh "gcloud run deploy ${_SERVICE} \ --image gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} \ - --no-allow-unauthenticated \ + --allow-unauthenticated \ --region ${_REGION} \ --service-account ${_SERVICE_ACCOUNT} \ --add-cloudsql-instances ${_CLOUD_SQL_CONNECTION_NAME} \ diff --git a/run/idp-sql/test/system.test.js b/run/idp-sql/test/system.test.js index f5fa42911f..af0fb8e6eb 100644 --- a/run/idp-sql/test/system.test.js +++ b/run/idp-sql/test/system.test.js @@ -57,12 +57,6 @@ describe('System Tests', () => { throw Error('"IDP_KEY" env var not found.'); } - // Get ID token for the authentication part - const {ID_TOKEN} = process.env; - if (!ID_TOKEN) { - throw Error('"ID_TOKEN" env var not found.'); - } - let BASE_URL, CUSTOM_TOKEN; before(async () => { // Deploy service using Cloud Build @@ -152,8 +146,7 @@ describe('System Tests', () => { method: 'POST', form: {team: 'DOGS'}, headers: { - Authorization: `Bearer ${CUSTOM_TOKEN.trim()}`, - "X-Serverless-Authorization": `Bearer ${ID_TOKEN}` + Authorization: `Bearer ${CUSTOM_TOKEN.trim()}` }, retry: { limit: 5, @@ -178,8 +171,7 @@ describe('System Tests', () => { method: 'POST', form: {team: 'DOGS'}, headers: { - Authorization: 'Bearer iam-a-token', - "X-Serverless-Authorization": `Bearer ${ID_TOKEN}` + Authorization: 'Bearer iam-a-token' }, retry: { limit: 5, From 94cb82a0098c9da79764461af7088221baac623b Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 24 Mar 2025 11:01:32 +1100 Subject: [PATCH 37/44] debugging: where 400 error coming from? --- run/idp-sql/test/system.test.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/run/idp-sql/test/system.test.js b/run/idp-sql/test/system.test.js index af0fb8e6eb..697aec27c5 100644 --- a/run/idp-sql/test/system.test.js +++ b/run/idp-sql/test/system.test.js @@ -91,6 +91,8 @@ describe('System Tests', () => { } // Retrieve ID token for testing + + console.log('Retrieving IDP token...'); const customToken = await admin.auth().createCustomToken('a-user-id'); const response = await got( `https://identitytoolkit.googleapis.com/v1/accounts:signInWithCustomToken?key=${IDP_KEY}`, @@ -111,6 +113,8 @@ describe('System Tests', () => { const tokens = JSON.parse(response.body); CUSTOM_TOKEN = tokens.idToken; if (!CUSTOM_TOKEN) throw Error('Unable to acquire an IDP token.'); + + console.log('Retrieved IDP token'); }); after(() => { From aafa923c7709fca2a5a975473a057e2e4d12da30 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 24 Mar 2025 11:10:17 +1100 Subject: [PATCH 38/44] reenable cleanup --- run/idp-sql/test/system.test.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/run/idp-sql/test/system.test.js b/run/idp-sql/test/system.test.js index 697aec27c5..3a58d2cdf2 100644 --- a/run/idp-sql/test/system.test.js +++ b/run/idp-sql/test/system.test.js @@ -125,8 +125,7 @@ describe('System Tests', () => { if (SAMPLE_VERSION) cleanUpCmd += `,_VERSION=${SAMPLE_VERSION}`; if (SERVICE_ACCOUNT) cleanUpCmd += `,_SERVICE_ACCOUNT=${SERVICE_ACCOUNT}`; - //TODO(glasnt): re-enable cleanup - //execSync(cleanUpCmd, {shell: true}); + execSync(cleanUpCmd, {shell: true}); }); it('Can successfully make a request', async () => { From 1903807b2d0a7a15e25d7222db8af3b705f5c1de Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 24 Mar 2025 11:13:58 +1100 Subject: [PATCH 39/44] try capturing the token error --- run/idp-sql/test/system.test.js | 38 ++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/run/idp-sql/test/system.test.js b/run/idp-sql/test/system.test.js index 3a58d2cdf2..e34c78aaa3 100644 --- a/run/idp-sql/test/system.test.js +++ b/run/idp-sql/test/system.test.js @@ -94,24 +94,28 @@ describe('System Tests', () => { console.log('Retrieving IDP token...'); const customToken = await admin.auth().createCustomToken('a-user-id'); - const response = await got( - `https://identitytoolkit.googleapis.com/v1/accounts:signInWithCustomToken?key=${IDP_KEY}`, - { - method: 'POST', - retry: { - limit: 5, - statusCodes: [404, 401, 403, 500], - methods: ['POST'], - }, - body: JSON.stringify({ - token: customToken, - returnSecureToken: true, - }), - } - ); + try { + const response = await got( + `https://identitytoolkit.googleapis.com/v1/accounts:signInWithCustomToken?key=${IDP_KEY}`, + { + method: 'POST', + retry: { + limit: 5, + statusCodes: [404, 401, 403, 500], + methods: ['POST'], + }, + body: JSON.stringify({ + token: customToken, + returnSecureToken: true, + }), + } + ); + const tokens = JSON.parse(response.body); + CUSTOM_TOKEN = tokens.idToken; + } catch (err) { + throw Error('IDP Token retrieval failed: ', err); + } - const tokens = JSON.parse(response.body); - CUSTOM_TOKEN = tokens.idToken; if (!CUSTOM_TOKEN) throw Error('Unable to acquire an IDP token.'); console.log('Retrieved IDP token'); From 717d351b3f20f211db2bc822e8f3aee617768aac Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 24 Mar 2025 11:16:49 +1100 Subject: [PATCH 40/44] lint --- run/idp-sql/test/system.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/run/idp-sql/test/system.test.js b/run/idp-sql/test/system.test.js index e34c78aaa3..be3eff3c7d 100644 --- a/run/idp-sql/test/system.test.js +++ b/run/idp-sql/test/system.test.js @@ -153,7 +153,7 @@ describe('System Tests', () => { method: 'POST', form: {team: 'DOGS'}, headers: { - Authorization: `Bearer ${CUSTOM_TOKEN.trim()}` + Authorization: `Bearer ${CUSTOM_TOKEN.trim()}`, }, retry: { limit: 5, @@ -178,7 +178,7 @@ describe('System Tests', () => { method: 'POST', form: {team: 'DOGS'}, headers: { - Authorization: 'Bearer iam-a-token' + Authorization: 'Bearer iam-a-token', }, retry: { limit: 5, From b69f4eba702b298ad0a45e97e7865813400411e6 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 24 Mar 2025 13:00:42 +1100 Subject: [PATCH 41/44] use explicitly named secret reference --- run/idp-sql/ci-setup.json | 2 +- run/idp-sql/test/system.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/run/idp-sql/ci-setup.json b/run/idp-sql/ci-setup.json index 8cc71f10d8..a1f34ed1b1 100644 --- a/run/idp-sql/ci-setup.json +++ b/run/idp-sql/ci-setup.json @@ -7,6 +7,6 @@ "DB_USER": "ci-user" }, "secrets": { - "IDP_KEY": "nodejs-docs-samples-tests/nodejs-docs-samples-idp-key", + "IDP_KEY": "nodejs-docs-samples-tests/long-door-651-idp-key", "DB_PASSWORD": "nodejs-docs-samples-tests/nodejs-docs-samples-test-postgres-instance-ci-user-password" } } diff --git a/run/idp-sql/test/system.test.js b/run/idp-sql/test/system.test.js index be3eff3c7d..7b4b57b1d2 100644 --- a/run/idp-sql/test/system.test.js +++ b/run/idp-sql/test/system.test.js @@ -113,7 +113,7 @@ describe('System Tests', () => { const tokens = JSON.parse(response.body); CUSTOM_TOKEN = tokens.idToken; } catch (err) { - throw Error('IDP Token retrieval failed: ', err); + throw Error('IDP Token retrieval failed: ', err.response.body); } if (!CUSTOM_TOKEN) throw Error('Unable to acquire an IDP token.'); From a0bcccfe503867ae65abb669df4396b5e8b9e6c6 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 24 Mar 2025 13:16:18 +1100 Subject: [PATCH 42/44] restore: try x-headers (debug) --- run/idp-sql/test/e2e_test_setup.yaml | 2 +- run/idp-sql/test/system.test.js | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/test/e2e_test_setup.yaml index 946d64e2c7..48eefe3ac3 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/test/e2e_test_setup.yaml @@ -58,7 +58,7 @@ steps: - | ./test/retry.sh "gcloud run deploy ${_SERVICE} \ --image gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} \ - --allow-unauthenticated \ + --no-allow-unauthenticated \ --region ${_REGION} \ --service-account ${_SERVICE_ACCOUNT} \ --add-cloudsql-instances ${_CLOUD_SQL_CONNECTION_NAME} \ diff --git a/run/idp-sql/test/system.test.js b/run/idp-sql/test/system.test.js index 7b4b57b1d2..855b2ad6a6 100644 --- a/run/idp-sql/test/system.test.js +++ b/run/idp-sql/test/system.test.js @@ -51,6 +51,12 @@ describe('System Tests', () => { throw Error('"DB_PASSWORD" env var not found.'); } + // Get ID token for the authentication part + const {ID_TOKEN} = process.env; + if (!ID_TOKEN) { + throw Error('"ID_TOKEN" env var not found.'); + } + // Get Firebase Key to create Id Tokens const {IDP_KEY} = process.env; if (!IDP_KEY) { @@ -154,6 +160,7 @@ describe('System Tests', () => { form: {team: 'DOGS'}, headers: { Authorization: `Bearer ${CUSTOM_TOKEN.trim()}`, + 'X-Serverless-Authorization': `Bearer ${ID_TOKEN}`, }, retry: { limit: 5, @@ -179,6 +186,7 @@ describe('System Tests', () => { form: {team: 'DOGS'}, headers: { Authorization: 'Bearer iam-a-token', + 'X-Serverless-Authorization': `Bearer ${ID_TOKEN}`, }, retry: { limit: 5, From 37518cf3db4e161d7c1d9cc23ec0bdb1eddf0818 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 24 Mar 2025 13:23:40 +1100 Subject: [PATCH 43/44] Revert "restore: try x-headers (debug)" This reverts commit a0bcccfe503867ae65abb669df4396b5e8b9e6c6. --- run/idp-sql/test/e2e_test_setup.yaml | 2 +- run/idp-sql/test/system.test.js | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/test/e2e_test_setup.yaml index 48eefe3ac3..946d64e2c7 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/test/e2e_test_setup.yaml @@ -58,7 +58,7 @@ steps: - | ./test/retry.sh "gcloud run deploy ${_SERVICE} \ --image gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} \ - --no-allow-unauthenticated \ + --allow-unauthenticated \ --region ${_REGION} \ --service-account ${_SERVICE_ACCOUNT} \ --add-cloudsql-instances ${_CLOUD_SQL_CONNECTION_NAME} \ diff --git a/run/idp-sql/test/system.test.js b/run/idp-sql/test/system.test.js index 855b2ad6a6..7b4b57b1d2 100644 --- a/run/idp-sql/test/system.test.js +++ b/run/idp-sql/test/system.test.js @@ -51,12 +51,6 @@ describe('System Tests', () => { throw Error('"DB_PASSWORD" env var not found.'); } - // Get ID token for the authentication part - const {ID_TOKEN} = process.env; - if (!ID_TOKEN) { - throw Error('"ID_TOKEN" env var not found.'); - } - // Get Firebase Key to create Id Tokens const {IDP_KEY} = process.env; if (!IDP_KEY) { @@ -160,7 +154,6 @@ describe('System Tests', () => { form: {team: 'DOGS'}, headers: { Authorization: `Bearer ${CUSTOM_TOKEN.trim()}`, - 'X-Serverless-Authorization': `Bearer ${ID_TOKEN}`, }, retry: { limit: 5, @@ -186,7 +179,6 @@ describe('System Tests', () => { form: {team: 'DOGS'}, headers: { Authorization: 'Bearer iam-a-token', - 'X-Serverless-Authorization': `Bearer ${ID_TOKEN}`, }, retry: { limit: 5, From 1335ed765d2739ba0378f2b09cda09ed9ad5f371 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Mon, 24 Mar 2025 13:23:55 +1100 Subject: [PATCH 44/44] revert custom audience (not required if not using ID_TOKEN) --- run/idp-sql/test/e2e_test_setup.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/test/e2e_test_setup.yaml index 946d64e2c7..f1d8ad1509 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/test/e2e_test_setup.yaml @@ -62,7 +62,6 @@ steps: --region ${_REGION} \ --service-account ${_SERVICE_ACCOUNT} \ --add-cloudsql-instances ${_CLOUD_SQL_CONNECTION_NAME} \ - --add-custom-audiences="https://action.test/" \ --update-secrets CLOUD_SQL_CREDENTIALS_SECRET=${_SERVICE}-secrets:latest" images: