diff --git a/.github/config/nodejs-dev.jsonc b/.github/config/nodejs-dev.jsonc index 400ee95119..4dbb31c0f5 100644 --- a/.github/config/nodejs-dev.jsonc +++ b/.github/config/nodejs-dev.jsonc @@ -202,6 +202,7 @@ "run/image-processing", "run/jobs", "run/logging-manual", + "run/idp-sql", "run/markdown-preview/editor", "run/markdown-preview/renderer", "run/pubsub", diff --git a/.github/config/nodejs-prod.jsonc b/.github/config/nodejs-prod.jsonc index fdb4d03609..b2a2a4cdf1 100644 --- a/.github/config/nodejs-prod.jsonc +++ b/.github/config/nodejs-prod.jsonc @@ -92,7 +92,6 @@ "functions/slack", // TypeError [ERR_INVALID_ARG_TYPE]: The "key" argument must be of type ... Received undefined "healthcare/fhir", // Error: Cannot find module 'whatwg-url' "iam/deny", // PERMISSION_DENIED: Permission iam.googleapis.com/denypolicies.create denied on resource cloudresourcemanager.googleapis.com/projects/long-door-651 - "run/idp-sql", // (untested) Error: Invalid contents in the credentials file "storagetransfer", // CredentialsError: Missing credentials in config, if using AWS_CONFIG_FILE, set AWS_SDK_LOAD_CONFIG=1 "video-intelligence", // PERMISSION_DENIED: The caller does not have permission "workflows", // SyntaxError: Cannot use import statement outside a module diff --git a/run/idp-sql/ci-setup.json b/run/idp-sql/ci-setup.json new file mode 100644 index 0000000000..a1f34ed1b1 --- /dev/null +++ b/run/idp-sql/ci-setup.json @@ -0,0 +1,12 @@ +{ + "env": { + "SERVICE_NAME": "idp-sql-${RUN_ID}", + "SAMPLE_VERSION": "${RUN_ID}", + "CLOUD_SQL_CONNECTION_NAME": "nodejs-docs-samples-tests:us-central1:test-postgres-instance", + "DB_NAME": "ci-database", + "DB_USER": "ci-user" + }, + "secrets": { + "IDP_KEY": "nodejs-docs-samples-tests/long-door-651-idp-key", + "DB_PASSWORD": "nodejs-docs-samples-tests/nodejs-docs-samples-test-postgres-instance-ci-user-password" } +} diff --git a/run/idp-sql/package.json b/run/idp-sql/package.json index 5f7970a3de..7bdfd66704 100644 --- a/run/idp-sql/package.json +++ b/run/idp-sql/package.json @@ -14,12 +14,14 @@ }, "scripts": { "start": "node index.js", - "test": "c8 mocha -p -j 2 test/app.test.js --timeout=120000 --exit", - "system-test": "c8 mocha -p -j 2 test/system.test.js --timeout=1800000 --exit" + "unit-test": "c8 mocha -p -j 2 test/app.test.js --timeout=120000 --exit", + "system-test": "test/proxy-setup.sh && c8 mocha -p -j 2 test/system.test.js --timeout=1800000 --exit", + "all-test": "npm run unit-test && npm run system-test", + "test": "npm -- run all-test" }, "dependencies": { "express": "^4.16.2", - "firebase-admin": "^12.0.0", + "firebase-admin": "^13.0.0", "gcp-metadata": "^6.0.0", "google-auth-library": "^9.0.0", "handlebars": "^4.7.6", diff --git a/run/idp-sql/test/e2e_test_cleanup.yaml b/run/idp-sql/test/e2e_test_cleanup.yaml index 7521b55ee7..e2c497f6cd 100644 --- a/run/idp-sql/test/e2e_test_cleanup.yaml +++ b/run/idp-sql/test/e2e_test_cleanup.yaml @@ -1,3 +1,18 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + steps: - id: 'Delete resources' @@ -12,11 +27,15 @@ steps: ./test/retry.sh "gcloud container images describe gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" \ "gcloud container images delete gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} --quiet" - ./test/retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM}" \ - "gcloud run services delete ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM} --quiet" + ./test/retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION}" \ + "gcloud run services delete ${_SERVICE} --region ${_REGION} --quiet" substitutions: _SERVICE: idp-sql _VERSION: manual _REGION: us-central1 - _PLATFORM: managed + +serviceAccount: 'projects/${PROJECT_ID}/serviceAccounts/${_SERVICE_ACCOUNT}' +options: + logging: CLOUD_LOGGING_ONLY + dynamicSubstitutions: true diff --git a/run/idp-sql/test/e2e_test_setup.yaml b/run/idp-sql/test/e2e_test_setup.yaml index 3053d19812..f1d8ad1509 100644 --- a/run/idp-sql/test/e2e_test_setup.yaml +++ b/run/idp-sql/test/e2e_test_setup.yaml @@ -1,3 +1,18 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + steps: - id: 'Add a Secret to Secret Manager' @@ -12,9 +27,13 @@ steps: sed -i "s/\"DB_USER\": \"postgres\"/\"DB_USER\": \"${_DB_USER}\"/" postgres-secrets.json ./test/retry.sh "gcloud secrets create ${_SERVICE}-secrets \ - --replication-policy="automatic" \ + --replication-policy=automatic \ --data-file=postgres-secrets.json" + ./test/retry.sh "gcloud secrets add-iam-policy-binding ${_SERVICE}-secrets \ + --member=serviceAccount:${_SERVICE_ACCOUNT} \ + --role=roles/secretmanager.secretAccessor" + - id: 'Build Container Image' name: 'gcr.io/cloud-builders/docker' entrypoint: '/bin/bash' @@ -37,11 +56,11 @@ steps: args: - '-c' - | - ./test/retry.sh "gcloud beta run deploy ${_SERVICE} \ + ./test/retry.sh "gcloud run deploy ${_SERVICE} \ --image gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} \ --allow-unauthenticated \ --region ${_REGION} \ - --platform ${_PLATFORM} \ + --service-account ${_SERVICE_ACCOUNT} \ --add-cloudsql-instances ${_CLOUD_SQL_CONNECTION_NAME} \ --update-secrets CLOUD_SQL_CREDENTIALS_SECRET=${_SERVICE}-secrets:latest" @@ -52,8 +71,13 @@ substitutions: _SERVICE: idp-sql _VERSION: manual _REGION: us-central1 - _PLATFORM: managed _CLOUD_SQL_CONNECTION_NAME: $PROJECT_ID:us-central1:idp-sql-instance _DB_NAME: postgres _DB_USER: postgres _DB_PASSWORD: password1234 + _SERVICE_ACCOUNT: ${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com + +serviceAccount: 'projects/${PROJECT_ID}/serviceAccounts/${_SERVICE_ACCOUNT}' +options: + logging: CLOUD_LOGGING_ONLY + dynamicSubstitutions: true diff --git a/run/idp-sql/test/proxy-setup.sh b/run/idp-sql/test/proxy-setup.sh new file mode 100755 index 0000000000..c1006a8e9a --- /dev/null +++ b/run/idp-sql/test/proxy-setup.sh @@ -0,0 +1,19 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Proof of concept: setting up proxy + +curl -o cloud-sql-proxy https://storage.googleapis.com/cloud-sql-connectors/cloud-sql-proxy/v2.15.1/cloud-sql-proxy.linux.amd64 +chmod +x cloud-sql-proxy +cloud-sql-proxy -dir=/cloudsql -instances=$INSTANCE_CONNECTION_NAME & diff --git a/run/idp-sql/test/retry.sh b/run/idp-sql/test/retry.sh index 78385733f0..0f36c2075a 100755 --- a/run/idp-sql/test/retry.sh +++ b/run/idp-sql/test/retry.sh @@ -59,7 +59,7 @@ do if ((attempt_num==max_attempts)) then echo "Attempt $attempt_num / $max_attempts failed! No more retries left!" - exit + exit 1 else echo "Attempt $attempt_num / $max_attempts failed!" sleep $((attempt_num++)) diff --git a/run/idp-sql/test/system.test.js b/run/idp-sql/test/system.test.js index 343718467c..7b4b57b1d2 100644 --- a/run/idp-sql/test/system.test.js +++ b/run/idp-sql/test/system.test.js @@ -32,6 +32,8 @@ describe('System Tests', () => { console.log('"SERVICE_NAME" env var not found. Defaulting to "idp-sql"'); SERVICE_NAME = 'idp-sql'; } + + const {SERVICE_ACCOUNT} = process.env; const {SAMPLE_VERSION} = process.env; const PLATFORM = 'managed'; const REGION = 'us-central1'; @@ -41,6 +43,9 @@ describe('System Tests', () => { if (!CLOUD_SQL_CONNECTION_NAME) { throw Error('"CLOUD_SQL_CONNECTION_NAME" env var not found.'); } + + const {DB_NAME} = process.env; + const {DB_USER} = process.env; const {DB_PASSWORD} = process.env; if (!DB_PASSWORD) { throw Error('"DB_PASSWORD" env var not found.'); @@ -52,15 +57,19 @@ describe('System Tests', () => { throw Error('"IDP_KEY" env var not found.'); } - let BASE_URL, ID_TOKEN; + let BASE_URL, CUSTOM_TOKEN; before(async () => { // Deploy service using Cloud Build let buildCmd = `gcloud builds submit --project ${GOOGLE_CLOUD_PROJECT} ` + '--config ./test/e2e_test_setup.yaml ' + - `--substitutions _SERVICE=${SERVICE_NAME},_PLATFORM=${PLATFORM},_REGION=${REGION}` + + `--substitutions _SERVICE=${SERVICE_NAME},_REGION=${REGION}` + `,_DB_PASSWORD=${DB_PASSWORD},_CLOUD_SQL_CONNECTION_NAME=${CLOUD_SQL_CONNECTION_NAME}`; + + if (SERVICE_ACCOUNT) buildCmd += `,_SERVICE_ACCOUNT=${SERVICE_ACCOUNT}`; if (SAMPLE_VERSION) buildCmd += `,_VERSION=${SAMPLE_VERSION}`; + if (DB_USER) buildCmd += `,_DB_USER=${DB_USER}`; + if (DB_NAME) buildCmd += `,_DB_NAME=${DB_NAME}`; console.log('Starting Cloud Build...'); execSync(buildCmd, {timeout: 240000, shell: true}); // timeout at 4 mins @@ -82,34 +91,43 @@ describe('System Tests', () => { } // Retrieve ID token for testing + + console.log('Retrieving IDP token...'); const customToken = await admin.auth().createCustomToken('a-user-id'); - const response = await got( - `https://identitytoolkit.googleapis.com/v1/accounts:signInWithCustomToken?key=${IDP_KEY}`, - { - method: 'POST', - retry: { - limit: 5, - statusCodes: [404, 401, 403, 500], - methods: ['POST'], - }, - body: JSON.stringify({ - token: customToken, - returnSecureToken: true, - }), - } - ); + try { + const response = await got( + `https://identitytoolkit.googleapis.com/v1/accounts:signInWithCustomToken?key=${IDP_KEY}`, + { + method: 'POST', + retry: { + limit: 5, + statusCodes: [404, 401, 403, 500], + methods: ['POST'], + }, + body: JSON.stringify({ + token: customToken, + returnSecureToken: true, + }), + } + ); + const tokens = JSON.parse(response.body); + CUSTOM_TOKEN = tokens.idToken; + } catch (err) { + throw Error('IDP Token retrieval failed: ', err.response.body); + } + + if (!CUSTOM_TOKEN) throw Error('Unable to acquire an IDP token.'); - const tokens = JSON.parse(response.body); - ID_TOKEN = tokens.idToken; - if (!ID_TOKEN) throw Error('Unable to acquire an ID token.'); + console.log('Retrieved IDP token'); }); after(() => { let cleanUpCmd = `gcloud builds submit --project ${GOOGLE_CLOUD_PROJECT} ` + '--config ./test/e2e_test_cleanup.yaml ' + - `--substitutions _SERVICE=${SERVICE_NAME},_PLATFORM=${PLATFORM},_REGION=${REGION}`; + `--substitutions _SERVICE=${SERVICE_NAME},_REGION=${REGION}`; if (SAMPLE_VERSION) cleanUpCmd += `,_VERSION=${SAMPLE_VERSION}`; + if (SERVICE_ACCOUNT) cleanUpCmd += `,_SERVICE_ACCOUNT=${SERVICE_ACCOUNT}`; execSync(cleanUpCmd, {shell: true}); }); @@ -128,14 +146,14 @@ describe('System Tests', () => { }); it('Can make a POST request with token', async () => { - assert(ID_TOKEN && ID_TOKEN.length > 0); + assert(CUSTOM_TOKEN && CUSTOM_TOKEN.length > 0); const options = { prefixUrl: BASE_URL.trim(), method: 'POST', form: {team: 'DOGS'}, headers: { - Authorization: `Bearer ${ID_TOKEN.trim()}`, + Authorization: `Bearer ${CUSTOM_TOKEN.trim()}`, }, retry: { limit: 5,