From 381e688da632d4047aacf30ed55d99171918af49 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 25 Feb 2025 15:47:57 +1100 Subject: [PATCH 01/11] ci(imagemagick): add test isolation for functions imagemagick --- .github/config/nodejs-prod.jsonc | 4 +--- functions/imagemagick/ci-setup.json | 6 ++++++ functions/v2/imagemagick/ci-setup.json | 6 ++++++ 3 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 functions/imagemagick/ci-setup.json create mode 100644 functions/v2/imagemagick/ci-setup.json diff --git a/.github/config/nodejs-prod.jsonc b/.github/config/nodejs-prod.jsonc index 5f908446a0..1bff3ad34c 100644 --- a/.github/config/nodejs-prod.jsonc +++ b/.github/config/nodejs-prod.jsonc @@ -42,7 +42,7 @@ ".kokoro/", ".prettierignore", ".prettierrc.js", - "cloud-samples-tools", // checked out by GH action in ci-*.yml + "cloud-samples-tools", // checked out by GH action in ci-*.yml "CODEOWNERS", "CODE_OF_CONDUCT.md", "CONTRIBUTING.md", @@ -87,10 +87,8 @@ "eventarc/pubsub", // (untested) Environment Variable 'SERVICE_NAME' not found "functions/billing", // Error: Request failed with status code 500 "functions/http/uploadFile", // npm error Missing script: "test" - "functions/imagemagick", // Error: A bucket name is needed to use Cloud Storage "functions/ocr/app", // Error: Bucket not provided. Make sure you have a "bucket" property in your request "functions/slack", // TypeError [ERR_INVALID_ARG_TYPE]: The "key" argument must be of type ... Received undefined - "functions/v2/imagemagick", // Error: A bucket name is needed to use Cloud Storage. "healthcare/fhir", // Error: Cannot find module 'whatwg-url' "iam/deny", // PERMISSION_DENIED: Permission iam.googleapis.com/denypolicies.create denied on resource cloudresourcemanager.googleapis.com/projects/long-door-651 "recaptcha_enterprise/snippets", // Cannot use import statement outside a module diff --git a/functions/imagemagick/ci-setup.json b/functions/imagemagick/ci-setup.json new file mode 100644 index 0000000000..6b5af2ce84 --- /dev/null +++ b/functions/imagemagick/ci-setup.json @@ -0,0 +1,6 @@ +{ + "env": { + "FUNCTIONS_BUCKET": "nodejs-docs-samples-tests", + "BLURRED_BUCKET_NAME": "nodejs-docs-samples-tests-imagick" + } +} diff --git a/functions/v2/imagemagick/ci-setup.json b/functions/v2/imagemagick/ci-setup.json new file mode 100644 index 0000000000..6b5af2ce84 --- /dev/null +++ b/functions/v2/imagemagick/ci-setup.json @@ -0,0 +1,6 @@ +{ + "env": { + "FUNCTIONS_BUCKET": "nodejs-docs-samples-tests", + "BLURRED_BUCKET_NAME": "nodejs-docs-samples-tests-imagick" + } +} From 8298a4e3ad34b4b7d29072b63ca1c1e6616c4333 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 26 Feb 2025 14:21:52 +1100 Subject: [PATCH 02/11] manual install package --- functions/imagemagick/test/index.test.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/functions/imagemagick/test/index.test.js b/functions/imagemagick/test/index.test.js index 5aa3b10e17..593bee0d07 100644 --- a/functions/imagemagick/test/index.test.js +++ b/functions/imagemagick/test/index.test.js @@ -15,7 +15,7 @@ 'use strict'; const assert = require('assert'); -const {spawn} = require('child_process'); +const {exec, spawn} = require('child_process'); const {Storage} = require('@google-cloud/storage'); const sinon = require('sinon'); const {request} = require('gaxios'); @@ -54,6 +54,11 @@ async function startFF(port) { return {ffProc, ffProcHandler}; } +// ImageMagick is available by default in Cloud Run Functions environments +// https://cloud.google.com/functions/1stgendocs/tutorials/imagemagick-1st-gen.md#importing_dependencies +// Manually install it for testing only. +exec("apt update && apt install imagemagick -y") + describe('functions/imagemagick tests', () => { before(async () => { let exists; From 943558d3d80ada15bba83cfce9035cf1de73787c Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 26 Feb 2025 14:57:16 +1100 Subject: [PATCH 03/11] debug reject failure --- functions/imagemagick/test/index.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/imagemagick/test/index.test.js b/functions/imagemagick/test/index.test.js index 593bee0d07..b41df91ed1 100644 --- a/functions/imagemagick/test/index.test.js +++ b/functions/imagemagick/test/index.test.js @@ -47,7 +47,7 @@ async function startFF(port) { let stderr = ''; ffProc.stdout.on('data', data => (stdout += data)); ffProc.stderr.on('data', data => (stderr += data)); - ffProc.on('error', reject); + ffProc.on('error', c => (c === 0 ? resolve(stdout) : reject(stderr))); ffProc.on('exit', c => (c === 0 ? resolve(stdout) : reject(stderr))); }); await waitPort({host: 'localhost', port}); From a54e7ad9e61f02dd9a7f356016968c7eada20659 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 26 Feb 2025 15:37:19 +1100 Subject: [PATCH 04/11] add update to v2 version of same sample --- functions/v2/imagemagick/test/integration.test.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/functions/v2/imagemagick/test/integration.test.js b/functions/v2/imagemagick/test/integration.test.js index e0a2ac56d2..bfc80c8875 100644 --- a/functions/v2/imagemagick/test/integration.test.js +++ b/functions/v2/imagemagick/test/integration.test.js @@ -33,6 +33,11 @@ const testFiles = { require('../index'); +// ImageMagick is available by default in Cloud Run Functions environments +// https://cloud.google.com/functions/1stgendocs/tutorials/imagemagick-1st-gen.md#importing_dependencies +// Manually install it for testing only. +exec("apt update && apt install imagemagick -y") + describe('functions/imagemagick tests', () => { before(async () => { let exists; From 377edac143b8fd0d933dc7aeda6a690df125d9ac Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 26 Feb 2025 15:42:10 +1100 Subject: [PATCH 05/11] correct import --- functions/v2/imagemagick/test/integration.test.js | 1 + 1 file changed, 1 insertion(+) diff --git a/functions/v2/imagemagick/test/integration.test.js b/functions/v2/imagemagick/test/integration.test.js index bfc80c8875..cc3602060a 100644 --- a/functions/v2/imagemagick/test/integration.test.js +++ b/functions/v2/imagemagick/test/integration.test.js @@ -15,6 +15,7 @@ 'use strict'; const assert = require('assert'); +const {exec} = require('child_process'); const {Storage} = require('@google-cloud/storage'); const sinon = require('sinon'); const supertest = require('supertest'); From 815f025baeec5cae956c73eb51ad7670095646ca Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 26 Feb 2025 16:51:40 +1100 Subject: [PATCH 06/11] sudo, lint --- functions/imagemagick/test/index.test.js | 2 +- functions/v2/imagemagick/test/integration.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/imagemagick/test/index.test.js b/functions/imagemagick/test/index.test.js index b41df91ed1..27976bff68 100644 --- a/functions/imagemagick/test/index.test.js +++ b/functions/imagemagick/test/index.test.js @@ -57,7 +57,7 @@ async function startFF(port) { // ImageMagick is available by default in Cloud Run Functions environments // https://cloud.google.com/functions/1stgendocs/tutorials/imagemagick-1st-gen.md#importing_dependencies // Manually install it for testing only. -exec("apt update && apt install imagemagick -y") +exec('sudo apt-get install imagemagick -y'); describe('functions/imagemagick tests', () => { before(async () => { diff --git a/functions/v2/imagemagick/test/integration.test.js b/functions/v2/imagemagick/test/integration.test.js index cc3602060a..6bb9ebfadb 100644 --- a/functions/v2/imagemagick/test/integration.test.js +++ b/functions/v2/imagemagick/test/integration.test.js @@ -37,7 +37,7 @@ require('../index'); // ImageMagick is available by default in Cloud Run Functions environments // https://cloud.google.com/functions/1stgendocs/tutorials/imagemagick-1st-gen.md#importing_dependencies // Manually install it for testing only. -exec("apt update && apt install imagemagick -y") +exec('sudo apt-get install imagemagick -y'); describe('functions/imagemagick tests', () => { before(async () => { From 1a6ab7317f03ae7009d968300c80d7f1f7fb601d Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Wed, 26 Feb 2025 16:55:19 +1100 Subject: [PATCH 07/11] wait for completion --- functions/imagemagick/test/index.test.js | 4 ++-- functions/v2/imagemagick/test/integration.test.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/functions/imagemagick/test/index.test.js b/functions/imagemagick/test/index.test.js index 27976bff68..9665d878fc 100644 --- a/functions/imagemagick/test/index.test.js +++ b/functions/imagemagick/test/index.test.js @@ -15,7 +15,7 @@ 'use strict'; const assert = require('assert'); -const {exec, spawn} = require('child_process'); +const {execSync, spawn} = require('child_process'); const {Storage} = require('@google-cloud/storage'); const sinon = require('sinon'); const {request} = require('gaxios'); @@ -57,7 +57,7 @@ async function startFF(port) { // ImageMagick is available by default in Cloud Run Functions environments // https://cloud.google.com/functions/1stgendocs/tutorials/imagemagick-1st-gen.md#importing_dependencies // Manually install it for testing only. -exec('sudo apt-get install imagemagick -y'); +execSync('sudo apt-get install imagemagick -y'); describe('functions/imagemagick tests', () => { before(async () => { diff --git a/functions/v2/imagemagick/test/integration.test.js b/functions/v2/imagemagick/test/integration.test.js index 6bb9ebfadb..82f5b8a43e 100644 --- a/functions/v2/imagemagick/test/integration.test.js +++ b/functions/v2/imagemagick/test/integration.test.js @@ -15,7 +15,7 @@ 'use strict'; const assert = require('assert'); -const {exec} = require('child_process'); +const {execSync} = require('child_process'); const {Storage} = require('@google-cloud/storage'); const sinon = require('sinon'); const supertest = require('supertest'); @@ -37,7 +37,7 @@ require('../index'); // ImageMagick is available by default in Cloud Run Functions environments // https://cloud.google.com/functions/1stgendocs/tutorials/imagemagick-1st-gen.md#importing_dependencies // Manually install it for testing only. -exec('sudo apt-get install imagemagick -y'); +execSync('sudo apt-get install imagemagick -y'); describe('functions/imagemagick tests', () => { before(async () => { From cf4b01c3e6ac5fdf6ba781a0c1df54f5e78ab684 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 4 Mar 2025 10:56:55 +1100 Subject: [PATCH 08/11] add tests to dev config --- .github/config/nodejs-dev.jsonc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/config/nodejs-dev.jsonc b/.github/config/nodejs-dev.jsonc index 8766848d7f..8c4d605ef9 100644 --- a/.github/config/nodejs-dev.jsonc +++ b/.github/config/nodejs-dev.jsonc @@ -144,6 +144,7 @@ "functions/http/httpContent", "functions/http/httpMethods", "functions/http/parseXML", + "functions/imagemagick", "functions/log/helloWorld", "functions/log/processEntry", "functions/memorystore/redis", @@ -170,6 +171,7 @@ "functions/v2/helloGCS", "functions/v2/helloPubSub", "functions/v2/httpLogging", + "functions/v2/imagemagick", "functions/v2/log/processEntry", "functions/v2/ocr/app", "functions/v2/responseStreaming", From f1d40f56436f66c2debf3e768c65104e20ffc572 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Tue, 4 Mar 2025 12:54:17 +1100 Subject: [PATCH 09/11] try bump timeout --- functions/imagemagick/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/imagemagick/package.json b/functions/imagemagick/package.json index 016d83e889..cf6fa1cc8b 100644 --- a/functions/imagemagick/package.json +++ b/functions/imagemagick/package.json @@ -12,7 +12,7 @@ "node": ">=12.0.0" }, "scripts": { - "test": "c8 mocha -p -j 2 test/*.test.js --timeout=20000 --exit" + "test": "c8 mocha -p -j 2 test/*.test.js --timeout=30000 --exit" }, "dependencies": { "@google-cloud/storage": "^7.0.0", From 757312b64a3f77d38e620bdca5055127ae90d8aa Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Thu, 6 Mar 2025 16:26:31 +1100 Subject: [PATCH 10/11] adapt ffprochandler from helloGCS test --- functions/imagemagick/test/index.test.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/functions/imagemagick/test/index.test.js b/functions/imagemagick/test/index.test.js index 9665d878fc..55fecb924c 100644 --- a/functions/imagemagick/test/index.test.js +++ b/functions/imagemagick/test/index.test.js @@ -47,8 +47,16 @@ async function startFF(port) { let stderr = ''; ffProc.stdout.on('data', data => (stdout += data)); ffProc.stderr.on('data', data => (stderr += data)); - ffProc.on('error', c => (c === 0 ? resolve(stdout) : reject(stderr))); - ffProc.on('exit', c => (c === 0 ? resolve(stdout) : reject(stderr))); + ffProc.on('exit', code => { + if (code === 0 || code === null) { + // code === null corresponds to a signal-kill + // (which doesn't necessarily indicate a test failure) + resolve(stdout); + } else { + stderr = `Error code: ${code}\n${stderr}`; + reject(new Error(stderr)); + } + }); }); await waitPort({host: 'localhost', port}); return {ffProc, ffProcHandler}; From cf5068dfa7d67f16b2c7fa12730fae114001c3c1 Mon Sep 17 00:00:00 2001 From: Katie McLaughlin Date: Fri, 7 Mar 2025 14:40:09 +1100 Subject: [PATCH 11/11] reduce delta --- .github/config/nodejs-prod.jsonc | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/config/nodejs-prod.jsonc b/.github/config/nodejs-prod.jsonc index 42754bee63..f9ec36469f 100644 --- a/.github/config/nodejs-prod.jsonc +++ b/.github/config/nodejs-prod.jsonc @@ -88,9 +88,8 @@ "dialogflow-cx", // NOT_FOUND: com.google.apps.framework.request.NotFoundException: Agent 'undefined' does not exist "dlp", // [ERR_REQUIRE_ESM]: require() of ES Module "document-ai", // [ERR_REQUIRE_ESM]: require() of ES Module - "functions/billing", // Error: Request failed with status code 500 - "functions/http/uploadFile", // npm error Missing script: "test" - "functions/ocr/app", // Error: Bucket not provided. Make sure you have a "bucket" property in your request + "functions/billing", // (untested) Error: Request failed with status code 500 + "functions/ocr/app", // (untested) Error: Bucket not provided. Make sure you have a "bucket" property in your request "functions/slack", // TypeError [ERR_INVALID_ARG_TYPE]: The "key" argument must be of type ... Received undefined "healthcare/fhir", // Error: Cannot find module 'whatwg-url' "iam/deny", // PERMISSION_DENIED: Permission iam.googleapis.com/denypolicies.create denied on resource cloudresourcemanager.googleapis.com/projects/long-door-651