Merge branch 'main' into ariwete-run-idp-sql

glasnt · web-flow · commit 2cb4f044265f · 2025-03-04T16:15:14.000+11:00
diff --git a/.github/config/nodejs-dev.jsonc b/.github/config/nodejs-dev.jsonc
@@ -85,6 +85,8 @@
     "appengine/hello-world/flexible_nodejs16_and_earlier",
     "appengine/hello-world/standard",
     "appengine/memcached",
+    "appengine/analytics",
+    "appengine/metadata/flexible",
     "appengine/metadata/flexible_nodejs16_and_earlier",
     "appengine/metadata/standard",
     "appengine/pubsub",
@@ -197,6 +199,7 @@
     "run/jobs",
     "run/logging-manual",
     "run/idp-sql",
+    "run/markdown-preview/editor",
     "run/markdown-preview/renderer",
     "run/pubsub",
     "run/system-package",
diff --git a/.github/config/nodejs-prod.jsonc b/.github/config/nodejs-prod.jsonc
@@ -76,8 +76,6 @@
 
     // TODO: fix these
     "ai-platform/snippets", // PERMISSION_DENIED: Permission denied: Consumer 'projects/undefined' has been suspended.
-    "appengine/analytics", // [ERR_REQUIRE_ESM]: require() of ES Module
-    "appengine/metadata/flexible", // [ERR_REQUIRE_ESM]: require() of ES Module
     "automl", // (untested) FAILED_PRECONDITION: Google Cloud AutoML Natural Language was retired on March 15, 2024. Please migrate to Vertex AI instead
     "cloud-sql/mysql/mysql", // (untested) Error: expected 200 "OK", got 500 "Internal Server Error"
     "cloud-sql/mysql/mysql2", // (untested) Error: Cannot find module './connect-connector-with-iam-authn.js'
@@ -98,7 +96,6 @@
     "healthcare/fhir", // Error: Cannot find module 'whatwg-url'
     "iam/deny", // PERMISSION_DENIED: Permission iam.googleapis.com/denypolicies.create denied on resource cloudresourcemanager.googleapis.com/projects/long-door-651
     "recaptcha_enterprise/snippets", // Cannot use import statement outside a module
-    "run/markdown-preview/editor", // (untested) Error: could not create an identity token: Cannot fetch ID token in this environment, use GCE or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to a service account credentials JSON file
     "scheduler", // SyntaxError: Cannot use import statement outside a module
     "storagetransfer", // CredentialsError: Missing credentials in config, if using AWS_CONFIG_FILE, set AWS_SDK_LOAD_CONFIG=1
     "video-intelligence", // PERMISSION_DENIED: The caller does not have permission
diff --git a/appengine/analytics/app.js b/appengine/analytics/app.js
@@ -15,8 +15,8 @@
 'use strict';
 
 // [START gae_flex_analytics_track_event]
-const express = require('express');
-const fetch = require('node-fetch');
+import express from 'express';
+import fetch from 'node-fetch';
 
 const app = express();
 app.enable('trust proxy');
@@ -77,4 +77,4 @@ app.listen(PORT, () => {
 
 // [END gae_flex_analytics_track_event]
 
-module.exports = app;
+export default app;
diff --git a/appengine/analytics/package.json b/appengine/analytics/package.json
@@ -9,6 +9,7 @@
     "type": "git",
     "url": "https://github.com/GoogleCloudPlatform/nodejs-docs-samples.git"
   },
+  "type": "module",
   "engines": {
     "node": "20.x"
   },
diff --git a/appengine/analytics/test/app.test.js b/appengine/analytics/test/app.test.js
@@ -1,6 +1,19 @@
-const supertest = require('supertest');
-const path = require('path');
-const app = require(path.join(__dirname, '../', 'app.js'));
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import supertest from 'supertest';
+import app from '../app.js';
 
 describe('gae_flex_analytics_track_event', () => {
   it('should be listening', async () => {
diff --git a/appengine/metadata/flexible/package.json b/appengine/metadata/flexible/package.json
@@ -9,6 +9,7 @@
     "type": "git",
     "url": "https://github.com/GoogleCloudPlatform/nodejs-docs-samples.git"
   },
+  "type": "module",
   "engines": {
     "node": ">= 18.x.x"
   },
diff --git a/appengine/metadata/flexible/server.js b/appengine/metadata/flexible/server.js
@@ -15,8 +15,8 @@
 'use strict';
 
 // [START gae_flex_metadata]
-const express = require('express');
-const fetch = require('node-fetch');
+import express from 'express';
+import fetch from 'node-fetch';
 
 const app = express();
 app.enable('trust proxy');
@@ -58,4 +58,4 @@ app.listen(PORT, () => {
   console.log('Press Ctrl+C to quit.');
 });
 // [END gae_flex_metadata]
-module.exports = app;
+export default app;
diff --git a/appengine/metadata/flexible/test/server.test.js b/appengine/metadata/flexible/test/server.test.js
@@ -1,6 +1,19 @@
-const path = require('path');
-const app = require(path.join(__dirname, '../', 'server.js'));
-const supertest = require('supertest');
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+import supertest from 'supertest';
+import app from '../server.js';
 
 describe('gae_flex_metadata', () => {
   it('should be listening', async () => {
diff --git a/run/markdown-preview/editor/ci-setup.json b/run/markdown-preview/editor/ci-setup.json
@@ -0,0 +1,6 @@
+{
+    "env": {
+        "SERVICE_NAME": "markdown-renderer-$RUN_ID",
+        "SAMPLE_VERSION": "$RUN_ID"
+    }
+}
diff --git a/run/markdown-preview/editor/package.json b/run/markdown-preview/editor/package.json
@@ -15,7 +15,9 @@
   "main": "main.js",
   "scripts": {
     "start": "node index.js",
-    "test": "c8 mocha -p -j 2 test/app.test.js --timeout=10000 --exit",
+    "test": "npm -- run all-test",
+    "all-test": "npm run unit-test && npm run system-test",
+    "unit-test": "c8 mocha -p -j 2 test/app.test.js --timeout=10000 --exit",
     "system-test": "c8 mocha -p -j 2 test/system.test.js --timeout=480000 --exit"
   },
   "dependencies": {
diff --git a/run/markdown-preview/editor/render.js b/run/markdown-preview/editor/render.js
@@ -37,13 +37,23 @@ const renderRequest = async markdown => {
   };
 
   try {
-    // Create a Google Auth client with the Renderer service url as the target audience.
-    if (!client) client = await auth.getIdTokenClient(serviceUrl);
-    // Fetch the client request headers and add them to the service request headers.
-    // The client request headers include an ID token that authenticates the request.
-    const clientHeaders = await client.getRequestHeaders();
-    serviceRequestOptions.headers['Authorization'] =
-      clientHeaders['Authorization'];
+    // [END cloudrun_secure_request]
+    // If we're in the test environment, use the envvar instead
+    if (process.env.ID_TOKEN) {
+      serviceRequestOptions.headers['Authorization'] =
+        'Bearer ' + process.env.ID_TOKEN;
+    } else {
+      // [START cloudrun_secure_request]
+      // Create a Google Auth client with the Renderer service url as the target audience.
+      if (!client) client = await auth.getIdTokenClient(serviceUrl);
+      // Fetch the client request headers and add them to the service request headers.
+      // The client request headers include an ID token that authenticates the request.
+      const clientHeaders = await client.getRequestHeaders();
+      serviceRequestOptions.headers['Authorization'] =
+        clientHeaders['Authorization'];
+      // [END cloudrun_secure_request]
+    }
+    // [START cloudrun_secure_request]
   } catch (err) {
     throw Error('could not create an identity token: ' + err.message);
   }
diff --git a/run/markdown-preview/editor/test/app.test.js b/run/markdown-preview/editor/test/app.test.js
@@ -56,7 +56,9 @@ describe('Integration tests', () => {
       await request.get('/render').retry(3).expect(404);
     });
 
-    it('responds 200 OK on "POST /render" with valid JSON', async () => {
+    // SKIP: this test is trying to call out to the RENDER_URL, which does not
+    // accept POST requests. TODO: setup correct mocking/workaround.
+    it.skip('responds 200 OK on "POST /render" with valid JSON', async () => {
       // A valid type will make a request to the /render endpoint.
       // TODO: This test outputs a JSON parsing SyntaxError from supertest but does not fail the assert.
       await request
diff --git a/run/markdown-preview/editor/test/e2e_test_cleanup.yaml b/run/markdown-preview/editor/test/e2e_test_cleanup.yaml
@@ -1,3 +1,18 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
 steps:
 
 - id: 'Delete resources'
@@ -9,17 +24,22 @@ steps:
       ./test/retry.sh "gcloud container images describe gcr.io/${PROJECT_ID}/renderer-${_SERVICE}:${_VERSION}" \
         "gcloud container images delete gcr.io/${PROJECT_ID}/renderer-${_SERVICE}:${_VERSION} --quiet"
 
-      ./test/retry.sh "gcloud run services describe renderer-${_SERVICE} --region ${_REGION} --platform ${_PLATFORM}" \
-        "gcloud run services delete renderer-${_SERVICE} --region ${_REGION} --platform ${_PLATFORM} --quiet"
+      ./test/retry.sh "gcloud run services describe renderer-${_SERVICE} --region ${_REGION}" \
+        "gcloud run services delete renderer-${_SERVICE} --region ${_REGION}  --quiet"
 
       ./test/retry.sh "gcloud container images describe gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}" \
         "gcloud container images delete gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} --quiet"
 
-      ./test/retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM}" \
-        "gcloud run services delete ${_SERVICE} --region ${_REGION} --platform ${_PLATFORM} --quiet"
+      ./test/retry.sh "gcloud run services describe ${_SERVICE} --region ${_REGION}" \
+        "gcloud run services delete ${_SERVICE} --region ${_REGION}  --quiet"
 
 substitutions:
   _SERVICE: editor
   _VERSION: manual
   _REGION: us-central1
-  _PLATFORM: managed
+  _SERVICE_ACCOUNT: ${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com
+
+serviceAccount: 'projects/${PROJECT_ID}/serviceAccounts/${_SERVICE_ACCOUNT}'
+options:
+    logging: CLOUD_LOGGING_ONLY
+    dynamicSubstitutions: true
diff --git a/run/markdown-preview/editor/test/e2e_test_setup.yaml b/run/markdown-preview/editor/test/e2e_test_setup.yaml
@@ -1,3 +1,18 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
 steps:
 
 - id: 'Build and Push Container Image - Editor'
@@ -17,15 +32,15 @@ steps:
   - |
     echo $(gcloud run services describe renderer-${_SERVICE} \
       --region ${_REGION} \
-      --format='value(status.url)' \
-      --platform ${_PLATFORM}) > _service_url
+      --format='value(status.url)') > _service_url
 
     ./test/retry.sh "gcloud run deploy ${_SERVICE} \
       --image gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION} \
       --no-allow-unauthenticated \
       --region ${_REGION} \
-      --platform ${_PLATFORM} \
-      --set-env-vars EDITOR_UPSTREAM_RENDER_URL=$(cat _service_url)"
+      --set-env-vars EDITOR_UPSTREAM_RENDER_URL=$(cat _service_url) \
+      --add-custom-audiences "https://action.test/" \
+      --service-account ${_SERVICE_ACCOUNT}"
 
 images:
 - gcr.io/${PROJECT_ID}/${_SERVICE}:${_VERSION}
@@ -34,4 +49,9 @@ substitutions:
   _SERVICE: editor
   _VERSION: manual
   _REGION: us-central1
-  _PLATFORM: managed
+  _SERVICE_ACCOUNT: ${PROJECT_NUMBER}@cloudbuild.gserviceaccount.com
+
+serviceAccount: 'projects/${PROJECT_ID}/serviceAccounts/${_SERVICE_ACCOUNT}'
+options:
+    logging: CLOUD_LOGGING_ONLY
+    dynamicSubstitutions: true
diff --git a/run/markdown-preview/editor/test/system.test.js b/run/markdown-preview/editor/test/system.test.js
@@ -15,8 +15,6 @@
 const assert = require('assert');
 const got = require('got');
 const {execSync} = require('child_process');
-const {GoogleAuth} = require('google-auth-library');
-const auth = new GoogleAuth();
 
 describe('End-to-End Tests', () => {
   // Retrieve Cloud Run service test config
@@ -29,18 +27,21 @@ describe('End-to-End Tests', () => {
     console.log('"SERVICE_NAME" env var not found. Defaulting to "editor"');
     SERVICE_NAME = 'editor';
   }
+  const {ID_TOKEN} = process.env;
+  if (!ID_TOKEN) throw Error('ID token not in envvar');
   const {SAMPLE_VERSION} = process.env;
-  const PLATFORM = 'managed';
+  const {SERVICE_ACCOUNT} = process.env;
   const REGION = 'us-central1';
 
-  let BASE_URL, ID_TOKEN;
+  let BASE_URL;
   before(async () => {
     // Deploy Renderer service
     let buildRendererCmd =
       `gcloud builds submit --project ${GOOGLE_CLOUD_PROJECT} ` +
       '--config ../renderer/test/e2e_test_setup.yaml ' +
-      `--substitutions _SERVICE=renderer-${SERVICE_NAME},_PLATFORM=${PLATFORM},_REGION=${REGION}`;
+      `--substitutions _SERVICE=renderer-${SERVICE_NAME},_REGION=${REGION}`;
     if (SAMPLE_VERSION) buildRendererCmd += `,_VERSION=${SAMPLE_VERSION}`;
+    if (SERVICE_ACCOUNT) buildRendererCmd += `,_SERVICE_ACCOUNT=${SERVICE_ACCOUNT}`;
 
     console.log('Starting Cloud Build for Renderer service...');
     execSync(buildRendererCmd, {cwd: '../renderer'});
@@ -50,8 +51,9 @@ describe('End-to-End Tests', () => {
     let buildCmd =
       `gcloud builds submit --project ${GOOGLE_CLOUD_PROJECT} ` +
       '--config ./test/e2e_test_setup.yaml ' +
-      `--substitutions _SERVICE=${SERVICE_NAME},_PLATFORM=${PLATFORM},_REGION=${REGION}`;
+      `--substitutions _SERVICE=${SERVICE_NAME},_REGION=${REGION}`;
     if (SAMPLE_VERSION) buildCmd += `,_VERSION=${SAMPLE_VERSION}`;
+    if (SERVICE_ACCOUNT) buildCmd += `,_SERVICE_ACCOUNT=${SERVICE_ACCOUNT}`;
 
     console.log('Starting Cloud Build for Editor service...');
     execSync(buildCmd, {timeout: 240000}); // timeout at 4 mins
@@ -60,23 +62,19 @@ describe('End-to-End Tests', () => {
     // Retrieve URL of Cloud Run service
     const url = execSync(
       `gcloud run services describe ${SERVICE_NAME} --project=${GOOGLE_CLOUD_PROJECT} ` +
-        `--platform=${PLATFORM} --region=${REGION} --format='value(status.url)'`
+        `--region=${REGION} --format='value(status.url)'`
     );
     BASE_URL = url.toString('utf-8').trim();
     if (!BASE_URL) throw Error('Cloud Run service URL not found');
-
-    const client = await auth.getIdTokenClient(BASE_URL);
-    const clientHeaders = await client.getRequestHeaders();
-    ID_TOKEN = clientHeaders['Authorization'];
-    if (!ID_TOKEN) throw Error('ID token could not be created');
   });
 
   after(() => {
     let cleanUpCmd =
       `gcloud builds submit --project ${GOOGLE_CLOUD_PROJECT} ` +
       '--config ./test/e2e_test_cleanup.yaml ' +
-      `--substitutions _SERVICE=${SERVICE_NAME},_PLATFORM=${PLATFORM},_REGION=${REGION}`;
+      `--substitutions _SERVICE=${SERVICE_NAME},_REGION=${REGION}`;
     if (SAMPLE_VERSION) cleanUpCmd += `,_VERSION=${SAMPLE_VERSION}`;
+    if (SERVICE_ACCOUNT) cleanUpCmd += `,_SERVICE_ACCOUNT=${SERVICE_ACCOUNT}`;
 
     execSync(cleanUpCmd);
   });
@@ -85,7 +83,7 @@ describe('End-to-End Tests', () => {
     const options = {
       prefixUrl: BASE_URL.trim(),
       headers: {
-        Authorization: ID_TOKEN.trim(),
+        Authorization: `Bearer ${ID_TOKEN.trim()}`,
       },
       retry: 3,
     };
@@ -97,7 +95,7 @@ describe('End-to-End Tests', () => {
     const options = {
       prefixUrl: BASE_URL.trim(),
       headers: {
-        Authorization: ID_TOKEN.trim(),
+        Authorization: `Bearer ${ID_TOKEN.trim()}`,
         'Content-Type': 'application/json',
       },
       method: 'POST',
diff --git a/run/markdown-preview/renderer/ci-setup.json b/run/markdown-preview/renderer/ci-setup.json
@@ -0,0 +1,6 @@
+{
+    "env": {
+        "SERVICE_NAME": "markdown-renderer-$RUN_ID",
+        "SAMPLE_VERSION": "$RUN_ID"
+    }
+}
diff --git a/run/markdown-preview/renderer/package.json b/run/markdown-preview/renderer/package.json
@@ -14,7 +14,9 @@
   "main": "index.js",
   "scripts": {
     "start": "node index.js",
-    "test": "c8 mocha -p -j 2 test/app.test.js --exit",
+    "test": "npm -- run all-test",
+    "all-test": "npm run unit-test && npm run system-test",
+    "unit-test": "c8 mocha -p -j 2 test/app.test.js --exit",
     "system-test": "c8 mocha -p -j 2 test/system.test.js --timeout=360000 --exit"
   },
   "dependencies": {
diff --git a/run/markdown-preview/renderer/test/e2e_test_cleanup.yaml b/run/markdown-preview/renderer/test/e2e_test_cleanup.yaml
diff --git a/run/markdown-preview/renderer/test/e2e_test_setup.yaml b/run/markdown-preview/renderer/test/e2e_test_setup.yaml
diff --git a/run/markdown-preview/renderer/test/system.test.js b/run/markdown-preview/renderer/test/system.test.js
