feat: testing isolation

.github/workflows/config/nodejs-e2e.json

@@ -0,0 +1,5 @@
+{
+  "match": ["run/hello-broken/", "run/helloworld/"],
+  "ignore": ["README.md", "node_modules/"],
+  "package": ["package.json"]
+}

.github/workflows/config/nodejs.json

@@ -0,0 +1,4 @@
+{
+  "ignore": ["README.md", "node_modules/"],
+  "package": ["package.json"]
+}

.github/workflows/experimental.yaml

@@ -0,0 +1,153 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: ⚗️ experimental
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - labeled
+
+jobs:
+  affected:
+    name: Finding affected tests
+    runs-on: ubuntu-latest
+    timeout-minutes: 1
+    outputs:
+      nodejs: ${{ steps.nodejs.outputs.packages }}
+      nodejs-e2e: ${{ steps.nodejs-e2e.outputs.packages }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ^1.22.0
+      - run: go install ./cmd/affected
+        working-directory: .github/workflows/samples-tools
+      - id: nodejs
+        run: |
+          PACKAGES=$(affected .github/workflows/config/nodejs.json)
+          echo "packages=$PACKAGES" | tee -a $GITHUB_OUTPUT
+      - id: nodejs-e2e
+        run: |
+          PACKAGES=$(affected .github/workflows/config/nodejs-e2e.json)
+          echo "packages=$PACKAGES" | tee -a $GITHUB_OUTPUT
+
+  nodejs-lint:
+    name: Node.js lint
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - run: make lint
+      - run: ./.github/workflows/utils/region-tags-tests.sh
+
+  nodejs-test:
+    name: Node.js test
+    needs: affected
+    strategy:
+      matrix:
+        package: ${{ fromJson(needs.affected.outputs.nodejs) }}
+        exclude:
+          - package: '.'
+          - package: ai-platform/snippets # PERMISSION_DENIED: Permission denied: Consumer 'projects/undefined' has been suspended.
+          - package: appengine/analytics # [ERR_REQUIRE_ESM]: require() of ES Module
+          - package: appengine/metadata/flexible # [ERR_REQUIRE_ESM]: require() of ES Module
+          - package: appengine/metadata/standard # [ERR_REQUIRE_ESM]: require() of ES Module
+          - package: automl # FAILED_PRECONDITION: Google Cloud AutoML Natural Language was retired on March 15, 2024. Please migrate to Vertex AI instead
+          - package: cloud-sql/mysql/mysql # Error: expected 200 "OK", got 500 "Internal Server Error"
+          - package: cloud-sql/mysql/mysql2 # Error: Cannot find module './connect-connector-with-iam-authn.js'
+          - package: cloud-sql/postgres/knex # CloudSQLConnectorError: Malformed instance connection name provided: expected format of "PROJECT:REGION:INSTANCE", got undefined
+          - package: cloud-sql/sqlserver/mssql # TypeError: The "config.server" property is required and must be of type string.
+          - package: cloud-sql/sqlserver/tedious # TypeError: The "config.server" property is required and must be of type string.
+          - package: compute # GoogleError: The resource 'projects/long-door-651/zones/us-central1-a/disks/disk-from-pool-name' was not found
+          - package: dataproc # GoogleError: Error submitting create cluster request: Multiple validation errors
+          - package: datastore/functions # [ERR_REQUIRE_ESM]: require() of ES Module
+          - package: dialogflow-cx # NOT_FOUND: com.google.apps.framework.request.NotFoundException: Agent 'undefined' does not exist
+          - package: dlp # [ERR_REQUIRE_ESM]: require() of ES Module
+          - package: document-ai # [ERR_REQUIRE_ESM]: require() of ES Module
+          - package: eventarc/audit-storage # Environment Variable 'SERVICE_NAME' not found
+          - package: eventarc/pubsub # Environment Variable 'SERVICE_NAME' not found
+          - package: functions/billing # Error: Request failed with status code 500
+          - package: functions/concepts # npm error Missing script: "test"
+          - package: functions/firebase # npm error Missing script: "test"
+          - package: functions/helloworld # npm error Missing script: "test"
+          - package: functions/http # npm error Missing script: "test"
+          - package: functions/http/uploadFile # npm error Missing script: "test"
+          - package: functions/imagemagick # Error: A bucket name is needed to use Cloud Storage
+          - package: functions/log # npm error Missing script: "test"
+          - package: functions/ocr/app # Error: Bucket not provided. Make sure you have a "bucket" property in your request
+          - package: functions/pubsub # npm error Missing script: "test"
+          - package: functions/slack # TypeError [ERR_INVALID_ARG_TYPE]: The "key" argument must be of type ... Received undefined
+          - package: functions/v2/imagemagick # Error: A bucket name is needed to use Cloud Storage.
+          - package: generative-ai/snippets # [VertexAI.ClientError]: got status: 403 Forbidden.
+          - package: healthcare/consent # GaxiosError: dataset not initialized
+          - package: healthcare/dicom # GaxiosError: dataset not initialized
+          - package: healthcare/fhir # Error: Cannot find module 'whatwg-url'
+          - package: healthcare/hl7v2 # Error: Cannot find module 'whatwg-url'
+          - package: iam/deny # PERMISSION_DENIED: Permission iam.googleapis.com/denypolicies.create denied on resource cloudresourcemanager.googleapis.com/projects/long-door-651
+          - package: memorystore/redis # npm error Missing script: "test"
+          - package: recaptcha_enterprise/demosite/app # Error: no test specified
+          - package: recaptcha_enterprise/snippets # Cannot use import statement outside a module
+          - package: run/idp-sql # Error: Invalid contents in the credentials file
+          - package: run/markdown-preview/editor # Error: could not create an identity token: Cannot fetch ID token in this environment, use GCE or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to a service account credentials JSON file
+          - package: run/system-package # Error: ENOENT: no such file or directory, access '/usr/bin/dot'
+          - package: scheduler # SyntaxError: Cannot use import statement outside a module
+          - package: storagetransfer # CredentialsError: Missing credentials in config, if using AWS_CONFIG_FILE, set AWS_SDK_LOAD_CONFIG=1
+          - package: video-intelligence # PERMISSION_DENIED: The caller does not have permission
+          - package: vision # REDIS: Error: connect ECONNREFUSED 127.0.0.1:6379
+          - package: workflows # SyntaxError: Cannot use import statement outside a module
+          - package: workflows/quickstart # [ERR_MODULE_NOT_FOUND]: Cannot find package 'ts-node' imported from ...
+      fail-fast: false # TODO: remove this, only for testing!
+    uses: ./.github/workflows/nodejs-test.yaml
+    with:
+      path: ${{ matrix.package }}
+      node-version: 20
+
+  # nodejs-e2e-test:
+  #   name: Node.js e2e-test
+  #   needs: affected
+  #   runs-on: ubuntu-latest
+  #   timeout-minutes: 60
+  #   permissions:
+  #     id-token: write # needed for google-github-actions/auth
+  #   strategy:
+  #     matrix:
+  #       node-version: [18, 22]
+  #       package: ${{ fromJson(needs.affected.outputs.nodejs-e2e) }}
+  #       exclude:
+  #         - package: '.'
+  #   steps:
+  #     - uses: actions/checkout@v4
+  #     - uses: actions/setup-node@v4
+  #       with:
+  #         node-version: ${{ matrix.node-version }}
+  #     - uses: google-github-actions/auth@v2
+  #       with:
+  #         project_id: long-door-651
+  #         workload_identity_provider: projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider
+  #         service_account: [email protected]
+  #         access_token_lifetime: 600s # 10 minutes
+  #     - run: make e2e-test dir=${{ matrix.package }}
+  #       env:
+  #         GOOGLE_SAMPLES_PROJECT: long-door-651

.github/workflows/nodejs-test.yaml

@@ -0,0 +1,47 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+on:
+  workflow_call:
+    inputs:
+      path:
+        required: true
+        type: string
+      node-version:
+        required: true
+        type: number
+
+jobs:
+  test:
+    name: ${{ inputs.node-version }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      id-token: write # needed for google-github-actions/auth
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+      - uses: actions/setup-node@v4
+        with:
+          node-version: ${{ inputs.node-version }}
+      - uses: google-github-actions/auth@v2
+        with:
+          project_id: long-door-651
+          workload_identity_provider: projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider
+          service_account: [email protected]
+          access_token_lifetime: 600s # 10 minutes
+      - run: make test dir=${{ inputs.path }}
+        env:
+          GOOGLE_SAMPLES_PROJECT: long-door-651

.github/workflows/samples-tools/.gitignore

@@ -0,0 +1,25 @@
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+go.work.sum
+
+# env file
+.env

.github/workflows/samples-tools/cmd/affected/main.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"slices"
+
+	"samples-tools/pkg/utils"
+)
+
+func main() {
+	usage := "usage: affected path/to/config.json [head-commit] [main-commit]"
+	configPath := utils.ArgRequired(1, usage)
+	headCommit := utils.ArgWithDefault(2, "HEAD")
+	mainCommit := utils.ArgWithDefault(3, "origin/main")
+
+	config, err := utils.LoadConfig(configPath)
+	if err != nil {
+		panic(err)
+	}
+
+	diffs, err := utils.Diffs(headCommit, mainCommit)
+	if err != nil {
+		panic(err)
+	}
+
+	matrix, err := affected(config, diffs)
+	if err != nil {
+		panic(err)
+	}
+
+	matrixJson, err := json.Marshal(matrix)
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Println(string(matrixJson))
+}
+
+func affected(config utils.Config, diffs []string) ([]string, error) {
+	uniquePackages := make(map[string]bool)
+	for _, diff := range diffs {
+		if !config.Matches(diff) {
+			continue
+		}
+		pkg := config.FindPackage(diff)
+		uniquePackages[pkg] = true
+	}
+
+	changed := make([]string, 0, len(uniquePackages))
+	for pkg := range uniquePackages {
+		changed = append(changed, pkg)
+	}
+
+	if !slices.Contains(changed, ".") {
+		return changed, nil
+	}
+
+	var packages []string
+	err := filepath.WalkDir(".",
+		func(path string, d os.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if path == "." {
+				return nil
+			}
+			if d.IsDir() && config.Matches(path) && config.IsPackageDir(path) {
+				packages = append(packages, path)
+				return nil
+			}
+			return nil
+		})
+	if err != nil {
+		return []string{}, err
+	}
+
+	if len(packages) > 256 {
+		return []string{}, fmt.Errorf("GitHub Actions only supports up to 256 packages, got %d packages", len(packages))
+	}
+	return packages, nil
+}

.github/workflows/samples-tools/go.mod

@@ -0,0 +1,3 @@
+module samples-tools
+
+go 1.22.0

.github/workflows/samples-tools/pkg/utils/args.go

@@ -0,0 +1,37 @@
+package utils
+
+import (
+	"fmt"
+	"os"
+	"strings"
+)
+
+func ArgRequired(index int, errorMessage string) string {
+	if len(os.Args) <= index {
+		panic(errorMessage)
+	}
+	return os.Args[index]
+}
+
+func ArgWithDefault(index int, defaultValue string) string {
+	if len(os.Args) <= index {
+		return defaultValue
+	}
+	return os.Args[index]
+}
+
+func InterpolateArgs(args []string, values []string) []string {
+	var result []string
+	for _, arg := range args {
+		for argIdx, value := range values {
+			arg = strings.ReplaceAll(arg, fmt.Sprintf("$%d", argIdx+1), value)
+		}
+		if arg := strings.TrimSpace(strings.TrimSuffix(arg, "$@")); arg != "" {
+			result = append(result, arg)
+		}
+		if strings.HasSuffix(arg, "$@") {
+			result = append(result, values...)
+		}
+	}
+	return result
+}