diff --git a/.github/cloud-samples-tools/.gitignore b/.github/cloud-samples-tools/.gitignore new file mode 100644 index 0000000000..6f72f89261 --- /dev/null +++ b/.github/cloud-samples-tools/.gitignore @@ -0,0 +1,25 @@ +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work +go.work.sum + +# env file +.env diff --git a/.github/cloud-samples-tools/README.md b/.github/cloud-samples-tools/README.md new file mode 100644 index 0000000000..b3b247923c --- /dev/null +++ b/.github/cloud-samples-tools/README.md @@ -0,0 +1,80 @@ +# Cloud Samples tools + +This is a collection of tools used for Cloud Samples maintenance and infrastructure. + +This tool has one function: + +- `affected` finds the affected packages given a list of diffs. + +## Config files + +For this tools, we refer to a **package** as an isolated directory, which contains a "package file". +For example, `package.json` in Node.js, `requirements.txt` in Python, `go.mod` in Go, or `pom.xml` in Java. + +Each language has different configurations. +We define them in config files in the repository, this way the tooling keeps language agnostic and each repository can have different configurations. + +The config file can be a `.json` file, or a `.jsonc` (JSON with comments) file. +For `.jsonc` files, it supports both `// single line comments` and `/* multi-line comments */`. + +For example: + +```jsonc +{ + // The package file where the tests should be run (required). + "package-file": "package.json", + + // Match diffs only on .js and .ts files + // Defaults to match all files. + "match": ["*.js", "*.ts"], + + // Ignore diffs on the README, text files, and anything under node_modules/. + // Defaults to not ignore anything. + "ignore": ["README.md", "*.txt", "node_modules/"], + + // Skip these packages, these could be handled by a different config. + // Defaults to not exclude anything. + "exclude-packages": ["path/to/slow-to-test", "special-config-package"], +} +``` + +For more information, see [`pkg/utils/config.go`](pkg/utils/config.go). + +## Building + +To build the tools, we must change to the directory where the tools package is defined. +We can run it in a subshell using parentheses to keep our working directory from changing. + +```sh +(cd .github/workflows/samples-tools && go build -o /tmp/tools ./cmd/*) +``` + +## Running the tools unit tests + +To the tools tests, we must change to the directory where the tools package is defined. +We can run it in a subshell using parentheses to keep our working directory from changing. + +```sh +(cd .github/workflows/samples-tools && go test ./...) +``` + +## Finding affected packages + +> This must run at the repository root directory. + +First, generate a file with all the diffs. +This file should be one file per line. + +You can use `git diff` to test on files that have changed in your branch. +You can also create the file manually if you want to test something without commiting changes to your branch. + +```sh +git --no-pager diff --name-only HEAD origin/main | tee /tmp/diffs.txt +``` + +Now we can check which packages have been affected. +We pass the config file and the diffs file as positional arguments. + +```sh +/tmp/tools affected .github/config/nodejs.jsonc /tmp/diffs.txt +``` diff --git a/.github/cloud-samples-tools/cmd/main.go b/.github/cloud-samples-tools/cmd/main.go new file mode 100644 index 0000000000..b79fdd2fab --- /dev/null +++ b/.github/cloud-samples-tools/cmd/main.go @@ -0,0 +1,96 @@ +/* + Copyright 2024 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + c "cloud-samples-tools/pkg/config" + "encoding/json" + "flag" + "fmt" + "log" + "os" + "strings" +) + +var usage = `usage: tools ... + +commands: + affected path/to/config.jsonc path/to/diffs.txt + run-all path/to/config.jsonc path/to/script.sh +` + +// Entry point to validate command line arguments. +func main() { + flag.Parse() + + command := flag.Arg(0) + if command == "" { + log.Fatalln("❌ no command specified\n", usage) + } + + switch command { + case "affected": + configFile := flag.Arg(1) + if configFile == "" { + log.Fatalln("❌ no config file specified\n", usage) + } + + diffsFile := flag.Arg(2) + if diffsFile == "" { + log.Fatalln("❌ no diffs file specified\n", usage) + } + + affectedCmd(configFile, diffsFile) + + default: + log.Fatalln("❌ unknown command: ", command, "\n", usage) + } +} + +// affected command entry point to validate inputs. +func affectedCmd(configFile string, diffsFile string) { + config, err := c.LoadConfig(configFile) + if err != nil { + log.Fatalln("❌ error loading the config file: ", configFile, "\n", err) + } + + diffsBytes, err := os.ReadFile(diffsFile) + if err != nil { + log.Fatalln("❌ error getting the diffs: ", diffsFile, "\n", err) + } + diffs := strings.Split(string(diffsBytes), "\n") + + packages, err := config.Affected(diffs) + if err != nil { + log.Fatalln("❌ error finding the affected packages.\n", err) + } + if len(packages) > 256 { + log.Fatalln( + "❌ Error: GitHub Actions only supports up to 256 packages, got ", + len(packages), + " packages, for more details see:\n", + "https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/running-variations-of-jobs-in-a-workflow", + ) + } + + packagesJson, err := json.Marshal(packages) + if err != nil { + log.Fatalln("❌ error marshaling packages to JSON.\n", err) + } + + fmt.Println(string(packagesJson)) +} diff --git a/.github/cloud-samples-tools/go.mod b/.github/cloud-samples-tools/go.mod new file mode 100644 index 0000000000..4375f164d4 --- /dev/null +++ b/.github/cloud-samples-tools/go.mod @@ -0,0 +1,3 @@ +module cloud-samples-tools + +go 1.22.0 diff --git a/.github/cloud-samples-tools/pkg/config/config.go b/.github/cloud-samples-tools/pkg/config/config.go new file mode 100644 index 0000000000..b3f959d78f --- /dev/null +++ b/.github/cloud-samples-tools/pkg/config/config.go @@ -0,0 +1,190 @@ +/* + Copyright 2024 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +import ( + "encoding/json" + "errors" + "io/fs" + "os" + "path/filepath" + "regexp" + "slices" + "strings" +) + +type Config struct { + // Filename to look for the root of a package. + PackageFile []string `json:"package-file"` + + // Pattern to match filenames or directories. + Match []string `json:"match"` + + // Pattern to ignore filenames or directories. + Ignore []string `json:"ignore"` + + // Packages to always exclude. + ExcludePackages []string `json:"exclude-packages"` +} + +var multiLineCommentsRegex = regexp.MustCompile(`(?s)\s*/\*.*?\*/`) +var singleLineCommentsRegex = regexp.MustCompile(`\s*//.*\s*`) + +// Saves the config to the given file. +func (c *Config) Save(file *os.File) error { + bytes, err := json.MarshalIndent(c, "", " ") + if err != nil { + return err + } + _, err = file.Write(bytes) + if err != nil { + return err + } + return nil +} + +// LoadConfig loads the config from the given path. +func LoadConfig(path string) (*Config, error) { + // Read the JSONC file. + sourceJsonc, err := os.ReadFile(path) + if err != nil { + return nil, err + } + + // Strip the comments and load the JSON. + sourceJson := multiLineCommentsRegex.ReplaceAll(sourceJsonc, []byte{}) + sourceJson = singleLineCommentsRegex.ReplaceAll(sourceJson, []byte{}) + + var config Config + err = json.Unmarshal(sourceJson, &config) + if err != nil { + return nil, err + } + + // Set default values if they are not set. + if config.PackageFile == nil { + return nil, errors.New("package-file is required") + } + if config.Match == nil { + config.Match = []string{"*"} + } + + return &config, nil +} + +// Match returns true if the path matches any of the patterns. +func Match(patterns []string, path string) bool { + filename := filepath.Base(path) + for _, pattern := range patterns { + if match, _ := filepath.Match(pattern, filename); match { + return true + } + if strings.Contains(path, pattern) { + return true + } + } + return false +} + +// Matches returns true if the path matches the config. +func (c *Config) Matches(path string) bool { + return Match(c.Match, path) && !Match(c.Ignore, path) +} + +// IsPackageDir returns true if the path is a package directory. +func (c *Config) IsPackageDir(dir string) bool { + for _, filename := range c.PackageFile { + packageFile := filepath.Join(dir, filename) + if fileExists(packageFile) { + return true + } + } + return false +} + +// FindPackage returns the package name for the given path. +func (c *Config) FindPackage(path string) string { + dir := filepath.Dir(path) + if dir == "." || c.IsPackageDir(dir) { + return dir + } + return c.FindPackage(dir) +} + +// FindAllPackages finds all the packages in the given root directory. +func (c *Config) FindAllPackages(root string) ([]string, error) { + var packages []string + err := fs.WalkDir(os.DirFS(root), ".", + func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if path == "." { + return nil + } + if slices.Contains(c.ExcludePackages, path) { + return nil + } + if d.IsDir() && c.Matches(path) && c.IsPackageDir(path) { + packages = append(packages, path) + return nil + } + return nil + }) + if err != nil { + return []string{}, err + } + return packages, nil +} + +// Affected returns the packages that have been affected from diffs. +// If there are diffs on at leat one global file affecting all packages, +// then this returns all packages matched by the config. +func (c *Config) Affected(diffs []string) ([]string, error) { + changed := c.Changed(diffs) + if slices.Contains(changed, ".") { + return c.FindAllPackages(".") + } + return changed, nil +} + +// Changed returns the packages that have changed. +// It only returns packages that are matched by the config, +// and are not excluded by the config. +func (c *Config) Changed(diffs []string) []string { + changedUnique := make(map[string]bool) + for _, diff := range diffs { + if !c.Matches(diff) { + continue + } + pkg := c.FindPackage(diff) + if slices.Contains(c.ExcludePackages, pkg) { + continue + } + changedUnique[pkg] = true + } + + if len(changedUnique) == 0 { + return []string{"."} + } + + changed := make([]string, 0, len(changedUnique)) + for pkg := range changedUnique { + changed = append(changed, pkg) + } + return changed +} diff --git a/.github/cloud-samples-tools/pkg/config/config_test.go b/.github/cloud-samples-tools/pkg/config/config_test.go new file mode 100644 index 0000000000..fc3cdd92e7 --- /dev/null +++ b/.github/cloud-samples-tools/pkg/config/config_test.go @@ -0,0 +1,198 @@ +package config_test + +import ( + c "cloud-samples-tools/pkg/config" + "os" + "path/filepath" + "reflect" + "testing" +) + +func TestLoadConfig(t *testing.T) { + tests := []struct { + filename string + config *c.Config + fails bool + }{ + { + filename: "empty.json", + fails: true, + }, + { + filename: "default-values.json", + config: &c.Config{ + PackageFile: []string{"package.json"}, + Match: []string{"*"}, + }, + }, + { + filename: "comments.jsonc", + config: &c.Config{ + PackageFile: []string{"package.json"}, + Match: []string{"*"}, + }, + }, + } + + for _, test := range tests { + path := filepath.Join("testdata", "config", test.filename) + got, err := c.LoadConfig(path) + if test.fails && err == nil { + t.Fatal("expected failure\n", got) + } + if !test.fails && err != nil { + t.Fatal("error loading config\n", err) + } + if !reflect.DeepEqual(test.config, got) { + t.Fatal("expected equal\n", test.config, got) + } + } +} + +func TestSaveLoadConfig(t *testing.T) { + file, err := os.CreateTemp("", "config-*.json") + if err != nil { + t.Fatal("error creating temp file\n", err) + } + defer os.Remove(file.Name()) + + config := c.Config{ + PackageFile: []string{"package.json"}, + Ignore: []string{"node_modules/", "*.md"}, + Match: []string{"*.js"}, + ExcludePackages: []string{"excluded"}, + } + err = config.Save(file) + if err != nil { + t.Fatal("error saving config\n", err) + } + + err = file.Close() + if err != nil { + t.Fatal("error closing file\n", err) + } + + loadedConfig, err := c.LoadConfig(file.Name()) + if err != nil { + t.Fatal("error loading config\n", err) + } + + if !reflect.DeepEqual(&config, loadedConfig) { + t.Fatal("expected equal\n", &config, loadedConfig) + } +} + +func TestMatch(t *testing.T) { + tests := []struct { + patterns []string + path string + expected bool + }{ + { + patterns: []string{}, + path: "path/to/file.js", + expected: false, + }, + { + patterns: []string{"*.js"}, + path: "path/to/file.js", + expected: true, + }, + { + patterns: []string{"path/to/"}, + path: "path/to/file.js", + expected: true, + }, + } + + for _, test := range tests { + got := c.Match(test.patterns, test.path) + if got != test.expected { + t.Fatal("expected equal\n", test.expected, got) + } + } +} + +func TestIsPackage(t *testing.T) { + config := c.Config{PackageFile: []string{"package.json"}} + tests := []struct { + path string + expected bool + }{ + { + path: filepath.Join("testdata", "path-does-not-exist"), + expected: false, + }, + { + path: filepath.Join("testdata", "my-package"), + expected: true, + }, + } + + for _, test := range tests { + got := config.IsPackageDir(test.path) + if test.expected != got { + t.Fatal("expected equal\n", test.expected, got) + } + } +} + +func TestFindPackage(t *testing.T) { + config := c.Config{PackageFile: []string{"package.json"}} + tests := []struct { + path string + expected string + }{ + { + path: filepath.Join("testdata", "my-file.txt"), + expected: ".", + }, + { + path: filepath.Join("testdata", "my-package", "my-file.txt"), + expected: filepath.Join("testdata", "my-package"), + }, + { + path: filepath.Join("testdata", "my-package", "subpackage", "my-file.txt"), + expected: filepath.Join("testdata", "my-package", "subpackage"), + }, + } + + for _, test := range tests { + got := config.FindPackage(test.path) + if test.expected != got { + t.Fatal("expected equal\n", test.expected, got) + } + } +} + +func TestChanged(t *testing.T) { + config := c.Config{ + PackageFile: []string{"package.json"}, + Match: []string{"*"}, + } + + tests := []struct { + diffs []string + expected []string + }{ + { + diffs: []string{filepath.Join("testdata", "file.txt")}, + expected: []string{"."}, + }, + { + diffs: []string{filepath.Join("testdata", "my-package", "file.txt")}, + expected: []string{filepath.Join("testdata", "my-package")}, + }, + { + diffs: []string{filepath.Join("testdata", "my-package", "subpackage", "file.txt")}, + expected: []string{filepath.Join("testdata", "my-package", "subpackage")}, + }, + } + + for _, test := range tests { + got := config.Changed(test.diffs) + if !reflect.DeepEqual(test.expected, got) { + t.Fatal("expected equal\n", test.expected, got) + } + } +} diff --git a/.github/cloud-samples-tools/pkg/config/testdata/config/comments.jsonc b/.github/cloud-samples-tools/pkg/config/testdata/config/comments.jsonc new file mode 100644 index 0000000000..193ff7e88d --- /dev/null +++ b/.github/cloud-samples-tools/pkg/config/testdata/config/comments.jsonc @@ -0,0 +1,8 @@ +/* + * Multi-line + * comment + */ +{ + // Single line comment. + "package-file": /* inline comment */ ["package.json"] // trailing comment +} diff --git a/.github/cloud-samples-tools/pkg/config/testdata/config/default-values.json b/.github/cloud-samples-tools/pkg/config/testdata/config/default-values.json new file mode 100644 index 0000000000..fd5eb9b8ea --- /dev/null +++ b/.github/cloud-samples-tools/pkg/config/testdata/config/default-values.json @@ -0,0 +1,3 @@ +{ + "package-file": ["package.json"] +} diff --git a/.github/cloud-samples-tools/pkg/config/testdata/config/empty.json b/.github/cloud-samples-tools/pkg/config/testdata/config/empty.json new file mode 100644 index 0000000000..0967ef424b --- /dev/null +++ b/.github/cloud-samples-tools/pkg/config/testdata/config/empty.json @@ -0,0 +1 @@ +{} diff --git a/.github/cloud-samples-tools/pkg/config/testdata/my-package/package.json b/.github/cloud-samples-tools/pkg/config/testdata/my-package/package.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/cloud-samples-tools/pkg/config/testdata/my-package/subpackage/package.json b/.github/cloud-samples-tools/pkg/config/testdata/my-package/subpackage/package.json new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.github/cloud-samples-tools/pkg/config/utils.go b/.github/cloud-samples-tools/pkg/config/utils.go new file mode 100644 index 0000000000..b227f9cd99 --- /dev/null +++ b/.github/cloud-samples-tools/pkg/config/utils.go @@ -0,0 +1,30 @@ +/* + Copyright 2024 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package config + +import ( + "errors" + "os" +) + +// fileExists returns true if the file exists. +func fileExists(path string) bool { + if _, err := os.Stat(path); errors.Is(err, os.ErrNotExist) { + return false + } + return true +} diff --git a/.github/config/nodejs.jsonc b/.github/config/nodejs.jsonc new file mode 100644 index 0000000000..47d2dd6fe8 --- /dev/null +++ b/.github/config/nodejs.jsonc @@ -0,0 +1,104 @@ +/* + Copyright 2024 Google LLC + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +{ + "package-file": ["package.json"], + "ignore": [ + ".eslintignore", + ".eslintrc.json", + ".github/.OwlBot.lock.yaml", + ".github/.OwlBot.yaml", + ".github/ISSUE_TEMPLATE/", + ".github/PULL_REQUEST_TEMPLATE.md", + ".github/auto-label.yaml", + ".github/blunderbuss.yaml", + ".github/cloud-samples-tools/", + ".github/flakybot.yaml", + ".github/header-checker-lint.yaml", + ".github/snippet-bot.yml", + ".github/trusted-contribution.yml", + ".gitignore", + ".kokoro/", + ".prettierignore", + ".prettierrc.js", + "CODEOWNERS", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "Makefile", + "README.md", + "SECURITY.md", + "buildsetup.sh", + "linkinator.config.json", + "node_modules/", + "owlbot.py", + "renovate.json" + ], + "exclude-packages": [ + "ai-platform/snippets", // PERMISSION_DENIED: Permission denied: Consumer 'projects/undefined' has been suspended. + "appengine/analytics", // [ERR_REQUIRE_ESM]: require() of ES Module + "appengine/metadata/flexible", // [ERR_REQUIRE_ESM]: require() of ES Module + "appengine/metadata/standard", // [ERR_REQUIRE_ESM]: require() of ES Module + "automl", // FAILED_PRECONDITION: Google Cloud AutoML Natural Language was retired on March 15, 2024. Please migrate to Vertex AI instead + "cloud-sql/mysql/mysql", // Error: expected 200 "OK", got 500 "Internal Server Error" + "cloud-sql/mysql/mysql2", // Error: Cannot find module './connect-connector-with-iam-authn.js' + "cloud-sql/postgres/knex", // CloudSQLConnectorError: Malformed instance connection name provided: expected format of "PROJECT:REGION:INSTANCE", got undefined + "cloud-sql/sqlserver/mssql", // TypeError: The "config.server" property is required and must be of type string. + "cloud-sql/sqlserver/tedious", // TypeError: The "config.server" property is required and must be of type string. + "compute", // GoogleError: The resource 'projects/long-door-651/zones/us-central1-a/disks/disk-from-pool-name' was not found + "dataproc", // GoogleError: Error submitting create cluster request: Multiple validation errors + "datastore/functions", // [ERR_REQUIRE_ESM]: require() of ES Module + "dialogflow-cx", // NOT_FOUND: com.google.apps.framework.request.NotFoundException: Agent 'undefined' does not exist + "dlp", // [ERR_REQUIRE_ESM]: require() of ES Module + "document-ai", // [ERR_REQUIRE_ESM]: require() of ES Module + "eventarc/audit-storage", // Environment Variable 'SERVICE_NAME' not found + "eventarc/pubsub", // Environment Variable 'SERVICE_NAME' not found + "functions/billing", // Error: Request failed with status code 500 + "functions/concepts", // npm error Missing script: "test" + "functions/firebase", // npm error Missing script: "test" + "functions/helloworld", // npm error Missing script: "test" + "functions/http", // npm error Missing script: "test" + "functions/http/uploadFile", // npm error Missing script: "test" + "functions/imagemagick", // Error: A bucket name is needed to use Cloud Storage + "functions/log", // npm error Missing script: "test" + "functions/ocr/app", // Error: Bucket not provided. Make sure you have a "bucket" property in your request + "functions/pubsub", // npm error Missing script: "test" + "functions/slack", // TypeError [ERR_INVALID_ARG_TYPE]: The "key" argument must be of type ... Received undefined + "functions/v2/imagemagick", // Error: A bucket name is needed to use Cloud Storage. + "generative-ai/snippets", // [VertexAI.ClientError]: got status: 403 Forbidden. + "healthcare/consent", // GaxiosError: dataset not initialized + "healthcare/dicom", // GaxiosError: dataset not initialized + "healthcare/fhir", // Error: Cannot find module 'whatwg-url' + "healthcare/hl7v2", // Error: Cannot find module 'whatwg-url' + "iam/deny", // PERMISSION_DENIED: Permission iam.googleapis.com/denypolicies.create denied on resource cloudresourcemanager.googleapis.com/projects/long-door-651 + "memorystore/redis", // npm error Missing script: "test" + "recaptcha_enterprise/demosite/app", // Error: no test specified + "recaptcha_enterprise/snippets", // Cannot use import statement outside a module + "run/idp-sql", // Error: Invalid contents in the credentials file + "run/markdown-preview/editor", // Error: could not create an identity token: Cannot fetch ID token in this environment, use GCE or set the GOOGLE_APPLICATION_CREDENTIALS environment variable to a service account credentials JSON file + "run/system-package", // Error: ENOENT: no such file or directory, access '/usr/bin/dot' + "scheduler", // SyntaxError: Cannot use import statement outside a module + "security-center/snippets", // Error: 3 INVALID_ARGUMENT: Fail to resolve resource 'organizations/undefined/locations/global' + "speech", // AssertionError: expected 'Transcription: Okay, I\'m here.\n Hi…' to match /Terrific. It's on the way./ + "storagetransfer", // CredentialsError: Missing credentials in config, if using AWS_CONFIG_FILE, set AWS_SDK_LOAD_CONFIG=1 + "talent", // AssertionError: expected '' to match /Job summary/ + "translate", // AssertionError: expected 'Languages:\n{ code: \'ab\', name: \'A…' to match /{ code: 'af', name: 'afrikáans' }/ + "video-intelligence", // PERMISSION_DENIED: The caller does not have permission + "vision", // REDIS: Error: connect ECONNREFUSED 127.0.0.1:6379 + "workflows", // SyntaxError: Cannot use import statement outside a module + "workflows/quickstart" // [ERR_MODULE_NOT_FOUND]: Cannot find package 'ts-node' imported from ... + ] +} diff --git a/.github/scripts/nodejs-test.sh b/.github/scripts/nodejs-test.sh new file mode 100644 index 0000000000..0a4322a668 --- /dev/null +++ b/.github/scripts/nodejs-test.sh @@ -0,0 +1,36 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e # exit on error +set -x # print commands as they are executed + +package=$1 + +if [ -z $package ]; then + echo "a package to test is required" + exit 1 +fi + +if [ -z $GOOGLE_SAMPLES_PROJECT ]; then + echo "GOOGLE_SAMPLES_PROJECT environment variable is required" + exit 1 +fi + +# Install the package dependencies. +cd $package +npm install || true +npm run build --if-present + +# Run the tests. +npm test diff --git a/.github/workflows/cloud-samples-tools-test.yaml b/.github/workflows/cloud-samples-tools-test.yaml new file mode 100644 index 0000000000..b6f9508c53 --- /dev/null +++ b/.github/workflows/cloud-samples-tools-test.yaml @@ -0,0 +1,60 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Cloud Samples Tools +on: + push: + branches: + - main + paths: + - .github/cloud-samples-tools/** + pull_request: + paths: + - .github/cloud-samples-tools/** + +env: + GO_VERSION: ^1.22.0 + +jobs: + lint: + name: Lint + runs-on: ubuntu-latest + timeout-minutes: 5 + defaults: + run: + working-directory: .github/cloud-samples-tools + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + - name: Go vet + run: go vet ./... + - name: Check format + run: test -z $(gofmt -l .) + + test: + name: Test + runs-on: ubuntu-latest + timeout-minutes: 5 + defaults: + run: + working-directory: .github/cloud-samples-tools + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + - name: Run tests + run: go test ./... diff --git a/.github/workflows/experimental.yaml b/.github/workflows/experimental.yaml new file mode 100644 index 0000000000..5484a90463 --- /dev/null +++ b/.github/workflows/experimental.yaml @@ -0,0 +1,90 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: ⚗️ experimental +on: + push: + branches: + - main + pull_request: + # schedule: + # # https://crontab.guru/#0_12_*_*_0 + # - cron: 0 12 * * 0 # At 12:00 on Sunday + +env: + GO_VERSION: ^1.22.0 + +jobs: + affected: + name: Finding affected tests + runs-on: ubuntu-latest + timeout-minutes: 2 + outputs: + nodejs: ${{ steps.nodejs.outputs.packages }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + - run: go build -o ${{ github.workspace }}/tools ./cmd/... + working-directory: .github/cloud-samples-tools + - name: Get diffs + run: git --no-pager diff --name-only HEAD origin/main | tee diffs.txt + - name: Find Node.js affected packages + id: nodejs + run: echo "packages=$(./tools affected .github/config/nodejs.jsonc diffs.txt)" | tee -a $GITHUB_OUTPUT + + nodejs-lint: + name: Node.js lint + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 20 + - run: make lint + - run: ./.github/workflows/utils/region-tags-tests.sh + + nodejs-test: + name: Node.js 20 test + needs: affected + runs-on: ubuntu-latest + permissions: + id-token: write # needed for google-github-actions/auth + strategy: + fail-fast: false + matrix: + package: ${{ fromJson(github.event_name == 'pull_request' && needs.affected.outputs.nodejs || '[]') }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - uses: actions/setup-node@v4 + with: + node-version: 20 + - uses: google-github-actions/auth@v2 + with: + project_id: long-door-651 + workload_identity_provider: projects/1046198160504/locations/global/workloadIdentityPools/github-actions-pool/providers/github-actions-provider + service_account: kokoro-system-test@long-door-651.iam.gserviceaccount.com + access_token_lifetime: 600s # 10 minutes + - name: Test ${{ matrix.package }} + run: | + npm install + bash .github/scripts/nodejs-test.sh ${{ matrix.package }} + env: + GOOGLE_SAMPLES_PROJECT: long-door-651 diff --git a/Makefile b/Makefile index bb15fcb493..fd98ddc7d9 100644 --- a/Makefile +++ b/Makefile @@ -22,7 +22,7 @@ test: check-env build cd ${dir} npm test -lint: +lint: build cd ${dir} npx gts fix npx gts lint