Create a registry package for products to self-register resources

.ci/containers/build-environment/Dockerfile

@@ -17,7 +17,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 1777 "$GOPATH"
 WORKDIR $GOPATH
 
 # terraform binary used by tfv/tgc
-COPY --from=hashicorp/terraform:1.11.0 /bin/terraform /bin/terraform
+COPY --from=hashicorp/terraform:1.14.3 /bin/terraform /bin/terraform
 
 SHELL ["/bin/bash", "-c"]
 

.ci/containers/go-plus/Dockerfile

@@ -29,7 +29,7 @@ RUN apt-get update && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-RUN wget https://releases.hashicorp.com/terraform/1.11.0/terraform_1.11.0_linux_amd64.zip \
-    && unzip terraform_1.11.0_linux_amd64.zip \
-    && rm terraform_1.11.0_linux_amd64.zip \
+RUN wget https://releases.hashicorp.com/terraform/1.14.3/terraform_1.14.3_linux_amd64.zip \
+    && unzip terraform_1.14.3_linux_amd64.zip \
+    && rm terraform_1.14.3_linux_amd64.zip \
     && mv ./terraform /bin/terraform

.ci/gcb-ingest-test-data.yaml

@@ -0,0 +1,19 @@
+---
+steps:
+    - name: 'gcr.io/graphite-docker-images/go-plus'
+      id: collect-nightly-test-status
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
+      secretEnv: ["TEAMCITY_TOKEN"]
+      args:
+        - 'collect-nightly-test-status'
+        - $_CUSTOM_DATE
+
+timeout: 3600s
+options:
+    machineType: 'N1_HIGHCPU_32'
+
+logsBucket: 'gs://cloudbuild-ingest-test-data-logs'
+availableSecrets:
+  secretManager:
+    - versionName: projects/673497134629/secrets/teamcity-token/versions/latest
+      env: TEAMCITY_TOKEN

.ci/gcb-test-failure-ticket.yaml

@@ -1,21 +1,11 @@
 ---
 steps:
-    - name: 'gcr.io/graphite-docker-images/go-plus'
-      id: collect-nightly-test-status
-      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
-      secretEnv: ["TEAMCITY_TOKEN"]
-      args:
-        - 'collect-nightly-test-status'
-        - $_CUSTOM_DATE
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: create-test-failure-ticket
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       secretEnv: ["GITHUB_TOKEN"]
-      waitFor: ["collect-nightly-test-status"]
       args:
         - 'create-test-failure-ticket'
-    - name: 'ubuntu'
-      args: ['sleep', '120']
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: manage-test-failure-ticket
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
@@ -31,7 +21,5 @@ options:
 logsBucket: 'gs://cloudbuild-test-failure-ticket-logs'
 availableSecrets:
   secretManager:
-    - versionName: projects/673497134629/secrets/teamcity-token/versions/latest
-      env: TEAMCITY_TOKEN
     - versionName: projects/673497134629/secrets/github-classic--repo-workflow/versions/latest
       env: GITHUB_TOKEN

.ci/infra/terraform/main.tf

@@ -295,6 +295,7 @@ module "project-services" {
     "firebaseremoteconfig.googleapis.com",
     "firebaserules.googleapis.com",
     "firebasestorage.googleapis.com",
+    "firebasevertexai.googleapis.com",
     "firestore.googleapis.com",
     "firestorekeyvisualizer.googleapis.com",
     "gkebackup.googleapis.com",

.ci/magician/cmd/collect_nightly_test_status.go

@@ -84,24 +84,27 @@ var collectNightlyTestStatusCmd = &cobra.Command{
 		tc := teamcity.NewClient(env["TEAMCITY_TOKEN"])
 		gcs := cloudstorage.NewClient()
 
-		now := time.Now()
-
 		loc, err := time.LoadLocation("America/Los_Angeles")
 		if err != nil {
 			return fmt.Errorf("Error loading location: %s", err)
 		}
-		date := now.In(loc)
+
+		now := time.Now().In(loc)
+		year, month, day := now.Date()
+
 		customDate := args[0]
 		// check if a specific date is provided
 		if customDate != "" {
 			parsedDate, err := time.Parse("2006-01-02", customDate) // input format YYYY-MM-DD
-			// Set the time to 7pm PT
-			date = time.Date(parsedDate.Year(), parsedDate.Month(), parsedDate.Day(), 19, 0, 0, 0, loc)
 			if err != nil {
 				return fmt.Errorf("invalid input time format: %w", err)
 			}
+			year, month, day = parsedDate.Date()
 		}
 
+		// Set the time to 7pm PT
+		date := time.Date(year, month, day, 19, 0, 0, 0, loc)
+
 		return execCollectNightlyTestStatus(date, tc, gcs)
 	},
 }
@@ -134,10 +137,31 @@ func execCollectNightlyTestStatus(now time.Time, tc TeamcityClient, gcs Cloudsto
 }
 
 func createTestReport(pVersion provider.Version, tc TeamcityClient, gcs CloudstorageClient, formattedStartCut, formattedFinishCut, date string) error {
+
+	// Check Queued Builds
+	queuedBuilds, err := tc.GetBuilds("queued", pVersion.TeamCityNightlyProjectName(), formattedFinishCut, formattedStartCut)
+	if err != nil {
+		return fmt.Errorf("failed to get queued builds: %w", err)
+	}
+	if len(queuedBuilds.Builds) > 0 {
+		fmt.Printf("%s Test unfinished: there are still %d builds queued.\n", strings.ToUpper(pVersion.String()), len(queuedBuilds.Builds))
+		return nil
+	}
+
+	// Check Running Builds
+	runningBuilds, err := tc.GetBuilds("running", pVersion.TeamCityNightlyProjectName(), formattedFinishCut, formattedStartCut)
+	if err != nil {
+		return fmt.Errorf("failed to get running builds: %w", err)
+	}
+	if len(runningBuilds.Builds) > 0 {
+		fmt.Printf("%s Test unfinished: there are still %d builds running.\n", strings.ToUpper(pVersion.String()), len(runningBuilds.Builds))
+		return nil
+	}
+
 	// Get all service test builds
-	builds, err := tc.GetBuilds(pVersion.TeamCityNightlyProjectName(), formattedFinishCut, formattedStartCut)
+	builds, err := tc.GetBuilds("finished", pVersion.TeamCityNightlyProjectName(), formattedFinishCut, formattedStartCut)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to get finished builds: %w", err)
 	}
 
 	var testInfoList []TestInfo

.ci/magician/cmd/interfaces.go

@@ -50,7 +50,7 @@ type CloudstorageClient interface {
 }
 
 type TeamcityClient interface {
-	GetBuilds(project, finishCut, startCut string) (teamcity.Builds, error)
+	GetBuilds(state, project, finishCut, startCut string) (teamcity.Builds, error)
 	GetTestResults(build teamcity.Build) (teamcity.TestResults, error)
 }
 

.ci/magician/cmd/vcr_cassette_update_test.go

@@ -317,7 +317,7 @@ func TestExecVCRCassetteUpdate(t *testing.T) {
 					{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-m", "-q", "cp", "/mock/dir/magic-modules/.ci/magician/cassettes/beta/*", "gs://vcr-nightly/beta/2024-07-08/buildID/main_cassettes_backup/fixtures/"}, map[string]string(nil)},
 					{"/mock/dir/magic-modules/.ci/magician", "git", []string{"clone", "https://hashicorp:token@github.com/hashicorp/terraform-provider-google-beta", "gopath/src/github.com/hashicorp/terraform-provider-google-beta"}, map[string]string(nil)},
 					{"gopath/src/github.com/hashicorp/terraform-provider-google-beta", "go", []string{"list", "./..."}, map[string]string(nil)},
-					{"gopath/src/github.com/hashicorp/terraform-provider-google-beta", "go", []string{"test", "", "-parallel", "32", "-v", "-run=TestAcc", "-timeout", "240m", "-ldflags=-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc", "-vet=off"}, map[string]string{
+					{"gopath/src/github.com/hashicorp/terraform-provider-google-beta", "go", []string{"test", "", "-parallel", "32", "-v", "-run=TestAcc", "-timeout", "360m", "-ldflags=-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc", "-vet=off"}, map[string]string{
 						"ACCTEST_PARALLELISM":            "32",
 						"GOOGLE_APPLICATION_CREDENTIALS": "/mock/dir/magic-modules/.ci/magician/sa_key.json",
 						"GOOGLE_CREDENTIALS":             "sa_key",
@@ -341,8 +341,8 @@ func TestExecVCRCassetteUpdate(t *testing.T) {
 		{
 			name: "replay failed then record",
 			cmdResults: map[string]string{
-				"gopath/src/github.com/hashicorp/terraform-provider-google-beta go [test  -parallel 32 -v -run=TestAcc -timeout 240m -ldflags=-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc -vet=off] map[ACCTEST_PARALLELISM:32 GOOGLE_APPLICATION_CREDENTIALS:/mock/dir/magic-modules/.ci/magician/sa_key.json GOOGLE_CREDENTIALS:sa_key GOOGLE_TEST_DIRECTORY: SA_KEY:sa_key TF_ACC:1 TF_ACC_REFRESH_AFTER_APPLY:1 TF_LOG:DEBUG TF_LOG_CORE:WARN TF_LOG_PATH_MASK:/mock/dir/magic-modules/.ci/magician/testlogs/replaying/beta/%s.log TF_LOG_SDK_FRAMEWORK:INFO TF_SCHEMA_PANIC_ON_ERROR:1 VCR_MODE:REPLAYING VCR_PATH:/mock/dir/magic-modules/.ci/magician/cassettes/beta]":                                           "--- FAIL: TestAccContainerNodePool_defaultDriverInstallation (590.29s)",
-				"gopath/src/github.com/hashicorp/terraform-provider-google-beta go [test  -parallel 1 -v -run=TestAccContainerNodePool_defaultDriverInstallation$ -timeout 240m -ldflags=-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc -vet=off] map[ACCTEST_PARALLELISM:1 GOOGLE_APPLICATION_CREDENTIALS:/mock/dir/magic-modules/.ci/magician/sa_key.json GOOGLE_CREDENTIALS:sa_key GOOGLE_TEST_DIRECTORY: SA_KEY:sa_key TF_ACC:1 TF_ACC_REFRESH_AFTER_APPLY:1 TF_LOG:DEBUG TF_LOG_CORE:WARN TF_LOG_PATH_MASK:/mock/dir/magic-modules/.ci/magician/testlogs/recording/beta/%s.log TF_LOG_SDK_FRAMEWORK:INFO TF_SCHEMA_PANIC_ON_ERROR:1 VCR_MODE:RECORDING VCR_PATH:/mock/dir/magic-modules/.ci/magician/cassettes/beta]": "--- PASS: TestAccContainerNodePool_defaultDriverInstallation (590.29s)",
+				"gopath/src/github.com/hashicorp/terraform-provider-google-beta go [test  -parallel 32 -v -run=TestAcc -timeout 360m -ldflags=-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc -vet=off] map[ACCTEST_PARALLELISM:32 GOOGLE_APPLICATION_CREDENTIALS:/mock/dir/magic-modules/.ci/magician/sa_key.json GOOGLE_CREDENTIALS:sa_key GOOGLE_TEST_DIRECTORY: SA_KEY:sa_key TF_ACC:1 TF_ACC_REFRESH_AFTER_APPLY:1 TF_LOG:DEBUG TF_LOG_CORE:WARN TF_LOG_PATH_MASK:/mock/dir/magic-modules/.ci/magician/testlogs/replaying/beta/%s.log TF_LOG_SDK_FRAMEWORK:INFO TF_SCHEMA_PANIC_ON_ERROR:1 VCR_MODE:REPLAYING VCR_PATH:/mock/dir/magic-modules/.ci/magician/cassettes/beta]":                                           "--- FAIL: TestAccContainerNodePool_defaultDriverInstallation (590.29s)",
+				"gopath/src/github.com/hashicorp/terraform-provider-google-beta go [test  -parallel 1 -v -run=TestAccContainerNodePool_defaultDriverInstallation$ -timeout 360m -ldflags=-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc -vet=off] map[ACCTEST_PARALLELISM:1 GOOGLE_APPLICATION_CREDENTIALS:/mock/dir/magic-modules/.ci/magician/sa_key.json GOOGLE_CREDENTIALS:sa_key GOOGLE_TEST_DIRECTORY: SA_KEY:sa_key TF_ACC:1 TF_ACC_REFRESH_AFTER_APPLY:1 TF_LOG:DEBUG TF_LOG_CORE:WARN TF_LOG_PATH_MASK:/mock/dir/magic-modules/.ci/magician/testlogs/recording/beta/%s.log TF_LOG_SDK_FRAMEWORK:INFO TF_SCHEMA_PANIC_ON_ERROR:1 VCR_MODE:RECORDING VCR_PATH:/mock/dir/magic-modules/.ci/magician/cassettes/beta]": "--- PASS: TestAccContainerNodePool_defaultDriverInstallation (590.29s)",
 			},
 			expectedCalls: map[string][]ParameterList{
 				"Run": {
@@ -351,7 +351,7 @@ func TestExecVCRCassetteUpdate(t *testing.T) {
 					{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-m", "-q", "cp", "/mock/dir/magic-modules/.ci/magician/cassettes/beta/*", "gs://vcr-nightly/beta/2024-07-08/buildID/main_cassettes_backup/fixtures/"}, map[string]string(nil)},
 					{"/mock/dir/magic-modules/.ci/magician", "git", []string{"clone", "https://hashicorp:token@github.com/hashicorp/terraform-provider-google-beta", "gopath/src/github.com/hashicorp/terraform-provider-google-beta"}, map[string]string(nil)},
 					{"gopath/src/github.com/hashicorp/terraform-provider-google-beta", "go", []string{"list", "./..."}, map[string]string(nil)},
-					{"gopath/src/github.com/hashicorp/terraform-provider-google-beta", "go", []string{"test", "", "-parallel", "32", "-v", "-run=TestAcc", "-timeout", "240m", "-ldflags=-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc", "-vet=off"}, map[string]string{
+					{"gopath/src/github.com/hashicorp/terraform-provider-google-beta", "go", []string{"test", "", "-parallel", "32", "-v", "-run=TestAcc", "-timeout", "360m", "-ldflags=-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc", "-vet=off"}, map[string]string{
 						"ACCTEST_PARALLELISM":            "32",
 						"GOOGLE_APPLICATION_CREDENTIALS": "/mock/dir/magic-modules/.ci/magician/sa_key.json",
 						"GOOGLE_CREDENTIALS":             "sa_key",
@@ -371,7 +371,7 @@ func TestExecVCRCassetteUpdate(t *testing.T) {
 					{"/mock/dir/magic-modules/.ci/magician", "gsutil", []string{"-h", "Content-Type:text/plain", "-q", "cp", "-r", "/mock/dir/magic-modules/.ci/magician/testlogs/replaying/beta/*", "gs://vcr-nightly/beta/2024-07-08/buildID/logs/build-log/"}, map[string]string(nil)},
 					// record
 					{"gopath/src/github.com/hashicorp/terraform-provider-google-beta", "go", []string{"list", "./..."}, map[string]string(nil)},
-					{"gopath/src/github.com/hashicorp/terraform-provider-google-beta", "go", []string{"test", "", "-parallel", "1", "-v", "-run=TestAccContainerNodePool_defaultDriverInstallation$", "-timeout", "240m", "-ldflags=-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc", "-vet=off"}, map[string]string{
+					{"gopath/src/github.com/hashicorp/terraform-provider-google-beta", "go", []string{"test", "", "-parallel", "1", "-v", "-run=TestAccContainerNodePool_defaultDriverInstallation$", "-timeout", "360m", "-ldflags=-X=github.com/hashicorp/terraform-provider-google-beta/version.ProviderVersion=acc", "-vet=off"}, map[string]string{
 						"ACCTEST_PARALLELISM":            "1",
 						"GOOGLE_APPLICATION_CREDENTIALS": "/mock/dir/magic-modules/.ci/magician/sa_key.json",
 						"GOOGLE_CREDENTIALS":             "sa_key",

.ci/magician/github/membership_data.go

@@ -134,16 +134,16 @@ var (
 		"shuyama1": {
 			vacations: []Vacation{
 				{
-					startDate: newDate(2025, 11, 25),
-					endDate:   newDate(2025, 12, 3),
+					startDate: newDate(2026, 01, 29),
+					endDate:   newDate(2026, 02, 04),
 				},
 			},
 		},
 		"SirGitsalot": {
 			vacations: []Vacation{
 				{
-					startDate: newDate(2025, 1, 18),
-					endDate:   newDate(2025, 1, 25),
+					startDate: newDate(2026, 1, 18),
+					endDate:   newDate(2026, 1, 27),
 				},
 			},
 		},

.ci/magician/teamcity/get.go

@@ -54,8 +54,8 @@ type FirstFailed struct {
 	Href string `json:"href"`
 }
 
-func (tc *Client) GetBuilds(project, finishCut, startCut string) (Builds, error) {
-	url := fmt.Sprintf("https://hashicorp.teamcity.com/app/rest/builds?locator=count:500,tag:cron-trigger,project:%s,branch:refs/heads/nightly-test,queuedDate:(date:%s,condition:before),queuedDate:(date:%s,condition:after)&fields=build(id,buildTypeId,buildConfName,webUrl,number,queuedDate,startDate,finishDate)", project, finishCut, startCut)
+func (tc *Client) GetBuilds(state, project, finishCut, startCut string) (Builds, error) {
+	url := fmt.Sprintf("https://hashicorp.teamcity.com/app/rest/builds?locator=state:%s,count:500,tag:cron-trigger,project:%s,branch:refs/heads/nightly-test,queuedDate:(date:%s,condition:before),queuedDate:(date:%s,condition:after)&fields=build(id,buildTypeId,buildConfName,webUrl,number,queuedDate,startDate,finishDate)", state, project, finishCut, startCut)
 
 	var builds Builds
 

.ci/magician/vcr/tester.go

@@ -74,13 +74,13 @@ type Tester struct {
 const accTestParallelism = 32
 const parallelJobs = 16
 
-const replayingTimeout = "240m"
+const replayingTimeout = "360m"
 
 var testResultsExpression = regexp.MustCompile(`(?m:^--- (PASS|FAIL|SKIP): (TestAcc\w+))`)
 
 var subtestResultsExpression = regexp.MustCompile(`(?m:^    --- (PASS|FAIL|SKIP): (TestAcc\w+)/(\w+))`)
 
-var testPanicExpression = regexp.MustCompile(`^panic: .*`)
+var testPanicExpression = regexp.MustCompile(`(?m:^panic: .*)`)
 
 var safeToLog = map[string]bool{
 	"ACCTEST_PARALLELISM":                        true,

docs/content/breaking-changes/make-a-breaking-change.md

@@ -121,7 +121,7 @@ The deprecation message will automatically show up in the resource documentation
 2. Update the [documentation for the field]({{< ref "/document/add-documentation" >}}) to include the deprecation notice. For example:
 
    ```markdown
-   * `api_field_name` - (Optional, [Beta](https://terraform.io/docs/providers/google/guides/provider_versions.html), Deprecated) FIELD_DESCRIPTION. `api_field_name` is deprecated and will be removed in a future major release. Use `other_field_name` instead.
+   * `api_field_name` - (Optional, [Beta](../guides/provider_versions.html.markdown), Deprecated) FIELD_DESCRIPTION. `api_field_name` is deprecated and will be removed in a future major release. Use `other_field_name` instead.
    ```
 {{< /tab >}}
 {{% /tabs %}}

docs/content/develop/add-fields.md

@@ -165,16 +165,12 @@ Replace `String` in the field type with one of the following options:
     description: |
       MULTILINE_FIELD_DESCRIPTION
     key_name: 'key_name'
-    key_description: |
-      MULTILINE_KEY_FIELD_DESCRIPTION
   # Map of primitive values
     value_type:
-      name: mapIntegerName
       type: Integer
 
   # Map of complex values
     value_type:
-      name: mapObjectName
       type: NestedObject
       properties:
       - name: 'FIELD_NAME'
@@ -185,7 +181,7 @@ Replace `String` in the field type with one of the following options:
 
 This type is used for general-case string -> non-string type mappings, use "KeyValuePairs" for string -> string mappings. Complex maps can't be represented natively in Terraform, and this type is transformed into an associative array (TypeSet) with the key merged into the object alongside other top-level fields.
 
-For `key_name` and `key_description`, provide a domain-appropriate name and description. For example, a map that references a specific type of resource would generally use the singular resource kind as the key name (such as "topic" for PubSub Topic) and a descriptor of the expected format depending on the context (such as resourceId vs full resource name).
+For `key_name`, provide a domain-appropriate field name. For example, a map that references a specific type of resource would generally use the singular resource kind as the key name (such as "topic" for PubSub Topic).
 
 {{< /tab >}}
 {{< /tabs >}}

docs/content/develop/custom-code.md

@@ -261,7 +261,7 @@ Any fields added in this way will need to be have documentation manually added u
 ```yaml
 docs:
   optional_properties: |
-    * `FIELD_NAME` - (Optional, [Beta](https://terraform.io/docs/providers/google/guides/provider_versions.html)) FIELD_DESCRIPTION
+    * `FIELD_NAME` - (Optional, [Beta](../guides/provider_versions.html.markdown)) FIELD_DESCRIPTION
 ```
 
 See [Add documentation (Handwritten)]({{< ref "/document/add-documentation" >}}) for more information about what to include in the field documentation.

docs/content/develop/promote-to-ga.md

@@ -65,7 +65,7 @@ For handwritten resources, modify the documentation as appropriate for your chan
 3. Add `Beta` as an annotation on any fields or subfields that remained beta-only. For example:
 
    ```markdown
-   * `FIELD_NAME` - (Optional, [Beta](https://terraform.io/docs/providers/google/guides/provider_versions.html)) FIELD_DESCRIPTION
+   * `FIELD_NAME` - (Optional, [Beta](../guides/provider_versions.html.markdown)) FIELD_DESCRIPTION
    ```
 
    Replace `FIELD_NAME` and `FIELD_DESCRIPTION` with the field's name and description.

docs/content/develop/set-up-dev-environment.md

@@ -12,7 +12,7 @@ development environment.
 
 1. [Install the gcloud CLI.](https://cloud.google.com/sdk/docs/install)
 1. In the Google Cloud console, on the project selector page, select or
-   [create a Google Cloud project](https://cloud.google.com/resource-manager/docs/creating-managing-projects).
+   [create a Google Cloud project](https://docs.cloud.google.com/resource-manager/docs/creating-managing-projects).
 
    > [!NOTE]
    > **Note:** If you don't already have a project to use for testing changes to