teamcity: add release_tests subproject (#14628) (#23714)

[upstream:ffd69b0e2cabca605cd233a0be544cc44181c6ad]

Signed-off-by: Modular Magician <[email protected]>

Author: modular-magician

.teamcity/components/builds/build_configuration_per_package.kt

@@ -20,15 +20,15 @@ import replaceCharsId
 
 // BuildConfigurationsForPackages accepts a map containing details of multiple packages in a provider and returns a list of build configurations for them all.
 // Intended to be used in projects where we're testing all packages, e.g. the nightly test projects
-fun BuildConfigurationsForPackages(packages: Map<String, Map<String, String>>, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List<String>, environmentVariables: AccTestConfiguration, testPrefix: String = "TestAcc"): List<BuildType> {
+fun BuildConfigurationsForPackages(packages: Map<String, Map<String, String>>, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List<String>, environmentVariables: AccTestConfiguration, testPrefix: String = "TestAcc", releaseDiffTest: String = "false"): List<BuildType> {
     val list = ArrayList<BuildType>()
 
     // Create build configurations for all packages, except sweeper
     packages.forEach { (packageName, info) ->
         val path: String = info.getValue("path").toString()
         val displayName: String = info.getValue("displayName").toString()
 
-        val pkg = PackageDetails(packageName, displayName, providerName, parentProjectName)
+        val pkg = PackageDetails(packageName, displayName, providerName, parentProjectName, releaseDiffTest)
         val buildConfig = pkg.buildConfiguration(path, vcsRoot, sharedResources, environmentVariables, testPrefix = testPrefix)
         list.add(buildConfig)
     }
@@ -38,12 +38,12 @@ fun BuildConfigurationsForPackages(packages: Map<String, Map<String, String>>, p
 
 // BuildConfigurationForSinglePackage accepts details of a single package in a provider and returns a build configuration for it
 // Intended to be used in short-lived projects where we're testing specific packages, e.g. feature branch testing
-fun BuildConfigurationForSinglePackage(packageName: String, packagePath: String, packageDisplayName: String, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List<String>, environmentVariables: AccTestConfiguration, testPrefix: String = "TestAcc"): BuildType{
-    val pkg = PackageDetails(packageName, packageDisplayName, providerName, parentProjectName)
+fun BuildConfigurationForSinglePackage(packageName: String, packagePath: String, packageDisplayName: String, providerName: String, parentProjectName: String, vcsRoot: GitVcsRoot, sharedResources: List<String>, environmentVariables: AccTestConfiguration, testPrefix: String = "TestAcc", releaseDiffTest: String = "false"): BuildType{
+    val pkg = PackageDetails(packageName, packageDisplayName, providerName, parentProjectName, releaseDiffTest)
     return pkg.buildConfiguration(packagePath, vcsRoot, sharedResources, environmentVariables, testPrefix = testPrefix)
 }
 
-class PackageDetails(private val packageName: String, private val displayName: String, private val providerName: String, private val parentProjectName: String) {
+class PackageDetails(private val packageName: String, private val displayName: String, private val providerName: String, private val parentProjectName: String, private val releaseDiffTest: String) {
 
     // buildConfiguration returns a BuildType for a service package
     // For BuildType docs, see https://teamcity.jetbrains.com/app/dsl-documentation/root/build-type/index.html
@@ -91,7 +91,7 @@ class PackageDetails(private val packageName: String, private val displayName: S
 
             params {
                 configureGoogleSpecificTestParameters(environmentVariables)
-                acceptanceTestBuildParams(parallelism, testPrefix, testTimeout)
+                acceptanceTestBuildParams(parallelism, testPrefix, testTimeout, releaseDiffTest)
                 terraformLoggingParameters(environmentVariables, providerName)
                 terraformCoreBinaryTesting()
                 terraformShouldPanicForSchemaErrors()
@@ -124,4 +124,4 @@ class PackageDetails(private val packageName: String, private val displayName: S
         var id = "%s_%s_PACKAGE_%s".format(this.parentProjectName, this.providerName, this.packageName)
         return replaceCharsId(id)
     }
-}
+}

.teamcity/components/builds/build_configuration_sweepers.kt

@@ -61,6 +61,7 @@ class SweeperDetails(private val sweeperName: String, private val parentProjectN
         // These hardcoded values affect the sweeper CLI command's behaviour
         val testPrefix = "TestAcc"
         val testTimeout = "12"
+        val releaseDiffTest = "false"
 
         return BuildType {
 
@@ -97,7 +98,7 @@ class SweeperDetails(private val sweeperName: String, private val parentProjectN
 
             params {
                 configureGoogleSpecificTestParameters(environmentVariables)
-                acceptanceTestBuildParams(parallelism, testPrefix, testTimeout)
+                acceptanceTestBuildParams(parallelism, testPrefix, testTimeout, releaseDiffTest)
                 sweeperParameters(sweeperRegions, sweeperRun)
                 terraformLoggingParameters(environmentVariables, providerName)
                 terraformCoreBinaryTesting()
@@ -131,4 +132,4 @@ class SweeperDetails(private val sweeperName: String, private val parentProjectN
         var id = "%s_%s".format(this.parentProjectName, this.sweeperName)
         return replaceCharsId(id)
     }
-}
+}

.teamcity/components/builds/build_configuration_vcr_recording.kt

@@ -27,6 +27,7 @@ class VcrDetails(private val providerName: String, private val buildId: String,
         val testTimeout = "12"
         val parallelism = DefaultParallelism
         val buildTimeout: Int = DefaultBuildTimeoutDuration
+        val releaseDiffTest = "false"
 
         // Path is just ./google(-beta) here, whereas nightly test builds use paths like ./google/something/specific
         // This helps VCR testing builds to run tests across multiple packages
@@ -70,7 +71,7 @@ class VcrDetails(private val providerName: String, private val buildId: String,
             params {
                 configureGoogleSpecificTestParameters(environmentVariables)
                 vcrEnvironmentVariables(environmentVariables, providerName)
-                acceptanceTestBuildParams(parallelism, testPrefix, testTimeout)
+                acceptanceTestBuildParams(parallelism, testPrefix, testTimeout, releaseDiffTest)
                 terraformLoggingParameters(environmentVariables, providerName)
                 terraformCoreBinaryTesting()
                 terraformShouldPanicForSchemaErrors()

.teamcity/components/builds/build_parameters.kt

@@ -205,11 +205,12 @@ fun ParametrizedWithType.configureGoogleSpecificTestParameters(config: AccTestCo
 
 // ParametrizedWithType.acceptanceTestBuildParams sets build params that affect how commands to run
 //  acceptance tests are templated
-fun ParametrizedWithType.acceptanceTestBuildParams(parallelism: Int, prefix: String, timeout: String) {
+fun ParametrizedWithType.acceptanceTestBuildParams(parallelism: Int, prefix: String, timeout: String, releaseDiffTest: String) {
     hiddenVariable("env.TF_ACC", "1", "Set to a value to run the Acceptance Tests")
     text("PARALLELISM", "%d".format(parallelism))
     text("TEST_PREFIX", prefix)
     text("TIMEOUT", timeout)
+    text("RELEASE_DIFF", "true")
 }
 
 // ParametrizedWithType.sweeperParameters sets build parameters that affect how sweepers are run
@@ -301,4 +302,4 @@ fun ParametrizedWithType.hiddenVariable(name: String, value: String, description
 
 fun ParametrizedWithType.hiddenPasswordVariable(name: String, value: String, description: String) {
     password(name, value, "", description, ParameterDisplay.HIDDEN)
-}
+}

.teamcity/components/constants.kt

@@ -8,6 +8,7 @@
 // Provider name that matches the name in the Registry
 const val ProviderNameGa = "google"
 const val ProviderNameBeta = "google-beta"
+const val ProviderNameBetaDiffTest = "google-beta-diff-test"
 
 // specifies the default hour (UTC) at which tests should be triggered, if enabled
 const val DefaultStartHour = 4
@@ -42,10 +43,11 @@ const val ServiceSweeperCronName = "$ServiceSweeperName - Cron"
 const val ServiceSweeperManualName = "$ServiceSweeperName - Manual"
 const val ProjectSweeperName = "Project Sweeper"
 const val NightlyTestsProjectId = "NightlyTests"
+const val WeeklyDiffTestsProjectId = "WeeklyDiffTests"
 const val MMUpstreamProjectId = "MMUpstreamTests"
 const val VcrRecordingProjectId = "VCRRecording"
 
 // Artifact rules controls which artifacts are uploaded to S3
 // https://www.jetbrains.com/help/teamcity/2024.07/configuring-general-settings.html#Artifact+Paths
 // The value below lacks a file extension, to allow upload of individual .txt files or a single .tar.gz file
-const val ArtifactRules = "%teamcity.build.checkoutDir%/debug*"
+const val ArtifactRules = "%teamcity.build.checkoutDir%/debug*"

.teamcity/components/projects/google_beta_subproject.kt

@@ -12,6 +12,7 @@ import builds.*
 import jetbrains.buildServer.configs.kotlin.Project
 import projects.reused.mmUpstream
 import projects.reused.nightlyTests
+import projects.reused.weeklyDiffTests
 import projects.reused.vcrRecording
 import replaceCharsId
 import vcs_roots.HashiCorpVCSRootBeta
@@ -41,6 +42,9 @@ fun googleSubProjectBeta(allConfig: AllContextParameters): Project {
         // This is only present for the Beta provider, as only TPGB VCR recordings are used.
         subProject(vcrRecording(betaId, ProviderNameBeta, HashiCorpVCSRootBeta, ModularMagicianVCSRootBeta, vcrConfig))
 
+        // Beta Diff Test project that uses hashicorp/terraform-provider-google-beta-diff-test
+        subProject(weeklyDiffTests(betaId + "_DIFF_TEST", ProviderNameBeta, HashiCorpVCSRootBeta, betaConfig, NightlyTriggerConfiguration(daysOfWeek = "SAT", nightlyTestsEnabled = false)))
+
         params {
             readOnlySettings()
         }

.teamcity/components/projects/reused/nightly_tests.kt

@@ -10,6 +10,7 @@ package projects.reused
 import NightlyTestsProjectId
 import ProviderNameBeta
 import ProviderNameGa
+import ProviderNameBetaDiffTest
 import ServiceSweeperName
 import SharedResourceNameBeta
 import SharedResourceNameGa
@@ -32,6 +33,7 @@ fun nightlyTests(parentProject:String, providerName: String, vcsRoot: GitVcsRoot
     when(providerName) {
         ProviderNameGa -> sharedResources = arrayListOf(SharedResourceNameGa)
         ProviderNameBeta -> sharedResources = arrayListOf(SharedResourceNameBeta)
+        ProviderNameBetaDiffTest -> sharedResources = arrayListOf(SharedResourceNameBeta)
         else -> throw Exception("Provider name not supplied when generating a nightly test subproject")
     }
 
@@ -48,6 +50,7 @@ fun nightlyTests(parentProject:String, providerName: String, vcsRoot: GitVcsRoot
     when(providerName) {
         ProviderNameGa -> sweepersList = SweepersListGa
         ProviderNameBeta -> sweepersList = SweepersListBeta
+        ProviderNameBetaDiffTest -> sweepersList = SweepersListBeta
         else -> throw Exception("Provider name not supplied when generating a nightly test subproject")
     }
     val serviceSweeperConfig = BuildConfigurationForServiceSweeper(providerName, ServiceSweeperName, sweepersList, projectId, vcsRoot, sharedResources, config)

.teamcity/components/projects/reused/weekly_diff_tests.kt

@@ -0,0 +1,72 @@
+/*
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+// This file is maintained in the GoogleCloudPlatform/magic-modules repository and copied into the downstream provider repositories. Any changes to this file in the downstream will be overwritten.
+
+package projects.reused
+
+import NightlyTestsProjectId
+import ProviderNameGa
+import ProviderNameBeta
+import ServiceSweeperName
+import SharedResourceNameBeta
+import SharedResourceNameGa
+import builds.*
+import generated.SweepersListBeta
+import generated.SweepersListGa
+import jetbrains.buildServer.configs.kotlin.Project
+import jetbrains.buildServer.configs.kotlin.vcs.GitVcsRoot
+import replaceCharsId
+
+fun weeklyDiffTests(parentProject:String, providerName: String, vcsRoot: GitVcsRoot, config: AccTestConfiguration, cron: NightlyTriggerConfiguration): Project {
+
+    var projectId = "${parentProject}_${NightlyTestsProjectId}"
+    projectId = replaceCharsId(projectId)
+
+    // Nightly test projects run all acceptance tests overnight
+    // Here we ensure the project uses the appropriate Shared Resource to ensure no clashes between builds and/or sweepers
+    var sharedResources: ArrayList<String>
+    when(providerName) {
+        ProviderNameGa -> sharedResources = arrayListOf(SharedResourceNameGa)
+        ProviderNameBeta -> sharedResources = arrayListOf(SharedResourceNameBeta)
+        else -> throw Exception("Provider name not supplied when generating a weekly diff test subproject")
+    }
+
+    // Create build configs to run acceptance tests for each package defined in packages.kt and services.kt files
+    // and add cron trigger to them all
+    val allPackages = getAllPackageInProviderVersion(providerName)
+    val packageBuildConfigs = BuildConfigurationsForPackages(allPackages, providerName, projectId, vcsRoot, sharedResources, config, releaseDiffTest = "true")
+    packageBuildConfigs.forEach { buildConfiguration ->
+        buildConfiguration.addTrigger(cron)
+    }
+
+    // Create build config for sweeping the nightly test project
+    var sweepersList: Map<String,Map<String,String>>
+    when(providerName) {
+        ProviderNameGa -> sweepersList = SweepersListGa
+        ProviderNameBeta -> sweepersList = SweepersListBeta
+        else -> throw Exception("Provider name not supplied when generating a weekly diff test subproject")
+    }
+    val serviceSweeperConfig = BuildConfigurationForServiceSweeper(providerName, ServiceSweeperName, sweepersList, projectId, vcsRoot, sharedResources, config)
+    val sweeperCron = cron.clone()
+    sweeperCron.startHour += 5  // Ensure triggered after the package test builds are triggered
+    serviceSweeperConfig.addTrigger(sweeperCron)
+
+    return Project {
+        id(projectId)
+        name = "Weekly Diff Tests"
+        description = "A project connected to the hashicorp/terraform-provider-${providerName} repository, where scheduled weekly diff tests run and users can trigger ad-hoc builds"
+
+        // Register build configs in the project
+        packageBuildConfigs.forEach { buildConfiguration ->
+            buildType(buildConfiguration)
+        }
+        buildType(serviceSweeperConfig)
+
+        params{
+            configureGoogleSpecificTestParameters(config)
+        }
+    }
+}

.teamcity/tests/test_utils.kt

@@ -15,6 +15,7 @@ import org.junit.Assert.fail
 const val gaProjectName = "Google"
 const val betaProjectName = "Google Beta"
 const val nightlyTestsProjectName = "Nightly Tests"
+const val weeklyDiffTestsProjectName = "Weekly Diff Tests"
 const val mmUpstreamProjectName = "Upstream MM Testing"
 const val projectSweeperProjectName = "Project Sweeper"
 

.teamcity/tests/weekly_diff_project.kt

@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) HashiCorp, Inc.
+ * SPDX-License-Identifier: MPL-2.0
+ */
+
+// This file is maintained in the GoogleCloudPlatform/magic-modules repository and copied into the downstream provider repositories. Any changes to this file in the downstream will be overwritten.
+
+package tests
+
+import jetbrains.buildServer.configs.kotlin.triggers.ScheduleTrigger
+import org.junit.Assert.assertTrue
+import org.junit.Test
+import projects.googleCloudRootProject
+
+class WeeklyDiffTestProjectsTests {
+    @Test
+    fun allBuildsShouldHaveTrigger() {
+        val root = googleCloudRootProject(testContextParameters())
+
+        // Find GA nightly test project
+        // var gaNightlyTestProject = getNestedProjectFromRoot(root, gaProjectName, weeklyDiffTestsProjectName)
+
+        // Find Beta nightly test project
+        var betaWeeklyDiffTestProject = getNestedProjectFromRoot(root, betaProjectName, weeklyDiffTestsProjectName)
+
+        // Make assertions about builds in both weekly diff test projects
+        (betaWeeklyDiffTestProject.buildTypes).forEach{bt ->
+            assertTrue("Build configuration `${bt.name}` should contain at least one trigger", bt.triggers.items.isNotEmpty())
+             // Look for at least one CRON trigger
+            var found: Boolean = false
+            lateinit var schedulingTrigger: ScheduleTrigger
+            for (item in bt.triggers.items){
+                if (item.type == "schedulingTrigger") {
+                    schedulingTrigger = item as ScheduleTrigger
+                    found = true
+                    break
+                }
+            }
+
+            assertTrue("Build configuration `${bt.name}` should contain a CRON/'schedulingTrigger' trigger", found)
+
+            // Check that weekly diff test is being ran on the nightly-test branch
+            var isNightlyTestBranch: Boolean = false
+            if (schedulingTrigger.branchFilter == "+:refs/heads/nightly-test"){
+                isNightlyTestBranch = true
+            }
+            assertTrue("Build configuration `${bt.name}` is using the nightly-test branch filter;", isNightlyTestBranch)
+        }
+    }
+}