chore(windows): run each JDK tests in parallel

Jenkinsfile

@@ -109,13 +109,38 @@ def parallelStages = [failFast: false]
                                 }
                                 junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'target/**/junit-results*.xml')
                             }
-                            // If the tests are passing for Linux AMD64, then we can build all the CPU architectures
-                            if (isUnix()) {
-                                stage('Multi-Arch Build') {
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
 
-                                    sh 'make every-build'
-                                }
-                            }
+parallelStages['multi-arch'] = {
+    def imageType = 'linux'
+    withEnv(["IMAGE_TYPE=${imageType}", "REGISTRY_ORG=${infra.isTrusted() ? 'jenkins' : 'jenkins4eval'}"]) {
+        int retryCounter = 0
+        retry(count: 2, conditions: [agent(), nonresumable()]) {
+            // Use local variable to manage concurrency and increment BEFORE spinning up any agent
+            final String resolvedAgentLabel = spotAgentSelector(agentSelector(imageType), retryCounter)
+            retryCounter++
+            node(resolvedAgentLabel) {
+                timeout(time: 60, unit: 'MINUTES') {
+                    checkout scm
+                    stage('Prepare Docker') {
+                        if (infra.isTrusted()) {
+                            echo 'INFO:Running on trusted.ci.jenkins.io, skipping'
+                        } else {
+                            sh 'make docker-init'
+                        }
+                    }
+                    // Publication of multi-arch images is already taken care of with `make publish`
+                    stage('Build') {
+                        if (infra.isTrusted()) {
+                            echo 'INFO: Running on trusted.ci.jenkins.io, skipping'
+                        } else {
+                            sh 'make every-build'
                         }
                     }
                 }
@@ -124,6 +149,7 @@ def parallelStages = [failFast: false]
     }
 }
 
+
 // Execute parallel stages
 parallel parallelStages
 // // vim: ft=groovy

build.ps1

@@ -21,6 +21,7 @@ Param(
 
 $ErrorActionPreference = 'Stop'
 $ProgressPreference = 'SilentlyContinue' # Disable Progress bar for faster downloads
+$pesterVersion = '5.3.3'
 
 if (![String]::IsNullOrWhiteSpace($env:TESTS_DEBUG)) {
     $TestsDebug = $env:TESTS_DEBUG
@@ -80,31 +81,32 @@ Function Test-CommandExists {
     }
 }
 
+# Ex: Test-Image -AgentType inbound-agent -RemotingVersion 3345.v03dee9b_f88fc -ImageName docker.io/jenkins/agent:jdk21-windowsservercore-ltsc2019 -JavaVersion 21.0.7_6 -TestNumber 0
 function Test-Image {
     param (
-        [String] $AgentTypeAndImageName
+        [String] $AgentType,
+        [String] $RemotingVersion,
+        [String] $ImageName,
+        [String] $JavaVersion,
+        [int] $TestNumber
     )
 
-    # Ex: agent|docker.io/jenkins/agent:jdk21-windowsservercore-ltsc2019|21.0.7_6
-    $items = $AgentTypeAndImageName.Split('|')
-    $agentType = $items[0]
-    $imageName = $items[1] -replace 'docker.io/', ''
-    $javaVersion = $items[2]
+    $imageName = $ImageName -replace 'docker.io/', ''
     $imageNameItems = $imageName.Split(':')
     $imageTag = $imageNameItems[1]
 
-    Write-Host "= TEST: Testing ${imageName} image:"
+    Write-Host "=== TEST: Test-Image -AgentType $AgentType -RemotingVersion $RemotingVersion -ImageName $ImageName -JavaVersion $JavaVersion -TestNumber $TestNumber"
 
     $env:IMAGE_NAME = $imageName
     $env:VERSION = "$RemotingVersion"
-    $env:JAVA_VERSION = "$javaVersion"
+    $env:JAVA_VERSION = "$JavaVersion"
 
-    $targetPath = '.\target\{0}\{1}' -f $agentType, $imageTag
+    $targetPath = '.\target\{0}\{1}\{2}\{3}' -f $AgentType, $imageTag, $aJavaVersion.Substring(0,2), $TestNumber
     if (Test-Path $targetPath) {
         Remove-Item -Recurse -Force $targetPath
     }
     New-Item -Path $targetPath -Type Directory | Out-Null
-    $configuration.Run.Path = 'tests\{0}.Tests.ps1' -f $agentType
+    $configuration.Run.Path = 'tests\{0}.{1}.Tests.ps1' -f $AgentType, $TestNumber
     $configuration.TestResult.OutputPath = '{0}\junit-results.xml' -f $targetPath
     $TestResults = Invoke-Pester -Configuration $configuration
     $failed = $false
@@ -124,7 +126,6 @@ function Test-Image {
 
 function Initialize-DockerComposeFile {
     param (
-        [String] $AgentType,
         [String] $ImageType,
         [String] $DockerComposeFile
     )
@@ -138,9 +139,6 @@ function Initialize-DockerComposeFile {
     # Override the list of Windows versions taken defined in docker-bake.hcl by the version from image type
     $env:WINDOWS_VERSION_OVERRIDE = $windowsVersion
 
-    # Override the list of agent types defined in docker-bake.hcl by the specified agent type
-    $env:WINDOWS_AGENT_TYPE_OVERRIDE = $AgentType
-
     # Retrieve the targets from docker buildx bake --print output
     # Remove the 'output' section (unsupported by docker compose)
     # For each target name as service key, return a map consisting of:
@@ -165,106 +163,138 @@ function Initialize-DockerComposeFile {
 
     # Remove override
     Remove-Item env:\WINDOWS_VERSION_OVERRIDE
-    Remove-Item env:\WINDOWS_AGENT_TYPE_OVERRIDE
 }
 
 Test-CommandExists 'docker'
 Test-CommandExists 'docker-compose'
 Test-CommandExists 'docker buildx'
 Test-CommandExists 'yq'
 
-foreach($agentType in $AgentTypes) {
-    $dockerComposeFile = 'build-windows_{0}_{1}.yaml' -f $AgentType, $ImageType
-    $baseDockerCmd = 'docker-compose --file={0}' -f $dockerComposeFile
-    $baseDockerBuildCmd = '{0} build --parallel --pull' -f $baseDockerCmd
+$testImageFunction = ${function:Test-Image}
+$workspacePath = (Get-Location).Path
+$dockerComposeFile = 'build-windows_{0}.yaml' -f $ImageType
+$baseDockerCmd = 'docker-compose --file={0}' -f $dockerComposeFile
+$baseDockerBuildCmd = '{0} build --parallel --pull' -f $baseDockerCmd
 
-    # Generate the docker compose file if it doesn't exists or if the parameter OverwriteDockerComposeFile is set
-    if ((Test-Path $dockerComposeFile) -and -not $OverwriteDockerComposeFile) {
-        Write-Host "= PREPARE: The docker compose file '$dockerComposeFile' containing the image definitions already exists."
-    } else {
-        Write-Host "= PREPARE: Initialize the docker compose file '$dockerComposeFile' containing the image definitions."
-        Initialize-DockerComposeFile -AgentType $AgentType -ImageType $ImageType -DockerComposeFile $dockerComposeFile
-    }
+# Generate the docker compose file if it doesn't exists or if the parameter OverwriteDockerComposeFile is set
+if ((Test-Path $dockerComposeFile) -and -not $OverwriteDockerComposeFile) {
+    Write-Host "= PREPARE: The docker compose file '$dockerComposeFile' containing the image definitions already exists."
+} else {
+    Write-Host "= PREPARE: Initialize the docker compose file '$dockerComposeFile' containing the image definitions."
+    Initialize-DockerComposeFile -ImageType $ImageType -DockerComposeFile $dockerComposeFile
+}
 
-    Write-Host '= PREPARE: List of images and tags to be processed:'
-    Invoke-Expression "$baseDockerCmd config"
+Write-Host '= PREPARE: List of images and tags to be processed:'
+Invoke-Expression "$baseDockerCmd config"
 
-    if ($target -eq 'build') {
-        Write-Host '= BUILD: Building all images...'
-        switch ($DryRun) {
-            $true { Write-Host "(dry-run) $baseDockerBuildCmd" }
-            $false { Invoke-Expression $baseDockerBuildCmd }
-        }
-        Write-Host '= BUILD: Finished building all images.'
+if ($target -eq 'build') {
+    Write-Host '= BUILD: Building all images...'
+    switch ($DryRun) {
+        $true { Write-Host "(dry-run) $baseDockerBuildCmd" }
+        $false { Invoke-Expression $baseDockerBuildCmd }
+    }
+    Write-Host "= BUILD: Finished building all images."
 
-        if ($lastExitCode -ne 0) {
-            exit $lastExitCode
-        }
+    if ($lastExitCode -ne 0) {
+        exit $lastExitCode
     }
+}
 
-    if ($target -eq 'test') {
-        if ($DryRun) {
-            Write-Host '= TEST: (dry-run) test harness'
-        } else {
-            Write-Host '= TEST: Starting test harness'
-
-            $mod = Get-InstalledModule -Name Pester -MinimumVersion 5.3.0 -MaximumVersion 5.3.3 -ErrorAction SilentlyContinue
-            if ($null -eq $mod) {
-                Write-Host '= TEST: Pester 5.3.x not found: installing...'
-                $module = 'C:\Program Files\WindowsPowerShell\Modules\Pester'
-                if (Test-Path $module) {
-                    takeown /F $module /A /R
-                    icacls $module /reset
-                    icacls $module /grant Administrators:'F' /inheritance:d /T
-                    Remove-Item -Path $module -Recurse -Force -Confirm:$false
-                }
-                Install-Module -Force -Name Pester -MaximumVersion 5.3.3
+if ($target -eq 'test') {
+    if ($DryRun) {
+        Write-Host '= TEST: (dry-run) test harness'
+    } else {
+        Write-Host '= TEST: Starting test harness'
+
+        $mod = Get-InstalledModule -Name Pester -MinimumVersion 5.3.0 -MaximumVersion 5.3.3 -ErrorAction SilentlyContinue
+        if ($null -eq $mod) {
+            Write-Host "= TEST: Pester $pesterVersion not found"
+            $module = 'C:\Program Files\WindowsPowerShell\Modules\Pester'
+            if (Test-Path $module) {
+                Write-Output '= TEST: taking ownership of existing Pester module to remove it...'
+                takeown /F $module /A /R | Out-Null
+                icacls $module /reset | Out-Null
+                icacls $module /grant Administrators:'F' /inheritance:d /T | Out-Null
+                Remove-Item -Path $module -Recurse -Force -Confirm:$false | Out-Null
             }
+            Write-Host '= TEST: Installing Pester'
+            Install-Module -Force -Name Pester -MaximumVersion $pesterVersion
+        }
 
-            Import-Module Pester
-            Write-Host '= TEST: Setting up Pester environment...'
-            $configuration = [PesterConfiguration]::Default
-            $configuration.Run.PassThru = $true
-            $configuration.Run.Path = '.\tests'
-            $configuration.Run.Exit = $true
-            $configuration.TestResult.Enabled = $true
-            $configuration.TestResult.OutputFormat = 'JUnitXml'
-            $configuration.Output.Verbosity = 'Diagnostic'
-            $configuration.CodeCoverage.Enabled = $false
-
-            Write-Host "= TEST: Testing all ${agentType} images..."
-            # Only fail the run afterwards in case of any test failures
-            $testFailed = $false
-            $imageDefinitions = Invoke-Expression "$baseDockerCmd config" | yq --unwrapScalar --output-format json '.services' | ConvertFrom-Json
-            foreach ($imageDefinition in $imageDefinitions.PSObject.Properties) {
-                $testFailed = $testFailed -or (Test-Image ('{0}|{1}|{2}' -f $agentType, $imageDefinition.Value.image, $imageDefinition.Value.build.args.JAVA_VERSION))
+        Write-Host '= TEST: Testing all images...'
+        $imageDefinitions = Invoke-Expression "$baseDockerCmd config" | yq --unwrapScalar --output-format json '.services' | ConvertFrom-Json
+
+        # Run Test-Image in parallel for each image definition
+        $jobs = @()
+        foreach ($imageDefinition in $imageDefinitions.PSObject.Properties) {
+            $image = $imageDefinition.Value.image
+            $agentType = $imageDefinition.Value.build.target
+            $javaVersion = $imageDefinition.Value.build.args.JAVA_VERSION
+            $testCount = 1
+            if ($AgentType -eq 'inbound-agent') {
+                $testCount = 5
             }
-
-            # Fail if any test failures
-            if ($testFailed -ne $false) {
-                Write-Error "Test stage failed for ${agentType}!"
-                exit 1
-            } else {
-                Write-Host "= TEST: stage passed for ${agentType}!"
+            Write-Host "= TEST: Starting ${testCount} ${image} test(s) in parallel..."
+            for ($i = 0; $i -lt $testCount; $i++) {
+                $jobName = '{0}-{1}-{2}' -f $agentType, $javaVersion.Substring(0,2), $i
+                $jobs += Start-Job -Name $jobName -ScriptBlock {
+                    param($anAgentType, $aRemotingVersion, $anImage, $aJavaVersion, $aTestImageFunction, $aWorkspacePath, $aTestNumber)
+
+                    Write-Host "== TEST: Setting up Pester environment for $anImage test #$aTestNumber..."
+                    Import-Module Pester
+                    $configuration = [PesterConfiguration]::Default
+                    $configuration.Run.PassThru = $true
+                    $configuration.Run.Path = '{0}\tests' -f $aWorkspacePath
+                    $configuration.Run.Exit = $true
+                    $configuration.TestResult.Enabled = $true
+                    $configuration.TestResult.OutputFormat = 'JUnitXml'
+                    $configuration.Output.Verbosity = 'Diagnostic'
+                    $configuration.CodeCoverage.Enabled = $false
+                    Set-Item -Path Function:Test-Image -Value $aTestImageFunction
+                    Set-Location -Path $aWorkspacePath
+
+                    Test-Image -AgentType $anAgentType -RemotingVersion $aRemotingVersion -ImageName $anImage -JavaVersion $aJavaVersion -TestNumber $aTestNumber
+                } -ArgumentList $agentType, $RemotingVersion, $image, $javaVersion, $testImageFunction, $workspacePath, $i
             }
         }
-    }
 
-    if ($target -eq 'publish') {
-        Write-Host '= PUBLISH: push all images and tags'
-        switch($DryRun) {
-            $true { Write-Host "(dry-run) $baseDockerCmd push" }
-            $false { Invoke-Expression "$baseDockerCmd push" }
+        # Wait for all jobs to finish and collect results
+        $testFailed = $false
+        foreach ($job in $jobs) {
+            $result = Receive-Job -Job $job -Wait
+            Write-Host ('== TEST: End of {0}' -f $job.Name)
+            if ($result.Failed) {
+                Write-Host '== TEST: Error(s), see the results below'
+                $result.Tests | ConvertTo-Json | Write-Host
+                $testFailed = $true
+            }
+            Remove-Job $job
         }
 
-        # Fail if any issues when publising the docker images
-        if ($lastExitCode -ne 0) {
-            Write-Error '= PUBLISH: failed!'
+        # Fail if any test failures
+        if ($testFailed -ne $false) {
+            Write-Error '= TEST: stage failed'
             exit 1
+        } else {
+            Write-Host '= TEST: stage passed'
         }
     }
 }
 
+if ($target -eq 'publish') {
+    Write-Host '= PUBLISH: push all images and tags'
+    switch($DryRun) {
+        $true { Write-Host "(dry-run) $baseDockerCmd push" }
+        $false { Invoke-Expression "$baseDockerCmd push" }
+    }
+
+    # Fail if any issues when publising the docker images
+    if ($lastExitCode -ne 0) {
+        Write-Error '= PUBLISH: failed!'
+        exit 1
+    }
+}
+
 if ($lastExitCode -ne 0) {
     Write-Error 'Build failed!'
 } else {