Run test template tests with Arcade (#51020)

Author: MiYanni

.vsts-ci.yml

@@ -102,6 +102,7 @@ extends:
           oneESCompat:
             templateFolderName: templates-official
             publishTaskPrefix: 1ES.
+          populateInternalRuntimeVariables: true
           runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64)
           locBranch: release/10.0.1xx
           # WORKAROUND: BinSkim requires the folder exist prior to scanning.
@@ -141,6 +142,14 @@ extends:
                 _SignType: real
               dependsOn: Official_windows_x64
               downloadManifestMsiPackages: true
+            ### TestTemplatesCG ###
+            # Note: This job is only used to allow the test templates to be built locally on the agent as opposed to Helix.
+            # The tests acquire the templates' PackageReferences from NuGet, which allows them to be scanned by CG (component governance).
+            # CG is only ran internally, so this job makes sense to only run alongside of the official jobs.
+            - categoryName: TestTemplatesCG
+              testProjects: $(Build.SourcesDirectory)/test/dotnet-new.IntegrationTests/dotnet-new.IntegrationTests.csproj
+              testRunnerAdditionalArguments: -class Microsoft.DotNet.Cli.New.IntegrationTests.DotnetNewTestTemplatesTests
+              publishXunitResults: true
 
       ############### LINUX ###############
       - template: /eng/pipelines/templates/jobs/sdk-job-matrix.yml@self
@@ -153,6 +162,7 @@ extends:
           oneESCompat:
             templateFolderName: templates-official
             publishTaskPrefix: 1ES.
+          populateInternalRuntimeVariables: true
           runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64)
           ${{ if and(eq(parameters.runTestBuild, false), ne(variables['Build.Reason'], 'PullRequest')) }}:
             timeoutInMinutes: 90
@@ -235,6 +245,7 @@ extends:
           oneESCompat:
             templateFolderName: templates-official
             publishTaskPrefix: 1ES.
+          populateInternalRuntimeVariables: true
           runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64)
           ${{ if and(eq(parameters.runTestBuild, false), ne(variables['Build.Reason'], 'PullRequest')) }}:
             timeoutInMinutes: 90
@@ -260,6 +271,8 @@ extends:
               vmImage: macOS-latest
               os: macOS
             helixTargetQueue: osx.13.arm64
+            populateInternalRuntimeVariables: true
+            runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64)
             macOSJobParameterSets:
             - categoryName: TestBuild
               targetArchitecture: arm64
@@ -270,7 +283,9 @@ extends:
         - template: /eng/dotnet-format/dotnet-format-integration.yml@self
           parameters:
             oneESCompat:
+              templateFolderName: templates-official
               publishTaskPrefix: 1ES.
+            populateInternalRuntimeVariables: true
             runtimeSourceProperties: /p:DotNetRuntimeSourceFeed=https://ci.dot.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64)
 
     ############### PUBLISH STAGE ###############

build/RunTestTemplateTests.ps1

@@ -2,6 +2,7 @@ parameters:
 - name: oneESCompat
   type: object
   default:
+    templateFolderName: templates
     publishTaskPrefix: ''
 
 - name: TestArguments
@@ -63,6 +64,9 @@ parameters:
 - name: runtimeSourceProperties
   type: string
   default: ''
+- name: populateInternalRuntimeVariables
+  type: boolean
+  default: false
 
 jobs:
 - job: Formatting_Check
@@ -77,8 +81,11 @@ jobs:
       os: windows
   timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
   steps:
-  - template: /eng/common/templates/steps/enable-internal-runtimes.yml
-  - template: /eng/common/templates/steps/enable-internal-sources.yml
+  - ${{ if eq(parameters.populateInternalRuntimeVariables, true) }}:
+    - template: /eng/common/${{ parameters.oneESCompat.templateFolderName }}/steps/enable-internal-sources.yml
+      parameters:
+        legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw)
+    - template: /eng/common/${{ parameters.oneESCompat.templateFolderName }}/steps/enable-internal-runtimes.yml
   - script: .\restore.cmd ${{ parameters.runtimeSourceProperties }}
     displayName: 🟣 Restore dependencies
   - script: |
@@ -106,8 +113,11 @@ jobs:
         os: windows
     timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
     steps:
-    - template: /eng/common/templates/steps/enable-internal-runtimes.yml
-    - template: /eng/common/templates/steps/enable-internal-sources.yml
+    - ${{ if eq(parameters.populateInternalRuntimeVariables, true) }}:
+      - template: /eng/common/${{ parameters.oneESCompat.templateFolderName }}/steps/enable-internal-sources.yml
+        parameters:
+          legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw)
+      - template: /eng/common/${{ parameters.oneESCompat.templateFolderName }}/steps/enable-internal-runtimes.yml
     - script: eng\dotnet-format\integration-test.cmd -repo '${{ testArgs._repo }}' -branchName '${{ testArgs._branchName }}' -sha '${{ testArgs._sha }}' -targetSolution '${{ testArgs._targetSolution }}' -useParentSdk ${{ testArgs._useParentSdk }} -testPath '$(Agent.TempDirectory)\temp' -stage 'prepare' -runtimeSourceProperties '${{ parameters.runtimeSourceProperties }}'
       displayName: 🟣 Prepare ${{ testArgs._repoName }} for formatting
 

eng/dotnet-format/dotnet-format-integration.yml

@@ -12,7 +12,9 @@ parameters:
   testProjects: $(Build.SourcesDirectory)/test/UnitTests.proj
   publishRetryConfig: false
   publishXunitResults: false
+  testRunnerAdditionalArguments: ''
   enableSbom: true
+  populateInternalRuntimeVariables: false
   timeoutInMinutes: 150
   ### ENV VARS ###
   testFullMSBuild: false
@@ -71,11 +73,11 @@ jobs:
           targetPath: $(Build.SourcesDirectory)/eng/BuildConfiguration
           artifactName: BuildConfiguration
 
-    # Populate internal runtime variables.
-    - template: /eng/common/templates/steps/enable-internal-sources.yml
-      parameters:
-        legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw)
-    - template: /eng/common/templates/steps/enable-internal-runtimes.yml
+    - ${{ if eq(parameters.populateInternalRuntimeVariables, true) }}:
+      - template: /eng/common/${{ parameters.oneESCompat.templateFolderName }}/steps/enable-internal-sources.yml
+        parameters:
+          legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw)
+      - template: /eng/common/${{ parameters.oneESCompat.templateFolderName }}/steps/enable-internal-runtimes.yml
 
     - ${{ if eq(parameters.downloadManifestMsiPackages, true) }}:
       - task: DownloadBuildArtifacts@1
@@ -84,7 +86,7 @@ jobs:
           downloadPath: $(Build.SourcesDirectory)/artifacts/downloaded-manifest-msi-packages
           itemPattern: '**/*Manifest-*.Msi.*.nupkg'
           checkDownloadedFiles: true
-        displayName: Download Manifest msi packages
+        displayName: 🟣 Download Manifest MSI Packages
 
     ############### BUILDING ###############
     - ${{ if eq(parameters.pool.os, 'windows') }}:
@@ -105,10 +107,6 @@ jobs:
           BuildConfig: $(buildConfiguration)
           TestFullMSBuild: ${{ parameters.testFullMSBuild }}
 
-      - ${{ if eq(parameters.targetArchitecture, 'x64') }}:
-        - powershell: build/RunTestTemplateTests.ps1
-          displayName: 🟣 Run Test Templates Tests
-
     - ${{ else }}:
       - script: |
           source $(Build.SourcesDirectory)/eng/common/native/init-os-and-arch.sh
@@ -141,12 +139,14 @@ jobs:
         - script: $(Build.SourcesDirectory)/artifacts/bin/redist/$(buildConfiguration)/dotnet/dotnet workload install wasm-tools --skip-manifest-update
           workingDirectory: $(Build.SourcesDirectory)/artifacts/bin
           displayName: 🟣 Install wasm-tools Workload
+
       # For the /p:Projects syntax for PowerShell, see: https://github.com/dotnet/msbuild/issues/471#issuecomment-1146466335
       - ${{ if eq(parameters.pool.os, 'windows') }}:
         - powershell: eng/common/build.ps1
             -restore -test -ci -prepareMachine -nativeToolsOnMachine
             -configuration $(buildConfiguration)
             /p:Projects=\`"${{ replace(parameters.testProjects, ';', '`;') }}\`"
+            /p:TestRunnerAdditionalArguments="${{ parameters.testRunnerAdditionalArguments }}"
             /p:TargetArchitecture=${{ parameters.targetArchitecture }}
             ${{ parameters.runtimeSourceProperties }}
             /p:CustomHelixTargetQueue=${{ parameters.helixTargetQueue }}
@@ -159,6 +159,7 @@ jobs:
             HelixAccessToken: $(HelixApiAccessToken)
             RunAoTTests: ${{ parameters.runAoTTests }}
             TestFullMSBuild: ${{ parameters.testFullMSBuild }}
+
       - ${{ else }}:
         # For the /p:Projects syntax for Bash, see: https://github.com/dotnet/msbuild/issues/471#issuecomment-1690189034
         # The /p:CustomHelixTargetQueue syntax is: <queue-name>@<container-url>
@@ -167,6 +168,7 @@ jobs:
             -restore -test -ci -prepareMachine
             -configuration $(buildConfiguration)
             '/p:Projects="${{ parameters.testProjects }}"'
+            /p:TestRunnerAdditionalArguments="${{ parameters.testRunnerAdditionalArguments }}"
             /p:TargetArchitecture=${{ parameters.targetArchitecture }}
             /p:TargetRid=${{ parameters.runtimeIdentifier }}
             ${{ parameters.osProperties }}

eng/pipelines/templates/jobs/sdk-build.yml

@@ -31,7 +31,6 @@ parameters:
     container: azureLinux30Amd64
     helixTargetContainer: $(helixTargetContainerPrefix)ubuntu-24.04-helix-amd64
     osProperties: /p:OSName=linux /p:BuildSdkDeb=true
-    runTests: true
     # Helix is hanging on this job using the container. See: https://github.com/dotnet/dnceng/issues/6000
     disableJob: true
   - categoryName: TemplateEngine