Merge remote-tracking branch 'origin/master' into Localization

vmapetr · vmapetr · commit 161c02b2821a · 2024-07-15T08:01:27.000Z
diff --git a/.azure-pipelines/build-job.yml b/.azure-pipelines/build-job.yml
@@ -64,6 +64,14 @@ parameters:
   - agent
   - pipelines-agent
 
+- name: disableSdl
+  type: boolean
+  default: false
+
+- name: justificationForDisablingSdl
+  type: string
+  default: ''
+
 jobs:
 
 - job: ${{ parameters.jobName }}
@@ -74,6 +82,13 @@ jobs:
   ${{ if ne(parameters.container, '') }}:
     container: ${{ parameters.container }}
 
+  ${{ if eq(parameters.disableSdl, true) }}:
+    parameters:
+      sdl:
+        codeql:
+          compiled:
+            enabled: false
+            justificationForDisabling: ${{ parameters.justificationForDisablingSdl}}
   variables:
     PACKAGE_TYPE: ${{ parameters.packageType }}
     ${{ if eq(parameters.os, 'win') }}:
diff --git a/.azure-pipelines/build-jobs.yml b/.azure-pipelines/build-jobs.yml
@@ -59,6 +59,10 @@ parameters:
   type: boolean
   default: false
 
+- name: disableSdl
+  type: boolean
+  default: false
+
 jobs:
 - template: /.azure-pipelines/build-job.yml@self
   parameters:
diff --git a/.azure-pipelines/pipeline.yml b/.azure-pipelines/pipeline.yml
@@ -271,6 +271,7 @@ extends:
       - ${{ if parameters.macOS_x64 }}:
         - template: /.azure-pipelines/build-jobs.yml@self
           parameters:
+            disableSdl: true
             jobName: build_osx
             displayName: macOS (x64)
             pool:
@@ -290,6 +291,7 @@ extends:
       - ${{ if parameters.macOS_arm64 }}:
         - template: /.azure-pipelines/build-jobs.yml@self
           parameters:
+            disableSdl: true
             jobName: build_osx_arm64
             displayName: macOS (ARM64)
             pool:
diff --git a/.vsts.release.yml b/.vsts.release.yml
@@ -1,5 +1,12 @@
 # This Yaml Document has been converted by ESAI Yaml Pipeline Conversion Tool.
 
+trigger:
+  paths:
+    include:
+    - release
+    - .azure-pipelines
+    - .vsts.release.yml
+
 schedules:
 - cron: '0 6 * * 2'
   displayName: Scheduled weekly run
@@ -88,11 +95,15 @@ extends:
             Write-Host "isRelease = $isRelease"
             Write-Host "##vso[task.setVariable variable=isRelease;isOutput=true]$isRelease"
 
+            $isTestRun = ($buildReason -eq 'IndividualCI' -or $buildReason -eq 'PullRequest')
+            Write-Host "isTestRun = $isTestRun"
+            Write-Host "##vso[task.setVariable variable=isTestRun;isOutput=true]$isTestRun"
+
             $isScheduledRelease = $isRelease -and $buildReason -eq 'Schedule'
             Write-Host "isScheduledRelease = $isScheduledRelease"
 
-            if ($isRelease) {
-              if ($isScheduledRelease) {
+            if ($isRelease -or $isTestRun) {
+              if ($isScheduledRelease -or $isTestRun) {
                 $majorAndMinorVersion = "3.$($currentSprint.sprint)"
                 $patchVersion = 0
                 ## Looking for a free patch version
@@ -116,6 +127,9 @@ extends:
                   Write-Error "Version parameter is required for manual release." -ErrorAction Stop
                 }
               }
+              if ($isTestRun) {
+                $agentVersion = '3.000.999'
+              }
               Write-Host "agentVersion = $agentVersion"
               Write-Host "##vso[task.setVariable variable=agentVersion;isOutput=true]$agentVersion"
 
@@ -137,10 +151,12 @@ extends:
       ################################################################################
         displayName: Create Release Branch
         variables:
+          IsTestRun: $[ stageDependencies.Verify_release.Set_variables.outputs['SetReleaseVariables.isTestRun'] ]
           IsRelease: $[ stageDependencies.Verify_release.Set_variables.outputs['SetReleaseVariables.isRelease'] ]
           ReleaseBranch: $[ stageDependencies.Verify_release.Set_variables.outputs['SetReleaseVariables.releaseBranch'] ]
           AgentVersion: $[ stageDependencies.Verify_release.Set_variables.outputs['SetReleaseVariables.agentVersion'] ]
-        condition: and(succeeded(), eq(variables.IsRelease, 'True'))
+        condition: and(succeeded(), or(eq(variables.IsRelease, 'True'), eq(variables.IsTestRun, 'True')))
+
         pool:
           name: 1ES-ABTT-Shared-Pool
           image: abtt-ubuntu-2204
@@ -178,10 +194,11 @@ extends:
             name: 1ES-Shared-Hosted-Pool_Windows-Server-2022
             demands: AzurePS
           variables:
+            IsTestRun: $[ stageDependencies.Verify_release.Set_variables.outputs['SetReleaseVariables.isTestRun'] ]
             IsRelease: $[ stageDependencies.Verify_release.Set_variables.outputs['SetReleaseVariables.isRelease'] ]
             ReleaseBranch: $[ stageDependencies.Verify_release.Set_variables.outputs['SetReleaseVariables.releaseBranch'] ]
             AgentVersion: $[ stageDependencies.Verify_release.Set_variables.outputs['SetReleaseVariables.agentVersion'] ]
-          condition: and(succeeded(), eq(variables.IsRelease, 'True'))
+          condition: and(succeeded(), or(eq(variables.IsRelease, 'True'), eq(variables.IsTestRun, 'True')))
           steps:
 
           # Clean
@@ -215,20 +232,46 @@ extends:
                 Select-AzSubscription -SubscriptionId $(SubscriptionId)
                 $storageContext = New-AzStorageContext -StorageAccountName vstsagentpackage -UseConnectedAccount
                 $versionDir = "$(AgentVersion)"
+                
+                $container = "agent"
+                $isTestContainer = "$(IsTestRun)"
+                Write-Host "isTestContainer = $isTestContainer"
+
+                if ($isTestContainer -eq "True") { 
+                  $container = "testagent"
+                  New-AzStorageContainer -Context $storageContext -Name $container -Permission Off
+                }
+                Write-Host "container = $container"
+                
                 Get-ChildItem -LiteralPath "$(System.ArtifactsDirectory)/agent" | ForEach-Object {
                   $target=$_
                   Get-ChildItem -LiteralPath "$(System.ArtifactsDirectory)/agent/$target" -Include "*.zip","*.tar.gz" | ForEach-Object {
                     $executable = $_
-                    Write-Host "Uploading $executable to BlobStorage vstsagentpackage/agent/$versionDir"
-                    Set-AzStorageBlobContent -Context $storageContext -Container agent -File "$(System.ArtifactsDirectory)/agent/$target/$executable" -Blob "$versionDir/$executable" -Force
-                    $uploadFiles.Add("/agent/$versionDir/$executable")
+                    Write-Host "Uploading $executable to BlobStorage vstsagentpackage/$container/$versionDir"
+                    Set-AzStorageBlobContent -Context $storageContext -Container $container -File "$(System.ArtifactsDirectory)/agent/$target/$executable" -Blob "$versionDir/$executable" -Force
+                    $uploadFiles.Add("/$container/$versionDir/$executable")
                   }
                 }
                 Write-Host "Purge Azure CDN Cache"
                 Clear-AzCdnEndpointContent -EndpointName vstsagentpackage -ProfileName vstsagentpackage -ResourceGroupName vstsagentpackage -ContentPath $uploadFiles
                 Write-Host "Force Refresh Azure CDN Cache"
                 Import-AzCdnEndpointContent -EndpointName vstsagentpackage -ProfileName vstsagentpackage -ResourceGroupName vstsagentpackage -ContentPath $uploadFiles
 
+          # Clean up blob container with test agent version
+          - task: AzurePowerShell@5
+            displayName: Delete Azure Blob container with test agent version
+            condition: and(succeeded(), eq(variables.IsTestRun, 'True'))
+            inputs:
+              azurePowerShellVersion: 'LatestVersion'
+              azureSubscription: 'azure-pipelines-agent-vstsagentpackage-oauth'
+              scriptType: 'InlineScript'
+              inline: |
+                Import-Module Azure, Az.Accounts, Az.Storage -ErrorAction Ignore -PassThru
+                Select-AzSubscription -SubscriptionId $(SubscriptionId)
+                $storageContext = New-AzStorageContext -StorageAccountName vstsagentpackage -UseConnectedAccount
+                $container = 'testagent'
+                Remove-AzStorageContainer -Name $container -Context $storageContext -Force
+                
           # Download all agent hashes created in previous phases
           - task: DownloadBuildArtifacts@0
             displayName: Download Agent Hashes
@@ -267,6 +310,7 @@ extends:
               $releaseCreated = Invoke-RestMethod @releaseParams
               Write-Host $releaseCreated
               $releaseId = $releaseCreated.id
+              Write-Host "##vso[task.setVariable variable=releaseId;isoutput=true]$releaseId"
               $assets = [System.IO.File]::ReadAllText("$(Build.SourcesDirectory)\assets.json").Replace("<AGENT_VERSION>","$(AgentVersion)")
               $assetsParams = @{
                 Uri = "https://uploads.github.com/repos/Microsoft/azure-pipelines-agent/releases/$releaseId/assets?name=assets.json"
@@ -279,6 +323,36 @@ extends:
               }
               Invoke-RestMethod @assetsParams
             displayName: Create agent release on Github
+            name: create_github_release
+
+          # Delete test agent release
+          - powershell: |
+              Write-Host "Deleting test github release."
+              $releaseId = $(create_github_release.releaseId) 
+              
+              $releaseParams = @{
+                Uri = "https://api.github.com/repos/Microsoft/azure-pipelines-agent/releases/$releaseId";
+                Method = 'DELETE';
+                Headers = @{
+                  Authorization = 'Basic ' + [Convert]::ToBase64String([Text.Encoding]::ASCII.GetBytes("vsts:$(GithubToken)"));
+                }
+                ContentType = 'application/json';
+              }
+              [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+              $releaseDeleted = Invoke-RestMethod @releaseParams
+              Write-Host $releaseDeleted.Id
+            displayName: Delete test agent release from Github
+            condition: and(succeeded(), eq(variables.IsTestRun, 'True'))
+
+          # Clean up test release branch
+          - powershell: |
+              git config --global user.email "azure-pipelines-bot@microsoft.com"
+              git config --global user.name "azure-pipelines-bot"
+              git status
+              git -c credential.helper='!f() { echo "username=pat"; echo "password=$(GithubToken)"; };f' push origin --delete $(ReleaseBranch)
+              git push --delete origin v$(AgentVersion)
+            displayName: Clean up test release branch
+            condition: and(succeeded(), eq(variables.IsTestRun, 'True'))
 
       - stage: CreatePRs
         dependsOn:
@@ -295,10 +369,11 @@ extends:
         ################################################################################
           displayName: Create PRs in AzureDevOps and ConfigChange
           variables:
+            IsTestRun: $[ stageDependencies.Verify_release.Set_variables.outputs['SetReleaseVariables.isTestRun'] ]
             IsRelease: $[ stageDependencies.Verify_release.Set_variables.outputs['SetReleaseVariables.isRelease'] ]
             ReleaseBranch: $[ stageDependencies.Verify_release.Set_variables.outputs['SetReleaseVariables.releaseBranch'] ]
             AgentVersion: $[ stageDependencies.Verify_release.Set_variables.outputs['SetReleaseVariables.agentVersion'] ]
-          condition: and(succeeded(), eq(variables.IsRelease, 'True'))
+          condition: and(succeeded(), or(eq(variables.IsRelease, 'True'), eq(variables.IsTestRun, 'True')))
           steps:
           - checkout: self
 
@@ -320,7 +395,7 @@ extends:
               cd release
               npm install
               ls
-              node createAdoPrs.js $(AgentVersion)
+              node createAdoPrs.js $(AgentVersion) --dryrun=$(IsTestRun)
             name: s_CreateAdoPrs
             displayName: Create PRs in AzureDevOps and ConfigChange
             env:
@@ -341,6 +416,7 @@ extends:
         - job: j_SendPRsNotifications
           displayName: Send Release PRs notifications
           variables:
+            IsTestRun: $[ stageDependencies.Verify_release.Set_variables.outputs['SetReleaseVariables.isTestRun'] ]
             IsRelease: $[ stageDependencies.Verify_release.Set_variables.outputs['SetReleaseVariables.isRelease'] ]
             AdoPrId: $[ stageDependencies.CreatePRs.create_ado_prs.outputs['s_CreateAdoPrs.AdoPrId'] ]
             AdoPrLink: $[ stageDependencies.CreatePRs.create_ado_prs.outputs['s_CreateAdoPrs.AdoPrLink'] ]
@@ -350,6 +426,7 @@ extends:
             and(
               not(${{ parameters.disableNotifications }}),
               eq(variables.IsRelease, 'True'),
+              eq(variables.IsTestRun, 'False'),
               not(${{ parameters.onlyGitHubRelease }})
             )
           steps:
diff --git a/release/createAdoPrs.js b/release/createAdoPrs.js
@@ -85,7 +85,7 @@ async function openPR(repo, project, sourceBranch, targetBranch, commitMessage,
 
     if (!fs.existsSync(repoPath)) {
         const gitUrl = `https://${process.env.PAT}@${orgUrl}/${project}/_git/${repo}`;
-        util.execInForeground(`${GIT} clone --depth 1 ${gitUrl} ${repoPath}`, null, opt.dryrun);
+        util.execInForeground(`${GIT} clone --depth 1 ${gitUrl} ${repoPath}`, null, opt.options.dryrun);
     }
 
     for (const targetToCommit of targetsToCommit) {
@@ -105,12 +105,12 @@ async function openPR(repo, project, sourceBranch, targetBranch, commitMessage,
     }
 
     for (const targetToCommit of targetsToCommit) {
-        util.execInForeground(`${GIT} add ${targetToCommit}`, repoPath, opt.dryrun);
+        util.execInForeground(`${GIT} add ${targetToCommit}`, repoPath, opt.options.dryrun);
     }
 
-    util.execInForeground(`${GIT} checkout -b ${sourceBranch}`, repoPath);
-    util.execInForeground(`${GIT} commit -m "${commitMessage}"`, repoPath);
-    util.execInForeground(`${GIT} push --force origin ${sourceBranch}`, repoPath);
+    util.execInForeground(`${GIT} checkout -b ${sourceBranch}`, repoPath, opt.options.dryrun);
+    util.execInForeground(`${GIT} commit -m "${commitMessage}"`, repoPath, opt.options.dryrun);
+    util.execInForeground(`${GIT} push --force origin ${sourceBranch}`, repoPath, opt.options.dryrun);
 
     const prefix = 'refs/heads/';
 
@@ -129,6 +129,8 @@ async function openPR(repo, project, sourceBranch, targetBranch, commitMessage,
 
     if (PR) {
         console.log('PR already exists');
+    } else if (opt.options.dryrun) {
+        return [-1, 'test']; // return without creating PR for test runs
     } else {
         console.log('PR does not exist; creating PR');
         PR = await gitApi.createPullRequest(pullRequest, repo, project);
@@ -167,8 +169,8 @@ async function main() {
         util.verifyMinimumNodeVersion();
         util.verifyMinimumGitVersion();
         createIntegrationFiles(agentVersion);
-        util.execInForeground(`${GIT} config --global user.email "${process.env.USEREMAIL}"`, null, opt.dryrun);
-        util.execInForeground(`${GIT} config --global user.name "${process.env.USERNAME}"`, null, opt.dryrun);
+        util.execInForeground(`${GIT} config --global user.email "${process.env.USEREMAIL}"`, null, opt.options.dryrun);
+        util.execInForeground(`${GIT} config --global user.name "${process.env.USERNAME}"`, null, opt.options.dryrun);
 
         const sprint = await getCurrentSprint();
 
diff --git a/src/Agent.Listener/CommandSettings.cs b/src/Agent.Listener/CommandSettings.cs
@@ -72,12 +72,18 @@ public CommandSettings(IHostContext context, string[] args, IScopedEnvironment e
                 context.SecretMasker.AddValue(Configure.SslClientCert, WellKnownSecretAliases.ConfigureSslClientCert);
                 context.SecretMasker.AddValue(Configure.Token, WellKnownSecretAliases.ConfigureToken);
                 context.SecretMasker.AddValue(Configure.WindowsLogonPassword, WellKnownSecretAliases.ConfigureWindowsLogonPassword);
+                context.SecretMasker.AddValue(Configure.TenantId, WellKnownSecretAliases.ConfigureTenantId);
+                context.SecretMasker.AddValue(Configure.ClientId, WellKnownSecretAliases.ConfigureClientId);
+                context.SecretMasker.AddValue(Configure.ClientSecret, WellKnownSecretAliases.ConfigureClientSecret);
             }
 
             if (Remove != null)
             {
                 context.SecretMasker.AddValue(Remove.Password, WellKnownSecretAliases.RemovePassword);
                 context.SecretMasker.AddValue(Remove.Token, WellKnownSecretAliases.RemoveToken);
+                context.SecretMasker.AddValue(Remove.TenantId, WellKnownSecretAliases.RemoveTenantId);
+                context.SecretMasker.AddValue(Remove.ClientId, WellKnownSecretAliases.RemoveClientId);
+                context.SecretMasker.AddValue(Remove.ClientSecret, WellKnownSecretAliases.RemoveClientSecret);
             }
 
             PrintArguments();
diff --git a/src/Agent.Sdk/Knob/AgentKnobs.cs b/src/Agent.Sdk/Knob/AgentKnobs.cs
@@ -582,7 +582,7 @@ public class AgentKnobs
         public static readonly Knob UseProcessHandlerV2 = new Knob(
             nameof(UseProcessHandlerV2),
             "Enables new Process handler (v2)",
-            new RuntimeKnobSource("AGENT_USE_PROCESS_HANDLER_V2"),
+            new PipelineFeatureSource("UseProcessHandlerV2"),
             new BuiltInDefaultKnobSource("false"));
 
         public static readonly Knob DisableDrainQueuesAfterTask = new Knob(
diff --git a/src/Agent.Sdk/Util/WellKnownSecretAliases.cs b/src/Agent.Sdk/Util/WellKnownSecretAliases.cs
@@ -11,8 +11,14 @@ public static class WellKnownSecretAliases
         public static readonly string ConfigureSslClientCert = "Configure.SslClientCert";
         public static readonly string ConfigureToken = "Configure.Token";
         public static readonly string ConfigureWindowsLogonPassword = "Configure.WindowsLogonPassword";
+        public static readonly string ConfigureClientId = "Configure.ClientId";
+        public static readonly string ConfigureClientSecret = "Configure.ClientSecret";
+        public static readonly string ConfigureTenantId = "Configure.TenantId";
         public static readonly string RemovePassword = "Remove.Password";
         public static readonly string RemoveToken = "Remove.Token";
+        public static readonly string RemoveClientId = "Remove.ClientId";
+        public static readonly string RemoveClientSecret = "Remove.ClientSecret";
+        public static readonly string RemoveTenantId = "Remove.TenantId";
 
         // Other known origins for secrets
         public static readonly string GitSourceProviderAuthHeader = "GitSourceProvider.AuthHeader";

Original file line number	Diff line number	Diff line change
`@@ -72,12 +72,18 @@ public CommandSettings(IHostContext context, string[] args, IScopedEnvironment e`
`72`	`72`	`context.SecretMasker.AddValue(Configure.SslClientCert, WellKnownSecretAliases.ConfigureSslClientCert);`
`73`	`73`	`context.SecretMasker.AddValue(Configure.Token, WellKnownSecretAliases.ConfigureToken);`
`74`	`74`	`context.SecretMasker.AddValue(Configure.WindowsLogonPassword, WellKnownSecretAliases.ConfigureWindowsLogonPassword);`
	`75`	`+ context.SecretMasker.AddValue(Configure.TenantId, WellKnownSecretAliases.ConfigureTenantId);`
	`76`	`+ context.SecretMasker.AddValue(Configure.ClientId, WellKnownSecretAliases.ConfigureClientId);`
	`77`	`+ context.SecretMasker.AddValue(Configure.ClientSecret, WellKnownSecretAliases.ConfigureClientSecret);`
`75`	`78`	`}`
`76`	`79`
`77`	`80`	`if (Remove != null)`
`78`	`81`	`{`
`79`	`82`	`context.SecretMasker.AddValue(Remove.Password, WellKnownSecretAliases.RemovePassword);`
`80`	`83`	`context.SecretMasker.AddValue(Remove.Token, WellKnownSecretAliases.RemoveToken);`
	`84`	`+ context.SecretMasker.AddValue(Remove.TenantId, WellKnownSecretAliases.RemoveTenantId);`
	`85`	`+ context.SecretMasker.AddValue(Remove.ClientId, WellKnownSecretAliases.RemoveClientId);`
	`86`	`+ context.SecretMasker.AddValue(Remove.ClientSecret, WellKnownSecretAliases.RemoveClientSecret);`
`81`	`87`	`}`
`82`	`88`
`83`	`89`	`PrintArguments();`