[v3.x] Add 1ES CI pipelines (#10247)

* Onboard v3.x to 1ES templates

* update nuget feeds

* Revert nuget.config feeds. Remove unused tools

* Fix buildNumber

* Use .net 8 SDK, fix version propagation

* Fix update build number

* Install additional .NET SDKs

* Fix/suppress cred scan hits in test files

* Remove performance app from legacy ci

* update 1es tests

* Test release

* Fix non-e2e env

* Fix webhost bin config path

* fix TestHelpers

* Fix missing variable

* Set AzureWebJobsEventHubPath in test env

* Fix .net 3.1 missing for tests

* Remove test change

* Fix .net 3.1 missing for tests

Author: jviau

.nuget/NuGet.Config

@@ -1,7 +1,8 @@
 <?xml version="1.0" encoding="utf-8"?>
 <configuration>
   <packageSources>
-    <add key="nuget.org" value="https://www.nuget.org/api/v2/" />
+    <clear />
+    <add key="nuget.org" value="https://api.nuget.org/v3/index.json" />
     <add key="AzureFunctionsTempStaging" value="https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/AzureFunctionsTempStaging/nuget/v3/index.json" />
     <add key="azure-appservice-test" value="https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/azure-appservice-test%40Local/nuget/v3/index.json" />
     <add key="Microsoft.Azure.Functions.PowerShellWorker" value="https://azfunc.pkgs.visualstudio.com/e6a70c92-4128-439f-8012-382fe78d6396/_packaging/Microsoft.Azure.Functions.PowerShellWorker/nuget/v3/index.json" />
@@ -10,4 +11,4 @@
     <!-- dotnet-tools is required for Microsoft.CodeAnalysis.CSharp.Analyzer.Testing.XUnit -->
     <add key="dotnet-tools" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json"/>
   </packageSources>
-</configuration>
+</configuration>

.nuget/packages.config

@@ -49,9 +49,6 @@ jobs:
     buildNumberSuffix: $[variables.buildNumberSuffixTemp]
     packSuffixSwitch: $[variables.packSuffixSwitchTemp]
     emgSuffixSwitch: $[variables.emgSuffixSwitchTemp]
-    majorVersion: $[dependencies.InitializePipeline.outputs['Initialize.MajorVersion']]
-    minorVersion: $[dependencies.InitializePipeline.outputs['Initialize.MinorVersion']]
-    patchVersion: $[dependencies.InitializePipeline.outputs['Initialize.PatchVersion']]
   pool:
     name: '1ES-Hosted-AzFunc'
     demands:
@@ -68,7 +65,7 @@ jobs:
     displayName: "Build artifacts"
     inputs:
       filePath: '$(Build.Repository.LocalPath)\build\build-extensions.ps1'
-      arguments: '-buildNumber "$(buildNumberSuffix)" -majorMinorVersion "$(majorVersion).$(minorVersion)" -patchVersion "$(patchVersion)" -suffix "$(suffix)" -commitHash "$(Build.SourceVersion)"'
+      arguments: '-buildNumber "$(buildNumberSuffix)" -suffix "$(suffix)" -commitHash "$(Build.SourceVersion)"'
   - task: PowerShell@2
     condition: eq(variables['RUNBUILDFORINTEGRATIONTESTS'], 'True')
     displayName: "Update host references"
@@ -112,14 +109,6 @@ jobs:
       publishVstsFeed: 'e6a70c92-4128-439f-8012-382fe78d6396/f37f760c-aebd-443e-9714-ce725cd427df'
       allowPackageConflicts: true
 
-  - task: DotNetCoreCLI@2
-    displayName: 'Build performance package'
-    inputs:
-      command: 'custom'
-      custom: 'pack'
-      arguments: '-o WebJobs.Script.Performance.App'
-      projects: |
-        **\WebJobs.Script.Performance.App.csproj
   - task: DotNetCoreCLI@2
     displayName: 'Build Abstractions and ExtensionsMetadataGenerator'
     inputs:
@@ -325,13 +314,6 @@ jobs:
       Verbosity: 'Information'
   - publish: $(Build.Repository.LocalPath)\packages
     artifact: NugetPackages
-  - task: ManifestGeneratorTask@0
-    displayName: 'SBOM Generation Task - Performance'
-    inputs:
-      BuildDropPath: '$(Build.Repository.LocalPath)\WebJobs.Script.Performance.App'
-      Verbosity: 'Information'
-  - publish: $(Build.Repository.LocalPath)\WebJobs.Script.Performance.App
-    artifact: Performance
 
 - job: RunUnitTests
   pool:

NuGet.config

@@ -0,0 +1,16 @@
+# Get major, minor and patchVersions
+$version = @{}
+[xml]$XMLContents = [xml](Get-Content -Path "$PSScriptRoot\common.props")
+$XMLContents.GetElementsByTagName("MajorVersion") |  ForEach-Object {
+  $version.Major = $_.InnerText
+}
+
+$XMLContents.GetElementsByTagName("MinorVersion") |  ForEach-Object {
+  $version.Minor = $_.InnerText
+}
+
+$XMLContents.GetElementsByTagName("PatchVersion") |  ForEach-Object {
+  $version.Patch = $_.InnerText
+}
+
+return $version

azure-pipelines.yml

@@ -1,12 +1,14 @@
 param (
   [string]$buildNumber,
-  [string]$majorMinorVersion,
-  [string]$patchVersion,
   [string]$suffix = "",
   [string]$commitHash = "N/A",
   [string]$hashesForHardlinksFile = "hashesForHardlinks.txt"
 )
 
+$version = & $PSScriptRoot\Get-AzureFunctionsVersion.ps1
+$majorMinorVersion = "$($version.Major).$($version.Minor)"
+$patchVersion = $version.Patch
+
 $extensionVersion = "$majorMinorVersion.$patchVersion"
 Write-Host "ExtensionVersion is $extensionVersion"
 Write-Host "BuildNumber is $buildNumber"

build/Get-AzureFunctionsVersion.ps1

@@ -16,14 +16,22 @@
     <CodeAnalysisRuleSet>$(MSBuildThisFileDirectory)..\src.ruleset</CodeAnalysisRuleSet>
     <NoWarn>$(NoWarn);NU1701</NoWarn>
     <DebugType>embedded</DebugType>
-    <UseSourceLink Condition="$(UseSourceLink) == '' And $(CI) != ''">true</UseSourceLink>
-	<Deterministic>True</Deterministic>
+    <Deterministic>True</Deterministic>
+    <NugetAudit>false</NugetAudit>
   </PropertyGroup>
 
   <PropertyGroup Condition="'$(UseSourceLink)' == 'true'">
     <SourceLink>$(BaseIntermediateOutputPath)source_link.json</SourceLink>
   </PropertyGroup>
-  
+
+  <PropertyGroup>
+    <!--
+      NOTE: source link does not actually work for v3.x in 1ES as the git remote url is Azure DevOps, not GitHub.
+      However, we do not release public nuget packages from this branch anymore so it is not even used.
+    -->
+    <UseSourceLink Condition="$(UseSourceLink) == '' And $(CI) != ''">true</UseSourceLink>
+  </PropertyGroup>
+
   <Target Name="GenerateSourceLink" BeforeTargets="CoreCompile" Condition="'$(UseSourceLink)' == 'true'">
     <PropertyGroup>
       <SrcRootDirectory>$([System.IO.Directory]::GetParent($(MSBuildThisFileDirectory.TrimEnd("\"))))</SrcRootDirectory>

build/build-extensions.ps1

@@ -17,29 +17,21 @@ if ($buildReason -eq "PullRequest") {
 }
 
 # Get major, minor and patchVersions
-[xml]$XMLContents = [xml](Get-Content -Path ".\build\common.props")
-$XMLContents.GetElementsByTagName("MajorVersion") |  ForEach-Object {
-  $majorVersion = $_.InnerText
-  Write-Host "##vso[task.setvariable variable=MajorVersion;isOutput=true]$majorVersion"
-  Write-Host "Setting 'MajorVersion' to $majorVersion"
-}
+$version = & $PSScriptRoot\Get-AzureFunctionsVersion.ps1
+Write-Host "##vso[task.setvariable variable=MajorVersion;isOutput=true]$($version.Major)"
+Write-Host "Setting 'MajorVersion' to $($version.Major)"
 
-$XMLContents.GetElementsByTagName("MinorVersion") |  ForEach-Object {
-  $minorVersion = $_.InnerText
-  Write-Host "##vso[task.setvariable variable=MinorVersion;isOutput=true]$minorVersion"
-  Write-Host "Setting 'MinorVersion' to $minorVersion"
-}
+Write-Host "##vso[task.setvariable variable=MinorVersion;isOutput=true]$($version.Minor)"
+Write-Host "Setting 'MinorVersion' to $($version.Minor)"
 
-$XMLContents.GetElementsByTagName("PatchVersion") |  ForEach-Object {
-  $patchVersion = $_.InnerText
-  Write-Host "##vso[task.setvariable variable=PatchVersion;isOutput=true]$patchVersion"
-  Write-Host "Setting 'PatchVersion' to $patchVersion"
-}
+Write-Host "##vso[task.setvariable variable=PatchVersion;isOutput=true]$($version.Patch)"
+Write-Host "Setting 'PatchVersion' to $($version.Patch)"
 
 #Update buildnumber with the same (Will be used by release pipelines)
-$customBuildNumber = "$majorVersion.$minorVersion.$patchVersion"
+$customBuildNumber = "$($version.Major).$($version.Minor).$($version.Patch)"
 if(($buildReason -eq "PullRequest") -or !($sourceBranch.ToLower().Contains("release")))
 {
   $customBuildNumber = "$customBuildNumber-$buildNumber"
 }
-Write-Host "##vso[build.updatebuildnumber]$customBuildNumber"
+
+Write-Host "##vso[build.updatebuildnumber]$customBuildNumber"

build/common.props

@@ -0,0 +1,18 @@
+trigger:
+  branches:
+    include:
+    - v3.x
+    - release/3.*
+
+resources:
+  repositories:
+  - repository: eng
+    type: git
+    name: engineering
+    ref: refs/tags/release
+
+variables:
+  - template: ci/variables/cfs.yml@eng
+
+extends:
+  template: ci/code-mirror.yml@eng

build/initialize-pipeline.ps1

@@ -0,0 +1,37 @@
+trigger: none # ensure this is not ran as a CI build
+
+pr:
+  branches:
+    include:
+    - v3.x
+    - release/3.*
+
+resources:
+  repositories:
+  - repository: 1es
+    type: git
+    name: 1ESPipelineTemplates/1ESPipelineTemplates
+    ref: refs/tags/release
+  - repository: eng
+    type: git
+    name: engineering
+    ref: refs/tags/release
+
+variables:
+  - template: /eng/ci/templates/variables/build.yml@self
+  - template: /ci/variables/cfs.yml@eng
+
+extends:
+  template: v1/1ES.Unofficial.PipelineTemplate.yml@1es
+  parameters:
+    pool:
+      name: 1es-pool-azfunc
+      image: 1es-windows-2022
+      os: windows
+
+    stages:
+    - stage: Test
+      jobs:
+      - template: /eng/ci/templates/jobs/initialize-pipeline.yml@self
+      - template: /eng/ci/templates/official/jobs/run-non-e2e-tests.yml@self
+      - template: /eng/ci/templates/official/jobs/run-integration-tests.yml@self