Refactor CI artifacts for release improvements (#10704)

* Consolidate artifacts

* Disable sdl spotBugs

* Remove PrivateSiteExtension

* stage release_notes.md as artifact

* Add script to generate private site extension

* Copy release_notes.md in site extension

* Flatten SiteExtension artifacts, custom hash task

* Generate release.json during build

* modifyOutputPath: false

* Use SHA256 for the hash

* Default private siteext to 64bit

* Default private siteext to zipping

* Remove binlog artifact to save space

* Update SiteExtension readme

* Update src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1

Co-authored-by: Lilian Kasem <[email protected]>

---------

Co-authored-by: Lilian Kasem <[email protected]>

Author: jviau

eng/ci/official-build.yml

@@ -70,6 +70,8 @@ extends:
       codeql:
         # Move codeql for source languages to source analysis stage
         runSourceLanguagesInSourceAnalysis: true
+      spotBugs:
+        enabled: false # Explicitly disabling as our JavaWorker package causes this to light up and subsequently fail.
 
     stages:
     - stage: Build

eng/ci/templates/official/jobs/build-artifacts-linux.yml

@@ -6,26 +6,19 @@ jobs:
     project: src/WebJobs.Script.WebHost/WebJobs.Script.WebHost.csproj
     configuration: release
     runtime: linux-x64
-    artifacts_path: $(Build.ArtifactStagingDirectory)
-    log_dir: $(artifacts_path)/log
-    zip_artifacts_path: $(artifacts_path)/Linux
+    drop_path: $(Build.ArtifactStagingDirectory)
+    linux_drop_path: $(drop_path)/linux
     build_args: '-v m -c $(configuration) -r $(runtime) --self-contained true'
 
   templateContext:
-    outputParentDirectory: $(artifacts_path)
+    outputParentDirectory: $(drop_path)
     outputs:
     # TODO: onboard to Azure Artifacts Drops to allow accessing this from docker linux pipeline in msazure
     # https://eng.ms/docs/cloud-ai-platform/devdiv/one-engineering-system-1es/1es-docs/azure-artifacts/artifact-services-onboarding
     - output: pipelineArtifact
       displayName: Publish linux artifacts
-      path: $(zip_artifacts_path)
-      artifact: Linux
-    - output: pipelineArtifact
-      displayName: Publish logs
-      path: $(log_dir)
-      artifact: Linux_Log
-      sbomEnabled: false
-      condition: always()
+      path: $(linux_drop_path)
+      artifact: drop_linux
 
   pool:
     name: 1es-pool-azfunc
@@ -41,15 +34,15 @@ jobs:
       command: custom
       custom: restore
       projects: $(project)
-      arguments: '-v m -r $(runtime) -p:PublishReadyToRun=true -bl:$(log_dir)/restore.binlog' # add PublishReadyToRun=true for https://github.com/dotnet/sdk/issues/20701
+      arguments: '-v m -r $(runtime) -p:PublishReadyToRun=true' # add PublishReadyToRun=true for https://github.com/dotnet/sdk/issues/20701
 
   - task: DotNetCoreCLI@2
     displayName: Build
     inputs:
       command: custom
       custom: build
       projects: $(project)
-      arguments: '$(build_args) --no-restore  -bl:$(log_dir)/build.binlog'
+      arguments: '$(build_args) --no-restore'
 
   - task: DotNetCoreCLI@2
     displayName: Publish
@@ -58,5 +51,6 @@ jobs:
       custom: publish
       publishWebProjects: false # we use our own publish logic
       zipAfterPublish: false # we use our own zip logic
+      modifyOutputPath: false
       projects: $(project)
-      arguments: '$(build_args) --no-build -p:ZipArtifactsPath=$(zip_artifacts_path) -bl:$(log_dir)/publish.binlog'
+      arguments: '$(build_args) --no-build -o $(linux_drop_path)/host'

eng/ci/templates/official/jobs/build-artifacts-windows.yml

@@ -8,45 +8,34 @@ jobs:
     os: windows
 
   variables:
-    nuget_package_path: $(Build.ArtifactStagingDirectory)/NugetPackages
-    log_dir: $(Build.ArtifactStagingDirectory)/log
+    drop_path: $(Build.ArtifactStagingDirectory)
+    nuget_drop_path: $(drop_path)/nuget
+    windows_drop_path: $(drop_path)/windows
 
   templateContext:
-    outputParentDirectory: $(Build.ArtifactStagingDirectory)
+    outputParentDirectory: $(drop_path)
     outputs:
     - output: pipelineArtifact
       displayName: Publish site extension
-      path: $(Build.ArtifactStagingDirectory)/SiteExtension
-      artifact: SiteExtension
-    - output: pipelineArtifact
-      displayName: Publish private site extension
-      path: $(Build.ArtifactStagingDirectory)/PrivateSiteExtension
-      artifact: PrivateSiteExtension
-    - output: pipelineArtifact
-      displayName: Publish site extension symbols
-      path: $(Build.ArtifactStagingDirectory)/SiteExtensionSymbols
-      artifact: Symbols
+      path: $(windows_drop_path)
+      artifact: drop_windows
     - output: pipelineArtifact
       displayName: Publish nuget packages
-      path: $(Build.ArtifactStagingDirectory)/NugetPackages
-      artifact: NugetPackages
-    - output: pipelineArtifact
-      displayName: Publish logs
-      path: $(log_dir)
-      artifact: Windows_Log
-      sbomEnabled: false
-      condition: always()
+      path: $(nuget_drop_path)
+      artifact: drop_nuget
 
   steps:
   - template: /eng/ci/templates/install-dotnet.yml@self
   - template: /eng/ci/templates/steps/build-site-ext.yml@self
+    parameters:
+      publishDir: $(windows_drop_path)
 
   - task: DotNetCoreCLI@2
     displayName: Build host packages
     inputs:
       command: custom
       custom: pack
-      arguments: -c release -o $(nuget_package_path)
+      arguments: -c release -o $(nuget_drop_path)
       projects: |
         **/WebJobs.Script.csproj
         **/WebJobs.Script.WebHost.csproj
@@ -74,7 +63,7 @@ jobs:
     inputs:
       command: custom
       custom: pack
-      arguments: '--no-build -c release -o $(nuget_package_path)'
+      arguments: '--no-build -c release -o $(nuget_drop_path)'
       projects: |
         **/WebJobs.Script.Abstractions.csproj
 
@@ -91,7 +80,7 @@ jobs:
     inputs:
       command: custom
       custom: pack
-      arguments: '--no-build -c release -o $(nuget_package_path)'
+      arguments: '--no-build -c release -o $(nuget_drop_path)'
       projects: |
         **/ExtensionsMetadataGenerator.csproj
 

eng/ci/templates/steps/build-site-ext.yml

@@ -2,6 +2,9 @@ parameters:
 - name: project
   type: string
   default: src/WebJobs.Script.SiteExtension/WebJobs.Script.SiteExtension.csproj
+- name: publishDir
+  type: string
+  default: ''
 
 steps:
 # Restore must be a separate step so we can pass in 'PublishReadyToRun=true'
@@ -11,15 +14,15 @@ steps:
     command: custom
     custom: restore
     projects: ${{ parameters.project }}
-    arguments: '-v m -p:PublishReadyToRun=true -bl:$(log_dir)/site_ext.restore.binlog'
+    arguments: '-v m -p:PublishReadyToRun=true'
 
 - task: DotNetCoreCLI@2
   displayName: Build site extension
   inputs:
     command: custom
     custom: build
     projects: ${{ parameters.project }}
-    arguments: '--no-restore -v m -c release -bl:$(log_dir)/site_ext.build.binlog'
+    arguments: '--no-restore -v m -c release'
 
 - task: DotNetCoreCLI@2
   displayName: Publish site extension
@@ -28,5 +31,9 @@ steps:
     custom: publish
     publishWebProjects: false # we use our own publish logic
     zipAfterPublish: false # we use our own zip logic
+    modifyOutputPath: false
     projects: ${{ parameters.project }}
-    arguments: '--no-build -v m -c release -p:ZipArtifactsPath=$(Build.ArtifactStagingDirectory) -bl:$(log_dir)/site_ext.publish.binlog'
+    ${{ if eq(parameters.publishDir, '') }}:
+      arguments: '--no-build -v m -c release'
+    ${{ else }}:
+      arguments: '--no-build -v m -c release -o ${{ parameters.publishDir }}'

src/WebJobs.Script.SiteExtension/New-PrivateSiteExtension.ps1

@@ -0,0 +1,130 @@
+<#
+    .SYNOPSIS
+    Produces a private site extension.
+
+    .DESCRIPTION
+    Takes in a published site extension and produces a private site extension.
+
+    .PARAMETER InputPath
+    The path of the published 'SiteExtension'. Leave null to scan for root in a child from here.
+
+    .PARAMETER OutputPath
+    The path to produce the private site extension to (either the zip file or folder). Leave null to compute this name.
+
+    .PARAMETER Bitness
+    The bitness to produce the private site extension with. Default is '64bit'.
+
+    .PARAMETER NoZip
+    [Switch] Include to produce site extension as a folder and not a zip.
+
+    .PARAMETER Force
+    [Switch] Include to overwrite existing files.
+
+    .INPUTS
+    None. You can't pipe objects to Update-Month.ps1.
+
+    .OUTPUTS
+    None. Update-Month.ps1 doesn't generate any output.
+#>
+
+param (
+    [string] $InputPath = $null,
+    [string] $OutputPath = $null,
+    [ValidateSet('x64', '64bit', 'x86', '32bit')][string] $Bitness = '64bit',
+    [switch] $NoZip,
+    [switch] $Force
+)
+
+$normalizeBitness = @{
+    'x64' = '64bit'
+    '64bit' = '64bit'
+    'x86' = '64bit'
+    '32bit' = '32bit'
+}
+
+$Bitness = $normalizeBitness[$Bitness]
+
+if (-not $InputPath)
+{
+    $InputPath = (Get-ChildItem -Path . -Filter "extension.xml" -Recurse).Directory.FullName
+}
+
+if (Test-Path (Join-Path $InputPath "WebJobs.Script.SiteExtension.csproj"))
+{
+    Write-Error "This script should not be ran in the WebJobs.Script.SiteExtension project folder. Run this script in the root of the published site extension folder."
+    exit 1
+}
+
+if (-not (Join-Path $InputPath "extension.xml" | Test-Path))
+{
+    Write-Error "Unable to find published site extension."
+    exit 1
+}
+
+if (-not $OutputPath)
+{
+    $runtime = $Bitness -eq '32bit' ? 'win-x86' : 'win-x64'
+    $leaf = (Split-Path $InputPath -Leaf)
+    $split = $leaf.IndexOf('.')
+    $OutputPath = "$($leaf.Substring(0, $split)).Private.$($leaf.Substring($split + 1)).$runtime"
+}
+
+function New-TemporaryDirectory {
+    $tmp = [System.IO.Path]::GetTempPath()
+    $name = (New-Guid).ToString("N")
+    return New-Item -ItemType Directory -Path (Join-Path $tmp $name)
+}
+
+function Write-Zip ($outputPath)
+{
+    if (Test-Path $outputPath) {
+        if ($Force) {
+            Remove-Item -Path $outputPath -Recurse -Force
+        } else {
+            Write-Error "OutputPath already exists. Use -Force to overwrite."
+            exit 1
+        }
+    }
+
+    $tempDir = New-TemporaryDirectory
+    Write-Folder $tempDir
+
+    Compress-Archive -Path "$tempDir/*" -DestinationPath $outputPath
+    Remove-Item -Path $tempDir -Recurse -Force
+}
+
+function Write-Folder ($outputPath)
+{
+    if (Test-Path "$outputPath/*") {
+        if ($Force) {
+            Remove-Item -Path $outputPath -Recurse -Force
+        } else {
+            Write-Error "OutputPath already exists. Use -Force to overwrite."
+            exit 1
+        }
+    }
+
+    $inputPath = Get-ChildItem -Path $InputPath -Attributes Directory
+    $outputPath = Join-Path $outputPath "SiteExtensions" "Functions"
+    New-Item -ItemType Directory -Path $outputPath | Out-Null
+
+    Copy-Item "$inputPath/applicationHost.xdt" -Destination $outputPath | Out-Null
+
+    $filesDest = (Join-Path $outputPath "$Bitness")
+    Copy-Item "$inputPath/$Bitness/" -Destination "$filesDest/" -Container -Recurse | Out-Null
+
+    $workerDest = (Join-Path $filesDest "workers")
+    Copy-Item "$inputPath/workers/" -Destination "$workerDest/" -Container -Recurse | Out-Null
+}
+
+if ($NoZip) {
+    Write-Folder $OutputPath
+} else {
+    if (-not $OutputPath.EndsWith(".zip")) {
+        $OutputPath = "$OutputPath.zip"
+    }
+
+    Write-Zip $OutputPath
+}
+
+Write-Host "Published private site extension to $OutputPath"

src/WebJobs.Script.SiteExtension/New-ReleaseJson.ps1

@@ -0,0 +1,41 @@
+param(
+    [string] $Version = $null,
+    [string[]] $Artifacts = $null,
+    [string] $CommitId = $null,
+    [string] $OutputPath
+)
+
+if (-not $Version) {
+    $Version = dotnet build $PSScriptRoot --getProperty:Version
+}
+
+if (-Not $Artifacts) {
+    $Artifacts = @(
+        "Functions.$Version"
+        "Functions.Symbols.$Version.win-x64"
+        "Functions.Symbols.$Version.win-x86"
+    )
+}
+
+if (-not $CommitId) {
+    $CommitId = $env:BUILD_SOURCEVERSION
+}
+
+if (-not $CommitId) {
+    $CommitId = (git rev-parse HEAD).Trim()
+}
+
+$obj = @{
+    name = $Version
+    artifacts = $Artifacts
+    tag = "v$Version"
+    commitId = $CommitId
+    releaseNotesFile = "release_notes.md"
+}
+
+if ($OutputPath) {
+    Write-Host "Writing $obj to $OutputPath"
+    $obj | ConvertTo-Json | Out-File $OutputPath
+}
+
+return $obj