Update host and consolidation CI pipelines (#4488)

Author: liliankasem

eng/ci/host-artifacts-build.yml

@@ -45,4 +45,38 @@ extends:
     stages:
       - stage: BuildCoreToolsHost
         jobs:
-        - template: /eng/ci/templates/official/jobs/host-build-pack.yml@self
+        - template: /eng/ci/templates/official/jobs/host-build-pack.yml@self
+
+      - stage: Merge
+        dependsOn: 'BuildCoreToolsHost'
+
+        jobs:
+        - job: MergeArtifacts
+          displayName: 'Merge artifacts'
+
+          templateContext:
+            type: artifactJob
+
+            inputs:
+              - input: pipelineArtifact
+                artifactName: func-host-windows-signed
+                targetPath: $(Build.ArtifactStagingDirectory)/func-cli-host/windows
+
+              - input: pipelineArtifact
+                artifactName: func-host-linux-signed
+                targetPath: $(Build.ArtifactStagingDirectory)/func-cli-host/linux
+
+            outputs:
+              - output: pipelineArtifact
+                targetPath: $(Build.ArtifactStagingDirectory)/func-cli-host
+                artifactName: 'func-cli-host'
+
+      - stage: UploadToNightlyFeed
+        dependsOn: 'Merge'
+        condition: and(succeeded(), eq(variables['Build.Reason'], 'schedule'))
+
+        jobs:
+        - template: /eng/ci/templates/official/jobs/upload-artifact-to-feed.yml@self
+          parameters:
+            artifactName: 'func-cli-host'
+            packageName: 'func-cli-host'

eng/ci/official-build.yml

@@ -66,8 +66,12 @@ extends:
         condition: and(succeeded(), eq(variables['Build.Reason'], 'schedule'))
 
         jobs:
-        - template: /eng/ci/templates/official/jobs/upload-cli-to-feed.yml@self
+        - template: /eng/ci/templates/official/jobs/upload-artifact-to-feed.yml@self
+          parameters:
+            artifactName: 'func-cli'
+            packageName: 'func-cli'
 
+      # Test Stages
       - stage: E2ETestWindows
         dependsOn: ''
 

eng/ci/templates/official/jobs/consolidate-cli-artifacts.yml

@@ -9,29 +9,24 @@ parameters:
 jobs:
 - job: AssembleArtifact${{ parameters.displayName }}
   displayName: "Assemble ${{ parameters.arch }}"
+
   templateContext:
     ${{ if eq(parameters.isNightlyBuild, false) }}:
       inputs:
         - input: pipelineArtifact
           pipeline: core-tools-host
-          artifactName: func-host-linux-signed
-          targetPath: $(Pipeline.Workspace)/core-tools-host/func-host-linux-signed
-        - input: pipelineArtifact
-          pipeline: core-tools-host
-          artifactName: func-host-windows-signed
-          targetPath: $(Pipeline.Workspace)/core-tools-host/func-host-windows-signed
-        - input: pipelineArtifact
-          pipeline: core-tools-inproc
-          artifactName: func-cli-inproc6
-          targetPath: $(Pipeline.Workspace)/core-tools-inproc/func-cli-inproc6
+          artifactName: func-cli-host
+          targetPath: $(Pipeline.Workspace)/func-cli-host/
+
         - input: pipelineArtifact
           pipeline: core-tools-inproc
-          artifactName: func-cli-inproc8
-          targetPath: $(Pipeline.Workspace)/core-tools-inproc/func-cli-inproc8
+          artifactName: func-cli-inproc
+          targetPath: $(Pipeline.Workspace)/func-cli-inproc/
+
         - input: pipelineArtifact
           pipeline: core-tools-default
           artifactName: func-cli
-          targetPath: $(Pipeline.Workspace)/core-tools-default/func-cli
+          targetPath: $(Pipeline.Workspace)/func-cli-default/func-cli
 
     outputParentDirectory: $(Build.ArtifactStagingDirectory)
     outputs:
@@ -48,54 +43,34 @@ jobs:
   steps:
   - checkout: self
 
-  # Conditional download of artifacts based on isNightlyBuild parameter
+  # Conditional download of artifacts based on isNightlyBuild parameter from ADO feed
   - ${{ if eq(parameters.isNightlyBuild, true) }}:
-    # Download from ADO feed for nightly builds
-    - task: DownloadPackage@1
-      displayName: 'Download core-tools-host (Linux signed) from feed'
-      inputs:
-        packageType: 'upack'
-        feed: 'internal/core-tools-nightly-build'
-        definition: 'func-host-linux-signed'
-        version: '*'
-        downloadPath: '$(Pipeline.Workspace)/core-tools-host/func-host-linux-signed'
-
-    - task: DownloadPackage@1
-      displayName: 'Download core-tools-host (Windows) from feed'
-      inputs:
-        packageType: 'upack'
-        feed: 'internal/core-tools-nightly-build'
-        definition: 'func-host-windows-signed'
-        version: '*'
-        downloadPath: '$(Pipeline.Workspace)/core-tools-host/func-host-windows-signed'
-
     - task: DownloadPackage@1
-      displayName: 'Download core-tools-inproc6 from feed'
+      displayName: 'Download func-cli-host from feed'
       inputs:
         packageType: 'upack'
         feed: 'internal/core-tools-nightly-build'
-        definition: 'func-cli-inproc6'
+        definition: 'func-cli-host'
         version: '*'
-        downloadPath: '$(Pipeline.Workspace)/core-tools-inproc/func-cli-inproc6'
+        downloadPath: '$(Pipeline.Workspace)/func-cli-host'
 
     - task: DownloadPackage@1
-      displayName: 'Download core-tools-inproc8 from feed'
+      displayName: 'Download func-cli-inproc from feed'
       inputs:
         packageType: 'upack'
         feed: 'internal/core-tools-nightly-build'
-        definition: 'func-cli-inproc8'
+        definition: 'func-cli-inproc'
         version: '*'
-        downloadPath: '$(Pipeline.Workspace)/core-tools-inproc/func-cli-inproc8'
+        downloadPath: '$(Pipeline.Workspace)/func-cli-inproc'
 
     - task: DownloadPackage@1
-      displayName: 'Download core-tools-default from feed'
+      displayName: 'Download func-cli from feed'
       inputs:
         packageType: 'upack'
         feed: 'internal/core-tools-nightly-build'
         definition: 'func-cli'
         version: '*'
-        downloadPath: '$(Pipeline.Workspace)/core-tools-default/func-cli'
-
+        downloadPath: '$(Pipeline.Workspace)/func-cli-default/func-cli'
 
   - task: DotNetCoreCLI@2
     displayName: "Run ArtifactAssembler"
@@ -104,29 +79,31 @@ jobs:
       projects: "$(Build.SourcesDirectory)/src/Cli/ArtifactAssembler/Azure.Functions.Cli.ArtifactAssembler.csproj"
       arguments: '-c release -- "Azure.Functions.Cli.${{ parameters.arch }}"'
       workingDirectory: '$(Pipeline.Workspace)'
-
-  - ${{ if eq(parameters.arch, 'min.win-x64') }}: # what about the other min.* architectures?
-    - task: PowerShell@2
-      displayName: 'Generate metadata.json file'
-      inputs:
-        targetType: filePath
-        filePath: '$(Build.SourcesDirectory)/eng/scripts/artifact-assembler/generate-metadata-file.ps1'
-        arguments: '-StagingDirectory "$(Pipeline.Workspace)/staging"'
-        workingDirectory: '$(Pipeline.Workspace)'
+    env:
+      # Directory names containing the artifacts
+      OUT_OF_PROC_ARTIFACT_ALIAS: 'func-cli-default'
+      IN_PROC_ARTIFACT_ALIAS: 'func-cli-inproc'
+      CORETOOLS_HOST_ARTIFACT_ALIAS: 'func-cli-host'
+      # Subdirectory names for the artifacts
+      OUT_OF_PROC_ARTIFACT_NAME: 'func-cli'
+      IN_PROC6_ARTIFACT_NAME: 'inproc6'
+      IN_PROC8_ARTIFACT_NAME: 'inproc8'
+      CORETOOLS_HOST_WINDOWS_ARTIFACT_NAME: 'windows'
+      CORETOOLS_HOST_LINUX_ARTIFACT_NAME: 'linux'
 
   - ${{ if startsWith(parameters.arch, 'win-x') }}:
     - task: PowerShell@2
       displayName: 'Generate MSI files'
       inputs:
         targetType: filePath
         filePath: '$(Build.SourcesDirectory)/eng/scripts/generate-msi-files.ps1'
-        arguments: '-ArtifactsPath "$(Pipeline.Workspace)/staging/coretools-cli"'
+        arguments: '-ArtifactsPath "$(Pipeline.Workspace)/staging/func-cli"'
         workingDirectory: '$(Build.SourcesDirectory)/build'
 
     - template: ci/sign-files.yml@eng
       parameters:
         displayName: Sign MSI files
-        folderPath: $(Pipeline.Workspace)/staging/coretools-cli
+        folderPath: $(Pipeline.Workspace)/staging/func-cli
         pattern: '*.msi'
         signType: 'inline'
         inlineOperation: |
@@ -159,25 +136,15 @@ jobs:
       inputs:
         targetType: filePath
         filePath: '$(Build.SourcesDirectory)/eng/scripts/artifact-assembler/test-vs-artifacts.ps1'
-        arguments: '-StagingDirectory "$(Pipeline.Workspace)/staging/coretools-visualstudio"'
+        arguments: '-StagingDirectory "$(Pipeline.Workspace)/staging/func-cli-visualstudio"'
 
   - ${{ if startsWith(parameters.arch, 'win') }}:
     - task: PowerShell@2
       displayName: 'Test Artifacts'
       inputs:
         targetType: filePath
         filePath: '$(Build.SourcesDirectory)/eng/scripts/artifact-assembler/test-artifacts.ps1'
-        arguments: '-StagingDirectory "$(Pipeline.Workspace)/staging/coretools-cli"'
-      env:
-        DirectoryToLogTo: $(Build.SourcesDirectory)/TestLogs
-      
-    - task: 1ES.PublishPipelineArtifact@1
-      condition: succeededOrFailed()
-      continueOnError: true
-      inputs:
-        targetPath: '$(Build.SourcesDirectory)/TestLogs'
-        artifactName: 'TestLogs_$(System.JobId)_$(System.JobAttempt)'
-        artifactType: 'pipeline'
+        arguments: '-StagingDirectory "$(Pipeline.Workspace)/staging/func-cli"'
 
   - task: DotNetCoreCLI@2
     displayName: 'Zip Artifacts'
@@ -198,10 +165,18 @@ jobs:
     displayName: 'Copy Files to Artifact Staging Directory'
     inputs:
       SourceFolder: '$(Pipeline.Workspace)/staging'
-      Contents: 'coretools-*/**'
+      Contents: 'func-cli-*/**'
       TargetFolder: '$(Build.ArtifactStagingDirectory)/core-tools'
 
   - ${{ if eq(parameters.arch, 'min.win-x64') }}:
+    - task: PowerShell@2
+      displayName: 'Generate metadata.json file'
+      inputs:
+        targetType: filePath
+        filePath: '$(Build.SourcesDirectory)/eng/scripts/artifact-assembler/generate-metadata-file.ps1'
+        arguments: '-StagingDirectory "$(Pipeline.Workspace)/staging"'
+        workingDirectory: '$(Pipeline.Workspace)'
+
     - task: CopyFiles@2
       displayName: 'Copy metadata.json to Artifact Staging Directory'
       inputs:

eng/ci/templates/official/jobs/host-build-pack.yml

@@ -116,24 +116,6 @@ jobs:
         nethost.dll
       TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/win-arm64
 
-  - ${{ if eq(variables['Build.Reason'], 'Schedule') }}:
-    - task: PowerShell@2
-      displayName: 'Generate nightly build package version for feed'
-      inputs:
-        targetType: filePath
-        filePath: '$(Build.SourcesDirectory)/eng/scripts/generate-nightly-build-version.ps1'
-
-    - task: UniversalPackages@0
-      displayName: Publish
-      inputs:
-        command: publish
-        publishDirectory: '$(Build.ArtifactStagingDirectory)/_coreToolsHostPackages'
-        vstsFeedPublish: 'internal/core-tools-nightly-build'
-        vstsFeedPackagePublish: 'func-host-windows-signed'
-        versionOption: custom
-        versionPublish: '$(NightlyBuildVersion)'
-        packagePublishDescription: 'Core tools host build (windows)'
-
   templateContext:
     outputParentDirectory: $(Build.ArtifactStagingDirectory)
     outputs:
@@ -239,34 +221,9 @@ jobs:
         libnethost.so
       TargetFolder: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages/linux-x64
 
-  - ${{ if eq(variables['Build.Reason'], 'Schedule') }}:
-    - task: PowerShell@2
-      displayName: 'Generate nightly build package version for feed'
-      inputs:
-        targetType: filePath
-        filePath: '$(Build.SourcesDirectory)/eng/scripts/generate-nightly-build-version.ps1'
-
-    - task: PowerShell@2
-      displayName: 'Print NightlyBuildVersion'
-      inputs:
-        targetType: 'inline'
-        script: |
-          Write-Host "NightlyBuildVersion value: $(NightlyBuildVersion)"
-
-    - task: UniversalPackages@0
-      displayName: Publish
-      inputs:
-        command: publish
-        publishDirectory: '$(Build.ArtifactStagingDirectory)/_coreToolsHostPackages'
-        vstsFeedPublish: 'internal/core-tools-nightly-build'
-        vstsFeedPackagePublish: 'func-host-linux-signed'
-        versionOption: custom
-        versionPublish: '$(NightlyBuildVersion)'
-        packagePublishDescription: 'Core tools host build (linux)'
-
   templateContext:
     outputParentDirectory: $(Build.ArtifactStagingDirectory)
     outputs:
     - output: pipelineArtifact
       path: $(Build.ArtifactStagingDirectory)/_coreToolsHostPackages
-      artifact: func-host-linux-signed
+      artifact: func-host-linux-signed

eng/ci/templates/official/jobs/publish-cli.yml

@@ -81,7 +81,7 @@ jobs:
         signPatterns: '*.osx-*.zip'
 
   # arm64 MSI signing is not supported yet, so we skip it for now. We also do not generate MSI for .min versions.
-  - ${{ if and(contains(parameters.runtime, 'win'), not(contains(parameters.runtime, 'arm64')), not(contains(parameters.runtime, 'min'))) }}:
+  - ${{ if startsWith(parameters.runtime, 'win-x') }}:
     - pwsh: >
         ./eng/scripts/generate-msi-files.ps1
         -artifactsPath "$(Build.Repository.LocalPath)/artifacts"

…tes/official/jobs/upload-cli-to-feed.yml …fficial/jobs/upload-artifact-to-feed.ymleng/ci/templates/official/jobs/upload-cli-to-feed.yml renamed to eng/ci/templates/official/jobs/upload-artifact-to-feed.yml eng/ci/templates/official/jobs/upload-cli-to-feed.yml renamed to eng/ci/templates/official/jobs/upload-artifact-to-feed.yml

@@ -1,6 +1,12 @@
+parameters:
+  - name: artifactName
+    type: string
+  - name: packageName
+    type: string
+
 jobs:
-- job: UploadCliToFeed
-  displayName: 'Upload CLI to Feed'
+- job: UploadArtifactToFeed
+  displayName: 'Upload artifact to Feed'
   timeoutInMinutes: "180"
 
   pool:
@@ -10,18 +16,18 @@ jobs:
 
   steps:
   - task: DownloadPipelineArtifact@2
-    displayName: Download CLI artifacts
+    displayName: Download artifact
     inputs:
-      artifact: 'func-cli-$(Build.BuildNumber)'
-      path: $(Build.ArtifactStagingDirectory)/func-cli
+      artifact: '{{ parameters.artifactName }}'
+      path: $(Build.ArtifactStagingDirectory)/drop
 
   - task: UniversalPackages@0
     displayName: Publish to feed
     inputs:
       command: publish
-      publishDirectory: '$(Build.ArtifactStagingDirectory)/func-cli'
+      publishDirectory: '$(Build.ArtifactStagingDirectory)/drop'
       vstsFeedPublish: 'internal/core-tools-nightly-build'
-      vstsFeedPackagePublish: 'func-cli'
+      vstsFeedPackagePublish: '{{ parameters.packageName }}'
       versionOption: custom
       versionPublish: '$(Build.BuildNumber)'
       packagePublishDescription: 'Core tools default nightly build'

src/Cli/ArtifactAssembler/Constants.cs

@@ -9,11 +9,11 @@ internal static class Constants
         internal const string InProc8DirectoryName = "in-proc8";
         internal const string InProc6DirectoryName = "in-proc6";
         internal const string CoreToolsHostDirectoryName = "host";
-        internal const string VisualStudioOutputArtifactDirectoryName = "coretools-visualstudio";
         internal const string InProcOutputArtifactNameSuffix = "_inproc";
         internal const string CoreToolsProductVersionPattern = @"(\d+\.\d+\.\d+(?:-[0-9A-Za-z\-\.]+)?(?:\+[0-9A-Za-z\-\.]+)?)$";
         internal const string ArtifactNameRegexPattern = @"^(.*?)(\d+\.\d+\.\d+)$";
         internal const string OutOfProcDirectoryName = "default";
-        internal const string CliOutputArtifactDirectoryName = "coretools-cli";
+        internal const string CliOutputArtifactDirectoryName = "func-cli";
+        internal const string VisualStudioOutputArtifactDirectoryName = "func-cli-visualstudio";
     }
 }