Pipeline V3 (Azure#37428)

* refactor package selection methodology to utilize Save-Package-Properties for package targeting
* introduce the python - pullrequest build definition that is triggered against all sdk/ directories, and builds/tests the packages which were changed
  * this also includes abstractions to reduce runtime by distributing sets of packages and duplicating matrix definition
* refactor all analyze checks which used to operate on ServiceDirectory to instead operate upon the packages present within a PackageInfoFolder from Save-Package-Properties
* miscellaneous tooling changes to account for new matrix possibilities

---------

Co-authored-by: Wes Haggard <[email protected]>
Co-authored-by: Patrick Hallisey <[email protected]>

Author: 3 people

.vscode/cspell.json

@@ -262,6 +262,7 @@
     "iohttp",
     "IOHTTP",
     "IOLRO",
+    "incompat",
     "inprogress",
     "ipconfiguration",
     "ipconfigurations",

eng/pipelines/templates/jobs/ci.tests.yml

@@ -79,9 +79,12 @@ jobs:
       value: ${{ parameters.InjectedPackages }}
 
     steps:
-    - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+    - template: /eng/pipelines/templates/steps/download-package-artifacts.yml
+
+    - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml
       parameters:
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
+        PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
 
     - template: /eng/common/pipelines/templates/steps/verify-agent-os.yml
       parameters:
@@ -112,7 +115,6 @@ jobs:
 
     - template: ../steps/build-test.yml
       parameters:
-        ServiceDirectory: ${{ parameters.ServiceDirectory }}
         TestMarkArgument: ${{ parameters.TestMarkArgument }}
         AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"'
         CoverageArg: $(CoverageArg)
@@ -122,12 +124,6 @@ jobs:
         InjectedPackages: $(InjectedPackages)
         TestProxy: ${{ parameters.TestProxy }}
         BeforeTestSteps:
-          - template: /eng/pipelines/templates/steps/download-package-artifacts.yml
-
-          - template: ../steps/set-dev-build.yml
-            parameters:
-              ServiceDirectory: ${{ parameters.ServiceDirectory }}
-
           - ${{ each step in parameters.BeforeTestSteps }}:
             -  ${{ step }}
         AfterTestSteps: ${{ parameters.AfterTestSteps }}

eng/pipelines/templates/jobs/ci.yml

@@ -71,18 +71,15 @@ jobs:
       image: $(LINUXVMIMAGE)
       os: linux
 
-    variables:
-      ComponentDetection.SourcePath: '$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}'
-
     steps:
-    - template: /eng/pipelines/templates/steps/build.yml
+    - template: /eng/pipelines/templates/steps/build-package-artifacts.yml
       parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
-        BuildTargetingString: ${{ parameters.BuildTargetingString }}
         BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
         TestPipeline: ${{ parameters.TestPipeline }}
-        ArtifactSuffix: linux
         Artifacts: ${{ parameters.Artifacts }}
+        ArtifactSuffix: linux
+        BuildTargetingString: ${{ parameters.BuildTargetingString }}
 
   - job: 'Build_Windows'
     timeoutInMinutes: 90
@@ -92,18 +89,15 @@ jobs:
       image: $(WINDOWSVMIMAGE)
       os: windows
 
-    variables:
-      ComponentDetection.SourcePath: '$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}'
-
     steps:
-    - template: /eng/pipelines/templates/steps/build.yml
+    - template: /eng/pipelines/templates/steps/build-package-artifacts.yml
       parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
-        BuildTargetingString: ${{ parameters.BuildTargetingString }}
         BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
         TestPipeline: ${{ parameters.TestPipeline }}
-        ArtifactSuffix: windows
         Artifacts: ${{ parameters.Artifacts }}
+        ArtifactSuffix: windows
+        BuildTargetingString: ${{ parameters.BuildTargetingString }}
 
   - job: 'Build_MacOS'
     timeoutInMinutes: 90
@@ -113,18 +107,15 @@ jobs:
       vmImage: $(MACVMIMAGE)
       os: macOS
 
-    variables:
-      ComponentDetection.SourcePath: '$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}'
-
     steps:
-    - template: /eng/pipelines/templates/steps/build.yml
+    - template: /eng/pipelines/templates/steps/build-package-artifacts.yml
       parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
-        BuildTargetingString: ${{ parameters.BuildTargetingString }}
         BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
         TestPipeline: ${{ parameters.TestPipeline }}
-        ArtifactSuffix: mac
         Artifacts: ${{ parameters.Artifacts }}
+        ArtifactSuffix: mac
+        BuildTargetingString: ${{ parameters.BuildTargetingString }}
 
   - job: 'Build_Extended'
     displayName: Build Extended
@@ -140,15 +131,46 @@ jobs:
       image: $(LINUXVMIMAGE)
       os: linux
 
-    variables:
-      ComponentDetection.SourcePath: '$(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}'
+    steps:
+    - template: /eng/pipelines/templates/steps/download-package-artifacts.yml
+
+    - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml
+      parameters:
+        BuildTargetingString: ${{ parameters.BuildTargetingString }}
+        PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
+        IncludeIndirect: false
+
+    - template: /eng/pipelines/templates/steps/build-extended-artifacts.yml
+      parameters:
+        ServiceDirectory: ${{ parameters.ServiceDirectory }}
+        BeforePublishSteps: ${{ parameters.BeforePublishSteps }}
+        BuildDocs: false
+        RunApiStubGen: true
+        TestPipeline: ${{ parameters.TestPipeline }}
+        Artifacts: ${{ parameters.Artifacts }}
+
+  - job: 'Build_Documentation'
+    displayName: Build Docs
+    dependsOn:
+      - 'Build_Linux'
+      - 'Build_Windows'
+      - 'Build_MacOS'
+
+    timeoutInMinutes: 90
+
+    pool:
+      name: $(LINUXPOOL)
+      image: $(LINUXVMIMAGE)
+      os: linux
 
     steps:
     - template: /eng/pipelines/templates/steps/download-package-artifacts.yml
 
-    - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+    - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml
       parameters:
         BuildTargetingString: ${{ parameters.BuildTargetingString }}
+        PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
+        IncludeIndirect: false
 
     - template: ../steps/build-extended-artifacts.yml
       parameters:
@@ -157,6 +179,7 @@ jobs:
         BuildDocs: ${{ parameters.BuildDocs }}
         TestPipeline: ${{ parameters.TestPipeline }}
         Artifacts: ${{ parameters.Artifacts }}
+        RunApiStubGen: false
 
   - job: 'Analyze'
     condition: and(succeeded(), ne(variables['Skip.Analyze'], 'true'))
@@ -172,10 +195,6 @@ jobs:
       os: linux
 
     steps:
-    - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
-      parameters:
-        BuildTargetingString: ${{ parameters.BuildTargetingString }}
-
     - template: /eng/common/pipelines/templates/steps/check-spelling.yml
       parameters:
         ContinueOnError: false
@@ -196,6 +215,12 @@ jobs:
         artifactName: 'packages_extended'
         targetPath: $(Build.ArtifactStagingDirectory)
 
+    - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml
+      parameters:
+        BuildTargetingString: ${{ parameters.BuildTargetingString }}
+        PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
+        IncludeIndirect: false
+
     - template: ../steps/analyze.yml
       parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}
@@ -219,6 +244,20 @@ jobs:
       MatrixConfigs: ${{ parameters.MatrixConfigs }}
       MatrixFilters: ${{ parameters.MatrixFilters }}
       MatrixReplace: ${{ parameters.MatrixReplace }}
+      PreGenerationSteps:
+
+        - template: /eng/pipelines/templates/steps/download-package-artifacts.yml
+
+        - task: Powershell@2
+          inputs:
+            pwsh: true
+            filePath: eng/scripts/distribute-packages-to-matrix.ps1
+            arguments: >-
+              -PackageInfoFolder "$(Build.ArtifactStagingDirectory)/PackageInfo"
+              -PlatformMatrix "${{ parameters.MatrixConfigs[0].Path }}"
+          displayName: 'Distribute Packages to Matrix'
+          condition: and(eq(variables['Build.Reason'], 'PullRequest'), eq('${{ parameters.ServiceDirectory }}','auto'))
+
       CloudConfig:
         Cloud: Public
       AdditionalParameters:
@@ -253,15 +292,12 @@ jobs:
         - pwsh: |
             python -m pip install "./tools/azure-sdk-tools[build]"
           displayName: 'Prep Environment'
-        - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
-          parameters:
-            BuildTargetingString: ${{ parameters.BuildTargetingString }}
         - task: PythonScript@0
           displayName: 'Ensure service coverage'
           inputs:
             scriptPath: '$(Build.SourcesDirectory)/scripts/devops_tasks/update_regression_services.py'
             arguments: >-
-              "$(TargetingString)"
+              "azure*"
               --service="${{ parameters.ServiceDirectory }}"
               --json=$(Build.SourcesDirectory)/eng/pipelines/templates/stages/regression-job-matrix.json
       CloudConfig:

eng/pipelines/templates/jobs/live.tests.yml

@@ -110,7 +110,7 @@ jobs:
         parameters:
           AgentImage: ${{ parameters.OSName }}
 
-      - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+      - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml
         parameters:
           BuildTargetingString: ${{ parameters.BuildTargetingString }}
 

eng/pipelines/templates/jobs/regression.yml

@@ -71,7 +71,7 @@ jobs:
 
     # Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`.
     steps:
-      - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
+      - template: /eng/pipelines/templates/steps/resolve-package-targeting.yml
         parameters:
           BuildTargetingString: ${{ parameters.BuildTargetingString }}
 

eng/pipelines/templates/stages/platform-matrix.json

@@ -38,4 +38,4 @@
       }
     }
   ]
-}
+}

eng/pipelines/templates/steps/analyze.yml

@@ -8,12 +8,7 @@ parameters:
   ValidateFormatting: false
   GenerateApiReviewForManualOnly: false
 
-# The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files:
-#     eng/pipelines/templates/jobs/ci.tests.yml
-#     eng/pipelines/templates/jobs/ci.yml
-#     eng/pipelines/templates/jobs/live.test.yml
-
-# Please use `$(TargetingString)` to refer to the python packages glob string. This was previously `${{ parameters.BuildTargetingString }}`.
+# Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml.
 steps:
   - template: /eng/pipelines/templates/steps/analyze_dependency.yml
     parameters:
@@ -32,14 +27,20 @@ steps:
       scriptPath: 'scripts/devops_tasks/set_tox_environment.py'
       arguments: '"$(TargetingString)" --team-project="$(System.TeamProject)" --service="${{ parameters.ServiceDirectory }}"'
 
-  - ${{ each artifact in parameters.Artifacts }}:
-    - ${{if ne(artifact.skipVerifyChangeLog, 'true')}}:
-      - template: /eng/common/pipelines/templates/steps/verify-changelog.yml
-        parameters:
-          PackageName: ${{artifact.name}}
-          ServiceName: ${{parameters.ServiceDirectory}}
-          ForRelease: false
-          Condition: succeededOrFailed()
+  - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}:
+    - template: /eng/common/pipelines/templates/steps/verify-changelogs.yml
+      parameters:
+        PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
+        Condition: succeededOrFailed()
+  - ${{ else }}:
+    - ${{ each artifact in parameters.Artifacts }}:
+      - ${{if ne(artifact.skipVerifyChangeLog, 'true')}}:
+        - template: /eng/common/pipelines/templates/steps/verify-changelog.yml
+          parameters:
+            PackageName: ${{artifact.name}}
+            ServiceName: ${{parameters.ServiceDirectory}}
+            ForRelease: false
+            Condition: succeededOrFailed()
 
   - script: |
       python -m pip install "./tools/azure-sdk-tools[build]" -q -I

eng/pipelines/templates/steps/analyze_dependency.yml

@@ -13,9 +13,9 @@ steps:
     displayName: 'Install Python Tools'
     condition: succeededOrFailed()
 
-  - template: /eng/common/pipelines/templates/steps/verify-readme.yml
+  - template: /eng/common/pipelines/templates/steps/verify-readmes.yml
     parameters:
-      ScanPath: ${{ parameters.ScanPath }}
+      PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo
       Condition: succeededOrFailed()
 
   - pwsh: |