Various fixes to support extension packages as dev_reqs (#35504)

Author: jalauzon-msft

eng/pipelines/templates/jobs/ci.tests.yml

@@ -78,7 +78,6 @@ jobs:
     - name: InjectedPackages
       value: ${{ parameters.InjectedPackages }}
 
-
     steps:
     - template: /eng/pipelines/templates/steps/targeting-string-resolve.yml
       parameters:

eng/pipelines/templates/jobs/ci.yml

@@ -245,6 +245,12 @@ jobs:
           Directory: sdk/${{ parameters.ServiceDirectory }}
         CheckLinkGuidance: $true
 
+    - task: DownloadPipelineArtifact@2
+      condition: succeededOrFailed()
+      inputs:
+        artifactName: 'packages_extended'
+        targetPath: $(Build.ArtifactStagingDirectory)
+
     - template: ../steps/analyze.yml
       parameters:
         ServiceDirectory: ${{ parameters.ServiceDirectory }}

eng/pipelines/templates/steps/analyze.yml

@@ -81,28 +81,31 @@ steps:
     condition: and(succeededOrFailed(), ne(variables['Skip.VerifySdist'],'true'))
     inputs:
      scriptPath: 'scripts/devops_tasks/dispatch_tox.py'
-     arguments: '"$(TargetingString)" --service=${{parameters.ServiceDirectory}} --toxenv=verifysdist'
+     arguments: '"$(TargetingString)" --service=${{parameters.ServiceDirectory}} --toxenv=verifysdist ${{ parameters.AdditionalTestArgs }}'
 
   - task: PythonScript@0
     displayName: 'Verify whl'
     condition: and(succeededOrFailed(), ne(variables['Skip.VerifyWhl'],'true'))
     inputs:
      scriptPath: 'scripts/devops_tasks/dispatch_tox.py'
-     arguments: '"$(TargetingString)" --service=${{parameters.ServiceDirectory}} --toxenv=verifywhl'
+     arguments: '"$(TargetingString)" --service=${{parameters.ServiceDirectory}} --toxenv=verifywhl ${{ parameters.AdditionalTestArgs }}'
 
   - template: run_mypy.yml
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
       TestMarkArgument: ${{ parameters.TestMarkArgument }}
+      AdditionalTestArgs: ${{ parameters.AdditionalTestArgs }}
 
   - template: run_pyright.yml
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
       TestMarkArgument: ${{ parameters.TestMarkArgument }}
+      AdditionalTestArgs: ${{ parameters.AdditionalTestArgs }}
 
   - template: run_pylint.yml
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
+      AdditionalTestArgs: ${{ parameters.AdditionalTestArgs }}
 
   - ${{ if parameters.ValidateFormatting }}:
     - template: run_black.yml
@@ -114,31 +117,25 @@ steps:
     displayName: 'Run Keyword Validation Check'
     inputs:
       scriptPath: 'scripts/devops_tasks/dispatch_tox.py'
-      arguments: '"$(TargetingString)" --service=${{parameters.ServiceDirectory}} --toxenv=verify_keywords'
+      arguments: '"$(TargetingString)" --service=${{parameters.ServiceDirectory}} --toxenv=verify_keywords ${{ parameters.AdditionalTestArgs }}'
     condition: and(succeededOrFailed(), ne(variables['Skip.KeywordCheck'],'true'))
 
-  - task: DownloadPipelineArtifact@2
-    condition: succeededOrFailed()
-    inputs:
-      artifactName: 'packages_extended'
-      targetPath: $(Build.ArtifactStagingDirectory)
-
   - template: ../steps/run_bandit.yml
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
       TestMarkArgument: ${{ parameters.TestMarkArgument }}
-      AdditionalTestArgs: ${{parameters.AdditionalTestArgs}}
+      AdditionalTestArgs: ${{ parameters.AdditionalTestArgs }}
 
   - template: /eng/pipelines/templates/steps/update_snippet.yml
     parameters:
       ScanPath: $(Build.SourcesDirectory)/sdk/${{ parameters.ServiceDirectory }}
-      AdditionalTestArgs: ${{parameters.AdditionalTestArgs}}
+      AdditionalTestArgs: ${{ parameters.AdditionalTestArgs }}
 
   - template: ../steps/run_breaking_changes.yml
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
       TestMarkArgument: ${{ parameters.TestMarkArgument }}
-      AdditionalTestArgs: ${{parameters.AdditionalTestArgs}}
+      AdditionalTestArgs: ${{ parameters.AdditionalTestArgs }}
 
   - template: /eng/common/pipelines/templates/steps/create-apireview.yml
     parameters:

eng/pipelines/templates/steps/build-extended-artifacts.yml

@@ -70,6 +70,7 @@ steps:
         "$(TargetingString)" 
         --service="${{ parameters.ServiceDirectory }}" 
         --toxenv=sphinx
+        --wheel_dir="$(Build.ArtifactStagingDirectory)"
 
   - pwsh: |
       Write-Host "##vso[task.setvariable variable=PIP_INDEX_URL]https://pypi.python.org/simple"
@@ -78,6 +79,7 @@ steps:
   - template: /eng/pipelines/templates/steps/run_apistub.yml
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
+      AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"'
 
   - ${{ parameters.BeforePublishSteps }}
 

eng/pipelines/templates/steps/run_apistub.yml

@@ -1,5 +1,7 @@
 parameters:
   ServiceDirectory: ''
+  AdditionalTestArgs: ''
+
 
 # The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files:
 #     eng/pipelines/templates/jobs/ci.tests.yml
@@ -31,4 +33,5 @@ steps:
         --service="${{ parameters.ServiceDirectory }}" 
         --toxenv="apistub"
         --disablecov
-        --filter-type="Omit_management"
+        --filter-type="Omit_management"
+        ${{ parameters.AdditionalTestArgs }}

eng/pipelines/templates/steps/run_black.yml

@@ -2,6 +2,7 @@ parameters:
   ServiceDirectory: ''
   ValidateFormatting: false
   EnvVars: {}
+  AdditionalTestArgs: ''
 
 steps:
   - task: UsePythonVersion@0

eng/pipelines/templates/steps/run_mypy.yml

@@ -3,6 +3,7 @@ parameters:
   ServiceDirectory: ''
   TestMarkArgument: ''
   EnvVars: {}
+  AdditionalTestArgs: ''
 
 # The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files:
 #     eng/pipelines/templates/jobs/ci.tests.yml
@@ -32,5 +33,6 @@ steps:
         --service="${{ parameters.ServiceDirectory }}"
         --toxenv="mypy"
         --disablecov
+        ${{ parameters.AdditionalTestArgs }}
     env: ${{ parameters.EnvVars }}
     condition: and(succeededOrFailed(), ne(variables['Skip.MyPy'],'true'))

eng/pipelines/templates/steps/run_pylint.yml

@@ -2,6 +2,7 @@ parameters:
   ServiceDirectory: ''
   TestMarkArgument: ''
   EnvVars: {}
+  AdditionalTestArgs: ''
 
 # The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files:
 #     eng/pipelines/templates/jobs/ci.tests.yml
@@ -32,5 +33,6 @@ steps:
         --toxenv="pylint"
         --disablecov
         --filter-type="Omit_management"
+        ${{ parameters.AdditionalTestArgs }}
     env: ${{ parameters.EnvVars }}
     condition: and(succeededOrFailed(), ne(variables['Skip.Pylint'],'true'))

eng/pipelines/templates/steps/run_pyright.yml

@@ -2,6 +2,7 @@ parameters:
   BuildTargetingString: 'azure-*'
   ServiceDirectory: ''
   TestMarkArgument: ''
+  AdditionalTestArgs: ''
 
 # The variable TargetingString is set by template `eng/pipelines/templates/steps/targeting-string-resolve.yml`. This template is invoked from yml files:
 #     eng/pipelines/templates/jobs/ci.tests.yml
@@ -31,6 +32,7 @@ steps:
         --service="${{ parameters.ServiceDirectory }}"
         --toxenv="pyright"
         --disablecov
+        ${{ parameters.AdditionalTestArgs }}
     condition: and(succeededOrFailed(), ne(variables['Skip.Pyright'],'true'))
 
   - task: PythonScript@0
@@ -43,4 +45,5 @@ steps:
         --service="${{ parameters.ServiceDirectory }}"
         --toxenv="verifytypes"
         --disablecov
+        ${{ parameters.AdditionalTestArgs }}
     condition: and(succeededOrFailed(), ne(variables['Skip.Verifytypes'],'true'))

tools/azure-sdk-tools/ci_tools/scenario/generation.py

@@ -74,6 +74,16 @@ def create_package_and_install(
 
     target_package = ParsedSetup.from_path(setup_py_path)
 
+    # ensure that discovered packages are always copied to the distribution directory regardless of other factors
+    for built_package in discovered_packages:
+        if os.getenv("PREBUILT_WHEEL_DIR") is not None:
+            package_path = os.path.join(os.environ["PREBUILT_WHEEL_DIR"], built_package)
+            if os.path.isfile(package_path):
+                built_pkg_path = package_path
+
+                # copy this file to the distribution directory just in case we need it later
+                shutil.copy(built_pkg_path, distribution_directory)
+
     if skip_install:
         logging.info("Flag to skip install whl is passed. Skipping package installation")
     else:
@@ -83,6 +93,7 @@ def create_package_and_install(
                 package_path = os.path.join(os.environ["PREBUILT_WHEEL_DIR"], built_package)
                 if os.path.isfile(package_path):
                     built_pkg_path = package_path
+
                     logging.info("Installing {w} from directory".format(w=built_package))
                 # it does't exist, so we need to error out
                 else: