Use a venv for build/test (#36340) (#36430)

* create `use-venv` yml which can utilize the current python to create a venv and prepend the PATH with it
* use a freshly created venv `build` and `test`

Co-authored-by: Scott Beddall <[email protected]>

Author: achauhan-scc

eng/pipelines/templates/jobs/regression.yml

@@ -79,6 +79,8 @@ jobs:
         displayName: 'Use Python 3.9'
         inputs:
           versionSpec: '3.9'
+    
+      - template: /eng/pipelines/templates/steps/use-venv.yml
 
       - task: DownloadPipelineArtifact@2
         inputs:

eng/pipelines/templates/steps/build-extended-artifacts.yml

@@ -47,6 +47,8 @@ steps:
     inputs:
       versionSpec: '3.11'
 
+  - template: /eng/pipelines/templates/steps/use-venv.yml
+
   - script: |
       python -m pip install setuptools==58.3.0
       python -m pip install -r eng/ci_tools.txt

eng/pipelines/templates/steps/build-package-artifacts.yml

@@ -49,27 +49,13 @@ steps:
     inputs:
       versionSpec: $(PythonVersion)
 
-  - bash: |
-      python -m venv $(Build.SourcesDirectory)/venv
-      source $(Build.SourcesDirectory)/venv/bin/activate
-      which python
-      python -m pip install --force -r eng/ci_tools.txt
-      python -m pip freeze --all
-      echo "##vso[task.setvariable variable=PATH]$PATH"
-    displayName: 'Prep Environment Linux/Mac'
-    condition: and(succeeded(), or(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OS'], 'Darwin')))
+  - template: /eng/pipelines/templates/steps/use-venv.yml
 
   - pwsh: |
-      python -m venv $(Build.SourcesDirectory)/venv
-      ./venv/Scripts/Activate.ps1
       which python
       python -m pip install --force -r eng\ci_tools.txt
       python -m pip freeze --all
-
-      $path = $env:PATH
-      Write-Host "##vso[task.setvariable variable=PATH]$path"
-    displayName: 'Prep Environment Windows'
-    condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
+    displayName: 'Prep Environment'
 
   - template: set-dev-build.yml@self
     parameters:
@@ -94,28 +80,16 @@ steps:
     displayName: 'Install QEMU Dependencies'
     condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
 
-  - bash: |
-      source $(Build.SourcesDirectory)/venv/bin/activate
-      which python
-      sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} --inactive
-    displayName: 'Generate Packages Linux/Mac'
-    condition: and(succeeded(), or(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OS'], 'Darwin')))
-    timeoutInMinutes: 80
-    env:
-      CIBW_BUILD_VERBOSITY: 3
-
   - pwsh: |
-      ./venv/Scripts/Activate.ps1
       which python
       sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} --inactive
-    displayName: 'Generate Packages Windows'
-    condition: and(succeededOrFailed(), eq(variables['Agent.OS'], 'Windows_NT'))
+    displayName: 'Generate Packages'
     timeoutInMinutes: 80
     env:
       CIBW_BUILD_VERBOSITY: 3
 
   - script: |
-      pip install twine==4.0.2 importlib-metadata==7.2.1
+      python -m pip install -r eng/release_requirements.txt
       twine check $(Build.ArtifactStagingDirectory)/**/*.whl
       twine check $(Build.ArtifactStagingDirectory)/**/*.tar.gz
     displayName: 'Verify Readme'

eng/pipelines/templates/steps/build-test.yml

@@ -33,6 +33,8 @@ steps:
     parameters:
       versionSpec: '${{ parameters.PythonVersion }}'
 
+  - template: /eng/pipelines/templates/steps/use-venv.yml
+
   - template: set-dev-build.yml
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}

eng/pipelines/templates/steps/use-venv.yml

@@ -0,0 +1,17 @@
+parameters:
+  - name: VirtualEnvironmentName
+    type: string
+    default: "venv"
+
+steps:
+  - pwsh: |
+      $(Build.SourcesDirectory)/eng/scripts/create-venv.ps1 `
+        -VenvName "${{ parameters.VirtualEnvironmentName }}" `
+        -RepoRoot "$(Build.SourcesDirectory)"
+    displayName: Create virtual environment
+
+  - pwsh: |
+      $(Build.SourcesDirectory)/eng/scripts/activate-venv.ps1 `
+        -VenvName "${{ parameters.VirtualEnvironmentName }}" `
+        -RepoRoot "$(Build.SourcesDirectory)"
+    displayName: Use ${{ parameters.VirtualEnvironmentName }} Virtual Environment

eng/scripts/activate-venv.ps1

@@ -0,0 +1,49 @@
+<#! 
+.SYNOPSIS
+Activates a virtual environment for a CI machine. Any further usages of "python" will utilize this virtual environment.
+
+.DESCRIPTION
+When activating a virtual environment, only a few things are actually functionally changed on the machine.
+
+# 1. PATH = path to the bin directory of the virtual env. "Scripts" on windows machines
+# 2. VIRTUAL_ENV = path to root of the virtual env
+# 3. VIRTUAL_ENV_PROMPT = the prompt that is displayed next to the CLI cursor when the virtual env is active
+# within a CI machine, we only need the PATH and VIRTUAL_ENV variables to be set.
+# 4. (optional and inconsistently) _OLD_VIRTUAL_PATH = the PATH before the virtual env was activated. This is not set in this script. 
+
+.PARAMETER VenvName
+The name of the virtual environment to activate.
+
+.PARAMETER RepoRoot
+The root of the repository.
+#>
+param (
+    [Parameter(Mandatory=$true)]
+    [string]$VenvName,
+    [Parameter(Mandatory=$true)]
+    [string]$RepoRoot # mandatory here, but $(Build.SourcesDirectory) will be passed in the template yaml
+)
+
+Set-StrictMode -Version 4
+$ErrorActionPreference = "Stop"
+
+$venvPath = Join-Path $RepoRoot $VenvName
+$venvBinPath = Join-Path $venvPath "bin"
+$env:VIRTUAL_ENV = $venvPath
+
+if (-not (Test-Path $venvPath)) {
+    Write-Error "Virtual environment '$venvPath' does not exist at $venvPath"
+    exit 1
+}
+
+if ($IsWindows) {
+    $venvBinPath = Join-Path $venvPath "Scripts"
+    $env:PATH = "$venvBinPath;$($env:PATH)"
+}
+else {
+    $env:PATH = "$venvBinPath`:$($env:PATH)"
+}
+
+Write-Host "Activating virtual environment '$VenvName' at $venvPath via AzDO to the value '$($env:PATH)'"
+Write-Host "##vso[task.setvariable variable=VIRTUAL_ENV]$($env:VIRTUAL_ENV)"
+Write-Host "##vso[task.prependpath]$($env:PATH)"

eng/scripts/create-venv.ps1

@@ -0,0 +1,32 @@
+<#! 
+.SYNOPSIS
+Creates a virtual environment for a CI machine.
+
+.DESCRIPTION
+If the virtual environment directory already exists, it will skip the creation. The location of the virtual environment will be stored in a variable
+named <VenvName>_LOCATION. The location will be RepoRoot + VenvName.
+
+.PARAMETER VenvName
+The name of the virtual environment which will be created.
+
+.PARAMETER RepoRoot
+The root of the repository.
+#>
+param(
+    [Parameter(Mandatory = $true)]
+    [string] $VenvName,
+    # The root of the repository should be $(Build.SourcesDirectory) passed in from template
+    [Parameter(Mandatory = $true)]
+    [string] $RepoRoot
+)
+
+$venvPath = Join-Path $RepoRoot $VenvName
+if (!(Test-Path $venvPath)) {
+    Write-Host "Creating virtual environment '$VenvName'."
+    python -m venv "$venvPath"
+    Write-Host "Virtual environment '$VenvName' created."
+    Write-Host "##vso[task.setvariable variable=$($VenvName)_LOCATION]$venvPath"
+}
+else {
+    Write-Host "Virtual environment '$VenvName' already exists. Skipping creation."
+}