cherry picking the fa2bb49 to fix the python issue

Author: scbedd

eng/ci_tools.txt

@@ -1,7 +1,7 @@
 # requirements leveraged by ci tools
-setuptools==67.6.0; python_version >= '3.5'
-virtualenv==20.24.3
-wheel==0.37.0
+setuptools==69.2.0; python_version >= '3.5'
+virtualenv==20.25.1
+wheel==0.43.0
 packaging==23.1
 tox==4.5.0
 pathlib2==2.3.5

eng/conda_test_requirements.txt

@@ -8,7 +8,7 @@ trio
 typing_extensions>=3.7.2
 cryptography
 adal
-setuptools==67.6.0; python_version >= '3.5'
+setuptools==69.2.0; python_version >= '3.5'
 pytest-asyncio==0.12.0
 -e sdk/core/azure-core/tests/testserver_tests/coretestserver
 azure-mgmt-storage

eng/pipelines/templates/steps/build-package-artifacts.yml

@@ -49,25 +49,25 @@ steps:
     inputs:
       versionSpec: $(PythonVersion)
 
+
+  - template: /eng/pipelines/templates/steps/use-venv.yml
+    parameters:
+      VirtualEnvironmentName: "venv"
+      Activate: false
+
   - bash: |
-      python -m venv $(Build.SourcesDirectory)/venv
-      source $(Build.SourcesDirectory)/venv/bin/activate
+      source $(VENV_LOCATION)/bin/activate
       which python
       python -m pip install --force -r eng/ci_tools.txt
       python -m pip freeze --all
-      echo "##vso[task.setvariable variable=PATH]$PATH"
     displayName: 'Prep Environment Linux/Mac'
     condition: and(succeeded(), or(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OS'], 'Darwin')))
 
   - pwsh: |
-      python -m venv $(Build.SourcesDirectory)/venv
-      ./venv/Scripts/Activate.ps1
+      . $(VENV_LOCATION)/Scripts/Activate.ps1
       which python
-      python -m pip install --force -r eng\ci_tools.txt
+      python -m pip install --force -r eng/ci_tools.txt
       python -m pip freeze --all
-
-      $path = $env:PATH
-      Write-Host "##vso[task.setvariable variable=PATH]$path"
     displayName: 'Prep Environment Windows'
     condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
 
@@ -95,7 +95,7 @@ steps:
     condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
 
   - bash: |
-      source $(Build.SourcesDirectory)/venv/bin/activate
+      source $(VENV_LOCATION)/bin/activate
       which python
       sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} --inactive
     displayName: 'Generate Packages Linux/Mac'
@@ -105,7 +105,7 @@ steps:
       CIBW_BUILD_VERBOSITY: 3
 
   - pwsh: |
-      ./venv/Scripts/Activate.ps1
+      . $(VENV_LOCATION)/Scripts/Activate.ps1
       which python
       sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} --inactive
     displayName: 'Generate Packages Windows'

eng/pipelines/templates/steps/build-test.yml

@@ -40,12 +40,12 @@ steps:
       $PSNativeCommandUseErrorActionPreference = $true
       python -m pip freeze
       python -m pip install pip==23.2.1
-      python -m pip install setuptools==67.6.0
-      python -m pip install wheel==0.37.0
+      python -m pip install wheel==0.43.0 --force-reinstall
+      python -m pip install setuptools==69.2.0 --force-reinstall
       python -m pip install -r eng/ci_tools.txt
       pip --version
       pip freeze
-    displayName: 'Prep Environment'
+    displayName: 'Prep Environment Linux/Mac'
 
   - ${{if eq(parameters.TestProxy, true) }}:
     - template: /eng/common/testproxy/test-proxy-tool.yml

eng/pipelines/templates/steps/use-venv.yml

@@ -0,0 +1,21 @@
+parameters:
+  - name: VirtualEnvironmentName
+    type: string
+    default: "venv"
+  - name: Activate
+    type: boolean
+    default: true
+
+steps:
+  - pwsh: |
+      $(Build.SourcesDirectory)/eng/scripts/create-venv.ps1 `
+        -VenvName "${{ parameters.VirtualEnvironmentName }}" `
+        -RepoRoot "$(Build.SourcesDirectory)"
+    displayName: Create virtual environment
+
+  - ${{ if eq(parameters.Activate, true) }}:
+    - pwsh: |
+        $(Build.SourcesDirectory)/eng/scripts/activate-venv.ps1 `
+          -VenvName "${{ parameters.VirtualEnvironmentName }}" `
+          -RepoRoot "$(Build.SourcesDirectory)"
+      displayName: Use ${{ parameters.VirtualEnvironmentName }} Virtual Environment

eng/regression_tools.txt

@@ -1,5 +1,5 @@
 # requirements leveraged by ci tools
-setuptools==67.6.0; python_version >= '3.5'
+setuptools==69.2.0; python_version >= '3.5'
 virtualenv==20.23.0
 wheel==0.37.0
 Jinja2==3.1.2

eng/scripts/create-venv.ps1

@@ -0,0 +1,33 @@
+<#! 
+.SYNOPSIS
+Creates a virtual environment for a CI machine.
+
+.DESCRIPTION
+If the virtual environment directory already exists, it will skip the creation. The location of the virtual environment will be stored in a variable
+named <VenvName>_LOCATION. The location will be RepoRoot + VenvName.
+
+.PARAMETER VenvName
+The name of the virtual environment which will be created.
+
+.PARAMETER RepoRoot
+The root of the repository.
+#>
+param(
+    [Parameter(Mandatory = $true)]
+    [string] $VenvName,
+    # The root of the repository should be $(Build.SourcesDirectory) passed in from template
+    [Parameter(Mandatory = $true)]
+    [string] $RepoRoot
+)
+
+$venvPath = Join-Path $RepoRoot $VenvName
+if (!(Test-Path $venvPath)) {
+    Write-Host "Creating virtual environment '$VenvName'."
+    python -m pip install virtualenv==20.25.1
+    python -m virtualenv "$venvPath"
+    Write-Host "Virtual environment '$VenvName' created."
+    Write-Host "##vso[task.setvariable variable=$($VenvName)_LOCATION]$venvPath"
+}
+else {
+    Write-Host "Virtual environment '$VenvName' already exists. Skipping creation."
+}