Conglomerated changes to optimize venv usage in pipeline

eng/pipelines/templates/steps/analyze.yml

@@ -48,7 +48,7 @@ steps:
             Condition: succeededOrFailed()
 
   - script: |
-      python -m pip install "./tools/azure-sdk-tools[build]" -q -I
+      uv pip install "./tools/azure-sdk-tools[build]"
       sdk_find_invalid_versions --always-succeed --service=${{parameters.ServiceDirectory}}
     displayName: Find Invalid Versions
     condition: succeededOrFailed()

eng/pipelines/templates/steps/build-extended-artifacts.yml

@@ -56,7 +56,8 @@ steps:
   - template: /eng/pipelines/templates/steps/use-venv.yml
 
   - script: |
-      python -m pip install -r eng/ci_tools.txt
+      which python
+      uv pip install -r eng/ci_tools.txt
     displayName: 'Prep Environment'
 
   - template: set-dev-build.yml

eng/pipelines/templates/steps/build-package-artifacts.yml

@@ -88,18 +88,19 @@ steps:
   - template: /eng/pipelines/templates/steps/use-venv.yml
     parameters:
       VirtualEnvironmentName: "venv"
-      Activate: false
       Condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux')))
 
   - pwsh: |
-      $(VENV_ACTIVATION_SCRIPT)
+      $ErrorActionPreference = 'Stop'
+      $PSNativeCommandUseErrorActionPreference = $true
       which python
-      python -m pip install --force -r eng/ci_tools.txt
+      uv pip install -r eng/ci_tools.txt
+
       if ($env:AGENT_OS -eq "Linux") {
         Write-Host "Installing release reqs"
-        python -m pip install -r eng/release_requirements.txt
+        uv pip install -r eng/release_requirements.txt
       }
-      python -m pip freeze --all
+      uv pip freeze
     displayName: 'Prep Environment'
     condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux')))
 
@@ -111,7 +112,6 @@ steps:
     condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
 
   - pwsh: |
-      $(VENV_ACTIVATION_SCRIPT)
       which python
       sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --inactive
     displayName: 'Generate Packages'
@@ -121,11 +121,13 @@ steps:
       CIBW_BUILD_VERBOSITY: 3
 
   - pwsh: |
-      $(VENV_ACTIVATION_SCRIPT)
-      twine check $(Build.ArtifactStagingDirectory)/**/*.whl
-      twine check $(Build.ArtifactStagingDirectory)/**/*.tar.gz
-    displayName: 'Verify Readme'
-    condition: and(succeededOrFailed(), eq(variables['Agent.OS'], 'Linux'))
+      which python
+      sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --inactive
+    displayName: 'Generate Packages'
+    condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux')))
+    timeoutInMinutes: 80
+    env:
+      CIBW_BUILD_VERBOSITY: 3
 
   - ${{ parameters.BeforePublishSteps }}
 

eng/pipelines/templates/steps/build-test.yml

@@ -26,23 +26,15 @@ steps:
       versionSpec: '${{ parameters.PythonVersion }}'
 
   - template: /eng/pipelines/templates/steps/use-venv.yml
-    parameters:
-      Activate: false
 
   - template: set-dev-build.yml
 
   - pwsh: |
-      if ($IsWindows) {
-        . $(VENV_LOCATION)/Scripts/Activate.ps1
-      }
-      else {
-        . $(VENV_LOCATION)/bin/activate.ps1
-      }
+      Write-Host (Get-Command python).Source
       $ErrorActionPreference = 'Stop'
       $PSNativeCommandUseErrorActionPreference = $true
-      python -m pip install --force -r eng/ci_tools.txt
-      python -m pip freeze --all
-      Write-Host (Get-Command python).Source
+      uv pip install --force -r eng/ci_tools.txt
+      uv pip freeze
     displayName: 'Prep Environment'
 
   - template: /eng/common/testproxy/test-proxy-tool.yml
@@ -75,12 +67,6 @@ steps:
           $env:AZURESUBSCRIPTION_CLIENT_ID = $account.Id;
           $env:AZURESUBSCRIPTION_TENANT_ID = $account.Tenants;
 
-          if ($IsWindows) {
-            . $(VENV_LOCATION)/Scripts/Activate.ps1
-          }
-          else {
-            . $(VENV_LOCATION)/bin/activate.ps1
-          }
           Write-Host (Get-Command python).Source
 
           if ($env:TESTMARKARGUMENT) {
@@ -104,12 +90,6 @@ steps:
 
   - ${{ else }}:
     - pwsh: |
-        if ($IsWindows) {
-          . $(VENV_LOCATION)/Scripts/Activate.ps1
-        }
-        else {
-          . $(VENV_LOCATION)/bin/activate.ps1
-        }
         Write-Host (Get-Command python).Source
 
         if ($env:TESTMARKARGUMENT) {

eng/pipelines/templates/steps/install-uv.yml

@@ -0,0 +1,26 @@
+
+steps:
+  - task: Bash@3
+    displayName: 'Install uv (Linux/macOS)'
+    inputs:
+      targetType: inline
+      script: |
+        curl -LsSf https://astral.sh/uv/install.sh | sh
+    condition: or(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OS'], 'Darwin'))
+
+  - task: Bash@3
+    inputs:
+      targetType: inline
+      script: |
+        echo "##vso[task.prependpath]$HOME/.local/bin"
+    displayName: 'Prepend path for MacOS'
+    condition: eq(variables['Agent.OS'], 'Darwin')
+
+  - task: PowerShell@2
+    displayName: 'Install uv (Windows)'
+    inputs:
+      targetType: inline
+      script: |
+        iex (irm https://astral.sh/uv/install.ps1)
+        Write-Host "##vso[task.prependpath]C:\Users\cloudtest\.local\bin"
+    condition: eq(variables['Agent.OS'], 'Windows_NT')

eng/pipelines/templates/steps/release-candidate-steps.yml

@@ -4,24 +4,20 @@ steps:
       versionSpec: $(PythonVersion)
 
   - template: /eng/pipelines/templates/steps/use-venv.yml
-    parameters:
-      Activate: false
 
   - pwsh: |
-      $(VENV_ACTIVATION_SCRIPT)
+      Write-Host (Get-Command python).Source
       $ErrorActionPreference = 'Stop'
       $PSNativeCommandUseErrorActionPreference = $true
-      python -m pip install -r $(Build.SourcesDirectory)/eng/ci_tools.txt
-      python -m pip freeze --all
-      Write-Host (Get-Command python).Source
+      uv pip install -r $(Build.SourcesDirectory)/eng/ci_tools.txt
+      uv pip freeze
     displayName: 'Install Dependencies'
 
   - template: /eng/common/testproxy/test-proxy-tool.yml
     parameters:
       runProxy: false
 
   - pwsh: |
-      $(VENV_ACTIVATION_SCRIPT)
       python ./scripts/devops_tasks/dispatch_tox.py "$(TargetedPackages)" --junitxml="junit/test_results.xml" --toxenv="whl" --filter-type="Build"
     displayName: 'Setup - Run Filtered Tests For Python $(PythonVersion)'
     env:

eng/pipelines/templates/steps/seed-virtualenv-wheels.yml

@@ -5,7 +5,7 @@ parameters:
 
 steps:
   - pwsh: |
-      python -m pip install virtualenv
+      uv pip install virtualenv
     displayName: Ensure virtualenv installed
 
   - pwsh: |

eng/pipelines/templates/steps/use-venv.yml

@@ -10,6 +10,8 @@ parameters:
     default: succeeded()
 
 steps:
+  - template: install-uv.yml
+
   - pwsh: |
       $(Build.SourcesDirectory)/eng/scripts/create-venv.ps1 `
         -VenvName "${{ parameters.VirtualEnvironmentName }}" `

eng/scripts/Language-Settings.ps1

@@ -161,9 +161,18 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory)
   $allPkgPropLines = $null
   try
   {
-    Push-Location $RepoRoot
-    $null = python -m pip install "./tools/azure-sdk-tools[build]" -q -I
-    $allPkgPropLines = python (Join-path eng scripts get_package_properties.py) -s $searchPath
+    # Use ‘uv pip install’ if uv is on PATH, otherwise fall back to python -m pip
+    if (Get-Command uv -ErrorAction SilentlyContinue) {
+      Write-Host "Using uv pip install"
+      $null = uv pip install "./tools/azure-sdk-tools[build]"
+      $freezeOutput = uv pip freeze
+      Write-Host "Pip freeze output: $freezeOutput"
+    } else {
+      Write-Host "Using python -m pip install"
+      $null = python -m pip install "./tools/azure-sdk-tools[build]" -q -I
+    }
+
+    Write-Host "Running get_package_properties.py to retrieve package properties"
   }
   catch
   {
@@ -412,7 +421,11 @@ function SetPackageVersion ($PackageName, $Version, $ServiceDirectory, $ReleaseD
   {
     $ReleaseDate = Get-Date -Format "yyyy-MM-dd"
   }
-  python -m pip install "$RepoRoot/tools/azure-sdk-tools[build]" -q -I
+  if (Get-Command uv -ErrorAction SilentlyContinue) {
+    uv pip install "$RepoRoot/tools/azure-sdk-tools[build]"
+  } else {
+    python -m pip install "$RepoRoot/tools/azure-sdk-tools[build]" -q -I
+  }
   sdk_set_version --package-name $PackageName --new-version $Version `
   --service $ServiceDirectory --release-date $ReleaseDate --replace-latest-entry-title $ReplaceLatestEntryTitle
 }

eng/scripts/activate-venv.ps1

@@ -1,4 +1,4 @@
-<#! 
+<#!
 .SYNOPSIS
 Activates a virtual environment for a CI machine. Any further usages of "python" will utilize this virtual environment.
 
@@ -9,7 +9,7 @@ When activating a virtual environment, only a few things are actually functional
 # 2. VIRTUAL_ENV = path to root of the virtual env
 # 3. VIRTUAL_ENV_PROMPT = the prompt that is displayed next to the CLI cursor when the virtual env is active
 # within a CI machine, we only need the PATH and VIRTUAL_ENV variables to be set.
-# 4. (optional and inconsistently) _OLD_VIRTUAL_PATH = the PATH before the virtual env was activated. This is not set in this script. 
+# 4. (optional and inconsistently) _OLD_VIRTUAL_PATH = the PATH before the virtual env was activated. This is not set in this script.
 
 .PARAMETER VenvName
 The name of the virtual environment to activate.
@@ -44,6 +44,11 @@ else {
     $env:PATH = "$venvBinPath`:$($env:PATH)"
 }
 
-Write-Host "Activating virtual environment '$VenvName' at $venvPath via AzDO to the value '$($env:PATH)'"
+Write-Host "Activating virtual environment '$VenvName' via VIRTUAL_ENV variable at $venvPath.'"
 Write-Host "##vso[task.setvariable variable=VIRTUAL_ENV]$($env:VIRTUAL_ENV)"
-Write-Host "##vso[task.prependpath]$($env:PATH)"
+
+Write-Host "Prepending path with $venvBinPath"
+Write-Host "##vso[task.prependpath]$venvBinPath"
+
+Write-Host "Unset of PYTHONHOME"
+Write-Host "##vso[task.setvariable variable=PYTHONHOME]"

eng/scripts/create-venv.ps1

@@ -21,15 +21,22 @@ param(
 )
 
 $venvPath = Join-Path $RepoRoot $VenvName
+
 if (!(Test-Path $venvPath)) {
     $invokingPython = (Get-Command "python").Source
-    Write-Host "Creating virtual environment '$VenvName' using python located at '$invokingPython'."
-    python -m pip install virtualenv==20.25.1
-    python -m virtualenv "$venvPath"
-    $pythonVersion = python --version
-    Write-Host "Virtual environment '$VenvName' created at directory path '$venvPath' utilizing python version $pythonVersion."
-    Write-Host "##vso[task.setvariable variable=$($VenvName)_LOCATION]$venvPath"
-    Write-Host "##vso[task.setvariable variable=$($VenvName)_ACTIVATION_SCRIPT]if(`$IsWindows){. $venvPath/Scripts/Activate.ps1;}else {. $venvPath/bin/activate.ps1}"
+
+    if (Get-Command uv -ErrorAction SilentlyContinue) {
+        Write-Host "Creating virtual environment '$VenvName' using uv."
+        uv venv $venvPath  --verbose
+    }
+    else {
+        Write-Host "Creating virtual environment '$VenvName' using virtualenv and python located at '$invokingPython'."
+        python -m pip install virtualenv==20.25.1
+        python -m virtualenv "$venvPath"
+        $pythonVersion = python --version
+        Write-Host "Virtual environment '$VenvName' created at directory path '$venvPath' utilizing python version $pythonVersion."
+        Write-Host "##vso[task.setvariable variable=$($VenvName)_LOCATION]$venvPath"
+    }
 }
 else {
     Write-Host "Virtual environment '$VenvName' already exists. Skipping creation."