diff --git a/eng/pipelines/templates/jobs/ci.tests.yml b/eng/pipelines/templates/jobs/ci.tests.yml index 8d7460e08ca5..487ab6745965 100644 --- a/eng/pipelines/templates/jobs/ci.tests.yml +++ b/eng/pipelines/templates/jobs/ci.tests.yml @@ -90,29 +90,6 @@ jobs: parameters: AgentImage: ${{ parameters.OSName }} - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.10' - - - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml - parameters: - PackageName: "azure-template" - ServiceDirectory: "template" - TestPipeline: ${{ parameters.TestPipeline }} - - - pwsh: | - python -m pip install "tools/azure-sdk-tools[build]" - displayName: Install build tooling - - - task: PythonScript@0 - displayName: 'Set Tox Environment' - inputs: - scriptPath: 'scripts/devops_tasks/set_tox_environment.py' - arguments: >- - --unsupported="$(UnsupportedToxEnvironments)" - --override="$(Run.ToxCustomEnvs)" - --team-project="$(System.TeamProject)" - - template: ../steps/build-test.yml parameters: TestMarkArgument: ${{ parameters.TestMarkArgument }} diff --git a/eng/pipelines/templates/jobs/ci.yml b/eng/pipelines/templates/jobs/ci.yml index 8716a14f30de..8d49bb9903ad 100644 --- a/eng/pipelines/templates/jobs/ci.yml +++ b/eng/pipelines/templates/jobs/ci.yml @@ -209,6 +209,14 @@ jobs: os: linux steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(PythonVersion)' + condition: succeededOrFailed() + inputs: + versionSpec: '$(PythonVersion)' + + - template: /eng/pipelines/templates/steps/use-venv.yml + - template: /eng/common/pipelines/templates/steps/check-spelling.yml parameters: ContinueOnError: false @@ -275,6 +283,7 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: '3.12' + - template: /eng/pipelines/templates/steps/use-venv.yml - template: /eng/common/pipelines/templates/steps/save-package-properties.yml parameters: ServiceDirectory: ${{parameters.ServiceDirectory}} @@ -310,8 +319,11 @@ jobs: - task: UsePythonVersion@0 inputs: versionSpec: '3.12' + - template: /eng/pipelines/templates/steps/use-venv.yml - pwsh: | - python -m pip install "./tools/azure-sdk-tools[build]" + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $(PIP_EXE) install "./tools/azure-sdk-tools[build]" displayName: 'Prep Environment' - task: PythonScript@0 displayName: 'Ensure service coverage' diff --git a/eng/pipelines/templates/steps/analyze.yml b/eng/pipelines/templates/steps/analyze.yml index 0e670721a1a4..6ab18778c379 100644 --- a/eng/pipelines/templates/steps/analyze.yml +++ b/eng/pipelines/templates/steps/analyze.yml @@ -48,7 +48,7 @@ steps: Condition: succeededOrFailed() - script: | - python -m pip install "./tools/azure-sdk-tools[build]" -q -I + $(PIP_EXE) install "./tools/azure-sdk-tools[build]" sdk_find_invalid_versions --always-succeed --service=${{parameters.ServiceDirectory}} displayName: Find Invalid Versions condition: succeededOrFailed() diff --git a/eng/pipelines/templates/steps/analyze_dependency.yml b/eng/pipelines/templates/steps/analyze_dependency.yml index 37da2dce5aef..4ed795ab7f7a 100644 --- a/eng/pipelines/templates/steps/analyze_dependency.yml +++ b/eng/pipelines/templates/steps/analyze_dependency.yml @@ -2,14 +2,8 @@ parameters: ScanPath: '' steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(PythonVersion)' - condition: succeededOrFailed() - inputs: - versionSpec: '$(PythonVersion)' - - pwsh: | - python -m pip install -r eng/ci_tools.txt + $(PIP_EXE) install -r eng/ci_tools.txt displayName: 'Install Python Tools' condition: succeededOrFailed() diff --git a/eng/pipelines/templates/steps/build-extended-artifacts.yml b/eng/pipelines/templates/steps/build-extended-artifacts.yml index 0505e7561997..05d923b93030 100644 --- a/eng/pipelines/templates/steps/build-extended-artifacts.yml +++ b/eng/pipelines/templates/steps/build-extended-artifacts.yml @@ -28,6 +28,8 @@ steps: inputs: versionSpec: '3.13' + - template: /eng/pipelines/templates/steps/use-venv.yml + - pwsh: | if ("${{ parameters.ServiceDirectory }}" -ne "auto") { @@ -53,10 +55,9 @@ steps: displayName: 'Tag scheduled builds' condition: and(eq(variables['Build.SourceBranchName'], variables['DefaultBranch']), eq(variables['Build.Reason'],'Schedule')) - - template: /eng/pipelines/templates/steps/use-venv.yml - - script: | - python -m pip install -r eng/ci_tools.txt + which python + $(PIP_EXE) install -r eng/ci_tools.txt displayName: 'Prep Environment' - template: set-dev-build.yml diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 84c002a911b1..cbe172654914 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -31,6 +31,8 @@ steps: inputs: versionSpec: $(PythonVersion) + - template: /eng/pipelines/templates/steps/use-venv.yml + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml@self parameters: PackageName: "azure-template" @@ -85,21 +87,17 @@ steps: parameters: PackagePropertiesFolder: $(Build.ArtifactStagingDirectory)/PackageInfo - - template: /eng/pipelines/templates/steps/use-venv.yml - parameters: - VirtualEnvironmentName: "venv" - Activate: false - Condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) - - pwsh: | - $(VENV_ACTIVATION_SCRIPT) + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true which python - python -m pip install --force -r eng/ci_tools.txt + $(PIP_EXE) install -r eng/ci_tools.txt + if ($env:AGENT_OS -eq "Linux") { Write-Host "Installing release reqs" - python -m pip install -r eng/release_requirements.txt + $(PIP_EXE) install -r eng/release_requirements.txt } - python -m pip freeze --all + $(PIP_EXE) freeze displayName: 'Prep Environment' condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) @@ -111,7 +109,6 @@ steps: condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) - pwsh: | - $(VENV_ACTIVATION_SCRIPT) which python sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --inactive displayName: 'Generate Packages' @@ -121,7 +118,7 @@ steps: CIBW_BUILD_VERBOSITY: 3 - pwsh: | - $(VENV_ACTIVATION_SCRIPT) + which python twine check $(Build.ArtifactStagingDirectory)/**/*.whl twine check $(Build.ArtifactStagingDirectory)/**/*.tar.gz displayName: 'Verify Readme' diff --git a/eng/pipelines/templates/steps/build-test.yml b/eng/pipelines/templates/steps/build-test.yml index 69f42f70c165..8364a124cb4e 100644 --- a/eng/pipelines/templates/steps/build-test.yml +++ b/eng/pipelines/templates/steps/build-test.yml @@ -17,6 +17,7 @@ parameters: TestProxy: false UseFederatedAuth: false ServiceConnection: '' + TestPipeline: false # Please use `$(TargetingString)` to refer to the python packages glob string. This variable is set from resolve-package-targeting.yml. @@ -26,25 +27,32 @@ steps: versionSpec: '${{ parameters.PythonVersion }}' - template: /eng/pipelines/templates/steps/use-venv.yml + + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml parameters: - Activate: false + PackageName: "azure-template" + ServiceDirectory: "template" + TestPipeline: ${{ parameters.TestPipeline }} - template: set-dev-build.yml - pwsh: | - if ($IsWindows) { - . $(VENV_LOCATION)/Scripts/Activate.ps1 - } - else { - . $(VENV_LOCATION)/bin/activate.ps1 - } + Write-Host (Get-Command python).Source $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true - python -m pip install --force -r eng/ci_tools.txt - python -m pip freeze --all - Write-Host (Get-Command python).Source + $(PIP_EXE) install -r eng/ci_tools.txt + $(PIP_EXE) freeze displayName: 'Prep Environment' + - task: PythonScript@0 + displayName: 'Set Tox Environment' + inputs: + scriptPath: 'scripts/devops_tasks/set_tox_environment.py' + arguments: >- + --unsupported="$(UnsupportedToxEnvironments)" + --override="$(Run.ToxCustomEnvs)" + --team-project="$(System.TeamProject)" + - template: /eng/common/testproxy/test-proxy-tool.yml parameters: runProxy: false @@ -75,12 +83,6 @@ steps: $env:AZURESUBSCRIPTION_CLIENT_ID = $account.Id; $env:AZURESUBSCRIPTION_TENANT_ID = $account.Tenants; - if ($IsWindows) { - . $(VENV_LOCATION)/Scripts/Activate.ps1 - } - else { - . $(VENV_LOCATION)/bin/activate.ps1 - } Write-Host (Get-Command python).Source if ($env:TESTMARKARGUMENT) { @@ -104,12 +106,6 @@ steps: - ${{ else }}: - pwsh: | - if ($IsWindows) { - . $(VENV_LOCATION)/Scripts/Activate.ps1 - } - else { - . $(VENV_LOCATION)/bin/activate.ps1 - } Write-Host (Get-Command python).Source if ($env:TESTMARKARGUMENT) { @@ -145,12 +141,6 @@ steps: - ${{ parameters.AfterTestSteps }} - pwsh: | - if ($IsWindows) { - . $(VENV_LOCATION)/Scripts/Activate.ps1 - } - else { - . $(VENV_LOCATION)/bin/activate.ps1 - } Write-Host (Get-Command python).Source python scripts/devops_tasks/create_coverage.py @@ -176,12 +166,6 @@ steps: $env:AZURESUBSCRIPTION_CLIENT_ID = $account.Id; $env:AZURESUBSCRIPTION_TENANT_ID = $account.Tenants; - if ($IsWindows) { - . $(VENV_LOCATION)/Scripts/Activate.ps1 - } - else { - . $(VENV_LOCATION)/bin/activate.ps1 - } Write-Host (Get-Command python).Source python scripts/devops_tasks/dispatch_tox.py "$(TargetingString)" ` diff --git a/eng/pipelines/templates/steps/install-uv.yml b/eng/pipelines/templates/steps/install-uv.yml new file mode 100644 index 000000000000..9d2a070ddc8f --- /dev/null +++ b/eng/pipelines/templates/steps/install-uv.yml @@ -0,0 +1,26 @@ + +steps: + - task: Bash@3 + displayName: 'Install uv (Linux/macOS)' + inputs: + targetType: inline + script: | + curl -LsSf https://astral.sh/uv/install.sh | sh + condition: or(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OS'], 'Darwin')) + + - task: Bash@3 + inputs: + targetType: inline + script: | + echo "##vso[task.prependpath]$HOME/.local/bin" + displayName: 'Prepend path for MacOS' + condition: eq(variables['Agent.OS'], 'Darwin') + + - task: PowerShell@2 + displayName: 'Install uv (Windows)' + inputs: + targetType: inline + script: | + iex (irm https://astral.sh/uv/install.ps1) + Write-Host "##vso[task.prependpath]$env:USERPROFILE\.local\bin" + condition: eq(variables['Agent.OS'], 'Windows_NT') diff --git a/eng/pipelines/templates/steps/release-candidate-steps.yml b/eng/pipelines/templates/steps/release-candidate-steps.yml index 720058f3b888..b5c2632a04ae 100644 --- a/eng/pipelines/templates/steps/release-candidate-steps.yml +++ b/eng/pipelines/templates/steps/release-candidate-steps.yml @@ -4,16 +4,13 @@ steps: versionSpec: $(PythonVersion) - template: /eng/pipelines/templates/steps/use-venv.yml - parameters: - Activate: false - pwsh: | - $(VENV_ACTIVATION_SCRIPT) + Write-Host (Get-Command python).Source $ErrorActionPreference = 'Stop' $PSNativeCommandUseErrorActionPreference = $true - python -m pip install -r $(Build.SourcesDirectory)/eng/ci_tools.txt - python -m pip freeze --all - Write-Host (Get-Command python).Source + $(PIP_EXE) install -r $(Build.SourcesDirectory)/eng/ci_tools.txt + $(PIP_EXE) freeze displayName: 'Install Dependencies' - template: /eng/common/testproxy/test-proxy-tool.yml @@ -21,7 +18,6 @@ steps: runProxy: false - pwsh: | - $(VENV_ACTIVATION_SCRIPT) python ./scripts/devops_tasks/dispatch_tox.py "$(TargetedPackages)" --junitxml="junit/test_results.xml" --toxenv="whl" --filter-type="Build" displayName: 'Setup - Run Filtered Tests For Python $(PythonVersion)' env: diff --git a/eng/pipelines/templates/steps/run_apistub.yml b/eng/pipelines/templates/steps/run_apistub.yml index 7928b0d22f74..ea84514f3ba0 100644 --- a/eng/pipelines/templates/steps/run_apistub.yml +++ b/eng/pipelines/templates/steps/run_apistub.yml @@ -16,8 +16,14 @@ steps: versionSpec: '3.10' condition: and(succeededOrFailed(), ne(variables['Skip.ApiStubGen'],'true')) - - script: | - python -m pip install -r eng/ci_tools.txt + - template: /eng/pipelines/templates/steps/use-venv.yml + parameters: + VirtualEnvironmentName: "venv-api-stub" + + - pwsh: | + $ErrorActionPreference = 'Stop' + $PSNativeCommandUseErrorActionPreference = $true + $(PIP_EXE) install -r eng/ci_tools.txt displayName: 'Prep Environment' condition: and(succeededOrFailed(), ne(variables['Skip.ApiStubGen'],'true')) diff --git a/eng/pipelines/templates/steps/seed-virtualenv-wheels.yml b/eng/pipelines/templates/steps/seed-virtualenv-wheels.yml index a3a178ca7a9f..6aefa40ad4a5 100644 --- a/eng/pipelines/templates/steps/seed-virtualenv-wheels.yml +++ b/eng/pipelines/templates/steps/seed-virtualenv-wheels.yml @@ -5,7 +5,7 @@ parameters: steps: - pwsh: | - python -m pip install virtualenv + $(PIP_EXE) install virtualenv displayName: Ensure virtualenv installed - pwsh: | diff --git a/eng/pipelines/templates/steps/set-dev-build.yml b/eng/pipelines/templates/steps/set-dev-build.yml index 26233110f3c8..74203bc0a263 100644 --- a/eng/pipelines/templates/steps/set-dev-build.yml +++ b/eng/pipelines/templates/steps/set-dev-build.yml @@ -9,7 +9,7 @@ steps: - template: /eng/common/pipelines/templates/steps/daily-dev-build-variable.yml - pwsh: | - python -m pip install "tools/azure-sdk-tools[build]" + $(PIP_EXE) install "tools/azure-sdk-tools[build]" sdk_set_dev_version "*" --build-id="$(Build.BuildNumber)" displayName: "Update package versions for dev build" condition: and(succeededOrFailed(), eq(variables['SetDevVersion'],'true'), ${{ parameters.Condition }}) diff --git a/eng/pipelines/templates/steps/use-venv.yml b/eng/pipelines/templates/steps/use-venv.yml index a25459c7a156..158a7e4a6e4e 100644 --- a/eng/pipelines/templates/steps/use-venv.yml +++ b/eng/pipelines/templates/steps/use-venv.yml @@ -8,8 +8,27 @@ parameters: - name: Condition type: string default: succeeded() + - name: UseUv + type: boolean + default: true steps: + - ${{ if eq(parameters.UseUv, true) }}: + - template: install-uv.yml + + - pwsh: | + if (Get-Command uv -ErrorAction SilentlyContinue) { + Write-Host "##vso[task.setvariable variable=PIP_EXE]uv pip" + } else { + Write-Error "Expected 'uv' command to be available, but it was not. Exiting with error." + exit 1 + } + displayName: Setting PIP_EXE to 'uv pip' + - ${{ else }}: + - pwsh: | + Write-Host "##vso[task.setvariable variable=PIP_EXE]python -m pip" + displayName: Setting PIP_EXE to 'python -m pip' + - pwsh: | $(Build.SourcesDirectory)/eng/scripts/create-venv.ps1 ` -VenvName "${{ parameters.VirtualEnvironmentName }}" ` @@ -23,4 +42,4 @@ steps: -VenvName "${{ parameters.VirtualEnvironmentName }}" ` -RepoRoot "$(Build.SourcesDirectory)" displayName: Use ${{ parameters.VirtualEnvironmentName }} Virtual Environment - condition: ${{ parameters.Condition }} + condition: ${{ parameters.Condition }} \ No newline at end of file diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 70a59b6fdb1e..a1fd500691ca 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -161,8 +161,18 @@ function Get-AllPackageInfoFromRepo ($serviceDirectory) $allPkgPropLines = $null try { - Push-Location $RepoRoot - $null = python -m pip install "./tools/azure-sdk-tools[build]" -q -I + # Use ‘uv pip install’ if uv is on PATH, otherwise fall back to python -m pip + if (Get-Command uv -ErrorAction SilentlyContinue) { + Write-Host "Using uv pip install" + $null = uv pip install "./tools/azure-sdk-tools[build]" + $freezeOutput = uv pip freeze + Write-Host "Pip freeze output: $freezeOutput" + } else { + Write-Host "Using python -m pip install" + $null = python -m pip install "./tools/azure-sdk-tools[build]" -q -I + } + + Write-Host "Running get_package_properties.py to retrieve package properties" $allPkgPropLines = python (Join-path eng scripts get_package_properties.py) -s $searchPath } catch @@ -412,7 +422,11 @@ function SetPackageVersion ($PackageName, $Version, $ServiceDirectory, $ReleaseD { $ReleaseDate = Get-Date -Format "yyyy-MM-dd" } - python -m pip install "$RepoRoot/tools/azure-sdk-tools[build]" -q -I + if (Get-Command uv -ErrorAction SilentlyContinue) { + uv pip install "$RepoRoot/tools/azure-sdk-tools[build]" + } else { + python -m pip install "$RepoRoot/tools/azure-sdk-tools[build]" -q -I + } sdk_set_version --package-name $PackageName --new-version $Version ` --service $ServiceDirectory --release-date $ReleaseDate --replace-latest-entry-title $ReplaceLatestEntryTitle } diff --git a/eng/scripts/activate-venv.ps1 b/eng/scripts/activate-venv.ps1 index 9d192212052e..82a73b37731c 100644 --- a/eng/scripts/activate-venv.ps1 +++ b/eng/scripts/activate-venv.ps1 @@ -1,4 +1,4 @@ -<#! +<#! .SYNOPSIS Activates a virtual environment for a CI machine. Any further usages of "python" will utilize this virtual environment. @@ -9,7 +9,7 @@ When activating a virtual environment, only a few things are actually functional # 2. VIRTUAL_ENV = path to root of the virtual env # 3. VIRTUAL_ENV_PROMPT = the prompt that is displayed next to the CLI cursor when the virtual env is active # within a CI machine, we only need the PATH and VIRTUAL_ENV variables to be set. -# 4. (optional and inconsistently) _OLD_VIRTUAL_PATH = the PATH before the virtual env was activated. This is not set in this script. +# 4. (optional and inconsistently) _OLD_VIRTUAL_PATH = the PATH before the virtual env was activated. This is not set in this script. .PARAMETER VenvName The name of the virtual environment to activate. @@ -44,6 +44,11 @@ else { $env:PATH = "$venvBinPath`:$($env:PATH)" } -Write-Host "Activating virtual environment '$VenvName' at $venvPath via AzDO to the value '$($env:PATH)'" +Write-Host "Activating virtual environment '$VenvName' via VIRTUAL_ENV variable at $venvPath.'" Write-Host "##vso[task.setvariable variable=VIRTUAL_ENV]$($env:VIRTUAL_ENV)" -Write-Host "##vso[task.prependpath]$($env:PATH)" \ No newline at end of file + +Write-Host "Prepending path with $venvBinPath" +Write-Host "##vso[task.prependpath]$venvBinPath" + +Write-Host "Unset of PYTHONHOME" +Write-Host "##vso[task.setvariable variable=PYTHONHOME]" \ No newline at end of file diff --git a/eng/scripts/create-venv.ps1 b/eng/scripts/create-venv.ps1 index 2824f6bc161d..1faee2584194 100644 --- a/eng/scripts/create-venv.ps1 +++ b/eng/scripts/create-venv.ps1 @@ -21,15 +21,22 @@ param( ) $venvPath = Join-Path $RepoRoot $VenvName + if (!(Test-Path $venvPath)) { $invokingPython = (Get-Command "python").Source - Write-Host "Creating virtual environment '$VenvName' using python located at '$invokingPython'." - python -m pip install virtualenv==20.25.1 - python -m virtualenv "$venvPath" + + if (Get-Command uv -ErrorAction SilentlyContinue) { + Write-Host "Creating virtual environment '$VenvName' using uv." + uv venv $venvPath --verbose + } + else { + Write-Host "Creating virtual environment '$VenvName' using virtualenv and python located at '$invokingPython'." + python -m pip install virtualenv==20.25.1 + python -m virtualenv "$venvPath" + } $pythonVersion = python --version Write-Host "Virtual environment '$VenvName' created at directory path '$venvPath' utilizing python version $pythonVersion." Write-Host "##vso[task.setvariable variable=$($VenvName)_LOCATION]$venvPath" - Write-Host "##vso[task.setvariable variable=$($VenvName)_ACTIVATION_SCRIPT]if(`$IsWindows){. $venvPath/Scripts/Activate.ps1;}else {. $venvPath/bin/activate.ps1}" } else { Write-Host "Virtual environment '$VenvName' already exists. Skipping creation."