Use platform-specific activation, pin setuptools 69.2.0 (#36454)

* pin setuptools to 69.2.0
* use virtualenv for our virtual environment creation instead of default venv
* source the virtual environment just before build steps, versus prepending path which was working unreliably
* remove venv use from test phase, as it wasn't actually having the effect we originally expected

Author: scbedd

.vscode/cspell.json

@@ -198,7 +198,6 @@
     "contosodataset",
     "centralus",
     "centraluseuap",
-    "corehttp",
     "creds",
     "ctoring",
     "ctxt",

eng/ci_tools.txt

@@ -1,6 +1,6 @@
 # requirements leveraged by ci tools
-setuptools==70.2.0; python_version >= '3.5'
-virtualenv==20.24.3
+setuptools==69.2.0; python_version >= '3.5'
+virtualenv==20.25.1
 wheel==0.43.0
 packaging==23.1
 tox==4.5.0

eng/conda_test_requirements.txt

@@ -8,7 +8,7 @@ trio
 typing_extensions>=3.7.2
 cryptography
 adal
-setuptools==70.2.0; python_version >= '3.5'
+setuptools==69.2.0; python_version >= '3.5'
 pytest-asyncio==0.12.0
 -e sdk/core/azure-core/tests/testserver_tests/coretestserver
 azure-mgmt-storage

eng/pipelines/templates/steps/build-package-artifacts.yml

@@ -52,14 +52,27 @@ steps:
     inputs:
       versionSpec: $(PythonVersion)
 
-  - ${{ if eq(parameters.UseVirtualEnvironment, true) }}:
-    - template: /eng/pipelines/templates/steps/use-venv.yml
+
+  - template: /eng/pipelines/templates/steps/use-venv.yml
+    parameters:
+      VirtualEnvironmentName: "venv"
+      Activate: false
+
+  - bash: |
+      source $(VENV_LOCATION)/bin/activate
+      which python
+      python -m pip install --force -r eng/ci_tools.txt
+      python -m pip freeze --all
+    displayName: 'Prep Environment Linux/Mac'
+    condition: and(succeeded(), or(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OS'], 'Darwin')))
 
   - pwsh: |
+      . $(VENV_LOCATION)/Scripts/Activate.ps1
       which python
       python -m pip install --force -r eng/ci_tools.txt
       python -m pip freeze --all
-    displayName: 'Prep Environment'
+    displayName: 'Prep Environment Windows'
+    condition: and(succeeded(), eq(variables['Agent.OS'], 'Windows_NT'))
 
   - template: set-dev-build.yml@self
     parameters:
@@ -84,10 +97,22 @@ steps:
     displayName: 'Install QEMU Dependencies'
     condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux'))
 
+  - bash: |
+      source $(VENV_LOCATION)/bin/activate
+      which python
+      sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} --inactive
+    displayName: 'Generate Packages Linux/Mac'
+    condition: and(succeeded(), or(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OS'], 'Darwin')))
+    timeoutInMinutes: 80
+    env:
+      CIBW_BUILD_VERBOSITY: 3
+
   - pwsh: |
+      . $(VENV_LOCATION)/Scripts/Activate.ps1
       which python
       sdk_build -d "$(Build.ArtifactStagingDirectory)" "$(TargetingString)" --service=${{parameters.ServiceDirectory}} --inactive
-    displayName: 'Generate Packages'
+    displayName: 'Generate Packages Windows'
+    condition: and(succeededOrFailed(), eq(variables['Agent.OS'], 'Windows_NT'))
     timeoutInMinutes: 80
     env:
       CIBW_BUILD_VERBOSITY: 3

eng/pipelines/templates/steps/build-test.yml

@@ -33,8 +33,6 @@ steps:
     parameters:
       versionSpec: '${{ parameters.PythonVersion }}'
 
-  - template: /eng/pipelines/templates/steps/use-venv.yml
-
   - template: set-dev-build.yml
     parameters:
       ServiceDirectory: ${{ parameters.ServiceDirectory }}
@@ -46,11 +44,11 @@ steps:
       python -m pip freeze
       python -m pip install pip==23.2.1
       python -m pip install wheel==0.43.0 --force-reinstall
-      python -m pip install setuptools==70.2.0 --force-reinstall
+      python -m pip install setuptools==69.2.0 --force-reinstall
       python -m pip install -r eng/ci_tools.txt
       pip --version
       pip freeze
-    displayName: 'Prep Environment'
+    displayName: 'Prep Environment Linux/Mac'
 
   - ${{if eq(parameters.TestProxy, true) }}:
     - template: /eng/common/testproxy/test-proxy-tool.yml

eng/pipelines/templates/steps/use-venv.yml

@@ -2,6 +2,9 @@ parameters:
   - name: VirtualEnvironmentName
     type: string
     default: "venv"
+  - name: Activate
+    type: boolean
+    default: true
 
 steps:
   - pwsh: |
@@ -10,8 +13,9 @@ steps:
         -RepoRoot "$(Build.SourcesDirectory)"
     displayName: Create virtual environment
 
-  - pwsh: |
-      $(Build.SourcesDirectory)/eng/scripts/activate-venv.ps1 `
-        -VenvName "${{ parameters.VirtualEnvironmentName }}" `
-        -RepoRoot "$(Build.SourcesDirectory)"
-    displayName: Use ${{ parameters.VirtualEnvironmentName }} Virtual Environment
+  - ${{ if eq(parameters.Activate, true) }}:
+    - pwsh: |
+        $(Build.SourcesDirectory)/eng/scripts/activate-venv.ps1 `
+          -VenvName "${{ parameters.VirtualEnvironmentName }}" `
+          -RepoRoot "$(Build.SourcesDirectory)"
+      displayName: Use ${{ parameters.VirtualEnvironmentName }} Virtual Environment

eng/regression_tools.txt

@@ -1,5 +1,5 @@
 # requirements leveraged by ci tools
-setuptools==70.2.0; python_version >= '3.5'
+setuptools==69.2.0; python_version >= '3.5'
 virtualenv==20.23.0
 wheel==0.43.0
 Jinja2==3.1.2

eng/scripts/create-venv.ps1

@@ -23,7 +23,8 @@ param(
 $venvPath = Join-Path $RepoRoot $VenvName
 if (!(Test-Path $venvPath)) {
     Write-Host "Creating virtual environment '$VenvName'."
-    python -m venv "$venvPath"
+    python -m pip install virtualenv==20.25.1
+    python -m virtualenv "$venvPath"
     Write-Host "Virtual environment '$VenvName' created."
     Write-Host "##vso[task.setvariable variable=$($VenvName)_LOCATION]$venvPath"
 }

eng/tox/tox_helper_tasks.py

@@ -28,26 +28,19 @@ def unzip_sdist_to_directory(containing_folder: str) -> str:
         tars = glob.glob(os.path.join(containing_folder, "*.tar.gz"))
         return unzip_file_to_directory(tars[0], containing_folder)
 
+
 def unzip_file_to_directory(path_to_zip_file: str, extract_location: str) -> str:
     if path_to_zip_file.endswith(".zip"):
         with zipfile.ZipFile(path_to_zip_file, "r") as zip_ref:
             zip_ref.extractall(extract_location)
             extracted_dir = os.path.basename(os.path.splitext(path_to_zip_file)[0])
             return os.path.join(extract_location, extracted_dir)
-    elif path_to_zip_file.endswith(".tar.gz") or path_to_zip_file.endswith(".tgz"):
-        with tarfile.open(path_to_zip_file, "r:gz") as tar_ref:
-            tar_ref.extractall(extract_location)
-            top_level_dir = None
-            for member in tar_ref.getmembers():
-                if member.isdir():
-                    top_level_dir = member.name.split('/')[0]
-                    break
-            if top_level_dir:
-                return os.path.join(extract_location, top_level_dir)
-            else:
-                raise ValueError("Failed to determine the top-level directory in the tar.gz archive")
     else:
-        raise ValueError("Unsupported file format")
+        with tarfile.open(path_to_zip_file) as tar_ref:
+            tar_ref.extractall(extract_location)
+            extracted_dir = os.path.basename(path_to_zip_file).replace(".tar.gz", "")
+            return os.path.join(extract_location, extracted_dir)
+
 
 def move_and_rename(source_location):
     new_location = os.path.join(os.path.dirname(source_location), "unzipped")

tools/azure-sdk-tools/ci_tools/build.py

@@ -189,21 +189,3 @@ def create_package(
 
     if enable_sdist:
         run([sys.executable, "setup.py", "sdist", "-d", dist], cwd=setup_parsed.folder, check=True)
-
-        # The file normalization changes in setuptools 67.3.0 intends source distribution file names to have the
-        # formatting a whl distributions. This patch avoids that for now.
-        normalized_package_name = setup_parsed.name.replace("-", "_")
-        generated_sdist_name = os.path.join(dist, f"{normalized_package_name}-{setup_parsed.version}.tar.gz")
-        destination = generated_sdist_name.replace(normalized_package_name, setup_parsed.name)
-
-        # in cases where there is no reformatting necessary (sdk/core/corehttp is a good example)
-        # skip this effort
-        if generated_sdist_name != destination:
-            if os.path.exists(generated_sdist_name):
-                if os.path.exists(destination):
-                    os.remove(destination)
-
-                os.rename(generated_sdist_name, destination)
-            else:
-                logger.log(level=logging.ERROR, msg=f"Unable to find a source distribution for {setup_parsed.name} that could be renamed without normalization.")
-