From bd4010739e3bdde22634aedd96346cce6648936a Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Fri, 10 Oct 2025 13:52:58 +0800 Subject: [PATCH 01/52] Update azure-pipelines.yml for Azure Pipelines --- azure-pipelines.yml | 1128 +++---------------------------------------- 1 file changed, 64 insertions(+), 1064 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f2fcdaca15e..88b860bd44c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,19 +1,3 @@ -resources: -- repo: self - -trigger: - batch: true - branches: - include: - - '*' - exclude: - - 'release*' - -pr: - branches: - include: - - '*' - variables: - template: ${{ variables.Pipeline.Workspace }}/.azure-pipelines/templates/variables.yml - name: Codeql.Enabled @@ -33,80 +17,9 @@ parameters: pool: pool-ubuntu-latest-arm64 jobs: -- job: CheckPullRequest - displayName: "Check the Format of Pull Request Title and Content" - condition: and(succeeded(), in(variables['System.PullRequest.TargetBranch'], 'dev', 'release', 'main')) - - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - bash: | - echo "Check Title of Pull Request: #$(System.PullRequest.PullRequestNumber)" - title=$(curl https://api.github.com/repos/$(Build.Repository.Name)/pulls/$(System.PullRequest.PullRequestNumber) | jq -r '.title') - echo $title - body=$(curl https://api.github.com/repos/$(Build.Repository.Name)/pulls/$(System.PullRequest.PullRequestNumber) | jq -r '.body') - echo $body - if [ "$(System.PullRequest.TargetBranch)" != "release" ] && echo $title | grep -iqF hotfix:; then - echo "Hotfix PR should target release branch." - exit 1 - fi - if [ "$(System.PullRequest.TargetBranch)" == "main" ]; then - echo "Feature PR should target dev branch." - exit 1 - fi - python scripts/ci/check_pull_request.py "$title" "$body" - -- job: RejectPullRequestToMasterBranch - displayName: "Reject Pull Request To Master Branch" - condition: and(succeeded(), eq(variables['System.PullRequest.TargetBranch'], 'master')) - - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - bash: | - echo "Reject pull request directly to master branch" - exit 1 - -- job: CredentialScanner - displayName: "Credential Scanner" - pool: - name: ${{ variables.windows_pool }} - steps: - - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3 - displayName: 'Run Credential Scanner' - inputs: - toolVersion: '2.1.17' - suppressionsFile: './scripts/ci/credscan/CredScanSuppressions.json' - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2 - displayName: 'Post Analysis' - inputs: - GdnBreakAllTools: false - GdnBreakGdnToolCredScan: true - GdnBreakGdnToolCredScanSeverity: Error - -- job: PolicyCheck - displayName: "Policy Check" - pool: - name: ${{ variables.windows_pool }} - steps: - - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@2 - displayName: 'Run Policy Check' - inputs: - targetType: F - result: PoliCheckResult.xml - - - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2 - displayName: 'Post Analysis' - inputs: - GdnBreakAllTools: false - GdnBreakGdnToolPoliCheck: true - GdnBreakGdnToolPoliCheckSeverity: Error - job: ExtractMetadata displayName: Extract Metadata - - condition: succeeded() pool: name: ${{ variables.ubuntu_pool }} steps: @@ -116,98 +29,27 @@ jobs: targetType: 'filePath' filePath: scripts/release/get_version.sh - - task: PublishPipelineArtifact@0 displayName: 'Publish Artifact: metadata' inputs: TargetPath: $(Build.ArtifactStagingDirectory) ArtifactName: metadata -- job: VerifyLinuxRequirements - displayName: 'Verify src/azure-cli/requirements.*.Linux.txt' - condition: succeeded() - pool: - name: ${{ variables.ubuntu_pool }} - - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - - bash: ./scripts/ci/dependency_check.sh - displayName: 'Verify src/azure-cli/requirements.py3.Linux.txt' - -- job: VerifyDarwinRequirements - displayName: 'Verify src/azure-cli/requirements.*.Darwin.txt' - condition: succeeded() - pool: - vmImage: ${{ variables.macos_pool }} - - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - - bash: ./scripts/ci/dependency_check.sh - displayName: 'Verify src/azure-cli/requirements.py3.Darwin.txt' - -- job: VerifyWindowsRequirements - displayName: 'Verify src/azure-cli/requirements.*.Windows.txt' - condition: succeeded() - pool: - name: ${{ variables.windows_pool }} - - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - - task: BatchScript@1 - inputs: - filename: ./scripts/ci/dependency_check.bat - displayName: 'Verify src/azure-cli/requirements.py3.Windows.txt' - -- job: VerifyVersions - displayName: Verify Command Module Versions - condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/release'), eq(variables['System.PullRequest.TargetBranch'], 'release'))) - - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - azdev verify history - - displayName: 'Verify History' - -- job: BuildWindowsMSI - displayName: Build Windows MSI +- job: BuildRpmPackagesAzureLinux + displayName: Build Rpm Package + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) strategy: matrix: - x86: - Platform: x86 - x64: - Platform: x64 - - dependsOn: ExtractMetadata - condition: succeeded() + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + image: mcr.microsoft.com/azurelinux/base/core:3.0 + artifact: rpm-azurelinux3.0-${{ arch.value }} + pool: ${{ arch.pool }} pool: - name: ${{ variables.windows_pool }} + name: $(pool) steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker - task: PipAuthenticate@1 condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') @@ -215,130 +57,23 @@ jobs: inputs: artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) - - script: | - set ARCH=$(Platform) - set TARGET=msi - set /p CLI_VERSION=<$(System.ArtifactsDirectory)/metadata/version - set - - build_scripts/windows/scripts/build.cmd - displayName: 'Build Windows MSI' + - task: Bash@3 + displayName: 'Build Rpm Package: Azure Linux' + inputs: + targetType: 'filePath' + filePath: scripts/release/rpm/pipeline_azurelinux.sh - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 displayName: 'SBOM' condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') inputs: - BuildDropPath: 'build_scripts/windows/out/' + BuildDropPath: $(Build.ArtifactStagingDirectory) - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: MSI' - inputs: - TargetPath: 'build_scripts/windows/out/' - ArtifactName: msi-$(Platform) - -- job: BuildWindowsZIP - displayName: Build Windows ZIP - strategy: - matrix: - x64: - Platform: x64 - - dependsOn: ExtractMetadata - condition: succeeded() - pool: - name: ${{ variables.windows_pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: PipAuthenticate@1 - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) - - - script: | - set ARCH=$(Platform) - set TARGET=zip - set /p CLI_VERSION=<$(System.ArtifactsDirectory)/metadata/version - set - - build_scripts/windows/scripts/build.cmd - displayName: 'Build Windows ZIP' - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - inputs: - BuildDropPath: 'build_scripts/windows/out/' - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: ZIP' - inputs: - TargetPath: 'build_scripts/windows/out/' - ArtifactName: zip-$(Platform) - -- job: TestMsiInstallation - displayName: Test MSI Installation - strategy: - matrix: - x86: - Platform: x86 - x64: - Platform: x64 - - dependsOn: BuildWindowsMSI - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: ${{ variables.windows_pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/msi' - artifactName: msi-$(Platform) - - - task: PowerShell@2 - displayName: Install and Load CLI - inputs: - filePath: build_scripts\windows\scripts\test_msi_installation.ps1 - -- job: TestZipInstallation - displayName: Test ZIP Installation - dependsOn: BuildWindowsZIP - strategy: - matrix: - x64: - Platform: x64 - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: ${{ variables.windows_pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/zip' - artifactName: zip-$(Platform) - - - task: PowerShell@2 - displayName: Expand and Load CLI + displayName: 'Publish Artifact: rpm-azurelinux' inputs: - filePath: build_scripts\windows\scripts\test_zip_installation.ps1 + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(artifact) - job: BuildDockerImageAzureLinux displayName: Build Docker Image Azure Linux @@ -382,6 +117,50 @@ jobs: TargetPath: $(Build.ArtifactStagingDirectory) ArtifactName: $(artifactName) +- job: TestRpmPackagesAzureLinux + displayName: Test Rpm Package + timeoutInMinutes: 180 + dependsOn: + - BuildRpmPackagesAzureLinux + - ExtractMetadata + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: $(pool) + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + image: mcr.microsoft.com/azurelinux/base/core:3.0 + artifact: rpm-azurelinux3.0-${{ arch.value }} + pool: ${{ arch.pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' + artifactName: $(artifact) + + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + + - bash: | + set -ex + + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$CLI_VERSION-1.*.rpm" -printf '%f\n') + + echo "== Test rpm package on ${IMAGE} ==" + docker pull $IMAGE + docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh" + + displayName: 'Test Rpm Package Azure Linux' + - job: TestDockerImageAzureLinux displayName: Test Docker Image Azure Linux dependsOn: @@ -423,782 +202,3 @@ jobs: docker load < $TAR_FILE docker run $IMAGE_NAME /bin/bash -c "time az self-test && time az --version && tdnf list --installed && sleep 5" displayName: 'Bash Script' - -- job: BuildPythonWheel - displayName: Build Python Wheels - - dependsOn: ExtractMetadata - condition: succeeded() - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - - task: PipAuthenticate@1 - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) - - - script: | - if [[ "$(Build.Reason)" == "PullRequest" ]]; then - branch=$(System.PullRequest.TargetBranch) - else - branch=$(Build.SourceBranchName) - fi - scripts/release/pypi/build.sh $branch - displayName: 'Run Wheel Build Script' - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: pypi' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: pypi - -- job: TestPythonWheel - displayName: Test Python Wheels - strategy: - matrix: - Python312: - python.version: '3.12' - Python313: - python.version: '3.13' - dependsOn: BuildPythonWheel - condition: succeeded() - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - task: DownloadPipelineArtifact@1 - displayName: 'Download PyPI Packages' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/pypi' - artifactName: pypi - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - - bash: | - #!/usr/bin/env bash - set -ex - - CLI_VERSION=`cat $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata/version` - echo "== Testing pip install on $PYTHON_VERSION ==" - cd $BUILD_ARTIFACTSTAGINGDIRECTORY/pypi - pip install --find-links ./ azure_cli-$CLI_VERSION*whl && az self-test && az --version && sleep 5 - displayName: 'Test pip Install' - -- job: TestCore - displayName: Unit Test for Core - timeoutInMinutes: 10 - pool: - name: ${{ variables.ubuntu_pool }} - strategy: - matrix: - Python312: - python.version: '3.12' - Python313: - python.version: '3.13' - steps: - - template: .azure-pipelines/templates/automation_test.yml - parameters: - pythonVersion: '$(python.version)' - module: 'azure-cli-core' - -- job: TestTelemetry - displayName: Unit Test for Telemetry - timeoutInMinutes: 10 - pool: - name: ${{ variables.ubuntu_pool }} - strategy: - matrix: - Python312: - python.version: '3.12' - Python313: - python.version: '3.13' - steps: - - template: .azure-pipelines/templates/automation_test.yml - parameters: - pythonVersion: '$(python.version)' - module: 'azure-cli-telemetry' - -- job: IntegrationTestAgainstProfiles - displayName: Integration Test against Profiles - dependsOn: BuildPythonWheel - condition: succeeded() - timeoutInMinutes: 20 - - pool: - name: ${{ variables.ubuntu_pool }} - strategy: - matrix: - Python312: - python.version: '3.12' - Python313: - python.version: '3.13' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - - bash: pip install --upgrade pip wheel setuptools - displayName: 'Install pip and wheel' - - bash: ./scripts/ci/test_profile_integration.sh - displayName: 'Run Integration Test against Profiles' - -- job: TestExtensionsLoading - displayName: Test Extensions Loading - condition: succeeded() - timeoutInMinutes: 80 - - pool: - name: ${{ variables.ubuntu_pool }} - strategy: - matrix: - Python313: - python.version: '3.13' - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - - bash: pip install --upgrade pip wheel setuptools - displayName: 'Install pip and wheel setuptools' - - bash: ./scripts/ci/test_extensions.sh - displayName: 'Load extensions' - -- job: BuildHomebrewFormula - displayName: Build Homebrew Formula - - dependsOn: BuildPythonWheel - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - bash: | - #!/bin/bash - - root=$(cd $(dirname $0); pwd) - - set -evx - - CLI_VERSION=`cat $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata/version` - HOMEBREW_UPSTREAM_URL=`curl -Ls -o /dev/null -w %{url_effective} https://api.github.com/repos/Azure/azure-cli/tarball/$BUILD_SOURCEVERSION` - - docker_files=$(cd $BUILD_SOURCESDIRECTORY/scripts/release/homebrew/docker; pwd) - src_files=$(cd $BUILD_SOURCESDIRECTORY/src; pwd) - - echo "Generating formula in docker container ... " - docker run -v $docker_files:/mnt/scripts \ - -v $src_files:/mnt/src \ - -e CLI_VERSION=$CLI_VERSION \ - -e HOMEBREW_UPSTREAM_URL=$HOMEBREW_UPSTREAM_URL \ - --name azurecli \ - mcr.microsoft.com/azurelinux/base/python:3 \ - /mnt/scripts/run.sh - - # clean up - rm -rf $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata - - docker cp azurecli:azure-cli.rb $BUILD_ARTIFACTSTAGINGDIRECTORY/azure-cli.rb - docker rm --force azurecli - displayName: 'Build homebrew formula' - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: homebrew' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: homebrew - -- job: TestHomebrewFormula - displayName: Test Homebrew Formula - - dependsOn: BuildHomebrewFormula - condition: succeeded() - pool: - vmImage: ${{ variables.macos_pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Homebrew' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/homebrew' - artifactName: homebrew - - - - bash: | - set -ev - # Force relink python@3.xx in Homebrew to resolve the conflict with pre-installed python 3.xx on macOS-12 image - # See: https://github.com/Azure/azure-cli/issues/29054 - python_version=3.13 - brew unlink python@$python_version && brew link --overwrite python@$python_version - - echo == Remove pre-installed azure-cli == - brew uninstall azure-cli - - echo == Install azure-cli.rb formula == - # Need to create a dummy homebrew tap to install the formula - git config --global user.email "you@example.com" - git config --global user.name "Your Name" - brew tap-new dev/azure-cli - cp $SYSTEM_ARTIFACTSDIRECTORY/homebrew/azure-cli.rb $(brew --repository)/Library/Taps/dev/homebrew-azure-cli/Formula/ - brew install --build-from-source dev/homebrew-azure-cli/azure-cli - - echo == Az Version == - az --version - - echo == Run Self-Test == - az self-test - - displayName: 'Bash Script' - -- job: TestHomebrewPackage - displayName: Test Homebrew Package - timeoutInMinutes: 180 - dependsOn: BuildHomebrewFormula - # condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - condition: false - pool: - vmImage: ${{ variables.macos_pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/homebrew' - artifactName: homebrew - - - - bash: ./scripts/release/homebrew/test_homebrew_package.sh - - - displayName: 'Test Homebrew Package' - - -- job: BuildRpmPackagesAzureLinux - displayName: Build Rpm Package - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - image: mcr.microsoft.com/azurelinux/base/core:3.0 - artifact: rpm-azurelinux3.0-${{ arch.value }} - pool: ${{ arch.pool }} - pool: - name: $(pool) - steps: - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - - task: PipAuthenticate@1 - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) - - - task: Bash@3 - displayName: 'Build Rpm Package: Azure Linux' - inputs: - targetType: 'filePath' - filePath: scripts/release/rpm/pipeline_azurelinux.sh - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: rpm-azurelinux' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(artifact) - -- job: TestRpmPackagesAzureLinux - displayName: Test Rpm Package - timeoutInMinutes: 180 - dependsOn: - - BuildRpmPackagesAzureLinux - - ExtractMetadata - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: $(pool) - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - image: mcr.microsoft.com/azurelinux/base/core:3.0 - artifact: rpm-azurelinux3.0-${{ arch.value }} - pool: ${{ arch.pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' - artifactName: $(artifact) - - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - - bash: | - set -ex - - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$CLI_VERSION-1.*.rpm" -printf '%f\n') - - echo "== Test rpm package on ${IMAGE} ==" - docker pull $IMAGE - docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh" - - displayName: 'Test Rpm Package Azure Linux' - -# TODO: rpmbuild on Red Hat UBI 8 is slow for unknown reason. Still working with Red Hat to investigate. -- job: BuildRpmPackages - displayName: Build Rpm Packages - # Do not run this job for Pull Requests due to the slowness - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: $(pool) - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Red Hat Universal Base Image 8 ${{ arch.name }}: - dockerfile: ubi - image: registry.access.redhat.com/ubi8/ubi:8.4 - artifact: rpm-ubi8-${{ arch.value }} - python_package: python3.12 - pool: ${{ arch.pool }} - Red Hat Universal Base Image 9 ${{ arch.name }}: - dockerfile: ubi - image: registry.access.redhat.com/ubi9/ubi:9.0.0 - artifact: rpm-ubi9-${{ arch.value }} - python_package: python3.12 - pool: ${{ arch.pool }} - Red Hat Universal Base Image 10 ${{ arch.name }}: - dockerfile: ubi - image: registry.access.redhat.com/ubi10/ubi:10.0 - artifact: rpm-ubi10-${{ arch.value }} - python_package: python3.12 - pool: ${{ arch.pool }} - steps: - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - task: PipAuthenticate@1 - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) - - task: Bash@3 - displayName: 'Build Rpm Package' - inputs: - targetType: 'filePath' - filePath: scripts/release/rpm/pipeline.sh - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: rpm' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(artifact) - - -- job: TestRpmPackage - displayName: Test Rpm Package - timeoutInMinutes: 180 - dependsOn: - - BuildRpmPackages - - ExtractMetadata - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: $(pool) - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Red Hat Universal Base Image 8 ${{ arch.name }}: - artifact: rpm-ubi8-${{ arch.value }} - distro: el8 - image: registry.access.redhat.com/ubi8/ubi:8.4 - python_package: python3.12 - python_cmd: python3.12 - pip_cmd: pip3.12 - pool: ${{ arch.pool }} - Red Hat Universal Base Image 9 ${{ arch.name }}: - artifact: rpm-ubi9-${{ arch.value }} - distro: el9 - image: registry.access.redhat.com/ubi9/ubi:9.0.0 - python_package: python3.12 - python_cmd: python3.12 - pip_cmd: pip3.12 - pool: ${{ arch.pool }} - Red Hat Universal Base Image 10 ${{ arch.name }}: - artifact: rpm-ubi10-${{ arch.value }} - distro: el10 - image: registry.access.redhat.com/ubi10/ubi:10.0 - python_package: python3.12 - python_cmd: python3.12 - pip_cmd: pip3.12 - pool: ${{ arch.pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' - artifactName: $(artifact) - - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - - bash: | - set -ex - - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$CLI_VERSION-1.${DISTRO}.*.rpm" -printf '%f\n') - - echo "== Test rpm package on ${IMAGE} ==" - docker pull $IMAGE - docker run --rm -e RPM_NAME=$RPM_NAME -e PYTHON_PACKAGE=${PYTHON_PACKAGE} -e PYTHON_CMD=${PYTHON_CMD} -e PIP_CMD=${PIP_CMD} -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" - - displayName: 'Test Rpm Package' - -- job: BuildDebPackages - displayName: Build Deb Packages - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: $(pool) - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - # https://wiki.ubuntu.com/Releases - Jammy ${{ arch.name }}: - # 22.04 - deb_system: ubuntu - distro: jammy - arch: ${{ arch.value }} - pool: ${{ arch.pool }} - - Noble ${{ arch.name }}: - # 24.04 - deb_system: ubuntu - distro: noble - arch: ${{ arch.value }} - pool: ${{ arch.pool }} - - # https://wiki.debian.org/DebianReleases - Bullseye ${{ arch.name }}: - # 11 - deb_system: debian - distro: bullseye - arch: ${{ arch.value }} - pool: ${{ arch.pool }} - Bookworm ${{ arch.name }}: - # 12 - deb_system: debian - distro: bookworm - arch: ${{ arch.value }} - pool: ${{ arch.pool }} - steps: - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - task: PipAuthenticate@1 - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) - - task: Bash@3 - displayName: 'Build $(deb_system) $(distro) $(arch) Package' - inputs: - targetType: 'filePath' - filePath: scripts/release/debian/pipeline.sh - env: - DISTRO_BASE_IMAGE: mcr.microsoft.com/mirror/docker/library/$(deb_system):$(distro) - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: $(deb_system) $(distro) $(arch)' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(deb_system)-$(distro)-$(arch) - -- job: TestDebPackages - timeoutInMinutes: 180 - displayName: Test Deb Packages - dependsOn: - - BuildDebPackages - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Jammy ${{ arch.name }}: - deb_system: ubuntu - distro: jammy - arch: ${{ arch.value }} - pool: ${{ arch.pool }} - Noble ${{ arch.name }}: - deb_system: ubuntu - distro: noble - arch: ${{ arch.value }} - pool: ${{ arch.pool }} - Bullseye ${{ arch.name }}: - deb_system: debian - distro: bullseye - arch: ${{ arch.value }} - pool: ${{ arch.pool }} - Bookworm ${{ arch.name }}: - deb_system: debian - distro: bookworm - arch: ${{ arch.value }} - pool: ${{ arch.pool }} - pool: - name: $(pool) - steps: - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download $(deb_system):$(distro) $(arch) Build' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/debian' - artifactName: $(deb_system)-$(distro)-$(arch) - - - task: Bash@3 - displayName: 'Test $(deb_system) $(distro) $(arch) Package' - env: - DISTRO_BASE_IMAGE: mcr.microsoft.com/mirror/docker/library/$(deb_system):$(distro) - inputs: - targetType: 'inline' - script: | - set -exv - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - - echo "== Test debian package on ${DISTRO} ==" - docker pull ${DISTRO_BASE_IMAGE} - docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" - -- job: CheckStyle - displayName: "Check CLI Style" - timeoutInMinutes: 120 - pool: - name: ${{ variables.ubuntu_multi_core_pool }} - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - azdev style - -- job: CheckHeaders - displayName: "Check License, History, and DocMap" - condition: and(not(contains(variables['Build.SourceBranch'], 'lts')), not(contains(variables['System.PullRequest.TargetBranch'], 'lts'))) - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - azdev verify license - azdev verify history - azdev verify document-map - -- job: PerformanceCheck - displayName: "PerformanceCheck" - strategy: - matrix: - Python312: - python.version: '3.12' - Python313: - python.version: '3.13' - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python $(python.version)' - inputs: - versionSpec: '$(python.version)' - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - azdev perf load-times - displayName: "Load Performance" - # - bash: | - # set -ev - # . env/bin/activate - - # azdev perf benchmark "version" "network vnet -h" "rest -h" "storage account" - # displayName: "Execution Performance" - -- job: CheckLinter - displayName: "Check CLI Linter" - - pool: - name: ${{ variables.ubuntu_multi_core_pool }} - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - python scripts/ci/service_name.py - if [[ "$(System.PullRequest.TargetBranch)" != "" ]]; then - # If CI is set to shallow fetch, target branch should be expilictly fetched. - git fetch origin --depth=1 $(System.PullRequest.TargetBranch) - azdev linter --ci-exclusions --min-severity medium --repo=./ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) - else - azdev linter --ci-exclusions --min-severity medium - fi - -- job: AzdevScan - condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) - displayName: "Secret Scan" - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.13' - inputs: - versionSpec: 3.13 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - git fetch origin --depth=1 $(System.PullRequest.TargetBranch) - declare -A secret_files - IFS_OLD=${IFS} - IFS=$'\n' - for FILE in `git diff --name-only --diff-filter=AM origin/$(System.PullRequest.TargetBranch)` ; do - echo $FILE - detected=$(azdev scan -f "$FILE" --continue-on-failure | python -c "import sys, json; print(json.load(sys.stdin)['secrets_detected'])") - if [ $detected == 'True' ]; then - printf "\033[0;31mDetected secrets from %s, You can run 'azdev mask' to remove secrets.\033[0m\n" "$FILE" - secret_files+=$FILE - fi - done - IFS=${IFS_OLD} - if [ "${#secret_files[@]}" -gt 0 ]; then - exit 1 - fi - -- job: CodegenCoverage - condition: in(variables['Build.Reason'], 'BatchedCI', 'IndividualCI') - timeoutInMinutes: 180 - displayName: "Codegen Coverage" - continueOnError: true - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' - inputs: - versionSpec: 3.11 - - template: .azure-pipelines/templates/azdev_setup.yml - - bash: | - set -ev - . env/bin/activate - # clone azure-cli-extensions - cd .. - git clone --depth 1 -b main https://github.com/Azure/azure-cli-extensions.git ./azure-cli-extensions - azdev extension repo add ./azure-cli-extensions - pip install setuptools==70.0.0 wheel==0.30.0 - azdev extension add "*" - pip install msrestazure markupsafe==2.0.1 - # Some extension will change the dependence, so run `azdev setup` again after all extensions installed. - azdev setup -c ./s -r ./azure-cli-extensions - - mkdir -p /tmp/module_stats - - find /mnt/vss/_work/1/s/src/azure-cli/azure/cli/command_modules/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n" | grep -v '^__pycache__$' > /mnt/vss/_work/1/s/scripts/ci/core_modules.txt - echo "=== Core Modules ===" - cat /mnt/vss/_work/1/s/scripts/ci/core_modules.txt - - find /mnt/vss/_work/1/azure-cli-extensions/src/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n" | grep -v '^__pycache__$' > /mnt/vss/_work/1/s/scripts/ci/extension_modules.txt - echo "=== Extension Modules ===" - cat /mnt/vss/_work/1/s/scripts/ci/extension_modules.txt - - for module in $(cat /mnt/vss/_work/1/s/scripts/ci/core_modules.txt); do - azdev statistics list-command-table $module --statistics-only > /tmp/module_stats/${module}.json || true - done - - for module in $(cat /mnt/vss/_work/1/s/scripts/ci/extension_modules.txt); do - azdev statistics list-command-table $module --statistics-only > /tmp/module_stats/${module}.json || true - done - azdev statistics list-command-table --statistics-only > /tmp/codegen_report.json || true - python /mnt/vss/_work/1/s/scripts/ci/codegen_report.py - env: - BUILD_ID: $(Build.BuildId) - BUILD_BRANCH: $(Build.SourceBranchName) - enabled: true From 5c114ad1dd0a4d5062d7fa55da065668f790254b Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Sat, 11 Oct 2025 14:44:28 +0800 Subject: [PATCH 02/52] Add argument parsing for RPM installation Added command line argument parsing to control RPM installation. --- .../release/rpm/test_azurelinux_in_docker.sh | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/scripts/release/rpm/test_azurelinux_in_docker.sh b/scripts/release/rpm/test_azurelinux_in_docker.sh index 69d5221632f..1d0f8e80708 100644 --- a/scripts/release/rpm/test_azurelinux_in_docker.sh +++ b/scripts/release/rpm/test_azurelinux_in_docker.sh @@ -3,9 +3,27 @@ # This script should be run in a Azure Linux docker container. set -exv +# Parse command line arguments +InstallRPM=true # Default value +while [[ $# -gt 0 ]]; do + case $1 in + --InstallRPM) + InstallRPM="$2" + shift 2 + ;; + *) + echo "Unknown option: $1" + exit 1 + ;; + esac +done + export USERNAME=azureuser -tdnf --nogpgcheck install /mnt/rpm/$RPM_NAME -y +# Install RPM package only if InstallRPM is true +if [[ "$InstallRPM" == "true" ]]; then + tdnf --nogpgcheck install /mnt/rpm/$RPM_NAME -y +fi tdnf install git gcc python3-devel python3-pip findutils ca-certificates -y From 3cf4742f2db9678417fdf92f7b2dec79243121f8 Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Sat, 11 Oct 2025 14:49:00 +0800 Subject: [PATCH 03/52] Add Azure pipeline for RPM and Docker image builds This YAML file defines an Azure pipeline for building and testing RPM packages and Docker images for Azure Linux. It includes jobs for extracting metadata, building RPM packages, building Docker images, and testing both RPM packages and Docker images across different architectures. --- .azure-pipelines/test-with-copa.yml | 210 ++++++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 .azure-pipelines/test-with-copa.yml diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml new file mode 100644 index 00000000000..dd06c760a37 --- /dev/null +++ b/.azure-pipelines/test-with-copa.yml @@ -0,0 +1,210 @@ +variables: +- template: ${{ variables.Pipeline.Workspace }}/.azure-pipelines/templates/variables.yml +- name: Codeql.Enabled + value: false +- name: ComponentDetection.ForceScan + value: eq(variables['Build.SourceBranch'], 'refs/heads/release') + +parameters: +- name: architectures + type: object + default: + - name: AMD64 + value: amd64 + pool: pool-ubuntu-latest-multi-core + - name: ARM64 + value: arm64 + pool: pool-ubuntu-latest-arm64 + +jobs: + +- job: ExtractMetadata + displayName: Extract Metadata + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: Bash@3 + displayName: 'Extract Version' + inputs: + targetType: 'filePath' + filePath: scripts/release/get_version.sh + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: metadata' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: metadata + +- job: BuildRpmPackagesAzureLinux + displayName: Build Rpm Package + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + image: mcr.microsoft.com/azurelinux/base/core:3.0 + artifact: rpm-azurelinux3.0-${{ arch.value }} + pool: ${{ arch.pool }} + pool: + name: $(pool) + steps: + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + + - task: PipAuthenticate@1 + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) + + - task: Bash@3 + displayName: 'Build Rpm Package: Azure Linux' + inputs: + targetType: 'filePath' + filePath: scripts/release/rpm/pipeline_azurelinux.sh + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: rpm-azurelinux' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(artifact) + +- job: BuildDockerImageAzureLinux + displayName: Build Docker Image Azure Linux + dependsOn: BuildRpmPackagesAzureLinux + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + pool: ${{ arch.pool }} + artifactName: docker-azurelinux3.0-${{ arch.value }} + dockerfile: azure-linux.dockerfile + packageArtifactName: rpm-azurelinux3.0-${{ arch.value }} + image: mcr.microsoft.com/azurelinux/base/core:3.0 + pool: + name: $(pool) + steps: + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/docker' + artifactName: $(packageArtifactName) + - bash: | + set -ex + mkdir docker-temp + mv $(Build.ArtifactStagingDirectory)/docker/*.rpm ./docker-temp/azure-cli.rpm + + bash scripts/release/docker/pipeline.sh + displayName: 'Build Docker' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + DockerImagesToScan: 'clibuild$BUILD_BUILDNUMBER:latest' + + - task: PublishPipelineArtifact@0 + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(artifactName) + +- job: TestRpmPackagesAzureLinux + displayName: Test Rpm Package + timeoutInMinutes: 180 + dependsOn: + - BuildRpmPackagesAzureLinux + - ExtractMetadata + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: $(pool) + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + image: mcr.microsoft.com/azurelinux/base/core:3.0 + artifact: rpm-azurelinux3.0-${{ arch.value }} + pool: ${{ arch.pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' + artifactName: $(artifact) + + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + + - bash: | + set -ex + + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$CLI_VERSION-1.*.rpm" -printf '%f\n') + + echo "== Test rpm package on ${IMAGE} ==" + docker pull $IMAGE + docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh" + + displayName: 'Test Rpm Package Azure Linux' + +- job: TestDockerImageAzureLinux + displayName: Test Docker Image Azure Linux + dependsOn: + - BuildDockerImageAzureLinux + - ExtractMetadata + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + pool: ${{ arch.pool }} + artifactName: docker-azurelinux3.0-${{ arch.value }} + pool: + name: $(pool) + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Docker Image' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/docker' + artifactName: $(artifactName) + + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + + - bash: | + set -exv + + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + IMAGE_NAME=clibuild$BUILD_BUILDNUMBER:latest + TAR_FILE=$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$CLI_VERSION.tar + + echo "== Test docker image ==" + + docker load < $TAR_FILE + + echo "== Run az self-test ==" + docker run $IMAGE_NAME /bin/bash -c "time az self-test && time az --version && tdnf list --installed" + + echo "== Run unit test ==" + docker run --rm -e -v $(pwd):/azure-cli $IMAGE_NAME /bin/bash "/azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false" + + displayName: 'Bash Script' From 662a5d4dc8bcd2219d775682959e3188dd9cf7fb Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Sat, 11 Oct 2025 14:51:11 +0800 Subject: [PATCH 04/52] Revert Change to default Azure CLI Pipeline --- azure-pipelines.yml | 1128 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 1064 insertions(+), 64 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 88b860bd44c..f2fcdaca15e 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,3 +1,19 @@ +resources: +- repo: self + +trigger: + batch: true + branches: + include: + - '*' + exclude: + - 'release*' + +pr: + branches: + include: + - '*' + variables: - template: ${{ variables.Pipeline.Workspace }}/.azure-pipelines/templates/variables.yml - name: Codeql.Enabled @@ -17,9 +33,80 @@ parameters: pool: pool-ubuntu-latest-arm64 jobs: +- job: CheckPullRequest + displayName: "Check the Format of Pull Request Title and Content" + condition: and(succeeded(), in(variables['System.PullRequest.TargetBranch'], 'dev', 'release', 'main')) + + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - bash: | + echo "Check Title of Pull Request: #$(System.PullRequest.PullRequestNumber)" + title=$(curl https://api.github.com/repos/$(Build.Repository.Name)/pulls/$(System.PullRequest.PullRequestNumber) | jq -r '.title') + echo $title + body=$(curl https://api.github.com/repos/$(Build.Repository.Name)/pulls/$(System.PullRequest.PullRequestNumber) | jq -r '.body') + echo $body + if [ "$(System.PullRequest.TargetBranch)" != "release" ] && echo $title | grep -iqF hotfix:; then + echo "Hotfix PR should target release branch." + exit 1 + fi + if [ "$(System.PullRequest.TargetBranch)" == "main" ]; then + echo "Feature PR should target dev branch." + exit 1 + fi + python scripts/ci/check_pull_request.py "$title" "$body" + +- job: RejectPullRequestToMasterBranch + displayName: "Reject Pull Request To Master Branch" + condition: and(succeeded(), eq(variables['System.PullRequest.TargetBranch'], 'master')) + + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - bash: | + echo "Reject pull request directly to master branch" + exit 1 + +- job: CredentialScanner + displayName: "Credential Scanner" + pool: + name: ${{ variables.windows_pool }} + steps: + - task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@3 + displayName: 'Run Credential Scanner' + inputs: + toolVersion: '2.1.17' + suppressionsFile: './scripts/ci/credscan/CredScanSuppressions.json' + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2 + displayName: 'Post Analysis' + inputs: + GdnBreakAllTools: false + GdnBreakGdnToolCredScan: true + GdnBreakGdnToolCredScanSeverity: Error + +- job: PolicyCheck + displayName: "Policy Check" + pool: + name: ${{ variables.windows_pool }} + steps: + - task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@2 + displayName: 'Run Policy Check' + inputs: + targetType: F + result: PoliCheckResult.xml + + - task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@2 + displayName: 'Post Analysis' + inputs: + GdnBreakAllTools: false + GdnBreakGdnToolPoliCheck: true + GdnBreakGdnToolPoliCheckSeverity: Error - job: ExtractMetadata displayName: Extract Metadata + + condition: succeeded() pool: name: ${{ variables.ubuntu_pool }} steps: @@ -29,27 +116,98 @@ jobs: targetType: 'filePath' filePath: scripts/release/get_version.sh + - task: PublishPipelineArtifact@0 displayName: 'Publish Artifact: metadata' inputs: TargetPath: $(Build.ArtifactStagingDirectory) ArtifactName: metadata -- job: BuildRpmPackagesAzureLinux - displayName: Build Rpm Package - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) +- job: VerifyLinuxRequirements + displayName: 'Verify src/azure-cli/requirements.*.Linux.txt' + condition: succeeded() + pool: + name: ${{ variables.ubuntu_pool }} + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + + - bash: ./scripts/ci/dependency_check.sh + displayName: 'Verify src/azure-cli/requirements.py3.Linux.txt' + +- job: VerifyDarwinRequirements + displayName: 'Verify src/azure-cli/requirements.*.Darwin.txt' + condition: succeeded() + pool: + vmImage: ${{ variables.macos_pool }} + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + + - bash: ./scripts/ci/dependency_check.sh + displayName: 'Verify src/azure-cli/requirements.py3.Darwin.txt' + +- job: VerifyWindowsRequirements + displayName: 'Verify src/azure-cli/requirements.*.Windows.txt' + condition: succeeded() + pool: + name: ${{ variables.windows_pool }} + + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + + - task: BatchScript@1 + inputs: + filename: ./scripts/ci/dependency_check.bat + displayName: 'Verify src/azure-cli/requirements.py3.Windows.txt' + +- job: VerifyVersions + displayName: Verify Command Module Versions + condition: and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/release'), eq(variables['System.PullRequest.TargetBranch'], 'release'))) + + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + azdev verify history + + displayName: 'Verify History' + +- job: BuildWindowsMSI + displayName: Build Windows MSI strategy: matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - image: mcr.microsoft.com/azurelinux/base/core:3.0 - artifact: rpm-azurelinux3.0-${{ arch.value }} - pool: ${{ arch.pool }} + x86: + Platform: x86 + x64: + Platform: x64 + + dependsOn: ExtractMetadata + condition: succeeded() pool: - name: $(pool) + name: ${{ variables.windows_pool }} steps: - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata - task: PipAuthenticate@1 condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') @@ -57,23 +215,130 @@ jobs: inputs: artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) - - task: Bash@3 - displayName: 'Build Rpm Package: Azure Linux' - inputs: - targetType: 'filePath' - filePath: scripts/release/rpm/pipeline_azurelinux.sh + - script: | + set ARCH=$(Platform) + set TARGET=msi + set /p CLI_VERSION=<$(System.ArtifactsDirectory)/metadata/version + set + + build_scripts/windows/scripts/build.cmd + displayName: 'Build Windows MSI' - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 displayName: 'SBOM' condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) + BuildDropPath: 'build_scripts/windows/out/' - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: rpm-azurelinux' + displayName: 'Publish Artifact: MSI' inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(artifact) + TargetPath: 'build_scripts/windows/out/' + ArtifactName: msi-$(Platform) + +- job: BuildWindowsZIP + displayName: Build Windows ZIP + strategy: + matrix: + x64: + Platform: x64 + + dependsOn: ExtractMetadata + condition: succeeded() + pool: + name: ${{ variables.windows_pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: PipAuthenticate@1 + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) + + - script: | + set ARCH=$(Platform) + set TARGET=zip + set /p CLI_VERSION=<$(System.ArtifactsDirectory)/metadata/version + set + + build_scripts/windows/scripts/build.cmd + displayName: 'Build Windows ZIP' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + inputs: + BuildDropPath: 'build_scripts/windows/out/' + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: ZIP' + inputs: + TargetPath: 'build_scripts/windows/out/' + ArtifactName: zip-$(Platform) + +- job: TestMsiInstallation + displayName: Test MSI Installation + strategy: + matrix: + x86: + Platform: x86 + x64: + Platform: x64 + + dependsOn: BuildWindowsMSI + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: ${{ variables.windows_pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/msi' + artifactName: msi-$(Platform) + + - task: PowerShell@2 + displayName: Install and Load CLI + inputs: + filePath: build_scripts\windows\scripts\test_msi_installation.ps1 + +- job: TestZipInstallation + displayName: Test ZIP Installation + dependsOn: BuildWindowsZIP + strategy: + matrix: + x64: + Platform: x64 + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: ${{ variables.windows_pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/zip' + artifactName: zip-$(Platform) + + - task: PowerShell@2 + displayName: Expand and Load CLI + inputs: + filePath: build_scripts\windows\scripts\test_zip_installation.ps1 - job: BuildDockerImageAzureLinux displayName: Build Docker Image Azure Linux @@ -117,50 +382,6 @@ jobs: TargetPath: $(Build.ArtifactStagingDirectory) ArtifactName: $(artifactName) -- job: TestRpmPackagesAzureLinux - displayName: Test Rpm Package - timeoutInMinutes: 180 - dependsOn: - - BuildRpmPackagesAzureLinux - - ExtractMetadata - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) - pool: - name: $(pool) - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - image: mcr.microsoft.com/azurelinux/base/core:3.0 - artifact: rpm-azurelinux3.0-${{ arch.value }} - pool: ${{ arch.pool }} - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' - artifactName: $(artifact) - - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - - bash: | - set -ex - - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$CLI_VERSION-1.*.rpm" -printf '%f\n') - - echo "== Test rpm package on ${IMAGE} ==" - docker pull $IMAGE - docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh" - - displayName: 'Test Rpm Package Azure Linux' - - job: TestDockerImageAzureLinux displayName: Test Docker Image Azure Linux dependsOn: @@ -202,3 +423,782 @@ jobs: docker load < $TAR_FILE docker run $IMAGE_NAME /bin/bash -c "time az self-test && time az --version && tdnf list --installed && sleep 5" displayName: 'Bash Script' + +- job: BuildPythonWheel + displayName: Build Python Wheels + + dependsOn: ExtractMetadata + condition: succeeded() + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + + - task: PipAuthenticate@1 + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) + + - script: | + if [[ "$(Build.Reason)" == "PullRequest" ]]; then + branch=$(System.PullRequest.TargetBranch) + else + branch=$(Build.SourceBranchName) + fi + scripts/release/pypi/build.sh $branch + displayName: 'Run Wheel Build Script' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: pypi' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: pypi + +- job: TestPythonWheel + displayName: Test Python Wheels + strategy: + matrix: + Python312: + python.version: '3.12' + Python313: + python.version: '3.13' + dependsOn: BuildPythonWheel + condition: succeeded() + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + - task: DownloadPipelineArtifact@1 + displayName: 'Download PyPI Packages' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/pypi' + artifactName: pypi + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + - bash: | + #!/usr/bin/env bash + set -ex + + CLI_VERSION=`cat $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata/version` + echo "== Testing pip install on $PYTHON_VERSION ==" + cd $BUILD_ARTIFACTSTAGINGDIRECTORY/pypi + pip install --find-links ./ azure_cli-$CLI_VERSION*whl && az self-test && az --version && sleep 5 + displayName: 'Test pip Install' + +- job: TestCore + displayName: Unit Test for Core + timeoutInMinutes: 10 + pool: + name: ${{ variables.ubuntu_pool }} + strategy: + matrix: + Python312: + python.version: '3.12' + Python313: + python.version: '3.13' + steps: + - template: .azure-pipelines/templates/automation_test.yml + parameters: + pythonVersion: '$(python.version)' + module: 'azure-cli-core' + +- job: TestTelemetry + displayName: Unit Test for Telemetry + timeoutInMinutes: 10 + pool: + name: ${{ variables.ubuntu_pool }} + strategy: + matrix: + Python312: + python.version: '3.12' + Python313: + python.version: '3.13' + steps: + - template: .azure-pipelines/templates/automation_test.yml + parameters: + pythonVersion: '$(python.version)' + module: 'azure-cli-telemetry' + +- job: IntegrationTestAgainstProfiles + displayName: Integration Test against Profiles + dependsOn: BuildPythonWheel + condition: succeeded() + timeoutInMinutes: 20 + + pool: + name: ${{ variables.ubuntu_pool }} + strategy: + matrix: + Python312: + python.version: '3.12' + Python313: + python.version: '3.13' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + - bash: pip install --upgrade pip wheel setuptools + displayName: 'Install pip and wheel' + - bash: ./scripts/ci/test_profile_integration.sh + displayName: 'Run Integration Test against Profiles' + +- job: TestExtensionsLoading + displayName: Test Extensions Loading + condition: succeeded() + timeoutInMinutes: 80 + + pool: + name: ${{ variables.ubuntu_pool }} + strategy: + matrix: + Python313: + python.version: '3.13' + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + - bash: pip install --upgrade pip wheel setuptools + displayName: 'Install pip and wheel setuptools' + - bash: ./scripts/ci/test_extensions.sh + displayName: 'Load extensions' + +- job: BuildHomebrewFormula + displayName: Build Homebrew Formula + + dependsOn: BuildPythonWheel + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - bash: | + #!/bin/bash + + root=$(cd $(dirname $0); pwd) + + set -evx + + CLI_VERSION=`cat $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata/version` + HOMEBREW_UPSTREAM_URL=`curl -Ls -o /dev/null -w %{url_effective} https://api.github.com/repos/Azure/azure-cli/tarball/$BUILD_SOURCEVERSION` + + docker_files=$(cd $BUILD_SOURCESDIRECTORY/scripts/release/homebrew/docker; pwd) + src_files=$(cd $BUILD_SOURCESDIRECTORY/src; pwd) + + echo "Generating formula in docker container ... " + docker run -v $docker_files:/mnt/scripts \ + -v $src_files:/mnt/src \ + -e CLI_VERSION=$CLI_VERSION \ + -e HOMEBREW_UPSTREAM_URL=$HOMEBREW_UPSTREAM_URL \ + --name azurecli \ + mcr.microsoft.com/azurelinux/base/python:3 \ + /mnt/scripts/run.sh + + # clean up + rm -rf $BUILD_ARTIFACTSTAGINGDIRECTORY/metadata + + docker cp azurecli:azure-cli.rb $BUILD_ARTIFACTSTAGINGDIRECTORY/azure-cli.rb + docker rm --force azurecli + displayName: 'Build homebrew formula' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: homebrew' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: homebrew + +- job: TestHomebrewFormula + displayName: Test Homebrew Formula + + dependsOn: BuildHomebrewFormula + condition: succeeded() + pool: + vmImage: ${{ variables.macos_pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Homebrew' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/homebrew' + artifactName: homebrew + + + - bash: | + set -ev + # Force relink python@3.xx in Homebrew to resolve the conflict with pre-installed python 3.xx on macOS-12 image + # See: https://github.com/Azure/azure-cli/issues/29054 + python_version=3.13 + brew unlink python@$python_version && brew link --overwrite python@$python_version + + echo == Remove pre-installed azure-cli == + brew uninstall azure-cli + + echo == Install azure-cli.rb formula == + # Need to create a dummy homebrew tap to install the formula + git config --global user.email "you@example.com" + git config --global user.name "Your Name" + brew tap-new dev/azure-cli + cp $SYSTEM_ARTIFACTSDIRECTORY/homebrew/azure-cli.rb $(brew --repository)/Library/Taps/dev/homebrew-azure-cli/Formula/ + brew install --build-from-source dev/homebrew-azure-cli/azure-cli + + echo == Az Version == + az --version + + echo == Run Self-Test == + az self-test + + displayName: 'Bash Script' + +- job: TestHomebrewPackage + displayName: Test Homebrew Package + timeoutInMinutes: 180 + dependsOn: BuildHomebrewFormula + # condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + condition: false + pool: + vmImage: ${{ variables.macos_pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/homebrew' + artifactName: homebrew + + + - bash: ./scripts/release/homebrew/test_homebrew_package.sh + + + displayName: 'Test Homebrew Package' + + +- job: BuildRpmPackagesAzureLinux + displayName: Build Rpm Package + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + image: mcr.microsoft.com/azurelinux/base/core:3.0 + artifact: rpm-azurelinux3.0-${{ arch.value }} + pool: ${{ arch.pool }} + pool: + name: $(pool) + steps: + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + + - task: PipAuthenticate@1 + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) + + - task: Bash@3 + displayName: 'Build Rpm Package: Azure Linux' + inputs: + targetType: 'filePath' + filePath: scripts/release/rpm/pipeline_azurelinux.sh + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: rpm-azurelinux' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(artifact) + +- job: TestRpmPackagesAzureLinux + displayName: Test Rpm Package + timeoutInMinutes: 180 + dependsOn: + - BuildRpmPackagesAzureLinux + - ExtractMetadata + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: $(pool) + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + image: mcr.microsoft.com/azurelinux/base/core:3.0 + artifact: rpm-azurelinux3.0-${{ arch.value }} + pool: ${{ arch.pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' + artifactName: $(artifact) + + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + + - bash: | + set -ex + + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$CLI_VERSION-1.*.rpm" -printf '%f\n') + + echo "== Test rpm package on ${IMAGE} ==" + docker pull $IMAGE + docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh" + + displayName: 'Test Rpm Package Azure Linux' + +# TODO: rpmbuild on Red Hat UBI 8 is slow for unknown reason. Still working with Red Hat to investigate. +- job: BuildRpmPackages + displayName: Build Rpm Packages + # Do not run this job for Pull Requests due to the slowness + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: $(pool) + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Red Hat Universal Base Image 8 ${{ arch.name }}: + dockerfile: ubi + image: registry.access.redhat.com/ubi8/ubi:8.4 + artifact: rpm-ubi8-${{ arch.value }} + python_package: python3.12 + pool: ${{ arch.pool }} + Red Hat Universal Base Image 9 ${{ arch.name }}: + dockerfile: ubi + image: registry.access.redhat.com/ubi9/ubi:9.0.0 + artifact: rpm-ubi9-${{ arch.value }} + python_package: python3.12 + pool: ${{ arch.pool }} + Red Hat Universal Base Image 10 ${{ arch.name }}: + dockerfile: ubi + image: registry.access.redhat.com/ubi10/ubi:10.0 + artifact: rpm-ubi10-${{ arch.value }} + python_package: python3.12 + pool: ${{ arch.pool }} + steps: + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + - task: PipAuthenticate@1 + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) + - task: Bash@3 + displayName: 'Build Rpm Package' + inputs: + targetType: 'filePath' + filePath: scripts/release/rpm/pipeline.sh + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: rpm' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(artifact) + + +- job: TestRpmPackage + displayName: Test Rpm Package + timeoutInMinutes: 180 + dependsOn: + - BuildRpmPackages + - ExtractMetadata + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: $(pool) + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Red Hat Universal Base Image 8 ${{ arch.name }}: + artifact: rpm-ubi8-${{ arch.value }} + distro: el8 + image: registry.access.redhat.com/ubi8/ubi:8.4 + python_package: python3.12 + python_cmd: python3.12 + pip_cmd: pip3.12 + pool: ${{ arch.pool }} + Red Hat Universal Base Image 9 ${{ arch.name }}: + artifact: rpm-ubi9-${{ arch.value }} + distro: el9 + image: registry.access.redhat.com/ubi9/ubi:9.0.0 + python_package: python3.12 + python_cmd: python3.12 + pip_cmd: pip3.12 + pool: ${{ arch.pool }} + Red Hat Universal Base Image 10 ${{ arch.name }}: + artifact: rpm-ubi10-${{ arch.value }} + distro: el10 + image: registry.access.redhat.com/ubi10/ubi:10.0 + python_package: python3.12 + python_cmd: python3.12 + pip_cmd: pip3.12 + pool: ${{ arch.pool }} + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' + artifactName: $(artifact) + + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + + - bash: | + set -ex + + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$CLI_VERSION-1.${DISTRO}.*.rpm" -printf '%f\n') + + echo "== Test rpm package on ${IMAGE} ==" + docker pull $IMAGE + docker run --rm -e RPM_NAME=$RPM_NAME -e PYTHON_PACKAGE=${PYTHON_PACKAGE} -e PYTHON_CMD=${PYTHON_CMD} -e PIP_CMD=${PIP_CMD} -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_rpm_in_docker.sh" + + displayName: 'Test Rpm Package' + +- job: BuildDebPackages + displayName: Build Deb Packages + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + pool: + name: $(pool) + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + # https://wiki.ubuntu.com/Releases + Jammy ${{ arch.name }}: + # 22.04 + deb_system: ubuntu + distro: jammy + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + + Noble ${{ arch.name }}: + # 24.04 + deb_system: ubuntu + distro: noble + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + + # https://wiki.debian.org/DebianReleases + Bullseye ${{ arch.name }}: + # 11 + deb_system: debian + distro: bullseye + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + Bookworm ${{ arch.name }}: + # 12 + deb_system: debian + distro: bookworm + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + steps: + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + - task: PipAuthenticate@1 + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + displayName: 'Pip Authenticate' + inputs: + artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) + - task: Bash@3 + displayName: 'Build $(deb_system) $(distro) $(arch) Package' + inputs: + targetType: 'filePath' + filePath: scripts/release/debian/pipeline.sh + env: + DISTRO_BASE_IMAGE: mcr.microsoft.com/mirror/docker/library/$(deb_system):$(distro) + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + + - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: $(deb_system) $(distro) $(arch)' + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(deb_system)-$(distro)-$(arch) + +- job: TestDebPackages + timeoutInMinutes: 180 + displayName: Test Deb Packages + dependsOn: + - BuildDebPackages + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Jammy ${{ arch.name }}: + deb_system: ubuntu + distro: jammy + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + Noble ${{ arch.name }}: + deb_system: ubuntu + distro: noble + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + Bullseye ${{ arch.name }}: + deb_system: debian + distro: bullseye + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + Bookworm ${{ arch.name }}: + deb_system: debian + distro: bookworm + arch: ${{ arch.value }} + pool: ${{ arch.pool }} + pool: + name: $(pool) + steps: + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download $(deb_system):$(distro) $(arch) Build' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/debian' + artifactName: $(deb_system)-$(distro)-$(arch) + + - task: Bash@3 + displayName: 'Test $(deb_system) $(distro) $(arch) Package' + env: + DISTRO_BASE_IMAGE: mcr.microsoft.com/mirror/docker/library/$(deb_system):$(distro) + inputs: + targetType: 'inline' + script: | + set -exv + CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` + + echo "== Test debian package on ${DISTRO} ==" + docker pull ${DISTRO_BASE_IMAGE} + docker run --rm -e DISTRO=${DISTRO} -e CLI_VERSION=$CLI_VERSION -v $SYSTEM_ARTIFACTSDIRECTORY/debian:/mnt/artifacts -v $(pwd):/azure-cli ${DISTRO_BASE_IMAGE} /bin/bash "/azure-cli/scripts/release/debian/test_deb_in_docker.sh" + +- job: CheckStyle + displayName: "Check CLI Style" + timeoutInMinutes: 120 + pool: + name: ${{ variables.ubuntu_multi_core_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + azdev style + +- job: CheckHeaders + displayName: "Check License, History, and DocMap" + condition: and(not(contains(variables['Build.SourceBranch'], 'lts')), not(contains(variables['System.PullRequest.TargetBranch'], 'lts'))) + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + azdev verify license + azdev verify history + azdev verify document-map + +- job: PerformanceCheck + displayName: "PerformanceCheck" + strategy: + matrix: + Python312: + python.version: '3.12' + Python313: + python.version: '3.13' + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python $(python.version)' + inputs: + versionSpec: '$(python.version)' + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + azdev perf load-times + displayName: "Load Performance" + # - bash: | + # set -ev + # . env/bin/activate + + # azdev perf benchmark "version" "network vnet -h" "rest -h" "storage account" + # displayName: "Execution Performance" + +- job: CheckLinter + displayName: "Check CLI Linter" + + pool: + name: ${{ variables.ubuntu_multi_core_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + python scripts/ci/service_name.py + if [[ "$(System.PullRequest.TargetBranch)" != "" ]]; then + # If CI is set to shallow fetch, target branch should be expilictly fetched. + git fetch origin --depth=1 $(System.PullRequest.TargetBranch) + azdev linter --ci-exclusions --min-severity medium --repo=./ --src=HEAD --tgt=origin/$(System.PullRequest.TargetBranch) + else + azdev linter --ci-exclusions --min-severity medium + fi + +- job: AzdevScan + condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) + displayName: "Secret Scan" + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.13' + inputs: + versionSpec: 3.13 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + git fetch origin --depth=1 $(System.PullRequest.TargetBranch) + declare -A secret_files + IFS_OLD=${IFS} + IFS=$'\n' + for FILE in `git diff --name-only --diff-filter=AM origin/$(System.PullRequest.TargetBranch)` ; do + echo $FILE + detected=$(azdev scan -f "$FILE" --continue-on-failure | python -c "import sys, json; print(json.load(sys.stdin)['secrets_detected'])") + if [ $detected == 'True' ]; then + printf "\033[0;31mDetected secrets from %s, You can run 'azdev mask' to remove secrets.\033[0m\n" "$FILE" + secret_files+=$FILE + fi + done + IFS=${IFS_OLD} + if [ "${#secret_files[@]}" -gt 0 ]; then + exit 1 + fi + +- job: CodegenCoverage + condition: in(variables['Build.Reason'], 'BatchedCI', 'IndividualCI') + timeoutInMinutes: 180 + displayName: "Codegen Coverage" + continueOnError: true + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - task: UsePythonVersion@0 + displayName: 'Use Python 3.11' + inputs: + versionSpec: 3.11 + - template: .azure-pipelines/templates/azdev_setup.yml + - bash: | + set -ev + . env/bin/activate + # clone azure-cli-extensions + cd .. + git clone --depth 1 -b main https://github.com/Azure/azure-cli-extensions.git ./azure-cli-extensions + azdev extension repo add ./azure-cli-extensions + pip install setuptools==70.0.0 wheel==0.30.0 + azdev extension add "*" + pip install msrestazure markupsafe==2.0.1 + # Some extension will change the dependence, so run `azdev setup` again after all extensions installed. + azdev setup -c ./s -r ./azure-cli-extensions + + mkdir -p /tmp/module_stats + + find /mnt/vss/_work/1/s/src/azure-cli/azure/cli/command_modules/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n" | grep -v '^__pycache__$' > /mnt/vss/_work/1/s/scripts/ci/core_modules.txt + echo "=== Core Modules ===" + cat /mnt/vss/_work/1/s/scripts/ci/core_modules.txt + + find /mnt/vss/_work/1/azure-cli-extensions/src/ -mindepth 1 -maxdepth 1 -type d -printf "%f\n" | grep -v '^__pycache__$' > /mnt/vss/_work/1/s/scripts/ci/extension_modules.txt + echo "=== Extension Modules ===" + cat /mnt/vss/_work/1/s/scripts/ci/extension_modules.txt + + for module in $(cat /mnt/vss/_work/1/s/scripts/ci/core_modules.txt); do + azdev statistics list-command-table $module --statistics-only > /tmp/module_stats/${module}.json || true + done + + for module in $(cat /mnt/vss/_work/1/s/scripts/ci/extension_modules.txt); do + azdev statistics list-command-table $module --statistics-only > /tmp/module_stats/${module}.json || true + done + azdev statistics list-command-table --statistics-only > /tmp/codegen_report.json || true + python /mnt/vss/_work/1/s/scripts/ci/codegen_report.py + env: + BUILD_ID: $(Build.BuildId) + BUILD_BRANCH: $(Build.SourceBranchName) + enabled: true From 45814984105c35d1b5c9602df045fd9fa8bd49d2 Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Sat, 11 Oct 2025 16:03:41 +0800 Subject: [PATCH 05/52] Update docker run commands to use -c option --- .azure-pipelines/test-with-copa.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index dd06c760a37..f929be0198e 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -157,7 +157,7 @@ jobs: echo "== Test rpm package on ${IMAGE} ==" docker pull $IMAGE - docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh" + docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash -c "/azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh" displayName: 'Test Rpm Package Azure Linux' @@ -205,6 +205,6 @@ jobs: docker run $IMAGE_NAME /bin/bash -c "time az self-test && time az --version && tdnf list --installed" echo "== Run unit test ==" - docker run --rm -e -v $(pwd):/azure-cli $IMAGE_NAME /bin/bash "/azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false" + docker run --rm -e -v $(pwd):/azure-cli $IMAGE_NAME /bin/bash -c "/azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false" displayName: 'Bash Script' From e7654a2cf757fae5b74af36928faa1cb341a85ee Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Sat, 11 Oct 2025 16:30:18 +0800 Subject: [PATCH 06/52] Fix script invocation in Azure Pipelines YAML --- .azure-pipelines/test-with-copa.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index f929be0198e..cfe329426a0 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -157,7 +157,7 @@ jobs: echo "== Test rpm package on ${IMAGE} ==" docker pull $IMAGE - docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash -c "/azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh" + docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh displayName: 'Test Rpm Package Azure Linux' @@ -205,6 +205,6 @@ jobs: docker run $IMAGE_NAME /bin/bash -c "time az self-test && time az --version && tdnf list --installed" echo "== Run unit test ==" - docker run --rm -e -v $(pwd):/azure-cli $IMAGE_NAME /bin/bash -c "/azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false" + docker run --rm -e -v $(pwd):/azure-cli $IMAGE_NAME /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false displayName: 'Bash Script' From 152bfe011cbeae3932ac0f570ac02287ca410e50 Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Sat, 11 Oct 2025 16:49:52 +0800 Subject: [PATCH 07/52] Fix docker run command by removing unnecessary option --- .azure-pipelines/test-with-copa.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index cfe329426a0..cb60db0cb27 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -205,6 +205,6 @@ jobs: docker run $IMAGE_NAME /bin/bash -c "time az self-test && time az --version && tdnf list --installed" echo "== Run unit test ==" - docker run --rm -e -v $(pwd):/azure-cli $IMAGE_NAME /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false + docker run --rm -v $(pwd):/azure-cli $IMAGE_NAME /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false displayName: 'Bash Script' From b71e01ba1a20ec358cc2061c41cd981e55533e9c Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Mon, 13 Oct 2025 10:17:10 +0800 Subject: [PATCH 08/52] Set timeout for TestDockerImageAzureLinux job --- .azure-pipelines/test-with-copa.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index cb60db0cb27..29c544c7988 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -163,6 +163,7 @@ jobs: - job: TestDockerImageAzureLinux displayName: Test Docker Image Azure Linux + timeoutInMinutes: 180 dependsOn: - BuildDockerImageAzureLinux - ExtractMetadata From 28fb3b395a21edb8d2b74e922c9788e70199cc47 Mon Sep 17 00:00:00 2001 From: YanaXu Date: Mon, 13 Oct 2025 10:31:05 +0800 Subject: [PATCH 09/52] add a new job to use copa --- .azure-pipelines/test-with-copa.yml | 45 ++++++++++++++++++++++++++++- 1 file changed, 44 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 29c544c7988..21e84692cf4 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -15,6 +15,10 @@ parameters: - name: ARM64 value: arm64 pool: pool-ubuntu-latest-arm64 + - name: ShouldTest + displayName: Test? + type: boolean + default: false jobs: @@ -123,7 +127,7 @@ jobs: dependsOn: - BuildRpmPackagesAzureLinux - ExtractMetadata - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule'), eq(${{parameters.ShouldTest}}, true)) pool: name: $(pool) strategy: @@ -163,6 +167,7 @@ jobs: - job: TestDockerImageAzureLinux displayName: Test Docker Image Azure Linux + condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) timeoutInMinutes: 180 dependsOn: - BuildDockerImageAzureLinux @@ -209,3 +214,41 @@ jobs: docker run --rm -v $(pwd):/azure-cli $IMAGE_NAME /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false displayName: 'Bash Script' + +- job: BuildCopaPatchedDockerImageAzureLinux + displayName: Build Copa Patched Docker Image Azure Linux + timeoutInMinutes: 180 + dependsOn: + - BuildDockerImageAzureLinux + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + pool: ${{ arch.pool }} + artifactName: docker-azurelinux3.0-${{ arch.value }} + pool: + name: $(pool) + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Docker Image' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/docker' + artifactName: $(artifactName) + + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + + - bash: | + set -exv + + IMAGE_NAME=clibuild$BUILD_BUILDNUMBER:latest + TAR_FILE=$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$CLI_VERSION.tar + + echo "== Test docker image ==" + + docker load < $TAR_FILE + echo "== docker image ==" + docker images + echo "== docker container ==" + docker ps + displayName: 'Bash Script' \ No newline at end of file From 7c9639f38033f49e326d0f31cd6ce2f0d1bb9a41 Mon Sep 17 00:00:00 2001 From: YanaXu Date: Mon, 13 Oct 2025 10:31:51 +0800 Subject: [PATCH 10/52] fix paramter format --- .azure-pipelines/test-with-copa.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 21e84692cf4..ddd807792dc 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -15,10 +15,10 @@ parameters: - name: ARM64 value: arm64 pool: pool-ubuntu-latest-arm64 - - name: ShouldTest - displayName: Test? - type: boolean - default: false +- name: ShouldTest + displayName: Test? + type: boolean + default: false jobs: From e7d2649b0fe3ec736391c14084073a799fd1368c Mon Sep 17 00:00:00 2001 From: YanaXu Date: Mon, 13 Oct 2025 15:07:08 +0800 Subject: [PATCH 11/52] change arm64 agent pool --- .azure-pipelines/test-with-copa.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index ddd807792dc..1007e642e48 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -14,7 +14,7 @@ parameters: pool: pool-ubuntu-latest-multi-core - name: ARM64 value: arm64 - pool: pool-ubuntu-latest-arm64 + pool: pool-ubuntu-arm64-temp - name: ShouldTest displayName: Test? type: boolean @@ -238,6 +238,14 @@ jobs: - bash: ./scripts/ci/install_docker.sh displayName: Install Docker + - task: Bash@3 + displayName: Install make and go for ARM64 Agent + condition: eq(variables['artifactName'], 'arm64') + inputs: + targetType: 'inline' + script: | + sudo apt update && sudo apt install make && sudo apt install -y golang-go + - bash: | set -exv From 00106a4d7c3b7d11c67dd4408a754df279fa649f Mon Sep 17 00:00:00 2001 From: YanaXu Date: Mon, 13 Oct 2025 16:56:33 +0800 Subject: [PATCH 12/52] print go version --- .azure-pipelines/test-with-copa.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 1007e642e48..66f5ec00b8b 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -244,6 +244,9 @@ jobs: inputs: targetType: 'inline' script: | + echo "-------------------------------------1" + sudo apt list -a golang-go + echo "-------------------------------------2" sudo apt update && sudo apt install make && sudo apt install -y golang-go - bash: | From 7644dd6eee1a6cc7e155244f1b0fdc9243de6f4a Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Wed, 15 Oct 2025 17:53:47 +0800 Subject: [PATCH 13/52] install copa --- .azure-pipelines/test-with-copa.yml | 90 ++++++++++++++++++++++------- 1 file changed, 69 insertions(+), 21 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 66f5ec00b8b..f5376926d0c 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -14,7 +14,7 @@ parameters: pool: pool-ubuntu-latest-multi-core - name: ARM64 value: arm64 - pool: pool-ubuntu-arm64-temp + pool: pool-ubuntu-latest-arm64 - name: ShouldTest displayName: Test? type: boolean @@ -239,27 +239,75 @@ jobs: displayName: Install Docker - task: Bash@3 - displayName: Install make and go for ARM64 Agent - condition: eq(variables['artifactName'], 'arm64') + displayName: Install PowerShell for ARM64 Agent + condition: and(succeeded(), eq(${{parameters.architectures.value}}, 'arm64')) inputs: targetType: 'inline' script: | - echo "-------------------------------------1" - sudo apt list -a golang-go - echo "-------------------------------------2" - sudo apt update && sudo apt install make && sudo apt install -y golang-go + curl -L -o /tmp/powershell.tar.gz https://github.com/PowerShell/PowerShell/releases/download/v7.4.11/powershell-7.4.11-linux-arm64.tar.gz + sudo mkdir -p /opt/microsoft/powershell/7 + sudo tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell/7 + sudo chmod +x /opt/microsoft/powershell/7/pwsh + sudo ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh + PATH=/usr/bin:/sbin:/usr/sbin:$PATH - - bash: | - set -exv - - IMAGE_NAME=clibuild$BUILD_BUILDNUMBER:latest - TAR_FILE=$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$CLI_VERSION.tar - - echo "== Test docker image ==" - - docker load < $TAR_FILE - echo "== docker image ==" - docker images - echo "== docker container ==" - docker ps - displayName: 'Bash Script' \ No newline at end of file + - task: Bash@3 + displayName: Install Az.Accounts for ARM64 Agents + condition: and(succeeded(), eq(${{parameters.architectures.value}}, 'arm64')) + inputs: + targetType: 'inline' + script: | + pwsh -command 'Register-PSRepository -Default -InstallationPolicy Trusted' + pwsh -command 'Install-Module -Name Az.Accounts -Repository PSGallery -AllowClobber -Force' + + - task: Bash@3 + displayName: Install Trivy + inputs: + targetType: 'inline' + script: | + set -eux + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin + echo "----------------------------------" + sudo trivy --version + + - task: Bash@3 + displayName: Install copacetic + inputs: + targetType: 'inline' + script: | + CopaDownloadUrl="https://github.com/project-copacetic/copacetic/releases/download/v$(CopaVersion)/copa_$(CopaVersion)_linux_amd64.tar.gz" + if [ "$(CPUArchitecture)" = "arm64" ]; then + CopaDownloadUrl="https://github.com/project-copacetic/copacetic/releases/download/v$(CopaVersion)/copa_$(CopaVersion)_linux_arm64.tar.gz" + fi + curl -L -o /tmp/copa.tar.gz $CopaDownloadUrl + sudo mkdir -p /opt/copacetic + sudo tar zxf /tmp/copa.tar.gz -C /opt/copacetic + sudo chmod +x /opt/copacetic/copa + sudo ln -s /opt/copacetic/copa /usr/bin/copa + PATH=/usr/bin:/sbin:/usr/sbin:$PATH + echo "----------------------------------" + sudo copa --version + echo "----------------------------------" + sudo copa --help + echo "----------------------------------" + sudo copa patch --help + echo "----------------------------------" + + - task: Bash@3 + displayName: Check Docker Image + inputs: + targetType: 'inline' + script: | + set -exv + + IMAGE_NAME=clibuild$BUILD_BUILDNUMBER:latest + TAR_FILE=$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$CLI_VERSION.tar + + echo "== Test docker image ==" + + docker load < $TAR_FILE + echo "== docker image ==" + docker images + echo "== docker container ==" + docker ps + \ No newline at end of file From 00a029dbde748301bcbcf1e5c5643d915f8824e1 Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Wed, 15 Oct 2025 17:59:38 +0800 Subject: [PATCH 14/52] fix variable --- .azure-pipelines/test-with-copa.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index f5376926d0c..0e6e07a80ec 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -226,6 +226,7 @@ jobs: Azure Linux 3.0 ${{ arch.name }}: pool: ${{ arch.pool }} artifactName: docker-azurelinux3.0-${{ arch.value }} + architecture: ${{ arch.value }} pool: name: $(pool) steps: @@ -240,7 +241,7 @@ jobs: - task: Bash@3 displayName: Install PowerShell for ARM64 Agent - condition: and(succeeded(), eq(${{parameters.architectures.value}}, 'arm64')) + condition: and(succeeded(), eq(variables['architecture'], 'arm64')) inputs: targetType: 'inline' script: | @@ -253,7 +254,7 @@ jobs: - task: Bash@3 displayName: Install Az.Accounts for ARM64 Agents - condition: and(succeeded(), eq(${{parameters.architectures.value}}, 'arm64')) + condition: and(succeeded(), eq(variables['architecture'], 'arm64')) inputs: targetType: 'inline' script: | @@ -310,4 +311,3 @@ jobs: docker images echo "== docker container ==" docker ps - \ No newline at end of file From e51480a34ce548035284956db310f0c43a224158 Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Fri, 17 Oct 2025 10:31:37 +0800 Subject: [PATCH 15/52] update get az cli version logic --- .azure-pipelines/test-with-copa.yml | 42 +++++++++++++++-------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 0e6e07a80ec..86ec4972e6d 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -22,26 +22,24 @@ parameters: jobs: -- job: ExtractMetadata - displayName: Extract Metadata +- job: GetAzCliVersion + displayName: Get Azure CLI Version pool: name: ${{ variables.ubuntu_pool }} steps: - task: Bash@3 - displayName: 'Extract Version' + name: GetAzCliVersion + displayName: Get CLI Version inputs: - targetType: 'filePath' - filePath: scripts/release/get_version.sh - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: metadata' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: metadata + targetType: 'inline' + script: | + ver=`cat src/azure-cli/azure/cli/__main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` + echo "Extracted Azure CLI version: $ver" + echo "##vso[task.setvariable variable=AzCliVersion]$ver" + echo "##vso[task.setvariable variable=AzCliVersion;isOutput=true]$ver" - job: BuildRpmPackagesAzureLinux displayName: Build Rpm Package - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) strategy: matrix: ${{ each arch in parameters.architectures }}: @@ -126,7 +124,7 @@ jobs: timeoutInMinutes: 180 dependsOn: - BuildRpmPackagesAzureLinux - - ExtractMetadata + - GetAzCliVersion condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule'), eq(${{parameters.ShouldTest}}, true)) pool: name: $(pool) @@ -137,6 +135,8 @@ jobs: image: mcr.microsoft.com/azurelinux/base/core:3.0 artifact: rpm-azurelinux3.0-${{ arch.value }} pool: ${{ arch.pool }} + variables: + AzCliVersion: $[ dependencies.GetAzCliVersion.outputs['GetAzCliVersion.AzCliVersion'] ] steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Metadata' @@ -155,9 +155,7 @@ jobs: - bash: | set -ex - - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$CLI_VERSION-1.*.rpm" -printf '%f\n') + RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$(AzCliVersion)-1.*.rpm" -printf '%f\n') echo "== Test rpm package on ${IMAGE} ==" docker pull $IMAGE @@ -171,7 +169,7 @@ jobs: timeoutInMinutes: 180 dependsOn: - BuildDockerImageAzureLinux - - ExtractMetadata + - GetAzCliVersion strategy: matrix: ${{ each arch in parameters.architectures }}: @@ -180,6 +178,8 @@ jobs: artifactName: docker-azurelinux3.0-${{ arch.value }} pool: name: $(pool) + variables: + AzCliVersion: $[ dependencies.GetAzCliVersion.outputs['GetAzCliVersion.AzCliVersion'] ] steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Metadata' @@ -199,9 +199,8 @@ jobs: - bash: | set -exv - CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` IMAGE_NAME=clibuild$BUILD_BUILDNUMBER:latest - TAR_FILE=$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$CLI_VERSION.tar + TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$(AzCliVersion).tar" echo "== Test docker image ==" @@ -220,6 +219,7 @@ jobs: timeoutInMinutes: 180 dependsOn: - BuildDockerImageAzureLinux + - GetAzCliVersion strategy: matrix: ${{ each arch in parameters.architectures }}: @@ -229,6 +229,8 @@ jobs: architecture: ${{ arch.value }} pool: name: $(pool) + variables: + AzCliVersion: $[ dependencies.GetAzCliVersion.outputs['GetAzCliVersion.AzCliVersion'] ] steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Docker Image' @@ -302,7 +304,7 @@ jobs: set -exv IMAGE_NAME=clibuild$BUILD_BUILDNUMBER:latest - TAR_FILE=$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$CLI_VERSION.tar + TAR_FILE=$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$(AzCliVersion).tar echo "== Test docker image ==" From 4b85a97ab442d0f474e2c8aa57edc1217f5d9acc Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Fri, 17 Oct 2025 10:41:55 +0800 Subject: [PATCH 16/52] print all env --- .azure-pipelines/test-with-copa.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 86ec4972e6d..e6c93f2837e 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -22,6 +22,20 @@ parameters: jobs: +- job: Check + displayName: Get Azure CLI Version + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - checkout: none + - task: Bash@3 + name: Check + displayName: Check + inputs: + targetType: 'inline' + script: | + env | sort + - job: GetAzCliVersion displayName: Get Azure CLI Version pool: From 819fc644614ac86b606749ca85d1b5d2ec9ae568 Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Fri, 17 Oct 2025 10:50:59 +0800 Subject: [PATCH 17/52] try not checkout the whole repo --- .azure-pipelines/test-with-copa.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index e6c93f2837e..31527cb5b27 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -23,7 +23,7 @@ parameters: jobs: - job: Check - displayName: Get Azure CLI Version + displayName: Check pool: name: ${{ variables.ubuntu_pool }} steps: @@ -35,7 +35,12 @@ jobs: targetType: 'inline' script: | env | sort - + curl -o __main__.py $BUILD_REPOSITORY_URI/blob/$BUILD_SOURCEBRANCHNAME/src/azure-cli/azure/cli/__main__.py + ver=`cat __main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` + echo "Extracted Azure CLI version: $ver" + echo "##vso[task.setvariable variable=AzCliVersion]$ver" + echo "##vso[task.setvariable variable=AzCliVersion;isOutput=true]$ver" + - job: GetAzCliVersion displayName: Get Azure CLI Version pool: From 001741b72a412677d8b20b5f23b0e0c522821c5b Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Fri, 17 Oct 2025 11:07:59 +0800 Subject: [PATCH 18/52] add debug log --- .azure-pipelines/test-with-copa.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 31527cb5b27..2fbb1977ae5 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -34,13 +34,19 @@ jobs: inputs: targetType: 'inline' script: | - env | sort curl -o __main__.py $BUILD_REPOSITORY_URI/blob/$BUILD_SOURCEBRANCHNAME/src/azure-cli/azure/cli/__main__.py + echo "----------------------------------1" + cat __main__.py + echo "----------------------------------2" + ver=`cat __main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` echo "Extracted Azure CLI version: $ver" echo "##vso[task.setvariable variable=AzCliVersion]$ver" echo "##vso[task.setvariable variable=AzCliVersion;isOutput=true]$ver" - + echo "----------------------------------3" + + env | sort + - job: GetAzCliVersion displayName: Get Azure CLI Version pool: From 785c542ef0db0c5e69a8fcafe824ac32676d0e4b Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Fri, 17 Oct 2025 11:15:38 +0800 Subject: [PATCH 19/52] update url --- .azure-pipelines/test-with-copa.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 2fbb1977ae5..3c5fd814789 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -34,7 +34,7 @@ jobs: inputs: targetType: 'inline' script: | - curl -o __main__.py $BUILD_REPOSITORY_URI/blob/$BUILD_SOURCEBRANCHNAME/src/azure-cli/azure/cli/__main__.py + curl -o __main__.py https://raw.githubusercontent.com/$BUILD_REPOSITORY_NAME/$BUILD_SOURCEBRANCH/src/azure-cli/azure/cli/__main__.py echo "----------------------------------1" cat __main__.py echo "----------------------------------2" From 281efc9689dedebdeeca71a04ff4e766fcb0f26b Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Fri, 17 Oct 2025 11:26:16 +0800 Subject: [PATCH 20/52] replace get az version job to save time --- .azure-pipelines/test-with-copa.yml | 32 +++++------------------------ 1 file changed, 5 insertions(+), 27 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 3c5fd814789..de0eb458d25 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -22,43 +22,21 @@ parameters: jobs: -- job: Check - displayName: Check - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - checkout: none - - task: Bash@3 - name: Check - displayName: Check - inputs: - targetType: 'inline' - script: | - curl -o __main__.py https://raw.githubusercontent.com/$BUILD_REPOSITORY_NAME/$BUILD_SOURCEBRANCH/src/azure-cli/azure/cli/__main__.py - echo "----------------------------------1" - cat __main__.py - echo "----------------------------------2" - - ver=`cat __main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` - echo "Extracted Azure CLI version: $ver" - echo "##vso[task.setvariable variable=AzCliVersion]$ver" - echo "##vso[task.setvariable variable=AzCliVersion;isOutput=true]$ver" - echo "----------------------------------3" - - env | sort - - job: GetAzCliVersion displayName: Get Azure CLI Version pool: name: ${{ variables.ubuntu_pool }} steps: + - checkout: none - task: Bash@3 name: GetAzCliVersion - displayName: Get CLI Version + displayName: Get Azure CLI Version inputs: targetType: 'inline' script: | - ver=`cat src/azure-cli/azure/cli/__main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` + curl -o __main__.py https://raw.githubusercontent.com/$BUILD_REPOSITORY_NAME/$BUILD_SOURCEBRANCH/src/azure-cli/azure/cli/__main__.py + ver=`cat __main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` + echo "Extracted Azure CLI version: $ver" echo "##vso[task.setvariable variable=AzCliVersion]$ver" echo "##vso[task.setvariable variable=AzCliVersion;isOutput=true]$ver" From 761b00bf1d83a7d97e2fb9f713defe05faecf725 Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Fri, 17 Oct 2025 14:41:38 +0800 Subject: [PATCH 21/52] refactor variables --- .azure-pipelines/test-with-copa.yml | 102 +++++++++++++++++----------- 1 file changed, 63 insertions(+), 39 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index de0eb458d25..828d7d927ec 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -4,6 +4,8 @@ variables: value: false - name: ComponentDetection.ForceScan value: eq(variables['Build.SourceBranch'], 'refs/heads/release') +- name: CopaVersion + value: 0.12.0-rc.1 parameters: - name: architectures @@ -22,24 +24,31 @@ parameters: jobs: -- job: GetAzCliVersion - displayName: Get Azure CLI Version +- job: PrepareAzCliVersion + displayName: Prepare Azure CLI Version pool: name: ${{ variables.ubuntu_pool }} steps: - checkout: none - task: Bash@3 - name: GetAzCliVersion - displayName: Get Azure CLI Version + name: PrepareAzCliVersion + displayName: Prepare Azure CLI Version inputs: targetType: 'inline' script: | curl -o __main__.py https://raw.githubusercontent.com/$BUILD_REPOSITORY_NAME/$BUILD_SOURCEBRANCH/src/azure-cli/azure/cli/__main__.py - ver=`cat __main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` + AzCLI_VERSION=`cat __main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` + + AzCLI_IMAGE_NAME="clibuild_$BUILD_BUILDNUMBER" + AzCLI_IMAGE_TAG="latest" - echo "Extracted Azure CLI version: $ver" - echo "##vso[task.setvariable variable=AzCliVersion]$ver" - echo "##vso[task.setvariable variable=AzCliVersion;isOutput=true]$ver" + echo "Azure CLI version: $AzCLI_VERSION" + echo "Azure CLI Image Name: $AzCLI_IMAGE_NAME" + echo "Azure CLI Image Tag: $AzCLI_IMAGE_TAG" + + echo "##vso[task.setvariable variable=AzCliVersion;isOutput=true]$AzCLI_VERSION" + echo "##vso[task.setvariable variable=AzCliImageName;isOutput=true]$AzCLI_IMAGE_NAME" + echo "##vso[task.setvariable variable=AzCliImageTag;isOutput=true]$AzCLI_IMAGE_TAG" - job: BuildRpmPackagesAzureLinux displayName: Build Rpm Package @@ -82,7 +91,9 @@ jobs: - job: BuildDockerImageAzureLinux displayName: Build Docker Image Azure Linux - dependsOn: BuildRpmPackagesAzureLinux + dependsOn: + - PrepareAzCliVersion + - BuildRpmPackagesAzureLinux strategy: matrix: ${{ each arch in parameters.architectures }}: @@ -94,6 +105,10 @@ jobs: image: mcr.microsoft.com/azurelinux/base/core:3.0 pool: name: $(pool) + variables: + AzCliVersion: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] + AzCliImageName: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] + AzCliImageTag: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] steps: - bash: ./scripts/ci/install_docker.sh displayName: Install Docker @@ -107,7 +122,14 @@ jobs: mkdir docker-temp mv $(Build.ArtifactStagingDirectory)/docker/*.rpm ./docker-temp/azure-cli.rpm - bash scripts/release/docker/pipeline.sh + docker build --no-cache \ + --build-arg IMAGE=$IMAGE \ + --tag $(AzCliImageName):$(AzCliImageTag) \ + --file $DOCKERFILE \ + $BUILD_SOURCESDIRECTORY + + docker save -o "$BUILD_STAGINGDIRECTORY/docker-azure-cli-$(AzCliVersion).tar" $(AzCliImageName):$(AzCliImageTag) + displayName: 'Build Docker' - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 @@ -115,7 +137,7 @@ jobs: condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') inputs: BuildDropPath: $(Build.ArtifactStagingDirectory) - DockerImagesToScan: 'clibuild$BUILD_BUILDNUMBER:latest' + DockerImagesToScan: $(AzCliDockerImageTag) - task: PublishPipelineArtifact@0 inputs: @@ -127,7 +149,7 @@ jobs: timeoutInMinutes: 180 dependsOn: - BuildRpmPackagesAzureLinux - - GetAzCliVersion + - PrepareAzCliVersion condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule'), eq(${{parameters.ShouldTest}}, true)) pool: name: $(pool) @@ -139,7 +161,7 @@ jobs: artifact: rpm-azurelinux3.0-${{ arch.value }} pool: ${{ arch.pool }} variables: - AzCliVersion: $[ dependencies.GetAzCliVersion.outputs['GetAzCliVersion.AzCliVersion'] ] + AzCliVersion: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Metadata' @@ -171,8 +193,8 @@ jobs: condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) timeoutInMinutes: 180 dependsOn: + - PrepareAzCliVersion - BuildDockerImageAzureLinux - - GetAzCliVersion strategy: matrix: ${{ each arch in parameters.architectures }}: @@ -182,7 +204,9 @@ jobs: pool: name: $(pool) variables: - AzCliVersion: $[ dependencies.GetAzCliVersion.outputs['GetAzCliVersion.AzCliVersion'] ] + AzCliVersion: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] + AzCliImageName: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] + AzCliImageTag: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Metadata' @@ -202,7 +226,6 @@ jobs: - bash: | set -exv - IMAGE_NAME=clibuild$BUILD_BUILDNUMBER:latest TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$(AzCliVersion).tar" echo "== Test docker image ==" @@ -210,19 +233,19 @@ jobs: docker load < $TAR_FILE echo "== Run az self-test ==" - docker run $IMAGE_NAME /bin/bash -c "time az self-test && time az --version && tdnf list --installed" + docker run $(AzCliImageName):$(AzCliImageTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" echo "== Run unit test ==" - docker run --rm -v $(pwd):/azure-cli $IMAGE_NAME /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false + docker run --rm -v $(pwd):/azure-cli $(AzCliImageName):$(AzCliImageTag) /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false - displayName: 'Bash Script' + displayName: 'Test Built Docker Image' - job: BuildCopaPatchedDockerImageAzureLinux displayName: Build Copa Patched Docker Image Azure Linux timeoutInMinutes: 180 dependsOn: - BuildDockerImageAzureLinux - - GetAzCliVersion + - PrepareAzCliVersion strategy: matrix: ${{ each arch in parameters.architectures }}: @@ -233,7 +256,9 @@ jobs: pool: name: $(pool) variables: - AzCliVersion: $[ dependencies.GetAzCliVersion.outputs['GetAzCliVersion.AzCliVersion'] ] + AzCliVersion: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] + AzCliImageName: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] + AzCliImageTag: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Docker Image' @@ -272,7 +297,7 @@ jobs: targetType: 'inline' script: | set -eux - curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/bin echo "----------------------------------" sudo trivy --version @@ -282,7 +307,7 @@ jobs: targetType: 'inline' script: | CopaDownloadUrl="https://github.com/project-copacetic/copacetic/releases/download/v$(CopaVersion)/copa_$(CopaVersion)_linux_amd64.tar.gz" - if [ "$(CPUArchitecture)" = "arm64" ]; then + if [ "$(architecture)" = "arm64" ]; then CopaDownloadUrl="https://github.com/project-copacetic/copacetic/releases/download/v$(CopaVersion)/copa_$(CopaVersion)_linux_arm64.tar.gz" fi curl -L -o /tmp/copa.tar.gz $CopaDownloadUrl @@ -290,29 +315,28 @@ jobs: sudo tar zxf /tmp/copa.tar.gz -C /opt/copacetic sudo chmod +x /opt/copacetic/copa sudo ln -s /opt/copacetic/copa /usr/bin/copa - PATH=/usr/bin:/sbin:/usr/sbin:$PATH - echo "----------------------------------" + sudo copa --version - echo "----------------------------------" sudo copa --help - echo "----------------------------------" sudo copa patch --help - echo "----------------------------------" - task: Bash@3 displayName: Check Docker Image inputs: targetType: 'inline' script: | - set -exv - - IMAGE_NAME=clibuild$BUILD_BUILDNUMBER:latest - TAR_FILE=$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$(AzCliVersion).tar - - echo "== Test docker image ==" + set -exv + + TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$(AzCliVersion).tar" + docker load < $TAR_FILE + + echo "== Check Docker Images ==" + docker images + + echo "== docker container ==" + docker ps + + echo "== Run az self-test ==" + docker run $(AzCliImageName):$(AzCliImageTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" + - docker load < $TAR_FILE - echo "== docker image ==" - docker images - echo "== docker container ==" - docker ps From e45bb4a607c472b3d86fab9ebd87085ed094dead Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Fri, 17 Oct 2025 15:09:43 +0800 Subject: [PATCH 22/52] add test --- .azure-pipelines/test-with-copa.yml | 134 +++++++++++- scripts/release/docker/az_cli_commands.sh | 199 ++++++++++++++++++ .../release/docker/test_az_cli_in_pipeline.sh | 135 ++++++++++++ 3 files changed, 460 insertions(+), 8 deletions(-) create mode 100644 scripts/release/docker/az_cli_commands.sh create mode 100644 scripts/release/docker/test_az_cli_in_pipeline.sh diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 828d7d927ec..e48d8c1af4f 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -21,6 +21,13 @@ parameters: displayName: Test? type: boolean default: false +- name: PatchLevel + displayName: library-patch-level? + default: "patch" + values: + - patch + - minor + - major jobs: @@ -41,14 +48,45 @@ jobs: AzCLI_IMAGE_NAME="clibuild_$BUILD_BUILDNUMBER" AzCLI_IMAGE_TAG="latest" + AzCLI_IMAGE_PATCHED_TAG="latest-patched" echo "Azure CLI version: $AzCLI_VERSION" echo "Azure CLI Image Name: $AzCLI_IMAGE_NAME" echo "Azure CLI Image Tag: $AzCLI_IMAGE_TAG" - + echo "Azure CLI Image Patched Tag: $AzCLI_IMAGE_PATCHED_TAG" + + echo "##vso[task.setvariable variable=AzCliVersion]$AzCLI_VERSION" + echo "##vso[task.setvariable variable=AzCliVersion;isOutput=true]$AzCLI_VERSION" echo "##vso[task.setvariable variable=AzCliImageName;isOutput=true]$AzCLI_IMAGE_NAME" echo "##vso[task.setvariable variable=AzCliImageTag;isOutput=true]$AzCLI_IMAGE_TAG" + echo "##vso[task.setvariable variable=AzCliImagePatchedTag;isOutput=true]$AzCLI_IMAGE_PATCHED_TAG" + + - task: PowerShell@2 + displayName: Set Build name + inputs: + targetType: inline + pwsh: true + script: | + # build name has 2 limitations + # - The maximum length of a build name is 255 characters. + # - Characters which are not allowed include '"', '/', ':', '<', '>', '\', '|', '?', '@', and '*'. + + $originalBuildName = "$(Build.BuildNumber)" + $newBuildName = "$originalBuildName - Azure CLI Docker Build $(AzCliVersion)" + + if("${{parameters.ShouldTest}}" -eq "true") { + $newBuildName = "$newBuildName with Test" + } else { + $newBuildName = "$newBuildName without Test" + } + + $notAllowedChars = @('"', '/', ':', '<', '>', '\', '|', '?', '@', '*') + $notAllowedChars | ForEach-Object {$newBuildName = $newBuildName.Replace($_, '_')} + if ( $newBuildName.Length -gt 200 ) { + $newBuildName = $newBuildName.Substring(0, 200) + } + Write-Host "##vso[build.updatebuildnumber]$newBuildName" - job: BuildRpmPackagesAzureLinux displayName: Build Rpm Package @@ -223,22 +261,35 @@ jobs: - bash: ./scripts/ci/install_docker.sh displayName: Install Docker + - task: Bash@3 + displayName: Create Test and Output Folders + inputs: + targetType: 'inline' + script: | + mkdir output + chmod +x ./scripts/release/docker/test_az_cli_in_pipeline.sh + ls ./.azure-pipelines + - bash: | set -exv - + + echo "== Load docker image ==" TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$(AzCliVersion).tar" - - echo "== Test docker image ==" - docker load < $TAR_FILE echo "== Run az self-test ==" docker run $(AzCliImageName):$(AzCliImageTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" + displayName: 'Run Simple Test' + + - bash: | + echo "== List docker image ==" + docker images + echo "== Run unit test ==" docker run --rm -v $(pwd):/azure-cli $(AzCliImageName):$(AzCliImageTag) /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false - displayName: 'Test Built Docker Image' + displayName: 'Run Unit Test' - job: BuildCopaPatchedDockerImageAzureLinux displayName: Build Copa Patched Docker Image Azure Linux @@ -259,6 +310,7 @@ jobs: AzCliVersion: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] AzCliImageName: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] AzCliImageTag: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] + AzCliImagePatchedTag: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImagePatchedTag'] ] steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Docker Image' @@ -338,5 +390,71 @@ jobs: echo "== Run az self-test ==" docker run $(AzCliImageName):$(AzCliImageTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" - - + + - task: Bash@3 + displayName: Create Output Folders + inputs: + targetType: 'inline' + script: | + pwd + mkdir output + + - task: Bash@3 + displayName: Scan with Trivy + inputs: + targetType: 'inline' + script: | + export IMAGE=$(AzCliImageName):$(AzCliImageTag) + OutputFile="trivy-scan-before.json" + sudo trivy image --pkg-types os,library --ignore-unfixed -f json -o $OutputFile $IMAGE + cp $OutputFile ./output/ + + - task: Bash@3 + displayName: Setup Docker BuildKit + inputs: + targetType: 'inline' + script: | + set -eux + # Enable Docker BuildKit + export DOCKER_BUILDKIT=1 + export BUILDKIT_PROGRESS=plain + + # Check Docker version and BuildKit support + sudo docker version + sudo docker buildx version || echo "buildx not available" + + # Try to create and use a new builder with containerd image store + sudo docker buildx create --name copa-builder --driver docker-container --use || true + sudo docker buildx inspect --bootstrap || echo "Failed to bootstrap buildx" + + # Alternative: Enable experimental features in Docker daemon + echo "Checking Docker daemon configuration..." + sudo docker info | grep -i experimental || echo "Experimental features not enabled" + + - task: Bash@3 + displayName: Patch with Copacetic + inputs: + targetType: 'inline' + script: | + set -eux + export DOCKER_BUILDKIT=1 + export BUILDKIT_PROGRESS=plain + + export COPA_EXPERIMENTAL=1 + export IMAGE=$(AzCliImageName):$(AzCliImageTag) + + echo "before copa patch-----------------------" + sudo COPA_EXPERIMENTAL=1 copa patch -i $IMAGE -r trivy-scan-before.json --pkg-types os,library --library-patch-level ${{parameters.PatchLevel}} --tag $(AzCliImagePatchedTag) + + echo "after copa patch-----------------------" + + - task: Bash@3 + displayName: Check Docker Image + inputs: + targetType: 'inline' + script: | + echo "== Check Docker Images ==" + docker images + + echo "== Run az self-test ==" + docker run $(AzCliImageName):$(AzCliImagePatchedTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" diff --git a/scripts/release/docker/az_cli_commands.sh b/scripts/release/docker/az_cli_commands.sh new file mode 100644 index 00000000000..2d039fbc3d7 --- /dev/null +++ b/scripts/release/docker/az_cli_commands.sh @@ -0,0 +1,199 @@ +# Basic Information +echo "=== Basic Information ===" +az account show +az --version +az extension list + +# Template Specs +echo "=== Template Specs ===" +az ts list + +# Accounts and Subscriptions +echo "=== Accounts and Subscriptions ===" +az account list +az account subscription list + +# Resource Groups +echo "=== Resource Groups ===" +az group list + +# Virtual Machines +echo "=== Virtual Machines ===" +az vm list +az vm image list --output json +# az vm size list --location eastus +az vmss list +az disk list +az snapshot list +az image list +az sig list +az vm availability-set list + +# Networking +echo "=== Networking ===" +az network vnet list +az network subnet list --vnet-name MyVNet --resource-group MyResourceGroup 2>/dev/null || echo 'Requires specific vnet and resource group' +az network nsg list +az network nic list +az network public-ip list +az network lb list +az network application-gateway list +az network dns zone list +az network vpn-gateway list +az network express-route list +az network route-table list +az network firewall list +az network private-endpoint list +az network private-link-service list +az network nat gateway list +az network traffic-manager profile list + +# Storage +echo "=== Storage ===" +az storage account list + +# Databases +echo "=== Databases ===" +az sql server list +az sql mi list +az mysql server list +az postgres server list +az postgres flexible-server list +az mysql flexible-server list +az mariadb server list +az cosmosdb list +az cosmosdb postgres cluster list --resource-group azure-cli-test-rg + +# Containers +echo "=== Containers ===" +az aks list +az acr list +az container list +az containerapp list +az containerapp env list +az aro list + +# App Services +echo "=== App Services ===" +az webapp list +az appservice plan list +az functionapp list + +# App Configuration +echo "=== App Configuration ===" +az appconfig list + +# Security and Identity +echo "=== Security and Identity ===" +az ad sp list --show-mine +az ad user list +az ad group list +az keyvault list +az policy assignment list +az policy definition list +az policy exemption list +az policy set-definition list +az role assignment list +az role definition list +az identity list +az security pricing list +az security contact list + +# Managed Services +echo "=== Managed Services ===" +az managedservices assignment list +az managedservices definition list + +# Monitoring +echo "=== Monitoring ===" +az monitor activity-log list --start-time 2025-09-01 --end-time 2025-09-10 2>/dev/null || echo 'Requires time range specification' +az monitor log-analytics workspace list + +# Backup +echo "=== Backup ===" +az backup vault list + +# Cognitive Services +echo "=== Cognitive Services ===" +az cognitiveservices account list + +# IoT +echo "=== IoT ===" +az iot hub list +az iot dps list +az iot central app list + +# Data Box Edge +echo "=== Data Box Edge ===" +az databoxedge device list + +# Events +echo "=== Events ===" +az eventgrid topic list +az eventgrid domain list +az eventhubs namespace list + +# Service Bus +echo "=== Service Bus ===" +az servicebus namespace list + +# Relay +echo "=== Relay ===" +az relay namespace list + +# Batch +echo "=== Batch ===" +az batch account list + +# CDN and Front Door +echo "=== CDN and Front Door ===" +az cdn profile list +az afd profile list + +# API Management +echo "=== API Management ===" +az apim list + +# Logic Apps +echo "=== Logic Apps ===" +az logic workflow list + +# Search +echo "=== Search ===" +az search service list --resource-group azure-cli-test-rg + +# HDInsight and Analytics +echo "=== HDInsight and Analytics ===" +az hdinsight list +az synapse workspace list + +# Signal R +echo "=== Signal R ===" +az signalr list + +# Locations and Availability +echo "=== Locations and Availability ===" +az account list-locations +az provider list + +# Resources +echo "=== Resources ===" +az resource list --output json + +# Tags +echo "=== Tags ===" +az tag list + +# NetApp Files +echo "=== NetApp Files ===" +az netappfiles account list + +# # Compute Fleet +# echo "=== Compute Fleet ===" +# az compute-fleet list + +# Budget and Cost +echo "=== Budget and Cost ===" +az consumption budget list 2>/dev/null || echo 'Budget command may require specific permissions' +az consumption usage list 2>/dev/null || echo 'Usage command may require specific permissions' +az billing account list \ No newline at end of file diff --git a/scripts/release/docker/test_az_cli_in_pipeline.sh b/scripts/release/docker/test_az_cli_in_pipeline.sh new file mode 100644 index 00000000000..3b4a52c86cd --- /dev/null +++ b/scripts/release/docker/test_az_cli_in_pipeline.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash + +# Set output file with timestamp +OUTPUT_FILE="/test/azure_cli_test_output.log" +OUTPUT_FILE_Result="/test/azure_cli_test_result.csv" + +# Function to run command and capture output +run_cmd() { + local cmd="$1" + + echo "$cmd" + echo "Command: $cmd" >> "$OUTPUT_FILE" + + # Create temporary files for stdout and stderr + local temp_stdout=$(mktemp) + local temp_stderr=$(mktemp) + local exit_code=0 + + # Run command and capture stdout and stderr separately + if ! eval "$cmd" > "$temp_stdout" 2> "$temp_stderr"; then + exit_code=$? + echo "ERROR: Command failed with exit code $exit_code" >&2 + fi + + # Read the outputs + local stdout_content=$(cat "$temp_stdout") + local stderr_content=$(cat "$temp_stderr") + + # Write to log files + cat "$temp_stdout" >> "$OUTPUT_FILE" + cat "$temp_stderr" >> "$OUTPUT_FILE" + + # Determine execution result + local execution_result="Success" + if [ $exit_code -ne 0 ]; then + execution_result="Fail" + fi + + # Check if output is JSON format and count items + local is_json="No" + local item_count="invalid" + + if [ -n "$stdout_content" ]; then + # Try to parse as JSON using jq if available, or python as fallback + if command -v jq >/dev/null 2>&1; then + if echo "$stdout_content" | jq . >/dev/null 2>&1; then + is_json="Yes" + # Check if it's an array or object + if echo "$stdout_content" | jq -e 'type' | grep -q "array"; then + item_count=$(echo "$stdout_content" | jq 'length') + elif echo "$stdout_content" | jq -e 'type' | grep -q "object"; then + item_count="1" + fi + fi + elif command -v python3 >/dev/null 2>&1; then + if python3 -c "import json; json.loads('''$stdout_content''')" 2>/dev/null; then + is_json="Yes" + # Check if it's an array or object + local json_type=$(python3 -c "import json; data=json.loads('''$stdout_content'''); print(type(data).__name__)" 2>/dev/null) + if [ "$json_type" = "list" ]; then + item_count=$(python3 -c "import json; print(len(json.loads('''$stdout_content''')))" 2>/dev/null) + elif [ "$json_type" = "dict" ]; then + item_count="1" + fi + fi + fi + fi + + # Extract warnings and errors from stderr + local warnings=$(echo "$stderr_content" | grep -i "warning" | tr '\n' ';' | sed 's/;$//') + local errors=$(echo "$stderr_content" | grep -i "error" | tr '\n' ';' | sed 's/;$//') + + # Escape quotes for CSV + cmd_escaped=$(echo "$cmd" | sed 's/"/""/g') + warnings_escaped=$(echo "$warnings" | sed 's/"/""/g') + errors_escaped=$(echo "$errors" | sed 's/"/""/g') + + # Write to CSV result file + echo "\"$cmd_escaped\",\"$execution_result\",\"$is_json\",\"$item_count\",\"$warnings_escaped\",\"$errors_escaped\"" >> "$OUTPUT_FILE_Result" + + # Cleanup temporary files + rm -f "$temp_stdout" "$temp_stderr" + + echo "" >> "$OUTPUT_FILE" +} + +echo "==================================================" +echo "Azure CLI Test Script - All Available List Commands" +echo "==================================================" +echo "Output will be saved to: $OUTPUT_FILE" +echo "" + +echo "==================================================" >> "$OUTPUT_FILE" +echo "Azure CLI Test Script - All Available List Commands" >> "$OUTPUT_FILE" +echo "Test started at: $(date)" >> "$OUTPUT_FILE" +echo "==================================================" >> "$OUTPUT_FILE" +echo "" >> "$OUTPUT_FILE" + +# Create CSV header for result file +echo "Command,Result,IsJSON,Count,Warnings,Errors" > "$OUTPUT_FILE_Result" + +# Execute commands from az_cli_commands.sh +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +az_commands_file="$script_dir/az_cli_commands.sh" + +if [ -f "$az_commands_file" ]; then + echo "Reading commands from: $az_commands_file" + while IFS= read -r line; do + # Skip empty lines and comments that start with # + if [[ -n "$line" && ! "$line" =~ ^[[:space:]]*# ]]; then + # Check if it's an echo command for section headers + if [[ "$line" =~ ^echo ]]; then + # Execute echo commands directly + eval "$line" + elif [[ "$line" =~ ^az ]]; then + # Execute az commands through run_cmd + run_cmd "$line" + fi + elif [[ "$line" =~ ^[[:space:]]*#[[:space:]]*echo ]]; then + # Handle commented echo commands (like # echo "=== Compute Fleet ===") + echo "$line" + fi + done < "$az_commands_file" +else + echo "Warning: Command file not found: $az_commands_file" +fi + +echo "==================================================" +echo "Azure CLI Test Completed" +echo "==================================================" +echo "Full output saved to: $OUTPUT_FILE" + +echo "==================================================" >> "$OUTPUT_FILE" +echo "Azure CLI Test Completed at: $(date)" >> "$OUTPUT_FILE" +echo "==================================================" >> "$OUTPUT_FILE" From 8339425373fec0891394651a05c9d9ad988c0296 Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Fri, 17 Oct 2025 15:29:21 +0800 Subject: [PATCH 23/52] use stage to refactor pipeline --- .azure-pipelines/test-with-copa.yml | 845 +++++++++++++++------------- 1 file changed, 445 insertions(+), 400 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index e48d8c1af4f..f660b3998fd 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -29,223 +29,188 @@ parameters: - minor - major -jobs: - -- job: PrepareAzCliVersion - displayName: Prepare Azure CLI Version - pool: - name: ${{ variables.ubuntu_pool }} - steps: - - checkout: none - - task: Bash@3 - name: PrepareAzCliVersion - displayName: Prepare Azure CLI Version - inputs: - targetType: 'inline' - script: | - curl -o __main__.py https://raw.githubusercontent.com/$BUILD_REPOSITORY_NAME/$BUILD_SOURCEBRANCH/src/azure-cli/azure/cli/__main__.py - AzCLI_VERSION=`cat __main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` - - AzCLI_IMAGE_NAME="clibuild_$BUILD_BUILDNUMBER" - AzCLI_IMAGE_TAG="latest" - AzCLI_IMAGE_PATCHED_TAG="latest-patched" - - echo "Azure CLI version: $AzCLI_VERSION" - echo "Azure CLI Image Name: $AzCLI_IMAGE_NAME" - echo "Azure CLI Image Tag: $AzCLI_IMAGE_TAG" - echo "Azure CLI Image Patched Tag: $AzCLI_IMAGE_PATCHED_TAG" - - echo "##vso[task.setvariable variable=AzCliVersion]$AzCLI_VERSION" - - echo "##vso[task.setvariable variable=AzCliVersion;isOutput=true]$AzCLI_VERSION" - echo "##vso[task.setvariable variable=AzCliImageName;isOutput=true]$AzCLI_IMAGE_NAME" - echo "##vso[task.setvariable variable=AzCliImageTag;isOutput=true]$AzCLI_IMAGE_TAG" - echo "##vso[task.setvariable variable=AzCliImagePatchedTag;isOutput=true]$AzCLI_IMAGE_PATCHED_TAG" - - - task: PowerShell@2 - displayName: Set Build name - inputs: - targetType: inline - pwsh: true - script: | - # build name has 2 limitations - # - The maximum length of a build name is 255 characters. - # - Characters which are not allowed include '"', '/', ':', '<', '>', '\', '|', '?', '@', and '*'. +stages: +- stage: PrepareInfo + displayName: Prepare Azure CLI Info + jobs: + - job: PrepareAzCliVersion + displayName: Prepare Azure CLI Info + pool: + name: ${{ variables.ubuntu_pool }} + steps: + - checkout: none + - task: Bash@3 + name: PrepareAzCliVersion + displayName: Prepare Azure CLI Info + inputs: + targetType: 'inline' + script: | + curl -o __main__.py https://raw.githubusercontent.com/$BUILD_REPOSITORY_NAME/$BUILD_SOURCEBRANCH/src/azure-cli/azure/cli/__main__.py + AzCLI_VERSION=`cat __main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` + + AzCLI_IMAGE_NAME="clibuild_$BUILD_BUILDNUMBER" + AzCLI_IMAGE_TAG="latest" + AzCLI_IMAGE_PATCHED_TAG="latest-patched" - $originalBuildName = "$(Build.BuildNumber)" - $newBuildName = "$originalBuildName - Azure CLI Docker Build $(AzCliVersion)" - - if("${{parameters.ShouldTest}}" -eq "true") { - $newBuildName = "$newBuildName with Test" - } else { - $newBuildName = "$newBuildName without Test" - } - - $notAllowedChars = @('"', '/', ':', '<', '>', '\', '|', '?', '@', '*') - $notAllowedChars | ForEach-Object {$newBuildName = $newBuildName.Replace($_, '_')} - if ( $newBuildName.Length -gt 200 ) { - $newBuildName = $newBuildName.Substring(0, 200) - } - Write-Host "##vso[build.updatebuildnumber]$newBuildName" - -- job: BuildRpmPackagesAzureLinux - displayName: Build Rpm Package - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - image: mcr.microsoft.com/azurelinux/base/core:3.0 - artifact: rpm-azurelinux3.0-${{ arch.value }} - pool: ${{ arch.pool }} - pool: - name: $(pool) - steps: - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - - task: PipAuthenticate@1 - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) - - - task: Bash@3 - displayName: 'Build Rpm Package: Azure Linux' - inputs: - targetType: 'filePath' - filePath: scripts/release/rpm/pipeline_azurelinux.sh - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: rpm-azurelinux' - inputs: - TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(artifact) - -- job: BuildDockerImageAzureLinux - displayName: Build Docker Image Azure Linux + echo "Azure CLI version: $AzCLI_VERSION" + echo "Azure CLI Image Name: $AzCLI_IMAGE_NAME" + echo "Azure CLI Image Tag: $AzCLI_IMAGE_TAG" + echo "Azure CLI Image Patched Tag: $AzCLI_IMAGE_PATCHED_TAG" + + echo "##vso[task.setvariable variable=AzCliVersion]$AzCLI_VERSION" + + echo "##vso[task.setvariable variable=AzCliVersion;isOutput=true]$AzCLI_VERSION" + echo "##vso[task.setvariable variable=AzCliImageName;isOutput=true]$AzCLI_IMAGE_NAME" + echo "##vso[task.setvariable variable=AzCliImageTag;isOutput=true]$AzCLI_IMAGE_TAG" + echo "##vso[task.setvariable variable=AzCliImagePatchedTag;isOutput=true]$AzCLI_IMAGE_PATCHED_TAG" + + - task: PowerShell@2 + displayName: Set Build name + inputs: + targetType: inline + pwsh: true + script: | + # build name has 2 limitations + # - The maximum length of a build name is 255 characters. + # - Characters which are not allowed include '"', '/', ':', '<', '>', '\', '|', '?', '@', and '*'. + + $originalBuildName = "$(Build.BuildNumber)" + $newBuildName = "$originalBuildName - Azure CLI Docker Build $(AzCliVersion)" + + if("${{parameters.ShouldTest}}" -eq "true") { + $newBuildName = "$newBuildName with Test" + } else { + $newBuildName = "$newBuildName without Test" + } + + $notAllowedChars = @('"', '/', ':', '<', '>', '\', '|', '?', '@', '*') + $notAllowedChars | ForEach-Object {$newBuildName = $newBuildName.Replace($_, '_')} + if ( $newBuildName.Length -gt 200 ) { + $newBuildName = $newBuildName.Substring(0, 200) + } + Write-Host "##vso[build.updatebuildnumber]$newBuildName" + +- stage: Build + displayName: Build RPM and Docker Images dependsOn: - - PrepareAzCliVersion - - BuildRpmPackagesAzureLinux - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - pool: ${{ arch.pool }} - artifactName: docker-azurelinux3.0-${{ arch.value }} - dockerfile: azure-linux.dockerfile - packageArtifactName: rpm-azurelinux3.0-${{ arch.value }} - image: mcr.microsoft.com/azurelinux/base/core:3.0 - pool: - name: $(pool) - variables: - AzCliVersion: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] - AzCliImageName: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] - AzCliImageTag: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] - steps: + - PrepareInfo + jobs: + - job: BuildRpmPackagesAzureLinux + displayName: Build Rpm Package + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + image: mcr.microsoft.com/azurelinux/base/core:3.0 + artifact: rpm-azurelinux3.0-${{ arch.value }} + pool: ${{ arch.pool }} + pool: + name: $(pool) + steps: - bash: ./scripts/ci/install_docker.sh displayName: Install Docker - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' + + - task: PipAuthenticate@1 + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + displayName: 'Pip Authenticate' inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/docker' - artifactName: $(packageArtifactName) - - bash: | - set -ex - mkdir docker-temp - mv $(Build.ArtifactStagingDirectory)/docker/*.rpm ./docker-temp/azure-cli.rpm - - docker build --no-cache \ - --build-arg IMAGE=$IMAGE \ - --tag $(AzCliImageName):$(AzCliImageTag) \ - --file $DOCKERFILE \ - $BUILD_SOURCESDIRECTORY + artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) - docker save -o "$BUILD_STAGINGDIRECTORY/docker-azure-cli-$(AzCliVersion).tar" $(AzCliImageName):$(AzCliImageTag) - - displayName: 'Build Docker' + - task: Bash@3 + displayName: 'Build Rpm Package: Azure Linux' + inputs: + targetType: 'filePath' + filePath: scripts/release/rpm/pipeline_azurelinux.sh - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 displayName: 'SBOM' condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') inputs: BuildDropPath: $(Build.ArtifactStagingDirectory) - DockerImagesToScan: $(AzCliDockerImageTag) - task: PublishPipelineArtifact@0 + displayName: 'Publish Artifact: rpm-azurelinux' inputs: TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(artifactName) - -- job: TestRpmPackagesAzureLinux - displayName: Test Rpm Package - timeoutInMinutes: 180 - dependsOn: - - BuildRpmPackagesAzureLinux - - PrepareAzCliVersion - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule'), eq(${{parameters.ShouldTest}}, true)) - pool: - name: $(pool) - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - image: mcr.microsoft.com/azurelinux/base/core:3.0 - artifact: rpm-azurelinux3.0-${{ arch.value }} - pool: ${{ arch.pool }} - variables: - AzCliVersion: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' - artifactName: $(artifact) - - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - - bash: | - set -ex - RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$(AzCliVersion)-1.*.rpm" -printf '%f\n') - - echo "== Test rpm package on ${IMAGE} ==" - docker pull $IMAGE - docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh - - displayName: 'Test Rpm Package Azure Linux' - -- job: TestDockerImageAzureLinux - displayName: Test Docker Image Azure Linux - condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) - timeoutInMinutes: 180 - dependsOn: - - PrepareAzCliVersion - - BuildDockerImageAzureLinux - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - pool: ${{ arch.pool }} - artifactName: docker-azurelinux3.0-${{ arch.value }} - pool: - name: $(pool) - variables: - AzCliVersion: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] - AzCliImageName: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] - AzCliImageTag: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] - steps: + ArtifactName: $(artifact) + + - job: BuildDockerImageAzureLinux + displayName: Build Docker Image + dependsOn: + - BuildRpmPackagesAzureLinux + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + pool: ${{ arch.pool }} + artifactName: docker-azurelinux3.0-${{ arch.value }} + dockerfile: azure-linux.dockerfile + packageArtifactName: rpm-azurelinux3.0-${{ arch.value }} + image: mcr.microsoft.com/azurelinux/base/core:3.0 + pool: + name: $(pool) + variables: + AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] + AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] + AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] + steps: + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + - task: DownloadPipelineArtifact@1 + displayName: 'Download Build Artifacts' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/docker' + artifactName: $(packageArtifactName) + - bash: | + set -ex + mkdir docker-temp + mv $(Build.ArtifactStagingDirectory)/docker/*.rpm ./docker-temp/azure-cli.rpm + + docker build --no-cache \ + --build-arg IMAGE=$IMAGE \ + --tag $(AzCliImageName):$(AzCliImageTag) \ + --file $DOCKERFILE \ + $BUILD_SOURCESDIRECTORY + + docker save -o "$BUILD_STAGINGDIRECTORY/docker-azure-cli-$(AzCliVersion).tar" $(AzCliImageName):$(AzCliImageTag) + + displayName: 'Build Docker' + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + DockerImagesToScan: $(AzCliDockerImageTag) + + - task: PublishPipelineArtifact@0 + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(artifactName) + + +- stage: Test1 + displayName: 1st Test - for Original Image + dependsOn: + - PrepareInfo + - Build + jobs: + - job: TestRpmPackagesAzureLinux + displayName: Test Rpm Package + timeoutInMinutes: 180 + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule'), eq(${{parameters.ShouldTest}}, true)) + pool: + name: $(pool) + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + image: mcr.microsoft.com/azurelinux/base/core:3.0 + artifact: rpm-azurelinux3.0-${{ arch.value }} + pool: ${{ arch.pool }} + variables: + AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] + steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Metadata' inputs: @@ -253,208 +218,288 @@ jobs: artifactName: metadata - task: DownloadPipelineArtifact@1 - displayName: 'Download Docker Image' + displayName: 'Download Build Artifacts' inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/docker' - artifactName: $(artifactName) + TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' + artifactName: $(artifact) - bash: ./scripts/ci/install_docker.sh displayName: Install Docker - - task: Bash@3 - displayName: Create Test and Output Folders - inputs: - targetType: 'inline' - script: | - mkdir output - chmod +x ./scripts/release/docker/test_az_cli_in_pipeline.sh - ls ./.azure-pipelines - - - bash: | - set -exv - - echo "== Load docker image ==" - TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$(AzCliVersion).tar" - docker load < $TAR_FILE - - echo "== Run az self-test ==" - docker run $(AzCliImageName):$(AzCliImageTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" - - displayName: 'Run Simple Test' - - bash: | - echo "== List docker image ==" - docker images - - echo "== Run unit test ==" - docker run --rm -v $(pwd):/azure-cli $(AzCliImageName):$(AzCliImageTag) /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false - - displayName: 'Run Unit Test' - -- job: BuildCopaPatchedDockerImageAzureLinux - displayName: Build Copa Patched Docker Image Azure Linux - timeoutInMinutes: 180 - dependsOn: - - BuildDockerImageAzureLinux - - PrepareAzCliVersion - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - pool: ${{ arch.pool }} - artifactName: docker-azurelinux3.0-${{ arch.value }} - architecture: ${{ arch.value }} - pool: - name: $(pool) - variables: - AzCliVersion: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] - AzCliImageName: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] - AzCliImageTag: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] - AzCliImagePatchedTag: $[ dependencies.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImagePatchedTag'] ] - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Docker Image' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/docker' - artifactName: $(artifactName) - - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - - task: Bash@3 - displayName: Install PowerShell for ARM64 Agent - condition: and(succeeded(), eq(variables['architecture'], 'arm64')) - inputs: - targetType: 'inline' - script: | - curl -L -o /tmp/powershell.tar.gz https://github.com/PowerShell/PowerShell/releases/download/v7.4.11/powershell-7.4.11-linux-arm64.tar.gz - sudo mkdir -p /opt/microsoft/powershell/7 - sudo tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell/7 - sudo chmod +x /opt/microsoft/powershell/7/pwsh - sudo ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh - PATH=/usr/bin:/sbin:/usr/sbin:$PATH - - - task: Bash@3 - displayName: Install Az.Accounts for ARM64 Agents - condition: and(succeeded(), eq(variables['architecture'], 'arm64')) - inputs: - targetType: 'inline' - script: | - pwsh -command 'Register-PSRepository -Default -InstallationPolicy Trusted' - pwsh -command 'Install-Module -Name Az.Accounts -Repository PSGallery -AllowClobber -Force' - - - task: Bash@3 - displayName: Install Trivy - inputs: - targetType: 'inline' - script: | - set -eux - curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/bin - echo "----------------------------------" - sudo trivy --version - - - task: Bash@3 - displayName: Install copacetic - inputs: - targetType: 'inline' - script: | - CopaDownloadUrl="https://github.com/project-copacetic/copacetic/releases/download/v$(CopaVersion)/copa_$(CopaVersion)_linux_amd64.tar.gz" - if [ "$(architecture)" = "arm64" ]; then - CopaDownloadUrl="https://github.com/project-copacetic/copacetic/releases/download/v$(CopaVersion)/copa_$(CopaVersion)_linux_arm64.tar.gz" - fi - curl -L -o /tmp/copa.tar.gz $CopaDownloadUrl - sudo mkdir -p /opt/copacetic - sudo tar zxf /tmp/copa.tar.gz -C /opt/copacetic - sudo chmod +x /opt/copacetic/copa - sudo ln -s /opt/copacetic/copa /usr/bin/copa - - sudo copa --version - sudo copa --help - sudo copa patch --help - - - task: Bash@3 - displayName: Check Docker Image - inputs: - targetType: 'inline' - script: | - set -exv - - TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$(AzCliVersion).tar" - docker load < $TAR_FILE - - echo "== Check Docker Images ==" - docker images - - echo "== docker container ==" - docker ps - - echo "== Run az self-test ==" - docker run $(AzCliImageName):$(AzCliImageTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" - - - task: Bash@3 - displayName: Create Output Folders - inputs: - targetType: 'inline' - script: | - pwd - mkdir output - - - task: Bash@3 - displayName: Scan with Trivy - inputs: - targetType: 'inline' - script: | - export IMAGE=$(AzCliImageName):$(AzCliImageTag) - OutputFile="trivy-scan-before.json" - sudo trivy image --pkg-types os,library --ignore-unfixed -f json -o $OutputFile $IMAGE - cp $OutputFile ./output/ - - - task: Bash@3 - displayName: Setup Docker BuildKit - inputs: - targetType: 'inline' - script: | - set -eux - # Enable Docker BuildKit - export DOCKER_BUILDKIT=1 - export BUILDKIT_PROGRESS=plain - - # Check Docker version and BuildKit support - sudo docker version - sudo docker buildx version || echo "buildx not available" - - # Try to create and use a new builder with containerd image store - sudo docker buildx create --name copa-builder --driver docker-container --use || true - sudo docker buildx inspect --bootstrap || echo "Failed to bootstrap buildx" - - # Alternative: Enable experimental features in Docker daemon - echo "Checking Docker daemon configuration..." - sudo docker info | grep -i experimental || echo "Experimental features not enabled" - - - task: Bash@3 - displayName: Patch with Copacetic - inputs: - targetType: 'inline' - script: | - set -eux - export DOCKER_BUILDKIT=1 - export BUILDKIT_PROGRESS=plain - - export COPA_EXPERIMENTAL=1 - export IMAGE=$(AzCliImageName):$(AzCliImageTag) - - echo "before copa patch-----------------------" - sudo COPA_EXPERIMENTAL=1 copa patch -i $IMAGE -r trivy-scan-before.json --pkg-types os,library --library-patch-level ${{parameters.PatchLevel}} --tag $(AzCliImagePatchedTag) - - echo "after copa patch-----------------------" - - - task: Bash@3 - displayName: Check Docker Image - inputs: - targetType: 'inline' - script: | - echo "== Check Docker Images ==" - docker images - - echo "== Run az self-test ==" - docker run $(AzCliImageName):$(AzCliImagePatchedTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" + set -ex + RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$(AzCliVersion)-1.*.rpm" -printf '%f\n') + + echo "== Test rpm package on ${IMAGE} ==" + docker pull $IMAGE + docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh + + displayName: 'Test Rpm Package Azure Linux' + + - job: TestDockerImageAzureLinux + displayName: Test Docker Image + condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) + timeoutInMinutes: 180 + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + pool: ${{ arch.pool }} + artifactName: docker-azurelinux3.0-${{ arch.value }} + pool: + name: $(pool) + variables: + AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] + AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] + AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Metadata' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' + artifactName: metadata + + - task: DownloadPipelineArtifact@1 + displayName: 'Download Docker Image' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/docker' + artifactName: $(artifactName) + + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + + - task: Bash@3 + displayName: Create Test and Output Folders + inputs: + targetType: 'inline' + script: | + mkdir output + chmod +x ./scripts/release/docker/test_az_cli_in_pipeline.sh + ls ./.azure-pipelines + + - bash: | + set -exv + + echo "== Load docker image ==" + TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$(AzCliVersion).tar" + docker load < $TAR_FILE + + echo "== Run az self-test ==" + docker run $(AzCliImageName):$(AzCliImageTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" + + displayName: 'Run Simple Test' + + - bash: | + echo "== List docker image ==" + docker images + + echo "== Run unit test ==" + docker run --rm -v $(pwd):/azure-cli $(AzCliImageName):$(AzCliImageTag) /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false + + displayName: 'Run Unit Test' + + +- stage: UseCopaToPatch + displayName: Patch with Copacetic + dependsOn: + - PrepareInfo + - Build + jobs: + - job: BuildCopaPatchedDockerImageAzureLinux + displayName: Build Copa Patched Docker Image + timeoutInMinutes: 180 + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + pool: ${{ arch.pool }} + artifactName: docker-azurelinux3.0-${{ arch.value }} + architecture: ${{ arch.value }} + pool: + name: $(pool) + variables: + AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] + AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] + AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] + AzCliImagePatchedTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImagePatchedTag'] ] + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Docker Image' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/docker' + artifactName: $(artifactName) + + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + + - task: Bash@3 + displayName: Install PowerShell for ARM64 Agent + condition: and(succeeded(), eq(variables['architecture'], 'arm64')) + inputs: + targetType: 'inline' + script: | + curl -L -o /tmp/powershell.tar.gz https://github.com/PowerShell/PowerShell/releases/download/v7.4.11/powershell-7.4.11-linux-arm64.tar.gz + sudo mkdir -p /opt/microsoft/powershell/7 + sudo tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell/7 + sudo chmod +x /opt/microsoft/powershell/7/pwsh + sudo ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh + PATH=/usr/bin:/sbin:/usr/sbin:$PATH + + - task: Bash@3 + displayName: Install Az.Accounts for ARM64 Agents + condition: and(succeeded(), eq(variables['architecture'], 'arm64')) + inputs: + targetType: 'inline' + script: | + pwsh -command 'Register-PSRepository -Default -InstallationPolicy Trusted' + pwsh -command 'Install-Module -Name Az.Accounts -Repository PSGallery -AllowClobber -Force' + + - task: Bash@3 + displayName: Install Trivy + inputs: + targetType: 'inline' + script: | + set -eux + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/bin + echo "----------------------------------" + sudo trivy --version + + - task: Bash@3 + displayName: Install copacetic + inputs: + targetType: 'inline' + script: | + CopaDownloadUrl="https://github.com/project-copacetic/copacetic/releases/download/v$(CopaVersion)/copa_$(CopaVersion)_linux_amd64.tar.gz" + if [ "$(architecture)" = "arm64" ]; then + CopaDownloadUrl="https://github.com/project-copacetic/copacetic/releases/download/v$(CopaVersion)/copa_$(CopaVersion)_linux_arm64.tar.gz" + fi + curl -L -o /tmp/copa.tar.gz $CopaDownloadUrl + sudo mkdir -p /opt/copacetic + sudo tar zxf /tmp/copa.tar.gz -C /opt/copacetic + sudo chmod +x /opt/copacetic/copa + sudo ln -s /opt/copacetic/copa /usr/bin/copa + + sudo copa --version + sudo copa --help + sudo copa patch --help + + - task: Bash@3 + displayName: Check Docker Image + inputs: + targetType: 'inline' + script: | + set -exv + + TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$(AzCliVersion).tar" + docker load < $TAR_FILE + + echo "== Check Docker Images ==" + docker images + + echo "== docker container ==" + docker ps + + echo "== Run az self-test ==" + docker run $(AzCliImageName):$(AzCliImageTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" + + - task: Bash@3 + displayName: Create Output Folders + inputs: + targetType: 'inline' + script: | + pwd + mkdir output + + - task: Bash@3 + displayName: Scan with Trivy + inputs: + targetType: 'inline' + script: | + export IMAGE=$(AzCliImageName):$(AzCliImageTag) + OutputFile="trivy-scan-before.json" + sudo trivy image --pkg-types os,library --ignore-unfixed -f json -o $OutputFile $IMAGE + cp $OutputFile ./output/ + + - task: Bash@3 + displayName: Setup Docker BuildKit + inputs: + targetType: 'inline' + script: | + set -eux + # Enable Docker BuildKit + export DOCKER_BUILDKIT=1 + export BUILDKIT_PROGRESS=plain + + # Check Docker version and BuildKit support + sudo docker version + sudo docker buildx version || echo "buildx not available" + + # Try to create and use a new builder with containerd image store + sudo docker buildx create --name copa-builder --driver docker-container --use || true + sudo docker buildx inspect --bootstrap || echo "Failed to bootstrap buildx" + + # Alternative: Enable experimental features in Docker daemon + echo "Checking Docker daemon configuration..." + sudo docker info | grep -i experimental || echo "Experimental features not enabled" + + - task: Bash@3 + displayName: Patch with Copacetic + inputs: + targetType: 'inline' + script: | + set -eux + export DOCKER_BUILDKIT=1 + export BUILDKIT_PROGRESS=plain + + export COPA_EXPERIMENTAL=1 + export IMAGE=$(AzCliImageName):$(AzCliImageTag) + + echo "before copa patch-----------------------" + sudo COPA_EXPERIMENTAL=1 copa patch -i $IMAGE -r trivy-scan-before.json --pkg-types os,library --library-patch-level ${{parameters.PatchLevel}} --tag $(AzCliImagePatchedTag) + + echo "after copa patch-----------------------" + + - task: Bash@3 + displayName: Check Docker Image + inputs: + targetType: 'inline' + script: | + echo "== Check Docker Images ==" + docker images + + echo "== Run az self-test ==" + docker run $(AzCliImageName):$(AzCliImagePatchedTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" + +- stage: Test2 + displayName: 2nd Test - for Copa Patched Image + dependsOn: + - PrepareInfo + - Build + - UseCopaToPatch + jobs: + - job: TestCopaPatchedDockerImageAzureLinux + displayName: Test Copa Patched Docker Image + timeoutInMinutes: 180 + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + pool: ${{ arch.pool }} + artifactName: docker-azurelinux3.0-${{ arch.value }} + architecture: ${{ arch.value }} + pool: + name: $(pool) + variables: + AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] + AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] + AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] + AzCliImagePatchedTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImagePatchedTag'] ] + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Docker Image' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/docker' + artifactName: $(artifactName) + + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker From ac27568c148781b23eae3a80f61f216e8dfc0d82 Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Fri, 17 Oct 2025 16:10:02 +0800 Subject: [PATCH 24/52] add smoke test --- .azure-pipelines/test-with-copa.yml | 152 ++++++++++++++++++++++++---- 1 file changed, 130 insertions(+), 22 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index f660b3998fd..76ccba7e0f4 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -188,7 +188,6 @@ stages: TargetPath: $(Build.ArtifactStagingDirectory) ArtifactName: $(artifactName) - - stage: Test1 displayName: 1st Test - for Original Image dependsOn: @@ -211,12 +210,6 @@ stages: variables: AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata - - task: DownloadPipelineArtifact@1 displayName: 'Download Build Artifacts' inputs: @@ -236,10 +229,10 @@ stages: displayName: 'Test Rpm Package Azure Linux' - - job: TestDockerImageAzureLinux - displayName: Test Docker Image + - job: TestDockerImageSmokeTest + displayName: Test Docker Image - Smoke Test condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) - timeoutInMinutes: 180 + timeoutInMinutes: 100 strategy: matrix: ${{ each arch in parameters.architectures }}: @@ -254,11 +247,119 @@ stages: AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] steps: - task: DownloadPipelineArtifact@1 - displayName: 'Download Metadata' + displayName: 'Download Docker Image' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/docker' + artifactName: $(artifactName) + + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + + - task: Bash@3 + displayName: Prepare Test and Output Folders inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/metadata' - artifactName: metadata + targetType: 'inline' + script: | + mkdir output + chmod +x ./scripts/release/docker/test_az_cli_in_pipeline.sh + ls ./.azure-pipelines + + - task: Bash@3 + displayName: 🚀 Self Test + inputs: + targetType: 'inline' + script: | + echo "== Load docker image ==" + TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$(AzCliVersion).tar" + docker load < $TAR_FILE + + echo "== Docker Images ==" + docker images + + echo "== Run az self-test ==" + docker run $(AzCliImageName):$(AzCliImageTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" + + #sudo docker run $(TestImage):$(TestImageTag) /bin/bash -c "time az self-test && time az --version && (tdnf list --installed || true) " + + - task: AzurePowerShell@5 + displayName: Get Azure Service Connection Info + inputs: + pwsh: true + azureSubscription: '$(TestAzureSubscription)' + ScriptType: 'InlineScript' + azurePowerShellVersion: 'LatestVersion' + Inline: | + $PSVersionTable + $AzAccessToken = (Get-AzAccessToken -AsSecureString).Token + $AzGraphToken = (Get-AzAccessToken -ResourceTypeName MSGraph -AsSecureString).Token + $ssPtr = [System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($AzAccessToken) + $ssPtrGraph = [System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($AzGraphToken) + try { + $AzAccessToken = [System.Runtime.InteropServices.Marshal]::PtrToStringBSTR($ssPtr) + $AzGraphToken = [System.Runtime.InteropServices.Marshal]::PtrToStringBSTR($ssPtrGraph) + } + finally { + [System.Runtime.InteropServices.Marshal]::ZeroFreeBSTR($ssPtr) + [System.Runtime.InteropServices.Marshal]::ZeroFreeBSTR($ssPtrGraph) + } + + $AzUserId = (Get-AzAccessToken).UserId + $AzSubscription = (Get-AzContext).Subscription + $AzTenant = (Get-AzContext).Tenant + + Write-Host "##vso[task.setvariable variable=AzAccessToken;issecret=true]$AzAccessToken" + Write-Host "##vso[task.setvariable variable=AzUserId;issecret=true]$AzUserId" + Write-Host "##vso[task.setvariable variable=AzSubscription;issecret=true]$AzSubscription" + Write-Host "##vso[task.setvariable variable=AzTenant;issecret=true]$AzTenant" + Write-Host "##vso[task.setvariable variable=AzGraphToken;issecret=true]$AzGraphToken" + + - task: PowerShell@2 + displayName: Get OIDC Token + inputs: + targetType: inline + pwsh: true + script: | + $token = "$(System.AccessToken)" + $header = @{authorization = "Bearer $token"} + $scUrl = "${env:SYSTEM_OIDCREQUESTURI}?api-version=7.1&serviceConnectionId=75a6a794-72e5-4618-b576-deea04f8b21d" + $result = Invoke-RestMethod -Uri $scUrl -Method Post -ContentType "application/json" -Headers $header + $OidcToken = $result.oidcToken + Write-Host "##vso[task.setvariable variable=OidcToken;issecret=true]$OidcToken" + + - task: Bash@3 + displayName: 🚀🚀 Smoke Test + inputs: + targetType: 'inline' + script: | + sudo docker run -v $(System.DefaultWorkingDirectory)/scripts/docker:/test $(AzCliImageName):$(AzCliImageTag) /bin/bash -c "az login --service-principal --username $(AzUserId) --tenant $(AzTenant) --federated-token $(OidcToken) && az account set --subscription $(AzSubscription) && /test/test_az_cli_in_pipeline.sh" + + echo "--------------------------------------------" + mv $(System.DefaultWorkingDirectory)/scripts/docker/azure_cli_test_output.log $(System.DefaultWorkingDirectory)/output/azure_cli_test_output_before.log + mv $(System.DefaultWorkingDirectory)/scripts/docker/azure_cli_test_result.csv $(System.DefaultWorkingDirectory)/output/azure_cli_test_result_before.csv + ls $(System.DefaultWorkingDirectory)/output + + - publish: $(System.DefaultWorkingDirectory)/output + artifact: $(AzCliVersion)-$(architecture)-smoke-test + displayName: Publish Artifact + + - job: TestDockerImageUnitTest + displayName: Test Docker Image - Unit Test + condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) + timeoutInMinutes: 180 + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + pool: ${{ arch.pool }} + artifactName: docker-azurelinux3.0-${{ arch.value }} + pool: + name: $(pool) + variables: + AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] + AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] + AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] + steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Docker Image' inputs: @@ -298,7 +399,6 @@ stages: displayName: 'Run Unit Test' - - stage: UseCopaToPatch displayName: Patch with Copacetic dependsOn: @@ -402,19 +502,12 @@ stages: echo "== Run az self-test ==" docker run $(AzCliImageName):$(AzCliImageTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" - - task: Bash@3 - displayName: Create Output Folders - inputs: - targetType: 'inline' - script: | - pwd - mkdir output - - task: Bash@3 displayName: Scan with Trivy inputs: targetType: 'inline' script: | + mkdir output export IMAGE=$(AzCliImageName):$(AzCliImageTag) OutputFile="trivy-scan-before.json" sudo trivy image --pkg-types os,library --ignore-unfixed -f json -o $OutputFile $IMAGE @@ -470,6 +563,21 @@ stages: echo "== Run az self-test ==" docker run $(AzCliImageName):$(AzCliImagePatchedTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" + - task: Bash@3 + displayName: Scan with Trivy again + inputs: + targetType: 'inline' + script: | + mkdir output + export IMAGE=$(AzCliImageName):$(AzCliImagePatchedTag) + OutputFile="trivy-scan-after.json" + sudo trivy image --pkg-types os,library --ignore-unfixed -f json -o $OutputFile $IMAGE + cp $OutputFile ./output/ + + - publish: $(System.DefaultWorkingDirectory)/output + artifact: $(AzCliVersion)-$(architecture)-copa + displayName: Publish Artifact + - stage: Test2 displayName: 2nd Test - for Copa Patched Image dependsOn: From 072fc93d1f18bcb1bcb948d257ded0efef307e2b Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Fri, 17 Oct 2025 16:19:45 +0800 Subject: [PATCH 25/52] add lines --- .azure-pipelines/test-with-copa.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 76ccba7e0f4..6fc70519a6c 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -479,17 +479,19 @@ stages: sudo chmod +x /opt/copacetic/copa sudo ln -s /opt/copacetic/copa /usr/bin/copa + echo "----------------------------------" sudo copa --version + echo "----------------------------------" sudo copa --help + echo "----------------------------------" sudo copa patch --help + echo "----------------------------------" - task: Bash@3 displayName: Check Docker Image inputs: targetType: 'inline' script: | - set -exv - TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$(AzCliVersion).tar" docker load < $TAR_FILE @@ -540,7 +542,6 @@ stages: inputs: targetType: 'inline' script: | - set -eux export DOCKER_BUILDKIT=1 export BUILDKIT_PROGRESS=plain From 093cab62d13a10ce6e9d70c28540e5b7ca794f21 Mon Sep 17 00:00:00 2001 From: YanaXu Date: Mon, 20 Oct 2025 10:37:14 +0800 Subject: [PATCH 26/52] install pwsh for smoke test --- .azure-pipelines/test-with-copa.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 6fc70519a6c..59047065994 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -239,6 +239,7 @@ stages: Azure Linux 3.0 ${{ arch.name }}: pool: ${{ arch.pool }} artifactName: docker-azurelinux3.0-${{ arch.value }} + architecture: ${{ arch.value }} pool: name: $(pool) variables: @@ -255,6 +256,28 @@ stages: - bash: ./scripts/ci/install_docker.sh displayName: Install Docker + - task: Bash@3 + displayName: Install PowerShell for ARM64 Agent + condition: and(succeeded(), eq(variables['architecture'], 'arm64')) + inputs: + targetType: 'inline' + script: | + curl -L -o /tmp/powershell.tar.gz https://github.com/PowerShell/PowerShell/releases/download/v7.4.11/powershell-7.4.11-linux-arm64.tar.gz + sudo mkdir -p /opt/microsoft/powershell/7 + sudo tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell/7 + sudo chmod +x /opt/microsoft/powershell/7/pwsh + sudo ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh + PATH=/usr/bin:/sbin:/usr/sbin:$PATH + + - task: Bash@3 + displayName: Install Az.Accounts for ARM64 Agents + condition: and(succeeded(), eq(variables['architecture'], 'arm64')) + inputs: + targetType: 'inline' + script: | + pwsh -command 'Register-PSRepository -Default -InstallationPolicy Trusted' + pwsh -command 'Install-Module -Name Az.Accounts -Repository PSGallery -AllowClobber -Force' + - task: Bash@3 displayName: Prepare Test and Output Folders inputs: From 4f90158ca001a0bfa455bfe9211ec3b5f24bd6d8 Mon Sep 17 00:00:00 2001 From: YanaXu Date: Mon, 20 Oct 2025 11:22:43 +0800 Subject: [PATCH 27/52] extract templates --- .../templates/test-docker-image-smoke.yml | 155 ++++++++++++ .../templates/test-docker-image-unit.yml | 78 ++++++ .azure-pipelines/test-with-copa.yml | 224 +++--------------- 3 files changed, 271 insertions(+), 186 deletions(-) create mode 100644 .azure-pipelines/templates/test-docker-image-smoke.yml create mode 100644 .azure-pipelines/templates/test-docker-image-unit.yml diff --git a/.azure-pipelines/templates/test-docker-image-smoke.yml b/.azure-pipelines/templates/test-docker-image-smoke.yml new file mode 100644 index 00000000000..9353023c620 --- /dev/null +++ b/.azure-pipelines/templates/test-docker-image-smoke.yml @@ -0,0 +1,155 @@ +parameters: +- name: architectures + type: object +- name: artifactNamePrefix + type: string + default: 'docker-azurelinux3.0' +- name: condition + type: string + default: 'succeeded()' +- name: timeoutInMinutes + type: number + default: 100 +- name: AzCliVersion + type: string + default: '' +- name: AzCliImageName + type: string + default: '' +- name: AzCliImageTag + type: string + default: '' + +jobs: +- job: TestDockerImageSmokeTest + displayName: Test Docker Image - Smoke Test + condition: ${{ parameters.condition }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + pool: ${{ arch.pool }} + artifactName: ${{ parameters.artifactNamePrefix }}-${{ arch.value }} + architecture: ${{ arch.value }} + pool: + name: $(pool) + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Docker Image' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/docker' + artifactName: $(artifactName) + + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + + - task: Bash@3 + displayName: Install PowerShell for ARM64 Agent + condition: and(succeeded(), eq(variables['architecture'], 'arm64')) + inputs: + targetType: 'inline' + script: | + curl -L -o /tmp/powershell.tar.gz https://github.com/PowerShell/PowerShell/releases/download/v7.4.11/powershell-7.4.11-linux-arm64.tar.gz + sudo mkdir -p /opt/microsoft/powershell/7 + sudo tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell/7 + sudo chmod +x /opt/microsoft/powershell/7/pwsh + sudo ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh + PATH=/usr/bin:/sbin:/usr/sbin:$PATH + + - task: Bash@3 + displayName: Install Az.Accounts for ARM64 Agents + condition: and(succeeded(), eq(variables['architecture'], 'arm64')) + inputs: + targetType: 'inline' + script: | + pwsh -command 'Register-PSRepository -Default -InstallationPolicy Trusted' + pwsh -command 'Install-Module -Name Az.Accounts -Repository PSGallery -AllowClobber -Force' + + - task: Bash@3 + displayName: Prepare Test and Output Folders + inputs: + targetType: 'inline' + script: | + mkdir output + chmod +x ./scripts/release/docker/test_az_cli_in_pipeline.sh + ls ./.azure-pipelines + + - task: Bash@3 + displayName: 🚀 Self Test + inputs: + targetType: 'inline' + script: | + echo "== Load docker image ==" + TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-${{ parameters.AzCliVersion }}.tar" + docker load < $TAR_FILE + + echo "== Docker Images ==" + docker images + + echo "== Run az self-test ==" + docker run ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash -c "time az self-test && time az --version && tdnf list --installed" + + #sudo docker run $(TestImage):$(TestImageTag) /bin/bash -c "time az self-test && time az --version && (tdnf list --installed || true) " + + - task: AzurePowerShell@5 + displayName: Get Azure Service Connection Info + inputs: + pwsh: true + azureSubscription: '$(TestAzureSubscription)' + ScriptType: 'InlineScript' + azurePowerShellVersion: 'LatestVersion' + Inline: | + $PSVersionTable + $AzAccessToken = (Get-AzAccessToken -AsSecureString).Token + $AzGraphToken = (Get-AzAccessToken -ResourceTypeName MSGraph -AsSecureString).Token + $ssPtr = [System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($AzAccessToken) + $ssPtrGraph = [System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($AzGraphToken) + try { + $AzAccessToken = [System.Runtime.InteropServices.Marshal]::PtrToStringBSTR($ssPtr) + $AzGraphToken = [System.Runtime.InteropServices.Marshal]::PtrToStringBSTR($ssPtrGraph) + } + finally { + [System.Runtime.InteropServices.Marshal]::ZeroFreeBSTR($ssPtr) + [System.Runtime.InteropServices.Marshal]::ZeroFreeBSTR($ssPtrGraph) + } + + $AzUserId = (Get-AzAccessToken).UserId + $AzSubscription = (Get-AzContext).Subscription + $AzTenant = (Get-AzContext).Tenant + + Write-Host "##vso[task.setvariable variable=AzAccessToken;issecret=true]$AzAccessToken" + Write-Host "##vso[task.setvariable variable=AzUserId;issecret=true]$AzUserId" + Write-Host "##vso[task.setvariable variable=AzSubscription;issecret=true]$AzSubscription" + Write-Host "##vso[task.setvariable variable=AzTenant;issecret=true]$AzTenant" + Write-Host "##vso[task.setvariable variable=AzGraphToken;issecret=true]$AzGraphToken" + + - task: PowerShell@2 + displayName: Get OIDC Token + inputs: + targetType: inline + pwsh: true + script: | + $token = "$(System.AccessToken)" + $header = @{authorization = "Bearer $token"} + $scUrl = "${env:SYSTEM_OIDCREQUESTURI}?api-version=7.1&serviceConnectionId=75a6a794-72e5-4618-b576-deea04f8b21d" + $result = Invoke-RestMethod -Uri $scUrl -Method Post -ContentType "application/json" -Headers $header + $OidcToken = $result.oidcToken + Write-Host "##vso[task.setvariable variable=OidcToken;issecret=true]$OidcToken" + + - task: Bash@3 + displayName: 🚀🚀 Smoke Test + inputs: + targetType: 'inline' + script: | + sudo docker run -v $(System.DefaultWorkingDirectory)/scripts/docker:/test ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash -c "az login --service-principal --username $(AzUserId) --tenant $(AzTenant) --federated-token $(OidcToken) && az account set --subscription $(AzSubscription) && /test/test_az_cli_in_pipeline.sh" + + echo "--------------------------------------------" + + mv $(System.DefaultWorkingDirectory)/scripts/docker/azure_cli_test_output.log $(System.DefaultWorkingDirectory)/output/azure_cli_test_output_before.log + mv $(System.DefaultWorkingDirectory)/scripts/docker/azure_cli_test_result.csv $(System.DefaultWorkingDirectory)/output/azure_cli_test_result_before.csv + ls $(System.DefaultWorkingDirectory)/output + + - publish: $(System.DefaultWorkingDirectory)/output + artifact: ${{ parameters.AzCliVersion }}-$(architecture)-smoke-test + displayName: Publish Artifact diff --git a/.azure-pipelines/templates/test-docker-image-unit.yml b/.azure-pipelines/templates/test-docker-image-unit.yml new file mode 100644 index 00000000000..8b7925473d0 --- /dev/null +++ b/.azure-pipelines/templates/test-docker-image-unit.yml @@ -0,0 +1,78 @@ +parameters: +- name: architectures + type: object +- name: artifactNamePrefix + type: string + default: 'docker-azurelinux3.0' +- name: artifactNameSuffix + type: string + default: '' +- name: ShouldTest + displayName: Test? + type: boolean + default: false +- name: timeoutInMinutes + type: number + default: 180 +- name: AzCliVersion + type: string + default: '' +- name: AzCliImageName + type: string + default: '' +- name: AzCliImageTag + type: string + default: '' + +jobs: +- job: TestDockerImageUnitTest + displayName: Test Docker Image - Unit Test + condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + strategy: + matrix: + ${{ each arch in parameters.architectures }}: + Azure Linux 3.0 ${{ arch.name }}: + pool: ${{ arch.pool }} + downloadArtifactName: ${{ parameters.artifactNamePrefix }}-${{ arch.value }}${{ parameters.artifactNameSuffix }} + pool: + name: $(pool) + steps: + - task: DownloadPipelineArtifact@1 + displayName: 'Download Docker Image' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/docker' + artifactName: $(downloadArtifactName) + + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker + + - task: Bash@3 + displayName: Create Test and Output Folders + inputs: + targetType: 'inline' + script: | + mkdir output + chmod +x ./scripts/release/docker/test_az_cli_in_pipeline.sh + ls ./.azure-pipelines + + - bash: | + set -exv + + echo "== Load docker image ==" + TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-${{ parameters.AzCliVersion }}.tar" + docker load < $TAR_FILE + + echo "== Run az self-test ==" + docker run ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash -c "time az self-test && time az --version && tdnf list --installed" + + displayName: 'Run Simple Test' + + - bash: | + echo "== List docker image ==" + docker images + + echo "== Run unit test ==" + docker run --rm -v $(pwd):/azure-cli ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false + + displayName: 'Run Unit Test' diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 59047065994..55174ff7bc6 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -229,198 +229,23 @@ stages: displayName: 'Test Rpm Package Azure Linux' - - job: TestDockerImageSmokeTest - displayName: Test Docker Image - Smoke Test - condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) - timeoutInMinutes: 100 - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - pool: ${{ arch.pool }} - artifactName: docker-azurelinux3.0-${{ arch.value }} - architecture: ${{ arch.value }} - pool: - name: $(pool) - variables: + - template: templates/test-docker-image-smoke.yml + parameters: + architectures: ${{ parameters.architectures }} + ShouldTest: ${{parameters.ShouldTest}} + timeoutInMinutes: 100 AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Docker Image' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/docker' - artifactName: $(artifactName) - - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - - task: Bash@3 - displayName: Install PowerShell for ARM64 Agent - condition: and(succeeded(), eq(variables['architecture'], 'arm64')) - inputs: - targetType: 'inline' - script: | - curl -L -o /tmp/powershell.tar.gz https://github.com/PowerShell/PowerShell/releases/download/v7.4.11/powershell-7.4.11-linux-arm64.tar.gz - sudo mkdir -p /opt/microsoft/powershell/7 - sudo tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell/7 - sudo chmod +x /opt/microsoft/powershell/7/pwsh - sudo ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh - PATH=/usr/bin:/sbin:/usr/sbin:$PATH - - - task: Bash@3 - displayName: Install Az.Accounts for ARM64 Agents - condition: and(succeeded(), eq(variables['architecture'], 'arm64')) - inputs: - targetType: 'inline' - script: | - pwsh -command 'Register-PSRepository -Default -InstallationPolicy Trusted' - pwsh -command 'Install-Module -Name Az.Accounts -Repository PSGallery -AllowClobber -Force' - - - task: Bash@3 - displayName: Prepare Test and Output Folders - inputs: - targetType: 'inline' - script: | - mkdir output - chmod +x ./scripts/release/docker/test_az_cli_in_pipeline.sh - ls ./.azure-pipelines - - - task: Bash@3 - displayName: 🚀 Self Test - inputs: - targetType: 'inline' - script: | - echo "== Load docker image ==" - TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$(AzCliVersion).tar" - docker load < $TAR_FILE - echo "== Docker Images ==" - docker images - - echo "== Run az self-test ==" - docker run $(AzCliImageName):$(AzCliImageTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" - - #sudo docker run $(TestImage):$(TestImageTag) /bin/bash -c "time az self-test && time az --version && (tdnf list --installed || true) " - - - task: AzurePowerShell@5 - displayName: Get Azure Service Connection Info - inputs: - pwsh: true - azureSubscription: '$(TestAzureSubscription)' - ScriptType: 'InlineScript' - azurePowerShellVersion: 'LatestVersion' - Inline: | - $PSVersionTable - $AzAccessToken = (Get-AzAccessToken -AsSecureString).Token - $AzGraphToken = (Get-AzAccessToken -ResourceTypeName MSGraph -AsSecureString).Token - $ssPtr = [System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($AzAccessToken) - $ssPtrGraph = [System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($AzGraphToken) - try { - $AzAccessToken = [System.Runtime.InteropServices.Marshal]::PtrToStringBSTR($ssPtr) - $AzGraphToken = [System.Runtime.InteropServices.Marshal]::PtrToStringBSTR($ssPtrGraph) - } - finally { - [System.Runtime.InteropServices.Marshal]::ZeroFreeBSTR($ssPtr) - [System.Runtime.InteropServices.Marshal]::ZeroFreeBSTR($ssPtrGraph) - } - - $AzUserId = (Get-AzAccessToken).UserId - $AzSubscription = (Get-AzContext).Subscription - $AzTenant = (Get-AzContext).Tenant - - Write-Host "##vso[task.setvariable variable=AzAccessToken;issecret=true]$AzAccessToken" - Write-Host "##vso[task.setvariable variable=AzUserId;issecret=true]$AzUserId" - Write-Host "##vso[task.setvariable variable=AzSubscription;issecret=true]$AzSubscription" - Write-Host "##vso[task.setvariable variable=AzTenant;issecret=true]$AzTenant" - Write-Host "##vso[task.setvariable variable=AzGraphToken;issecret=true]$AzGraphToken" - - - task: PowerShell@2 - displayName: Get OIDC Token - inputs: - targetType: inline - pwsh: true - script: | - $token = "$(System.AccessToken)" - $header = @{authorization = "Bearer $token"} - $scUrl = "${env:SYSTEM_OIDCREQUESTURI}?api-version=7.1&serviceConnectionId=75a6a794-72e5-4618-b576-deea04f8b21d" - $result = Invoke-RestMethod -Uri $scUrl -Method Post -ContentType "application/json" -Headers $header - $OidcToken = $result.oidcToken - Write-Host "##vso[task.setvariable variable=OidcToken;issecret=true]$OidcToken" - - - task: Bash@3 - displayName: 🚀🚀 Smoke Test - inputs: - targetType: 'inline' - script: | - sudo docker run -v $(System.DefaultWorkingDirectory)/scripts/docker:/test $(AzCliImageName):$(AzCliImageTag) /bin/bash -c "az login --service-principal --username $(AzUserId) --tenant $(AzTenant) --federated-token $(OidcToken) && az account set --subscription $(AzSubscription) && /test/test_az_cli_in_pipeline.sh" - - echo "--------------------------------------------" - - mv $(System.DefaultWorkingDirectory)/scripts/docker/azure_cli_test_output.log $(System.DefaultWorkingDirectory)/output/azure_cli_test_output_before.log - mv $(System.DefaultWorkingDirectory)/scripts/docker/azure_cli_test_result.csv $(System.DefaultWorkingDirectory)/output/azure_cli_test_result_before.csv - ls $(System.DefaultWorkingDirectory)/output - - - publish: $(System.DefaultWorkingDirectory)/output - artifact: $(AzCliVersion)-$(architecture)-smoke-test - displayName: Publish Artifact - - - job: TestDockerImageUnitTest - displayName: Test Docker Image - Unit Test - condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) - timeoutInMinutes: 180 - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: - pool: ${{ arch.pool }} - artifactName: docker-azurelinux3.0-${{ arch.value }} - pool: - name: $(pool) - variables: + - template: templates/test-docker-image-unit.yml + parameters: + architectures: ${{ parameters.architectures }} + ShouldTest: ${{parameters.ShouldTest}} + timeoutInMinutes: 180 AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] - steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Docker Image' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/docker' - artifactName: $(artifactName) - - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - - task: Bash@3 - displayName: Create Test and Output Folders - inputs: - targetType: 'inline' - script: | - mkdir output - chmod +x ./scripts/release/docker/test_az_cli_in_pipeline.sh - ls ./.azure-pipelines - - - bash: | - set -exv - - echo "== Load docker image ==" - TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$(AzCliVersion).tar" - docker load < $TAR_FILE - - echo "== Run az self-test ==" - docker run $(AzCliImageName):$(AzCliImageTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" - - displayName: 'Run Simple Test' - - - bash: | - echo "== List docker image ==" - docker images - - echo "== Run unit test ==" - docker run --rm -v $(pwd):/azure-cli $(AzCliImageName):$(AzCliImageTag) /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false - - displayName: 'Run Unit Test' - stage: UseCopaToPatch displayName: Patch with Copacetic @@ -437,6 +262,7 @@ stages: Azure Linux 3.0 ${{ arch.name }}: pool: ${{ arch.pool }} artifactName: docker-azurelinux3.0-${{ arch.value }} + patchedArtifactName: docker-azurelinux3.0-${{ arch.value }}-patched architecture: ${{ arch.value }} pool: name: $(pool) @@ -496,6 +322,7 @@ stages: if [ "$(architecture)" = "arm64" ]; then CopaDownloadUrl="https://github.com/project-copacetic/copacetic/releases/download/v$(CopaVersion)/copa_$(CopaVersion)_linux_arm64.tar.gz" fi + echo "Download copa from: $CopaDownloadUrl" curl -L -o /tmp/copa.tar.gz $CopaDownloadUrl sudo mkdir -p /opt/copacetic sudo tar zxf /tmp/copa.tar.gz -C /opt/copacetic @@ -572,7 +399,10 @@ stages: export IMAGE=$(AzCliImageName):$(AzCliImageTag) echo "before copa patch-----------------------" - sudo COPA_EXPERIMENTAL=1 copa patch -i $IMAGE -r trivy-scan-before.json --pkg-types os,library --library-patch-level ${{parameters.PatchLevel}} --tag $(AzCliImagePatchedTag) + + # currently it failed; let's fake the patched image + # sudo COPA_EXPERIMENTAL=1 copa patch -i $IMAGE -r trivy-scan-before.json --pkg-types os,library --library-patch-level ${{parameters.PatchLevel}} --tag $(AzCliImagePatchedTag) + sudo docker tag $(AzCliImageName):$(AzCliImageTag) $(AzCliImageName):$(AzCliImagePatchedTag) echo "after copa patch-----------------------" @@ -602,6 +432,28 @@ stages: artifact: $(AzCliVersion)-$(architecture)-copa displayName: Publish Artifact + - task: Bash@3 + displayName: Save Docker Image + inputs: + targetType: 'inline' + script: | + echo "== Check Docker Images ==" + docker images + + echo "== Save Docker Image to tar file ==" + docker save -o "$BUILD_STAGINGDIRECTORY/docker-azure-cli-$(AzCliVersion)-patched.tar" $(AzCliImageName):$(AzCliImagePatchedTag) + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory) + DockerImagesToScan: $(AzCliImagePatchedTag) + + - task: PublishPipelineArtifact@0 + inputs: + TargetPath: $(Build.ArtifactStagingDirectory) + ArtifactName: $(patchedArtifactName) + - stage: Test2 displayName: 2nd Test - for Copa Patched Image dependsOn: From 2f20ad6c3c9ab75b1b911d71e3bbebb17c08604e Mon Sep 17 00:00:00 2001 From: YanaXu Date: Mon, 20 Oct 2025 11:24:50 +0800 Subject: [PATCH 28/52] fix a bug --- .azure-pipelines/templates/test-docker-image-smoke.yml | 10 +++++++--- .azure-pipelines/test-with-copa.yml | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines/templates/test-docker-image-smoke.yml b/.azure-pipelines/templates/test-docker-image-smoke.yml index 9353023c620..c0d2bc0d69a 100644 --- a/.azure-pipelines/templates/test-docker-image-smoke.yml +++ b/.azure-pipelines/templates/test-docker-image-smoke.yml @@ -4,9 +4,13 @@ parameters: - name: artifactNamePrefix type: string default: 'docker-azurelinux3.0' -- name: condition +- name: artifactNameSuffix type: string - default: 'succeeded()' + default: '' +- name: ShouldTest + displayName: Test? + type: boolean + default: false - name: timeoutInMinutes type: number default: 100 @@ -30,7 +34,7 @@ jobs: ${{ each arch in parameters.architectures }}: Azure Linux 3.0 ${{ arch.name }}: pool: ${{ arch.pool }} - artifactName: ${{ parameters.artifactNamePrefix }}-${{ arch.value }} + artifactName: ${{ parameters.artifactNamePrefix }}-${{ arch.value }}${{ parameters.artifactNameSuffix }} architecture: ${{ arch.value }} pool: name: $(pool) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 55174ff7bc6..9457168a62c 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -232,7 +232,7 @@ stages: - template: templates/test-docker-image-smoke.yml parameters: architectures: ${{ parameters.architectures }} - ShouldTest: ${{parameters.ShouldTest}} + ShouldTest: ${{ parameters.ShouldTest }} timeoutInMinutes: 100 AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] @@ -241,7 +241,7 @@ stages: - template: templates/test-docker-image-unit.yml parameters: architectures: ${{ parameters.architectures }} - ShouldTest: ${{parameters.ShouldTest}} + ShouldTest: ${{ parameters.ShouldTest }} timeoutInMinutes: 180 AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] From 3b8ca4b26435cb5c22e6eac107e0e9a7351def9f Mon Sep 17 00:00:00 2001 From: YanaXu Date: Mon, 20 Oct 2025 11:26:16 +0800 Subject: [PATCH 29/52] fix a bug 2 --- .azure-pipelines/templates/test-docker-image-smoke.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/templates/test-docker-image-smoke.yml b/.azure-pipelines/templates/test-docker-image-smoke.yml index c0d2bc0d69a..2453a939668 100644 --- a/.azure-pipelines/templates/test-docker-image-smoke.yml +++ b/.azure-pipelines/templates/test-docker-image-smoke.yml @@ -27,7 +27,7 @@ parameters: jobs: - job: TestDockerImageSmokeTest displayName: Test Docker Image - Smoke Test - condition: ${{ parameters.condition }} + condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) timeoutInMinutes: ${{ parameters.timeoutInMinutes }} strategy: matrix: From 791c459b6747f1ebd52faab09430c16858329d41 Mon Sep 17 00:00:00 2001 From: YanaXu Date: Mon, 20 Oct 2025 14:13:39 +0800 Subject: [PATCH 30/52] update variable --- .azure-pipelines/test-with-copa.yml | 49 +++++++++++++++-------------- 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 9457168a62c..291e960f95c 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -40,7 +40,7 @@ stages: steps: - checkout: none - task: Bash@3 - name: PrepareAzCliVersion + name: SetAzCliVersionTask displayName: Prepare Azure CLI Info inputs: targetType: 'inline' @@ -94,6 +94,10 @@ stages: displayName: Build RPM and Docker Images dependsOn: - PrepareInfo + variables: + AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliVersion'] ] + AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliImageName'] ] + AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliImageTag'] ] jobs: - job: BuildRpmPackagesAzureLinux displayName: Build Rpm Package @@ -149,10 +153,7 @@ stages: image: mcr.microsoft.com/azurelinux/base/core:3.0 pool: name: $(pool) - variables: - AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] - AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] - AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] + steps: - bash: ./scripts/ci/install_docker.sh displayName: Install Docker @@ -193,6 +194,10 @@ stages: dependsOn: - PrepareInfo - Build + variables: + AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliVersion'] ] + AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliImageName'] ] + AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliImageTag'] ] jobs: - job: TestRpmPackagesAzureLinux displayName: Test Rpm Package @@ -207,8 +212,6 @@ stages: image: mcr.microsoft.com/azurelinux/base/core:3.0 artifact: rpm-azurelinux3.0-${{ arch.value }} pool: ${{ arch.pool }} - variables: - AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Build Artifacts' @@ -234,24 +237,27 @@ stages: architectures: ${{ parameters.architectures }} ShouldTest: ${{ parameters.ShouldTest }} timeoutInMinutes: 100 - AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] - AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] - AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] + AzCliVersion: $(AzCliVersion) + AzCliImageName: $(AzCliImageName) + AzCliImageTag: $(AzCliImageTag) - template: templates/test-docker-image-unit.yml parameters: architectures: ${{ parameters.architectures }} ShouldTest: ${{ parameters.ShouldTest }} timeoutInMinutes: 180 - AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] - AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] - AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] - + AzCliVersion: $(AzCliVersion) + AzCliImageName: $(AzCliImageName) + AzCliImageTag: $(AzCliImageTag) - stage: UseCopaToPatch displayName: Patch with Copacetic dependsOn: - PrepareInfo - Build + variables: + AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliVersion'] ] + AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliImageName'] ] + AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliImageTag'] ] jobs: - job: BuildCopaPatchedDockerImageAzureLinux displayName: Build Copa Patched Docker Image @@ -266,11 +272,7 @@ stages: architecture: ${{ arch.value }} pool: name: $(pool) - variables: - AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] - AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] - AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] - AzCliImagePatchedTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImagePatchedTag'] ] + steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Docker Image' @@ -460,6 +462,10 @@ stages: - PrepareInfo - Build - UseCopaToPatch + variables: + AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliVersion'] ] + AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliImageName'] ] + AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliImageTag'] ] jobs: - job: TestCopaPatchedDockerImageAzureLinux displayName: Test Copa Patched Docker Image @@ -473,11 +479,6 @@ stages: architecture: ${{ arch.value }} pool: name: $(pool) - variables: - AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliVersion'] ] - AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageName'] ] - AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImageTag'] ] - AzCliImagePatchedTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersion.AzCliImagePatchedTag'] ] steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Docker Image' From a11432749e1d4d158bb437311728faa116242a92 Mon Sep 17 00:00:00 2001 From: YanaXu Date: Mon, 20 Oct 2025 15:06:27 +0800 Subject: [PATCH 31/52] refactor test --- .../templates/test-docker-image-smoke.yml | 10 +- .../templates/test-docker-image-unit.yml | 8 +- .azure-pipelines/test-with-copa.yml | 106 ++++++++++-------- 3 files changed, 70 insertions(+), 54 deletions(-) diff --git a/.azure-pipelines/templates/test-docker-image-smoke.yml b/.azure-pipelines/templates/test-docker-image-smoke.yml index 2453a939668..c6a275ddbb2 100644 --- a/.azure-pipelines/templates/test-docker-image-smoke.yml +++ b/.azure-pipelines/templates/test-docker-image-smoke.yml @@ -1,10 +1,10 @@ parameters: - name: architectures type: object -- name: artifactNamePrefix +- name: downloadArtifactNamePrefix type: string default: 'docker-azurelinux3.0' -- name: artifactNameSuffix +- name: downloadArtifactNameSuffix type: string default: '' - name: ShouldTest @@ -32,9 +32,9 @@ jobs: strategy: matrix: ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: + ${{ arch.name }}: pool: ${{ arch.pool }} - artifactName: ${{ parameters.artifactNamePrefix }}-${{ arch.value }}${{ parameters.artifactNameSuffix }} + downloadArtifactName: ${{ parameters.downloadArtifactNamePrefix }}-${{ arch.value }}${{ parameters.downloadArtifactNameSuffix }} architecture: ${{ arch.value }} pool: name: $(pool) @@ -43,7 +43,7 @@ jobs: displayName: 'Download Docker Image' inputs: TargetPath: '$(Build.ArtifactStagingDirectory)/docker' - artifactName: $(artifactName) + artifactName: $(downloadArtifactName) - bash: ./scripts/ci/install_docker.sh displayName: Install Docker diff --git a/.azure-pipelines/templates/test-docker-image-unit.yml b/.azure-pipelines/templates/test-docker-image-unit.yml index 8b7925473d0..347ddc6659c 100644 --- a/.azure-pipelines/templates/test-docker-image-unit.yml +++ b/.azure-pipelines/templates/test-docker-image-unit.yml @@ -1,10 +1,10 @@ parameters: - name: architectures type: object -- name: artifactNamePrefix +- name: downloadArtifactNamePrefix type: string default: 'docker-azurelinux3.0' -- name: artifactNameSuffix +- name: downloadArtifactNameSuffix type: string default: '' - name: ShouldTest @@ -32,9 +32,9 @@ jobs: strategy: matrix: ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: + ${{ arch.name }}: pool: ${{ arch.pool }} - downloadArtifactName: ${{ parameters.artifactNamePrefix }}-${{ arch.value }}${{ parameters.artifactNameSuffix }} + downloadArtifactName: ${{ parameters.downloadArtifactNamePrefix }}-${{ arch.value }}${{ parameters.downloadArtifactNameSuffix }} pool: name: $(pool) steps: diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 291e960f95c..066d3110290 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -40,7 +40,7 @@ stages: steps: - checkout: none - task: Bash@3 - name: SetAzCliVersionTask + name: PrepareAzCliVersionTask displayName: Prepare Azure CLI Info inputs: targetType: 'inline' @@ -95,18 +95,18 @@ stages: dependsOn: - PrepareInfo variables: - AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliVersion'] ] - AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliImageName'] ] - AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliImageTag'] ] + AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliVersion'] ] + AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImageName'] ] + AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImageTag'] ] jobs: - job: BuildRpmPackagesAzureLinux displayName: Build Rpm Package strategy: matrix: ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: + ${{ arch.name }}: image: mcr.microsoft.com/azurelinux/base/core:3.0 - artifact: rpm-azurelinux3.0-${{ arch.value }} + uploadArtifact: rpm-azurelinux3.0-${{ arch.value }} pool: ${{ arch.pool }} pool: name: $(pool) @@ -136,7 +136,7 @@ stages: displayName: 'Publish Artifact: rpm-azurelinux' inputs: TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(artifact) + ArtifactName: $(uploadArtifact) - job: BuildDockerImageAzureLinux displayName: Build Docker Image @@ -145,23 +145,24 @@ stages: strategy: matrix: ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: + ${{ arch.name }}: pool: ${{ arch.pool }} - artifactName: docker-azurelinux3.0-${{ arch.value }} + downloadArtifactName: rpm-azurelinux3.0-${{ arch.value }} + uploadArtifactName: docker-azurelinux3.0-${{ arch.value }} dockerfile: azure-linux.dockerfile - packageArtifactName: rpm-azurelinux3.0-${{ arch.value }} image: mcr.microsoft.com/azurelinux/base/core:3.0 pool: name: $(pool) - steps: - bash: ./scripts/ci/install_docker.sh displayName: Install Docker + - task: DownloadPipelineArtifact@1 displayName: 'Download Build Artifacts' inputs: TargetPath: '$(Build.ArtifactStagingDirectory)/docker' - artifactName: $(packageArtifactName) + artifactName: $(downloadArtifactName) + - bash: | set -ex mkdir docker-temp @@ -187,7 +188,7 @@ stages: - task: PublishPipelineArtifact@0 inputs: TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(artifactName) + ArtifactName: $(uploadArtifactName) - stage: Test1 displayName: 1st Test - for Original Image @@ -195,9 +196,9 @@ stages: - PrepareInfo - Build variables: - AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliVersion'] ] - AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliImageName'] ] - AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliImageTag'] ] + AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliVersion'] ] + AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImageName'] ] + AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImageTag'] ] jobs: - job: TestRpmPackagesAzureLinux displayName: Test Rpm Package @@ -208,16 +209,16 @@ stages: strategy: matrix: ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: + ${{ arch.name }}: image: mcr.microsoft.com/azurelinux/base/core:3.0 - artifact: rpm-azurelinux3.0-${{ arch.value }} + downloadArtifactName: rpm-azurelinux3.0-${{ arch.value }} pool: ${{ arch.pool }} steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Build Artifacts' inputs: TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' - artifactName: $(artifact) + artifactName: $(downloadArtifactName) - bash: ./scripts/ci/install_docker.sh displayName: Install Docker @@ -229,7 +230,6 @@ stages: echo "== Test rpm package on ${IMAGE} ==" docker pull $IMAGE docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh - displayName: 'Test Rpm Package Azure Linux' - template: templates/test-docker-image-smoke.yml @@ -249,15 +249,17 @@ stages: AzCliVersion: $(AzCliVersion) AzCliImageName: $(AzCliImageName) AzCliImageTag: $(AzCliImageTag) -- stage: UseCopaToPatch + +- stage: PatchWithCopa displayName: Patch with Copacetic dependsOn: - PrepareInfo - Build variables: - AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliVersion'] ] - AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliImageName'] ] - AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliImageTag'] ] + AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliVersion'] ] + AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImageName'] ] + AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImageTag'] ] + AzCliImagePatchedTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImagePatchedTag'] ] jobs: - job: BuildCopaPatchedDockerImageAzureLinux displayName: Build Copa Patched Docker Image @@ -265,10 +267,10 @@ stages: strategy: matrix: ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: + ${{ arch.name }}: pool: ${{ arch.pool }} - artifactName: docker-azurelinux3.0-${{ arch.value }} - patchedArtifactName: docker-azurelinux3.0-${{ arch.value }}-patched + downloadArtifactName: docker-azurelinux3.0-${{ arch.value }} + uploadArtifactName: docker-azurelinux3.0-${{ arch.value }}-patched architecture: ${{ arch.value }} pool: name: $(pool) @@ -278,7 +280,7 @@ stages: displayName: 'Download Docker Image' inputs: TargetPath: '$(Build.ArtifactStagingDirectory)/docker' - artifactName: $(artifactName) + artifactName: $(downloadArtifactName) - bash: ./scripts/ci/install_docker.sh displayName: Install Docker @@ -432,7 +434,7 @@ stages: - publish: $(System.DefaultWorkingDirectory)/output artifact: $(AzCliVersion)-$(architecture)-copa - displayName: Publish Artifact + displayName: Publish trivy and copa Artifact - task: Bash@3 displayName: Save Docker Image @@ -443,7 +445,7 @@ stages: docker images echo "== Save Docker Image to tar file ==" - docker save -o "$BUILD_STAGINGDIRECTORY/docker-azure-cli-$(AzCliVersion)-patched.tar" $(AzCliImageName):$(AzCliImagePatchedTag) + docker save -o "$BUILD_STAGINGDIRECTORY/docker-azure-cli-$(AzCliVersion).tar" $(AzCliImageName):$(AzCliImagePatchedTag) - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 displayName: 'SBOM' @@ -452,39 +454,53 @@ stages: DockerImagesToScan: $(AzCliImagePatchedTag) - task: PublishPipelineArtifact@0 + displayName: Publish docker Artifact inputs: TargetPath: $(Build.ArtifactStagingDirectory) - ArtifactName: $(patchedArtifactName) + ArtifactName: $(uploadArtifactName) - stage: Test2 displayName: 2nd Test - for Copa Patched Image dependsOn: - PrepareInfo - Build - - UseCopaToPatch + - PatchWithCopa variables: - AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliVersion'] ] - AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliImageName'] ] - AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['SetAzCliVersionTask.AzCliImageTag'] ] + AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliVersion'] ] + AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImageName'] ] + AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImageTag'] ] + AzCliImagePatchedTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImagePatchedTag'] ] jobs: - - job: TestCopaPatchedDockerImageAzureLinux + - job: TestCopaPatchedDockerImage displayName: Test Copa Patched Docker Image timeoutInMinutes: 180 strategy: matrix: ${{ each arch in parameters.architectures }}: - Azure Linux 3.0 ${{ arch.name }}: + ${{ arch.name }}: pool: ${{ arch.pool }} - artifactName: docker-azurelinux3.0-${{ arch.value }} + downloadArtifactName: docker-azurelinux3.0-${{ arch.value }}-patched architecture: ${{ arch.value }} pool: name: $(pool) steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Docker Image' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/docker' - artifactName: $(artifactName) + - template: templates/test-docker-image-smoke.yml + parameters: + architectures: ${{ parameters.architectures }} + ShouldTest: ${{ parameters.ShouldTest }} + downloadArtifactNameSuffix: '-patched' + timeoutInMinutes: 100 + AzCliVersion: $(AzCliVersion) + AzCliImageName: $(AzCliImageName) + AzCliImageTag: $(AzCliImagePatchedTag) + - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker + - template: templates/test-docker-image-unit.yml + parameters: + architectures: ${{ parameters.architectures }} + ShouldTest: ${{ parameters.ShouldTest }} + downloadArtifactNameSuffix: '-patched' + timeoutInMinutes: 180 + AzCliVersion: $(AzCliVersion) + AzCliImageName: $(AzCliImageName) + AzCliImageTag: $(AzCliImagePatchedTag) From 58032e174db7a2d493f9164c2b9323df2ba2dff5 Mon Sep 17 00:00:00 2001 From: YanaXu Date: Mon, 20 Oct 2025 15:10:31 +0800 Subject: [PATCH 32/52] format --- .azure-pipelines/test-with-copa.yml | 38 ++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 066d3110290..8853ba59777 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -484,23 +484,23 @@ stages: pool: name: $(pool) steps: - - template: templates/test-docker-image-smoke.yml - parameters: - architectures: ${{ parameters.architectures }} - ShouldTest: ${{ parameters.ShouldTest }} - downloadArtifactNameSuffix: '-patched' - timeoutInMinutes: 100 - AzCliVersion: $(AzCliVersion) - AzCliImageName: $(AzCliImageName) - AzCliImageTag: $(AzCliImagePatchedTag) - + - template: templates/test-docker-image-smoke.yml + parameters: + architectures: ${{ parameters.architectures }} + ShouldTest: ${{ parameters.ShouldTest }} + downloadArtifactNameSuffix: '-patched' + timeoutInMinutes: 100 + AzCliVersion: $(AzCliVersion) + AzCliImageName: $(AzCliImageName) + AzCliImageTag: $(AzCliImagePatchedTag) + - - template: templates/test-docker-image-unit.yml - parameters: - architectures: ${{ parameters.architectures }} - ShouldTest: ${{ parameters.ShouldTest }} - downloadArtifactNameSuffix: '-patched' - timeoutInMinutes: 180 - AzCliVersion: $(AzCliVersion) - AzCliImageName: $(AzCliImageName) - AzCliImageTag: $(AzCliImagePatchedTag) + - template: templates/test-docker-image-unit.yml + parameters: + architectures: ${{ parameters.architectures }} + ShouldTest: ${{ parameters.ShouldTest }} + downloadArtifactNameSuffix: '-patched' + timeoutInMinutes: 180 + AzCliVersion: $(AzCliVersion) + AzCliImageName: $(AzCliImageName) + AzCliImageTag: $(AzCliImagePatchedTag) From 6dab4408f8a2482c2a3a870fa0cea1defdd73707 Mon Sep 17 00:00:00 2001 From: YanaXu Date: Mon, 20 Oct 2025 15:12:28 +0800 Subject: [PATCH 33/52] remove not used jobs --- .azure-pipelines/templates/test-docker-image-smoke.yml | 1 - .azure-pipelines/templates/test-docker-image-unit.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.azure-pipelines/templates/test-docker-image-smoke.yml b/.azure-pipelines/templates/test-docker-image-smoke.yml index c6a275ddbb2..defb7440cc3 100644 --- a/.azure-pipelines/templates/test-docker-image-smoke.yml +++ b/.azure-pipelines/templates/test-docker-image-smoke.yml @@ -24,7 +24,6 @@ parameters: type: string default: '' -jobs: - job: TestDockerImageSmokeTest displayName: Test Docker Image - Smoke Test condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) diff --git a/.azure-pipelines/templates/test-docker-image-unit.yml b/.azure-pipelines/templates/test-docker-image-unit.yml index 347ddc6659c..0418270b52c 100644 --- a/.azure-pipelines/templates/test-docker-image-unit.yml +++ b/.azure-pipelines/templates/test-docker-image-unit.yml @@ -24,7 +24,6 @@ parameters: type: string default: '' -jobs: - job: TestDockerImageUnitTest displayName: Test Docker Image - Unit Test condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) From 97f824b889c4e425b217167fd3420cdcc73d6600 Mon Sep 17 00:00:00 2001 From: YanaXu Date: Mon, 20 Oct 2025 15:18:12 +0800 Subject: [PATCH 34/52] revert --- .azure-pipelines/templates/test-docker-image-smoke.yml | 1 + .azure-pipelines/templates/test-docker-image-unit.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.azure-pipelines/templates/test-docker-image-smoke.yml b/.azure-pipelines/templates/test-docker-image-smoke.yml index defb7440cc3..c6a275ddbb2 100644 --- a/.azure-pipelines/templates/test-docker-image-smoke.yml +++ b/.azure-pipelines/templates/test-docker-image-smoke.yml @@ -24,6 +24,7 @@ parameters: type: string default: '' +jobs: - job: TestDockerImageSmokeTest displayName: Test Docker Image - Smoke Test condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) diff --git a/.azure-pipelines/templates/test-docker-image-unit.yml b/.azure-pipelines/templates/test-docker-image-unit.yml index 0418270b52c..347ddc6659c 100644 --- a/.azure-pipelines/templates/test-docker-image-unit.yml +++ b/.azure-pipelines/templates/test-docker-image-unit.yml @@ -24,6 +24,7 @@ parameters: type: string default: '' +jobs: - job: TestDockerImageUnitTest displayName: Test Docker Image - Unit Test condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) From c350614f04eedd9e894808699a16131be46ea4b7 Mon Sep 17 00:00:00 2001 From: YanaXu Date: Mon, 20 Oct 2025 15:20:25 +0800 Subject: [PATCH 35/52] format --- .azure-pipelines/test-with-copa.yml | 50 +++++++++++------------------ 1 file changed, 18 insertions(+), 32 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 8853ba59777..c9e6e8633ed 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -471,36 +471,22 @@ stages: AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImageTag'] ] AzCliImagePatchedTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImagePatchedTag'] ] jobs: - - job: TestCopaPatchedDockerImage - displayName: Test Copa Patched Docker Image - timeoutInMinutes: 180 - strategy: - matrix: - ${{ each arch in parameters.architectures }}: - ${{ arch.name }}: - pool: ${{ arch.pool }} - downloadArtifactName: docker-azurelinux3.0-${{ arch.value }}-patched - architecture: ${{ arch.value }} - pool: - name: $(pool) - steps: - - template: templates/test-docker-image-smoke.yml - parameters: - architectures: ${{ parameters.architectures }} - ShouldTest: ${{ parameters.ShouldTest }} - downloadArtifactNameSuffix: '-patched' - timeoutInMinutes: 100 - AzCliVersion: $(AzCliVersion) - AzCliImageName: $(AzCliImageName) - AzCliImageTag: $(AzCliImagePatchedTag) - + - template: templates/test-docker-image-smoke.yml + parameters: + architectures: ${{ parameters.architectures }} + ShouldTest: ${{ parameters.ShouldTest }} + downloadArtifactNameSuffix: '-patched' + timeoutInMinutes: 100 + AzCliVersion: $(AzCliVersion) + AzCliImageName: $(AzCliImageName) + AzCliImageTag: $(AzCliImagePatchedTag) - - template: templates/test-docker-image-unit.yml - parameters: - architectures: ${{ parameters.architectures }} - ShouldTest: ${{ parameters.ShouldTest }} - downloadArtifactNameSuffix: '-patched' - timeoutInMinutes: 180 - AzCliVersion: $(AzCliVersion) - AzCliImageName: $(AzCliImageName) - AzCliImageTag: $(AzCliImagePatchedTag) + - template: templates/test-docker-image-unit.yml + parameters: + architectures: ${{ parameters.architectures }} + ShouldTest: ${{ parameters.ShouldTest }} + downloadArtifactNameSuffix: '-patched' + timeoutInMinutes: 180 + AzCliVersion: $(AzCliVersion) + AzCliImageName: $(AzCliImageName) + AzCliImageTag: $(AzCliImagePatchedTag) From e952c8c9a95c6b8e85317ddc6196d20d2b584210 Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Tue, 21 Oct 2025 11:27:49 +0800 Subject: [PATCH 36/52] remove powershell usage --- .../templates/test-docker-image-smoke.yml | 85 ++++--------------- .azure-pipelines/test-with-copa.yml | 60 ++++++++----- 2 files changed, 55 insertions(+), 90 deletions(-) diff --git a/.azure-pipelines/templates/test-docker-image-smoke.yml b/.azure-pipelines/templates/test-docker-image-smoke.yml index c6a275ddbb2..fab70d69458 100644 --- a/.azure-pipelines/templates/test-docker-image-smoke.yml +++ b/.azure-pipelines/templates/test-docker-image-smoke.yml @@ -23,6 +23,18 @@ parameters: - name: AzCliImageTag type: string default: '' +- name: TestAzUserId + type: string + default: '' +- name: TestAzSubscription + type: string + default: '' +- name: TestAzTenant + type: string + default: '' +- name: TestServiceConnectionId + type: string + default: '' jobs: - job: TestDockerImageSmokeTest @@ -48,28 +60,6 @@ jobs: - bash: ./scripts/ci/install_docker.sh displayName: Install Docker - - task: Bash@3 - displayName: Install PowerShell for ARM64 Agent - condition: and(succeeded(), eq(variables['architecture'], 'arm64')) - inputs: - targetType: 'inline' - script: | - curl -L -o /tmp/powershell.tar.gz https://github.com/PowerShell/PowerShell/releases/download/v7.4.11/powershell-7.4.11-linux-arm64.tar.gz - sudo mkdir -p /opt/microsoft/powershell/7 - sudo tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell/7 - sudo chmod +x /opt/microsoft/powershell/7/pwsh - sudo ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh - PATH=/usr/bin:/sbin:/usr/sbin:$PATH - - - task: Bash@3 - displayName: Install Az.Accounts for ARM64 Agents - condition: and(succeeded(), eq(variables['architecture'], 'arm64')) - inputs: - targetType: 'inline' - script: | - pwsh -command 'Register-PSRepository -Default -InstallationPolicy Trusted' - pwsh -command 'Install-Module -Name Az.Accounts -Repository PSGallery -AllowClobber -Force' - - task: Bash@3 displayName: Prepare Test and Output Folders inputs: @@ -96,57 +86,16 @@ jobs: #sudo docker run $(TestImage):$(TestImageTag) /bin/bash -c "time az self-test && time az --version && (tdnf list --installed || true) " - - task: AzurePowerShell@5 - displayName: Get Azure Service Connection Info - inputs: - pwsh: true - azureSubscription: '$(TestAzureSubscription)' - ScriptType: 'InlineScript' - azurePowerShellVersion: 'LatestVersion' - Inline: | - $PSVersionTable - $AzAccessToken = (Get-AzAccessToken -AsSecureString).Token - $AzGraphToken = (Get-AzAccessToken -ResourceTypeName MSGraph -AsSecureString).Token - $ssPtr = [System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($AzAccessToken) - $ssPtrGraph = [System.Runtime.InteropServices.Marshal]::SecureStringToBSTR($AzGraphToken) - try { - $AzAccessToken = [System.Runtime.InteropServices.Marshal]::PtrToStringBSTR($ssPtr) - $AzGraphToken = [System.Runtime.InteropServices.Marshal]::PtrToStringBSTR($ssPtrGraph) - } - finally { - [System.Runtime.InteropServices.Marshal]::ZeroFreeBSTR($ssPtr) - [System.Runtime.InteropServices.Marshal]::ZeroFreeBSTR($ssPtrGraph) - } - - $AzUserId = (Get-AzAccessToken).UserId - $AzSubscription = (Get-AzContext).Subscription - $AzTenant = (Get-AzContext).Tenant - - Write-Host "##vso[task.setvariable variable=AzAccessToken;issecret=true]$AzAccessToken" - Write-Host "##vso[task.setvariable variable=AzUserId;issecret=true]$AzUserId" - Write-Host "##vso[task.setvariable variable=AzSubscription;issecret=true]$AzSubscription" - Write-Host "##vso[task.setvariable variable=AzTenant;issecret=true]$AzTenant" - Write-Host "##vso[task.setvariable variable=AzGraphToken;issecret=true]$AzGraphToken" - - - task: PowerShell@2 - displayName: Get OIDC Token - inputs: - targetType: inline - pwsh: true - script: | - $token = "$(System.AccessToken)" - $header = @{authorization = "Bearer $token"} - $scUrl = "${env:SYSTEM_OIDCREQUESTURI}?api-version=7.1&serviceConnectionId=75a6a794-72e5-4618-b576-deea04f8b21d" - $result = Invoke-RestMethod -Uri $scUrl -Method Post -ContentType "application/json" -Headers $header - $OidcToken = $result.oidcToken - Write-Host "##vso[task.setvariable variable=OidcToken;issecret=true]$OidcToken" - - task: Bash@3 displayName: 🚀🚀 Smoke Test inputs: targetType: 'inline' script: | - sudo docker run -v $(System.DefaultWorkingDirectory)/scripts/docker:/test ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash -c "az login --service-principal --username $(AzUserId) --tenant $(AzTenant) --federated-token $(OidcToken) && az account set --subscription $(AzSubscription) && /test/test_az_cli_in_pipeline.sh" + scUrl="${SYSTEM_OIDCREQUESTURI}?api-version=7.1&serviceConnectionId=$(ServiceConnectionId)" + response=$(curl -s -X POST "$scUrl" -H "Authorization: Bearer $(System.AccessToken)" -H "Content-Type: application/json" -d '') + OidcToken=$(echo "$response" | jq -r '.oidcToken') + + sudo docker run -v $(System.DefaultWorkingDirectory)/scripts/docker:/test ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash -c "az login --service-principal --username $(AzUserId) --tenant $(AzTenant) --federated-token $OidcToken && az account set --subscription $(AzSubscription) && /test/test_az_cli_in_pipeline.sh" echo "--------------------------------------------" diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index c9e6e8633ed..8d2b9222c31 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -90,6 +90,32 @@ stages: } Write-Host "##vso[build.updatebuildnumber]$newBuildName" + - task: AzureCLI@2 + name: PrepareServiceConnectionInfoTask + displayName: Get Azure Service Connection Info + inputs: + azureSubscription: '$(TestAzureSubscription)' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + az account show + + # Get user, subscription and tenant info + AzUserId=$(az account show --query user.name -o tsv) + AzSubscription=$(az account show --query id -o tsv) + AzTenant=$(az account show --query tenantId -o tsv) + + echo "Azure User ID: $AzUserId" + echo "Azure Subscription: $AzSubscription" + echo "Azure Tenant: $AzTenant" + echo "Azure Service Connection Id: $AZURESUBSCRIPTION_SERVICE_CONNECTION_ID" + + # Set pipeline variables as secrets + echo "##vso[task.setvariable variable=AzUserId;issecret=true;isOutput=true]$AzUserId" + echo "##vso[task.setvariable variable=AzSubscription;issecret=true;isOutput=true]$AzSubscription" + echo "##vso[task.setvariable variable=AzTenant;issecret=true;isOutput=true]$AzTenant" + echo "##vso[task.setvariable variable=ServiceConnectionId;isOutput=true]$AZURESUBSCRIPTION_SERVICE_CONNECTION_ID" + - stage: Build displayName: Build RPM and Docker Images dependsOn: @@ -199,6 +225,10 @@ stages: AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliVersion'] ] AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImageName'] ] AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImageTag'] ] + TestAzUserId: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareServiceConnectionInfoTask.AzUserId'] ] + TestAzSubscription: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareServiceConnectionInfoTask.AzSubscription'] ] + TestAzTenant: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareServiceConnectionInfoTask.AzTenant'] ] + TestServiceConnectionId: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareServiceConnectionInfoTask.ServiceConnectionId'] ] jobs: - job: TestRpmPackagesAzureLinux displayName: Test Rpm Package @@ -240,6 +270,10 @@ stages: AzCliVersion: $(AzCliVersion) AzCliImageName: $(AzCliImageName) AzCliImageTag: $(AzCliImageTag) + TestAzUserId: $(TestAzUserId) + TestAzSubscription: $(TestAzSubscription) + TestAzTenant: $(TestAzTenant) + TestServiceConnectionId: $(TestServiceConnectionId) - template: templates/test-docker-image-unit.yml parameters: @@ -285,28 +319,6 @@ stages: - bash: ./scripts/ci/install_docker.sh displayName: Install Docker - - task: Bash@3 - displayName: Install PowerShell for ARM64 Agent - condition: and(succeeded(), eq(variables['architecture'], 'arm64')) - inputs: - targetType: 'inline' - script: | - curl -L -o /tmp/powershell.tar.gz https://github.com/PowerShell/PowerShell/releases/download/v7.4.11/powershell-7.4.11-linux-arm64.tar.gz - sudo mkdir -p /opt/microsoft/powershell/7 - sudo tar zxf /tmp/powershell.tar.gz -C /opt/microsoft/powershell/7 - sudo chmod +x /opt/microsoft/powershell/7/pwsh - sudo ln -s /opt/microsoft/powershell/7/pwsh /usr/bin/pwsh - PATH=/usr/bin:/sbin:/usr/sbin:$PATH - - - task: Bash@3 - displayName: Install Az.Accounts for ARM64 Agents - condition: and(succeeded(), eq(variables['architecture'], 'arm64')) - inputs: - targetType: 'inline' - script: | - pwsh -command 'Register-PSRepository -Default -InstallationPolicy Trusted' - pwsh -command 'Install-Module -Name Az.Accounts -Repository PSGallery -AllowClobber -Force' - - task: Bash@3 displayName: Install Trivy inputs: @@ -480,6 +492,10 @@ stages: AzCliVersion: $(AzCliVersion) AzCliImageName: $(AzCliImageName) AzCliImageTag: $(AzCliImagePatchedTag) + TestAzUserId: $(TestAzUserId) + TestAzSubscription: $(TestAzSubscription) + TestAzTenant: $(TestAzTenant) + TestServiceConnectionId: $(TestServiceConnectionId) - template: templates/test-docker-image-unit.yml parameters: From 4ab5b53a1589928919f813702184767a7eaed2b6 Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Tue, 21 Oct 2025 11:30:04 +0800 Subject: [PATCH 37/52] format --- .azure-pipelines/test-with-copa.yml | 50 ++++++++++++++--------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 8d2b9222c31..3fcc2ac5689 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -90,31 +90,31 @@ stages: } Write-Host "##vso[build.updatebuildnumber]$newBuildName" - - task: AzureCLI@2 - name: PrepareServiceConnectionInfoTask - displayName: Get Azure Service Connection Info - inputs: - azureSubscription: '$(TestAzureSubscription)' - scriptType: 'bash' - scriptLocation: 'inlineScript' - inlineScript: | - az account show - - # Get user, subscription and tenant info - AzUserId=$(az account show --query user.name -o tsv) - AzSubscription=$(az account show --query id -o tsv) - AzTenant=$(az account show --query tenantId -o tsv) - - echo "Azure User ID: $AzUserId" - echo "Azure Subscription: $AzSubscription" - echo "Azure Tenant: $AzTenant" - echo "Azure Service Connection Id: $AZURESUBSCRIPTION_SERVICE_CONNECTION_ID" - - # Set pipeline variables as secrets - echo "##vso[task.setvariable variable=AzUserId;issecret=true;isOutput=true]$AzUserId" - echo "##vso[task.setvariable variable=AzSubscription;issecret=true;isOutput=true]$AzSubscription" - echo "##vso[task.setvariable variable=AzTenant;issecret=true;isOutput=true]$AzTenant" - echo "##vso[task.setvariable variable=ServiceConnectionId;isOutput=true]$AZURESUBSCRIPTION_SERVICE_CONNECTION_ID" + - task: AzureCLI@2 + name: PrepareServiceConnectionInfoTask + displayName: Get Azure Service Connection Info + inputs: + azureSubscription: '$(TestAzureSubscription)' + scriptType: 'bash' + scriptLocation: 'inlineScript' + inlineScript: | + az account show + + # Get user, subscription and tenant info + AzUserId=$(az account show --query user.name -o tsv) + AzSubscription=$(az account show --query id -o tsv) + AzTenant=$(az account show --query tenantId -o tsv) + + echo "Azure User ID: $AzUserId" + echo "Azure Subscription: $AzSubscription" + echo "Azure Tenant: $AzTenant" + echo "Azure Service Connection Id: $AZURESUBSCRIPTION_SERVICE_CONNECTION_ID" + + # Set pipeline variables as secrets + echo "##vso[task.setvariable variable=AzUserId;issecret=true;isOutput=true]$AzUserId" + echo "##vso[task.setvariable variable=AzSubscription;issecret=true;isOutput=true]$AzSubscription" + echo "##vso[task.setvariable variable=AzTenant;issecret=true;isOutput=true]$AzTenant" + echo "##vso[task.setvariable variable=ServiceConnectionId;isOutput=true]$AZURESUBSCRIPTION_SERVICE_CONNECTION_ID" - stage: Build displayName: Build RPM and Docker Images From 64717f153c1a9cceb3a09314f7d1bb59f820a065 Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Tue, 21 Oct 2025 13:13:09 +0800 Subject: [PATCH 38/52] fix variable --- .azure-pipelines/templates/test-docker-image-smoke.yml | 4 ++-- .azure-pipelines/test-with-copa.yml | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.azure-pipelines/templates/test-docker-image-smoke.yml b/.azure-pipelines/templates/test-docker-image-smoke.yml index fab70d69458..8b9de9331e9 100644 --- a/.azure-pipelines/templates/test-docker-image-smoke.yml +++ b/.azure-pipelines/templates/test-docker-image-smoke.yml @@ -91,11 +91,11 @@ jobs: inputs: targetType: 'inline' script: | - scUrl="${SYSTEM_OIDCREQUESTURI}?api-version=7.1&serviceConnectionId=$(ServiceConnectionId)" + scUrl="${SYSTEM_OIDCREQUESTURI}?api-version=7.1&serviceConnectionId=${{parameters.TestServiceConnectionId}}" response=$(curl -s -X POST "$scUrl" -H "Authorization: Bearer $(System.AccessToken)" -H "Content-Type: application/json" -d '') OidcToken=$(echo "$response" | jq -r '.oidcToken') - sudo docker run -v $(System.DefaultWorkingDirectory)/scripts/docker:/test ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash -c "az login --service-principal --username $(AzUserId) --tenant $(AzTenant) --federated-token $OidcToken && az account set --subscription $(AzSubscription) && /test/test_az_cli_in_pipeline.sh" + sudo docker run -v $(System.DefaultWorkingDirectory)/scripts/docker:/test ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash -c "az login --service-principal --username ${{parameters.TestAzUserId}} --tenant ${{parameters.TestAzTenant}} --federated-token $OidcToken && az account set --subscription ${{parameters.TestAzSubscription}} && /test/test_az_cli_in_pipeline.sh" echo "--------------------------------------------" diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 3fcc2ac5689..9e2e1be83aa 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -482,6 +482,10 @@ stages: AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImageName'] ] AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImageTag'] ] AzCliImagePatchedTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImagePatchedTag'] ] + TestAzUserId: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareServiceConnectionInfoTask.AzUserId'] ] + TestAzSubscription: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareServiceConnectionInfoTask.AzSubscription'] ] + TestAzTenant: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareServiceConnectionInfoTask.AzTenant'] ] + TestServiceConnectionId: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareServiceConnectionInfoTask.ServiceConnectionId'] ] jobs: - template: templates/test-docker-image-smoke.yml parameters: From 672aa97a79efd233ce823380d494bf0b753466bc Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Tue, 21 Oct 2025 16:02:13 +0800 Subject: [PATCH 39/52] fix smoke test --- .../templates/test-docker-image-smoke.yml | 11 +-- .azure-pipelines/test-with-copa.yml | 82 ++++++++----------- 2 files changed, 40 insertions(+), 53 deletions(-) diff --git a/.azure-pipelines/templates/test-docker-image-smoke.yml b/.azure-pipelines/templates/test-docker-image-smoke.yml index 8b9de9331e9..c20ddf7a70f 100644 --- a/.azure-pipelines/templates/test-docker-image-smoke.yml +++ b/.azure-pipelines/templates/test-docker-image-smoke.yml @@ -67,7 +67,6 @@ jobs: script: | mkdir output chmod +x ./scripts/release/docker/test_az_cli_in_pipeline.sh - ls ./.azure-pipelines - task: Bash@3 displayName: 🚀 Self Test @@ -84,8 +83,6 @@ jobs: echo "== Run az self-test ==" docker run ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash -c "time az self-test && time az --version && tdnf list --installed" - #sudo docker run $(TestImage):$(TestImageTag) /bin/bash -c "time az self-test && time az --version && (tdnf list --installed || true) " - - task: Bash@3 displayName: 🚀🚀 Smoke Test inputs: @@ -95,14 +92,14 @@ jobs: response=$(curl -s -X POST "$scUrl" -H "Authorization: Bearer $(System.AccessToken)" -H "Content-Type: application/json" -d '') OidcToken=$(echo "$response" | jq -r '.oidcToken') - sudo docker run -v $(System.DefaultWorkingDirectory)/scripts/docker:/test ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash -c "az login --service-principal --username ${{parameters.TestAzUserId}} --tenant ${{parameters.TestAzTenant}} --federated-token $OidcToken && az account set --subscription ${{parameters.TestAzSubscription}} && /test/test_az_cli_in_pipeline.sh" + docker run -v $(System.DefaultWorkingDirectory)/scripts/release/docker:/test ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash -c "az login --service-principal --username ${{parameters.TestAzUserId}} --tenant ${{parameters.TestAzTenant}} --federated-token $OidcToken && az account set --subscription ${{parameters.TestAzSubscription}} && /test/test_az_cli_in_pipeline.sh" echo "--------------------------------------------" - mv $(System.DefaultWorkingDirectory)/scripts/docker/azure_cli_test_output.log $(System.DefaultWorkingDirectory)/output/azure_cli_test_output_before.log - mv $(System.DefaultWorkingDirectory)/scripts/docker/azure_cli_test_result.csv $(System.DefaultWorkingDirectory)/output/azure_cli_test_result_before.csv + mv $(System.DefaultWorkingDirectory)/scripts/release/docker/azure_cli_test_output.log $(System.DefaultWorkingDirectory)/output/azure_cli_test_output.log + mv $(System.DefaultWorkingDirectory)/scripts/release/docker/azure_cli_test_result.csv $(System.DefaultWorkingDirectory)/output/azure_cli_test_result.csv ls $(System.DefaultWorkingDirectory)/output - publish: $(System.DefaultWorkingDirectory)/output - artifact: ${{ parameters.AzCliVersion }}-$(architecture)-smoke-test + artifact: ${{ parameters.AzCliVersion }}-$(architecture)-smoke-test${{ parameters.downloadArtifactNameSuffix }} displayName: Publish Artifact diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 9e2e1be83aa..3add79ed90b 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -295,7 +295,7 @@ stages: AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImageTag'] ] AzCliImagePatchedTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImagePatchedTag'] ] jobs: - - job: BuildCopaPatchedDockerImageAzureLinux + - job: BuildCopaPatchedDockerImage displayName: Build Copa Patched Docker Image timeoutInMinutes: 180 strategy: @@ -318,6 +318,28 @@ stages: - bash: ./scripts/ci/install_docker.sh displayName: Install Docker + + - task: Bash@3 + displayName: Configure Docker Daemon + inputs: + targetType: 'inline' + script: | + if [ -f /etc/docker/daemon.json ]; then + echo "Docker daemon.json exists, printing content:" + cat /etc/docker/daemon.json + else + echo "Docker daemon.json does not exist, creating it..." + sudo mkdir -p /etc/docker + echo '{"features": { "containerd-snapshotter": true }}' | sudo tee /etc/docker/daemon.json + echo "Created /etc/docker/daemon.json with containerd-snapshotter enabled" + fi + + - task: Bash@3 + displayName: Restart Docker + inputs: + targetType: 'inline' + script: | + sudo systemctl restart docker - task: Bash@3 displayName: Install Trivy @@ -327,7 +349,7 @@ stages: set -eux curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/bin echo "----------------------------------" - sudo trivy --version + trivy --version - task: Bash@3 displayName: Install copacetic @@ -345,13 +367,13 @@ stages: sudo chmod +x /opt/copacetic/copa sudo ln -s /opt/copacetic/copa /usr/bin/copa - echo "----------------------------------" - sudo copa --version - echo "----------------------------------" - sudo copa --help - echo "----------------------------------" - sudo copa patch --help - echo "----------------------------------" + echo "---------------- copa --version ------------------" + copa --version + echo "---------------- copa --help ------------------" + copa --help + echo "---------------- copa patch --help ------------------" + copa patch --help + echo "-----------------------------------------------------" - task: Bash@3 displayName: Check Docker Image @@ -378,49 +400,17 @@ stages: mkdir output export IMAGE=$(AzCliImageName):$(AzCliImageTag) OutputFile="trivy-scan-before.json" - sudo trivy image --pkg-types os,library --ignore-unfixed -f json -o $OutputFile $IMAGE + trivy image --pkg-types os,library --ignore-unfixed -f json -o $OutputFile $IMAGE cp $OutputFile ./output/ - - task: Bash@3 - displayName: Setup Docker BuildKit - inputs: - targetType: 'inline' - script: | - set -eux - # Enable Docker BuildKit - export DOCKER_BUILDKIT=1 - export BUILDKIT_PROGRESS=plain - - # Check Docker version and BuildKit support - sudo docker version - sudo docker buildx version || echo "buildx not available" - - # Try to create and use a new builder with containerd image store - sudo docker buildx create --name copa-builder --driver docker-container --use || true - sudo docker buildx inspect --bootstrap || echo "Failed to bootstrap buildx" - - # Alternative: Enable experimental features in Docker daemon - echo "Checking Docker daemon configuration..." - sudo docker info | grep -i experimental || echo "Experimental features not enabled" - - task: Bash@3 displayName: Patch with Copacetic inputs: targetType: 'inline' script: | - export DOCKER_BUILDKIT=1 - export BUILDKIT_PROGRESS=plain - export COPA_EXPERIMENTAL=1 export IMAGE=$(AzCliImageName):$(AzCliImageTag) - - echo "before copa patch-----------------------" - - # currently it failed; let's fake the patched image - # sudo COPA_EXPERIMENTAL=1 copa patch -i $IMAGE -r trivy-scan-before.json --pkg-types os,library --library-patch-level ${{parameters.PatchLevel}} --tag $(AzCliImagePatchedTag) - sudo docker tag $(AzCliImageName):$(AzCliImageTag) $(AzCliImageName):$(AzCliImagePatchedTag) - - echo "after copa patch-----------------------" + copa patch -i $IMAGE -r trivy-scan-before.json --pkg-types os,library --library-patch-level ${{parameters.PatchLevel}} --tag $(AzCliImagePatchedTag) - task: Bash@3 displayName: Check Docker Image @@ -441,12 +431,12 @@ stages: mkdir output export IMAGE=$(AzCliImageName):$(AzCliImagePatchedTag) OutputFile="trivy-scan-after.json" - sudo trivy image --pkg-types os,library --ignore-unfixed -f json -o $OutputFile $IMAGE + trivy image --pkg-types os,library --ignore-unfixed -f json -o $OutputFile $IMAGE cp $OutputFile ./output/ - publish: $(System.DefaultWorkingDirectory)/output - artifact: $(AzCliVersion)-$(architecture)-copa - displayName: Publish trivy and copa Artifact + artifact: $(AzCliVersion)-$(architecture)-trivy + displayName: Publish trivy Artifact - task: Bash@3 displayName: Save Docker Image From 272774839007d301a1f5f822cf538d2cdda809fe Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Tue, 21 Oct 2025 16:28:33 +0800 Subject: [PATCH 40/52] workaround unit test --- .azure-pipelines/templates/test-docker-image-unit.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/templates/test-docker-image-unit.yml b/.azure-pipelines/templates/test-docker-image-unit.yml index 347ddc6659c..2b18545f2d7 100644 --- a/.azure-pipelines/templates/test-docker-image-unit.yml +++ b/.azure-pipelines/templates/test-docker-image-unit.yml @@ -73,6 +73,6 @@ jobs: docker images echo "== Run unit test ==" - docker run --rm -v $(pwd):/azure-cli ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false + docker run --rm -e AZ_INSTALLER='' -v $(pwd):/azure-cli ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false displayName: 'Run Unit Test' From 2aecff3b76aad018042e0693a42fe31ec20b653e Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Tue, 21 Oct 2025 16:41:37 +0800 Subject: [PATCH 41/52] fix folder --- .azure-pipelines/test-with-copa.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 3add79ed90b..1e213d7e541 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -428,7 +428,6 @@ stages: inputs: targetType: 'inline' script: | - mkdir output export IMAGE=$(AzCliImageName):$(AzCliImagePatchedTag) OutputFile="trivy-scan-after.json" trivy image --pkg-types os,library --ignore-unfixed -f json -o $OutputFile $IMAGE From ececaf16b36b3138159e6adf856f44c5d7588f2f Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Wed, 22 Oct 2025 10:21:06 +0800 Subject: [PATCH 42/52] switch the steps --- .../templates/test-docker-image-smoke.yml | 2 +- .../templates/test-docker-image-unit.yml | 6 +-- .azure-pipelines/test-with-copa.yml | 40 ++++++++++--------- 3 files changed, 25 insertions(+), 23 deletions(-) diff --git a/.azure-pipelines/templates/test-docker-image-smoke.yml b/.azure-pipelines/templates/test-docker-image-smoke.yml index c20ddf7a70f..4e18ef4b271 100644 --- a/.azure-pipelines/templates/test-docker-image-smoke.yml +++ b/.azure-pipelines/templates/test-docker-image-smoke.yml @@ -38,7 +38,7 @@ parameters: jobs: - job: TestDockerImageSmokeTest - displayName: Test Docker Image - Smoke Test + displayName: Smoke Test - Docker Image condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) timeoutInMinutes: ${{ parameters.timeoutInMinutes }} strategy: diff --git a/.azure-pipelines/templates/test-docker-image-unit.yml b/.azure-pipelines/templates/test-docker-image-unit.yml index 2b18545f2d7..90091a55a40 100644 --- a/.azure-pipelines/templates/test-docker-image-unit.yml +++ b/.azure-pipelines/templates/test-docker-image-unit.yml @@ -26,7 +26,7 @@ parameters: jobs: - job: TestDockerImageUnitTest - displayName: Test Docker Image - Unit Test + displayName: Unit Test - Docker Image condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) timeoutInMinutes: ${{ parameters.timeoutInMinutes }} strategy: @@ -66,7 +66,7 @@ jobs: echo "== Run az self-test ==" docker run ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash -c "time az self-test && time az --version && tdnf list --installed" - displayName: 'Run Simple Test' + displayName: '🚀 Run Simple Test' - bash: | echo "== List docker image ==" @@ -75,4 +75,4 @@ jobs: echo "== Run unit test ==" docker run --rm -e AZ_INSTALLER='' -v $(pwd):/azure-cli ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false - displayName: 'Run Unit Test' + displayName: '🚀🚀 Run Unit Test' diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 1e213d7e541..d8771a5a9c2 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -31,17 +31,17 @@ parameters: stages: - stage: PrepareInfo - displayName: Prepare Azure CLI Info + displayName: Prepare Build Info jobs: - job: PrepareAzCliVersion - displayName: Prepare Azure CLI Info + displayName: Prepare Build Info pool: name: ${{ variables.ubuntu_pool }} steps: - checkout: none - task: Bash@3 name: PrepareAzCliVersionTask - displayName: Prepare Azure CLI Info + displayName: Prepare Build Info inputs: targetType: 'inline' script: | @@ -126,7 +126,7 @@ stages: AzCliImageTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImageTag'] ] jobs: - job: BuildRpmPackagesAzureLinux - displayName: Build Rpm Package + displayName: Build Rpm strategy: matrix: ${{ each arch in parameters.architectures }}: @@ -231,7 +231,7 @@ stages: TestServiceConnectionId: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareServiceConnectionInfoTask.ServiceConnectionId'] ] jobs: - job: TestRpmPackagesAzureLinux - displayName: Test Rpm Package + displayName: Test - Rpm Package timeoutInMinutes: 180 condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule'), eq(${{parameters.ShouldTest}}, true)) pool: @@ -423,20 +423,6 @@ stages: echo "== Run az self-test ==" docker run $(AzCliImageName):$(AzCliImagePatchedTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" - - task: Bash@3 - displayName: Scan with Trivy again - inputs: - targetType: 'inline' - script: | - export IMAGE=$(AzCliImageName):$(AzCliImagePatchedTag) - OutputFile="trivy-scan-after.json" - trivy image --pkg-types os,library --ignore-unfixed -f json -o $OutputFile $IMAGE - cp $OutputFile ./output/ - - - publish: $(System.DefaultWorkingDirectory)/output - artifact: $(AzCliVersion)-$(architecture)-trivy - displayName: Publish trivy Artifact - - task: Bash@3 displayName: Save Docker Image inputs: @@ -460,6 +446,22 @@ stages: TargetPath: $(Build.ArtifactStagingDirectory) ArtifactName: $(uploadArtifactName) + - task: Bash@3 + displayName: Scan with Trivy again + continueOnError: true + inputs: + targetType: 'inline' + script: | + set -eux + export IMAGE=$(AzCliImageName):$(AzCliImagePatchedTag) + OutputFile="trivy-scan-after.json" + trivy image --pkg-types os,library --ignore-unfixed -f json -o $OutputFile $IMAGE + cp $OutputFile ./output/ + + - publish: $(System.DefaultWorkingDirectory)/output + artifact: $(AzCliVersion)-$(architecture)-trivy + displayName: Publish trivy Artifact + - stage: Test2 displayName: 2nd Test - for Copa Patched Image dependsOn: From 8cfeded6af496c9ff2cca2439006c3dd21c171ba Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Wed, 22 Oct 2025 10:58:58 +0800 Subject: [PATCH 43/52] update display name --- .../templates/test-docker-image-smoke.yml | 21 +++------- .../templates/test-docker-image-unit.yml | 14 +------ .azure-pipelines/test-with-copa.yml | 39 ++++++++++--------- 3 files changed, 28 insertions(+), 46 deletions(-) diff --git a/.azure-pipelines/templates/test-docker-image-smoke.yml b/.azure-pipelines/templates/test-docker-image-smoke.yml index 4e18ef4b271..c285bd7f573 100644 --- a/.azure-pipelines/templates/test-docker-image-smoke.yml +++ b/.azure-pipelines/templates/test-docker-image-smoke.yml @@ -1,14 +1,10 @@ parameters: - name: architectures type: object -- name: downloadArtifactNamePrefix - type: string - default: 'docker-azurelinux3.0' - name: downloadArtifactNameSuffix type: string default: '' - name: ShouldTest - displayName: Test? type: boolean default: false - name: timeoutInMinutes @@ -46,7 +42,8 @@ jobs: ${{ each arch in parameters.architectures }}: ${{ arch.name }}: pool: ${{ arch.pool }} - downloadArtifactName: ${{ parameters.downloadArtifactNamePrefix }}-${{ arch.value }}${{ parameters.downloadArtifactNameSuffix }} + downloadArtifactName: ${{ parameters.AzCliVersion }}-docker-${{ arch.value }}${{ parameters.downloadArtifactNameSuffix }} + uploadArtifactName: ${{ parameters.AzCliVersion }}-docker-${{ arch.value }}-smoke-test${{ parameters.downloadArtifactNameSuffix }} architecture: ${{ arch.value }} pool: name: $(pool) @@ -60,14 +57,6 @@ jobs: - bash: ./scripts/ci/install_docker.sh displayName: Install Docker - - task: Bash@3 - displayName: Prepare Test and Output Folders - inputs: - targetType: 'inline' - script: | - mkdir output - chmod +x ./scripts/release/docker/test_az_cli_in_pipeline.sh - - task: Bash@3 displayName: 🚀 Self Test inputs: @@ -92,14 +81,16 @@ jobs: response=$(curl -s -X POST "$scUrl" -H "Authorization: Bearer $(System.AccessToken)" -H "Content-Type: application/json" -d '') OidcToken=$(echo "$response" | jq -r '.oidcToken') + chmod +x ./scripts/release/docker/test_az_cli_in_pipeline.sh docker run -v $(System.DefaultWorkingDirectory)/scripts/release/docker:/test ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash -c "az login --service-principal --username ${{parameters.TestAzUserId}} --tenant ${{parameters.TestAzTenant}} --federated-token $OidcToken && az account set --subscription ${{parameters.TestAzSubscription}} && /test/test_az_cli_in_pipeline.sh" echo "--------------------------------------------" + mkdir output mv $(System.DefaultWorkingDirectory)/scripts/release/docker/azure_cli_test_output.log $(System.DefaultWorkingDirectory)/output/azure_cli_test_output.log mv $(System.DefaultWorkingDirectory)/scripts/release/docker/azure_cli_test_result.csv $(System.DefaultWorkingDirectory)/output/azure_cli_test_result.csv ls $(System.DefaultWorkingDirectory)/output - publish: $(System.DefaultWorkingDirectory)/output - artifact: ${{ parameters.AzCliVersion }}-$(architecture)-smoke-test${{ parameters.downloadArtifactNameSuffix }} - displayName: Publish Artifact + artifact: $(uploadArtifactName) + displayName: Publish Test Artifact diff --git a/.azure-pipelines/templates/test-docker-image-unit.yml b/.azure-pipelines/templates/test-docker-image-unit.yml index 90091a55a40..767b13f58b2 100644 --- a/.azure-pipelines/templates/test-docker-image-unit.yml +++ b/.azure-pipelines/templates/test-docker-image-unit.yml @@ -8,7 +8,6 @@ parameters: type: string default: '' - name: ShouldTest - displayName: Test? type: boolean default: false - name: timeoutInMinutes @@ -34,7 +33,7 @@ jobs: ${{ each arch in parameters.architectures }}: ${{ arch.name }}: pool: ${{ arch.pool }} - downloadArtifactName: ${{ parameters.downloadArtifactNamePrefix }}-${{ arch.value }}${{ parameters.downloadArtifactNameSuffix }} + downloadArtifactName: ${{ parameters.AzCliVersion }}-docker-${{ arch.value }}${{ parameters.downloadArtifactNameSuffix }} pool: name: $(pool) steps: @@ -47,15 +46,6 @@ jobs: - bash: ./scripts/ci/install_docker.sh displayName: Install Docker - - task: Bash@3 - displayName: Create Test and Output Folders - inputs: - targetType: 'inline' - script: | - mkdir output - chmod +x ./scripts/release/docker/test_az_cli_in_pipeline.sh - ls ./.azure-pipelines - - bash: | set -exv @@ -66,7 +56,7 @@ jobs: echo "== Run az self-test ==" docker run ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash -c "time az self-test && time az --version && tdnf list --installed" - displayName: '🚀 Run Simple Test' + displayName: '🚀 Run Self Test' - bash: | echo "== List docker image ==" diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index d8771a5a9c2..3d4a2162175 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -92,7 +92,7 @@ stages: - task: AzureCLI@2 name: PrepareServiceConnectionInfoTask - displayName: Get Azure Service Connection Info + displayName: Get Test Azure Service Connection Info inputs: azureSubscription: '$(TestAzureSubscription)' scriptType: 'bash' @@ -132,7 +132,7 @@ stages: ${{ each arch in parameters.architectures }}: ${{ arch.name }}: image: mcr.microsoft.com/azurelinux/base/core:3.0 - uploadArtifact: rpm-azurelinux3.0-${{ arch.value }} + uploadArtifact: $(AzCliVersion)-rpm-${{ arch.value }} pool: ${{ arch.pool }} pool: name: $(pool) @@ -159,7 +159,7 @@ stages: BuildDropPath: $(Build.ArtifactStagingDirectory) - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: rpm-azurelinux' + displayName: 'Publish RPM Artifact' inputs: TargetPath: $(Build.ArtifactStagingDirectory) ArtifactName: $(uploadArtifact) @@ -173,8 +173,8 @@ stages: ${{ each arch in parameters.architectures }}: ${{ arch.name }}: pool: ${{ arch.pool }} - downloadArtifactName: rpm-azurelinux3.0-${{ arch.value }} - uploadArtifactName: docker-azurelinux3.0-${{ arch.value }} + downloadArtifactName: $(AzCliVersion)-rpm-${{ arch.value }} + uploadArtifactName: $(AzCliVersion)-docker-${{ arch.value }} dockerfile: azure-linux.dockerfile image: mcr.microsoft.com/azurelinux/base/core:3.0 pool: @@ -184,7 +184,7 @@ stages: displayName: Install Docker - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' + displayName: 'Download RPM Package' inputs: TargetPath: '$(Build.ArtifactStagingDirectory)/docker' artifactName: $(downloadArtifactName) @@ -201,8 +201,8 @@ stages: $BUILD_SOURCESDIRECTORY docker save -o "$BUILD_STAGINGDIRECTORY/docker-azure-cli-$(AzCliVersion).tar" $(AzCliImageName):$(AzCliImageTag) - - displayName: 'Build Docker' + + displayName: 'Build Docker Image' - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 displayName: 'SBOM' @@ -212,6 +212,7 @@ stages: DockerImagesToScan: $(AzCliDockerImageTag) - task: PublishPipelineArtifact@0 + displayName: Publish docker Artifact inputs: TargetPath: $(Build.ArtifactStagingDirectory) ArtifactName: $(uploadArtifactName) @@ -241,11 +242,11 @@ stages: ${{ each arch in parameters.architectures }}: ${{ arch.name }}: image: mcr.microsoft.com/azurelinux/base/core:3.0 - downloadArtifactName: rpm-azurelinux3.0-${{ arch.value }} + downloadArtifactName: $(AzCliVersion)-rpm-${{ arch.value }} pool: ${{ arch.pool }} steps: - task: DownloadPipelineArtifact@1 - displayName: 'Download Build Artifacts' + displayName: 'Download RPM Artifacts' inputs: TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' artifactName: $(downloadArtifactName) @@ -260,7 +261,7 @@ stages: echo "== Test rpm package on ${IMAGE} ==" docker pull $IMAGE docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh - displayName: 'Test Rpm Package Azure Linux' + displayName: 'Test Rpm Package' - template: templates/test-docker-image-smoke.yml parameters: @@ -285,7 +286,7 @@ stages: AzCliImageTag: $(AzCliImageTag) - stage: PatchWithCopa - displayName: Patch with Copacetic + displayName: Patch with Copa dependsOn: - PrepareInfo - Build @@ -303,8 +304,8 @@ stages: ${{ each arch in parameters.architectures }}: ${{ arch.name }}: pool: ${{ arch.pool }} - downloadArtifactName: docker-azurelinux3.0-${{ arch.value }} - uploadArtifactName: docker-azurelinux3.0-${{ arch.value }}-patched + downloadArtifactName: $(AzCliVersion)-docker-${{ arch.value }} + uploadArtifactName: $(AzCliVersion)-docker-${{ arch.value }}-patched architecture: ${{ arch.value }} pool: name: $(pool) @@ -352,7 +353,7 @@ stages: trivy --version - task: Bash@3 - displayName: Install copacetic + displayName: Install Copa inputs: targetType: 'inline' script: | @@ -404,7 +405,7 @@ stages: cp $OutputFile ./output/ - task: Bash@3 - displayName: Patch with Copacetic + displayName: Patch with Copa inputs: targetType: 'inline' script: | @@ -413,7 +414,7 @@ stages: copa patch -i $IMAGE -r trivy-scan-before.json --pkg-types os,library --library-patch-level ${{parameters.PatchLevel}} --tag $(AzCliImagePatchedTag) - task: Bash@3 - displayName: Check Docker Image + displayName: Check Patched Docker Image inputs: targetType: 'inline' script: | @@ -424,7 +425,7 @@ stages: docker run $(AzCliImageName):$(AzCliImagePatchedTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" - task: Bash@3 - displayName: Save Docker Image + displayName: Save Patched Docker Image inputs: targetType: 'inline' script: | @@ -460,7 +461,7 @@ stages: - publish: $(System.DefaultWorkingDirectory)/output artifact: $(AzCliVersion)-$(architecture)-trivy - displayName: Publish trivy Artifact + displayName: Publish Trivy Artifact - stage: Test2 displayName: 2nd Test - for Copa Patched Image From 82d0d637db5fd10c23a2ae4d6434324b6d7b2290 Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Wed, 22 Oct 2025 13:58:57 +0800 Subject: [PATCH 44/52] update artifact folder --- .azure-pipelines/test-with-copa.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 3d4a2162175..12d2ca20e12 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -186,13 +186,13 @@ stages: - task: DownloadPipelineArtifact@1 displayName: 'Download RPM Package' inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/docker' + TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' artifactName: $(downloadArtifactName) - bash: | set -ex mkdir docker-temp - mv $(Build.ArtifactStagingDirectory)/docker/*.rpm ./docker-temp/azure-cli.rpm + mv $(Build.ArtifactStagingDirectory)/rpm/*.rpm ./docker-temp/azure-cli.rpm docker build --no-cache \ --build-arg IMAGE=$IMAGE \ @@ -433,18 +433,19 @@ stages: docker images echo "== Save Docker Image to tar file ==" - docker save -o "$BUILD_STAGINGDIRECTORY/docker-azure-cli-$(AzCliVersion).tar" $(AzCliImageName):$(AzCliImagePatchedTag) + mkdir "$BUILD_STAGINGDIRECTORY/copa" + docker save -o "$BUILD_STAGINGDIRECTORY/copa/docker-azure-cli-$(AzCliVersion).tar" $(AzCliImageName):$(AzCliImagePatchedTag) - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 displayName: 'SBOM' inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) + BuildDropPath: $(Build.ArtifactStagingDirectory)/copa DockerImagesToScan: $(AzCliImagePatchedTag) - task: PublishPipelineArtifact@0 displayName: Publish docker Artifact inputs: - TargetPath: $(Build.ArtifactStagingDirectory) + TargetPath: $(Build.ArtifactStagingDirectory)/copa ArtifactName: $(uploadArtifactName) - task: Bash@3 From a5557e4cdcb160a3250f99e1b60c98ed3b48cfdb Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Wed, 22 Oct 2025 14:48:24 +0800 Subject: [PATCH 45/52] workaround unit test --- .azure-pipelines/templates/test-docker-image-unit.yml | 2 +- .azure-pipelines/test-with-copa.yml | 1 + scripts/release/rpm/test_azurelinux_in_docker.sh | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/templates/test-docker-image-unit.yml b/.azure-pipelines/templates/test-docker-image-unit.yml index 767b13f58b2..2e76050fa64 100644 --- a/.azure-pipelines/templates/test-docker-image-unit.yml +++ b/.azure-pipelines/templates/test-docker-image-unit.yml @@ -63,6 +63,6 @@ jobs: docker images echo "== Run unit test ==" - docker run --rm -e AZ_INSTALLER='' -v $(pwd):/azure-cli ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false + docker run --rm -v $(pwd):/azure-cli ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false displayName: '🚀🚀 Run Unit Test' diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 12d2ca20e12..f35d54a3a49 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -398,6 +398,7 @@ stages: inputs: targetType: 'inline' script: | + set -eux mkdir output export IMAGE=$(AzCliImageName):$(AzCliImageTag) OutputFile="trivy-scan-before.json" diff --git a/scripts/release/rpm/test_azurelinux_in_docker.sh b/scripts/release/rpm/test_azurelinux_in_docker.sh index 1d0f8e80708..84c9a1c4e21 100644 --- a/scripts/release/rpm/test_azurelinux_in_docker.sh +++ b/scripts/release/rpm/test_azurelinux_in_docker.sh @@ -18,6 +18,8 @@ while [[ $# -gt 0 ]]; do esac done +unset AZ_INSTALLER + export USERNAME=azureuser # Install RPM package only if InstallRPM is true From 2fbfcca487b76b2bdce60946fb5f1c5b36a979c2 Mon Sep 17 00:00:00 2001 From: Yan Xu Date: Wed, 22 Oct 2025 15:57:04 +0800 Subject: [PATCH 46/52] fix artifact folder --- .azure-pipelines/test-with-copa.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index f35d54a3a49..8d6de23cf99 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -200,21 +200,21 @@ stages: --file $DOCKERFILE \ $BUILD_SOURCESDIRECTORY - docker save -o "$BUILD_STAGINGDIRECTORY/docker-azure-cli-$(AzCliVersion).tar" $(AzCliImageName):$(AzCliImageTag) + mkdir $(Build.ArtifactStagingDirectory)/docker + docker save -o "$(Build.ArtifactStagingDirectory)/docker/docker-azure-cli-$(AzCliVersion).tar" $(AzCliImageName):$(AzCliImageTag) displayName: 'Build Docker Image' - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 displayName: 'SBOM' - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory) + BuildDropPath: $(Build.ArtifactStagingDirectory)/docker DockerImagesToScan: $(AzCliDockerImageTag) - task: PublishPipelineArtifact@0 displayName: Publish docker Artifact inputs: - TargetPath: $(Build.ArtifactStagingDirectory) + TargetPath: $(Build.ArtifactStagingDirectory)/docker ArtifactName: $(uploadArtifactName) - stage: Test1 @@ -419,11 +419,11 @@ stages: inputs: targetType: 'inline' script: | - echo "== Check Docker Images ==" - docker images + echo "== Check Docker Images ==" + docker images - echo "== Run az self-test ==" - docker run $(AzCliImageName):$(AzCliImagePatchedTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" + echo "== Run az self-test ==" + docker run $(AzCliImageName):$(AzCliImagePatchedTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" - task: Bash@3 displayName: Save Patched Docker Image From 4a5f42b3e2b99fe5bdba8651b7608e41b80a5e72 Mon Sep 17 00:00:00 2001 From: YanaXu Date: Thu, 23 Oct 2025 12:47:09 +0800 Subject: [PATCH 47/52] yml format --- .../templates/test-docker-image-unit.yml | 50 ++- .azure-pipelines/test-with-copa.yml | 373 +++++++++--------- 2 files changed, 210 insertions(+), 213 deletions(-) diff --git a/.azure-pipelines/templates/test-docker-image-unit.yml b/.azure-pipelines/templates/test-docker-image-unit.yml index 2e76050fa64..98d85670252 100644 --- a/.azure-pipelines/templates/test-docker-image-unit.yml +++ b/.azure-pipelines/templates/test-docker-image-unit.yml @@ -37,32 +37,30 @@ jobs: pool: name: $(pool) steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Docker Image' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/docker' - artifactName: $(downloadArtifactName) + - task: DownloadPipelineArtifact@1 + displayName: 'Download Docker Image' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/docker' + artifactName: $(downloadArtifactName) - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker - - bash: | - set -exv + - bash: | + set -exv + + echo "== Load docker image ==" + TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-${{ parameters.AzCliVersion }}.tar" + docker load < $TAR_FILE + + echo "== Run az self-test ==" + docker run ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash -c "time az self-test && time az --version && tdnf list --installed" + displayName: '🚀 Run Self Test' - echo "== Load docker image ==" - TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-${{ parameters.AzCliVersion }}.tar" - docker load < $TAR_FILE - - echo "== Run az self-test ==" - docker run ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash -c "time az self-test && time az --version && tdnf list --installed" - - displayName: '🚀 Run Self Test' - - - bash: | - echo "== List docker image ==" - docker images - - echo "== Run unit test ==" - docker run --rm -v $(pwd):/azure-cli ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false - - displayName: '🚀🚀 Run Unit Test' + - bash: | + echo "== List docker image ==" + docker images + + echo "== Run unit test ==" + docker run --rm -v $(pwd):/azure-cli ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false + displayName: '🚀🚀 Run Unit Test' diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 8d6de23cf99..9381fb0115d 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -25,9 +25,9 @@ parameters: displayName: library-patch-level? default: "patch" values: - - patch - - minor - - major + - patch + - minor + - major stages: - stage: PrepareInfo @@ -180,42 +180,41 @@ stages: pool: name: $(pool) steps: - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker - - - task: DownloadPipelineArtifact@1 - displayName: 'Download RPM Package' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' - artifactName: $(downloadArtifactName) - - - bash: | - set -ex - mkdir docker-temp - mv $(Build.ArtifactStagingDirectory)/rpm/*.rpm ./docker-temp/azure-cli.rpm - - docker build --no-cache \ - --build-arg IMAGE=$IMAGE \ - --tag $(AzCliImageName):$(AzCliImageTag) \ - --file $DOCKERFILE \ - $BUILD_SOURCESDIRECTORY + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker - mkdir $(Build.ArtifactStagingDirectory)/docker - docker save -o "$(Build.ArtifactStagingDirectory)/docker/docker-azure-cli-$(AzCliVersion).tar" $(AzCliImageName):$(AzCliImageTag) + - task: DownloadPipelineArtifact@1 + displayName: 'Download RPM Package' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/rpm' + artifactName: $(downloadArtifactName) - displayName: 'Build Docker Image' + - bash: | + set -ex + mkdir docker-temp + mv $(Build.ArtifactStagingDirectory)/rpm/*.rpm ./docker-temp/azure-cli.rpm + + docker build --no-cache \ + --build-arg IMAGE=$IMAGE \ + --tag $(AzCliImageName):$(AzCliImageTag) \ + --file $DOCKERFILE \ + $BUILD_SOURCESDIRECTORY - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory)/docker - DockerImagesToScan: $(AzCliDockerImageTag) + mkdir $(Build.ArtifactStagingDirectory)/docker + docker save -o "$(Build.ArtifactStagingDirectory)/docker/docker-azure-cli-$(AzCliVersion).tar" $(AzCliImageName):$(AzCliImageTag) + displayName: 'Build Docker Image' - - task: PublishPipelineArtifact@0 - displayName: Publish docker Artifact - inputs: - TargetPath: $(Build.ArtifactStagingDirectory)/docker - ArtifactName: $(uploadArtifactName) + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory)/docker + DockerImagesToScan: $(AzCliDockerImageTag) + + - task: PublishPipelineArtifact@0 + displayName: Publish docker Artifact + inputs: + TargetPath: $(Build.ArtifactStagingDirectory)/docker + ArtifactName: $(uploadArtifactName) - stage: Test1 displayName: 1st Test - for Original Image @@ -311,159 +310,159 @@ stages: name: $(pool) steps: - - task: DownloadPipelineArtifact@1 - displayName: 'Download Docker Image' - inputs: - TargetPath: '$(Build.ArtifactStagingDirectory)/docker' - artifactName: $(downloadArtifactName) - - - bash: ./scripts/ci/install_docker.sh - displayName: Install Docker + - task: DownloadPipelineArtifact@1 + displayName: 'Download Docker Image' + inputs: + TargetPath: '$(Build.ArtifactStagingDirectory)/docker' + artifactName: $(downloadArtifactName) + + - bash: ./scripts/ci/install_docker.sh + displayName: Install Docker - - task: Bash@3 - displayName: Configure Docker Daemon - inputs: - targetType: 'inline' - script: | - if [ -f /etc/docker/daemon.json ]; then - echo "Docker daemon.json exists, printing content:" - cat /etc/docker/daemon.json - else - echo "Docker daemon.json does not exist, creating it..." - sudo mkdir -p /etc/docker - echo '{"features": { "containerd-snapshotter": true }}' | sudo tee /etc/docker/daemon.json - echo "Created /etc/docker/daemon.json with containerd-snapshotter enabled" - fi - - - task: Bash@3 - displayName: Restart Docker - inputs: - targetType: 'inline' - script: | - sudo systemctl restart docker - - - task: Bash@3 - displayName: Install Trivy - inputs: - targetType: 'inline' - script: | - set -eux - curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/bin - echo "----------------------------------" - trivy --version - - - task: Bash@3 - displayName: Install Copa - inputs: - targetType: 'inline' - script: | - CopaDownloadUrl="https://github.com/project-copacetic/copacetic/releases/download/v$(CopaVersion)/copa_$(CopaVersion)_linux_amd64.tar.gz" - if [ "$(architecture)" = "arm64" ]; then - CopaDownloadUrl="https://github.com/project-copacetic/copacetic/releases/download/v$(CopaVersion)/copa_$(CopaVersion)_linux_arm64.tar.gz" - fi - echo "Download copa from: $CopaDownloadUrl" - curl -L -o /tmp/copa.tar.gz $CopaDownloadUrl - sudo mkdir -p /opt/copacetic - sudo tar zxf /tmp/copa.tar.gz -C /opt/copacetic - sudo chmod +x /opt/copacetic/copa - sudo ln -s /opt/copacetic/copa /usr/bin/copa - - echo "---------------- copa --version ------------------" - copa --version - echo "---------------- copa --help ------------------" - copa --help - echo "---------------- copa patch --help ------------------" - copa patch --help - echo "-----------------------------------------------------" - - - task: Bash@3 - displayName: Check Docker Image - inputs: - targetType: 'inline' - script: | - TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$(AzCliVersion).tar" - docker load < $TAR_FILE - - echo "== Check Docker Images ==" - docker images - - echo "== docker container ==" - docker ps - - echo "== Run az self-test ==" - docker run $(AzCliImageName):$(AzCliImageTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" - - - task: Bash@3 - displayName: Scan with Trivy - inputs: - targetType: 'inline' - script: | - set -eux - mkdir output - export IMAGE=$(AzCliImageName):$(AzCliImageTag) - OutputFile="trivy-scan-before.json" - trivy image --pkg-types os,library --ignore-unfixed -f json -o $OutputFile $IMAGE - cp $OutputFile ./output/ - - - task: Bash@3 - displayName: Patch with Copa - inputs: - targetType: 'inline' - script: | - export COPA_EXPERIMENTAL=1 - export IMAGE=$(AzCliImageName):$(AzCliImageTag) - copa patch -i $IMAGE -r trivy-scan-before.json --pkg-types os,library --library-patch-level ${{parameters.PatchLevel}} --tag $(AzCliImagePatchedTag) - - - task: Bash@3 - displayName: Check Patched Docker Image - inputs: - targetType: 'inline' - script: | - echo "== Check Docker Images ==" - docker images - - echo "== Run az self-test ==" - docker run $(AzCliImageName):$(AzCliImagePatchedTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" - - - task: Bash@3 - displayName: Save Patched Docker Image - inputs: - targetType: 'inline' - script: | - echo "== Check Docker Images ==" - docker images - - echo "== Save Docker Image to tar file ==" - mkdir "$BUILD_STAGINGDIRECTORY/copa" - docker save -o "$BUILD_STAGINGDIRECTORY/copa/docker-azure-cli-$(AzCliVersion).tar" $(AzCliImageName):$(AzCliImagePatchedTag) - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' - inputs: - BuildDropPath: $(Build.ArtifactStagingDirectory)/copa - DockerImagesToScan: $(AzCliImagePatchedTag) - - - task: PublishPipelineArtifact@0 - displayName: Publish docker Artifact - inputs: - TargetPath: $(Build.ArtifactStagingDirectory)/copa - ArtifactName: $(uploadArtifactName) - - - task: Bash@3 - displayName: Scan with Trivy again - continueOnError: true - inputs: - targetType: 'inline' - script: | - set -eux - export IMAGE=$(AzCliImageName):$(AzCliImagePatchedTag) - OutputFile="trivy-scan-after.json" - trivy image --pkg-types os,library --ignore-unfixed -f json -o $OutputFile $IMAGE - cp $OutputFile ./output/ - - - publish: $(System.DefaultWorkingDirectory)/output - artifact: $(AzCliVersion)-$(architecture)-trivy - displayName: Publish Trivy Artifact + - task: Bash@3 + displayName: Configure Docker Daemon + inputs: + targetType: 'inline' + script: | + if [ -f /etc/docker/daemon.json ]; then + echo "Docker daemon.json exists, printing content:" + cat /etc/docker/daemon.json + else + echo "Docker daemon.json does not exist, creating it..." + sudo mkdir -p /etc/docker + echo '{"features": { "containerd-snapshotter": true }}' | sudo tee /etc/docker/daemon.json + echo "Created /etc/docker/daemon.json with containerd-snapshotter enabled" + fi + + - task: Bash@3 + displayName: Restart Docker + inputs: + targetType: 'inline' + script: | + sudo systemctl restart docker + + - task: Bash@3 + displayName: Install Trivy + inputs: + targetType: 'inline' + script: | + set -eux + curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/bin + echo "----------------------------------" + trivy --version + + - task: Bash@3 + displayName: Install Copa + inputs: + targetType: 'inline' + script: | + CopaDownloadUrl="https://github.com/project-copacetic/copacetic/releases/download/v$(CopaVersion)/copa_$(CopaVersion)_linux_amd64.tar.gz" + if [ "$(architecture)" = "arm64" ]; then + CopaDownloadUrl="https://github.com/project-copacetic/copacetic/releases/download/v$(CopaVersion)/copa_$(CopaVersion)_linux_arm64.tar.gz" + fi + echo "Download copa from: $CopaDownloadUrl" + curl -L -o /tmp/copa.tar.gz $CopaDownloadUrl + sudo mkdir -p /opt/copacetic + sudo tar zxf /tmp/copa.tar.gz -C /opt/copacetic + sudo chmod +x /opt/copacetic/copa + sudo ln -s /opt/copacetic/copa /usr/bin/copa + + echo "---------------- copa --version ------------------" + copa --version + echo "---------------- copa --help ------------------" + copa --help + echo "---------------- copa patch --help ------------------" + copa patch --help + echo "-----------------------------------------------------" + + - task: Bash@3 + displayName: Check Docker Image + inputs: + targetType: 'inline' + script: | + TAR_FILE="$SYSTEM_ARTIFACTSDIRECTORY/docker/docker-azure-cli-$(AzCliVersion).tar" + docker load < $TAR_FILE + + echo "== Check Docker Images ==" + docker images + + echo "== docker container ==" + docker ps + + echo "== Run az self-test ==" + docker run $(AzCliImageName):$(AzCliImageTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" + + - task: Bash@3 + displayName: Scan with Trivy + inputs: + targetType: 'inline' + script: | + set -eux + mkdir output + export IMAGE=$(AzCliImageName):$(AzCliImageTag) + OutputFile="trivy-scan-before.json" + trivy image --pkg-types os,library --ignore-unfixed -f json -o $OutputFile $IMAGE + cp $OutputFile ./output/ + + - task: Bash@3 + displayName: Patch with Copa + inputs: + targetType: 'inline' + script: | + export COPA_EXPERIMENTAL=1 + export IMAGE=$(AzCliImageName):$(AzCliImageTag) + copa patch -i $IMAGE -r trivy-scan-before.json --pkg-types os,library --library-patch-level ${{parameters.PatchLevel}} --tag $(AzCliImagePatchedTag) + + - task: Bash@3 + displayName: Check Patched Docker Image + inputs: + targetType: 'inline' + script: | + echo "== Check Docker Images ==" + docker images + + echo "== Run az self-test ==" + docker run $(AzCliImageName):$(AzCliImagePatchedTag) /bin/bash -c "time az self-test && time az --version && tdnf list --installed" + + - task: Bash@3 + displayName: Save Patched Docker Image + inputs: + targetType: 'inline' + script: | + echo "== Check Docker Images ==" + docker images + + echo "== Save Docker Image to tar file ==" + mkdir "$BUILD_STAGINGDIRECTORY/copa" + docker save -o "$BUILD_STAGINGDIRECTORY/copa/docker-azure-cli-$(AzCliVersion).tar" $(AzCliImageName):$(AzCliImagePatchedTag) + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: 'SBOM' + inputs: + BuildDropPath: $(Build.ArtifactStagingDirectory)/copa + DockerImagesToScan: $(AzCliImagePatchedTag) + + - task: PublishPipelineArtifact@0 + displayName: Publish docker Artifact + inputs: + TargetPath: $(Build.ArtifactStagingDirectory)/copa + ArtifactName: $(uploadArtifactName) + + - task: Bash@3 + displayName: Scan with Trivy again + continueOnError: true + inputs: + targetType: 'inline' + script: | + set -eux + export IMAGE=$(AzCliImageName):$(AzCliImagePatchedTag) + OutputFile="trivy-scan-after.json" + trivy image --pkg-types os,library --ignore-unfixed -f json -o $OutputFile $IMAGE + cp $OutputFile ./output/ + + - publish: $(System.DefaultWorkingDirectory)/output + artifact: $(AzCliVersion)-$(architecture)-trivy + displayName: Publish Trivy Artifact - stage: Test2 displayName: 2nd Test - for Copa Patched Image From 2255852175a1937ab599fc629b7c31d6d8c29550 Mon Sep 17 00:00:00 2001 From: YanaXu Date: Thu, 23 Oct 2025 13:55:25 +0800 Subject: [PATCH 48/52] remove not used condition --- .azure-pipelines/test-with-copa.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 9381fb0115d..6af9754027c 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -141,7 +141,6 @@ stages: displayName: Install Docker - task: PipAuthenticate@1 - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') displayName: 'Pip Authenticate' inputs: artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) @@ -154,7 +153,6 @@ stages: - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 displayName: 'SBOM' - condition: startsWith(variables['Build.SourceBranch'], 'refs/heads/release') inputs: BuildDropPath: $(Build.ArtifactStagingDirectory) @@ -233,7 +231,7 @@ stages: - job: TestRpmPackagesAzureLinux displayName: Test - Rpm Package timeoutInMinutes: 180 - condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule'), eq(${{parameters.ShouldTest}}, true)) + condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) pool: name: $(pool) strategy: From e37d12497fdf25373144c3dfeca298582e584517 Mon Sep 17 00:00:00 2001 From: YanaXu Date: Thu, 23 Oct 2025 14:18:49 +0800 Subject: [PATCH 49/52] refine --- .../templates/test-docker-image-smoke.yml | 2 +- .../templates/test-docker-image-unit.yml | 6 +-- .azure-pipelines/test-with-copa.yml | 46 ++++++++----------- 3 files changed, 23 insertions(+), 31 deletions(-) diff --git a/.azure-pipelines/templates/test-docker-image-smoke.yml b/.azure-pipelines/templates/test-docker-image-smoke.yml index c285bd7f573..02a724de491 100644 --- a/.azure-pipelines/templates/test-docker-image-smoke.yml +++ b/.azure-pipelines/templates/test-docker-image-smoke.yml @@ -34,7 +34,7 @@ parameters: jobs: - job: TestDockerImageSmokeTest - displayName: Smoke Test - Docker Image + displayName: Smoke Test - Docker condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) timeoutInMinutes: ${{ parameters.timeoutInMinutes }} strategy: diff --git a/.azure-pipelines/templates/test-docker-image-unit.yml b/.azure-pipelines/templates/test-docker-image-unit.yml index 98d85670252..26b8bd38b5b 100644 --- a/.azure-pipelines/templates/test-docker-image-unit.yml +++ b/.azure-pipelines/templates/test-docker-image-unit.yml @@ -25,7 +25,7 @@ parameters: jobs: - job: TestDockerImageUnitTest - displayName: Unit Test - Docker Image + displayName: Unit Test - Docker condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) timeoutInMinutes: ${{ parameters.timeoutInMinutes }} strategy: @@ -55,7 +55,7 @@ jobs: echo "== Run az self-test ==" docker run ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash -c "time az self-test && time az --version && tdnf list --installed" - displayName: '🚀 Run Self Test' + displayName: '🚀 Self Test' - bash: | echo "== List docker image ==" @@ -63,4 +63,4 @@ jobs: echo "== Run unit test ==" docker run --rm -v $(pwd):/azure-cli ${{ parameters.AzCliImageName }}:${{ parameters.AzCliImageTag }} /bin/bash /azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh --InstallRPM false - displayName: '🚀🚀 Run Unit Test' + displayName: '🚀🚀 Unit Test' diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index 6af9754027c..fab0398b35a 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -1,11 +1,5 @@ variables: - template: ${{ variables.Pipeline.Workspace }}/.azure-pipelines/templates/variables.yml -- name: Codeql.Enabled - value: false -- name: ComponentDetection.ForceScan - value: eq(variables['Build.SourceBranch'], 'refs/heads/release') -- name: CopaVersion - value: 0.12.0-rc.1 parameters: - name: architectures @@ -22,7 +16,7 @@ parameters: type: boolean default: false - name: PatchLevel - displayName: library-patch-level? + displayName: copa-library-patch-level? default: "patch" values: - patch @@ -31,7 +25,7 @@ parameters: stages: - stage: PrepareInfo - displayName: Prepare Build Info + displayName: Prepare jobs: - job: PrepareAzCliVersion displayName: Prepare Build Info @@ -41,7 +35,7 @@ stages: - checkout: none - task: Bash@3 name: PrepareAzCliVersionTask - displayName: Prepare Build Info + displayName: Get Azure CLI Version inputs: targetType: 'inline' script: | @@ -55,7 +49,7 @@ stages: echo "Azure CLI version: $AzCLI_VERSION" echo "Azure CLI Image Name: $AzCLI_IMAGE_NAME" echo "Azure CLI Image Tag: $AzCLI_IMAGE_TAG" - echo "Azure CLI Image Patched Tag: $AzCLI_IMAGE_PATCHED_TAG" + echo "Azure CLI Image Copa Patched Tag: $AzCLI_IMAGE_PATCHED_TAG" echo "##vso[task.setvariable variable=AzCliVersion]$AzCLI_VERSION" @@ -98,8 +92,6 @@ stages: scriptType: 'bash' scriptLocation: 'inlineScript' inlineScript: | - az account show - # Get user, subscription and tenant info AzUserId=$(az account show --query user.name -o tsv) AzSubscription=$(az account show --query id -o tsv) @@ -117,7 +109,7 @@ stages: echo "##vso[task.setvariable variable=ServiceConnectionId;isOutput=true]$AZURESUBSCRIPTION_SERVICE_CONNECTION_ID" - stage: Build - displayName: Build RPM and Docker Images + displayName: Build dependsOn: - PrepareInfo variables: @@ -131,7 +123,7 @@ stages: matrix: ${{ each arch in parameters.architectures }}: ${{ arch.name }}: - image: mcr.microsoft.com/azurelinux/base/core:3.0 + image: $(BaseImage) uploadArtifact: $(AzCliVersion)-rpm-${{ arch.value }} pool: ${{ arch.pool }} pool: @@ -140,10 +132,10 @@ stages: - bash: ./scripts/ci/install_docker.sh displayName: Install Docker - - task: PipAuthenticate@1 - displayName: 'Pip Authenticate' - inputs: - artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) + # - task: PipAuthenticate@1 + # displayName: 'Pip Authenticate' + # inputs: + # artifactFeeds: $(AZURE_ARTIFACTS_FEEDS) - task: Bash@3 displayName: 'Build Rpm Package: Azure Linux' @@ -152,7 +144,7 @@ stages: filePath: scripts/release/rpm/pipeline_azurelinux.sh - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' + displayName: 'Generate SBOM' inputs: BuildDropPath: $(Build.ArtifactStagingDirectory) @@ -163,7 +155,7 @@ stages: ArtifactName: $(uploadArtifact) - job: BuildDockerImageAzureLinux - displayName: Build Docker Image + displayName: Build Docker dependsOn: - BuildRpmPackagesAzureLinux strategy: @@ -174,7 +166,7 @@ stages: downloadArtifactName: $(AzCliVersion)-rpm-${{ arch.value }} uploadArtifactName: $(AzCliVersion)-docker-${{ arch.value }} dockerfile: azure-linux.dockerfile - image: mcr.microsoft.com/azurelinux/base/core:3.0 + image: $(BaseImage) pool: name: $(pool) steps: @@ -203,7 +195,7 @@ stages: displayName: 'Build Docker Image' - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: 'SBOM' + displayName: 'Generate SBOM' inputs: BuildDropPath: $(Build.ArtifactStagingDirectory)/docker DockerImagesToScan: $(AzCliDockerImageTag) @@ -215,7 +207,7 @@ stages: ArtifactName: $(uploadArtifactName) - stage: Test1 - displayName: 1st Test - for Original Image + displayName: 1st Test - Original Image dependsOn: - PrepareInfo - Build @@ -229,7 +221,7 @@ stages: TestServiceConnectionId: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareServiceConnectionInfoTask.ServiceConnectionId'] ] jobs: - job: TestRpmPackagesAzureLinux - displayName: Test - Rpm Package + displayName: Test - Rpm timeoutInMinutes: 180 condition: and(succeeded(), eq(${{parameters.ShouldTest}}, true)) pool: @@ -238,7 +230,7 @@ stages: matrix: ${{ each arch in parameters.architectures }}: ${{ arch.name }}: - image: mcr.microsoft.com/azurelinux/base/core:3.0 + image: $(BaseImage) downloadArtifactName: $(AzCliVersion)-rpm-${{ arch.value }} pool: ${{ arch.pool }} steps: @@ -294,7 +286,7 @@ stages: AzCliImagePatchedTag: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImagePatchedTag'] ] jobs: - job: BuildCopaPatchedDockerImage - displayName: Build Copa Patched Docker Image + displayName: Patch with Copa timeoutInMinutes: 180 strategy: matrix: @@ -463,7 +455,7 @@ stages: displayName: Publish Trivy Artifact - stage: Test2 - displayName: 2nd Test - for Copa Patched Image + displayName: 2nd Test - Copa Patched Image dependsOn: - PrepareInfo - Build From 4de6f75173dd16c34a541980e1cf12f5cc1b4c86 Mon Sep 17 00:00:00 2001 From: YanaXu Date: Thu, 23 Oct 2025 14:22:59 +0800 Subject: [PATCH 50/52] update shouldTest as default true --- .azure-pipelines/test-with-copa.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index fab0398b35a..ba807a1ce51 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -14,7 +14,7 @@ parameters: - name: ShouldTest displayName: Test? type: boolean - default: false + default: true - name: PatchLevel displayName: copa-library-patch-level? default: "patch" From 94f4886f590082d9bb849295f6236f54695259e3 Mon Sep 17 00:00:00 2001 From: YanaXu Date: Thu, 23 Oct 2025 14:28:54 +0800 Subject: [PATCH 51/52] merge tasks --- .azure-pipelines/test-with-copa.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index ba807a1ce51..a2c80f0e6a2 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -323,12 +323,6 @@ stages: echo '{"features": { "containerd-snapshotter": true }}' | sudo tee /etc/docker/daemon.json echo "Created /etc/docker/daemon.json with containerd-snapshotter enabled" fi - - - task: Bash@3 - displayName: Restart Docker - inputs: - targetType: 'inline' - script: | sudo systemctl restart docker - task: Bash@3 From 591f56a3a6eb41d463493e0670ebb03b42e2c9cb Mon Sep 17 00:00:00 2001 From: YanaXu Date: Thu, 30 Oct 2025 14:07:21 +0800 Subject: [PATCH 52/52] support no patchable vulnerabilities found --- .azure-pipelines/test-with-copa.yml | 36 ++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/test-with-copa.yml b/.azure-pipelines/test-with-copa.yml index a2c80f0e6a2..1c097f33c63 100644 --- a/.azure-pipelines/test-with-copa.yml +++ b/.azure-pipelines/test-with-copa.yml @@ -390,15 +390,44 @@ stages: - task: Bash@3 displayName: Patch with Copa + name: PatchWithCopaTask inputs: targetType: 'inline' script: | export COPA_EXPERIMENTAL=1 export IMAGE=$(AzCliImageName):$(AzCliImageTag) - copa patch -i $IMAGE -r trivy-scan-before.json --pkg-types os,library --library-patch-level ${{parameters.PatchLevel}} --tag $(AzCliImagePatchedTag) + copa patch -i $IMAGE -r trivy-scan-before.json --pkg-types os,library --library-patch-level ${{parameters.PatchLevel}} --tag $(AzCliImagePatchedTag) 2>&1 | tee copa-patch.log + + echo "------------- copa patched result: -------------" + cat copa-patch.log + echo "------------------------------------------------" + + PatchableVulnerabilitiesFound=true + if grep -q "Error" copa-patch.log; then + if grep -q "Error: no patchable vulnerabilities found" copa-patch.log; then + echo "no patchable vulnerabilities found" + PatchableVulnerabilitiesFound=false + else + echo "Found error in copa-patch.log:" + grep "Error" copa-patch.log + exit 1 + fi + else + PATCHED_IMAGE="$(AzCliImageName):$(AzCliImagePatchedTag)" + if ! docker image ls --format "table {{.Repository}}:{{.Tag}}" | grep -q "^$PATCHED_IMAGE$"; then + echo "Error: Patched image $PATCHED_IMAGE does not exist" + exit 1 + fi + fi + echo "##vso[task.setvariable variable=PatchableVulnerabilitiesFound]$PatchableVulnerabilitiesFound" + echo "##vso[task.setvariable variable=PatchableVulnerabilitiesFound;isOutput=true]$PatchableVulnerabilitiesFound" + + echo "== Check Docker Images ==" + docker images - task: Bash@3 displayName: Check Patched Docker Image + condition: and(succeeded(), eq(variables['PatchableVulnerabilitiesFound'], 'true')) inputs: targetType: 'inline' script: | @@ -410,6 +439,7 @@ stages: - task: Bash@3 displayName: Save Patched Docker Image + condition: and(succeeded(), eq(variables['PatchableVulnerabilitiesFound'], 'true')) inputs: targetType: 'inline' script: | @@ -422,18 +452,21 @@ stages: - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 displayName: 'SBOM' + condition: and(succeeded(), eq(variables['PatchableVulnerabilitiesFound'], 'true')) inputs: BuildDropPath: $(Build.ArtifactStagingDirectory)/copa DockerImagesToScan: $(AzCliImagePatchedTag) - task: PublishPipelineArtifact@0 displayName: Publish docker Artifact + condition: and(succeeded(), eq(variables['PatchableVulnerabilitiesFound'], 'true')) inputs: TargetPath: $(Build.ArtifactStagingDirectory)/copa ArtifactName: $(uploadArtifactName) - task: Bash@3 displayName: Scan with Trivy again + condition: and(succeeded(), eq(variables['PatchableVulnerabilitiesFound'], 'true')) continueOnError: true inputs: targetType: 'inline' @@ -454,6 +487,7 @@ stages: - PrepareInfo - Build - PatchWithCopa + condition: and(succeeded(), eq(dependencies.PatchWithCopa.outputs['BuildCopaPatchedDockerImage.PatchWithCopaTask.PatchableVulnerabilitiesFound'], 'true')) variables: AzCliVersion: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliVersion'] ] AzCliImageName: $[ stageDependencies.PrepareInfo.PrepareAzCliVersion.outputs['PrepareAzCliVersionTask.AzCliImageName'] ]