Add API View to build (Azure#2396)

* Add API View to build
* Use toolchain defaulting instead explicit toolchain in each cargo call

Author: hallipr

Cargo.lock

@@ -39,7 +39,6 @@ jobs:
       pwsh: true
       filePath: $(Build.SourcesDirectory)/eng/scripts/Analyze-Code.ps1
       arguments: >
-        -Toolchain '$(Toolchain)'
         -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo'
         -SkipPackageAnalysis:('$(NoPackagesChanged)' -eq 'true')
 

eng/pipelines/templates/jobs/analyze.yml

@@ -70,7 +70,6 @@ jobs:
       filePath: $(Build.SourcesDirectory)/eng/scripts/Test-Packages.ps1
       arguments: >
         -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo'
-        -Toolchain '$(Toolchain)'
 
   - ${{ if eq(parameters.TestProxy, true) }}:
     - pwsh: |

eng/pipelines/templates/jobs/ci.tests.yml

@@ -121,8 +121,8 @@ jobs:
           $env:AZURESUBSCRIPTION_CLIENT_ID = $account.Id;
           $env:AZURESUBSCRIPTION_TENANT_ID = $account.Tenants;
 
-          Write-Host "./eng/scripts/Test-Packages.ps1 -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo' -Toolchain '$(Toolchain)'"
-          ./eng/scripts/Test-Packages.ps1 -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo' -Toolchain '$(Toolchain)'
+          Write-Host "./eng/scripts/Test-Packages.ps1 -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo'"
+          ./eng/scripts/Test-Packages.ps1 -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo'
         workingDirectory: $(Build.SourcesDirectory)
   - ${{ else }}:
     - task: Powershell@2
@@ -139,7 +139,6 @@ jobs:
         filePath: $(Build.SourcesDirectory)/eng/scripts/Test-Packages.ps1
         arguments: >
           -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo'
-          -Toolchain '$(Toolchain)'
 
   - ${{ parameters.PostSteps }}
 

eng/pipelines/templates/jobs/live.tests.yml

@@ -32,7 +32,8 @@ jobs:
 
   - template: /eng/pipelines/templates/steps/use-rust.yml@self
     parameters:
-      Toolchain: nightly
+      WorkingDirectory: $(System.DefaultWorkingDirectory)/eng/tools/generate_api_report
+      SetDefault: false
 
   - template: /eng/pipelines/templates/steps/use-rust.yml@self
     parameters:
@@ -69,3 +70,9 @@ jobs:
       ArtifactPath: $(Build.ArtifactStagingDirectory)
       ArtifactName: ${{ parameters.PipelineArtifactName }}
       CustomCondition: and(succeededOrFailed(), ne(variables['NoPackagesChanged'],'true'))
+
+  - template: /eng/common/pipelines/templates/steps/create-apireview.yml
+    parameters:
+      Artifacts: ${{ parameters.Artifacts }}
+      
+  - template: /eng/common/pipelines/templates/steps/detect-api-changes.yml

eng/pipelines/templates/jobs/pack.yml

@@ -93,7 +93,7 @@ stages:
                     $token = $env:CARGO_REGISTRY_TOKEN
                     $crateName = '${{artifact.name}}'
 
-                    $manifestPath = "$(Pipeline.Workspace)/drop/$crateName/Cargo.toml"
+                    $manifestPath = "$(Pipeline.Workspace)/drop/$crateName/contents/Cargo.toml"
                     Write-Host "> cargo publish --manifest-path `"$manifestPath`""
                     cargo publish --manifest-path $manifestPath
                     if (!$?) {
@@ -112,23 +112,8 @@ stages:
                   env:
                     CARGO_REGISTRY_TOKEN: $(azure-sdk-cratesio-token)
 
-        # - job: CreateApiReview
-        #   displayName: "Api Review"
-        #   condition: and(succeeded(), ne(variables['Skip.CreateApiReview'], 'true'))
-        #   dependsOn: PublishPackage
-        #   steps:
-        #   - template: /eng/common/pipelines/templates/steps/create-apireview.yml
-        #     parameters:
-        #       ArtifactPath: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}
-        #       Artifacts: ${{parameters.Artifacts}}
-        #       ConfigFileDir: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/PackageInfo
-        #       MarkPackageAsShipped: true
-        #       ArtifactName: ${{parameters.PipelineArtifactName}}
-        #       SourceRootPath: $(Pipeline.Workspace)/azure-sdk-for-rust
-        #       PackageName: ${{artifact.name}}
-
         - job: UpdatePackageVersion
-          displayName: "Update Package Versions"
+          displayName: "API Review and Package Version Update"
           condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true'))
           dependsOn: PublishPackage
           steps:
@@ -141,6 +126,20 @@ stages:
             parameters:
               Toolchain: nightly
 
+          - download: current
+            displayName: Download ${{parameters.PipelineArtifactName}} artifact
+            artifact: ${{parameters.PipelineArtifactName}}
+
+          - template: /eng/common/pipelines/templates/steps/create-apireview.yml
+            parameters:
+              ArtifactPath: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}
+              Artifacts: ${{parameters.Artifacts}}
+              ConfigFileDir: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/PackageInfo
+              MarkPackageAsShipped: true
+              ArtifactName: ${{parameters.PipelineArtifactName}}
+              SourceRootPath: $(System.DefaultWorkingDirectory)
+              PackageName: ${{artifact.name}}
+
           # Apply the version increment to each library, which updates the Cargo.toml and changelog files.
           - task: PowerShell@2
             displayName: Increment ${{artifact.name}} version

eng/pipelines/templates/stages/archetype-rust-release.yml

@@ -1,42 +1,100 @@
 parameters:
   - name: Toolchain
     type: string
-    default: stable
+    default: active
+    # Expexted values: 'stable', 'nightly', 'msrv', 'active' or a specific toolchain version
+    # 'msrv' will read the MSRV from azure_core
+    # 'active' will use the active toolchain for the working directory, which will be from rust-toolchain.toml, from a
+    #   folder override or the rustup default toolchain
   - name: MaxAttempts
     type: number
     default: 3
+  - name: WorkingDirectory
+    type: string
+    default: $(System.DefaultWorkingDirectory)
+  - name: SetDefault
+    type: boolean
+    default: true
 
 steps:
-  - pwsh: |
-      $toolchain = '${{ parameters.Toolchain }}'
-      if ($toolchain -eq 'msrv') {
-          Write-Host "Reading MSRV from azure_core"
-          $toolchain = cargo read-manifest --manifest-path ./sdk/core/azure_core/Cargo.toml | ConvertFrom-Json | Select-Object -ExpandProperty rust_version
-      }
+- pwsh: |
+    $Toolchain = '${{ parameters.Toolchain }}'
+    $MaxAttempts = ${{ parameters.MaxAttempts }}  
+    $SetDefault = $${{ parameters.SetDefault }}
+
+    function Invoke-LoggedCommand($command) {
+        Write-Host "##[group]$command"
+        Invoke-Expression $command
+        Write-Host "##[endgroup]"
+        if ($LASTEXITCODE -ne 0) {
+            Write-Host "Command failed: $command"
+            exit $LASTEXITCODE
+        }
+    }
+
+    if ($Toolchain -eq 'msrv') {
+        Write-Host "Reading MSRV from azure_core"
+        $toolchainArg = cargo read-manifest --manifest-path "$(System.DefaultWorkingDirectory)/sdk/core/azure_core/Cargo.toml"
+          | ConvertFrom-Json
+          | Select-Object -ExpandProperty rust_version
+    }
+    elseif ($Toolchain -eq 'active') {
+        # You can't call 'rustup install' without a toolchain before rustup 1.28.0. If know we'll be doing that, make sure
+        # we have the latest rustup installed
+        
+        # Depending on the version of rustup currently installed, simply calling `rustup --version` will install the
+        # active toolchain per rust-toolchain.toml if it's not already installed. We should check the rust version
+        # outside of our repo's context to avoid any rustup-toolchain file influence.
+
+        Push-Location '$(Pipeline.Workspace)'
+        Invoke-LoggedCommand "rustup --version" | Tee-Object -Variable rustupVersion
+        
+        if ($rustupVersion -match 'rustup (\d+).(\d+).\d+') {
+            $major = $matches[1]
+            $minor = $matches[2]
+            if ($major -lt 1 -or $minor -lt 28) {
+                Invoke-LoggedCommand "rustup self update"
+            }
+        }
+        Pop-Location
 
-      Write-Host "Setting Toolchain variable to $toolchain"
-      Write-Host "##vso[task.setvariable variable=Toolchain]$toolchain"
+        $toolchainArg = ''
+    }
+    else {
+        $toolchainArg = $toolchain
+    }
 
-      $attempts = 0
-      $maxAttempts = ${{ parameters.MaxAttempts }}
+    $attempts = 0
 
-      while ($true) {
+    while ($true) {
         $attempts++
-        Write-Host "> rustup toolchain install --no-self-update $toolchain"
-        rustup toolchain install --no-self-update $toolchain
 
-        if ($?) { exit 0 }
+        Invoke-LoggedCommand "rustup install --no-self-update $toolchainArg"
 
-        if ($attempts -lt $maxAttempts) {
-          Write-Host "Failed to install $toolchain, attempt $attempts, retrying..."
-        } else {
-          Write-Host "Failed to install $toolchain after $attempts attempts."
-          exit 1
+        if ($?) { break }
+
+        if ($attempts -lt $MaxAttempts) {
+            Write-Host "Install failed, attempt $attempts, retrying..."
+        }
+        else {
+            Write-Host "Install failed after $attempts attempts."
+            exit 1
         }
 
         # Failures to update are usually caused by file locks in Windows.
         # Sleep for a few seconds to give the blocking process a chance to release the lock.
         Start-Sleep -Seconds 3
-      }
+    }
+
+    if ($SetDefault) {
+        if ($Toolchain -eq 'active') {
+            $toolchainArg = rustup show active-toolchain -v | Select-Object -First 1
+        }
+
+        Invoke-LoggedCommand "rustup default $toolchainArg"
+    }
+
+    Invoke-LoggedCommand "rustup show"
 
-    displayName: "Use Rust ${{ parameters.Toolchain }}"
+  displayName: "Use Rust ${{ parameters.Toolchain }}"
+  workingDirectory: ${{ parameters.WorkingDirectory }}

eng/pipelines/templates/steps/use-rust.yml

@@ -2,7 +2,6 @@
 
 #Requires -Version 7.0
 param(
-  [string]$Toolchain = 'stable',
   [string]$PackageInfoDirectory,
   [switch]$CheckWasm = $true,
   [switch]$SkipPackageAnalysis
@@ -15,26 +14,25 @@ Set-StrictMode -Version 2.0
 
 Write-Host @"
 Analyzing code with
-    Toolchain: '$Toolchain'
     RUSTFLAGS: '${env:RUSTFLAGS}'
     RUSTDOCFLAGS: '${env:RUSTDOCFLAGS}'
 "@
 
 if ($CheckWasm) {
-  Invoke-LoggedCommand "rustup target add --toolchain $Toolchain wasm32-unknown-unknown"
+  Invoke-LoggedCommand "rustup target add wasm32-unknown-unknown"
 }
 
-Invoke-LoggedCommand "cargo +$Toolchain check --package azure_core --all-features --all-targets --keep-going"
+Invoke-LoggedCommand "cargo check --package azure_core --all-features --all-targets --keep-going"
 
-Invoke-LoggedCommand "cargo +$Toolchain fmt --all -- --check"
+Invoke-LoggedCommand "cargo fmt --all -- --check"
 
-Invoke-LoggedCommand "cargo +$Toolchain clippy --workspace --all-features --all-targets --keep-going --no-deps"
+Invoke-LoggedCommand "cargo clippy --workspace --all-features --all-targets --keep-going --no-deps"
 
 if ($CheckWasm) {
-  Invoke-LoggedCommand "cargo +$Toolchain clippy --target=wasm32-unknown-unknown --workspace --keep-going --no-deps"
+  Invoke-LoggedCommand "cargo clippy --target=wasm32-unknown-unknown --workspace --keep-going --no-deps"
 }
 
-Invoke-LoggedCommand "cargo +$Toolchain doc --workspace --no-deps --all-features"
+Invoke-LoggedCommand "cargo doc --workspace --no-deps --all-features"
 
 # Verify package dependencies
 

eng/scripts/Analyze-Code.ps1

@@ -119,6 +119,7 @@ function Get-AllPackageInfoFromRepo ([string] $ServiceDirectory) {
 function Get-rust-AdditionalValidationPackagesFromPackageSet ($packagesWithChanges, $diff, $allPackageProperties) {
   # if the change was in a service directory, but not in a package directory, test all the packages in the service directory
   [array]$serviceFiles = ($diff.ChangedFiles + $diff.DeletedFiles) | ForEach-Object { $_ -replace '\\', '/' } | Where-Object { $_ -match "^sdk/.+/" }
+  
   # remove files that target any specific package
   foreach ($package in $allPackageProperties) {
     $packagePathPattern = "^$( [Regex]::Escape($package.DirectoryPath.Replace('\', '/')) )/"
@@ -127,15 +128,14 @@ function Get-rust-AdditionalValidationPackagesFromPackageSet ($packagesWithChang
 
   $affectedServiceDirectories = $serviceFiles | ForEach-Object { $_ -replace '^sdk/(.+?)/.*', '$1' } | Sort-Object -Unique
 
-  $affectedPackages = $allPackageProperties | Where-Object { $affectedServiceDirectories -contains $_.ServiceDirectory }
-
-  [array]$additionalPackages = $affectedPackages | Where-Object { $packagesWithChanges -notcontains $_ }
+  $affectedPackages = @($allPackageProperties | Where-Object { $affectedServiceDirectories -contains $_.ServiceDirectory })
+  $additionalPackages = @($affectedPackages | Where-Object { $packagesWithChanges -notcontains $_ })
 
   foreach ($package in $additionalPackages) {
     $package.IncludedForValidation = $true
   }
 
-  return $additionalPackages
+  return $additionalPackages ?? @()
 }
 
 function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$workingDirectory) {
@@ -167,3 +167,21 @@ function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$working
     ReadmeContent  = $readmeContent
   }
 }
+
+function Find-rust-Artifacts-For-Apireview([string]$ArtifactPath, [string]$packageName) {
+  [array]$files = Get-ChildItem -Path $ArtifactPath -Recurse -Filter "$packageName.rust.json"
+
+  if (!$files) {
+    Write-Host "$($packageName) does not have api review json"
+    return $null
+  }
+  elseif ($files.Count -ne 1) {
+    Write-Host "$($artifactPath) should contain only one api review for $($packageName)"
+    Write-Host "Number of files $($files.Count)"
+    return $null
+  }
+  $packages = @{
+    $files[0].Name = $files[0].FullName
+  }
+  return $packages
+}