Use native cargo commands for crates.io work (Azure#2140)

Author: hallipr

eng/pipelines/templates/jobs/ci.yml

@@ -155,7 +155,7 @@ jobs:
       filePath: $(Build.SourcesDirectory)/eng/scripts/Analyze-Code.ps1
       arguments: >
         -Toolchain '$(Toolchain)'
-        -PackageInfoPath '$(Build.ArtifactStagingDirectory)/PackageInfo'
+        -PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo'
         -SkipPackageAnalysis:('$(NoPackagesChanged)' -eq 'true')
 
   - template: /eng/common/pipelines/templates/steps/check-spelling.yml

eng/pipelines/templates/stages/archetype-rust-release.yml

@@ -63,6 +63,13 @@ stages:
         - deployment: PublishPackage
           displayName: "Publish to Crates.io"
           condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true'))
+          templateContext:
+            type: releaseJob  # Required, this indicates this deployment job is a release job
+            isProduction: true  # Required, must be 'true' or 'false'
+            inputs:  # All input build artifacts must be declared here
+            - input: pipelineArtifact  # Required, type of the input artifact
+              artifactName: ${{parameters.PipelineArtifactName}}  # Required, name of the pipeline artifact
+              targetPath: $(Pipeline.Workspace)/drop  # Optional, specifies where the artifact is downloaded to
           environment: ${{parameters.Environment}}
           # This timeout shouldn't be necessary once we're able to parallelize better. Right now,
           # this is here to ensure larger areas (30+) libraries don't time out.
@@ -77,18 +84,20 @@ stages:
               deploy:
                 steps:
                 - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+                  parameters:
+                    skipCheckoutNone: true
 
-                - download: current
-                  displayName: Download ${{parameters.PipelineArtifactName}} artifact
-                  artifact: ${{parameters.PipelineArtifactName}}
+                - template: /eng/pipelines/templates/steps/use-rust.yml@self
+                  parameters:
+                    Toolchain: stable
 
                 - task: PowerShell@2
                   displayName: Publish Crate
                   inputs:
                     targetType: filePath
                     filePath: $(Build.SourcesDirectory)/eng/scripts/Publish-Crates.ps1
                     arguments: >
-                      -PackagesPath '$(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}'
+                      -PackagesPath '$(Pipeline.Workspace)/drop'
                       -CrateNames '${{artifact.name}}'
                       -Token $(azure-sdk-cratesio-token)
                       -AdditionalOwners 'heaths','hallipr'

eng/pipelines/templates/steps/test-packages.yml

@@ -30,7 +30,7 @@ steps:
       pwsh: true
       filePath: $(Build.SourcesDirectory)/eng/scripts/Test-Packages.ps1
       arguments: >
-        -PackageInfoPath '$(Pipeline.Workspace)/${{parameters.BuildArtifactName}}/PackageInfo'
+        -PackageInfoDirectory '$(Pipeline.Workspace)/${{parameters.BuildArtifactName}}/PackageInfo'
         -Toolchain '$(Toolchain)'
         -UnitTests $${{ parameters.UnitTests }}
         -FunctionalTests $${{ parameters.FunctionalTests }}

eng/scripts/Analyze-Code.ps1

@@ -3,7 +3,7 @@
 #Requires -Version 7.0
 param(
   [string]$Toolchain = 'stable',
-  [string]$PackageInfoPath,
+  [string]$PackageInfoDirectory,
   [switch]$CheckWasm = $true,
   [switch]$SkipPackageAnalysis
 )
@@ -12,7 +12,6 @@ $ErrorActionPreference = 'Stop'
 Set-StrictMode -Version 2.0
 
 . (Join-Path $PSScriptRoot '..' 'common' 'scripts' 'common.ps1')
-. (Join-Path $EngCommonScriptsDir 'Helpers' 'CommandInvocation-Helpers.ps1')
 
 Write-Host "Analyzing code with
     Toolchain: '$Toolchain'`n"
@@ -41,12 +40,12 @@ Invoke-LoggedCommand "cargo +$Toolchain doc --workspace --no-deps --all-features
 $verifyDependenciesScript = Join-Path $RepoRoot 'eng' 'scripts' 'verify-dependencies.rs' -Resolve
 
 if (!$SkipPackageAnalysis) {
-  if (!(Test-Path $PackageInfoPath)) {
-    Write-Error "Package info path '$PackageInfoPath' does not exist."
+  if (!(Test-Path $PackageInfoDirectory)) {
+    Write-Error "Package info path '$PackageInfoDirectory' does not exist."
     exit 1
   }
 
-  $packagesToTest = Get-ChildItem $PackageInfoPath -Filter "*.json" -Recurse
+  $packagesToTest = Get-ChildItem $PackageInfoDirectory -Filter "*.json" -Recurse
   | Get-Content -Raw
   | ConvertFrom-Json
 

eng/scripts/Language-Settings.ps1

@@ -1,11 +1,13 @@
 $Language = "rust"
 $LanguageDisplayName = "Rust"
 $PackageRepository = "crates.io"
-$packagePattern = "cargo-metadata.json"
+$packagePattern = "Cargo.toml"
 #$MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/main/_data/releases/latest/rust-packages.csv"
 $GithubUri = "https://github.com/Azure/azure-sdk-for-rust"
 $PackageRepositoryUri = "https://crates.io/crates"
 
+. (Join-Path $EngCommonScriptsDir "Helpers" "PSModule-Helpers.ps1")
+
 function SetPackageVersion ($PackageName, $Version, $ServiceDirectory, $ReleaseDate, $ReplaceLatestEntryTitle = $true) {
   if ($null -eq $ReleaseDate) {
     $ReleaseDate = Get-Date -Format "yyyy-MM-dd"
@@ -141,28 +143,19 @@ function Get-rust-AdditionalValidationPackagesFromPackageSet ($packagesWithChang
 }
 
 function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$workingDirectory) {
-  #$pkg will be a FileInfo object for the cargo-metadata.json file in a package artifact directory
-  $package = Get-Content -Path $pkg.FullName -Raw | ConvertFrom-Json
-  $packageName = $package.name
-  $packageVersion = $package.vers
-
-  $crateFile = Get-ChildItem $pkg.DirectoryName -Filter '*.crate'
+  #$pkg will be a FileInfo object for the Cargo.toml file in a package artifact directory
+  $package = cargo read-manifest --manifest-path $pkg.FullName
 
-  New-Item -Path $workingDirectory -ItemType Directory -Force | Out-Null
-  $workFolder = Join-Path $workingDirectory $crateFile.BaseName
-  if (Test-Path $workFolder) {
-    Remove-item $workFolder -Recurse -Force | Out-Null
-  }
+  $packageName = $package.name
+  $packageVersion = $package.version
 
-  # This will extract the contents of the crate file into a folder matching the file name
-  tar -xvzf $crateFile.FullName -C $workingDirectory | Out-Null
+  $changeLogLoc = Get-ChildItem -Path $pkg.DirectoryName -Filter "CHANGELOG.md" | Select-Object -First 1
+  $readmeContentLoc = Get-ChildItem -Path $pkg.DirectoryName -Filter "README.md" | Select-Object -First 1
 
-  $changeLogLoc = Get-ChildItem -Path $workFolder -Filter "CHANGELOG.md" | Select-Object -First 1
   if ($changeLogLoc) {
     $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $packageVersion
   }
 
-  $readmeContentLoc = Get-ChildItem -Path $workFolder -Filter "README.md" | Select-Object -First 1
   if ($readmeContentLoc) {
     $readmeContent = Get-Content -Raw $readmeContentLoc
   }

eng/scripts/Pack-Crates.ps1

@@ -12,7 +12,6 @@ param(
 $ErrorActionPreference = 'Stop'
 
 . (Join-Path $PSScriptRoot '..' 'common' 'scripts' 'common.ps1')
-. (Join-Path $EngCommonScriptsDir 'Helpers' 'CommandInvocation-Helpers.ps1')
 
 if ($OutputPath) {
   $OutputPath = New-Item -ItemType Directory -Path $OutputPath -Force | Select-Object -ExpandProperty FullName
@@ -168,61 +167,6 @@ function Add-PathVersions($packages) {
   }
 }
 
-function Get-ApiMetadata($package) {
-  $packagePath = Split-Path -Path $package.manifest_path -Parent
-  $readmePath = Join-Path -Path $packagePath -ChildPath $package.readme
-  $jsonBody = [ordered]@{
-    'name'          = $package.name
-    'vers'          = $package.version
-    'deps'          = @()
-    'features'      = $package.features
-    'authors'       = $package.authors
-    'description'   = $package.description
-    'documentation' = $package.documentation
-    'homepage'      = $package.homepage
-    'readme'        = if ($package.readme -and (Test-Path -Path $readmePath)) {
-      Get-Content -Path $readmePath -Raw
-    }
-    else {
-      $null
-    }
-    'readme_file'   = $package.readme
-    'keywords'      = $package.keywords
-    'categories'    = $package.categories
-    'license'       = $package.license
-    'license_file'  = $package.license_file
-    'repository'    = $package.repository
-    'links'         = $package.links
-    'rust_version'  = $package.rust_version
-  }
-
-  foreach ($dependency in $package.dependencies) {
-    $jsonBody.deps += @{
-      'name'                  = $dependency.name
-      'version_req'           = $dependency.req
-      'features'              = $dependency.features
-      'optional'              = $dependency.optional
-      'default_features'      = $dependency.default_features
-      'target'                = $dependency.target
-      'kind'                  = $dependency.kind
-      'explicit_name_in_toml' = $dependency.rename
-    }
-  }
-
-  return $jsonBody
-}
-
-function New-ApiPutFile($crateMetadata, $CrateFilePath) {
-  $metadataBytes = [Text.Encoding]::Utf8.GetBytes($crateMetadata)
-  $metadataLengthBytes = [BitConverter]::GetBytes([UInt32]$metadataBytes.Length)
-  $crateBytes = [IO.File]::ReadAllBytes($CrateFilePath)
-  $crateLengthBytes = [BitConverter]::GetBytes([UInt32]$crateBytes.Length)
-
-  $bytes += $metadataLengthBytes + $metadataBytes + $crateLengthBytes + $crateBytes
-
-  return $bytes
-}
-
 Push-Location $RepoRoot
 try {
   $localRegistryPath = Initialize-VendorDirectory
@@ -245,14 +189,7 @@ try {
 
     Invoke-LoggedCommand `
       -GroupOutput `
-      -Command "cargo package --package $packageName --config `"source.crates-io.replace-with='local'`" --config `"source.local.directory='$localRegistryPath'`" --allow-dirty"
-
-    $crateFile = "$RepoRoot/target/package/$packageName-$packageVersion.crate"
-
-    if (-not (Test-Path -Path $crateFile)) {
-      Write-Error "Building the package '$packageName' didn't produce a crate file in the expected location: '$crateFile'"
-      exit 1
-    }
+      -Command "cargo publish --dry-run --package $packageName --registry crates-io --config `"source.crates-io.replace-with='local'`" --config `"source.local.directory='$localRegistryPath'`" --allow-dirty"
 
     # copy the package to the local registry
     Add-CrateToLocalRegistry `
@@ -261,31 +198,15 @@ try {
 
     if ($OutputPath -and $package.OutputPackage) {
       $packageOutputPath = "$OutputPath/$packageName"
+      $targetPackagePath = "$RepoRoot/target/package/$packageName-$packageVersion"
+
       if (Test-Path -Path $packageOutputPath) {
         Remove-Item -Path $packageOutputPath -Recurse -Force
       }
 
       Write-Host "Copying package '$packageName' to '$packageOutputPath'"
-
       New-Item -ItemType Directory -Path $packageOutputPath -Force | Out-Null
-      Copy-Item -Path $crateFile -Destination $packageOutputPath
-      # Copy package's Cargo.toml to the output directory
-      Copy-Item -Path "$RepoRoot/target/package/$packageName-$packageVersion/Cargo.toml" -Destination $packageOutputPath
-      # Write package metadata to the output directory
-
-      $metadataFile = "$packageOutputPath/cargo-metadata.json"
-      $uploadFile = "$packageOutputPath/cargo-put.bin"
-
-      $crateMetadata = Get-ApiMetadata $package | ConvertTo-Json -Depth 10
-
-      Write-Host "Writing crates.io request metadata to '$metadataFile'"
-      $crateMetadata | Out-File -FilePath "$metadataFile" -Encoding utf8
-
-      $uploadBytes = New-ApiPutFile $crateMetadata $crateFile
-      Write-Host "Writing crates.io request bundle to '$uploadFile'"
-      [IO.File]::WriteAllBytes($uploadFile, $uploadBytes)
-      
-      Get-ApiMetadata $package | ConvertTo-Json -Depth 100 | Out-File -FilePath "$packageOutputPath/cargo-metadata.json" -Encoding utf8
+      Copy-Item -Path $targetPackagePath -Destination $packageOutputPath -Recurse -Exclude "Cargo.toml.orig"
     }
   }
 

eng/scripts/Publish-Crates.ps1

@@ -11,34 +11,22 @@ param(
 $ErrorActionPreference = 'Stop'
 Set-StrictMode -Version 2.0
 
-function TryAddOwners($crateName) {
-  foreach ($owner in $AdditionalOwners) {
-    Write-Host "Adding owner: '$owner' to crate: '$crateName'"
-    # https://doc.rust-lang.org/cargo/reference/registry-web-api.html#owners-add
-    # ignore errors is owner already exists
-    $body = @{ users = @($owner) } | ConvertTo-Json
-
-    $response = Invoke-WebRequest -Method Put -Uri "https://crates.io/api/v1/crates/$crateName/owners" `
-      -Headers @{ Accept = 'application/json'; Authorization = $Token } `
-      -ContentType 'application/json' `
-      -Body $body`
-      -SkipHttpErrorCheck
-
-    if ($response.StatusCode -ge 400 -and $response.Content -notmatch 'already an owner') {
-      Write-Host "Failed to add owner: '$owner' to crate: '$crateName'"
-      Write-Host "Response: $($response.Content)"
-      exit 1
-    }
-  }
-}
-
 foreach ($crateName in $CrateNames) {
   Write-Host "Publishing packae: '$crateName'"
+  $manifestPath = "$PackagesPath/$crateName/Cargo.toml"
   # https://doc.rust-lang.org/cargo/reference/registry-web-api.html#publish
-  Invoke-WebRequest -Method Put -Uri 'https://crates.io/api/v1/crates/new' `
-    -Headers @{ Accept = 'application/json'; Authorization = $Token } `
-    -ContentType 'application/json' `
-    -InFile "$PackagesPath/$crateName/cargo-put.bin"
+  Write-Host "> cargo publish --manifest-path `"$manifestPath`" --token <TOKEN>"
+  cargo publish --manifest-path $manifestPath --token $Token
+  if (!$?) {
+    Write-Error "Failed to publish package: '$crateName'"
+    exit 1
+  }
 
-  TryAddOwners $crateName
+  $existingOwners = (Invoke-LoggedCommand cargo owner --list $crateName) -replace " \(.*", ""
+  $missingOwners = $AdditionalOwners | Where-Object { $existingOwners -notcontains $_ }
+
+  foreach ($owner in $missingOwners) {
+    Write-Host "> cargo owner --add $owner $crateName --token <TOKEN>"
+    cargo owner --add $owner $crateName --token $Token
+  }
 }

eng/scripts/Test-Packages.ps1

@@ -5,28 +5,27 @@ param(
   [string]$Toolchain = 'stable',
   [bool]$UnitTests = $true,
   [bool]$FunctionalTests = $true,
-  [string]$PackageInfoPath
+  [string]$PackageInfoDirectory
 )
 
 $ErrorActionPreference = 'Stop'
 Set-StrictMode -Version 2.0
 
 . "$PSScriptRoot\..\common\scripts\common.ps1"
-. (Join-Path $EngCommonScriptsDir "Helpers" CommandInvocation-Helpers.ps1)
 
 Write-Host "Testing packages with
     Toolchain: '$Toolchain'
     UnitTests: '$UnitTests'
     FunctionalTests: '$FunctionalTests'
-    PackageInfoPath: '$PackageInfoPath'"
+    PackageInfoDirectory: '$PackageInfoDirectory'"
 
-if ($PackageInfoPath) {
-  if (!(Test-Path $PackageInfoPath)) {
-    Write-Error "Package info path '$PackageInfoPath' does not exist."
+if ($PackageInfoDirectory) {
+  if (!(Test-Path $PackageInfoDirectory)) {
+    Write-Error "Package info path '$PackageInfoDirectory' does not exist."
     exit 1
   }
 
-  $packagesToTest = Get-ChildItem $PackageInfoPath -Filter "*.json" -Recurse
+  $packagesToTest = Get-ChildItem $PackageInfoDirectory -Filter "*.json" -Recurse
   | Get-Content -Raw
   | ConvertFrom-Json
 }

eng/scripts/Yank-Crates.ps1

@@ -2,29 +2,26 @@
 
 #Requires -Version 7.0
 param(
-  [string]$PackagesPath,
+  [string]$PackageInfoDirectory,
   [string[]]$CrateNames,
   [string]$Token
 )
 
 $ErrorActionPreference = 'Stop'
-#Set-StrictMode -Version 2.0
+Set-StrictMode -Version 2.0
 
 $hasErrors = $false
 foreach ($crateName in $crateNames) {
-  $crate = Get-Content "$PackagesPath/$crateName/cargo-metadata.json" -Raw | ConvertFrom-Json
-  $crateVersion = $crate.vers
+  $crate = Get-Content "$PackageInfoDirectory/$crateName.json" -Raw | ConvertFrom-Json
+  $crateVersion = $crate.Version
 
   Write-Host "Yanking crate: '$crateName@$crateVersion'"
 
-  # https://doc.rust-lang.org/cargo/reference/registry-web-api.html#yank
-  $response = Invoke-WebRequest -Method Delete -Uri "https://crates.io/api/v1/crates/$crateName/$crateVersion/yank" `
-    -Headers @{ Accept = 'application/json'; Authorization = $Token } `
-    -SkipHttpErrorCheck
-  
-  if ($response.StatusCode -ge 400) {
+  Write-Host "cargo yank $crateName --version $crateVersion --token <TOKEN>"
+  cargo yank $crateName --version $crateVersion --token $Token
+
+  if (!$?) {
     Write-Host "Failed to yank crate: '$crateName@$crateVersion'"
-    Write-Host "Response: $($response.Content)"
     $hasErrors = $true
   }
 }