Skip to content

Commit 75b2e34

Browse files
authored
ESRP publishing (#3065)
## Do I need to review this as a service owner? Probably not. This PR is for EngSys. You're probably seeing this review because I updated your `ci.yml` file to use "release intent" which allows you to choose which packages to release at release build time. We in EngSys will merge this after reviewing. ## Notes Fixes #3035 * Release packages using ESRP * Release intent (check boxes to release packages) * You can run live tests without releasing packages * Packages are released in dependency order * Dependency enforcement (build fails if a dependency is not released or being released in the current build) Updated runs with release intent: Sample pipeline run with 2 release crates: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=5424835&view=results Sample pipeline run with no releases and live tests: https://dev.azure.com/azure-sdk/internal/_build/results?buildId=5424839&view=logs&j=b766ebde-1fdb-5f11-1350-46ddc53b23cf&t=9494b470-f365-50d2-5008-6394593b39c5 -- In this case note that necessary dependencies were built (e.g. `typespec`) even though it was outside the scope of the service. Test failures notwithstanding. Sample pipeline run with no releases (also no live tests): https://dev.azure.com/azure-sdk/internal/_build/results?buildId=5424838&view=results Example unified version increment PR: #3134 Sample releases: * https://github.com/Azure/azure-sdk-for-rust/releases/tag/azure_canary_core%401.0.0-beta.5397226 * https://github.com/Azure/azure-sdk-for-rust/releases/tag/azure_canary%401.0.0-beta.5397226 ## Open questions * Right now `Pack-Crates.ps1` fails in release scenarios when releasing a package that has unreleased dependencies (e.g. `dependency_crate = { workspace = true }`. The expectation here is that dependencies will go from `workspace = true` to `{ version = "1.2.3" }` when releasing. Is this a correct expectation?
1 parent 2c0cf1b commit 75b2e34

File tree

14 files changed

+342
-171
lines changed

14 files changed

+342
-171
lines changed

eng/pipelines/templates/jobs/pack.yml

Lines changed: 40 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -55,15 +55,46 @@ jobs:
5555
ServiceDirectory: ${{ parameters.ServiceDirectory }}
5656
PackageInfoDirectory: $(Build.ArtifactStagingDirectory)/PackageInfo
5757

58-
- task: Powershell@2
59-
displayName: "Pack Crates"
60-
condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true'))
61-
inputs:
62-
pwsh: true
63-
filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1
64-
arguments: >
65-
-OutputPath '$(Build.ArtifactStagingDirectory)'
66-
-PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo'
58+
- ${{ if eq('auto', parameters.ServiceDirectory) }}:
59+
- task: Powershell@2
60+
displayName: Pack Crates
61+
condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true'))
62+
inputs:
63+
pwsh: true
64+
filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1
65+
arguments: >
66+
-OutputPath '$(Build.ArtifactStagingDirectory)'
67+
-PackageInfoDirectory '$(Build.ArtifactStagingDirectory)/PackageInfo'
68+
69+
- ${{ else }}:
70+
- pwsh: |
71+
$artifacts = '${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json
72+
$requireDependencies = $true
73+
$artifactsToBuild = $artifacts | Where-Object { $_.releaseInBatch -eq 'True' }
74+
75+
if (!$artifactsToBuild) {
76+
Write-Host "No packages to release. Building all packages in the service directory with no dependency validation."
77+
$artifactsToBuild = $artifacts
78+
$requireDependencies = $false
79+
}
80+
81+
$packageNames = $artifactsToBuild.name -join ','
82+
83+
Write-Host "##vso[task.setvariable variable=PackageNames]$packageNames"
84+
Write-Host "##vso[task.setvariable variable=RequireDependencies]$requireDependencies"
85+
displayName: Create package list
86+
87+
- task: Powershell@2
88+
displayName: Pack Crates
89+
condition: and(succeeded(), ne(variables['NoPackagesChanged'],'true'))
90+
inputs:
91+
pwsh: true
92+
filePath: $(Build.SourcesDirectory)/eng/scripts/Pack-Crates.ps1
93+
arguments: >
94+
-OutputPath '$(Build.ArtifactStagingDirectory)'
95+
-PackageNames $(PackageNames)
96+
-RequireDependencies:$$(RequireDependencies)
97+
-OutBuildOrderFile '$(Build.ArtifactStagingDirectory)/release-order.json'
6798
6899
- template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
69100
parameters:

eng/pipelines/templates/stages/archetype-rust-release.yml

Lines changed: 109 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,13 @@ parameters:
1414
- name: DevFeedName
1515
type: string
1616
default: 'public/azure-sdk-for-rust'
17-
- name: Environment
18-
type: string
19-
default: 'cratesio'
2017

2118
stages:
2219
- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
2320
- ${{ if in(variables['Build.Reason'], 'Manual', '') }}:
24-
- ${{ each artifact in parameters.Artifacts }}:
25-
- stage: Release_${{artifact.safeName}}
26-
displayName: "Release: ${{artifact.name}}"
21+
- ${{ if gt(length(parameters.Artifacts), 0) }}:
22+
- stage: Release_Batch
23+
displayName: "Releasing: ${{length(parameters.Artifacts)}} crates"
2724
dependsOn: ${{parameters.DependsOn}}
2825
condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-rust-pr'))
2926
variables:
@@ -50,16 +47,17 @@ stages:
5047

5148
- template: /eng/common/pipelines/templates/steps/retain-run.yml
5249

53-
- script: |
54-
echo "##vso[build.addbuildtag]${{artifact.name}}"
55-
displayName: Add build tag '${{artifact.name}}'
50+
- ${{ each artifact in parameters.Artifacts }}:
51+
- script: |
52+
echo "##vso[build.addbuildtag]${{artifact.name}}"
53+
displayName: Add build tag '${{artifact.name}}'
5654
57-
- template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml
58-
parameters:
59-
ArtifactLocation: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/${{artifact.name}}
60-
PackageRepository: Crates.io
61-
ReleaseSha: $(Build.SourceVersion)
62-
WorkingDirectory: $(Pipeline.Workspace)/_work
55+
- template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml
56+
parameters:
57+
ArtifactLocation: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/${{artifact.name}}
58+
PackageRepository: Crates.io
59+
ReleaseSha: $(Build.SourceVersion)
60+
WorkingDirectory: $(Pipeline.Workspace)/_work
6361

6462
- deployment: PublishPackage
6563
displayName: "Publish to Crates.io"
@@ -71,7 +69,10 @@ stages:
7169
- input: pipelineArtifact # Required, type of the input artifact
7270
artifactName: ${{parameters.PipelineArtifactName}} # Required, name of the pipeline artifact
7371
targetPath: $(Pipeline.Workspace)/drop # Optional, specifies where the artifact is downloaded to
74-
environment: ${{parameters.Environment}}
72+
${{if parameters.TestPipeline}}:
73+
environment: none
74+
${{else}}:
75+
environment: cratesio
7576
# This timeout shouldn't be necessary once we're able to parallelize better. Right now,
7677
# this is here to ensure larger areas (30+) libraries don't time out.
7778
timeoutInMinutes: 120
@@ -84,33 +85,77 @@ stages:
8485
runOnce:
8586
deploy:
8687
steps:
87-
- template: /eng/pipelines/templates/steps/use-rust.yml@self
88-
parameters:
89-
Toolchain: stable
90-
91-
- pwsh: |
92-
$additionalOwners = @('heaths', 'hallipr')
93-
$token = $env:CARGO_REGISTRY_TOKEN
94-
$crateName = '${{artifact.name}}'
95-
96-
$manifestPath = "$(Pipeline.Workspace)/drop/$crateName/contents/Cargo.toml"
97-
Write-Host "> cargo publish --manifest-path `"$manifestPath`""
98-
cargo publish --manifest-path $manifestPath
99-
if (!$?) {
100-
Write-Error "Failed to publish package: '$crateName'"
101-
exit 1
102-
}
103-
104-
$existingOwners = (cargo owner --list $crateName) -replace " \(.*", ""
105-
$missingOwners = $additionalOwners | Where-Object { $existingOwners -notcontains $_ }
106-
107-
foreach ($owner in $missingOwners) {
108-
Write-Host "> cargo owner --add $owner $crateName"
109-
cargo owner --add $owner $crateName
110-
}
111-
displayName: Publish Crate
112-
env:
113-
CARGO_REGISTRY_TOKEN: $(azure-sdk-cratesio-token)
88+
- pwsh: |
89+
Write-Host "##vso[task.setvariable variable=ArtifactIndex]0"
90+
displayName: Set ArtifactIndex to 0
91+
92+
- ${{ each artifact in parameters.Artifacts }}:
93+
- pwsh: |
94+
# Read artifact release order from release-order.json
95+
# and use ArtifactIndex to select the right one
96+
$index = [int]'$(ArtifactIndex)'
97+
$artifacts = Get-Content '$(Pipeline.Workspace)/drop/release-order.json' | ConvertFrom-Json
98+
if ($index -ge $artifacts.Count) {
99+
Write-Error "ArtifactIndex $index is out of range (0..$($artifacts.Count - 1))"
100+
exit 1
101+
}
102+
103+
$artifactName = $artifacts[$index]
104+
Write-Host "Releasing artifact $artifactName (index $index)"
105+
106+
$artifactRootPath = '$(Pipeline.Workspace)/drop'
107+
$outDir = '$(Pipeline.Workspace)/esrp-release'
108+
109+
if (Test-Path $outDir) {
110+
Write-Host "Cleaning output directory: $outDir"
111+
Remove-Item -Path $outDir -Recurse -Force
112+
}
113+
New-Item -ItemType Directory -Path $outDir -Force | Out-Null
114+
115+
Write-Host "Artifact name: $artifactName"
116+
117+
$packageMetadataPath = "$artifactRootPath/PackageInfo/$artifactName.json"
118+
if (!(Test-Path $packageMetadataPath)) {
119+
Write-Error "Package metadata file not found: $packageMetadataPath"
120+
exit 1
121+
}
122+
123+
$packageMetadata = Get-Content -Raw $packageMetadataPath | ConvertFrom-Json
124+
$packageVersion = $packageMetadata.version
125+
Write-Host "Package version: $packageVersion"
126+
127+
$cratePath = "$artifactRootPath/$artifactName/$artifactName-$packageVersion.crate"
128+
Copy-Item `
129+
-Path $cratePath `
130+
-Destination $outDir
131+
Write-Host "Contents of $outDir"
132+
Get-ChildItem -Path $outDir | ForEach-Object { Write-Host $_.FullName }
133+
displayName: 'Copy crate for ESRP'
134+
135+
- task: EsrpRelease@10
136+
displayName: 'ESRP Release'
137+
inputs:
138+
connectedservicename: 'Azure SDK PME Managed Identity'
139+
ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b'
140+
DomainTenantId: '975f013f-7f24-47e8-a7d3-abc4752bf346'
141+
Usemanagedidentity: true
142+
KeyVaultName: 'kv-azuresdk-codesign'
143+
SignCertName: 'azure-sdk-esrp-release-certificate'
144+
intent: 'packagedistribution'
145+
contenttype: 'Rust'
146+
contentsource: 'Folder'
147+
folderlocation: '$(Pipeline.Workspace)/esrp-release'
148+
waitforreleasecompletion: true
149+
owners: ${{ coalesce(variables['Build.RequestedForEmail'], '[email protected]') }}
150+
approvers: ${{ coalesce(variables['Build.RequestedForEmail'], '[email protected]') }}
151+
serviceendpointurl: 'https://api.esrp.microsoft.com/'
152+
mainpublisher: 'ESRPRELPACMANTEST'
153+
154+
- pwsh: |
155+
$index = [int]'$(ArtifactIndex)' + 1
156+
Write-Host "Setting ArtifactIndex to $index"
157+
Write-Host "##vso[task.setvariable variable=ArtifactIndex]$index"
158+
displayName: Increment ArtifactIndex
114159
115160
- job: UpdatePackageVersion
116161
displayName: "API Review and Package Version Update"
@@ -130,69 +175,32 @@ stages:
130175
displayName: Download ${{parameters.PipelineArtifactName}} artifact
131176
artifact: ${{parameters.PipelineArtifactName}}
132177

133-
- template: /eng/common/pipelines/templates/steps/create-apireview.yml
134-
parameters:
135-
ArtifactPath: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}
136-
Artifacts: ${{parameters.Artifacts}}
137-
ConfigFileDir: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/PackageInfo
138-
MarkPackageAsShipped: true
139-
ArtifactName: ${{parameters.PipelineArtifactName}}
140-
SourceRootPath: $(System.DefaultWorkingDirectory)
141-
PackageName: ${{artifact.name}}
142-
143-
# Apply the version increment to each library, which updates the Cargo.toml and changelog files.
144-
- task: PowerShell@2
145-
displayName: Increment ${{artifact.name}} version
146-
inputs:
147-
targetType: filePath
148-
filePath: $(Build.SourcesDirectory)/eng/scripts/Update-PackageVersion.ps1
149-
arguments: >
150-
-ServiceDirectory '${{parameters.ServiceDirectory}}'
151-
-PackageName '${{artifact.name}}'
178+
- ${{each artifact in parameters.Artifacts }}:
179+
- template: /eng/common/pipelines/templates/steps/create-apireview.yml
180+
parameters:
181+
ArtifactPath: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}
182+
Artifacts: ${{parameters.Artifacts}}
183+
ConfigFileDir: $(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/PackageInfo
184+
MarkPackageAsShipped: true
185+
ArtifactName: ${{parameters.PipelineArtifactName}}
186+
SourceRootPath: $(System.DefaultWorkingDirectory)
187+
PackageName: ${{artifact.name}}
188+
189+
# Apply the version increment to each library, which updates the Cargo.toml and changelog files.
190+
- task: PowerShell@2
191+
displayName: Increment ${{artifact.name}} version
192+
inputs:
193+
targetType: filePath
194+
filePath: $(Build.SourcesDirectory)/eng/scripts/Update-PackageVersion.ps1
195+
arguments: >
196+
-ServiceDirectory '${{parameters.ServiceDirectory}}'
197+
-PackageName '${{artifact.name}}'
152198
153199
- template: /eng/common/pipelines/templates/steps/create-pull-request.yml
154200
parameters:
155201
PRBranchName: increment-package-version-${{parameters.ServiceDirectory}}-$(Build.BuildId)
156-
CommitMsg: "Increment package version after release of ${{ artifact.name }}"
202+
CommitMsg: "Increment package version after release of ${{ join(', ', parameters.Artifacts.*.name) }}"
157203
PRTitle: "Increment versions for ${{parameters.ServiceDirectory}} releases"
158204
CloseAfterOpenForTesting: '${{parameters.TestPipeline}}'
159205
${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}:
160206
BaseBranchName: main
161-
162-
- ${{ if eq(parameters.TestPipeline, true) }}:
163-
- job: ManualApproval
164-
displayName: "Manual approval"
165-
dependsOn: PublishPackage
166-
condition: ne(variables['Skip.PublishPackage'], 'true')
167-
pool: server
168-
timeoutInMinutes: 120 # 2 hours
169-
steps:
170-
- task: ManualValidation@1
171-
timeoutInMinutes: 60 # 1 hour
172-
inputs:
173-
notifyUsers: '' # Required, but empty string allowed
174-
allowApproversToApproveTheirOwnRuns: true
175-
instructions: "Approve yank of ${{ artifact.name }}"
176-
onTimeout: 'resume'
177-
178-
- job: YankCrates
179-
displayName: "Yank Crates"
180-
dependsOn: ManualApproval
181-
condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true'))
182-
steps:
183-
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
184-
185-
- download: current
186-
displayName: Download ${{parameters.PipelineArtifactName}} artifact
187-
artifact: ${{parameters.PipelineArtifactName}}
188-
189-
- task: PowerShell@2
190-
displayName: Yank Crates
191-
env:
192-
CARGO_REGISTRY_TOKEN: $(azure-sdk-cratesio-token)
193-
inputs:
194-
targetType: filePath
195-
filePath: $(Build.SourcesDirectory)/eng/scripts/Yank-Crates.ps1
196-
arguments:
197-
-CrateNames '${{artifact.name}}'
198-
-PackageInfoDirectory '$(Pipeline.Workspace)/${{parameters.PipelineArtifactName}}/PackageInfo'

eng/pipelines/templates/stages/archetype-sdk-client.yml

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,9 @@ extends:
148148
parameters:
149149
DependsOn: "Build"
150150
ServiceDirectory: ${{ parameters.ServiceDirectory }}
151-
Artifacts: ${{ parameters.Artifacts }}
151+
Artifacts:
152+
- ${{ each artifact in parameters.Artifacts }}:
153+
- ${{ if ne(artifact.releaseInBatch, 'false')}}:
154+
- ${{ artifact }}
152155
TestPipeline: ${{ eq(parameters.ServiceDirectory, 'canary') }}
153156
PipelineArtifactName: packages

eng/scripts/Language-Settings.ps1

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
$Language = "rust"
22
$LanguageDisplayName = "Rust"
33
$PackageRepository = "crates.io"
4-
$packagePattern = "Cargo.toml"
4+
$packagePattern = "*.crate"
55
#$MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/main/_data/releases/latest/rust-packages.csv"
66
$GithubUri = "https://github.com/Azure/azure-sdk-for-rust"
77
$PackageRepositoryUri = "https://crates.io/crates"
@@ -139,15 +139,32 @@ function Get-rust-AdditionalValidationPackagesFromPackageSet ($packagesWithChang
139139
return $additionalPackages ?? @()
140140
}
141141

142+
# $GetPackageInfoFromPackageFileFn = "Get-${Language}-PackageInfoFromPackageFile"
142143
function Get-rust-PackageInfoFromPackageFile([IO.FileInfo]$pkg, [string]$workingDirectory) {
143-
#$pkg will be a FileInfo object for the Cargo.toml file in a package artifact directory
144-
$package = cargo read-manifest --manifest-path $pkg.FullName | ConvertFrom-Json
144+
# Create a temporary folder for extraction
145+
$extractionPath = [System.IO.Path]::Combine([System.IO.Path]::GetTempPath(), [System.IO.Path]::GetRandomFileName())
146+
New-Item -ItemType Directory -Path $extractionPath | Out-Null
147+
148+
# Extract the .crate file (which is a tarball) to the temporary folder
149+
tar -xvf $pkg.FullName -C $extractionPath
150+
$cargoTomlPath = [System.IO.Path]::Combine($extractionPath, $pkg.BaseName, 'Cargo.toml')
151+
152+
Write-Host "Reading package info from $cargoTomlPath"
153+
if (!(Test-Path $cargoTomlPath)) {
154+
$message = "The Cargo.toml file was not found in the package artifact at $cargoTomlPath"
155+
LogError $message
156+
throw $message
157+
}
158+
159+
$package = cargo read-manifest --manifest-path $cargoTomlPath | ConvertFrom-Json
145160

146161
$packageName = $package.name
147162
$packageVersion = $package.version
148163

149-
$changeLogLoc = Get-ChildItem -Path $pkg.DirectoryName -Filter "CHANGELOG.md" | Select-Object -First 1
150-
$readmeContentLoc = Get-ChildItem -Path $pkg.DirectoryName -Filter "README.md" | Select-Object -First 1
164+
$packageAssetPath = [System.IO.Path]::Combine($extractionPath, "$packageName-$packageVersion")
165+
166+
$changeLogLoc = Get-ChildItem -Path $packageAssetPath -Filter "CHANGELOG.md" | Select-Object -First 1
167+
$readmeContentLoc = Get-ChildItem -Path $packageAssetPath -Filter "README.md" | Select-Object -First 1
151168

152169
if ($changeLogLoc) {
153170
$releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $packageVersion

0 commit comments

Comments
 (0)