Skip to content

Commit 681016a

Browse files
brettsambrettsam
committed
improving secret checkouts
1 parent 1c538e2 commit 681016a

File tree

4 files changed

+122
-90
lines changed

4 files changed

+122
-90
lines changed

‎azure-pipelines.yml‎

Lines changed: 44 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -180,7 +180,6 @@ jobs:
180180
artifact: packages
181181

182182
- job: RunUnitTests
183-
dependsOn: InitializePipeline
184183
pool:
185184
vmImage: 'windows-2019'
186185
steps:
@@ -201,11 +200,8 @@ jobs:
201200
**\WebJobs.Script.Tests.csproj
202201
203202
- job: RunNonE2EIntegrationTests
204-
dependsOn: InitializePipeline
205203
pool:
206204
vmImage: 'windows-2019'
207-
variables:
208-
keyVaultSuffix: $[ dependencies.InitializePipeline.outputs['Initialize.LeaseBlob'] ]
209205
steps:
210206
- task: UseDotNet@2
211207
inputs:
@@ -215,11 +211,27 @@ jobs:
215211
- task: UseNode@1
216212
inputs:
217213
version: '10.x'
214+
- task: PowerShell@2
215+
displayName: 'Install Az.Storage Powershell module'
216+
inputs:
217+
targetType: 'inline'
218+
script: 'Install-Module -Name Az.Storage -RequiredVersion 1.11.0 -Scope CurrentUser -Force -AllowClobber'
219+
- task: AzureKeyVault@1
220+
inputs:
221+
# Note: This is actually a Service Connection in DevOps, not an Azure subscription name
222+
azureSubscription: 'Azure-Functions-Host-CI'
223+
keyVaultName: 'azure-functions-host-ci'
224+
secretsFilter: '*'
225+
- task: PowerShell@2
226+
displayName: 'Checkout secrets'
227+
inputs:
228+
filePath: '$(Build.Repository.LocalPath)\build\checkout-secrets.ps1'
229+
arguments: '-connectionString ''$(Storage-azurefunctionshostci0)'''
218230
- task: AzureKeyVault@1
219231
inputs:
220232
# Note: This is actually a Service Connection in DevOps, not an Azure subscription name
221233
azureSubscription: 'Azure-Functions-Host-CI'
222-
keyVaultName: azure-functions-host-$(keyVaultSuffix)
234+
keyVaultName: azure-functions-host-$(LeaseBlob)
223235
secretsFilter: '*'
224236
- task: DotNetCoreCLI@2
225237
displayName: 'Non-E2E integration tests'
@@ -236,13 +248,16 @@ jobs:
236248
AzureWebJobsEventHubReceiver: $(EventHub)
237249
AzureWebJobsSecretStorageKeyVaultConnectionString: $(KeyVaultConnectionString)
238250
AzureWebJobsSecretStorageKeyVaultName: $(KeyVaultName)
251+
- task: PowerShell@2
252+
condition: always()
253+
displayName: 'Checkin secrets'
254+
inputs:
255+
filePath: '$(Build.Repository.LocalPath)\build\checkin-secrets.ps1'
256+
arguments: '-connectionString ''$(Storage-azurefunctionshostci0)'' -leaseBlob $(LeaseBlob) -leaseToken $(LeaseToken)'
239257

240258
- job: RunIntegrationTests
241-
dependsOn: InitializePipeline
242259
pool:
243260
vmImage: 'windows-2019'
244-
variables:
245-
keyVaultSuffix: $[ dependencies.InitializePipeline.outputs['Initialize.LeaseBlob'] ]
246261
steps:
247262
- task: UseDotNet@2
248263
inputs:
@@ -256,11 +271,27 @@ jobs:
256271
inputs:
257272
versionSpec: '3.7.x'
258273
addToPath: true
274+
- task: PowerShell@2
275+
displayName: 'Install Az.Storage Powershell module'
276+
inputs:
277+
targetType: 'inline'
278+
script: 'Install-Module -Name Az.Storage -RequiredVersion 1.11.0 -Scope CurrentUser -Force -AllowClobber'
279+
- task: AzureKeyVault@1
280+
inputs:
281+
# Note: This is actually a Service Connection in DevOps, not an Azure subscription name
282+
azureSubscription: 'Azure-Functions-Host-CI'
283+
keyVaultName: 'azure-functions-host-ci'
284+
secretsFilter: '*'
285+
- task: PowerShell@2
286+
displayName: 'Checkout secrets'
287+
inputs:
288+
filePath: '$(Build.Repository.LocalPath)\build\checkout-secrets.ps1'
289+
arguments: '-connectionString ''$(Storage-azurefunctionshostci0)'''
259290
- task: AzureKeyVault@1
260291
inputs:
261292
# Note: This is actually a Service Connection in DevOps, not an Azure subscription name
262293
azureSubscription: 'Azure-Functions-Host-CI'
263-
keyVaultName: azure-functions-host-$(keyVaultSuffix)
294+
keyVaultName: azure-functions-host-$(LeaseBlob)
264295
secretsFilter: '*'
265296
- task: PowerShell@2
266297
displayName: 'Set environment variables'
@@ -375,28 +406,9 @@ jobs:
375406
arguments: '--filter "Group=ContainerInstanceTests"'
376407
projects: |
377408
**\WebJobs.Script.Tests.Integration.csproj
378-
379-
- job: FinishPipeline
380-
dependsOn:
381-
- InitializePipeline
382-
- BuildArtifacts
383-
- RunUnitTests
384-
- RunIntegrationTests
385-
- RunNonE2EIntegrationTests
386-
condition: always()
387-
variables:
388-
leaseBlob: $[ dependencies.InitializePipeline.outputs['Initialize.LeaseBlob'] ]
389-
leaseToken: $[ dependencies.InitializePipeline.outputs['Initialize.LeaseToken'] ]
390-
pool:
391-
vmImage: 'windows-2019'
392-
steps:
393-
- task: AzureKeyVault@1
394-
inputs:
395-
azureSubscription: 'Azure-Functions-Host-CI'
396-
keyVaultName: 'azure-functions-host-ci'
397-
secretsFilter: '*'
398409
- task: PowerShell@2
399-
displayName: 'Finish'
410+
condition: always()
411+
displayName: 'Checkin secrets'
400412
inputs:
401-
filePath: '$(Build.Repository.LocalPath)\build\finish-pipeline.ps1'
402-
arguments: '-connectionString ''$(Storage-azurefunctionshostci0)'' -leaseBlob $(leaseBlob) -leaseToken $(leaseToken)'
413+
filePath: '$(Build.Repository.LocalPath)\build\checkin-secrets.ps1'
414+
arguments: '-connectionString ''$(Storage-azurefunctionshostci0)'' -leaseBlob $(LeaseBlob) -leaseToken $(LeaseToken)'

‎build/finish-pipeline.ps1‎ renamed to ‎build/checkin-secrets.ps1‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@ if ($leaseToken -eq "") {
1616

1717
Write-Host "Breaking lease for $leaseBlob."
1818

19-
$storageContext = New-AzureStorageContext -ConnectionString $connectionString
20-
$blob = Get-AzureStorageBlob -Context $storageContext -Container "ci-locks" -Blob $leaseBlob
19+
$storageContext = New-AzStorageContext -ConnectionString $connectionString
20+
$blob = Get-AzStorageBlob -Context $storageContext -Container "ci-locks" -Blob $leaseBlob
2121

22-
$accessCondition = New-Object -TypeName Microsoft.WindowsAzure.Storage.AccessCondition
22+
$accessCondition = New-Object -TypeName Microsoft.Azure.Storage.AccessCondition
2323
$accessCondition.LeaseId = $leaseToken
2424
$blob.ICloudBlob.ReleaseLease($accessCondition)

‎build/checkout-secrets.ps1‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
param (
2+
[string]$connectionString = ""
3+
)
4+
5+
function AcquireLease($blob) {
6+
try {
7+
return $blob.ICloudBlob.AcquireLease($null, $null, $null, $null, $null)
8+
} catch {
9+
Write-Host " Error: $_"
10+
return $null
11+
}
12+
}
13+
14+
# use this for tracking metadata in lease blobs
15+
$buildName = "3.0." + $env:buildNumber + "_" + $env:SYSTEM_JOBDISPLAYNAME
16+
17+
$azVersion = "1.11.0"
18+
Import-Module Az.Storage
19+
$azModule = Get-Module -Name Az.Storage
20+
if ($azModule.Version -ne $azVersion) {
21+
throw "Az.Storage module version $azVersion was not found. Current version: $($azModule.Version)"
22+
}
23+
24+
# get a blob lease to prevent test overlap
25+
$storageContext = New-AzStorageContext -ConnectionString $connectionString
26+
27+
While($true) {
28+
$blobs = Get-AzStorageBlob -Context $storageContext -Container "ci-locks"
29+
$token = $null
30+
31+
# shuffle the blobs for random ordering
32+
$blobs = $blobs | Sort-Object {Get-Random}
33+
34+
Write-Host "Looking for unleased ci-lock blobs (list is shuffled):"
35+
Foreach ($blob in $blobs) {
36+
$name = $blob.Name
37+
$leaseStatus = $blob.ICloudBlob.Properties.LeaseStatus
38+
39+
Write-Host " ${name}: $leaseStatus"
40+
41+
if ($leaseStatus -eq "Locked") {
42+
continue
43+
}
44+
45+
Write-Host " Attempting to acquire lease on $name."
46+
$token = AcquireLease $blob
47+
if ($token -ne $null) {
48+
Write-Host " Lease acquired on $name. LeaseId: '$token'"
49+
Write-Host "##vso[task.setvariable variable=LeaseBlob]$name"
50+
Write-Host "##vso[task.setvariable variable=LeaseToken]$token"
51+
try {
52+
$blob.ICloudBlob.FetchAttributes()
53+
$blob.ICloudBlob.Metadata["Build"] = $buildName
54+
$accessCondition = New-Object -TypeName Microsoft.Azure.Storage.AccessCondition
55+
$accessCondition.LeaseId = $token
56+
$blob.ICloudBlob.SetMetadata($accessCondition)
57+
} catch {
58+
# best effort
59+
Write-Host "Warning: unable to update blob metadata. Continuing. $_"
60+
}
61+
break
62+
} else {
63+
Write-Host " Lease not acquired on $name."
64+
}
65+
}
66+
67+
if ($token -ne $null) {
68+
break
69+
}
70+
71+
$delay = 30
72+
Write-Host "No lease acquired. Waiting $delay seconds to try again. This run cannot begin until it acquires a lease on a CI test environment."
73+
Start-Sleep -s $delay
74+
Write-Host ""
75+
}

‎build/initialize-pipeline.ps1‎

Lines changed: 0 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,3 @@
1-
param (
2-
[string]$connectionString = ""
3-
)
4-
5-
function AcquireLease($blob) {
6-
try {
7-
return $blob.ICloudBlob.AcquireLease($null, $null, $null, $null, $null)
8-
} catch {
9-
Write-Host " Error: $_"
10-
return $null
11-
}
12-
}
13-
141
$buildReason = $env:BUILD_REASON
152

163
if ($buildReason -eq "PullRequest") {
@@ -22,46 +9,4 @@ if ($buildReason -eq "PullRequest") {
229
Write-Host "##vso[task.setvariable variable=BuildArtifacts;isOutput=true]true"
2310
Write-Host "Setting 'BuildArtifacts' to true."
2411
}
25-
}
26-
27-
# get a blob lease to prevent test overlap
28-
$storageContext = New-AzureStorageContext -ConnectionString $connectionString
29-
While($true) {
30-
$blobs = Get-AzureStorageBlob -Context $storageContext -Container "ci-locks"
31-
$token = $null
32-
33-
# shuffle the blobs for random ordering
34-
$blobs = $blobs | Sort-Object {Get-Random}
35-
36-
Write-Host "Looking for unleased ci-lock blobs (list is shuffled):"
37-
Foreach ($blob in $blobs) {
38-
$name = $blob.Name
39-
$leaseStatus = $blob.ICloudBlob.Properties.LeaseStatus
40-
41-
Write-Host " ${name}: $leaseStatus"
42-
43-
if ($leaseStatus -eq "Locked") {
44-
continue
45-
}
46-
47-
Write-Host " Attempting to acquire lease on $name."
48-
$token = AcquireLease $blob
49-
if ($token -ne $null) {
50-
Write-Host " Lease acquired on $name. LeaseId: '$token'"
51-
Write-Host "##vso[task.setvariable variable=LeaseBlob;isOutput=true]$name"
52-
Write-Host "##vso[task.setvariable variable=LeaseToken;isOutput=true]$token"
53-
break
54-
} else {
55-
Write-Host " Lease not acquired on $name."
56-
}
57-
}
58-
59-
if ($token -ne $null) {
60-
break
61-
}
62-
63-
$delay = 30
64-
Write-Host "No lease acquired. Waiting $delay seconds to try again. This run cannot begin until it acquires a lease on a CI test environment."
65-
Start-Sleep -s $delay
66-
Write-Host ""
6712
}

0 commit comments

Comments
 (0)