Skip to content

Commit bdc9e7b

Browse files
JimSuplizioJimSuplizio
authored
Add namespaces to package info (#37092)
* Add namespaces to packageInfo json as part of the build * Update script description * Move step above the publish-1es-artifact step * Fix the sync of the ArtifactsList convert to actually make it an array * Fix the table output in the powershell script to remove GroupId which was a copy/paste from java * Update comment for feedback
1 parent e1084e5 commit bdc9e7b

File tree

3 files changed

+161
-33
lines changed

3 files changed

+161
-33
lines changed

‎eng/pipelines/templates/steps/build-extended-artifacts.yml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,16 @@ steps:
8080

8181
- ${{ parameters.BeforePublishSteps }}
8282

83+
- task: Powershell@2
84+
inputs:
85+
filePath: $(Build.SourcesDirectory)/eng/scripts/Save-Package-Namespaces-Property.ps1
86+
arguments: >
87+
-ArtifactStagingDirectory "$(Build.ArtifactStagingDirectory)"
88+
-ArtifactsList ('${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json)
89+
pwsh: true
90+
workingDirectory: $(Pipeline.Workspace)
91+
displayName: Update package properties with namespaces
92+
8393
- template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml
8494
parameters:
8595
ArtifactPath: '$(Build.ArtifactStagingDirectory)'

‎eng/scripts/Save-Package-Namespaces-Property.ps1‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
<#
2+
.SYNOPSIS
3+
Given an artifact staging directory and the artifacts list, loop through all artifacts
4+
that docs that aren't skipping docs creation and add the namespace to the PackageInfo
5+
json file.
6+
7+
.DESCRIPTION
8+
Given an artifact staging directory and the artifacts list, loop through all artifacts
9+
that docs that aren't skipping docs creation and add the namespace to the PackageInfo
10+
json file. This is done using the .whl file from the build.
11+
12+
.PARAMETER ArtifactStagingDirectory
13+
The root directory of the staged artifacts. The PackageInfo files will be in the
14+
PackageInfo subdirectory. The whl files are going to be in the subdirectory which
15+
is the same as the artifact's name but artifact name in the file's name will have
16+
underscores instead of dashes.
17+
18+
.PARAMETER ArtifactsList
19+
The list of artifacts to gather namespaces for, this is only done for libraries that are
20+
producing docs.
21+
-ArtifactsList ('${{ convertToJson(parameters.Artifacts) }}' | ConvertFrom-Json)
22+
#>
23+
[CmdletBinding()]
24+
Param (
25+
[Parameter(Mandatory = $True)]
26+
[string] $ArtifactStagingDirectory,
27+
[Parameter(Mandatory=$true)]
28+
[array] $ArtifactsList
29+
)
30+
31+
$ArtifactsList = $ArtifactsList | Where-Object -Not "skipPublishDocMs"
32+
33+
. (Join-Path $PSScriptRoot ".." common scripts common.ps1)
34+
35+
if (-not $ArtifactsList) {
36+
Write-Host "ArtifactsList is empty, nothing to process. This can happen if skipPublishDocMs is set to true for all libraries being built."
37+
exit 0
38+
}
39+
40+
Write-Host "ArtifactStagingDirectory=$ArtifactStagingDirectory"
41+
if (-not (Test-Path -Path $ArtifactStagingDirectory)) {
42+
LogError "ArtifactStagingDirectory '$ArtifactStagingDirectory' does not exist."
43+
exit 1
44+
}
45+
46+
Write-Host ""
47+
Write-Host "ArtifactsList:"
48+
$ArtifactsList | Format-Table -Property Name | Out-String | Write-Host
49+
50+
$packageInfoDirectory = Join-Path $ArtifactStagingDirectory "PackageInfo"
51+
52+
$foundError = $false
53+
# At this point the packageInfo files should have been already been created.
54+
# The only thing being done here is adding or updating namespaces for libraries
55+
# that will be producing docs.
56+
foreach($artifact in $ArtifactsList) {
57+
# Get the version from the packageInfo file
58+
$packageInfoFile = Join-Path $packageInfoDirectory "$($artifact.Name).json"
59+
Write-Host "processing $($packageInfoFile.FullName)"
60+
$packageInfo = ConvertFrom-Json (Get-Content $packageInfoFile -Raw)
61+
$version = $packageInfo.Version
62+
# If the dev version is set, use that. This will be set for nightly builds
63+
if ($packageInfo.DevVersion) {
64+
$version = $packageInfo.DevVersion
65+
}
66+
# From the $packageInfo piece together the path to the javadoc jar file
67+
$WhlDir = Join-Path $ArtifactStagingDirectory $packageInfo.Name
68+
$WhlName = $packageInfo.Name.Replace("-","_")
69+
$WhlFile = Get-ChildItem -Path $WhlDir -File -Filter "$whlName-$version*.whl"
70+
71+
if (!(Test-Path $WhlFile -PathType Leaf)) {
72+
LogError "Whl file for, $($packageInfo.Name), was not found in $WhlDir. Please ensure that a .whl file is being produced for the library."
73+
$foundError = $true
74+
continue
75+
}
76+
$namespaces = Get-NamespacesFromWhlFile $packageInfo.Name $version -PythonWhlFile $WhlFile
77+
if ($namespaces.Count -gt 0) {
78+
Write-Host "Adding/Updating Namespaces property with the following namespaces:"
79+
$namespaces | Write-Host
80+
if ($packageInfo.PSobject.Properties.Name -contains "Namespaces") {
81+
Write-Host "Contains Namespaces property, updating"
82+
$packageInfo.Namespaces = $namespaces
83+
}
84+
else {
85+
Write-Host "Adding Namespaces property"
86+
$packageInfo = $packageInfo | Add-Member -MemberType NoteProperty -Name Namespaces -Value $namespaces -PassThru
87+
}
88+
$packageInfoJson = ConvertTo-Json -InputObject $packageInfo -Depth 100
89+
Write-Host "The updated packageInfo for $packageInfoFile is:"
90+
Write-Host "$packageInfoJson"
91+
Set-Content `
92+
-Path $packageInfoFile `
93+
-Value $packageInfoJson
94+
} else {
95+
LogError "Unable to determine namespaces for $($packageInfo.Name). Please ensure that skipPublishDocMs isn't incorrectly set to true."
96+
$foundError = $true
97+
}
98+
}
99+
100+
if ($foundError) {
101+
exit 1
102+
}
103+
exit 0

‎eng/scripts/docs/Docs-ToC.ps1‎

Lines changed: 48 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -28,48 +28,63 @@ function Get-NamespacesFromWhlFile {
2828
param(
2929
[Parameter(Mandatory=$true)] [string]$Library,
3030
[Parameter(Mandatory=$true)] [string]$Version,
31-
[Parameter(Mandatory=$false)] [string]$ExtraIndexUrl = ""
31+
[Parameter(Mandatory=$false)] [string]$ExtraIndexUrl = "",
32+
[Parameter(Mandatory=$false)] [string]$PythonWhlFile = $null
3233
)
3334

3435
$destination = (Join-Path ([System.IO.Path]::GetTempPath()) "$Library$Version")
36+
New-Item $destination -ItemType Directory | Out-Null
3537
$namespaces = @()
3638

3739
try {
3840

41+
if ($PythonWhlFile) {
42+
if (Test-Path $PythonWhlFile -PathType Leaf) {
43+
Write-Host "Copying $PythonWhlFile to $destination"
44+
Copy-Item $PythonWhlFile -Destination $destination
45+
} else {
46+
LogWarning "$PythonWhlFile, does not exist."
47+
}
48+
} else {
49+
$success = Get-WhlFile $Library $Version $destination $ExtraIndexUrl
50+
if (-not $success) {
51+
LogWarning "Could not download Whl file for $Library $Version"
52+
}
53+
}
3954
# Pulling the whl file generates output, make sure it's sent to null so
4055
# it's not returned as part of this function.
41-
$success = Get-WhlFile $Library $Version $destination $ExtraIndexUrl
42-
if ($success) {
43-
44-
# Each library gets its own temporary directory. There should only be one whl
45-
# file in the destination directory
46-
$whlFile = Get-ChildItem -Path $destination -File -Filter "*.whl" | Select-Object -First 1
47-
$unpackDir = Join-Path -Path $destination -ChildPath "$Library-$Version"
48-
Expand-Archive -Path $whlFile -DestinationPath $unpackDir
49-
50-
# Look for any directory that contains __init__.py with the following exceptions:
51-
# 1. *.dist-info directories shouldn't be included in the results.
52-
# 2. If any subdirectory starts with "_" it's internal and needs to be skipped
53-
# 3. If there's a root level directory named "azure" with an __init__.py file then
54-
# needs to be skipped. This doesn't happen with libraries released from the
55-
# azure-sdk-for-python repository but there are older libraries that are in the
56-
# docs directories which are/were released outside of the repository where this
57-
# is true.
58-
$rootLevelAzureDir = Join-Path -Path $unpackDir -ChildPath "azure"
59-
$namespaceDirs = Get-ChildItem -Path $unpackDir -Recurse -Filter "__init__.py" |
60-
Where-Object{$_.DirectoryName -notlike "*.dist-info"} |
61-
Where-Object{$_.DirectoryName -notlike "*$([IO.Path]::DirectorySeparatorChar)_*" } |
62-
Where-Object{$_.DirectoryName -ine $rootLevelAzureDir}
63-
foreach($namespaceDir in $namespaceDirs) {
64-
# Strip off the root directy, everything left will be subDir1/subDir2/../subDirN.
65-
# The directory separators will be replaced with periods to compute the
66-
# namespace
67-
$partialDir = $namespaceDir.DirectoryName.Replace($unpackDir + $([IO.Path]::DirectorySeparatorChar), "")
68-
$namespaces += $partialDir.Replace([IO.Path]::DirectorySeparatorChar, ".")
69-
# Since only the primary namespace is being pulled, break out of the loop after
70-
# the first one.
71-
break
72-
}
56+
# Each library gets its own temporary directory. There should only be one whl
57+
# file in the destination directory
58+
$whlFile = Get-ChildItem -Path $destination -File -Filter "*.whl" | Select-Object -First 1
59+
# If we can't download the file or the passed in file doesn't exist then the whlFile
60+
# won't exist, there's nothing to process and an empty namesapces list will be returned
61+
if ($whlFile) {
62+
$unpackDir = Join-Path -Path $destination -ChildPath "$Library-$Version"
63+
Expand-Archive -Path $whlFile -DestinationPath $unpackDir
64+
65+
# Look for any directory that contains __init__.py with the following exceptions:
66+
# 1. *.dist-info directories shouldn't be included in the results.
67+
# 2. If any subdirectory starts with "_" it's internal and needs to be skipped
68+
# 3. If there's a root level directory named "azure" with an __init__.py file then
69+
# needs to be skipped. This doesn't happen with libraries released from the
70+
# azure-sdk-for-python repository but there are older libraries that are in the
71+
# docs directories which are/were released outside of the repository where this
72+
# is true.
73+
$rootLevelAzureDir = Join-Path -Path $unpackDir -ChildPath "azure"
74+
$namespaceDirs = Get-ChildItem -Path $unpackDir -Recurse -Filter "__init__.py" |
75+
Where-Object{$_.DirectoryName -notlike "*.dist-info"} |
76+
Where-Object{$_.DirectoryName -notlike "*$([IO.Path]::DirectorySeparatorChar)_*" } |
77+
Where-Object{$_.DirectoryName -ine $rootLevelAzureDir}
78+
foreach($namespaceDir in $namespaceDirs) {
79+
# Strip off the root directy, everything left will be subDir1/subDir2/../subDirN.
80+
# The directory separators will be replaced with periods to compute the
81+
# namespace
82+
$partialDir = $namespaceDir.DirectoryName.Replace($unpackDir + $([IO.Path]::DirectorySeparatorChar), "")
83+
$namespaces += $partialDir.Replace([IO.Path]::DirectorySeparatorChar, ".")
84+
# Since only the primary namespace is being pulled, break out of the loop after
85+
# the first one.
86+
break
87+
}
7388
}
7489
}
7590
finally {

0 commit comments

Comments
 (0)