Skip to content

Commit 07a8328

Browse files
JimSuplizioJimSuplizio
committed
Updates for docs ToC generation
1 parent bea2d61 commit 07a8328

File tree

2 files changed

+166
-24
lines changed

2 files changed

+166
-24
lines changed

‎eng/scripts/Language-Settings.ps1‎

Lines changed: 26 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,29 @@ function Get-python-PackageInfoFromPackageFile ($pkg, $workingDirectory)
128128
}
129129
}
130130

131+
# This is the GetDocsMsDevLanguageSpecificPackageInfoFn implementation
132+
function Get-python-DocsMsDevLanguageSpecificPackageInfo($packageInfo, $packageSourceOverride) {
133+
# If the default namespace isn't in the package info then it needs to be added
134+
if (!$packageInfo.Namespaces) {
135+
# If the Version is INGORE that means it's a source install and those
136+
# ones need to be done by hand
137+
if ($packageInfo.Version -ine "IGNORE") {
138+
$version = $packageInfo.Version
139+
# If the dev version is set, use that
140+
if ($packageInfo.DevVersion) {
141+
$version = $packageInfo.DevVersion
142+
}
143+
$namespaces = Get-NamespacesFromWhlFile $packageInfo.Name $version $packageSourceOverride
144+
if ($namespaces.Count -gt 0) {
145+
$packageInfo | Add-Member -Type NoteProperty -Name "Namespaces" -Value $namespaces
146+
} else {
147+
LogWarning "Unable to find namespaces $($packageInfo.Name):$version"
148+
}
149+
}
150+
}
151+
return $packageInfo
152+
}
153+
131154
# Stage and Upload Docs to blob Storage
132155
function Publish-python-GithubIODocs ($DocLocation, $PublicArtifactLocation)
133156
{
@@ -618,8 +641,8 @@ function Validate-Python-DocMsPackages ($PackageInfo, $PackageInfos, $PackageSou
618641

619642
$allSucceeded = $true
620643
foreach ($item in $PackageInfos) {
621-
# Some packages
622-
if ($item.Version -eq 'IGNORE') {
644+
# Some packages
645+
if ($item.Version -eq 'IGNORE') {
623646
continue
624647
}
625648

@@ -656,7 +679,7 @@ function Get-python-DirectoriesForGeneration () {
656679

657680
function Update-python-GeneratedSdks([string]$PackageDirectoriesFile) {
658681
$packageDirectories = Get-Content $PackageDirectoriesFile | ConvertFrom-Json
659-
682+
660683
$directoriesWithErrors = @()
661684

662685
foreach ($directory in $packageDirectories) {

‎eng/scripts/docs/Docs-ToC.ps1‎

Lines changed: 140 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,84 @@
1-
function GetOnboardingFileForMoniker($docRepoLocation, $moniker) {
1+
# Download the .whl file into the given destination directory.
2+
function Get-WhlFile {
3+
param(
4+
[Parameter(Mandatory=$true)] [string]$Library,
5+
[Parameter(Mandatory=$true)] [string]$Version,
6+
[Parameter(Mandatory=$true)] [string]$Destination,
7+
[Parameter(Mandatory=$false)] [string]$ExtraIndexUrl = ""
8+
)
9+
$pipCommandArgs = "$Library==$Version"
10+
if ($ExtraIndexUrl) {
11+
$pipCommandArgs += " --extra-index-url $ExtraIndexUrl"
12+
}
13+
14+
# download the whl file
15+
Write-Host "pip download --quiet --only-binary :all: --dest $Destination --no-cache --no-deps $pipCommandArgs"
16+
pip download --quiet --only-binary :all: --dest $Destination --no-cache --no-deps $pipCommandArgs
17+
if($LASTEXITCODE -ne 0) {
18+
return $false
19+
}
20+
return $true
21+
}
22+
23+
# Given a library and version, download the .whl file and unpack it to get the namespaces.
24+
# A temp directory is used for the download and unpack which is cleaned up afterwards.
25+
function Get-NamespacesFromWhlFile {
26+
param(
27+
[Parameter(Mandatory=$true)] [string]$Library,
28+
[Parameter(Mandatory=$true)] [string]$Version,
29+
[Parameter(Mandatory=$false)] [string]$ExtraIndexUrl = ""
30+
)
31+
32+
$destination = (Join-Path ([System.IO.Path]::GetTempPath()) "$Library$Version")
33+
$namespaces = @()
34+
35+
try {
36+
37+
# Pulling the whl file generates output, make sure it's sent to null so
38+
# it's not returned as part of this function.
39+
$success = Get-WhlFile $Library $Version $destination $ExtraIndexUrl
40+
if ($success) {
41+
42+
# Each library gets its own temporary directory. There should only be one whl
43+
# file in the destination directory
44+
$whlFile = Get-ChildItem -Path $destination -File -Filter "*.whl" | Select-Object -First 1
45+
$unpackDir = Join-Path -Path $destination -ChildPath "$Library-$Version"
46+
Expand-Archive -Path $whlFile -DestinationPath $unpackDir
47+
48+
# Look for any directory that contains __init__.py with the following exceptions:
49+
# 1. *.dist-info directories shouldn't be included in the results.
50+
# 2. If any subdirectory starts with "_" it's internal and needs to be skipped
51+
# 3. If there's a root level directory named "azure" with an __init__.py file then
52+
# needs to be skipped. This doesn't happen with libraries released from the
53+
# azure-sdk-for-python repository but there are older libraries that are in the
54+
# docs directories which are/were released outside of the repository where this
55+
# is true.
56+
$rootLevelAzureDir = Join-Path -Path $unpackDir -ChildPath "azure"
57+
$namespaceDirs = Get-ChildItem -Path $unpackDir -Recurse -Filter "__init__.py" |
58+
Where-Object{$_.DirectoryName -notlike "*.dist-info"} |
59+
Where-Object{$_.DirectoryName -notlike "*$([IO.Path]::DirectorySeparatorChar)_*" } |
60+
Where-Object{$_.DirectoryName -ine $rootLevelAzureDir}
61+
foreach($namespaceDir in $namespaceDirs) {
62+
# strip off the root directy, everything left will be subDir1/subDir2 which needs
63+
$partialDir = $namespaceDir.DirectoryName.Replace($unpackDir + $([IO.Path]::DirectorySeparatorChar), "")
64+
$namespaces += $partialDir.Replace([IO.Path]::DirectorySeparatorChar, ".")
65+
# Since only the primary namespace is being pulled, break out of the loop after
66+
# the first one.
67+
break
68+
}
69+
}
70+
}
71+
finally {
72+
# Clean up the temp directory if it was created
73+
if (Test-Path $destination) {
74+
Remove-Item $destination -Recurse -Force
75+
}
76+
}
77+
# Make sure this always returns an array
78+
Write-Output -NoEnumerate $namespaces
79+
}
80+
81+
function GetOnboardingFileForMoniker($docRepoLocation, $moniker) {
282
$packageOnboardingFile = 'ci-configs/packages-latest.json'
383
if ($moniker -eq 'preview') {
484
$packageOnboardingFile = 'ci-configs/packages-preview.json'
@@ -21,7 +101,7 @@ function Get-python-OnboardedDocsMsPackagesForMoniker($DocRepoLocation, $moniker
21101
continue
22102
}
23103
$packageName = $spec.package_info.name
24-
104+
25105
$jsonFile = "$DocRepoLocation/metadata/$moniker/$packageName.json"
26106
if (Test-Path $jsonFile) {
27107
$onboardedPackages[$packageName] = ConvertFrom-Json (Get-Content $jsonFile -Raw)
@@ -55,7 +135,7 @@ function Get-python-OnboardedDocsMsPackages($DocRepoLocation) {
55135
return $onboardedPackages
56136
}
57137

58-
function GetPackageLevelReadme($packageMetadata) {
138+
function GetPackageLevelReadme($packageMetadata) {
59139
# Fallback for package name
60140
$packageLevelReadmeName = $packageMetadata.Package
61141
if ($packageLevelReadmeName.StartsWith('azure-')) {
@@ -70,19 +150,70 @@ function GetPackageLevelReadme($packageMetadata) {
70150
}
71151
return $packageLevelReadmeName
72152
}
73-
74-
function Get-python-PackageLevelReadme($packageMetadata) {
153+
154+
# This function is called within a loop. To prevent multiple reads of the same
155+
# file data, this uses a script-scoped cache variable.
156+
$script:PackageMetadataJsonLookup = $null
157+
function GetPackageMetadataJsonLookup($docRepoLocation) {
158+
if ($script:PackageMetadataJsonLookup) {
159+
return $script:PackageMetadataJsonLookup
160+
}
161+
162+
$script:PackageMetadataJsonLookup = @{}
163+
$packageJsonFiles = Get-ChildItem $docRepoLocation/metadata/ -Filter *.json -Recurse
164+
foreach ($packageJsonFile in $packageJsonFiles) {
165+
$packageJson = Get-Content $packageJsonFile -Raw | ConvertFrom-Json -AsHashtable
166+
167+
if (!$script:PackageMetadataJsonLookup.ContainsKey($packageJson.Name)) {
168+
$script:PackageMetadataJsonLookup[$packageJson.Name] = @($packageJson)
169+
} else {
170+
$script:PackageMetadataJsonLookup[$packageJson.Name] += $packageJson
171+
}
172+
}
173+
174+
return $script:PackageMetadataJsonLookup
175+
}
176+
177+
# Grab the namespaces from the json file
178+
function Get-Toc-Children($package, $docRepoLocation) {
179+
$packageTable = GetPackageMetadataJsonLookup $docRepoLocation
180+
181+
$namespaces = @()
182+
if ($packageTable.ContainsKey($package)) {
183+
foreach ($entry in $packageTable[$package]) {
184+
if ($entry.ContainsKey('Namespaces')) {
185+
$namespaces += $entry['Namespaces']
186+
}
187+
}
188+
}
189+
# Sort the array and clean out any dupes (there shouldn't be any but better safe than sorry)
190+
$namespaces = $namespaces | Sort-Object -Unique
191+
# Ensure that this always returns an array, even if there's one item or 0 items
192+
Write-Output -NoEnumerate $namespaces
193+
}
194+
195+
function Get-python-PackageLevelReadme($packageMetadata) {
75196
return GetPackageLevelReadme -packageMetadata $packageMetadata
76197
}
77198

78-
function Get-python-DocsMsTocData($packageMetadata, $docRepoLocation) {
199+
# Defined in common.ps1
200+
# $GetDocsMsTocDataFn = "Get-${Language}-DocsMsTocData"
201+
function Get-python-DocsMsTocData($packageMetadata, $docRepoLocation, $PackageSourceOverride) {
79202
$packageLevelReadmeName = GetPackageLevelReadme -packageMetadata $packageMetadata
80-
81203
$packageTocHeader = GetDocsTocDisplayName $packageMetadata
204+
205+
# Get-Toc-Children always returns an array, even if there's only 1 item or it's empty
206+
$children = Get-Toc-Children `
207+
-package $packageMetadata.Package `
208+
-docRepoLocation $docRepoLocation
209+
if ($children.Count -eq 0) {
210+
LogWarning "Did not find the package namespaces for $($packageMetadata.Package)"
211+
}
212+
82213
$output = [PSCustomObject]@{
83214
PackageLevelReadmeHref = "~/docs-ref-services/{moniker}/$packageLevelReadmeName-readme.md"
84215
PackageTocHeader = $packageTocHeader
85-
TocChildren = @($packageMetadata.Package)
216+
TocChildren = $children
86217
}
87218

88219
return $output
@@ -113,19 +244,7 @@ function Get-python-UpdatedDocsMsToc($toc) {
113244
$functionService = [PSCustomObject]@{
114245
name = 'Functions';
115246
landingPageType = 'Service';
116-
children = @('azure-functions', 'azure-functions-durable')
117-
}
118-
119-
# The network service is not onboarded into the regular Python build because
120-
# it takes many hours to build.
121-
$networkService = [PSCustomObject]@{
122-
name = 'Network';
123-
href = "~/docs-ref-services/{moniker}/network.md";
124-
items = @([PSCustomObject]@{
125-
name = 'Management';
126-
landingPageType = 'Service';
127-
children = @('azure-mgmt-network')
128-
})
247+
children = @('azure.functions', 'azure.durable_functions')
129248
}
130249

131250
# Add new services which are not onboarded in obvious ways in the CI config.

0 commit comments

Comments
 (0)