Skip to content

Commit 202fee4

Browse files
JimSuplizioJimSuplizio
authored
Merge 1638ce2 into 4348985 
2 parents 4348985 + 1638ce2 commit 202fee4

File tree

2 files changed

+174
-25
lines changed

2 files changed

+174
-25
lines changed

‎eng/scripts/Language-Settings.ps1‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,34 @@ function Get-python-PackageInfoFromPackageFile ($pkg, $workingDirectory)
128128
}
129129
}
130130

131+
# This is the GetDocsMsDevLanguageSpecificPackageInfoFn implementation
132+
function Get-python-DocsMsDevLanguageSpecificPackageInfo($packageInfo, $packageSourceOverride) {
133+
# If the default namespace isn't in the package info then it needs to be added
134+
# Can't check if (!$packageInfo.Namespaces) in strict mode because Namespaces won't exist
135+
# at all.
136+
if (!($packageInfo | Get-Member Namespaces)) {
137+
# If the Version is INGORE that means it's a source install and those
138+
# ones need to be done by hand
139+
if ($packageInfo.Version -ine "IGNORE") {
140+
$version = $packageInfo.Version
141+
# If the dev version is set, use that
142+
if ($packageInfo.DevVersion) {
143+
$version = $packageInfo.DevVersion
144+
}
145+
$namespaces = Get-NamespacesFromWhlFile $packageInfo.Name $version $packageSourceOverride
146+
if ($namespaces.Count -gt 0) {
147+
$packageInfo | Add-Member -Type NoteProperty -Name "Namespaces" -Value $namespaces
148+
} else {
149+
LogWarning "Unable to find namespaces for $($packageInfo.Name):$version, using the package name."
150+
$tempNamespaces = @()
151+
$tempNamespaces += $packageInfo.Name
152+
$packageInfo | Add-Member -Type NoteProperty -Name "Namespaces" -Value $tempNamespaces
153+
}
154+
}
155+
}
156+
return $packageInfo
157+
}
158+
131159
# Stage and Upload Docs to blob Storage
132160
function Publish-python-GithubIODocs ($DocLocation, $PublicArtifactLocation)
133161
{
@@ -618,8 +646,8 @@ function Validate-Python-DocMsPackages ($PackageInfo, $PackageInfos, $PackageSou
618646

619647
$allSucceeded = $true
620648
foreach ($item in $PackageInfos) {
621-
# Some packages
622-
if ($item.Version -eq 'IGNORE') {
649+
# If the Version is INGORE that means it's a source install and those aren't run through ValidatePackage
650+
if ($item.Version -eq 'IGNORE') {
623651
continue
624652
}
625653

@@ -656,7 +684,7 @@ function Get-python-DirectoriesForGeneration () {
656684

657685
function Update-python-GeneratedSdks([string]$PackageDirectoriesFile) {
658686
$packageDirectories = Get-Content $PackageDirectoriesFile | ConvertFrom-Json
659-
687+
660688
$directoriesWithErrors = @()
661689

662690
foreach ($directory in $packageDirectories) {

‎eng/scripts/docs/Docs-ToC.ps1‎

Lines changed: 143 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,86 @@
1-
function GetOnboardingFileForMoniker($docRepoLocation, $moniker) {
1+
# Download the .whl file into the given destination directory.
2+
function Get-WhlFile {
3+
param(
4+
[Parameter(Mandatory=$true)] [string]$Library,
5+
[Parameter(Mandatory=$true)] [string]$Version,
6+
[Parameter(Mandatory=$true)] [string]$Destination,
7+
[Parameter(Mandatory=$false)] [string]$ExtraIndexUrl = ""
8+
)
9+
10+
$LibArg= "$Library==$Version"
11+
if ($ExtraIndexUrl) {
12+
$ExtraIndexArg = "--extra-index-url=$ExtraIndexUrl"
13+
Write-Host "python -m pip download --quiet --only-binary=:all: --dest $Destination --no-deps $ExtraIndexArg $LibArg"
14+
python -m pip download --quiet --only-binary=:all: --dest $Destination --no-deps $ExtraIndexArg $LibArg
15+
} else {
16+
Write-Host "python -m pip download --quiet --only-binary=:all: --dest $Destination --no-deps $LibArg"
17+
python -m pip download --quiet --only-binary=:all: --dest $Destination --no-deps $LibArg
18+
}
19+
if($LASTEXITCODE -ne 0) {
20+
return $false
21+
}
22+
return $true
23+
}
24+
25+
# Given a library and version, download the .whl file and unpack it to get the namespaces.
26+
# A temp directory is used for the download and unpack which is cleaned up afterwards.
27+
function Get-NamespacesFromWhlFile {
28+
param(
29+
[Parameter(Mandatory=$true)] [string]$Library,
30+
[Parameter(Mandatory=$true)] [string]$Version,
31+
[Parameter(Mandatory=$false)] [string]$ExtraIndexUrl = ""
32+
)
33+
34+
$destination = (Join-Path ([System.IO.Path]::GetTempPath()) "$Library$Version")
35+
$namespaces = @()
36+
37+
try {
38+
39+
# Pulling the whl file generates output, make sure it's sent to null so
40+
# it's not returned as part of this function.
41+
$success = Get-WhlFile $Library $Version $destination $ExtraIndexUrl
42+
if ($success) {
43+
44+
# Each library gets its own temporary directory. There should only be one whl
45+
# file in the destination directory
46+
$whlFile = Get-ChildItem -Path $destination -File -Filter "*.whl" | Select-Object -First 1
47+
$unpackDir = Join-Path -Path $destination -ChildPath "$Library-$Version"
48+
Expand-Archive -Path $whlFile -DestinationPath $unpackDir
49+
50+
# Look for any directory that contains __init__.py with the following exceptions:
51+
# 1. *.dist-info directories shouldn't be included in the results.
52+
# 2. If any subdirectory starts with "_" it's internal and needs to be skipped
53+
# 3. If there's a root level directory named "azure" with an __init__.py file then
54+
# needs to be skipped. This doesn't happen with libraries released from the
55+
# azure-sdk-for-python repository but there are older libraries that are in the
56+
# docs directories which are/were released outside of the repository where this
57+
# is true.
58+
$rootLevelAzureDir = Join-Path -Path $unpackDir -ChildPath "azure"
59+
$namespaceDirs = Get-ChildItem -Path $unpackDir -Recurse -Filter "__init__.py" |
60+
Where-Object{$_.DirectoryName -notlike "*.dist-info"} |
61+
Where-Object{$_.DirectoryName -notlike "*$([IO.Path]::DirectorySeparatorChar)_*" } |
62+
Where-Object{$_.DirectoryName -ine $rootLevelAzureDir}
63+
foreach($namespaceDir in $namespaceDirs) {
64+
# strip off the root directy, everything left will be subDir1/subDir2 which needs
65+
$partialDir = $namespaceDir.DirectoryName.Replace($unpackDir + $([IO.Path]::DirectorySeparatorChar), "")
66+
$namespaces += $partialDir.Replace([IO.Path]::DirectorySeparatorChar, ".")
67+
# Since only the primary namespace is being pulled, break out of the loop after
68+
# the first one.
69+
break
70+
}
71+
}
72+
}
73+
finally {
74+
# Clean up the temp directory if it was created
75+
if (Test-Path $destination) {
76+
Remove-Item $destination -Recurse -Force
77+
}
78+
}
79+
# Make sure this always returns an array
80+
Write-Output -NoEnumerate $namespaces
81+
}
82+
83+
function GetOnboardingFileForMoniker($docRepoLocation, $moniker) {
284
$packageOnboardingFile = 'ci-configs/packages-latest.json'
385
if ($moniker -eq 'preview') {
486
$packageOnboardingFile = 'ci-configs/packages-preview.json'
@@ -21,7 +103,7 @@ function Get-python-OnboardedDocsMsPackagesForMoniker($DocRepoLocation, $moniker
21103
continue
22104
}
23105
$packageName = $spec.package_info.name
24-
106+
25107
$jsonFile = "$DocRepoLocation/metadata/$moniker/$packageName.json"
26108
if (Test-Path $jsonFile) {
27109
$onboardedPackages[$packageName] = ConvertFrom-Json (Get-Content $jsonFile -Raw)
@@ -55,7 +137,7 @@ function Get-python-OnboardedDocsMsPackages($DocRepoLocation) {
55137
return $onboardedPackages
56138
}
57139

58-
function GetPackageLevelReadme($packageMetadata) {
140+
function GetPackageLevelReadme($packageMetadata) {
59141
# Fallback for package name
60142
$packageLevelReadmeName = $packageMetadata.Package
61143
if ($packageLevelReadmeName.StartsWith('azure-')) {
@@ -70,19 +152,70 @@ function GetPackageLevelReadme($packageMetadata) {
70152
}
71153
return $packageLevelReadmeName
72154
}
73-
74-
function Get-python-PackageLevelReadme($packageMetadata) {
155+
156+
# This function is called within a loop. To prevent multiple reads of the same
157+
# file data, this uses a script-scoped cache variable.
158+
$script:PackageMetadataJsonLookup = $null
159+
function GetPackageMetadataJsonLookup($docRepoLocation) {
160+
if ($script:PackageMetadataJsonLookup) {
161+
return $script:PackageMetadataJsonLookup
162+
}
163+
164+
$script:PackageMetadataJsonLookup = @{}
165+
$packageJsonFiles = Get-ChildItem $docRepoLocation/metadata/ -Filter *.json -Recurse
166+
foreach ($packageJsonFile in $packageJsonFiles) {
167+
$packageJson = Get-Content $packageJsonFile -Raw | ConvertFrom-Json -AsHashtable
168+
169+
if (!$script:PackageMetadataJsonLookup.ContainsKey($packageJson.Name)) {
170+
$script:PackageMetadataJsonLookup[$packageJson.Name] = @($packageJson)
171+
} else {
172+
$script:PackageMetadataJsonLookup[$packageJson.Name] += $packageJson
173+
}
174+
}
175+
176+
return $script:PackageMetadataJsonLookup
177+
}
178+
179+
# Grab the namespaces from the json file
180+
function Get-Toc-Children($package, $docRepoLocation) {
181+
$packageTable = GetPackageMetadataJsonLookup $docRepoLocation
182+
183+
$namespaces = @()
184+
if ($packageTable.ContainsKey($package)) {
185+
foreach ($entry in $packageTable[$package]) {
186+
if ($entry.ContainsKey('Namespaces')) {
187+
$namespaces += $entry['Namespaces']
188+
}
189+
}
190+
}
191+
# Sort the array and clean out any dupes (there shouldn't be any but better safe than sorry)
192+
$namespaces = $namespaces | Sort-Object -Unique
193+
# Ensure that this always returns an array, even if there's one item or 0 items
194+
Write-Output -NoEnumerate $namespaces
195+
}
196+
197+
function Get-python-PackageLevelReadme($packageMetadata) {
75198
return GetPackageLevelReadme -packageMetadata $packageMetadata
76199
}
77200

78-
function Get-python-DocsMsTocData($packageMetadata, $docRepoLocation) {
201+
# Defined in common.ps1
202+
# $GetDocsMsTocDataFn = "Get-${Language}-DocsMsTocData"
203+
function Get-python-DocsMsTocData($packageMetadata, $docRepoLocation, $PackageSourceOverride) {
79204
$packageLevelReadmeName = GetPackageLevelReadme -packageMetadata $packageMetadata
80-
81205
$packageTocHeader = GetDocsTocDisplayName $packageMetadata
206+
207+
# Get-Toc-Children always returns an array, even if there's only 1 item or it's empty
208+
$children = Get-Toc-Children `
209+
-package $packageMetadata.Package `
210+
-docRepoLocation $docRepoLocation
211+
if ($children.Count -eq 0) {
212+
LogWarning "Did not find the package namespaces for $($packageMetadata.Package)"
213+
}
214+
82215
$output = [PSCustomObject]@{
83216
PackageLevelReadmeHref = "~/docs-ref-services/{moniker}/$packageLevelReadmeName-readme.md"
84217
PackageTocHeader = $packageTocHeader
85-
TocChildren = @($packageMetadata.Package)
218+
TocChildren = $children
86219
}
87220

88221
return $output
@@ -113,19 +246,7 @@ function Get-python-UpdatedDocsMsToc($toc) {
113246
$functionService = [PSCustomObject]@{
114247
name = 'Functions';
115248
landingPageType = 'Service';
116-
children = @('azure-functions', 'azure-functions-durable')
117-
}
118-
119-
# The network service is not onboarded into the regular Python build because
120-
# it takes many hours to build.
121-
$networkService = [PSCustomObject]@{
122-
name = 'Network';
123-
href = "~/docs-ref-services/{moniker}/network.md";
124-
items = @([PSCustomObject]@{
125-
name = 'Management';
126-
landingPageType = 'Service';
127-
children = @('azure-mgmt-network')
128-
})
249+
children = @('azure.functions', 'azure.durable_functions')
129250
}
130251

131252
# Add new services which are not onboarded in obvious ways in the CI config.
@@ -134,7 +255,7 @@ function Get-python-UpdatedDocsMsToc($toc) {
134255
# (e.g. "Functions"), sorting the resulting list, then re-adding the ultimate
135256
# item to the end. This ensures that the "Other" service is at the bottom as
136257
# intended.
137-
$sortableServices = $services[0..($services.Length - 2)] + $functionService + $networkService
258+
$sortableServices = $services[0..($services.Length - 2)] + $functionService
138259
$toc[0].items = ($sortableServices | Sort-Object -Property name) + $services[-1]
139260

140261
# PowerShell outputs a single object if the output is an array with only one

0 commit comments

Comments
 (0)