Skip to content

Commit 20ee0ac

Browse files
authored
learn.microsoft.com docs onboarding engsys (#6818)
* Add metadata fetching * Update docindex.yml to use new params and variables * Add onboarding logic and update .gitignore to allow eng/scripts/docs edits * Test: point to different branch * Skip onboarding legacy moniker * Add docs to release process * REVERT: Test eng/common changes to Update-DocsMsPackages.ps1 * Remove nightly branch work, those require hosted docs * Make CHANGELOG.md comply with rules * Fix relevant spelling issues * TEST: Add GitHub token * Branch * push branch * Set DefinitionId * Test branch * Token * Remove unnecessary condition * Commitish * Specify monikers * Revert test-related changes
1 parent 6378718 commit 20ee0ac

File tree

8 files changed

+321
-0
lines changed

8 files changed

+321
-0
lines changed

.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -351,6 +351,7 @@ build/
351351

352352
#Doxygen
353353
[Dd]ocs/
354+
!eng/scripts/docs
354355

355356
# vscode
356357
**/.vscode/*

.vscode/cspell.json

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,19 @@
310310
"zulu"
311311
],
312312
"overrides": [
313+
{
314+
"filename": "**/eng/pipelines/docindex.yml",
315+
"words": [
316+
"msdocs",
317+
"SDKCI"
318+
]
319+
},
320+
{
321+
"filename": "**/eng/pipelines/templates/stages/archetype-cpp-release.yml",
322+
"words": [
323+
"docsms"
324+
]
325+
},
313326
{
314327
"filename": "CMakePresets.json",
315328
"words": [

eng/pipelines/docindex.yml

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,99 @@
11
trigger: none
22

3+
parameters:
4+
- name: StartMainSDKCIRun
5+
displayName: |
6+
Kick off the main SDK CI docs run when manually running the pipeline
7+
type: boolean
8+
default: false
9+
10+
variables:
11+
- template: /eng/pipelines/templates/variables/globals.yml
12+
- template: /eng/pipelines/templates/variables/image.yml
313
jobs:
414
- template: /eng/common/pipelines/templates/jobs/docindex.yml
15+
16+
- job: UpdateDocsMsBuildConfig
17+
pool:
18+
name: $(LINUXPOOL)
19+
demands: ImageOverride -equals $(LINUXVMIMAGE)
20+
variables:
21+
DocRepoLocation: $(Pipeline.Workspace)/docs
22+
DocRepoOwner: MicrosoftDocs
23+
DocRepoName: azure-docs-sdk-cpp
24+
25+
steps:
26+
# Sync docs repo (this can be sparse)
27+
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
28+
parameters:
29+
SkipCheckoutNone: true
30+
Paths:
31+
- ci-configs/
32+
- metadata/
33+
- docs-ref-mapping/
34+
- docs-ref-services/
35+
Repositories:
36+
- Name: $(DocRepoOwner)/$(DocRepoName)
37+
WorkingDirectory: $(DocRepoLocation)
38+
39+
- task: Powershell@2
40+
inputs:
41+
pwsh: true
42+
filePath: eng/common/scripts/Update-DocsMsPackages.ps1
43+
arguments: -DocRepoLocation $(DocRepoLocation) -SkipLegacyOnboarding
44+
displayName: Update Docs Onboarding for main branch
45+
46+
- task: Powershell@2
47+
inputs:
48+
pwsh: true
49+
filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1
50+
arguments: >-
51+
-DocRepoLocation $(DocRepoLocation)
52+
-Monikers @('latest','preview')
53+
displayName: Generate Service Level Readme for main branch
54+
55+
- task: Powershell@2
56+
inputs:
57+
pwsh: true
58+
filePath: eng/common/scripts/Update-DocsMsToc.ps1
59+
arguments: >-
60+
-DocRepoLocation $(DocRepoLocation)
61+
-OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml
62+
displayName: Generate ToC for main branch
63+
64+
- task: Powershell@2
65+
inputs:
66+
pwsh: true
67+
filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1
68+
arguments: >-
69+
-DocRepoLocation $(DocRepoLocation)
70+
displayName: Verify Required Docs Json Members
71+
72+
# Push changes to docs repo
73+
- template: /eng/common/pipelines/templates/steps/set-default-branch.yml
74+
parameters:
75+
WorkingDirectory: $(DocRepoLocation)
76+
77+
- template: /eng/common/pipelines/templates/steps/git-push-changes.yml
78+
parameters:
79+
BaseRepoBranch: $(DefaultBranch)
80+
BaseRepoOwner: $(DocRepoOwner)
81+
CommitMsg: "Update docs CI configuration Build: $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)"
82+
TargetRepoName: $(DocRepoName)
83+
TargetRepoOwner: $(DocRepoOwner)
84+
WorkingDirectory: $(DocRepoLocation)
85+
86+
- task: AzureCLI@2
87+
displayName: Queue Docs CI build
88+
condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(${{ parameters.StartMainSDKCIRun }}, true)))
89+
inputs:
90+
azureSubscription: msdocs-apidrop-connection
91+
scriptType: pscore
92+
scriptLocation: inlineScript
93+
inlineScript: |
94+
$accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv
95+
eng/common/scripts/Queue-Pipeline.ps1 `
96+
-Organization "apidrop" `
97+
-Project "Content%20CI" `
98+
-DefinitionId 8056 `
99+
-BearerToken $accessToken

eng/pipelines/templates/stages/archetype-cpp-release.yml

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,9 @@ parameters:
88
IntegrationDependsOn:
99
- PrePublishBuild
1010

11+
TargetDocRepoOwner: MicrosoftDocs
12+
TargetDocRepoName: azure-docs-sdk-cpp
13+
1114
stages:
1215
- ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}:
1316
- ${{ each artifact in parameters.Artifacts }}:
@@ -115,6 +118,40 @@ stages:
115118
FolderForUpload: '$(Pipeline.Workspace)/docs/${{ artifact.Name }}'
116119
TargetLanguage: 'cpp'
117120
ArtifactLocation: '$(Pipeline.Workspace)/packages/${{artifact.Name}}'
121+
122+
- job: PublishDocsMs
123+
displayName: Docs.MS Release
124+
condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true'))
125+
126+
# C++ learn docs are generated from hosted doxygen docs so this
127+
# can only run after the static docs have been published.
128+
dependsOn: PublishDocs
129+
130+
pool:
131+
name: $(LINUXPOOL)
132+
image: $(LINUXVMIMAGE)
133+
os: linux
134+
steps:
135+
- template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
136+
parameters:
137+
Paths:
138+
- sdk/**/*.md
139+
- .github/CODEOWNERS
140+
- download: current
141+
- template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml
142+
parameters:
143+
PackageInfoLocations:
144+
- $(Pipeline.Workspace)/packages/PackageInfo/${{artifact.name}}.json
145+
RepoId: Azure/azure-sdk-for-cpp
146+
WorkingDirectory: $(System.DefaultWorkingDirectory)
147+
TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}}
148+
TargetDocRepoName: ${{parameters.TargetDocRepoName}}
149+
Language: cpp
150+
SparseCheckoutPaths:
151+
- docs-ref-services/
152+
- metadata/
153+
- ci-configs/packages-latest.json
154+
- ci-configs/packages-preview.json
118155

119156
- ${{ if ne(artifact.skipPublishVcpkg, 'true') }}:
120157
- job: PublishVcpkg

eng/scripts/Language-Settings.ps1

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@ $LanguageDisplayName = "C++"
33
$PackageRepository = "CPP"
44
$packagePattern = "package-info.json"
55
$MetadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/main/_data/releases/latest/cpp-packages.csv"
6+
$GithubUri = "https://github.com/Azure/azure-sdk-for-cpp"
7+
$PackageRepositoryUri = "https://github.com/microsoft/vcpkg/tree/master/ports"
8+
9+
. "$PSScriptRoot/docs/Docs-ToC.ps1"
610

711
$VersionRegex = '(#define AZURE_\w+_VERSION_MAJOR )(?<major>[0-9]+)(\s+#define AZURE_\w+_VERSION_MINOR )(?<minor>[0-9]+)(\s+#define AZURE_\w+_VERSION_PATCH )(?<patch>[0-9]+)(\s+#define AZURE_\w+_VERSION_PRERELEASE )"(?<prerelease>[a-zA-Z0-9.]*)"';
812
function Get-VersionHppLocation ($ServiceDirectory, $PackageName) {
@@ -11,6 +15,30 @@ function Get-VersionHppLocation ($ServiceDirectory, $PackageName) {
1115
return $versionHppLocation
1216
}
1317

18+
function Get-cpp-DocsMsMetadataForPackage($PackageInfo) {
19+
$readmeName = $PackageInfo.Name.ToLower()
20+
21+
# Readme names (which are used in the URL) should not include redundant terms
22+
# when viewed in URL form. For example:
23+
# https://docs.microsoft.com/en-us/dotnet/api/overview/azure/storage.blobs-readme
24+
# Note how the end of the URL doesn't look like:
25+
# ".../azure/azure.storage.blobs-readme"
26+
27+
# This logic eliminates a preceding "azure-" in the readme filename.
28+
# "azure-storage-blobs" -> "storage-blobs"
29+
if ($readmeName.StartsWith('azure-')) {
30+
$readmeName = $readmeName.Substring(6)
31+
}
32+
33+
Write-Host "Docs.ms Readme name: $($readmeName)"
34+
New-Object PSObject -Property @{
35+
DocsMsReadMeName = $readmeName
36+
LatestReadMeLocation = 'docs-ref-services/latest'
37+
PreviewReadMeLocation = 'docs-ref-services/preview'
38+
Suffix = ''
39+
}
40+
}
41+
1442
function Get-cpp-PackageInfoFromRepo($pkgPath, $serviceDirectory)
1543
{
1644
$pkgName = Split-Path -Leaf $pkgPath
Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
. "$PSScriptRoot/Docs-ToC.ps1"
2+
3+
# $SetDocsPackageOnboarding = "Set-${Language}-DocsPackageOnboarding"
4+
function Set-cpp-DocsPackageOnboarding($moniker, $metadata, $docRepoLocation, $packageSourceOverride) {
5+
$onboardingFile = GetOnboardingFile `
6+
-docRepoLocation $docRepoLocation `
7+
-moniker $moniker
8+
9+
$onboardingSpec = Get-Content $onboardingFile -Raw | ConvertFrom-Json -AsHashtable
10+
11+
$packagesToOnboard = @()
12+
foreach ($package in $metadata) {
13+
$packageSpec = [ordered]@{
14+
name = $package.Name
15+
tool = 'cpp-doxygen-1.9.1'
16+
url = "https://azuresdkdocs.z19.web.core.windows.net/cpp/$($package.Name)/$($package.Version)/annotated.html"
17+
}
18+
19+
$packagesToOnboard += $packageSpec
20+
}
21+
22+
$onboardingSpec['packages'] = $packagesToOnboard
23+
24+
Set-Content `
25+
-Path $onboardingFile `
26+
-Value ($onboardingSpec | ConvertTo-Json -Depth 100)
27+
}
28+
29+
# $GetDocsPackagesAlreadyOnboarded = "Get-${Language}-DocsPackagesAlreadyOnboarded"
30+
function Get-cpp-DocsPackagesAlreadyOnboarded($docRepoLocation, $moniker) {
31+
$packageOnboardingFile = GetOnboardingFile `
32+
-docRepoLocation $DocRepoLocation `
33+
-moniker $moniker
34+
35+
$onboardedPackages = @{}
36+
$onboardingSpec = ConvertFrom-Json (Get-Content $packageOnboardingFile -Raw)
37+
foreach ($spec in $onboardingSpec.packages) {
38+
$packageInfo = GetPackageInfoFromDocsMsConfig $spec
39+
$onboardedPackages[$packageInfo.Name] = $packageInfo
40+
}
41+
42+
return $onboardedPackages
43+
}
44+
45+
function GetPackageInfoFromDocsMsConfig($spec) {
46+
if (!$spec) {
47+
throw "Spec must not be empty"
48+
}
49+
50+
# "https://azuresdkdocs.z19.web.core.windows.net/cpp/azure-security-attestation/1.1.0/annotated.html" => 1.1.0
51+
$version = $spec.url.Split('/')[-2]
52+
if (!$version) {
53+
throw "Could not determine version from URL: $($spec.url)"
54+
}
55+
56+
return @{
57+
Name = $spec.name
58+
Version = $version
59+
}
60+
}

eng/scripts/docs/Docs-ToC.ps1

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
function GetOnboardingFile($docRepoLocation, $moniker) {
2+
$packageOnboardingFile = "$docRepoLocation/ci-configs/packages-latest.json"
3+
if ("preview" -eq $moniker) {
4+
$packageOnboardingFile = "$docRepoLocation/ci-configs/packages-preview.json"
5+
}
6+
7+
return $packageOnboardingFile
8+
}
9+
10+
# $GetOnboardedDocsMsPackagesFn = "Get-${Language}-OnboardedDocsMsPackages"
11+
function Get-cpp-OnboardedDocsMsPackages($DocRepoLocation) {
12+
$packageOnboardingFiles = @(
13+
"$DocRepoLocation/ci-configs/packages-latest.json",
14+
"$DocRepoLocation/ci-configs/packages-preview.json")
15+
16+
$onboardedPackages = @{}
17+
foreach ($file in $packageOnboardingFiles) {
18+
$onboardingSpec = ConvertFrom-Json (Get-Content $file -Raw)
19+
foreach ($spec in $onboardingSpec.packages) {
20+
$packageName = $spec.name
21+
$onboardedPackages[$packageName] = $null
22+
}
23+
}
24+
25+
return $onboardedPackages
26+
}
27+
28+
# $GetOnboardedDocsMsPackagesForMonikerFn = "Get-${Language}-OnboardedDocsMsPackagesForMoniker"
29+
function Get-cpp-OnboardedDocsMsPackagesForMoniker($DocRepoLocation, $moniker) {
30+
$packageOnboardingFile = GetOnboardingFile `
31+
-docRepoLocation $DocRepoLocation `
32+
-moniker $moniker
33+
34+
$onboardedPackages = @{}
35+
$onboardingSpec = ConvertFrom-Json (Get-Content $packageOnboardingFile -Raw)
36+
foreach ($spec in $onboardingSpec.packages) {
37+
$jsonFile = "$DocRepoLocation/metadata/$($moniker)/$($spec.name).json"
38+
if (Test-Path $jsonFile) {
39+
$onboardedPackages[$spec.name] = ConvertFrom-Json (Get-Content $jsonFile -Raw)
40+
}
41+
else {
42+
$onboardedPackages[$spec.name] = $null
43+
}
44+
}
45+
46+
return $onboardedPackages
47+
}
48+
49+
# $GetPackageLevelReadmeFn = "Get-${Language}-PackageLevelReadme"
50+
function Get-cpp-PackageLevelReadme($packageMetadata) {
51+
return GetPackageReadmeName -packageMetadata $packageMetadata
52+
}
53+
54+
function GetPackageReadmeName($packageMetadata) {
55+
# Fallback to get package-level readme name if metadata file info does not exist
56+
$packageLevelReadmeName = $packageMetadata.Package.ToLower().Replace('azure-', '')
57+
58+
# If there is a metadata json for the package use the DocsMsReadmeName from
59+
# the metadata function
60+
if ($packageMetadata.PSObject.Members.Name -contains "FileMetadata") {
61+
$readmeMetadata = &$GetDocsMsMetadataForPackageFn -PackageInfo $packageMetadata.FileMetadata
62+
$packageLevelReadmeName = $readmeMetadata.DocsMsReadMeName
63+
}
64+
return $packageLevelReadmeName
65+
}
66+
67+
# $GetDocsMsTocDataFn = "Get-${Language}-DocsMsTocData"
68+
function Get-cpp-DocsMsTocData($packageMetadata, $docRepoLocation) {
69+
$packageLevelReadmeName = GetPackageReadmeName -packageMetadata $packageMetadata
70+
$packageTocHeader = GetDocsTocDisplayName $packageMetadata
71+
$output = [PSCustomObject]@{
72+
PackageLevelReadmeHref = "~/docs-ref-services/{moniker}/$packageLevelReadmeName-readme.md"
73+
PackageTocHeader = $packageTocHeader
74+
TocChildren = @($packageMetadata.Package)
75+
}
76+
77+
return $output
78+
}
79+
80+
# $GetRepositoryLinkFn = "Get-${Language}-RepositoryLink"
81+
function Get-cpp-RepositoryLink($packageInfo) {
82+
return "$PackageRepositoryUri/$($packageInfo.Package)-cpp"
83+
}
84+
85+

sdk/template/azure-template/CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
## 1.0.0 (Unreleased)
44

5+
### Other Changes
6+
57
* Validate template project ahead of release
68

79
## 1.0.0-beta.25 (2021-01-12)

0 commit comments

Comments
 (0)