Skip to content

Commit 7bb9e41

Browse files
anthony-c-martinanthony-c-martin
authored
Merge pull request #3042 from Azure/autorestv2
Leverage `autorest.bicep` for schema generation
2 parents 19b2df0 + b501a7e commit 7bb9e41

File tree

10 files changed

+307
-143
lines changed

10 files changed

+307
-143
lines changed

‎.github/workflows/generate-schemas.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@ jobs:
2424
steps:
2525
- name: Checkout repo
2626
uses: actions/checkout@v4
27+
with:
28+
submodules: recursive
2729

2830
- name: Clone azure-rest-api-specs
2931
uses: actions/checkout@v4
@@ -41,6 +43,18 @@ jobs:
4143
run: npm ci
4244
working-directory: generator
4345

46+
- name: Build bicep-types
47+
run: |
48+
npm ci
49+
npm run build
50+
working-directory: bicep-types-az/bicep-types/src/bicep-types
51+
52+
- name: Build autorest.bicep
53+
run: |
54+
npm ci
55+
npm run build
56+
working-directory: bicep-types-az/src/autorest.bicep
57+
4458
- name: Run generator
4559
run: |
4660
rm -Rf "$GITHUB_WORKSPACE/schemas"

‎.github/workflows/generate-single.yml

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,15 @@ on:
1414
jobs:
1515
update-schemas:
1616
name: Update Schemas
17+
permissions:
18+
contents: write
1719
runs-on: ubuntu-latest
1820

1921
steps:
2022
- name: Checkout repo
2123
uses: actions/checkout@v4
24+
with:
25+
submodules: recursive
2226

2327
- name: Clone azure-rest-api-specs
2428
uses: actions/checkout@v4
@@ -36,26 +40,29 @@ jobs:
3640
run: npm ci
3741
working-directory: generator
3842

43+
- name: Build bicep-types
44+
run: |
45+
npm ci
46+
npm run build
47+
working-directory: bicep-types-az/bicep-types/src/bicep-types
48+
49+
- name: Build autorest.bicep
50+
run: |
51+
npm ci
52+
npm run build
53+
working-directory: bicep-types-az/src/autorest.bicep
54+
3955
- name: Run generator
4056
run: |
4157
npm run generate-single -- \
4258
--specs-dir "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" \
4359
--base-path '${{ github.event.inputs.single_path }}'
4460
working-directory: generator
4561

46-
- name: Create Pull Request
47-
uses: peter-evans/create-pull-request@v6
62+
- name: Push to git branch
63+
uses: stefanzweifel/git-auto-commit-action@v5
4864
with:
49-
committer: GitHub <[email protected]>
50-
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
51-
signoff: false
52-
branch: autogenerate-${{ github.event.inputs.single_path }}
53-
delete-branch: true
54-
title: |
55-
Update Generated Schemas (${{ github.event.inputs.single_path }})
56-
body: |
57-
Update Generated Schemas (${{ github.event.inputs.single_path }})
58-
commit-message: |
59-
Update Generated Schemas (${{ github.event.inputs.single_path }})
60-
labels: autogenerate
61-
draft: false
65+
commit_message: Update Generated Schemas (${{ github.event.inputs.single_path }})
66+
branch: autogenerate-single/${{ github.event.inputs.single_path }}
67+
push_options: '--force'
68+
create_branch: true

‎.gitmodules

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
[submodule "bicep-types-az"]
2+
path = bicep-types-az
3+
url = https://github.com/Azure/bicep-types-az

‎bicep-types-az

Submodule bicep-types-az added at 4f20acb

‎generator/autogenlist.ts

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,9 @@ const disabledProviders: AutoGenConfig[] = [
3232
disabledForAutogen: true,
3333
},
3434
{
35-
// Disabled until the unexpected character error in the swagger spec is fixed
3635
basePath: 'cdn/resource-manager',
3736
namespace: 'Microsoft.Cdn',
38-
disabledForAutogen: true,
37+
useAutorestV2: true,
3938
},
4039
{
4140
// Disabled until the enum mismatch in the swagger spec is fixed

‎generator/autorest.ts

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
import path from 'path';
4+
import os from 'os';
5+
import { findRecursive, lowerCaseContains, executeCmd, fileExists } from './utils';
6+
import * as constants from './constants';
7+
import { ReadmeTag, AutoGenConfig, CodeBlock } from './models';
8+
import * as cm from '@ts-common/commonmark-to-markdown'
9+
import * as yaml from 'js-yaml'
10+
import { readFile, writeFile } from 'fs/promises';
11+
12+
const autorestBinary = os.platform() === 'win32' ? 'autorest.cmd' : 'autorest';
13+
export const apiVersionRegex = /^\d{4}-\d{2}-\d{2}(|-preview)$/;
14+
15+
async function execAutoRest(tmpFolder: string, params: string[]) {
16+
await executeCmd(__dirname, `${__dirname}/node_modules/.bin/${autorestBinary}`, params);
17+
if (!fileExists(tmpFolder)) {
18+
return [];
19+
}
20+
21+
return await findRecursive(tmpFolder, p => path.extname(p) === '.json');
22+
}
23+
24+
export async function runAutorest(readme: string, tmpFolder: string) {
25+
const autoRestParams = [
26+
`--version=${constants.autorestCoreVersion}`,
27+
`--use=@autorest/azureresourceschema@${constants.azureresourceschemaVersion}`,
28+
'--azureresourceschema',
29+
`--output-folder=${tmpFolder}`,
30+
'--multiapi',
31+
'--pass-thru:subset-reducer',
32+
'--pass-thru:schema-validator-swagger',
33+
readme,
34+
];
35+
36+
if (constants.autoRestVerboseOutput) {
37+
autoRestParams.push('--verbose');
38+
}
39+
40+
return await execAutoRest(tmpFolder, autoRestParams);
41+
}
42+
43+
44+
45+
export async function generateAutorestConfig(readme: string, autoGenConfig: AutoGenConfig) {
46+
const content = (await readFile(readme)).toString();
47+
const markdownEx = cm.parse(content);
48+
const fileSet = new Set<string>();
49+
for (const node of cm.iterate(markdownEx.markDown)) {
50+
// We're only interested in yaml code blocks
51+
if (node.type !== 'code_block' || !node.info || !node.literal ||
52+
!node.info.trim().startsWith('yaml')) {
53+
continue;
54+
}
55+
56+
const DOC = (yaml.load(node.literal) as CodeBlock);
57+
if (DOC) {
58+
const inputFile = DOC['input-file'];
59+
if (typeof inputFile === 'string') {
60+
fileSet.add(inputFile);
61+
} else if (inputFile instanceof Array) {
62+
for (const i of inputFile) {
63+
fileSet.add(i);
64+
}
65+
}
66+
}
67+
}
68+
69+
let readmeTag = {} as ReadmeTag;
70+
for (const inputFile of fileSet) {
71+
const pathComponents = inputFile.split("/");
72+
73+
if (!autoGenConfig.useNamespaceFromConfig &&
74+
!lowerCaseContains(pathComponents, autoGenConfig.namespace)) {
75+
continue;
76+
}
77+
78+
const apiVersion = pathComponents.filter(p => p.match(apiVersionRegex) !== null)[0];
79+
if (!apiVersion) {
80+
continue;
81+
}
82+
83+
readmeTag[apiVersion] ??= readmeTag[apiVersion] || [];
84+
readmeTag[apiVersion].push(inputFile);
85+
}
86+
87+
if (autoGenConfig.readmeTag) {
88+
readmeTag = {...readmeTag, ...autoGenConfig.readmeTag };
89+
}
90+
91+
const schemaReadmeContent = compositeSchemaReadme(readmeTag);
92+
93+
const schemaReadme = readme.replace(/\.md$/i, '.azureresourceschema.md');
94+
95+
await writeFile(schemaReadme, schemaReadmeContent);
96+
}
97+
98+
function compositeSchemaReadme(readmeTag: ReadmeTag): string {
99+
let content =
100+
`## AzureResourceSchema
101+
102+
### AzureResourceSchema multi-api
103+
104+
\`\`\` yaml $(azureresourceschema) && $(multiapi)
105+
${yaml.dump({ 'batch': Object.keys(readmeTag).map(apiVersion => ({ 'tag': `schema-${apiVersion}`})) }, { lineWidth: 1000 })}
106+
\`\`\`
107+
108+
`
109+
for (const apiVersion of Object.keys(readmeTag)) {
110+
content +=
111+
`
112+
### Tag: schema-${apiVersion} and azureresourceschema
113+
114+
\`\`\` yaml $(tag) == 'schema-${apiVersion}' && $(azureresourceschema)
115+
${yaml.dump({ 'input-file': readmeTag[apiVersion] }, { lineWidth: 1000})}
116+
\`\`\`
117+
`
118+
}
119+
return content;
120+
}

‎generator/autorestV2.ts

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
// Copyright (c) Microsoft Corporation.
2+
// Licensed under the MIT License.
3+
import path from 'path';
4+
import os from 'os';
5+
import { findRecursive, executeCmd, fileExists } from './utils';
6+
import * as constants from './constants';
7+
import { readFile, writeFile } from 'fs/promises';
8+
import * as markdown from '@ts-common/commonmark-to-markdown'
9+
import * as yaml from 'js-yaml'
10+
11+
const autorestBinary = os.platform() === 'win32' ? 'autorest.cmd' : 'autorest';
12+
13+
const rootDir = `${__dirname}/../`;
14+
const extensionDir = path.resolve(`${rootDir}/bicep-types-az/src/autorest.bicep/`);
15+
16+
export async function generateAutorestV2Config(readmePath: string, bicepReadmePath: string) {
17+
// We expect a path format convention of <provider>/(any/number/of/intervening/folders)/<yyyy>-<mm>-<dd>(|-preview)/<filename>.json
18+
// This information is used to generate individual tags in the generated autorest configuration
19+
// eslint-disable-next-line no-useless-escape
20+
const pathRegex = /^(\$\(this-folder\)\/|)([^\/]+)(?:\/[^\/]+)*\/(\d{4}-\d{2}-\d{2}(|-preview))\/.*\.json$/i;
21+
22+
const readmeContents = await readFile(readmePath, { encoding: 'utf8' });
23+
const readmeMarkdown = markdown.parse(readmeContents);
24+
25+
const inputFiles = new Set<string>();
26+
// we need to look for all autorest configuration elements containing input files, and collect that list of files. These will look like (e.g.):
27+
// ```yaml $(tag) == 'someTag'
28+
// input-file:
29+
// - path/to/file.json
30+
// - path/to/other_file.json
31+
// ```
32+
for (const node of markdown.iterate(readmeMarkdown.markDown)) {
33+
// We're only interested in yaml code blocks
34+
if (node.type !== 'code_block' || !node.info || !node.literal ||
35+
!node.info.trim().startsWith('yaml')) {
36+
continue;
37+
}
38+
39+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
40+
const yamlData = yaml.load(node.literal) as any;
41+
if (yamlData) {
42+
// input-file may be a single string or an array of strings
43+
const inputFile = yamlData['input-file'];
44+
if (typeof inputFile === 'string') {
45+
inputFiles.add(inputFile);
46+
} else if (inputFile instanceof Array) {
47+
for (const i of inputFile) {
48+
inputFiles.add(i);
49+
}
50+
}
51+
}
52+
}
53+
54+
const filesByTag: Record<string, string[]> = {};
55+
for (const file of inputFiles) {
56+
const normalizedFile = normalizeJsonPath(file);
57+
const match = pathRegex.exec(normalizedFile);
58+
if (match) {
59+
// Generate a unique tag. We can't process all of the different API versions in one autorest pass
60+
// because there are constraints on naming uniqueness (e.g. naming of definitions), so we want to pass over
61+
// each API version separately.
62+
const tagName = `${match[2].toLowerCase()}-${match[3].toLowerCase()}`;
63+
if (!filesByTag[tagName]) {
64+
filesByTag[tagName] = [];
65+
}
66+
67+
filesByTag[tagName].push(normalizedFile);
68+
} else {
69+
console.warn(`WARNING: Unable to parse swagger path "${file}"`);
70+
}
71+
}
72+
73+
let generatedContent = `##Bicep
74+
75+
### Bicep multi-api
76+
\`\`\`yaml $(bicep) && $(multiapi)
77+
${yaml.dump({ 'batch': Object.keys(filesByTag).map(tag => ({ 'tag': tag })) }, { lineWidth: 1000 })}
78+
\`\`\`
79+
`;
80+
81+
for (const tag of Object.keys(filesByTag)) {
82+
generatedContent += `### Tag: ${tag} and bicep
83+
\`\`\`yaml $(tag) == '${tag}' && $(bicep)
84+
${yaml.dump({ 'input-file': filesByTag[tag] }, { lineWidth: 1000})}
85+
\`\`\`
86+
`;
87+
}
88+
89+
await writeFile(bicepReadmePath, generatedContent);
90+
}
91+
92+
function normalizeJsonPath(jsonPath: string) {
93+
// eslint-disable-next-line no-useless-escape
94+
return path.normalize(jsonPath).replace(/[\\\/]/g, '/');
95+
}
96+
97+
async function execAutoRest(tmpFolder: string, params: string[]) {
98+
await executeCmd(__dirname, `${__dirname}/node_modules/.bin/${autorestBinary}`, params);
99+
if (!fileExists(tmpFolder)) {
100+
return [];
101+
}
102+
103+
return await findRecursive(tmpFolder, p => path.extname(p) === '.json');
104+
}
105+
106+
export async function runAutorestV2(readme: string, tmpFolder: string) {
107+
const autoRestParams = [
108+
`--use=@autorest/modelerfour`,
109+
`--use=${extensionDir}`,
110+
'--bicep',
111+
`--output-folder=${tmpFolder}`,
112+
'--multiapi',
113+
'--title=none',
114+
// This is necessary to avoid failures such as "ERROR: Semantic violation: Discriminator must be a required property." blocking type generation.
115+
// In an ideal world, we'd raise issues in https://github.com/Azure/azure-rest-api-specs and force RP teams to fix them, but this isn't very practical
116+
// as new validations are added continuously, and there's often quite a lag before teams will fix them - we don't want to be blocked by this in generating types.
117+
`--skip-semantics-validation`,
118+
`--arm-schema=true`,
119+
readme,
120+
];
121+
122+
if (constants.autoRestVerboseOutput) {
123+
autoRestParams.push('--verbose');
124+
}
125+
126+
return await execAutoRest(tmpFolder, autoRestParams);
127+
}

0 commit comments

Comments
 (0)