Skip to content

Commit 1569e6a

Browse files
jorgecotillojorgecotillo
authored
Merge pull request #2070 from Azure/jcotillo/schema_gen_batch
Schema gen - batched
2 parents fb85c18 + b9dbd4c commit 1569e6a

File tree

2 files changed

+253
-8
lines changed

2 files changed

+253
-8
lines changed

‎.github/workflows/generate-schemas-batch.yml

Lines changed: 207 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,207 @@
1+
name: Generate Schemas batch
2+
3+
on:
4+
workflow_dispatch:
5+
inputs:
6+
api_specs_ref:
7+
description: 'Git ref or full SHA for https://github.com/Azure/azure-rest-api-specs.'
8+
required: true
9+
default: 'main'
10+
11+
env:
12+
SUMMARY_LOG_DIR: /tmp/summary
13+
SUMMARY_LOG_PATH: /tmp/summary/summary.log
14+
SUMMARY_LOG_ARTIFACT_NAME: summary-log
15+
AUTOGENERATE_BRANCH_NAME: autogenerate
16+
MAIN_BRANCH_NAME: main
17+
jobs:
18+
batch-0:
19+
name: Update Schemas Batch 0
20+
runs-on: ubuntu-latest
21+
steps:
22+
- uses: actions/[email protected]
23+
24+
- name: Clone azure-rest-api-specs
25+
uses: actions/[email protected]
26+
with:
27+
repository: Azure/azure-rest-api-specs
28+
path: workflow-temp/azure-rest-api-specs
29+
ref: ${{ github.event.inputs.api_specs_ref }}
30+
31+
- name: Install generator npm packages
32+
run: npm ci
33+
working-directory: generator
34+
35+
- name: Create initial log description
36+
run: mkdir -p /tmp/summary
37+
38+
- id: generate
39+
name: Run generator
40+
run: |
41+
npm run generate-all -- --local-path "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" --summary-log-path $SUMMARY_LOG_PATH --batch-count 4 --batch-index 0
42+
working-directory: generator
43+
44+
- name: Commit changes
45+
uses: EndBug/add-and-commit@v7
46+
with:
47+
author_name: github-actions
48+
author_email: [email protected]
49+
message: 'Autogenerate schemas batch 0'
50+
add: '*.*'
51+
52+
- name: Upload summary log
53+
uses: actions/upload-artifact@v2
54+
with:
55+
name: ${{ env.SUMMARY_LOG_ARTIFACT_NAME }}
56+
path: ${{ env.SUMMARY_LOG_PATH }}
57+
batch-1:
58+
needs: batch-0
59+
name: Update Schemas Batch 1
60+
runs-on: ubuntu-latest
61+
steps:
62+
- uses: actions/[email protected]
63+
64+
- name: Clone azure-rest-api-specs
65+
uses: actions/[email protected]
66+
with:
67+
repository: Azure/azure-rest-api-specs
68+
path: workflow-temp/azure-rest-api-specs
69+
ref: ${{ github.event.inputs.api_specs_ref }}
70+
71+
- name: Install generator npm packages
72+
run: npm ci
73+
working-directory: generator
74+
75+
- name: Download summary log artifact
76+
uses: actions/download-artifact@v2
77+
with:
78+
name: ${{ env.SUMMARY_LOG_ARTIFACT_NAME }}
79+
path: ${{ env.SUMMARY_LOG_DIR }}
80+
81+
- id: generate
82+
name: Run generator
83+
run: |
84+
npm run generate-all -- --local-path "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" --summary-log-path $SUMMARY_LOG_PATH --batch-count 4 --batch-index 1
85+
working-directory: generator
86+
87+
- name: Commit changes
88+
uses: EndBug/add-and-commit@v7
89+
with:
90+
author_name: github-actions
91+
author_email: [email protected]
92+
message: 'Autogenerate schemas batch 1'
93+
add: '*.*'
94+
95+
- name: Upload summary log
96+
uses: actions/upload-artifact@v2
97+
with:
98+
name: ${{ env.SUMMARY_LOG_ARTIFACT_NAME }}
99+
path: ${{ env.SUMMARY_LOG_PATH }}
100+
batch-2:
101+
needs: batch-1
102+
name: Update Schemas Batch 2
103+
runs-on: ubuntu-latest
104+
steps:
105+
- uses: actions/[email protected]
106+
107+
- name: Clone azure-rest-api-specs
108+
uses: actions/[email protected]
109+
with:
110+
repository: Azure/azure-rest-api-specs
111+
path: workflow-temp/azure-rest-api-specs
112+
ref: ${{ github.event.inputs.api_specs_ref }}
113+
114+
- name: Install generator npm packages
115+
run: npm ci
116+
working-directory: generator
117+
118+
- name: Download summary log artifact
119+
uses: actions/download-artifact@v2
120+
with:
121+
name: ${{ env.SUMMARY_LOG_ARTIFACT_NAME }}
122+
path: ${{ env.SUMMARY_LOG_DIR }}
123+
124+
- id: generate
125+
name: Run generator
126+
run: |
127+
npm run generate-all -- --local-path "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" --summary-log-path $SUMMARY_LOG_PATH --batch-count 4 --batch-index 2
128+
working-directory: generator
129+
130+
- name: Commit changes
131+
uses: EndBug/add-and-commit@v7
132+
with:
133+
author_name: github-actions
134+
author_email: [email protected]
135+
message: 'Autogenerate schemas batch 2'
136+
add: '*.*'
137+
138+
- name: Upload summary log
139+
uses: actions/upload-artifact@v2
140+
with:
141+
name: ${{ env.SUMMARY_LOG_ARTIFACT_NAME }}
142+
path: ${{ env.SUMMARY_LOG_PATH }}
143+
batch-3:
144+
needs: batch-2
145+
name: Update Schemas Batch 3
146+
runs-on: ubuntu-latest
147+
steps:
148+
- uses: actions/[email protected]
149+
150+
- name: Clone azure-rest-api-specs
151+
uses: actions/[email protected]
152+
with:
153+
repository: Azure/azure-rest-api-specs
154+
path: workflow-temp/azure-rest-api-specs
155+
ref: ${{ github.event.inputs.api_specs_ref }}
156+
157+
- name: Install generator npm packages
158+
run: npm ci
159+
working-directory: generator
160+
161+
- name: Download summary log artifact
162+
uses: actions/download-artifact@v2
163+
with:
164+
name: ${{ env.SUMMARY_LOG_ARTIFACT_NAME }}
165+
path: ${{ env.SUMMARY_LOG_DIR }}
166+
167+
- id: generate
168+
name: Run generator
169+
run: |
170+
npm run generate-all -- --local-path "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" --summary-log-path $SUMMARY_LOG_PATH --batch-count 4 --batch-index 3
171+
172+
summary="$(<$SUMMARY_LOG_PATH)"
173+
summary="${summary//'%'/'%25'}"
174+
summary="${summary//$'\n'/'%0A'}"
175+
summary="${summary//$'\r'/'%0D'}"
176+
echo "::set-output name=summary::$summary"
177+
working-directory: generator
178+
179+
- id: get_swagger_gh_uri
180+
name: Get GitHub URI for azure-rest-api-specs
181+
run: |
182+
git_sha=`git rev-parse HEAD`
183+
echo "::set-output name=gh_uri::https://github.com/Azure/azure-rest-api-specs/tree/$git_sha"
184+
working-directory: workflow-temp/azure-rest-api-specs
185+
186+
- name: Create Pull Request
187+
uses: peter-evans/[email protected]
188+
with:
189+
committer: GitHub <[email protected]>
190+
author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
191+
signoff: false
192+
branch: ${{ env.MAIN_BRANCH_NAME }}
193+
title: |
194+
Update Generated Schemas ${{ github.event.inputs.single_path && format('(single path: {0})', github.event.inputs.single_path) || '' }}
195+
body: |
196+
Update Generated Schemas ${{ github.event.inputs.single_path && format('(single path: {0})', github.event.inputs.single_path) || '' }}
197+
198+
Generate schemas for ${{ steps.get_swagger_gh_uri.outputs.gh_uri }}
199+
200+
Summary
201+
${{ steps.generate.outputs.summary }}
202+
commit-message: |
203+
Update Generated Schemas ${{ github.event.inputs.single_path && format('(single path: {0})', github.event.inputs.single_path) || '' }}
204+
205+
Generate schemas for ${{ steps.get_swagger_gh_uri.outputs.gh_uri }}
206+
labels: ${{ env.AUTOGENERATE_BRANCH_NAME }}
207+
draft: false

‎generator/cmd/generateall.ts

Lines changed: 46 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -9,20 +9,31 @@ import { flatten, keys } from 'lodash';
99
import { executeSynchronous, chunker, writeJsonFile } from '../utils';
1010
import { Package } from '../models';
1111
import yargs from 'yargs';
12+
import path from 'path';
13+
14+
import { createWriteStream } from 'fs';
15+
import stripAnsi from 'strip-ansi';
1216

1317
const argsConfig = yargs
1418
.strict()
1519
.option('batch-count', { type: 'number', desc: 'If running in batch mode, the total number of batch jobs running' })
1620
.option('batch-index', { type: 'number', desc: 'If running in batch mode, the index of this batch job' })
1721
.option('local-path', { type: 'string', desc: 'The local path to the azure-rest-api-specs repo' })
1822
.option('readme-files', { type: 'array', desc: 'The list of readme.md files to generate schemas for' })
19-
.option('output-path', { type: 'string', desc: 'The base path to save schema output' });
23+
.option('output-path', { type: 'string', desc: 'The base path to save schema output' })
24+
.option('summary-log-path', { type: 'string', desc: 'The path to store generation summary information. File will be saved in md format.' });
25+
26+
interface ILogger {
27+
out: (data: string) => void;
28+
}
2029

2130
executeSynchronous(async () => {
2231
const args = await argsConfig.parseAsync();
2332

2433
let basePaths;
2534
let localPath = args['local-path'];
35+
let summaryPath = args['summary-log-path'];
36+
2637
if (!localPath) {
2738
localPath = constants.specsRepoPath;
2839
basePaths = await cloneAndGenerateBasePaths(localPath, constants.specsRepoUri, constants.specsRepoCommitHash);
@@ -31,14 +42,23 @@ executeSynchronous(async () => {
3142
basePaths = await generateBasePaths(localPath);
3243
}
3344

45+
if (!summaryPath) {
46+
// generate default full path
47+
summaryPath = path.join(constants.specsRepoPath, 'summary.log');
48+
}
49+
50+
// resolve absolute path
51+
summaryPath = await resolveAbsolutePath(summaryPath);
52+
3453
if (args['batch-count'] !== undefined && args['batch-index'] !== undefined) {
3554
basePaths = chunker(basePaths, args['batch-count'])[args['batch-index']];
3655
}
3756

3857
const schemaConfigs: SchemaConfiguration[] = [];
39-
const errors = [];
4058
const packages: Package[] = [];
4159

60+
const summaryLogger = await getLogger(summaryPath);
61+
4262
for (const basePath of basePaths) {
4363
const readme = await validateAndReturnReadmePath(localPath, basePath);
4464
const namespaces = keys(await getApiVersionsByNamespace(readme));
@@ -76,7 +96,15 @@ executeSynchronous(async () => {
7696
console.log(chalk.red(`Caught exception processing autogenlist entry ${autoGenConfig.basePath}.`));
7797
console.log(chalk.red(error));
7898

79-
errors.push(error);
99+
// Use markdown formatting as this summary will be included in the PR description
100+
logOut(summaryLogger,
101+
`<details>
102+
<summary>Failed to generate types for path '${basePath}'</summary>
103+
\`\`\`
104+
${error}
105+
\`\`\`
106+
</details>
107+
`);
80108
}
81109
packages.push(pkg);
82110
}
@@ -87,10 +115,20 @@ executeSynchronous(async () => {
87115
if (args['output-path']) {
88116
const outputPath = await resolveAbsolutePath(args['output-path']);
89117
await writeJsonFile(outputPath, { packages });
90-
} else {
91-
if (errors.length > 0) {
92-
throw new Error(`Autogeneration failed with ${errors.length} errors. See logs for detailed information.`);
93-
}
94118
}
95-
96119
});
120+
121+
function logOut(logger: ILogger, line: string) {
122+
logger.out(`${line}\n`);
123+
}
124+
125+
async function getLogger(logFilePath: string): Promise<ILogger> {
126+
const logFileStream = createWriteStream(logFilePath, { flags: 'a' });
127+
128+
return {
129+
out: (data: string) => {
130+
process.stdout.write(data);
131+
logFileStream.write(stripAnsi(data));
132+
}
133+
};
134+
}

0 commit comments

Comments
 (0)