Merge pull request #2924 from anthony-c-martin/main

anthony-c-martin · web-flow · commit c45181b46810 · 2023-07-21T16:09:49.000-04:00
Support batching in GitHub Actions
diff --git a/.github/workflows/generate-schemas-batch.yml b/.github/workflows/generate-schemas-batch.yml
@@ -1,8 +1,8 @@
-name: Generate Schemas batch
+name: Generate Schemas Batch
 
 on:
   schedule:
-    - cron:  '0 1 * * 3'
+    - cron: '45 5 * * SUN'
   workflow_dispatch:
     inputs:
       api_specs_ref:
@@ -11,215 +11,106 @@ on:
         default: 'main'
 
 env:
-  SUMMARY_LOG_DIR: /tmp/summary
-  SUMMARY_LOG_PATH: /tmp/summary/summary.log
-  SUMMARY_LOG_ARTIFACT_NAME: summary-log
-  AUTOGENERATE_BRANCH_NAME: autogenerate
-  TEMP_AUTOGENERATE_BRANCH_NAME: autogenerate_tmp
-  AUTHOR: ${{ github.actor }}
-  AUTHOR_EMAIL: ${{ github.actor }}@users.noreply.github.com
+  # This must be kept in sync with the arguments passed to the "batch" matrix
+  BATCH_COUNT: 20
+
 jobs:
-  batch-0:
-    name: Update Schemas Batch 0
+  generate:
+    name: Update Schemas Batch ${{ matrix.batch }}
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        batch: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
     steps:
-      - uses: actions/checkout@v2.3.5
-        with:
-          ref: ${{ env.AUTOGENERATE_BRANCH_NAME }}
-
-      - uses: peterjgrainger/action-create-branch@v2.0.1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          branch: ${{ env.TEMP_AUTOGENERATE_BRANCH_NAME }}
+      - name: Checkout repo
+        uses: actions/checkout@v3
 
       - name: Clone azure-rest-api-specs
-        uses: actions/checkout@v2.3.5
+        uses: actions/checkout@v3
         with:
           repository: Azure/azure-rest-api-specs
           path: workflow-temp/azure-rest-api-specs
           ref: ${{ github.event.inputs.api_specs_ref }}
-      
-      - name: Install generator npm packages
-        run: npm ci
-        working-directory: generator
 
-      - name: Create initial log description
-        run: mkdir -p /tmp/summary
-      
-      - id: generate
-        name: Run generator
-        run: |
-          npm run generate-all -- --local-path "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" --summary-log-path $SUMMARY_LOG_PATH --batch-count 4 --batch-index 0
-        working-directory: generator
-
-      - name: Commit changes
-        uses: EndBug/add-and-commit@v8
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
         with:
-          default_author: github_actor
-          message: 'Autogenerate schemas batch 0'
-          new_branch: ${{ env.TEMP_AUTOGENERATE_BRANCH_NAME }}
-
-      - name: Upload summary log
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ env.SUMMARY_LOG_ARTIFACT_NAME }}
-          path: ${{ env.SUMMARY_LOG_PATH }}
-  batch-1:
-    needs: batch-0
-    name: Update Schemas Batch 1
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2.3.5
-        with:
-          ref: ${{ env.TEMP_AUTOGENERATE_BRANCH_NAME }}
-
-      - name: Clone azure-rest-api-specs
-        uses: actions/checkout@v2.3.5
-        with:
-          repository: Azure/azure-rest-api-specs
-          path: workflow-temp/azure-rest-api-specs
-          ref: ${{ github.event.inputs.api_specs_ref }}
+          node-version: 16.x
       
       - name: Install generator npm packages
         run: npm ci
         working-directory: generator
 
-      - name: Download summary log artifact
-        uses: actions/download-artifact@v2
-        with:
-          name: ${{ env.SUMMARY_LOG_ARTIFACT_NAME }}
-          path: ${{ env.SUMMARY_LOG_DIR }}
-      
-      - id: generate
-        name: Run generator
+      - name: Run generator
         run: |
-          npm run generate-all -- --local-path "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" --summary-log-path $SUMMARY_LOG_PATH --batch-count 4 --batch-index 1
+          rm -Rf "$GITHUB_WORKSPACE/schemas"
+          rm -Rf "$GITHUB_WORKSPACE/summary.log"
+          mkdir -p "$GITHUB_WORKSPACE/schemas"
+
+          npm run generate-all -- \
+            --local-path "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" \
+            --batch-count ${{ env.BATCH_COUNT }} \
+            --batch-index ${{ matrix.batch }} \
+            --summary-log-path "$GITHUB_WORKSPACE/summary.log" \
+            --combine-batch-mode true
         working-directory: generator
 
-      - name: Commit changes
-        uses: EndBug/add-and-commit@v8
+      - name: Upload Schemas
+        uses: actions/upload-artifact@v3
         with:
-          default_author: github_actor
-          message: 'Autogenerate schemas batch 1'
-          new_branch: ${{ env.TEMP_AUTOGENERATE_BRANCH_NAME }}
+          name: batch-${{ matrix.batch }}-schemas
+          path: schemas
+          if-no-files-found: error
 
       - name: Upload summary log
-        uses: actions/upload-artifact@v2
+        uses: actions/upload-artifact@v3
         with:
-          name: ${{ env.SUMMARY_LOG_ARTIFACT_NAME }}
-          path: ${{ env.SUMMARY_LOG_PATH }}
-  batch-2:
-    needs: batch-1
-    name: Update Schemas Batch 2
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2.3.5
-        with:
-          ref: ${{ env.TEMP_AUTOGENERATE_BRANCH_NAME }}
+          name: batch-${{ matrix.batch }}-summary
+          path: summary.log
+          if-no-files-found: error
 
-      - name: Clone azure-rest-api-specs
-        uses: actions/checkout@v2.3.5
-        with:
-          repository: Azure/azure-rest-api-specs
-          path: workflow-temp/azure-rest-api-specs
-          ref: ${{ github.event.inputs.api_specs_ref }}
-      
-      - name: Install generator npm packages
-        run: npm ci
-        working-directory: generator
-
-      - name: Download summary log artifact
-        uses: actions/download-artifact@v2
-        with:
-          name: ${{ env.SUMMARY_LOG_ARTIFACT_NAME }}
-          path: ${{ env.SUMMARY_LOG_DIR }}
-      
-      - id: generate
-        name: Run generator
-        run: |
-          npm run generate-all -- --local-path "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" --summary-log-path $SUMMARY_LOG_PATH --batch-count 4 --batch-index 2
-        working-directory: generator
-
-      - name: Commit changes
-        uses: EndBug/add-and-commit@v8
-        with:
-          default_author: github_actor
-          message: 'Autogenerate schemas batch 2'
-          new_branch: ${{ env.TEMP_AUTOGENERATE_BRANCH_NAME }}
-
-      - name: Upload summary log
-        uses: actions/upload-artifact@v2
-        with:
-          name: ${{ env.SUMMARY_LOG_ARTIFACT_NAME }}
-          path: ${{ env.SUMMARY_LOG_PATH }}
-  batch-3:
-    needs: batch-2
-    name: Update Schemas Batch 3
+  combine:
+    needs: generate
+    name: Combine Schema Batches
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2.3.5
+      - name: Checkout repo
+        uses: actions/checkout@v3
+
+      - name: Download batch results
+        uses: actions/download-artifact@v3
         with:
-          ref: ${{ env.TEMP_AUTOGENERATE_BRANCH_NAME }}
+          path: workflow-temp
 
-      - name: Clone azure-rest-api-specs
-        uses: actions/checkout@v2.3.5
+      - name: Setup Node.js
+        uses: actions/setup-node@v3
         with:
-          repository: Azure/azure-rest-api-specs
-          path: workflow-temp/azure-rest-api-specs
-          ref: ${{ github.event.inputs.api_specs_ref }}
+          node-version: 16.x
       
       - name: Install generator npm packages
         run: npm ci
         working-directory: generator
 
-      - name: Download summary log artifact
-        uses: actions/download-artifact@v2
-        with:
-          name: ${{ env.SUMMARY_LOG_ARTIFACT_NAME }}
-          path: ${{ env.SUMMARY_LOG_DIR }}
-      
-      - id: generate
-        name: Run generator
+      - name: Combine batches
         run: |
-          npm run generate-all -- --local-path "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" --summary-log-path $SUMMARY_LOG_PATH --batch-count 4 --batch-index 3
-
-          summary="$(<$SUMMARY_LOG_PATH)"
-          summary="${summary//'%'/'%25'}"
-          summary="${summary//$'\n'/'%0A'}"
-          summary="${summary//$'\r'/'%0D'}"
-          echo "::set-output name=summary::$summary"
+          npm run combine-batches -- \
+            --input-path "$GITHUB_WORKSPACE/workflow-temp" \
+            --batch-count ${{ env.BATCH_COUNT }}
         working-directory: generator
 
-      - id: get_swagger_gh_uri
-        name: Get GitHub URI for azure-rest-api-specs
-        run: |
-          git_sha=`git rev-parse HEAD`
-          echo "::set-output name=gh_uri::https://github.com/Azure/azure-rest-api-specs/tree/$git_sha"
-        working-directory: workflow-temp/azure-rest-api-specs
-
       - name: Create Pull Request
-        uses: peter-evans/create-pull-request@v4.1.3
+        uses: peter-evans/create-pull-request@v5
         with:
           committer: GitHub <noreply@github.com>
-          author: ${{ env.AUTHOR }} <${{ env.AUTHOR_EMAIL }}>
+          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
           signoff: false
-          branch: main
-          branch-suffix: short-commit-hash
+          branch: autogenerate-batch
           delete-branch: true
-          token: ${{ secrets.GITHUB_TOKEN }}
           title: |
-            Update Generated Schemas ${{ github.event.inputs.single_path && format('(single path: {0})', github.event.inputs.single_path) || '' }}
+            Update Generated Schemas
           body:  |
-            Update Generated Schemas ${{ github.event.inputs.single_path && format('(single path: {0})', github.event.inputs.single_path) || '' }}
-
-            Generate schemas for ${{ steps.get_swagger_gh_uri.outputs.gh_uri }}
-
-            Summary
-            ${{ steps.generate.outputs.summary }}
+            Update Generated Schemas
           commit-message: |
-            Update Generated Schemas ${{ github.event.inputs.single_path && format('(single path: {0})', github.event.inputs.single_path) || '' }}
-
-            Generate schemas for ${{ steps.get_swagger_gh_uri.outputs.gh_uri }}
-          labels: automerge
+            Update Generated Schemas
+          labels: autogenerate
           draft: false
diff --git a/generator/cmd/combine-batches.ts b/generator/cmd/combine-batches.ts
@@ -0,0 +1,44 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+import path from 'path';
+import { SchemaConfiguration, saveAutoGeneratedSchemaRefs } from '../generate';
+import { executeSynchronous, readJsonFile, } from '../utils';
+import yargs from 'yargs';
+import { flatten, uniq } from 'lodash';
+import { cp } from 'fs/promises';
+import { schemasBasePath } from '../constants';
+
+const argsConfig = yargs
+  .strict()
+  .option('input-path', { type: 'string', desc: 'The path to find batch results' })
+  .option('batch-count', { type: 'number', desc: 'The total number of batches' });
+
+executeSynchronous(async () => {
+  const args = await argsConfig.parseAsync();
+
+  const inputPath = args['input-path'];
+  const batchCount = args['batch-count'];
+  if (!inputPath || !batchCount) {
+    throw 'Bad input';
+  }
+
+  let schemaConfigs: SchemaConfiguration[] = [];
+  for (let i = 0; i < batchCount; i++) {
+    const schemasPath = path.join(inputPath, `batch-${i}-schemas`);
+    const batchSchemaConfigs: SchemaConfiguration[] = await readJsonFile(`${schemasPath}/schemaconfig.json`);
+
+    const relativePaths = uniq(batchSchemaConfigs.map(x => x.relativePath));
+    for (const relativePath of relativePaths) {
+      const source = path.join(schemasPath, relativePath);
+      const dest = path.join(schemasBasePath, relativePath);
+
+      console.log(`copying ${source} to ${dest}`);
+      await cp(source, dest, { force: true });
+    }
+
+    console.log(`schema config for batch ${i}: ${JSON.stringify(batchSchemaConfigs, null, 2)}`);
+    schemaConfigs = [...schemaConfigs, ...batchSchemaConfigs];
+  }
+
+  await saveAutoGeneratedSchemaRefs(flatten(schemaConfigs));
+});
diff --git a/generator/cmd/generateall.ts b/generator/cmd/generateall.ts
@@ -13,6 +13,7 @@ import path from 'path';
 
 import { createWriteStream } from 'fs';
 import stripAnsi from 'strip-ansi';
+import { schemasBasePath } from '../constants';
 
 const argsConfig = yargs
   .strict()
@@ -21,6 +22,7 @@ const argsConfig = yargs
   .option('local-path', { type: 'string', desc: 'The local path to the azure-rest-api-specs repo' })
   .option('readme-files', { type: 'array', desc: 'The list of readme.md files to generate schemas for' })
   .option('output-path', { type: 'string', desc: 'The base path to save schema output' })
+  .option('combine-batch-mode', { type: 'boolean', desc: 'If true, we rely on the combine-batches command to summarize the results.' })
   .option('summary-log-path', { type: 'string', desc: 'The path to store generation summary information. File will be saved in md format.' });
 
 interface ILogger {
@@ -33,6 +35,7 @@ executeSynchronous(async () => {
     let basePaths;
     let localPath = args['local-path'];
     let summaryPath = args['summary-log-path'];
+    const combineBatchMode = args['combine-batch-mode'] ?? false;
 
     // localPath refers to the specs repo (azure-rest-api-specs)
     if (!localPath) {
@@ -53,8 +56,12 @@ executeSynchronous(async () => {
     // resolve absolute path
     summaryPath = await resolveAbsolutePath(summaryPath);
 
-    if (args['batch-count'] !== undefined && args['batch-index'] !== undefined) {
-        basePaths = chunker(basePaths, args['batch-count'])[args['batch-index']];
+    const batchIndex = args['batch-index'];
+    const batchCount = args['batch-count'];
+    if (batchCount !== undefined && batchIndex !== undefined) {
+        basePaths = chunker(basePaths, batchCount)[batchIndex];
+
+        console.log(`Generating following base paths for batch ${batchIndex}: ${JSON.stringify(basePaths, null, 2)}`);
     }
 
     const schemaConfigs: SchemaConfiguration[] = [];
@@ -81,7 +88,9 @@ executeSynchronous(async () => {
                 });
             }
 
-            await clearAutoGeneratedSchemaRefs(filteredAutoGenList);
+            if (!combineBatchMode) {
+                await clearAutoGeneratedSchemaRefs(filteredAutoGenList);
+            }
 
             for (const autoGenConfig of filteredAutoGenList) {
                 const pkg = {
@@ -135,7 +144,12 @@ executeSynchronous(async () => {
         
     }
 
-    await saveAutoGeneratedSchemaRefs(flatten(schemaConfigs));
+    if (combineBatchMode) {
+        const schemaConfigFile = path.join(schemasBasePath, 'schemaconfig.json');
+        await writeJsonFile(schemaConfigFile, schemaConfigs);
+    } else {
+        await saveAutoGeneratedSchemaRefs(flatten(schemaConfigs));
+    }
 
     if (args['output-path']) {
         const outputPath = await resolveAbsolutePath(args['output-path']);
diff --git a/generator/package.json b/generator/package.json
