chore: log integ test output to the file system (#416)

Write the (huge) text logs of integ test files to the file system if
configured to do so (with `$INTEG_LOGS`), and use that feature in the
GitHub logs to publish the logs as an artifact.

At the same time, also start playing with Markdown reports of GitHub
actions by writing to `$GITHUB_STEP_SUMMARY` to see what that looks
like.

---
By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache-2.0 license

---------

Signed-off-by: github-actions <github-actions@github.com>
Co-authored-by: github-actions <github-actions@github.com>

Author: rix0rrr

.github/workflows/integ.yml

@@ -48,20 +48,26 @@ export function integTest(
         throw new Error('FAIL_FAST requested and currently failing. Stopping test early.');
       }
 
-      return await callback({
+      const ret = await callback({
         output,
         randomString: randomString(),
         name,
         log(s: string) {
           output.write(`${s}\n`);
         },
       });
+
+      await writeLog(name, true, output.toString());
+
+      return ret;
     } catch (e: any) {
+      // Print the buffered output, only if the test fails.
       failed = true;
 
-      // Print the buffered output, only if the test fails.
       output.write(e.message);
       output.write(e.stack);
+
+      await writeLog(name, false, output.toString());
       process.stderr.write(`[INTEG TEST::${name}] Failed: ${e}\n`);
 
       const isGitHub = !!process.env.GITHUB_RUN_ID;
@@ -108,3 +114,48 @@ export function randomString() {
   // Crazy
   return Math.random().toString(36).replace(/[^a-z0-9]+/g, '');
 }
+
+/**
+ * Write log files
+ *
+ * Write a text log to `${INTEG_LOGS}/[FAILED-]description-of-test.txt`, and a single
+ * line of a Markdown table to `${INTEG_LOGS}/md/1-description-of-test.md`.
+ *
+ * The latter are designed to be globcatted to $GITHUB_STEP_SUMMARY after tests
+ * (we don't write there directly to avoid concurrency issues with multiple processes
+ * reading and mutating the same file).
+ *
+ * We do use `atomicWrite` to write files -- it's only necessary for the header file,
+ * which gets overwritten by every test, just to make sure it properly exists (shouldn't
+ * end up empty or with interleaved contents). The other writes are not
+ * contended and don't need to be atomic, but the function is just ergonomic to use.
+ */
+async function writeLog(testName: string, success: boolean, output: string) {
+  if (process.env.INTEG_LOGS) {
+    const slug = slugify(testName);
+    const logFileName = `${process.env.INTEG_LOGS}/${success ? '' : 'FAILED-'}${slug}.txt`;
+    await atomicWrite(logFileName, output);
+
+    // Sort failures before successes, and the table header before all
+    await atomicWrite(`${process.env.INTEG_LOGS}/md/0-header.md`, [
+      '| Result | Test Name |',
+      '|--------|-----------|',
+    ].map(x => `${x}\n`).join(''));
+
+    const mdFileName = `${process.env.INTEG_LOGS}/md/${success ? '2' : '1'}-${slug}.md`;
+    const firstColumn = success ? 'pass ✅' : 'fail ❌';
+    await atomicWrite(mdFileName, `| ${firstColumn} | ${testName} |\n`);
+  }
+}
+
+function slugify(x: string) {
+  return x.replace(/[^a-zA-Z0-9_,]+/g, '-');
+}
+
+async function atomicWrite(fileName: string, contents: string) {
+  await fs.promises.mkdir(path.dirname(fileName), { recursive: true });
+
+  const tmp = `${fileName}.${process.pid}`;
+  await fs.promises.writeFile(tmp, contents);
+  await fs.promises.rename(tmp, fileName);
+}

packages/@aws-cdk-testing/cli-integ/lib/integ-test.ts

@@ -416,6 +416,23 @@ export class CdkCliIntegTestsWorkflow extends Component {
             CDK_MAJOR_VERSION: '2',
             RELEASE_TAG: 'latest',
             GITHUB_TOKEN: '${{ secrets.GITHUB_TOKEN }}',
+            INTEG_LOGS: 'logs',
+          },
+        },
+        {
+          name: 'Set workflow summary',
+          run: [
+            'echo "## Test results" >> $GITHUB_STEP_SUMMARY',
+            'cat logs/md/*.md >> $GITHUB_STEP_SUMMARY',
+          ].join('\n'),
+        },
+        {
+          name: 'Upload logs',
+          uses: 'actions/upload-artifact@v4.4.0',
+          with: {
+            name: 'logs',
+            path: 'logs/',
+            overwrite: 'true',
           },
         },
       ],