fix(build): fix hash file parsing (#1950)

In 2da3c31, we moved file hash computation for our produced artifacts
from the JSON feed generation step to the newly introduced "download artifacts"
step of the `release_publish` tasks (because we need to download artifacts
before we can submit them to the papertrail service). We used the standard-ish
SHASUMS format (output of `sha256sum`/input of `sha256sum -c`) for that, but
failed to parse it properly when consuming it.

This fixes that and extends the download center config tests to also perform
the full "download artifacts" step as part of the test setup, to ensure that
the two parts work together properly.

Author: addaleax

packages/build/src/download-center/config.spec.ts

@@ -1,8 +1,9 @@
 import type { DownloadCenterConfig } from '@mongodb-js/dl-center/dist/download-center-config';
-import { getPackageFile, type PackageInformationProvider } from '../packaging';
+import { type PackageInformationProvider } from '../packaging';
 import { expect } from 'chai';
 import sinon from 'sinon';
-import { ALL_PACKAGE_VARIANTS, type PackageVariant } from '../config';
+import type { Config } from '../config';
+import { type PackageVariant } from '../config';
 import {
   createVersionConfig,
   createDownloadCenterConfig,
@@ -13,6 +14,10 @@ import {
 import { promises as fs } from 'fs';
 import path from 'path';
 import fetch from 'node-fetch';
+import { createServer } from 'http';
+import { once } from 'events';
+import { runDownloadAndListArtifacts } from '../run-download-and-list-artifacts';
+import type { AddressInfo } from 'net';
 
 const packageInformation = (version: string) =>
   ((packageVariant: PackageVariant) => {
@@ -33,7 +38,7 @@ const packageInformation = (version: string) =>
 
 describe('DownloadCenter config', function () {
   let outputDir: string;
-  beforeEach(async function () {
+  before(async function () {
     outputDir = path.join(
       __dirname,
       '..',
@@ -45,26 +50,32 @@ describe('DownloadCenter config', function () {
     );
     await fs.mkdir(outputDir, { recursive: true });
 
-    const testVersions = ['2.0.1', '2.0.0', '1.2.2'];
-    const allFiles = testVersions
-      .flatMap((version) =>
-        ALL_PACKAGE_VARIANTS.map(
-          (packageVariant) =>
-            getPackageFile(packageVariant, packageInformation(version)).path
-        )
-      )
-      .flatMap((file) => [file, `${file}.sig`]);
-    await fs.writeFile(
-      path.join(outputDir, 'SHASUMS1.txt'),
-      allFiles.map((f) => `ghjklm  ${f}`).join('\n')
-    );
-    await fs.writeFile(
-      path.join(outputDir, 'SHASUMS256.txt'),
-      allFiles.map((f) => `abcdef  ${f}`).join('\n')
-    );
+    const httpServer = createServer((req, res) => {
+      res.end(req.url);
+    });
+    httpServer.listen(0);
+    await once(httpServer, 'listening');
+    try {
+      const testVersions = ['2.0.1', '2.0.0', '1.2.2'];
+
+      for (const version of testVersions) {
+        const config: Partial<Config> = {
+          outputDir,
+          packageInformation: packageInformation(version),
+        };
+        await runDownloadAndListArtifacts(
+          config as Config,
+          `http://localhost:${(httpServer.address() as AddressInfo).port}/`,
+          'append-to-hash-file-for-testing'
+        );
+      }
+    } finally {
+      httpServer.close();
+      await once(httpServer, 'close');
+    }
   });
 
-  afterEach(async function () {
+  after(async function () {
     await fs.rm(outputDir, { recursive: true });
   });
 
@@ -376,8 +387,9 @@ describe('DownloadCenter config', function () {
           archive: {
             type: 'zip',
             url: `${baseUrl}mongosh-1.2.2-darwin-x64.zip`,
-            sha256: 'abcdef',
-            sha1: 'ghjklm',
+            sha256:
+              'ed8922ef572c307634150bf78ea02338349949f29245123884b1318cdc402dd9',
+            sha1: '4f1b987549703c58940be9f8582f4032374b1074',
           },
         });
       });

packages/build/src/download-center/config.ts

@@ -293,13 +293,13 @@ async function getHashes(
         );
         const line = content
           .split('\n')
-          .find((line) => line.endsWith(packagedFilename));
+          .find((line) => line.trim().startsWith(packagedFilename));
         if (!line) {
           throw new Error(
             `Could not find entry for ${packagedFilename} in ${filename}`
           );
         }
-        return [hash, line.trim().split(/\s/)[0]] as const;
+        return [hash, line.trim().split(/\s+/)[1]] as const;
       })
     )
   ) as { sha1: string; sha256: string };

packages/build/src/run-download-and-list-artifacts.ts

@@ -17,7 +17,8 @@ export const hashListFiles = [
 
 export async function runDownloadAndListArtifacts(
   config: Config,
-  publicArtifactBaseUrl: string = ARTIFACTS_URL_PUBLIC_BASE
+  publicArtifactBaseUrl: string = ARTIFACTS_URL_PUBLIC_BASE,
+  hashFileWriteMode: 'normal' | 'append-to-hash-file-for-testing' = 'normal'
 ): Promise<void> {
   const requiredConfigKeys: (keyof Config)[] = ['outputDir'];
   for (const key of requiredConfigKeys) {
@@ -66,7 +67,10 @@ export async function runDownloadAndListArtifacts(
         .filter(Boolean)
         .map((file) => `${file?.filename}  ${file?.[hash]}`)
         .join('\n') + '\n';
-    await fs.writeFile(filepath, contents);
+    await (hashFileWriteMode === 'normal' ? fs.writeFile : fs.appendFile)(
+      filepath,
+      contents
+    );
     console.log('wrote hash list to', filepath);
   }
 }