Validate asset hashes before uploading

Author: motiz88

scripts/releases/upload-release-assets-for-dotslash.js

@@ -12,6 +12,7 @@
 
 const {
   validateAndParseDotSlashFile,
+  validateDotSlashArtifactData,
   processDotSlashFileInPlace,
 } = require('./utils/dotslash-utils');
 const {REPO_ROOT} = require('../shared/consts');
@@ -104,6 +105,7 @@ async function uploadReleaseAssetsForDotSlash(
         providers,
         // NOTE: We mostly ignore suggestedFilename in favour of reading the actual asset URLs
         suggestedFilename,
+        artifactInfo,
       ) => {
         let upstreamUrl, targetReleaseAssetInfo;
         for (const provider of providers) {
@@ -170,6 +172,10 @@ async function uploadReleaseAssetsForDotSlash(
             // NOTE: Using curl because we have seen issues with fetch() on GHA
             // and the Meta CDN. ¯\_(ツ)_/¯
             const {data, headers} = await getWithCurl(upstreamUrl);
+            console.log(
+              `[${targetReleaseAssetInfo.name}] Validating download...`,
+            );
+            await validateDotSlashArtifactData(data, artifactInfo);
             if (dryRun) {
               console.log(
                 `[${targetReleaseAssetInfo.name}] Dry run: Not uploading to release.`,

scripts/releases/utils/__tests__/__snapshots__/dotslash-utils-test.js.snap

@@ -66,6 +66,11 @@ Array [
       },
     ],
     "test-linux-x86_64.tar.gz",
+    Object {
+      "digest": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "hash": "sha256",
+      "size": 0,
+    },
   ],
   Array [
     Array [
@@ -79,6 +84,19 @@ Array [
       },
     ],
     "test-macos-aarch64.zip",
+    Object {
+      "digest": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+      "hash": "sha256",
+      "size": 0,
+    },
   ],
 ]
 `;
+
+exports[`validateDotSlashArtifactData blake3 failure on digest mismatch 1`] = `"blake3 mismatch: expected 2623f14eac39a9cc7b211cda9c52bcb9949ccd63aed4040a6a1a9f5f9b9431fa, got af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262"`;
+
+exports[`validateDotSlashArtifactData blake3 failure on size mismatch 1`] = `"size mismatch: expected 1, got 0"`;
+
+exports[`validateDotSlashArtifactData sha256 failure on digest mismatch 1`] = `"sha256 mismatch: expected 558b2587b199594ac439b9464e14ea72429bf6998c4fbfa941c1cf89244c0b3e, got e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"`;
+
+exports[`validateDotSlashArtifactData sha256 failure on size mismatch 1`] = `"size mismatch: expected 1, got 0"`;

scripts/releases/utils/__tests__/dotslash-utils-test.js

@@ -16,6 +16,7 @@ const {
   processDotSlashFileInPlace,
   dangerouslyResignGeneratedFile,
   validateAndParseDotSlashFile,
+  validateDotSlashArtifactData,
 } = require('../dotslash-utils');
 
 jest.useRealTimers();
@@ -172,3 +173,77 @@ describe('dangerouslyResignGeneratedFile', () => {
 }`);
   });
 });
+
+describe('validateDotSlashArtifactData', () => {
+  test('blake3 success', async () => {
+    await expect(
+      validateDotSlashArtifactData(Buffer.from([]), {
+        hash: 'blake3',
+        digest:
+          'af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262',
+        size: 0,
+      }),
+    ).resolves.toBeUndefined();
+  });
+
+  test('blake3 failure on size mismatch', async () => {
+    await expect(
+      validateDotSlashArtifactData(Buffer.from([]), {
+        hash: 'blake3',
+        digest:
+          'af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262',
+        size: 1,
+      }),
+    ).rejects.toThrowErrorMatchingSnapshot();
+  });
+
+  test('blake3 failure on digest mismatch', async () => {
+    await expect(
+      validateDotSlashArtifactData(Buffer.from([]), {
+        hash: 'blake3',
+        digest:
+          'af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262'
+            .split('')
+            .reverse()
+            .join(''),
+        size: 0,
+      }),
+    ).rejects.toThrowErrorMatchingSnapshot();
+  });
+
+  test('sha256 success', async () => {
+    await expect(
+      validateDotSlashArtifactData(Buffer.from([]), {
+        hash: 'sha256',
+        digest:
+          'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
+        size: 0,
+      }),
+    ).resolves.toBeUndefined();
+  });
+
+  test('sha256 failure on size mismatch', async () => {
+    await expect(
+      validateDotSlashArtifactData(Buffer.from([]), {
+        hash: 'sha256',
+        digest:
+          'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855',
+        size: 1,
+      }),
+    ).rejects.toThrowErrorMatchingSnapshot();
+  });
+
+  test('sha256 failure on digest mismatch', async () => {
+    await expect(
+      validateDotSlashArtifactData(Buffer.from([]), {
+        hash: 'sha256',
+        digest:
+          'e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855'
+            .split('')
+            .reverse()
+            .join(''),
+        size: 0,
+      }),
+    ).rejects.toThrowErrorMatchingSnapshot();
+  });
+});

scripts/releases/utils/dotslash-utils.js

@@ -15,6 +15,8 @@ const {parse, modify, applyEdits} = require('jsonc-parser');
 const signedsource = require('signedsource');
 const dotslash = require('@motizilberman/dotslash');
 const execFile = require('util').promisify(require('child_process').execFile);
+const os = require('os');
+const path = require('path');
 
 /*::
 type DotSlashProvider = {
@@ -27,6 +29,14 @@ type DotSlashProvider = {
   name: string,
 };
 
+type DotSlashPlatformSpec = {
+  providers: DotSlashProvider[],
+  hash: 'blake3' | 'sha256',
+  digest: string,
+  size: number,
+  ...
+}
+
 type JSONCFormattingOptions = {
   tabSize?: number,
   insertSpaces?: boolean,
@@ -36,6 +46,11 @@ type JSONCFormattingOptions = {
 type DotSlashProvidersTransformFn = (
   providers: $ReadOnlyArray<DotSlashProvider>,
   suggestedFilename: string,
+  artifactInfo: $ReadOnlyArray<{
+    hash: 'blake3' | 'sha256',
+    digest: string,
+    size: number,
+  }>
 ) => ?$ReadOnlyArray<DotSlashProvider>;
 */
 
@@ -82,13 +97,17 @@ async function processDotSlashFileInPlace(
     splitShebangFromContents(originalContents);
   const json = parse(originalContentsJson);
   let intermediateContentsJson = originalContentsJson;
-  for (const [platform, platformSpec] of Object.entries(json.platforms)) {
+  for (const [platform, platformSpec] of Object.entries(json.platforms) /*::
+   as $ReadOnlyArray<[string, DotSlashPlatformSpec]>
+  */) {
     const providers = platformSpec.providers;
     const suggestedFilename =
       `${sanitizeFileNameComponent(json.name)}-${platform}` +
       (platformSpec.format ? `.${platformSpec.format}` : '');
+    const {hash, digest, size} = platformSpec;
     const newProviders =
-      transformProviders(providers, suggestedFilename) ?? providers;
+      transformProviders(providers, suggestedFilename, {hash, digest, size}) ??
+      providers;
     if (newProviders !== providers) {
       const edits = modify(
         intermediateContentsJson,
@@ -126,9 +145,43 @@ async function dangerouslyResignGeneratedFile(
   await fs.writeFile(filename, newContents);
 }
 
+async function validateDotSlashArtifactData(
+  data /*: Buffer */,
+  platformSpec /*: $ReadOnly<{
+    digest: string,
+    hash: 'blake3' | 'sha256',
+    size: number,
+    ...
+  }> */,
+) /*: Promise<void> */ {
+  const {digest: expectedDigest, hash, size} = platformSpec;
+  if (data.length !== size) {
+    throw new Error(`size mismatch: expected ${size}, got ${data.length}`);
+  }
+  const hashFunction = hash === 'blake3' ? 'b3sum' : 'sha256';
+
+  const tempDir = await fs.mkdtemp(
+    path.join(os.tmpdir(), 'validate-artifact-hash-'),
+  );
+  try {
+    const tempFile = path.join(tempDir, 'data');
+    await fs.writeFile(tempFile, data);
+    const {stdout} = await execFile(dotslash, ['--', hashFunction, tempFile]);
+    const actualDigest = stdout.trim();
+    if (actualDigest !== expectedDigest) {
+      throw new Error(
+        `${hash} mismatch: expected ${expectedDigest}, got ${actualDigest}`,
+      );
+    }
+  } finally {
+    await fs.rm(tempDir, {recursive: true, force: true});
+  }
+}
+
 module.exports = {
   DEFAULT_FORMATTING_OPTIONS,
   processDotSlashFileInPlace,
   dangerouslyResignGeneratedFile,
   validateAndParseDotSlashFile,
+  validateDotSlashArtifactData,
 };