ci(md): Upload md files to R2 (#14171)

Author: BYK

package.json

@@ -38,6 +38,7 @@
   },
   "dependencies": {
     "@ariakit/react": "^0.4.5",
+    "@aws-sdk/client-s3": "^3.837.0",
     "@emotion/core": "^11.0.0",
     "@emotion/react": "^11.11.0",
     "@emotion/styled": "^11.0.0",

scripts/generate-md-exports.mjs

@@ -1,12 +1,14 @@
 #!/usr/bin/env node
 /* eslint-disable no-console */
+import {ListObjectsV2Command, PutObjectCommand, S3Client} from '@aws-sdk/client-s3';
 import {selectAll} from 'hast-util-select';
 import {createHash} from 'node:crypto';
 import {createReadStream, createWriteStream, existsSync} from 'node:fs';
-import {mkdir, opendir, readFile, rm} from 'node:fs/promises';
+import {mkdir, opendir, readFile, rm, writeFile} from 'node:fs/promises';
 import {cpus} from 'node:os';
 import * as path from 'node:path';
-import {Readable} from 'node:stream';
+import {compose, Readable} from 'node:stream';
+import {text} from 'node:stream/consumers';
 import {pipeline} from 'node:stream/promises';
 import {fileURLToPath} from 'node:url';
 import {isMainThread, parentPort, Worker, workerData} from 'node:worker_threads';
@@ -23,17 +25,40 @@ import {unified} from 'unified';
 import {remove} from 'unist-util-remove';
 
 const CACHE_COMPRESS_LEVEL = 4;
+const R2_BUCKET = 'sentry-docs';
+const accessKeyId = process.env.R2_ACCESS_KEY_ID;
+const secretAccessKey = process.env.R2_SECRET_ACCESS_KEY;
 
-function taskFinishHandler(data) {
-  if (data.failedTasks.length === 0) {
-    console.log(
-      `💰 Worker[${data.id}]: Cache hits: ${data.cacheHits} (${Math.round((data.cacheHits / data.success) * 100)}%)`
-    );
-    console.log(`✅ Worker[${data.id}]: converted ${data.success} files successfully.`);
+function getS3Client() {
+  return new S3Client({
+    endpoint: 'https://773afa1f62ff86c80db4f24f7ff1e9c8.r2.cloudflarestorage.com',
+    region: 'auto',
+    credentials: {
+      accessKeyId,
+      secretAccessKey,
+    },
+    retryMode: 'adaptive',
+  });
+}
+
+async function uploadToCFR2(s3Client, relativePath, data) {
+  const command = new PutObjectCommand({
+    Bucket: R2_BUCKET,
+    Key: relativePath,
+    Body: data,
+    ContentType: 'text/markdown',
+  });
+  await s3Client.send(command);
+  return;
+}
+
+function taskFinishHandler({id, success, failedTasks}) {
+  if (failedTasks.length === 0) {
+    console.log(`✅ Worker[${id}]: converted ${success} files successfully.`);
     return false;
   }
-  console.error(`❌ Worker[${data.id}]: ${data.failedTasks.length} files failed:`);
-  console.error(data.failedTasks);
+  console.error(`❌ Worker[${id}]: ${failedTasks.length} files failed:`);
+  console.error(failedTasks);
   return true;
 }
 
@@ -68,13 +93,34 @@ async function createWork() {
   const numWorkers = Math.max(Math.floor(cpus().length / 2), 2);
   const workerTasks = new Array(numWorkers).fill(null).map(() => []);
 
+  const existingFilesOnR2 = null;
+  if (accessKeyId && secretAccessKey) {
+    console.log(`☁️ Getting existing hashes from R2...`);
+    const s3Client = getS3Client();
+    let continuationToken = undefined;
+    do {
+      const response = await s3Client.send(
+        new ListObjectsV2Command({
+          Bucket: R2_BUCKET,
+          ContinuationToken: continuationToken,
+        })
+      );
+      continuationToken = response.NextContinuationToken;
+      for (const {Key, ETag} of response.Contents) {
+        existingFilesOnR2.set(Key, ETag.slice(1, -1)); // Remove quotes from ETag
+      }
+    } while (continuationToken);
+    console.log(`✅ Found ${existingFilesOnR2.size} existing files on R2.`);
+  }
+
   console.log(`🔎 Discovering files to convert...`);
 
   let numFiles = 0;
   let workerIdx = 0;
   // Need a high buffer size here otherwise Node skips some subdirectories!
   // See https://github.com/nodejs/node/issues/48820
   const dir = await opendir(INPUT_DIR, {recursive: true, bufferSize: 1024});
+
   for await (const dirent of dir) {
     if (dirent.name.endsWith('.html') && dirent.isFile()) {
       const sourcePath = path.join(dirent.parentPath || dirent.path, dirent.name);
@@ -84,7 +130,13 @@ async function createWork() {
       );
       await mkdir(targetDir, {recursive: true});
       const targetPath = path.join(targetDir, dirent.name.slice(0, -5) + '.md');
-      workerTasks[workerIdx].push({sourcePath, targetPath});
+      const relativePath = path.relative(OUTPUT_DIR, targetPath);
+      workerTasks[workerIdx].push({
+        sourcePath,
+        targetPath,
+        relativePath,
+        r2Hash: existingFilesOnR2 ? existingFilesOnR2.get(relativePath) : null,
+      });
       workerIdx = (workerIdx + 1) % numWorkers;
       numFiles++;
     }
@@ -96,7 +148,12 @@ async function createWork() {
   const workerPromises = new Array(numWorkers - 1).fill(null).map((_, id) => {
     return new Promise((resolve, reject) => {
       const worker = new Worker(selfPath, {
-        workerData: {id, noCache, cacheDir: CACHE_DIR, tasks: workerTasks[id]},
+        workerData: {
+          id,
+          noCache,
+          cacheDir: CACHE_DIR,
+          tasks: workerTasks[id],
+        },
       });
       let hasErrors = false;
       worker.on('message', data => (hasErrors = taskFinishHandler(data)));
@@ -113,10 +170,10 @@ async function createWork() {
   // The main thread can also process tasks -- That's 65% more bullet per bullet! -Cave Johnson
   workerPromises.push(
     processTaskList({
-      noCache,
-      cacheDir: CACHE_DIR,
-      tasks: workerTasks[workerTasks.length - 1],
       id: workerTasks.length - 1,
+      tasks: workerTasks[workerTasks.length - 1],
+      cacheDir: CACHE_DIR,
+      noCache,
     }).then(data => {
       if (taskFinishHandler(data)) {
         throw new Error(`Worker[${data.id}] had some errors.`);
@@ -133,25 +190,24 @@ async function createWork() {
 const md5 = data => createHash('md5').update(data).digest('hex');
 
 async function genMDFromHTML(source, target, {cacheDir, noCache}) {
-  const text = (await readFile(source, {encoding: 'utf8'}))
+  const leanHTML = (await readFile(source, {encoding: 'utf8'}))
     // Remove all script tags, as they are not needed in markdown
     // and they are not stable across builds, causing cache misses
     .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '');
-  const hash = md5(text);
-  const cacheFile = path.join(cacheDir, hash);
+  const cacheKey = md5(leanHTML);
+  const cacheFile = path.join(cacheDir, cacheKey);
   if (!noCache) {
     try {
-      await pipeline(
-        createReadStream(cacheFile),
-        createBrotliDecompress(),
-        createWriteStream(target, {
-          encoding: 'utf8',
-        })
+      const data = await text(
+        compose(createReadStream(cacheFile), createBrotliDecompress())
       );
+      await writeFile(target, data, {encoding: 'utf8'});
 
-      return true;
-    } catch {
-      // pass
+      return {cacheHit: true, data};
+    } catch (err) {
+      if (err.code !== 'ENOENT') {
+        console.warn(`Error using cache file ${cacheFile}:`, err);
+      }
     }
   }
 
@@ -178,7 +234,7 @@ async function genMDFromHTML(source, target, {cacheDir, noCache}) {
       .use(() => tree => remove(tree, {type: 'inlineCode', value: ''}))
       .use(remarkGfm)
       .use(remarkStringify)
-      .process(text)
+      .process(leanHTML)
   );
   const reader = Readable.from(data);
 
@@ -203,23 +259,62 @@ async function genMDFromHTML(source, target, {cacheDir, noCache}) {
     ).catch(err => console.warn('Error writing cache file:', err)),
   ]);
 
-  return false;
+  return {cacheHit: false, data};
 }
 
 async function processTaskList({id, tasks, cacheDir, noCache}) {
+  const s3Client = getS3Client();
   const failedTasks = [];
-  let cacheHits = 0;
-  for (const {sourcePath, targetPath} of tasks) {
+  let cacheMisses = [];
+  let r2CacheMisses = [];
+  console.log(`🤖 Worker[${id}]: Starting to process ${tasks.length} files...`);
+  for (const {sourcePath, targetPath, relativePath, r2Hash} of tasks) {
     try {
-      cacheHits += await genMDFromHTML(sourcePath, targetPath, {
+      const {data, cacheHit} = await genMDFromHTML(sourcePath, targetPath, {
         cacheDir,
         noCache,
       });
+      if (!cacheHit) {
+        cacheMisses.push(relativePath);
+      }
+
+      if (r2Hash !== null) {
+        const fileHash = md5(data);
+        if (r2Hash !== fileHash) {
+          r2CacheMisses.push(relativePath);
+          console.log(
+            `📤 Worker[${id}]: Uploading ${relativePath} to R2, hash mismatch: ${r2Hash} !== ${fileHash}`
+          );
+          await uploadToCFR2(s3Client, relativePath, data);
+        }
+      }
     } catch (error) {
       failedTasks.push({sourcePath, targetPath, error});
     }
   }
-  return {id, success: tasks.length - failedTasks.length, failedTasks, cacheHits};
+  const success = tasks.length - failedTasks.length;
+  if (r2CacheMisses.length / tasks.length > 0.1) {
+    console.warn(
+      `⚠️ Worker[${id}]: More than 10% of files had a different hash on R2, this might indicate a problem with the cache or the generation process.`
+    );
+  } else if (r2CacheMisses.length > 0) {
+    console.log(
+      `📤 Worker[${id}]: Updated the following files on R2: \n${r2CacheMisses.map(n => ` - ${n}`).join('\n')}`
+    );
+  }
+  if (cacheMisses.length / tasks.length > 0.1) {
+    console.warn(`⚠️ Worker[${id}]: More than 10% cache miss rate during build.`);
+  } else if (cacheMisses.length > 0) {
+    console.log(
+      `❇️ Worker[${id}]: Updated cache for the following files: \n${cacheMisses.map(n => ` - ${n}`).join('\n')}`
+    );
+  }
+
+  return {
+    id,
+    success,
+    failedTasks,
+  };
 }
 
 async function doWork(work) {