chore: debug remove stale objects (#372)

Author: Aslemammad

.github/workflows/remove-stale.yml

@@ -5,8 +5,17 @@ jobs:
   rm_stale_packages:
     runs-on: ubuntu-latest
     steps:
-      - name: Send POST request
+      - name: Send POST request and capture output
         run: |
           curl -X POST \
+            -N \
             -H "sb-rm-stale-key: ${{ secrets.NITRO_RM_STALE_KEY }}" \
-            https://pkg.pr.new/rm/stale
+            https://pkg.pr.new/rm/stale \
+            | tee rm-stale.log
+
+      - name: Upload rm-stale.log as artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: rm-stale-log
+          path: rm-stale.log
+

packages/app/server/routes/rm/stale.post.ts

@@ -1,31 +1,48 @@
 import type { H3Event } from "h3";
 
 export default eventHandler(async (event) => {
+  setResponseHeader(event, "Transfer-Encoding", "chunked");
+  setResponseHeader(event, "Cache-Control", "no-cache");
+  setResponseHeader(event, "Content-Type", "text/plain");
+
   const rmStaleKeyHeader = getHeader(event, "sb-rm-stale-key");
+  const signal = toWebRequest(event).signal;
   const { rmStaleKey } = useRuntimeConfig(event);
+
   if (rmStaleKeyHeader !== rmStaleKey) {
     throw createError({
       status: 403,
     });
   }
-  return {
-    ok: true,
-    removed: [
-      ...(await Promise.all([
-        iterateAndDelete(event, {
-          prefix: usePackagesBucket.base,
-          limit: 100,
-        }),
-        iterateAndDelete(event, {
-          prefix: useTemplatesBucket.base,
-          limit: 100,
-        }),
-      ]).then((results) => results.flat())),
-    ],
-  };
+
+  const { readable, writable } = new TransformStream()
+
+  event.waitUntil(
+    (async () => {
+      // const writer = writable.getWriter()
+      // console.log('here')
+      // await writer.ready
+      // await writer.write(new TextEncoder().encode("start\n"))
+      // writer.releaseLock()
+
+      await iterateAndDelete(event, writable, signal, {
+        prefix: usePackagesBucket.base,
+        limit: 100,
+      })
+      await iterateAndDelete(event, writable, signal, {
+        prefix: useTemplatesBucket.base,
+        limit: 100,
+      })
+      await writable.close()
+    })()
+  )
+
+  return readable
 });
 
-async function iterateAndDelete(event: H3Event, opts: R2ListOptions) {
+async function iterateAndDelete(event: H3Event, writable: WritableStream, signal: AbortSignal, opts: R2ListOptions) {
+  const writer = writable.getWriter()
+  await writer.ready
   const binding = useBinding(event);
 
   let truncated = true;
@@ -34,34 +51,58 @@ async function iterateAndDelete(event: H3Event, opts: R2ListOptions) {
   const downloadedAtBucket = useDownloadedAtBucket(event);
   const today = Date.parse(new Date().toString());
 
-  const removed: string[] = [];
-  while (truncated) {
+  while (truncated && !signal.aborted) {
     // TODO: Avoid using context.cloudflare and migrate to unstorage, but it does not have truncated for now
     const next = await binding.list({
       ...opts,
       cursor,
     });
     for (const object of next.objects) {
+      if (signal.aborted) {
+        break;
+      }
       const uploaded = Date.parse(object.uploaded.toString());
       // remove the object anyway if it's 6 months old already
-      if ((today - uploaded) / (1000 * 3600 * 24 * 30 * 6) >= 1) {
-        removed.push(object.key);
-        event.context.cloudflare.context.waitUntil(binding.delete(object.key));
-        event.context.cloudflare.context.waitUntil(
-          downloadedAtBucket.removeItem(object.key),
-        );
+      // Use calendar-accurate 6 months check
+      const uploadedDate = new Date(uploaded);
+      const sixMonthsAgo = new Date(today);
+      sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6);
+      if (uploadedDate <= sixMonthsAgo) {
+        const downloadedAt = (await downloadedAtBucket.getItem(object.key))!;
+        await writer.write(new TextEncoder().encode(JSON.stringify({
+          key: object.key,
+          uploaded: new Date(object.uploaded),
+          downloadedAt: downloadedAt ? new Date(downloadedAt) : null,
+        }) + "\n"))
+        // event.context.cloudflare.context.waitUntil(binding.delete(object.key));
+        // event.context.cloudflare.context.waitUntil(
+        //   downloadedAtBucket.removeItem(object.key),
+        // );
+      }
+      const downloadedAt = (await downloadedAtBucket.getItem(object.key));
+
+      if (!downloadedAt) {
+        continue;
       }
-      const downloadedAt = (await downloadedAtBucket.getItem(object.key))!;
       // if it has not been downloaded in the last month and it's at least 1 month old
+      // Calendar-accurate 1 month checks
+      const downloadedAtDate = new Date(downloadedAt);
+      const oneMonthAgo = new Date(today);
+      oneMonthAgo.setMonth(oneMonthAgo.getMonth() - 1);
+      const uploadedDate2 = new Date(uploaded); // uploaded already parsed above
       if (
-        !((today - downloadedAt) / (1000 * 3600 * 24 * 30) < 1) &&
-        (today - uploaded) / (1000 * 3600 * 24 * 30) >= 1
+        downloadedAtDate <= oneMonthAgo &&
+        uploadedDate2 <= oneMonthAgo
       ) {
-        removed.push(object.key);
-        event.context.cloudflare.context.waitUntil(binding.delete(object.key));
-        event.context.cloudflare.context.waitUntil(
-          downloadedAtBucket.removeItem(object.key),
-        );
+        await writer.write(new TextEncoder().encode(JSON.stringify({
+          key: object.key,
+          uploaded: new Date(object.uploaded),
+          downloadedAt: new Date(downloadedAt),
+        }) + "\n"))
+        // event.context.cloudflare.context.waitUntil(binding.delete(object.key));
+        // event.context.cloudflare.context.waitUntil(
+        //   downloadedAtBucket.removeItem(object.key),
+        // );
       }
     }
 
@@ -70,5 +111,5 @@ async function iterateAndDelete(event: H3Event, opts: R2ListOptions) {
       cursor = next.cursor;
     }
   }
-  return removed;
+  writer.releaseLock()
 }