chore: some changes to remove stale

Author: Aslemammad

packages/app/server/routes/rm/stale.post.ts

@@ -5,25 +5,22 @@ export default eventHandler(async (event) => {
   const signal = toWebRequest(event).signal;
   const { rmStaleKey } = useRuntimeConfig(event);
 
-  // if (rmStaleKeyHeader !== rmStaleKey) {
-  //   throw createError({
-  //     status: 403,
-  //   });
-  // }
+  if (rmStaleKeyHeader !== rmStaleKey) {
+    throw createError({
+      status: 403,
+    });
+  }
 
   const { bucket, cursor, remove } = await readBody<{ bucket: 'packages' | 'templates'; cursor: string | null; remove: boolean }>(event);
 
+  setResponseHeader(event, "Content-Type", "application/json");
+
   const result = await iterateAndDelete(event, signal, {
     prefix: bucket === 'packages' ? usePackagesBucket.base : useTemplatesBucket.base,
-    limit: 1000,
     cursor: cursor || undefined,
   }, remove);
 
-  setResponseHeader(event, "Content-Type", "application/json");
-
-  return {
-    result,
-  };
+  return result;
 });
 
 // Helper for concurrency limiting
@@ -54,68 +51,63 @@ async function iterateAndDelete(event: H3Event, signal: AbortSignal, opts: R2Lis
   const binding = useBinding(event);
   let truncated = true;
   let cursor: string | undefined = opts.cursor;
-  let processed = 0;
   const removedItems: Array<{ key: string; uploaded: Date; downloadedAt?: Date }> = [];
   const downloadedAtBucket = useDownloadedAtBucket(event);
   const today = Date.parse(new Date().toString());
   const CONCURRENCY = 10;
 
-  while (truncated && !signal.aborted) {
-    if (removedItems.length >= 100 || processed >= 10000) {
-      break;
-    }
-    const next = await binding.list({
-      ...opts,
-      cursor,
-    });
-    processed += next.objects.length;
-
-    await mapWithConcurrency(next.objects, CONCURRENCY, async (object) => {
-      if (signal.aborted) return;
-      const uploaded = Date.parse(object.uploaded.toString());
-      const uploadedDate = new Date(uploaded);
-      const sixMonthsAgo = new Date(today);
-      sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6);
-      if (uploadedDate <= sixMonthsAgo) {
-        removedItems.push({
-          key: object.key,
-          uploaded: new Date(object.uploaded),
-        });
-        if (remove) {
-          await binding.delete(object.key);
-          await downloadedAtBucket.removeItem(object.key);
-        }
-        return;
-      }
-      const downloadedAt = await downloadedAtBucket.getItem(object.key);
-      if (!downloadedAt) {
-        return;
+  const next = await binding.list({
+    ...opts,
+    limit: 1000,
+    cursor,
+  });
+
+  await mapWithConcurrency(next.objects, CONCURRENCY, async (object) => {
+    if (signal.aborted) return;
+    const uploaded = Date.parse(object.uploaded.toString());
+    const uploadedDate = new Date(uploaded);
+    const sixMonthsAgo = new Date(today);
+    sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6);
+    if (uploadedDate <= sixMonthsAgo) {
+      removedItems.push({
+        key: object.key,
+        uploaded: new Date(object.uploaded),
+      });
+      if (remove) {
+        await binding.delete(object.key);
+        await downloadedAtBucket.removeItem(object.key);
       }
-      const downloadedAtDate = new Date(downloadedAt);
-      const oneMonthAgo = new Date(today);
-      oneMonthAgo.setMonth(oneMonthAgo.getMonth() - 1);
-      const uploadedDate2 = new Date(uploaded);
-      if (
-        downloadedAtDate <= oneMonthAgo &&
-        uploadedDate2 <= oneMonthAgo
-      ) {
-        removedItems.push({
-          key: object.key,
-          uploaded: new Date(object.uploaded),
-          downloadedAt: new Date(downloadedAt),
-        });
-        if (remove) {
-          await binding.delete(object.key);
-          await downloadedAtBucket.removeItem(object.key);
-        }
+      return;
+    }
+    const downloadedAt = await downloadedAtBucket.getItem(object.key);
+    if (!downloadedAt) {
+      return;
+    }
+    const downloadedAtDate = new Date(downloadedAt);
+    const oneMonthAgo = new Date(today);
+    oneMonthAgo.setMonth(oneMonthAgo.getMonth() - 1);
+    const uploadedDate2 = new Date(uploaded);
+    if (
+      downloadedAtDate <= oneMonthAgo &&
+      uploadedDate2 <= oneMonthAgo
+    ) {
+      removedItems.push({
+        key: object.key,
+        uploaded: new Date(object.uploaded),
+        downloadedAt: new Date(downloadedAt),
+      });
+      if (remove) {
+        await binding.delete(object.key);
+        await downloadedAtBucket.removeItem(object.key);
       }
-    });
-
-    truncated = next.truncated;
-    if (next.truncated) {
-      cursor = next.cursor;
     }
+  });
+
+  truncated = next.truncated;
+  if (next.truncated) {
+    cursor = next.cursor;
   }
+
   return {
     cursor,
     truncated,

script/remove-stale.js

@@ -34,17 +34,9 @@ async function processBucket(bucket) {
 
       const json = await res.json();
 
-      console.log(`[${bucket}] Batch ${batch} - Removed items:`, json.result.removedItems.length);
-      if (json.result.removedItems.length > 0) {
-        for (const item of json.result.removedItems) {
-          console.log(`  -`, item);
-        }
-      }
-      console.log('just fetched', json.result.removedItems.length)
-      cursor = json.result.cursor;
-      truncated = json.result.truncated;
-      console.log('cursor', cursor)
-      console.log('truncated', truncated)
+      console.log(`[${bucket}] Batch ${batch} - Removed items:`, json.removedItems.length);
+      cursor = json.cursor;
+      truncated = json.truncated;
       batch++;
       if (!truncated) {
         console.log(`[${bucket}] Completed. Total batches: ${batch}`);