Skip to content

Commit 3e5daeb

Browse files
AslemammadAslemammad
authored
debug: rm stale (#374)
1 parent 4222e36 commit 3e5daeb

File tree

3 files changed

+130
-81
lines changed

3 files changed

+130
-81
lines changed

‎.github/workflows/remove-stale.js‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
const endpoint = process.env.STALE_ENDPOINT;
2+
const staleKey = process.env.STALE_KEY;
3+
const remove = process.env.STALE_REMOVE === 'true';
4+
5+
if (!endpoint || !staleKey) {
6+
console.error('STALE_ENDPOINT and STALE_KEY environment variables are required.');
7+
process.exit(1);
8+
}
9+
10+
async function processBucket(bucket) {
11+
let cursor = null;
12+
let batch = 0;
13+
let truncated = true;
14+
while (truncated) {
15+
const body = {
16+
bucket,
17+
cursor,
18+
remove,
19+
};
20+
try {
21+
const res = await fetch(endpoint, {
22+
method: 'POST',
23+
headers: {
24+
'Content-Type': 'application/json',
25+
'sb-rm-stale-key': staleKey,
26+
},
27+
body: JSON.stringify(body),
28+
});
29+
const text = await res.text();
30+
let json;
31+
try {
32+
json = JSON.parse(text);
33+
} catch (e) {
34+
console.error(`[${bucket}] Batch ${batch} - Failed to parse response:`, text);
35+
process.exit(1);
36+
}
37+
if (!res.ok) {
38+
console.error(`[${bucket}] Batch ${batch} - Request failed:`, json);
39+
process.exit(1);
40+
}
41+
console.log(`[${bucket}] Batch ${batch} - Removed items:`, json.result.removedItems.length);
42+
if (json.result.removedItems.length > 0) {
43+
for (const item of json.result.removedItems) {
44+
console.log(` -`, item);
45+
}
46+
}
47+
cursor = json.result.cursor;
48+
truncated = json.result.truncated;
49+
batch++;
50+
if (!truncated) {
51+
console.log(`[${bucket}] Completed. Total batches: ${batch}`);
52+
}
53+
} catch (e) {
54+
console.error(`[${bucket}] Batch ${batch} - Request failed:`, e);
55+
process.exit(1);
56+
}
57+
}
58+
}
59+
60+
(async () => {
61+
for (const bucket of ['packages', 'templates']) {
62+
console.log(`Processing bucket: ${bucket}`);
63+
await processBucket(bucket);
64+
}
65+
})();

‎.github/workflows/remove-stale.yml‎

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,24 @@
11
on:
22
workflow_dispatch:
3+
inputs:
4+
remove:
5+
description: 'Actually delete items (true/false)'
6+
required: false
7+
default: 'false'
38

49
jobs:
510
rm_stale_packages:
611
runs-on: ubuntu-latest
712
steps:
8-
- name: Send POST request and capture output
9-
run: |
10-
curl -X POST \
11-
-N \
12-
-H "sb-rm-stale-key: ${{ secrets.NITRO_RM_STALE_KEY }}" \
13-
https://pkg.pr.new/rm/stale \
14-
| tee rm-stale.log
15-
16-
- name: Upload rm-stale.log as artifact
17-
uses: actions/upload-artifact@v4
13+
- name: Set up Node.js
14+
uses: actions/setup-node@v4
1815
with:
19-
name: rm-stale-log
20-
path: rm-stale.log
16+
node-version: 'lts/*'
17+
18+
- name: Run remove-stale.js
19+
run: |
20+
node .github/workflows/remove-stale.js
21+
env:
22+
STALE_ENDPOINT: "https://pkg.pr.new/rm/stale"
23+
STALE_KEY: ${{ secrets.NITRO_RM_STALE_KEY }}
24+
STALE_REMOVE: "${{ github.event.inputs.remove }}"

‎packages/app/server/routes/rm/stale.post.ts‎

Lines changed: 49 additions & 69 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,8 @@
11
import type { H3Event } from "h3";
22

33
export default eventHandler(async (event) => {
4-
setResponseHeader(event, "Transfer-Encoding", "chunked");
54
setResponseHeader(event, "Cache-Control", "no-cache");
6-
setResponseHeader(event, "Content-Type", "text/plain");
5+
setResponseHeader(event, "Content-Type", "application/json");
76

87
const rmStaleKeyHeader = getHeader(event, "sb-rm-stale-key");
98
const signal = toWebRequest(event).signal;
@@ -15,49 +14,31 @@ export default eventHandler(async (event) => {
1514
});
1615
}
1716

18-
const { readable, writable } = new TransformStream();
17+
const { bucket, cursor, remove } = await readBody<{ bucket: 'packages' | 'templates'; cursor: string | null; remove: boolean }>(event);
1918

20-
event.waitUntil(
21-
(async () => {
22-
// const writer = writable.getWriter()
23-
// console.log('here')
24-
// await writer.ready
25-
// await writer.write(new TextEncoder().encode("start\n"))
26-
// writer.releaseLock()
19+
const result = await iterateAndDelete(event, signal, {
20+
prefix: bucket === 'packages' ? usePackagesBucket.base : useTemplatesBucket.base,
21+
limit: 100,
22+
cursor: cursor || undefined,
23+
}, remove);
2724

28-
await iterateAndDelete(event, writable, signal, {
29-
prefix: usePackagesBucket.base,
30-
limit: 100,
31-
});
32-
await iterateAndDelete(event, writable, signal, {
33-
prefix: useTemplatesBucket.base,
34-
limit: 100,
35-
});
36-
await writable.close();
37-
})(),
38-
);
39-
40-
return readable;
25+
return {
26+
result,
27+
};
4128
});
4229

43-
async function iterateAndDelete(
44-
event: H3Event,
45-
writable: WritableStream,
46-
signal: AbortSignal,
47-
opts: R2ListOptions,
48-
) {
49-
const writer = writable.getWriter();
50-
await writer.ready;
30+
async function iterateAndDelete(event: H3Event, signal: AbortSignal, opts: R2ListOptions, remove: boolean) {
5131
const binding = useBinding(event);
52-
5332
let truncated = true;
5433
let cursor: string | undefined;
55-
34+
const removedItems: Array<{ key: string; uploaded: Date; downloadedAt?: Date }> = [];
5635
const downloadedAtBucket = useDownloadedAtBucket(event);
5736
const today = Date.parse(new Date().toString());
5837

5938
while (truncated && !signal.aborted) {
60-
// TODO: Avoid using context.cloudflare and migrate to unstorage, but it does not have truncated for now
39+
if (removedItems.length >= 1000) {
40+
break
41+
}
6142
const next = await binding.list({
6243
...opts,
6344
cursor,
@@ -67,59 +48,58 @@ async function iterateAndDelete(
6748
break;
6849
}
6950
const uploaded = Date.parse(object.uploaded.toString());
70-
// remove the object anyway if it's 6 months old already
71-
// Use calendar-accurate 6 months check
51+
// removedItems.push({
52+
// key: object.key,
53+
// uploaded: new Date(object.uploaded),
54+
// });
7255
const uploadedDate = new Date(uploaded);
7356
const sixMonthsAgo = new Date(today);
7457
sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6);
7558
if (uploadedDate <= sixMonthsAgo) {
76-
const downloadedAt = (await downloadedAtBucket.getItem(object.key))!;
77-
await writer.write(
78-
new TextEncoder().encode(
79-
JSON.stringify({
80-
key: object.key,
81-
uploaded: new Date(object.uploaded),
82-
downloadedAt: downloadedAt ? new Date(downloadedAt) : null,
83-
}) + "\n",
84-
),
85-
);
86-
// event.context.cloudflare.context.waitUntil(binding.delete(object.key));
87-
// event.context.cloudflare.context.waitUntil(
88-
// downloadedAtBucket.removeItem(object.key),
89-
// );
59+
removedItems.push({
60+
key: object.key,
61+
uploaded: new Date(object.uploaded),
62+
});
63+
if (remove) {
64+
await binding.delete(object.key);
65+
await downloadedAtBucket.removeItem(object.key);
66+
}
67+
continue;
9068
}
9169
const downloadedAt = await downloadedAtBucket.getItem(object.key);
9270

9371
if (!downloadedAt) {
9472
continue;
9573
}
96-
// if it has not been downloaded in the last month and it's at least 1 month old
97-
// Calendar-accurate 1 month checks
9874
const downloadedAtDate = new Date(downloadedAt);
9975
const oneMonthAgo = new Date(today);
10076
oneMonthAgo.setMonth(oneMonthAgo.getMonth() - 1);
101-
const uploadedDate2 = new Date(uploaded); // uploaded already parsed above
102-
if (downloadedAtDate <= oneMonthAgo && uploadedDate2 <= oneMonthAgo) {
103-
await writer.write(
104-
new TextEncoder().encode(
105-
JSON.stringify({
106-
key: object.key,
107-
uploaded: new Date(object.uploaded),
108-
downloadedAt: new Date(downloadedAt),
109-
}) + "\n",
110-
),
111-
);
112-
// event.context.cloudflare.context.waitUntil(binding.delete(object.key));
113-
// event.context.cloudflare.context.waitUntil(
114-
// downloadedAtBucket.removeItem(object.key),
115-
// );
77+
const uploadedDate2 = new Date(uploaded);
78+
if (
79+
downloadedAtDate <= oneMonthAgo &&
80+
uploadedDate2 <= oneMonthAgo
81+
) {
82+
removedItems.push({
83+
key: object.key,
84+
uploaded: new Date(object.uploaded),
85+
downloadedAt: new Date(downloadedAt),
86+
});
87+
if (remove) {
88+
await binding.delete(object.key);
89+
await downloadedAtBucket.removeItem(object.key);
90+
}
91+
11692
}
11793
}
118-
11994
truncated = next.truncated;
12095
if (next.truncated) {
12196
cursor = next.cursor;
12297
}
12398
}
124-
writer.releaseLock();
99+
return {
100+
cursor,
101+
truncated,
102+
removedItems,
103+
};
125104
}
105+

0 commit comments

Comments
 (0)