Skip to content

Commit 1b34350

Browse files
aster-voidclaude
andcommitted
modules/admin: enhance migration with URL processing and cleanup tools
- Add content image URL replacement during migration - Upload ./image.webp references to S3 and replace in content - Process members pageContent, articles content, projects content - Add cleanup invalid URLs feature (nullify non-S3 URLs) - Add delete all migrated data feature for clean re-migration - Copy no-image.svg fallback from utcode.net 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <[email protected]>
1 parent a8e613c commit 1b34350

File tree

7 files changed

+372
-42
lines changed

7 files changed

+372
-42
lines changed

secrets.dev.yaml

Lines changed: 20 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,30 @@
11
DATABASE_URL: ENC[AES256_GCM,data:DT1kdctK4pK5GF/Ysn3ULxVOK3Vu+7cb7sA=,iv:rqxTwZ31IwSJ4IcrKxdiTQLP7KNAmkf8NBTU5sOZpA0=,tag:KqrGB6S1dgKniSbA+z5pmA==,type:str]
22
BETTER_AUTH_URL: ENC[AES256_GCM,data:HTm98h7WLoXX92kkFAB3SF8DQE3s,iv:/yFx+YaJHkGi5+mMIDc2dW7bWXEPrLfl4udJAbfx75c=,tag:GxJutrvSDayFdIam7DBWsA==,type:str]
33
BETTER_AUTH_SECRET: ENC[AES256_GCM,data:iWFfgHGgQhqxHQPS0MrhOQuCaHFi5Y474Ret+WMkjoeVjo5wGQzGmQaaRcE=,iv:CWOE30RroxpvX9Mm9D/1M9jFhEaILwAqAVxcNhpxGKc=,tag:UXWoHgK39mrc+x9LMSCYiA==,type:str]
4-
GITHUB_CLIENT_ID: ""
5-
GITHUB_CLIENT_SECRET: ""
64
S3_ENDPOINT: ENC[AES256_GCM,data:8DKqTQi0rLcxlbtQZOYaQOtstRlv,iv:ipAyykxrhdicmfT3l1oI4MSjIN6lVs5EiPVKrke8cJQ=,tag:SCRhvHIkQKJdz0UzhJ6rjQ==,type:str]
75
S3_ACCESS_KEY: ENC[AES256_GCM,data:iZkREoP5BhiHvw==,iv:Urs+Ux5JDVsfJleuFAKrFxHF9Xg3/m7uEph0kln5Hv0=,tag:l5tXx04AVGfdAiRJJHb3nw==,type:str]
86
S3_SECRET_KEY: ENC[AES256_GCM,data:53MOStu3RyL3CA==,iv:5nzXZ3BV5gYgTj0Kt4YJEDhidLXRqQiq4YyCe4fiXCo=,tag:RkJ+WKqylheG5azXkhRlnw==,type:str]
97
S3_BUCKET: ENC[AES256_GCM,data:NOun,iv:UhkU0jiq8pee9jboSLDZT3XK+fG9NrLozhWaEAKxqRM=,tag:omC3c8OYgS2KbbpwIwMWvA==,type:str]
108
S3_PUBLIC_URL: ENC[AES256_GCM,data:o/JRmwxFNqjrMHZJ/eoynCIMcjNIC7G3nw==,iv:8Ut2k/uuhobj7i37J3P3Vb/AUhFLofSoWB6X+Y3g39o=,tag:j1XOf1meTwI0ZoqoYeEvjg==,type:str]
9+
#ENC[AES256_GCM,data:cMR8p8WSoYZev2lnoiZQN08G,iv:3YUTmZWlyQp8ZCdGKOdscsWMizXB/lvcI6igKhUYrtE=,tag:dXYOd1JAk5TOYwqu9t/8fA==,type:comment]
1110
UNSAFE_DISABLE_AUTH: ENC[AES256_GCM,data:qQfxNQ==,iv:j250CMhk9ZPKEXrVjh+Ls99gEpqhAzQSdCtBhbyDRkA=,tag:9R7/YtAI+VgJUKAPcTHpQg==,type:str]
11+
#ENC[AES256_GCM,data:Wa3MOB+PLFQu914=,iv:ehB7lgR6bMcLsORKQJ3na5G88UDccNx7FMur6mRSlpM=,tag:3wzRIF6tJAPaS9aZMfyJJA==,type:comment]
12+
CLOUDFLARE_ZONE_ID: ""
13+
CLOUDFLARE_API_TOKEN: ""
14+
GITHUB_CLIENT_ID: ""
15+
GITHUB_CLIENT_SECRET: ""
1216
sops:
13-
age:
14-
- recipient: age1gsn5pg5pkk89zsukc94gqwlu7zyrep0prm7pwc787tcegapgmvgqzeq64c
15-
enc: |
16-
-----BEGIN AGE ENCRYPTED FILE-----
17-
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBaZ0NjbVpFbVJ5TS8xSlNW
18-
a0hmOXRrQ2xpYkVJWHhVRm9lODRTUldxeTE0CnRGemp4dEZKV29JbjhPL2RVa3lG
19-
WGNTdE9RYmZ0L1BQRjQyQ1dwU0x3dlkKLS0tIFc4NXE4ZFRhYmFtWlR2M3lpWVV1
20-
Y2M0Q3NYeFBVTjZKOFNIS0x6dnRPUEEKCt8q9U5HaLS0Mm2JNmODlECAYjN8nn+1
21-
zlbwhcAUqbhFd8HteSbp8jTgp14PUuig/c2c3JDOmgQfiITZaxLayQ==
22-
-----END AGE ENCRYPTED FILE-----
23-
lastmodified: "2025-12-23T06:23:33Z"
24-
mac: ENC[AES256_GCM,data:lyuVzf2Zm8UTIMQNFCeg1BIKR9pfHOorS08alTMKJRHwdqKsziDu9lm86/6iOmUFpyasoG1J2sZN95ltq0IPOooV9sm28QvlNon8HBaNGkgvVRfCeU9vuKQrcp7OOSGI2Vqe38ZIveyBbhTPtqjoLuJfnXkoFiV2s1kPgySxFLM=,iv:OsDm31BPcCveHRqPpn8IpO3GvwKJ/DfCx/eYbWHGceY=,tag:xvVJ2K6xE12HykR8CIpcGA==,type:str]
25-
unencrypted_suffix: _unencrypted
26-
version: 3.11.0
17+
age:
18+
- recipient: age1gsn5pg5pkk89zsukc94gqwlu7zyrep0prm7pwc787tcegapgmvgqzeq64c
19+
enc: |
20+
-----BEGIN AGE ENCRYPTED FILE-----
21+
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSBaZ0NjbVpFbVJ5TS8xSlNW
22+
a0hmOXRrQ2xpYkVJWHhVRm9lODRTUldxeTE0CnRGemp4dEZKV29JbjhPL2RVa3lG
23+
WGNTdE9RYmZ0L1BQRjQyQ1dwU0x3dlkKLS0tIFc4NXE4ZFRhYmFtWlR2M3lpWVV1
24+
Y2M0Q3NYeFBVTjZKOFNIS0x6dnRPUEEKCt8q9U5HaLS0Mm2JNmODlECAYjN8nn+1
25+
zlbwhcAUqbhFd8HteSbp8jTgp14PUuig/c2c3JDOmgQfiITZaxLayQ==
26+
-----END AGE ENCRYPTED FILE-----
27+
lastmodified: "2025-12-23T09:13:19Z"
28+
mac: ENC[AES256_GCM,data:bvImv8g/K8wHa07RKBEPtTInOAdvNaJjTMt2A5EWBjPnoroHazfz9rrL2KMuBYJxZyXd3eHYb+L+MRZC154qcahU83KXeYZazReP99jRC0iPNIGeqCYaCeJ1vRfiM8kzPluScGavEbE272jfigjfXps2+duWPgk49gUjNvMAJQU=,iv:Fq+fokmZibCDLuSFuNT5XwNT4KYRbkw0yKcGzYashOE=,tag:ceu8QgNoxivwFX2IDFnnNw==,type:str]
29+
unencrypted_suffix: _unencrypted
30+
version: 3.11.0

secrets.prod.yaml

Lines changed: 16 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,20 @@ S3_ACCESS_KEY: ENC[AES256_GCM,data:JH70m0+9+KlfjXlDzVqJGYLFy8xRN3fxIV0YwmB9aRE=,
88
S3_SECRET_KEY: ENC[AES256_GCM,data:QEc9T5yC+iZoV403CqqBTJCsDCdzH2FV0vQoRlEU6vM0oK5PrHiTOXUwe4WMSFCiz3ioBvgQOKHoan963U0gZA==,iv:0qZE002nF25N91pvITA65qBSyi2mDzRXNP/2pcBnI0o=,tag:gFQU5CJDr0FDWiWcFl05Ng==,type:str]
99
S3_BUCKET: ENC[AES256_GCM,data:kKVmATCe2JYggI5bXjch9kiCbqVb,iv:WBVmx8TWKbJ3xWHiGy3/Os4xlPfcuDMz3xoWnzsTato=,tag:BqnD6HXBC4A9qqeREcB6Sw==,type:str]
1010
S3_PUBLIC_URL: ENC[AES256_GCM,data:yF9V11xmPsO9fAyozCVUWwC+z16+guLMNA==,iv:n8mEi0E8bvxK68hHZ6yd8dBlfi10sETNNXQBMt1ygrw=,tag:7ZK8/MM2xVTi8N65q0+v5g==,type:str]
11+
CLOUDFLARE_ZONE_ID: ENC[AES256_GCM,data:1f/0GcTM0bAiR/OevKD2W7GX9uENyyPpGiIQD8vvVRQ=,iv:352ETtiKjNyXTmxBlzD3w5WImmsO/pcW0ihsph1vHow=,tag:92DCIGeA8HK9hVJrmBm0Qw==,type:str]
12+
CLOUDFLARE_API_TOKEN: ENC[AES256_GCM,data:G1mLIuB0vbB6r2arvlxwy5bMmRRYi4p+EQC1JqUYz9YLGIctkI2I5Q==,iv:9veja4XIz0XzX2QD5CDIGHUmuxXMDnDxW0m6l6zr74c=,tag:ibsmoZDansTHzsNidu5T0A==,type:str]
1113
sops:
12-
age:
13-
- recipient: age1gsn5pg5pkk89zsukc94gqwlu7zyrep0prm7pwc787tcegapgmvgqzeq64c
14-
enc: |
15-
-----BEGIN AGE ENCRYPTED FILE-----
16-
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwRWovR2lYZjk5UzQ0Y2Ri
17-
QXAxam5LaEY2S3hFREVEUHJzdDNQbCs2T3hnCkprMDNlamZPQmhvTXh1SUdOUHNX
18-
OEhTYWlLMVdsaVRuUDdPRVZvRmc3VkEKLS0tIGlOK1JScmJtQVJPWXErZkc2VVNR
19-
NHZSdEp1WGc4S2ppK1ZKcDZnU0JRQmMKJj4n6DBAozt2DTlYFIhd1jWWLGMYalAM
20-
+Txthl+D2hQNSITfK7jzVXIhk4z3g2ahNXscRmgrU9u4V0Jj6Gzz/g==
21-
-----END AGE ENCRYPTED FILE-----
22-
lastmodified: "2025-12-23T06:48:31Z"
23-
mac: ENC[AES256_GCM,data:51C/AiU7T7aqHxqvhl/+sc82nigPHlKzLDXqXDyfR5PGfNKsnP7/u/X3xsp6VhNv+itnRCp/29rG3qjKnbjm04uS28bIRmSzaUrDK6/b3ASnJIexhsuAeZFsjqPaJ0IpECdrmn7D2ZXrJQw7qx1LpKlFsHBK8awlE1JDcmdAFXY=,iv:hF63GDjwaKZ5t+/oN5heoFSLHuIJ9tgvqyHNoaDxmjA=,tag:L9gvHacn2UAi693OkVydGA==,type:str]
24-
unencrypted_suffix: _unencrypted
25-
version: 3.11.0
14+
age:
15+
- recipient: age1gsn5pg5pkk89zsukc94gqwlu7zyrep0prm7pwc787tcegapgmvgqzeq64c
16+
enc: |
17+
-----BEGIN AGE ENCRYPTED FILE-----
18+
YWdlLWVuY3J5cHRpb24ub3JnL3YxCi0+IFgyNTUxOSAwRWovR2lYZjk5UzQ0Y2Ri
19+
QXAxam5LaEY2S3hFREVEUHJzdDNQbCs2T3hnCkprMDNlamZPQmhvTXh1SUdOUHNX
20+
OEhTYWlLMVdsaVRuUDdPRVZvRmc3VkEKLS0tIGlOK1JScmJtQVJPWXErZkc2VVNR
21+
NHZSdEp1WGc4S2ppK1ZKcDZnU0JRQmMKJj4n6DBAozt2DTlYFIhd1jWWLGMYalAM
22+
+Txthl+D2hQNSITfK7jzVXIhk4z3g2ahNXscRmgrU9u4V0Jj6Gzz/g==
23+
-----END AGE ENCRYPTED FILE-----
24+
lastmodified: "2025-12-23T09:16:16Z"
25+
mac: ENC[AES256_GCM,data:jFCDcGCZA8hzIwZy5TibMg/PMB6UAobWpxdJ7XlzfMcvCWWV5Azf0wGYWLA0jWdZ2nOB2sdCODGfWKzBHuyceUSbBNFC+yHlyEnsZZyudzfQZ1v4sSRoalYuzhK+r/IWi/BYMNu2HkZ2nkUF1ma/FWirb93w30MFvUePZlX06N4=,iv:yaVZRVOMEMe55tgl/QzQPRjjzz2lrZyB1obclrZVIgE=,tag:L13UTthVJshWD1EcJP18LQ==,type:str]
26+
unencrypted_suffix: _unencrypted
27+
version: 3.11.0

src/lib/data/private/migration.remote.ts

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
import { command, query } from "$app/server";
22
import { requireUtCodeMember } from "$lib/server/database/auth.server";
3-
import { startDataMigration } from "$lib/server/services/migration/index.server";
3+
import {
4+
startDataMigration,
5+
startDeleteAll,
6+
startImageCleanup,
7+
} from "$lib/server/services/migration/index.server";
48
import { getMigrationState, resetMigration } from "$lib/server/services/migration/state.server";
59
import type { MigrationState } from "$lib/shared/types/migration";
610

@@ -18,3 +22,13 @@ export const reset = command(async (): Promise<void> => {
1822
await requireUtCodeMember();
1923
resetMigration();
2024
});
25+
26+
export const cleanup = command(async (): Promise<{ started: boolean; message: string }> => {
27+
await requireUtCodeMember();
28+
return startImageCleanup();
29+
});
30+
31+
export const deleteAll = command(async (): Promise<{ started: boolean; message: string }> => {
32+
await requireUtCodeMember();
33+
return startDeleteAll();
34+
});

src/lib/server/services/migration/index.server.ts

Lines changed: 70 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,14 @@ import { rm } from "node:fs/promises";
1010
import { tmpdir } from "node:os";
1111
import { join } from "node:path";
1212
import { completeMigration, failMigration, isRunning, log, startMigration } from "./state.server";
13-
import { migrateArticles, migrateImages, migrateMembers, migrateProjects } from "./workers.server";
13+
import {
14+
cleanupInvalidImageUrls,
15+
deleteAllMigratedData,
16+
migrateArticles,
17+
migrateImages,
18+
migrateMembers,
19+
migrateProjects,
20+
} from "./workers.server";
1421

1522
const REPO_URL = "https://github.com/ut-code/utcode.net.git";
1623

@@ -51,6 +58,68 @@ export function startDataMigration(): { started: boolean; message: string } {
5158
return { started: true, message: "Migration started" };
5259
}
5360

61+
export function startImageCleanup(): { started: boolean; message: string } {
62+
if (isRunning()) {
63+
return { started: false, message: "Migration already in progress" };
64+
}
65+
66+
startMigration();
67+
log("=== Image URL Cleanup Started ===");
68+
69+
runCleanupAsync().catch(console.error);
70+
71+
return { started: true, message: "Cleanup started" };
72+
}
73+
74+
async function runCleanupAsync(): Promise<void> {
75+
try {
76+
const result = await cleanupInvalidImageUrls(log);
77+
78+
log("=== Cleanup Complete ===");
79+
completeMigration({
80+
members: { created: result.members.cleaned, skipped: result.members.skipped, errors: 0 },
81+
articles: { created: result.articles.cleaned, skipped: result.articles.skipped, errors: 0 },
82+
projects: { created: result.projects.cleaned, skipped: result.projects.skipped, errors: 0 },
83+
images: { created: 0, skipped: 0, errors: 0 },
84+
});
85+
} catch (e) {
86+
const msg = e instanceof Error ? e.message : String(e);
87+
log(`=== Cleanup Failed: ${msg} ===`);
88+
failMigration(msg);
89+
}
90+
}
91+
92+
export function startDeleteAll(): { started: boolean; message: string } {
93+
if (isRunning()) {
94+
return { started: false, message: "Migration already in progress" };
95+
}
96+
97+
startMigration();
98+
log("=== Delete All Data Started ===");
99+
100+
runDeleteAllAsync().catch(console.error);
101+
102+
return { started: true, message: "Delete started" };
103+
}
104+
105+
async function runDeleteAllAsync(): Promise<void> {
106+
try {
107+
const result = await deleteAllMigratedData(log);
108+
109+
log("=== Delete Complete ===");
110+
completeMigration({
111+
members: { created: result.members.deleted, skipped: 0, errors: 0 },
112+
articles: { created: result.articles.deleted, skipped: 0, errors: 0 },
113+
projects: { created: result.projects.deleted, skipped: 0, errors: 0 },
114+
images: { created: 0, skipped: 0, errors: 0 },
115+
});
116+
} catch (e) {
117+
const msg = e instanceof Error ? e.message : String(e);
118+
log(`=== Delete Failed: ${msg} ===`);
119+
failMigration(msg);
120+
}
121+
}
122+
54123
async function runMigrationAsync(): Promise<void> {
55124
let repoPath: string | null = null;
56125

0 commit comments

Comments
 (0)