diff --git a/.eslintrc.cjs b/.eslintrc.cjs new file mode 100644 index 0000000..4796612 --- /dev/null +++ b/.eslintrc.cjs @@ -0,0 +1,23 @@ +module.exports = { + root: true, + env: { + node: true, + es2022: true, + }, + extends: ["eslint:recommended", "prettier"], + parserOptions: { + ecmaVersion: "latest", + }, + ignorePatterns: ["node_modules/"], + overrides: [ + { + files: ["web/**/*.js"], + env: { + browser: true, + }, + }, + ], + rules: { + "no-console": "off", + }, +}; diff --git a/.gitignore b/.gitignore index 62aecbc..d9d7ba2 100644 --- a/.gitignore +++ b/.gitignore @@ -134,3 +134,6 @@ flomo sendedIds.json memo.json + +AGENTS.md +.DS_Store diff --git a/README.md b/README.md index ec1041b..98ea210 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,42 @@ 最好使用 memo@0.10.4 以上版本，因为之前的版本不支持设置创建时间的功能。 +# Web 控制台（可视化） + +如果你不想手动执行命令行，可以使用内置 Web 控制台完成导入和删除。 + +## 启动 + +```bash +pnpm install +pnpm web +``` + +启动后访问： + +```text +http://127.0.0.1:3131 +``` + +## 页面操作步骤 + +1. 在 `Session` 区域填写： + - `API Host`（例如 `http://localhost:5230/api/v1` 或 `http://localhost:5230/api/v2`） + - `Access Token` + 点击 `Save Session`。 +2. 选择任务并执行： + - Flomo：优先在页面直接选择 flomo 导出文件夹（包含 `index.html` 和 `file/` 资源目录），然后点击 `执行 Flomo 导入` + - 微信读书：优先在页面直接选择 TXT 文件，然后点击 `执行微信读书导入` + - 如需兼容旧流程，可在“高级（可选）”里填写本地路径（例如 `./flomo/index.html`、`./weixin.txt`） + - 删除已导入数据：点击 `删除已导入 Memo` +3. 在 `Live Logs` 观察实时日志，在 `Status` 查看任务状态。 + +## 说明 + +- 当前仅支持单任务串行执行，运行中再次启动任务会返回冲突提示。 +- 会话凭据只保存在服务进程内存中，页面刷新或服务重启后需要重新填写。 +- 删除逻辑依赖 `sendedIds.json`，默认存放在系统临时目录 `memos-import-artifacts`（可通过 `MEMOS_ARTIFACT_DIR` 覆盖）。 + # flomo ## 导出 flomo 数据 @@ -48,7 +84,7 @@ node ./src/main.js ./flomo/index.html > 删除同步数据仅支持删除脚本创建的内容，创建的 tag 请手动删除，因为无法确认 tag 是否有被其他内容使用。 -执行完同步数据后如果不符合预期，可以执行下面的命令删除同步的数据。删除会读取同步完成写入到 `sendedIds.json` 文件数据，所以需要保证这个文件存在。 +执行完同步数据后如果不符合预期，可以执行下面的命令删除同步的数据。删除会读取同步完成写入的 `sendedIds.json`，默认路径为系统临时目录 `memos-import-artifacts`（可通过 `MEMOS_ARTIFACT_DIR` 覆盖）。 ```bash node ./src/delete.js diff --git a/package.json b/package.json index cea2859..031ee40 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,10 @@ "description": "", "main": "index.js", "scripts": { - "run": "node ./src/weixin.js http://localhost:5230/api/memo?openId=8deb3079-055b-48a1-a1e4-499ce493fa0c ./weixin.txt", - "lint": "eslint ." + "run": "node ./src/weixin.js http://localhost:5230/api/v1 YOUR_ACCESS_TOKEN ./weixin.txt", + "lint": "ESLINT_USE_FLAT_CONFIG=false eslint .", + "test": "node --test tests/*.test.js", + "web": "node ./src/server/index.js" }, "keywords": [], "author": "", @@ -18,7 +20,8 @@ "fs-extra": "^11.1.0", "mime": "^3.0.0", "turndown": "^7.1.2", - "url-parse": "^1.5.10" + "url-parse": "^1.5.10", + "busboy": "^1.6.0" }, "devDependencies": { "@babel/core": "^7.20.12", @@ -26,4 +29,4 @@ "eslint": "^8.34.0", "eslint-config-prettier": "^8.6.0" } -} \ No newline at end of file +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b550dee..a9f9601 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,6 +11,9 @@ importers: axios: specifier: ^1.6.0 version: 1.7.9 + busboy: + specifier: ^1.6.0 + version: 1.6.0 chalk: specifier: ^4.1.2 version: 4.1.2 @@ -239,6 +242,10 @@ packages: engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} hasBin: true + busboy@1.6.0: + resolution: {integrity: sha512-8SFQbg/0hQ9xy3UNTB0YEnsNBbWfhf7RtnzpL7TkBiTBRfrQ9Fxcnz7VJsleJpyp6rVLvXiuORqjlHi5q+PYuA==} + engines: {node: '>=10.16.0'} + callsites@3.1.0: resolution: {integrity: sha512-P8BjAsXvZS+VIDUI11hHCQEv74YT67YUi5JJFNWIqL235sBmjX4+qx9Muvls5ivyNENctx46xQLQ3aTuE7ssaQ==} engines: {node: '>=6'} @@ -661,6 +668,10 @@ packages: resolution: {integrity: sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==} engines: {node: '>=8'} + streamsearch@1.1.0: + resolution: {integrity: sha512-Mcc5wHehp9aXz1ax6bZUyY5afg9u2rv5cqQI3mRrYkGC8rW2hM02jWuwjtL++LS5qinSyhj2QfLyNsuc+VsExg==} + engines: {node: '>=10.0.0'} + strip-ansi@6.0.1: resolution: {integrity: sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==} engines: {node: '>=8'} @@ -957,6 +968,10 @@ snapshots: node-releases: 2.0.19 update-browserslist-db: 1.1.2(browserslist@4.24.4) + busboy@1.6.0: + dependencies: + streamsearch: 1.1.0 + callsites@3.1.0: {} caniuse-lite@1.0.30001699: {} @@ -1374,6 +1389,8 @@ snapshots: shebang-regex@3.0.0: {} + streamsearch@1.1.0: {} + strip-ansi@6.0.1: dependencies: ansi-regex: 5.0.1 diff --git a/src/core/apiClient.js b/src/core/apiClient.js new file mode 100644 index 0000000..4a67f36 --- /dev/null +++ b/src/core/apiClient.js @@ -0,0 +1,118 @@ +const fs = require("fs-extra"); +const path = require("path"); +const mime = require("mime"); +const axios = require("axios"); +const parse = require("url-parse"); + +const DEFAULT_SLEEP_MS = 1000; + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function getVersion(openApi) { + if ((openApi || "").includes("/v2")) return "/v2"; + return "/v1"; +} + +function getRequestUrl(openApi, requestPath) { + const { origin } = parse(openApi || ""); + return `${origin}${requestPath}`; +} + +function extractMemoName(responseData) { + return responseData?.data?.name || responseData?.data?.data?.name; +} + +function createApiClient(options) { + const { openApi, accessToken, sleepMs = DEFAULT_SLEEP_MS } = options; + + if (!openApi) { + throw new Error("openApi is required"); + } + if (!accessToken) { + throw new Error("accessToken is required"); + } + + const version = getVersion(openApi); + const headers = { + Authorization: `Bearer ${accessToken}`, + }; + + return { + async uploadFile(filePath) { + const readFile = fs.readFileSync(filePath); + const response = await axios({ + method: "post", + url: getRequestUrl(openApi, `/api${version}/resources`), + data: { + content: readFile.toString("base64"), + filename: path.basename(filePath), + type: mime.getType(filePath) || undefined, + }, + headers, + }); + + return response.data; + }, + + async sendMemo(memo) { + const response = await axios({ + method: "post", + url: getRequestUrl(openApi, `/api${version}/memos`), + data: memo, + headers: { + ...headers, + "Content-Type": "application/json; charset=UTF-8", + }, + }); + + await sleep(sleepMs); + return response; + }, + + async updateMemo(memoName, createTime) { + return axios({ + method: "patch", + url: getRequestUrl(openApi, `/api${version}/${memoName}`), + data: { createTime }, + headers: { + ...headers, + "Content-Type": "application/json; charset=UTF-8", + }, + }); + }, + + async setMemoResources(memoName, resources) { + return axios({ + method: "patch", + url: getRequestUrl(openApi, `/api${version}/${memoName}/resources`), + data: { + resources, + }, + headers: { + ...headers, + "Content-Type": "application/json; charset=UTF-8", + }, + }); + }, + + async deleteMemo(memoName) { + return axios({ + method: "delete", + url: getRequestUrl(openApi, `/api${version}/${memoName}`), + headers: { + ...headers, + "Content-Type": "application/json; charset=UTF-8", + }, + }); + }, + + extractMemoName, + }; +} + +module.exports = { + createApiClient, + extractMemoName, +}; diff --git a/src/core/artifactStore.js b/src/core/artifactStore.js new file mode 100644 index 0000000..2e03b66 --- /dev/null +++ b/src/core/artifactStore.js @@ -0,0 +1,18 @@ +const os = require("os"); +const path = require("path"); +const fs = require("fs-extra"); + +const ARTIFACT_DIR_ENV = "MEMOS_ARTIFACT_DIR"; + +function getDefaultArtifactDir() { + const fromEnv = process.env[ARTIFACT_DIR_ENV]; + const dir = fromEnv ? path.resolve(fromEnv) : path.join(os.tmpdir(), "memos-import-artifacts"); + fs.ensureDirSync(dir); + return dir; +} + +module.exports = { + ARTIFACT_DIR_ENV, + getDefaultArtifactDir, +}; + diff --git a/src/core/deleteImported.js b/src/core/deleteImported.js new file mode 100644 index 0000000..54e4d5e --- /dev/null +++ b/src/core/deleteImported.js @@ -0,0 +1,42 @@ +const path = require("path"); +const fs = require("fs-extra"); +const { createApiClient } = require("./apiClient"); +const { getDefaultArtifactDir } = require("./artifactStore"); + +function noop() {} + +async function deleteImported(options) { + const { openApi, accessToken, artifactDir = getDefaultArtifactDir(), onEvent = noop } = options; + + const client = createApiClient({ openApi, accessToken }); + const idsFilePath = path.join(artifactDir, "sendedIds.json"); + + if (!fs.existsSync(idsFilePath)) { + throw new Error(`sendedIds.json not found: ${idsFilePath}`); + } + + const ids = fs.readJSONSync(idsFilePath); + + onEvent({ type: "started", message: "Delete started", data: { total: ids.length } }); + + let current = 0; + for (const id of ids) { + current += 1; + onEvent({ type: "progress", message: `Deleting ${current}/${ids.length}`, data: { current, total: ids.length } }); + await client.deleteMemo(id); + onEvent({ type: "success", message: `Deleted: ${id}` }); + } + + const result = { + total: ids.length, + success: ids.length, + failed: 0, + }; + + onEvent({ type: "finished", message: "Delete finished", data: result }); + return result; +} + +module.exports = { + deleteImported, +}; diff --git a/src/core/importFlomo.js b/src/core/importFlomo.js new file mode 100644 index 0000000..8851f44 --- /dev/null +++ b/src/core/importFlomo.js @@ -0,0 +1,98 @@ +const path = require("path"); +const fs = require("fs-extra"); +const { createApiClient } = require("./apiClient"); +const { parseFlomoHtml } = require("./parseFlomo"); +const { getDefaultArtifactDir } = require("./artifactStore"); + +function noop() {} + +async function importFlomo(options) { + const { + openApi, + accessToken, + htmlPath, + artifactDir = getDefaultArtifactDir(), + onEvent = noop, + sleepMs, + apiClient, + } = options; + + if (!htmlPath) { + throw new Error("htmlPath is required"); + } + + const client = apiClient || createApiClient({ openApi, accessToken, sleepMs }); + const memoJsonPath = path.join(artifactDir, "memo.json"); + const sentIdsPath = path.join(artifactDir, "sendedIds.json"); + + fs.removeSync(memoJsonPath); + fs.removeSync(sentIdsPath); + + const html = fs.readFileSync(htmlPath, "utf8"); + const memoArr = parseFlomoHtml(html); + const sendedMemoNames = []; + + onEvent({ type: "started", message: "Flomo import started", data: { total: memoArr.length } }); + onEvent({ type: "log", message: "Uploading resources" }); + + for (const memo of memoArr) { + const resources = []; + for (const filePath of memo.files) { + const fullPath = path.resolve(path.dirname(htmlPath), filePath); + if (!fs.existsSync(fullPath)) { + onEvent({ type: "log", message: `资源文件不存在，已跳过: ${filePath}` }); + continue; + } + onEvent({ type: "log", message: `Uploading file: ${filePath}` }); + const uploaded = await client.uploadFile(fullPath); + resources.push(uploaded); + } + memo.resources = resources; + } + + fs.writeJSONSync(memoJsonPath, memoArr, { spaces: 2 }); + + const sendOrder = [...memoArr].reverse(); + let currentCount = 0; + + for (const memo of sendOrder) { + currentCount += 1; + onEvent({ + type: "progress", + message: `Sending memo ${currentCount}/${sendOrder.length}`, + data: { current: currentCount, total: sendOrder.length }, + }); + + const response = await client.sendMemo({ + content: memo.content, + }); + + const memoName = client.extractMemoName(response); + if (!memoName) { + throw new Error("Cannot resolve memo name from response"); + } + + sendedMemoNames.push(memoName); + + await client.updateMemo(memoName, new Date(memo.time).toISOString()); + await client.setMemoResources(memoName, memo.resources || []); + + fs.writeJSONSync(sentIdsPath, sendedMemoNames, { spaces: 2 }); + onEvent({ type: "success", message: `Memo sent: ${memoName}` }); + } + + const result = { + total: sendOrder.length, + success: sendedMemoNames.length, + failed: 0, + sentIdsPath, + memoJsonPath, + }; + + onEvent({ type: "finished", message: "Flomo import finished", data: result }); + return result; +} + +module.exports = { + importFlomo, +}; diff --git a/src/core/importWeixin.js b/src/core/importWeixin.js new file mode 100644 index 0000000..d4c51df --- /dev/null +++ b/src/core/importWeixin.js @@ -0,0 +1,71 @@ +const path = require("path"); +const fs = require("fs-extra"); +const { createApiClient } = require("./apiClient"); +const { parseWeixinText } = require("./parseWeixin"); +const { getDefaultArtifactDir } = require("./artifactStore"); + +function noop() {} + +async function importWeixin(options) { + const { + openApi, + accessToken, + txtPath, + artifactDir = getDefaultArtifactDir(), + onEvent = noop, + sleepMs, + } = options; + + if (!txtPath) { + throw new Error("txtPath is required"); + } + + const client = createApiClient({ openApi, accessToken, sleepMs }); + const sentIdsPath = path.join(artifactDir, "sendedIds.json"); + + fs.removeSync(sentIdsPath); + + const text = fs.readFileSync(txtPath, "utf8"); + const parsed = parseWeixinText(text); + const sendedMemoNames = []; + + onEvent({ type: "started", message: "Weixin import started", data: { total: parsed.notes.length } }); + + let currentCount = 0; + for (const note of parsed.notes) { + currentCount += 1; + onEvent({ + type: "progress", + message: `Sending memo ${currentCount}/${parsed.notes.length}`, + data: { current: currentCount, total: parsed.notes.length }, + }); + + const response = await client.sendMemo({ + content: `${note.content}\n\n章节: ${note.chapterTitle}\n\n${parsed.tag}`, + }); + + const memoName = client.extractMemoName(response); + if (!memoName) { + throw new Error("Cannot resolve memo name from response"); + } + + sendedMemoNames.push(memoName); + fs.writeJSONSync(sentIdsPath, sendedMemoNames, { spaces: 2 }); + + onEvent({ type: "success", message: `Memo sent: ${memoName}` }); + } + + const result = { + total: parsed.notes.length, + success: sendedMemoNames.length, + failed: 0, + sentIdsPath, + }; + + onEvent({ type: "finished", message: "Weixin import finished", data: result }); + return result; +} + +module.exports = { + importWeixin, +}; diff --git a/src/core/parseFlomo.js b/src/core/parseFlomo.js new file mode 100644 index 0000000..778276d --- /dev/null +++ b/src/core/parseFlomo.js @@ -0,0 +1,50 @@ +const cheerio = require("cheerio"); +const TurndownService = require("turndown"); + +function parseFlomoHtml(html) { + const $ = cheerio.load(html); + const memoArr = []; + + const memos = $(".memo"); + + for (const memo of memos) { + const time = $(memo).find(".time").text(); + let content = ""; + let tags = []; + const files = []; + + $(memo) + .find(".content") + .each((_, contentHtml) => { + const turndownService = new TurndownService(); + const text = turndownService.turndown($(contentHtml).html()); + content += `${content ? "\n" : ""}${text}`; + }); + + const tagReg = /#(\S*)/g; + const tagMatch = content.match(tagReg); + if (tagMatch) { + tags = tagMatch.map((item) => item.replace("#", "")).filter(Boolean); + } + + $(memo) + .find(".files img") + .each((_, img) => { + files.push($(img).attr("src")); + }); + + memoArr.push({ + time, + content, + files, + tags, + }); + } + + memoArr.sort((a, b) => new Date(b.time) - new Date(a.time)); + return memoArr; +} + +module.exports = { + parseFlomoHtml, +}; diff --git a/src/core/parseWeixin.js b/src/core/parseWeixin.js new file mode 100644 index 0000000..d9483a7 --- /dev/null +++ b/src/core/parseWeixin.js @@ -0,0 +1,71 @@ +function parseWeixinText(fileContent) { + const contentParse = { + bookInfo: [], + chapterInfo: [], + }; + + const fileContentArr = fileContent.split("\n"); + + let curContent = []; + let curChapterTitle = ""; + + for (const index in fileContentArr) { + const line = fileContentArr[index]; + + if (line.length && line.startsWith("◆ ")) { + curChapterTitle = line; + } else { + curContent.push(line); + } + + if (!line.length && !fileContentArr[index - 1]?.length) { + if (!contentParse.bookInfo.length) { + contentParse.bookInfo = curContent; + } else { + contentParse.chapterInfo.push({ + title: curChapterTitle, + content: curContent, + }); + } + + curContent = []; + } + } + + const bookName = (contentParse.bookInfo[0] || "").replaceAll("《", "").replaceAll("》", ""); + const tag = `#微信读书/${bookName}`; + + const notes = []; + + for (const chapter of contentParse.chapterInfo) { + if (chapter.title.includes("◆ 点评")) continue; + + const chapterTitle = chapter.title.replace("◆ ", "").trim(); + + let currentLines = []; + + for (const line of chapter.content) { + if (line.length) { + currentLines.push(line.replaceAll(">>", ">")); + } else { + if (currentLines.length) { + notes.push({ + chapterTitle, + content: currentLines.join("\n"), + }); + } + currentLines = []; + } + } + } + + return { + bookName, + tag, + notes, + }; +} + +module.exports = { + parseWeixinText, +}; diff --git a/src/delete.js b/src/delete.js index cc8c078..418d7ce 100644 --- a/src/delete.js +++ b/src/delete.js @@ -1,12 +1,24 @@ -const path = require("path"); -const fs = require("fs-extra"); -const { deleteMemo } = require("./utils/api"); +const { deleteImported } = require("./core/deleteImported"); -const idsFilePath = path.join(process.cwd(), "sendedIds.json"); -const ids = fs.readJSONSync(idsFilePath); +const [, , openApi, accessToken] = process.argv; -for (const id of ids) { - deleteMemo(id).then(() => { - console.log("delete success", id); - }); +if (!openApi || !accessToken) { + console.error("Usage: node ./src/delete.js "); + process.exit(1); } + +deleteImported({ + openApi, + accessToken, + onEvent(event) { + const payload = event.data ? ` ${JSON.stringify(event.data)}` : ""; + console.log(`[${event.type}] ${event.message}${payload}`); + }, +}) + .then((result) => { + console.log("Delete summary:", result); + }) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/src/main.js b/src/main.js index ac1daea..fa852d8 100644 --- a/src/main.js +++ b/src/main.js @@ -1,132 +1,25 @@ -const fs = require("fs-extra"); -const cheerio = require("cheerio"); -var TurndownService = require("turndown"); -const chalk = require("chalk"); +const { importFlomo } = require("./core/importFlomo"); -const { htmlPath, getFilePath, mergePromise, errorTip } = require("./utils/utils"); -const { uploadFile, sendMemo, updateMemo, setMemoResources } = require("./utils/api"); +const [, , openApi, accessToken, htmlPath] = process.argv; -fs.removeSync("./memo.json"); -fs.removeSync("./sendedIds.json"); - -const sendedMemoNames = []; -const memoArr = []; - -const $ = cheerio.load(fs.readFileSync(htmlPath, "utf8")); - -const memos = $(".memo"); - -for (const memo of memos) { - const time = $(memo).find(".time").text(); - let content = ""; - let tags = []; - let files = []; - - $(memo) - .find(".content") - .each((index, html) => { - let text = $(html).html(); - - var turndownService = new TurndownService(); - text = turndownService.turndown(text); - - content += `${content ? "\n" : ""}${text}`; - }, ""); - - // 正则通过 #xxx 获取标签 - const tagReg = /#(\S*)/g; - const tagMatch = content.match(tagReg); - if (tagMatch) { - tags = tagMatch.map((item) => item.replace("#", "")).filter((tag) => !!tag); - } - - $(memo) - .find(".files img") - .each((index, img) => { - const src = $(img).attr("src"); - files.push(src); - }); - - memoArr.push({ - time, - content, - files, - tags, - }); -} - -memoArr.sort((a, b) => { - return new Date(b.time) - new Date(a.time); -}); - -async function uploadFileHandler() { - console.log(chalk.green("======================= 上传资源 =======================")); - for (const memo of memoArr) { - memoArr.resourceList = memoArr.resourceList || []; - const uploadFilePromiseArr = []; - if (memo.files.length) { - for (const filePath of memo.files) { - const fullPath = getFilePath(filePath); - uploadFilePromiseArr.push(() => { - console.log(chalk.green("开始上传"), filePath); - return uploadFile(fullPath); - }); - } - } - - await mergePromise(uploadFilePromiseArr).then((res) => { - memo.resources = [...(memo.resources || []), ...res]; - }); - } - - console.log(chalk.green("======================= 上传资源 end =======================")); +if (!openApi || !accessToken || !htmlPath) { + console.error("Usage: node ./src/main.js "); + process.exit(1); } -async function sendMemoHandler() { - const sendMemoPromiseArr = []; - - fs.writeJSONSync("./memo.json", memoArr); - - console.log(chalk.green("======================= 发送 Memo =======================")); - console.log(chalk.blue(`总计待发送: ${memoArr.length} 条`)); - - let currentCount = 0; - const totalCount = memoArr.length; - - for (const memo of memoArr) { - let content = memo.content; - - memo.tags.forEach((tag) => { - content += ` #${tag}`; - }); - - sendMemoPromiseArr.unshift(async () => { - try { - currentCount++; - console.log(chalk.yellow(`正在发送 [${currentCount}/${totalCount}]`)); - return await sendMemo({ - content: memo.content, - // createdTs: new Date(memo.time).getTime() / 1000, - }).then(async (res) => { - sendedMemoNames.push(res?.data?.name || res?.data?.data?.name); - - await updateMemo(res?.data?.name, new Date(memo.time).toISOString()); - - await setMemoResources(res?.data?.name, memo.resources); - - console.log(chalk.green(`发送成功 [${currentCount}/${totalCount}]`)); - - fs.writeJSONSync("./sendedIds.json", sendedMemoNames); - }); - } catch (error) { - console.log(chalk.red(`发送失败 [${currentCount}/${totalCount}]`)); - errorTip(error); - } - }); - } - - await mergePromise(sendMemoPromiseArr); - console.log(chalk.green("======================= 发送 Memo 完成 =======================")); -} - -uploadFileHandler().then(sendMemoHandler); +importFlomo({ + openApi, + accessToken, + htmlPath, + onEvent(event) { + const payload = event.data ? ` ${JSON.stringify(event.data)}` : ""; + console.log(`[${event.type}] ${event.message}${payload}`); + }, +}) + .then((result) => { + console.log("Flomo import summary:", result); + }) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/src/server/index.js b/src/server/index.js new file mode 100644 index 0000000..741acdd --- /dev/null +++ b/src/server/index.js @@ -0,0 +1,485 @@ +const http = require("http"); +const os = require("os"); +const path = require("path"); +const crypto = require("crypto"); +const fs = require("fs-extra"); +const Busboy = require("busboy"); +const { importFlomo } = require("../core/importFlomo"); +const { importWeixin } = require("../core/importWeixin"); +const { deleteImported } = require("../core/deleteImported"); +const { + isSupportedUploadKind, + isSupportedFileForKind, + sanitizeRelativePath, + resolveFlomoEntryHtml, +} = require("./uploadUtils"); + +const HOST = process.env.HOST || "127.0.0.1"; +const PORT = Number(process.env.PORT || 3131); +const WEB_ROOT = path.resolve(__dirname, "../../web"); +const UPLOAD_ROOT = path.join(os.tmpdir(), "memos-import-upload"); +const UPLOAD_LIMIT_BYTES = 20 * 1024 * 1024; +const UPLOAD_TTL_MS = 30 * 60 * 1000; + +let session = null; +let activeJobId = null; +const jobs = new Map(); +const uploads = new Map(); + +fs.removeSync(UPLOAD_ROOT); +fs.ensureDirSync(UPLOAD_ROOT); + +function sendJson(res, status, payload) { + res.writeHead(status, { "Content-Type": "application/json; charset=utf-8" }); + res.end(JSON.stringify(payload)); +} + +function sendEvent(client, event) { + client.write(`event: ${event.type}\n`); + client.write(`data: ${JSON.stringify(event)}\n\n`); +} + +function appendEvent(job, event) { + job.events.push(event); + for (const client of job.clients) { + sendEvent(client, event); + } +} + +function parseBody(req) { + return new Promise((resolve, reject) => { + let body = ""; + req.on("data", (chunk) => { + body += chunk; + }); + req.on("end", () => { + if (!body) { + resolve({}); + return; + } + try { + resolve(JSON.parse(body)); + } catch (_error) { + reject(new Error("请求体不是合法 JSON")); + } + }); + req.on("error", reject); + }); +} + +function getContentType(filePath) { + if (filePath.endsWith(".html")) return "text/html; charset=utf-8"; + if (filePath.endsWith(".css")) return "text/css; charset=utf-8"; + if (filePath.endsWith(".js")) return "application/javascript; charset=utf-8"; + return "text/plain; charset=utf-8"; +} + +function createJob(type) { + const id = `${Date.now()}-${Math.random().toString(16).slice(2, 8)}`; + const job = { + id, + type, + status: "running", + createdAt: new Date().toISOString(), + events: [], + clients: new Set(), + result: null, + error: null, + uploadId: null, + }; + jobs.set(id, job); + activeJobId = id; + return job; +} + +function cleanupUpload(uploadId) { + if (!uploadId) return; + const record = uploads.get(uploadId); + if (!record) return; + + uploads.delete(uploadId); + fs.remove(record.tempDir || record.tempPath).catch(() => {}); +} + +async function runJob(job, runner) { + try { + const result = await runner((event) => { + appendEvent(job, { + ...event, + ts: Date.now(), + }); + }); + job.status = "finished"; + job.result = result; + } catch (error) { + job.status = "failed"; + job.error = toPublicError(error); + appendEvent(job, { + type: "error", + message: job.error.message, + ts: Date.now(), + }); + } finally { + cleanupUpload(job.uploadId); + activeJobId = null; + } +} + +function toPublicError(error) { + if (!error) { + return { message: "任务执行失败" }; + } + + if (["ENOENT", "EISDIR", "ENOTDIR", "EACCES", "EPERM"].includes(error.code)) { + return { message: "文件读取失败，请检查导出文件或路径是否正确" }; + } + + const rawMessage = String(error.message || "任务执行失败"); + const message = rawMessage.replace(/\/[^\s]+/g, ""); + return { message }; +} + +function requireSession(res) { + if (!session?.openApi || !session?.accessToken) { + sendJson(res, 400, { error: "请先调用 /api/session 保存会话" }); + return false; + } + return true; +} + +function resolveInputPath(inputPath) { + if (!inputPath) return null; + if (path.isAbsolute(inputPath)) return inputPath; + return path.resolve(process.cwd(), inputPath); +} + +function parseMultipartUpload(req) { + return new Promise((resolve, reject) => { + const uploadToken = crypto.randomUUID(); + const tempDir = path.join(UPLOAD_ROOT, uploadToken); + fs.ensureDirSync(tempDir); + + const busboy = Busboy({ + headers: req.headers, + limits: { files: 5000, fileSize: UPLOAD_LIMIT_BYTES, fields: 10 }, + }); + + let kind = ""; + let fileCount = 0; + let totalBytes = 0; + let originalName = ""; + let hasFile = false; + let hasError = false; + let fileSizeExceeded = false; + const savedRelativePaths = []; + const pendingWrites = []; + + function fail(error) { + if (hasError) return; + hasError = true; + fs.remove(tempDir).catch(() => {}); + reject(error); + } + + busboy.on("field", (name, value) => { + if (name === "kind") { + kind = String(value || "").trim(); + } + }); + + busboy.on("file", (_name, file, info) => { + hasFile = true; + const incomingName = info?.filename || "upload.bin"; + const safeRelativePath = sanitizeRelativePath(incomingName) || path.basename(incomingName); + const targetPath = path.join(tempDir, safeRelativePath); + + if (!targetPath.startsWith(tempDir)) { + file.resume(); + fail(new Error("上传文件路径非法")); + return; + } + + fs.ensureDirSync(path.dirname(targetPath)); + const writeStream = fs.createWriteStream(targetPath); + fileCount += 1; + if (!originalName) { + originalName = path.basename(safeRelativePath); + } + savedRelativePaths.push(safeRelativePath); + + file.on("data", (chunk) => { + totalBytes += chunk.length; + }); + + file.on("limit", () => { + fileSizeExceeded = true; + }); + + const writePromise = new Promise((resolveWrite, rejectWrite) => { + file.on("error", rejectWrite); + writeStream.on("error", rejectWrite); + writeStream.on("finish", resolveWrite); + }); + + pendingWrites.push(writePromise); + file.pipe(writeStream); + }); + + busboy.on("error", fail); + + busboy.on("finish", async () => { + try { + await Promise.all(pendingWrites); + + if (hasError) return; + if (fileSizeExceeded) { + fail(new Error("文件超过 20MB 限制")); + return; + } + if (!hasFile || !fileCount) { + fail(new Error("请上传文件")); + return; + } + if (!isSupportedUploadKind(kind)) { + fail(new Error("kind 仅支持 flomo 或 weixin")); + return; + } + + for (const relativePath of savedRelativePaths) { + if (!isSupportedFileForKind(kind, relativePath)) { + fail(new Error(`文件类型不支持: ${relativePath}`)); + return; + } + } + + if (kind === "weixin" && fileCount !== 1) { + fail(new Error("微信读书仅支持上传一个 TXT 文件")); + return; + } + + let tempPath = path.join(tempDir, savedRelativePaths[0]); + let htmlRelativePath = ""; + + if (kind === "flomo") { + htmlRelativePath = resolveFlomoEntryHtml(savedRelativePaths); + if (!htmlRelativePath) { + fail(new Error("未找到 Flomo HTML 入口文件（index.html）")); + return; + } + tempPath = path.join(tempDir, htmlRelativePath); + } + + resolve({ + kind, + tempDir, + tempPath, + htmlRelativePath, + originalName, + fileCount, + size: totalBytes, + }); + } catch (error) { + fail(error); + } + }); + + req.pipe(busboy); + }); +} + +function gcExpiredUploads() { + const now = Date.now(); + for (const [uploadId, record] of uploads.entries()) { + if (now - record.createdAt > UPLOAD_TTL_MS) { + cleanupUpload(uploadId); + } + } +} + +setInterval(gcExpiredUploads, 5 * 60 * 1000).unref(); + +const server = http.createServer(async (req, res) => { + try { + const reqUrl = new URL(req.url, `http://${req.headers.host}`); + + if (req.method === "GET" && reqUrl.pathname === "/api/health") { + sendJson(res, 200, { ok: true, activeJobId }); + return; + } + + if (req.method === "POST" && reqUrl.pathname === "/api/session") { + const body = await parseBody(req); + if (!body.openApi || !body.accessToken) { + sendJson(res, 400, { error: "openApi 和 accessToken 不能为空" }); + return; + } + session = { + openApi: body.openApi, + accessToken: body.accessToken, + }; + sendJson(res, 200, { ok: true }); + return; + } + + if (req.method === "POST" && reqUrl.pathname === "/api/uploads") { + const parsed = await parseMultipartUpload(req); + const uploadId = crypto.randomUUID(); + uploads.set(uploadId, { + ...parsed, + createdAt: Date.now(), + }); + + sendJson(res, 201, { + uploadId, + kind: parsed.kind, + originalName: parsed.originalName, + fileCount: parsed.fileCount || 1, + size: parsed.size, + }); + return; + } + + if (req.method === "POST" && reqUrl.pathname.startsWith("/api/jobs/")) { + if (activeJobId) { + sendJson(res, 409, { error: "已有任务正在执行", activeJobId }); + return; + } + + if (!requireSession(res)) return; + + const body = await parseBody(req); + const type = reqUrl.pathname.split("/").pop(); + const job = createJob(type); + + if (type === "flomo") { + let htmlPath = null; + + if (body.uploadId) { + const upload = uploads.get(body.uploadId); + if (!upload || upload.kind !== "flomo") { + jobs.delete(job.id); + activeJobId = null; + sendJson(res, 400, { error: "uploadId 无效或已过期" }); + return; + } + job.uploadId = body.uploadId; + htmlPath = upload.tempPath; + } else { + htmlPath = resolveInputPath(body.htmlPath); + if (!htmlPath || !fs.existsSync(htmlPath)) { + jobs.delete(job.id); + activeJobId = null; + sendJson(res, 400, { error: "htmlPath 不存在" }); + return; + } + } + + runJob(job, (onEvent) => importFlomo({ ...session, htmlPath, onEvent })); + } else if (type === "weixin") { + let txtPath = null; + + if (body.uploadId) { + const upload = uploads.get(body.uploadId); + if (!upload || upload.kind !== "weixin") { + jobs.delete(job.id); + activeJobId = null; + sendJson(res, 400, { error: "uploadId 无效或已过期" }); + return; + } + job.uploadId = body.uploadId; + txtPath = upload.tempPath; + } else { + txtPath = resolveInputPath(body.txtPath); + if (!txtPath || !fs.existsSync(txtPath)) { + jobs.delete(job.id); + activeJobId = null; + sendJson(res, 400, { error: "txtPath 不存在" }); + return; + } + } + + runJob(job, (onEvent) => importWeixin({ ...session, txtPath, onEvent })); + } else if (type === "delete") { + runJob(job, (onEvent) => deleteImported({ ...session, onEvent })); + } else { + jobs.delete(job.id); + activeJobId = null; + sendJson(res, 404, { error: "未知任务类型" }); + return; + } + + sendJson(res, 202, { jobId: job.id, status: job.status }); + return; + } + + if (req.method === "GET" && /^\/api\/jobs\/[^/]+\/events$/.test(reqUrl.pathname)) { + const parts = reqUrl.pathname.split("/"); + const jobId = parts[3]; + const job = jobs.get(jobId); + if (!job) { + sendJson(res, 404, { error: "任务不存在" }); + return; + } + + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }); + + for (const event of job.events) { + sendEvent(res, event); + } + + job.clients.add(res); + req.on("close", () => { + job.clients.delete(res); + }); + return; + } + + if (req.method === "GET" && /^\/api\/jobs\/[^/]+\/result$/.test(reqUrl.pathname)) { + const parts = reqUrl.pathname.split("/"); + const jobId = parts[3]; + const job = jobs.get(jobId); + if (!job) { + sendJson(res, 404, { error: "任务不存在" }); + return; + } + + sendJson(res, 200, { + id: job.id, + type: job.type, + status: job.status, + createdAt: job.createdAt, + result: job.result, + error: job.error, + }); + return; + } + + if (req.method === "GET") { + const pathname = reqUrl.pathname === "/" ? "/index.html" : reqUrl.pathname; + const filePath = path.resolve(WEB_ROOT, `.${pathname}`); + + if (!filePath.startsWith(WEB_ROOT)) { + sendJson(res, 403, { error: "禁止访问" }); + return; + } + + if (fs.existsSync(filePath) && fs.statSync(filePath).isFile()) { + res.writeHead(200, { "Content-Type": getContentType(filePath) }); + fs.createReadStream(filePath).pipe(res); + return; + } + } + + sendJson(res, 404, { error: "Not Found" }); + } catch (error) { + sendJson(res, 500, { error: error.message }); + } +}); + +server.listen(PORT, HOST, () => { + console.log(`Web console is running at http://${HOST}:${PORT}`); +}); diff --git a/src/server/uploadUtils.js b/src/server/uploadUtils.js new file mode 100644 index 0000000..efcf4c0 --- /dev/null +++ b/src/server/uploadUtils.js @@ -0,0 +1,57 @@ +const path = require("path"); + +const SUPPORTED_KINDS = new Set(["flomo", "weixin"]); + +function isSupportedUploadKind(kind) { + return SUPPORTED_KINDS.has(kind); +} + +function normalizeExt(filename) { + return path.extname(filename || "").toLowerCase(); +} + +function isSupportedFileForKind(kind, filename) { + if (kind === "flomo") { + return true; + } + if (kind === "weixin") { + const ext = normalizeExt(filename); + return ext === ".txt"; + } + return false; +} + +function sanitizeRelativePath(inputPath) { + const normalized = path + .normalize(String(inputPath || "")) + .replace(/^(\.\.(\/|\\|$))+/, "") + .replace(/^[/\\]+/, ""); + if (!normalized || normalized === ".") { + return ""; + } + return normalized; +} + +function resolveFlomoEntryHtml(savedRelativePaths) { + const htmlCandidates = savedRelativePaths + .filter((item) => { + const ext = path.extname(item).toLowerCase(); + return ext === ".html" || ext === ".htm"; + }) + .sort((a, b) => { + const aIsIndex = /(^|[/\\])index\.html?$/i.test(a); + const bIsIndex = /(^|[/\\])index\.html?$/i.test(b); + if (aIsIndex && !bIsIndex) return -1; + if (!aIsIndex && bIsIndex) return 1; + return a.length - b.length; + }); + + return htmlCandidates[0] || ""; +} + +module.exports = { + isSupportedUploadKind, + isSupportedFileForKind, + sanitizeRelativePath, + resolveFlomoEntryHtml, +}; diff --git a/src/utils/api.js b/src/utils/api.js index 5e41e1e..4b62924 100644 --- a/src/utils/api.js +++ b/src/utils/api.js @@ -2,12 +2,11 @@ const fs = require("fs-extra"); const path = require("path"); const mime = require("mime"); const axios = require("axios"); -const FormData = require("form-data"); const { getRequestUrl, getAccessToken, openApi } = require("./utils"); const SLEEP = 1000; -default_header = { +const defaultHeader = { Authorization: `Bearer ${getAccessToken()}`, }; const getVersion = () => { @@ -26,7 +25,7 @@ exports.uploadFile = async (filePath) => { filename: path.basename(filePath), type: mime.getType(filePath) || undefined, }, - headers: default_header, + headers: defaultHeader, }).then((res) => res.data); }; @@ -36,7 +35,7 @@ exports.sendMemo = async (memo) => { url: getRequestUrl(`/api${getVersion()}/memos`), data: memo, headers: { - ...default_header, + ...defaultHeader, "Content-Type": "application/json; charset=UTF-8", }, }).then(async (res) => { @@ -52,7 +51,7 @@ exports.updateMemo = async (memoName, createTime) => { url: getRequestUrl(`/api${getVersion()}/${memoName}`), data: { createTime }, headers: { - ...default_header, + ...defaultHeader, "Content-Type": "application/json; charset=UTF-8", }, }); @@ -66,7 +65,7 @@ exports.setMemoResources = async (memoName, resources) => { resources: resources, }, headers: { - ...default_header, + ...defaultHeader, "Content-Type": "application/json; charset=UTF-8", }, }; @@ -79,7 +78,7 @@ exports.deleteMemo = async (memoName) => { method: "delete", url: getRequestUrl(`/api${getVersion()}/${memoName}`), headers: { - ...default_header, + ...defaultHeader, "Content-Type": "application/json; charset=UTF-8", }, }); diff --git a/src/weixin.js b/src/weixin.js index fe8a27e..5ef0f17 100644 --- a/src/weixin.js +++ b/src/weixin.js @@ -1,90 +1,25 @@ -const fs = require("fs-extra"); -const chalk = require("chalk"); +const { importWeixin } = require("./core/importWeixin"); -const { htmlPath, getFilePath, mergePromise, errorTip } = require("./utils/utils"); -const { sendMemo } = require("./utils/api"); +const [, , openApi, accessToken, txtPath] = process.argv; -fs.removeSync("./sendedIds.json"); - -const contentParse = { - bookInfo: [], - chapterInfo: [], -}; - -const fullPath = getFilePath(htmlPath); - -const fileContent = fs.readFileSync(fullPath, "utf8"); - -const fileContentArr = fileContent.split("\n"); - -let curContent = []; -let curChapterTitle = ""; -for (const index in fileContentArr) { - const line = fileContentArr[index]; - - if (line.length && line.startsWith("◆ ")) { - curChapterTitle = line; - } else { - curContent.push(line); - } - - if (!line.length && !fileContentArr[index - 1].length) { - if (!contentParse.bookInfo.length) { - contentParse.bookInfo = curContent; - } else { - contentParse.chapterInfo.push({ - title: curChapterTitle, - content: curContent, - }); - } - - curContent = []; - } -} - -// fs.writeJsonSync("./content.json", contentParse); - -const bookname = contentParse.bookInfo[0].replaceAll("《", "").replaceAll("》", ""); -const tag = `#微信读书/${bookname}`; - -const sendMemoPromiseArr = []; - -for (const chapter of contentParse.chapterInfo) { - if (chapter.title.includes("◆ 点评")) continue; - - const chapterTitle = chapter.title.replace("◆ ", "").trim(); - - let curContent = []; - for (const index in chapter.content) { - const line = chapter.content[index]; - - if (line.length) { - curContent.push(line.replaceAll(">>", ">")); - } else { - if (curContent.length) { - const content = curContent.join("\n"); - sendMemoPromiseArr.push(async () => { - try { - return await sendMemo({ - content: `${content}\n\n章节: ${chapterTitle}\n\n${tag}`, - }).then((res) => { - console.log(chalk.green("success"), res.data.content); - return res; - }); - } catch (error) { - errorTip(error); - } - }); - } - - curContent = []; - } - } +if (!openApi || !accessToken || !txtPath) { + console.error("Usage: node ./src/weixin.js "); + process.exit(1); } -let sendedMemoNames = []; -mergePromise(sendMemoPromiseArr).then((res) => { - sendedMemoNames = res.map((item) => item.data.name); - - fs.writeJSONSync("./sendedIds.json", sendedMemoNames); -}); +importWeixin({ + openApi, + accessToken, + txtPath, + onEvent(event) { + const payload = event.data ? ` ${JSON.stringify(event.data)}` : ""; + console.log(`[${event.type}] ${event.message}${payload}`); + }, +}) + .then((result) => { + console.log("Weixin import summary:", result); + }) + .catch((error) => { + console.error(error); + process.exit(1); + }); diff --git a/tests/import-flomo.test.js b/tests/import-flomo.test.js new file mode 100644 index 0000000..7a041d2 --- /dev/null +++ b/tests/import-flomo.test.js @@ -0,0 +1,46 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); +const path = require("path"); +const os = require("os"); +const fs = require("fs-extra"); + +const { importFlomo } = require("../src/core/importFlomo"); + +test("importFlomo should skip missing resource files and continue sending memos", async () => { + const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "memos-flomo-test-")); + const htmlPath = path.join(tmpDir, "index.html"); + + fs.writeFileSync( + htmlPath, + `
2024-01-01 00:00:00

hello

` + ); + + let sendMemoCount = 0; + const events = []; + + const mockClient = { + uploadFile: async () => { + throw new Error("should not be called for missing file"); + }, + sendMemo: async () => { + sendMemoCount += 1; + return { data: { name: `memos/${sendMemoCount}` } }; + }, + updateMemo: async () => {}, + setMemoResources: async () => {}, + extractMemoName: (res) => res?.data?.name, + }; + + const result = await importFlomo({ + openApi: "http://localhost:5230/api/v1", + accessToken: "token", + htmlPath, + artifactDir: tmpDir, + onEvent: (event) => events.push(event), + apiClient: mockClient, + }); + + assert.equal(result.success, 1); + assert.equal(sendMemoCount, 1); + assert.equal(events.some((e) => e.type === "log" && e.message.includes("资源文件不存在，已跳过")), true); +}); diff --git a/tests/parsers.test.js b/tests/parsers.test.js new file mode 100644 index 0000000..ca18751 --- /dev/null +++ b/tests/parsers.test.js @@ -0,0 +1,55 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); + +const { parseFlomoHtml } = require("../src/core/parseFlomo"); +const { parseWeixinText } = require("../src/core/parseWeixin"); + +test("parseFlomoHtml should parse memos, tags, files and sort by time desc", () => { + const html = ` +
+
2024-01-01 10:00:00
+

First #tag1

+
+
+
+
2024-01-02 10:00:00
+

Second #tag2

+
+
`; + + const memos = parseFlomoHtml(html); + + assert.equal(memos.length, 2); + assert.equal(memos[0].content.includes("Second"), true); + assert.deepEqual(memos[0].tags, ["tag2"]); + assert.deepEqual(memos[0].files, ["b.png"]); + assert.deepEqual(memos[1].tags, ["tag1"]); +}); + +test("parseWeixinText should parse chapters and skip review section", () => { + const content = [ + "《Book Name》", + "author", + "", + "", + "◆ Chapter 1", + "line one", + "", + "line two", + "", + "", + "◆ 点评", + "ignore line", + "", + "", + ].join("\n"); + + const parsed = parseWeixinText(content); + + assert.equal(parsed.bookName, "Book Name"); + assert.equal(parsed.tag, "#微信读书/Book Name"); + assert.equal(parsed.notes.length, 2); + assert.equal(parsed.notes[0].chapterTitle, "Chapter 1"); + assert.equal(parsed.notes[0].content, "line one"); + assert.equal(parsed.notes[1].content, "line two"); +}); diff --git a/tests/upload-utils.test.js b/tests/upload-utils.test.js new file mode 100644 index 0000000..f73460c --- /dev/null +++ b/tests/upload-utils.test.js @@ -0,0 +1,35 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); + +const { + isSupportedUploadKind, + isSupportedFileForKind, + sanitizeRelativePath, + resolveFlomoEntryHtml, +} = require("../src/server/uploadUtils"); + +test("isSupportedUploadKind should accept flomo and weixin", () => { + assert.equal(isSupportedUploadKind("flomo"), true); + assert.equal(isSupportedUploadKind("weixin"), true); + assert.equal(isSupportedUploadKind("other"), false); +}); + +test("isSupportedFileForKind should validate ext by kind", () => { + assert.equal(isSupportedFileForKind("flomo", "index.html"), true); + assert.equal(isSupportedFileForKind("flomo", "index.htm"), true); + assert.equal(isSupportedFileForKind("flomo", "file/abc.png"), true); + assert.equal(isSupportedFileForKind("flomo", "assets/app.css"), true); + + assert.equal(isSupportedFileForKind("weixin", "notes.txt"), true); + assert.equal(isSupportedFileForKind("weixin", "notes.html"), false); +}); + +test("sanitizeRelativePath should remove unsafe prefixes", () => { + assert.equal(sanitizeRelativePath("../a/b/index.html"), "a/b/index.html"); + assert.equal(sanitizeRelativePath("/tmp/a.txt"), "tmp/a.txt"); +}); + +test("resolveFlomoEntryHtml should choose index.html first", () => { + const result = resolveFlomoEntryHtml(["file/a.png", "index.html", "nested/other.html"]); + assert.equal(result, "index.html"); +}); diff --git a/web/app.js b/web/app.js new file mode 100644 index 0000000..7f66b43 --- /dev/null +++ b/web/app.js @@ -0,0 +1,185 @@ +const statusEl = document.getElementById("status"); +const logsEl = document.getElementById("logs"); + +function setStatus(text) { + statusEl.textContent = text; +} + +function appendLog(message) { + logsEl.textContent += `${message}\n`; + logsEl.scrollTop = logsEl.scrollHeight; +} + +async function requestJson(url, options) { + const response = await fetch(url, options || {}); + const payload = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(payload.error || `请求失败: ${response.status}`); + } + return payload; +} + +async function requestJsonWithBody(url, body) { + return requestJson(url, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify(body || {}), + }); +} + +async function uploadFile(kind, file) { + const formData = new FormData(); + formData.append("kind", kind); + formData.append("file", file); + + const result = await requestJson("/api/uploads", { + method: "POST", + body: formData, + }); + + appendLog(`[上传] ${result.originalName} (${result.size} bytes)`); + return result.uploadId; +} + +async function uploadFolder(kind, files) { + const formData = new FormData(); + formData.append("kind", kind); + + for (const file of files) { + const relativePath = file.webkitRelativePath || file.name; + formData.append("file", file, relativePath); + } + + const result = await requestJson("/api/uploads", { + method: "POST", + body: formData, + }); + + appendLog(`[上传] 已上传目录，共 ${result.fileCount} 个文件`); + return result.uploadId; +} + +async function saveSession() { + const openApi = document.getElementById("openApi").value.trim(); + const accessToken = document.getElementById("accessToken").value.trim(); + + await requestJsonWithBody("/api/session", { openApi, accessToken }); + + setStatus("会话已保存"); + appendLog("[会话] 已保存"); +} + +function attachSse(jobId) { + const events = new EventSource(`/api/jobs/${jobId}/events`); + + const relay = (type) => { + events.addEventListener(type, (event) => { + const data = JSON.parse(event.data); + appendLog(`[${data.type}] ${data.message || ""}`); + if (data.type === "finished") { + setStatus("已完成"); + } + if (data.type === "error") { + setStatus("失败"); + } + }); + }; + + ["started", "progress", "log", "success", "error", "finished"].forEach(relay); + + return events; +} + +async function runJob(type, body) { + logsEl.textContent = ""; + const jobLabelMap = { + flomo: "Flomo 导入", + weixin: "微信读书导入", + delete: "删除任务", + }; + setStatus(`运行中: ${jobLabelMap[type] || type}`); + + const result = await requestJsonWithBody(`/api/jobs/${type}`, body || {}); + + appendLog(`[任务] 已启动: ${result.jobId}`); + const sse = attachSse(result.jobId); + + const poll = setInterval(async () => { + try { + const job = await requestJson(`/api/jobs/${result.jobId}/result`, { method: "GET" }); + if (job.status !== "running") { + clearInterval(poll); + sse.close(); + appendLog(`[任务] 状态=${job.status}`); + appendLog(`[任务] 结果=${JSON.stringify(job.result || job.error || {})}`); + } + } catch (error) { + clearInterval(poll); + sse.close(); + appendLog(`[错误] ${error.message}`); + setStatus("失败"); + } + }, 1000); +} + +document.getElementById("saveSession").addEventListener("click", async () => { + try { + await saveSession(); + } catch (error) { + appendLog(`[错误] ${error.message}`); + setStatus("失败"); + } +}); + +document.getElementById("runFlomo").addEventListener("click", async () => { + const htmlPath = document.getElementById("htmlPath").value.trim(); + const flomoFolderFiles = Array.from(document.getElementById("flomoFolder").files || []); + + try { + if (flomoFolderFiles.length) { + const uploadId = await uploadFolder("flomo", flomoFolderFiles); + await runJob("flomo", { uploadId }); + return; + } + + if (!htmlPath) { + throw new Error("请先选择 Flomo 导出文件夹，或填写 Flomo 文件路径"); + } + + await runJob("flomo", { htmlPath }); + } catch (error) { + appendLog(`[错误] ${error.message}`); + setStatus("失败"); + } +}); + +document.getElementById("runWeixin").addEventListener("click", async () => { + const txtPath = document.getElementById("txtPath").value.trim(); + const weixinFile = document.getElementById("weixinFile").files[0]; + + try { + if (weixinFile) { + const uploadId = await uploadFile("weixin", weixinFile); + await runJob("weixin", { uploadId }); + return; + } + + if (!txtPath) { + throw new Error("请先选择微信读书 TXT 文件，或填写文件路径"); + } + + await runJob("weixin", { txtPath }); + } catch (error) { + appendLog(`[错误] ${error.message}`); + setStatus("失败"); + } +}); + +document.getElementById("runDelete").addEventListener("click", async () => { + try { + await runJob("delete", {}); + } catch (error) { + appendLog(`[错误] ${error.message}`); + setStatus("失败"); + } +}); diff --git a/web/index.html b/web/index.html new file mode 100644 index 0000000..18b0026 --- /dev/null +++ b/web/index.html @@ -0,0 +1,81 @@ + + + + + + Memos 导入控制台 + + + +
+
+

Memos 导入控制台

+

通过可视化页面执行 Flomo / 微信读书导入与删除任务，并实时查看日志。

+
+ +
+

会话配置

+
+ + +
+ +
+ +
+

任务

+
+ + +
+
+ + +
+ +
+ + +
+
+ + +
+ +
+ +
+
+ +
+

状态

+ 
空闲
+
+ +
+

实时日志

+ 

+      

+    

+
+    
+  
+
diff --git a/web/style.css b/web/style.css
new file mode 100644
index 0000000..c78e92f
--- /dev/null
+++ b/web/style.css
@@ -0,0 +1,102 @@
+:root {
+  --bg: #f5f7fb;
+  --card: #ffffff;
+  --ink: #1f2937;
+  --muted: #6b7280;
+  --accent: #0f766e;
+  --danger: #b91c1c;
+  --border: #d1d5db;
+}
+
+* {
+  box-sizing: border-box;
+}
+
+body {
+  margin: 0;
+  font-family: "Iowan Old Style", "Palatino Linotype", serif;
+  color: var(--ink);
+  background: radial-gradient(circle at top right, #dbeafe, #f5f7fb 45%);
+}
+
+.page {
+  width: min(960px, 100% - 2rem);
+  margin: 2rem auto;
+  display: grid;
+  gap: 1rem;
+}
+
+.panel {
+  background: var(--card);
+  border: 1px solid var(--border);
+  border-radius: 12px;
+  padding: 1rem;
+}
+
+h1,
+h2 {
+  margin: 0 0 0.5rem;
+}
+
+p {
+  margin: 0;
+  color: var(--muted);
+}
+
+.field-grid {
+  display: grid;
+  grid-template-columns: repeat(auto-fit, minmax(260px, 1fr));
+  gap: 0.75rem;
+}
+
+.job-group {
+  display: grid;
+  grid-template-columns: 1fr auto;
+  gap: 0.75rem;
+  margin-bottom: 0.75rem;
+}
+
+label {
+  display: grid;
+  gap: 0.25rem;
+  color: var(--muted);
+}
+
+input {
+  border: 1px solid var(--border);
+  border-radius: 8px;
+  padding: 0.6rem 0.7rem;
+  font-size: 0.95rem;
+}
+
+button {
+  border: none;
+  border-radius: 8px;
+  padding: 0.65rem 0.9rem;
+  background: var(--accent);
+  color: #fff;
+  cursor: pointer;
+}
+
+button.danger {
+  background: var(--danger);
+}
+
+pre {
+  margin: 0;
+  padding: 0.75rem;
+  border-radius: 8px;
+  border: 1px solid var(--border);
+  background: #f9fafb;
+  min-height: 72px;
+  max-height: 280px;
+  overflow: auto;
+  white-space: pre-wrap;
+  word-break: break-word;
+}
+
+@media (max-width: 640px) {
+  .job-group {
+    grid-template-columns: 1fr;
+  }
+}