Skip to content

Commit dd06b88

Browse files
committed
feat(storage): 新增Discord Bot存储驱动并实现分片断点续传支持
feat(storage): 新增MIRROR(清华、中科大、阿里云镜像站配置)存储驱动及浏览器伪装功能 per(storage): 增强存储配置的安全性与密钥处理 - 新增支持分块上传的Discord存储驱动 - 优化对输入值的掩码占位符检测,防止将掩码值写回数据库,确保真实密钥的安全性。 - 实现面向开源软件镜像站点的新MIRROR存储驱动(只读模式) - 通过MasqueradeClient为存储驱动添加浏览器伪装功能 - 支持预置镜像站配置(清华、中科大、阿里云)及自动填充端点 - 将镜像驱动集成至StorageFactory并配置验证规则 - 为现有驱动添加类浏览器头部信息以提升兼容性
1 parent f6ad6bc commit dd06b88

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+4912
-119
lines changed

backend/package.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "cloudpaste-api",
3-
"version": "1.8.0",
3+
"version": "1.9.0",
44
"description": "CloudPaste API基于Cloudflare Workers和D1数据库",
55
"main": "unified-entry.js",
66
"scripts": {
@@ -27,15 +27,15 @@
2727
"@aws-sdk/client-s3": "3.726.1",
2828
"@aws-sdk/lib-storage": "3.726.1",
2929
"@aws-sdk/s3-request-presigner": "3.726.1",
30-
"@huggingface/hub": "^2.7.1",
3130
"@hono/node-server": "^1.19.6",
31+
"@huggingface/hub": "^2.7.1",
3232
"better-sqlite3": "^12.5.0",
33+
"cron-parser": "^5.4.0",
3334
"fast-xml-parser": "^5.2.5",
3435
"file-type": "^21.0.0",
3536
"hono": "^4.10.6",
3637
"mime-types": "^3.0.1",
3738
"node-schedule": "^2.1.1",
38-
"cron-parser": "^5.4.0",
3939
"tsx": "^4.21.0",
4040
"webdav": "^5.8.0"
4141
},

backend/src/repositories/StorageConfigRepository.js

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,10 @@ export class StorageConfigRepository extends BaseRepository {
3131
row,
3232
});
3333

34+
// 注意:row(表字段)优先级应高于 config_json(投影字段)
3435
const merged = {
35-
...row,
3636
...(projected && typeof projected === "object" ? projected : {}),
37+
...row,
3738
};
3839

3940
// 保留原始 config_json 对象(非枚举属性,避免对外暴露)

backend/src/routes/fs/multipart.js

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { MountManager } from "../../storage/managers/MountManager.js";
55
import { FileSystem } from "../../storage/fs/FileSystem.js";
66
import { getEncryptionSecret } from "../../utils/environmentUtils.js";
77
import { usePolicy } from "../../security/policies/policies.js";
8-
import { findUploadSessionById, updateUploadSessionById } from "../../utils/uploadSessions.js";
8+
import { findUploadSessionById, normalizeUploadSessionUserId, updateUploadSessionById } from "../../utils/uploadSessions.js";
99
import { validateFsItemName } from "../../storage/fs/utils/FsInputValidator.js";
1010

1111
/**
@@ -93,6 +93,24 @@ export const registerMultipartRoutes = (router, helpers) => {
9393
return { db: c.env.DB, encryptionSecret: getEncryptionSecret(c), repositoryFactory: c.get("repos"), userInfo, userIdOrInfo, userType };
9494
};
9595

96+
const assertUploadSessionOwnedByUser = (sessionRow, userIdOrInfo, userType) => {
97+
if (!sessionRow) {
98+
throw new ValidationError("未找到对应的上传会话");
99+
}
100+
101+
const expectedUserId = normalizeUploadSessionUserId(userIdOrInfo, userType);
102+
const rowUserId = String(sessionRow.user_id || "");
103+
const rowUserType = String(sessionRow.user_type || "");
104+
105+
// 必须至少匹配 user_id;user_type 为空时视为兼容旧数据(不做强校验)
106+
const idMatches = rowUserId === String(expectedUserId || "");
107+
const typeMatches = !rowUserType || rowUserType === String(userType || "");
108+
109+
if (!idMatches || !typeMatches) {
110+
throw new AuthenticationError("上传会话不属于当前用户,拒绝访问");
111+
}
112+
};
113+
96114
const assertValidFileName = (fileName) => {
97115
const result = validateFsItemName(fileName);
98116
if (result.valid) return;
@@ -148,6 +166,9 @@ export const registerMultipartRoutes = (router, helpers) => {
148166
assertValidFileName(fileName);
149167
}
150168

169+
const sessionRow = await findUploadSessionById(db, { id: uploadId });
170+
assertUploadSessionOwnedByUser(sessionRow, userIdOrInfo, userType);
171+
151172
const mountManager = new MountManager(db, encryptionSecret, repositoryFactory, { env: c.env });
152173
const fileSystem = new FileSystem(mountManager);
153174
const safeParts = Array.isArray(parts) ? parts : [];
@@ -167,6 +188,9 @@ export const registerMultipartRoutes = (router, helpers) => {
167188

168189
assertValidFileName(fileName);
169190

191+
const sessionRow = await findUploadSessionById(db, { id: uploadId });
192+
assertUploadSessionOwnedByUser(sessionRow, userIdOrInfo, userType);
193+
170194
const mountManager = new MountManager(db, encryptionSecret, repositoryFactory, { env: c.env });
171195
const fileSystem = new FileSystem(mountManager);
172196
await fileSystem.abortFrontendMultipartUpload(path, uploadId, fileName, userIdOrInfo, userType);
@@ -200,6 +224,9 @@ export const registerMultipartRoutes = (router, helpers) => {
200224

201225
assertValidFileName(fileName);
202226

227+
const sessionRow = await findUploadSessionById(db, { id: uploadId });
228+
assertUploadSessionOwnedByUser(sessionRow, userIdOrInfo, userType);
229+
203230
const mountManager = new MountManager(db, encryptionSecret, repositoryFactory, { env: c.env });
204231
const fileSystem = new FileSystem(mountManager);
205232
const result = await fileSystem.listMultipartParts(path, uploadId, fileName, userIdOrInfo, userType);
@@ -219,6 +246,9 @@ export const registerMultipartRoutes = (router, helpers) => {
219246

220247
const safePartNumbers = Array.isArray(partNumbers) ? partNumbers : [];
221248

249+
const sessionRow = await findUploadSessionById(db, { id: uploadId });
250+
assertUploadSessionOwnedByUser(sessionRow, userIdOrInfo, userType);
251+
222252
// 状态机推进:请求分片 URL 视为“开始上传”
223253
try {
224254
await updateUploadSessionById(db, {
@@ -266,9 +296,7 @@ export const registerMultipartRoutes = (router, helpers) => {
266296
const contentLength = contentLengthHeader ? Number.parseInt(contentLengthHeader, 10) || 0 : 0;
267297

268298
const sessionRow = await findUploadSessionById(db, { id: uploadId });
269-
if (!sessionRow) {
270-
throw new ValidationError("未找到对应的上传会话");
271-
}
299+
assertUploadSessionOwnedByUser(sessionRow, userIdOrInfo, userType);
272300

273301
const mountManager = new MountManager(db, encryptionSecret, repositoryFactory, { env: c.env });
274302
const fileSystem = new FileSystem(mountManager);

backend/src/routes/systemRoutes.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ systemRoutes.get("/api/version", async (c) => {
9797
const isDocker = runtimeEnv === "docker";
9898

9999
// 统一的默认版本配置
100-
const DEFAULT_VERSION = "1.8.0";
100+
const DEFAULT_VERSION = "1.9.0";
101101
const DEFAULT_NAME = "cloudpaste-api";
102102

103103
let version = DEFAULT_VERSION;

backend/src/services/storageConfigService.js

Lines changed: 146 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ import { MountManager } from "../storage/managers/MountManager.js";
55
import { ApiStatus } from "../constants/index.js";
66
import { AppError, ValidationError, NotFoundError, DriverError } from "../http/errors.js";
77
import { CAPABILITIES } from "../storage/interfaces/capabilities/index.js";
8+
import { invalidateFsCache } from "../cache/invalidation.js";
9+
import { FsSearchIndexStore } from "../storage/fs/search/FsSearchIndexStore.js";
810

911
/**
1012
* 计算存储配置在 WebDAV 渠道下支持的策略列表
@@ -33,6 +35,81 @@ function computeWebDavSupportedPolicies(cfg) {
3335
// 去重
3436
return Array.from(new Set(policies));
3537
}
38+
39+
/**
40+
* 判断一个输入值是否像“掩码占位符”(例如 *****1234)
41+
* - 这是为了防止编辑表单时把 masked 值写回数据库,覆盖真实密钥
42+
* - 前端应该已做过滤,但后端必须兜底
43+
* @param {any} value
44+
* @returns {boolean}
45+
*/
46+
function isMaskedSecretPlaceholder(value) {
47+
if (typeof value !== "string") return false;
48+
const s = value.trim();
49+
if (!s) return false;
50+
// maskSecret 默认会生成很多个 * + 尾部少量可见字符
51+
// 这里用一个“非常保守但够用”的判定:至少 3 个 * 开头
52+
return /^\*{3,}.+$/.test(s);
53+
}
54+
55+
function getTypeConfigSchema(storageType) {
56+
if (!storageType) return null;
57+
const meta = StorageFactory.getTypeMetadata(storageType);
58+
return meta?.configSchema || null;
59+
}
60+
61+
function getSecretFieldDefsFromSchema(schema) {
62+
const fields = schema?.fields;
63+
if (!Array.isArray(fields)) return [];
64+
return fields.filter((f) => f && typeof f === "object" && f.type === "secret" && typeof f.name === "string" && f.name);
65+
}
66+
67+
function shouldSkipSecretWrite(value, { allowEmpty = true } = {}) {
68+
if (value === undefined || value === null) return true;
69+
if (typeof value === "string") {
70+
const s = value.trim();
71+
if (allowEmpty && s.length === 0) return true;
72+
if (isMaskedSecretPlaceholder(s)) return true;
73+
}
74+
return false;
75+
}
76+
77+
async function encryptSecretsInConfigJson(storageType, configJson, encryptionSecret, { requireRequiredSecrets = false } = {}) {
78+
const schema = getTypeConfigSchema(storageType);
79+
const secretFields = getSecretFieldDefsFromSchema(schema);
80+
if (!secretFields.length) return configJson;
81+
82+
for (const f of secretFields) {
83+
const key = f.name;
84+
const raw = configJson?.[key];
85+
86+
// required 校验(仅用于 create)
87+
if (requireRequiredSecrets) {
88+
const missing =
89+
raw === undefined ||
90+
raw === null ||
91+
(typeof raw === "string" && raw.trim().length === 0) ||
92+
isMaskedSecretPlaceholder(raw);
93+
if (f.required === true && missing) {
94+
throw new ValidationError(`缺少必填字段: ${key}`);
95+
}
96+
}
97+
98+
// 跳过空值(update 时应“保留原值”)
99+
if (shouldSkipSecretWrite(raw, { allowEmpty: true })) {
100+
continue;
101+
}
102+
103+
// 明确拒绝 masked 占位符写入(无论 create/update)
104+
if (isMaskedSecretPlaceholder(raw)) {
105+
throw new ValidationError(`字段 ${key} 不能是掩码占位符,请重新填写`);
106+
}
107+
108+
configJson[key] = await encryptValue(String(raw), encryptionSecret);
109+
}
110+
111+
return configJson;
112+
}
36113
import { encryptValue, buildSecretView } from "../utils/crypto.js";
37114
import { generateStorageConfigId } from "../utils/common.js";
38115

@@ -119,6 +196,9 @@ export async function createStorageConfig(db, configData, adminId, encryptionSec
119196
for (const f of requiredS3) {
120197
if (!configData[f]) throw new ValidationError(`缺少必填字段: ${f}`);
121198
}
199+
if (isMaskedSecretPlaceholder(configData.access_key_id) || isMaskedSecretPlaceholder(configData.secret_access_key)) {
200+
throw new ValidationError("S3 密钥字段疑似为掩码占位符(*****1234),请重新填写真实值");
201+
}
122202
const encryptedAccessKey = await encryptValue(configData.access_key_id, encryptionSecret);
123203
const encryptedSecretKey = await encryptValue(configData.secret_access_key, encryptionSecret);
124204
configJson = {
@@ -139,6 +219,9 @@ export async function createStorageConfig(db, configData, adminId, encryptionSec
139219
for (const f of requiredWebDav) {
140220
if (!configData[f]) throw new ValidationError(`缺少必填字段: ${f}`);
141221
}
222+
if (isMaskedSecretPlaceholder(configData.password)) {
223+
throw new ValidationError("WebDAV 密码字段疑似为掩码占位符(*****1234),请重新填写真实值");
224+
}
142225

143226
let endpoint_url = String(configData.endpoint_url).trim();
144227
try {
@@ -173,8 +256,9 @@ export async function createStorageConfig(db, configData, adminId, encryptionSec
173256
total_storage_bytes: totalStorageBytes,
174257
};
175258
} else {
176-
const { name, storage_type, is_public, is_default, ...rest } = configData;
259+
const { name, storage_type, is_public, is_default, remark, url_proxy, ...rest } = configData;
177260
configJson = rest || {};
261+
await encryptSecretsInConfigJson(configData.storage_type, configJson, encryptionSecret, { requireRequiredSecrets: true });
178262
}
179263
const createData = {
180264
id,
@@ -203,6 +287,10 @@ export async function updateStorageConfig(db, id, updateData, adminId, encryptio
203287
const exists = await repo.findByIdAndAdminWithSecrets(id, adminId);
204288
if (!exists) throw new NotFoundError("存储配置不存在");
205289

290+
// 是否有“驱动私有配置”字段变化(比如 endpoint/bucket/token 等)
291+
// - 仅改 name/remark/is_public/is_default/status 这种“展示/开关”字段,不需要清索引
292+
// - 一旦改了驱动配置,旧索引可能对应的是“旧数据源/旧路径”,必须标记 not_ready 让你重建
293+
let driverConfigChanged = false;
206294
const topPatch = {};
207295
if (updateData.name) topPatch.name = updateData.name;
208296
if (updateData.is_public !== undefined) topPatch.is_public = updateData.is_public ? 1 : 0;
@@ -215,17 +303,32 @@ export async function updateStorageConfig(db, id, updateData, adminId, encryptio
215303
if (exists?.__config_json__ && typeof exists.__config_json__ === "object") {
216304
cfg = { ...exists.__config_json__ };
217305
}
306+
307+
// 本类型 schema 中声明的 secret 字段集合
308+
const schema = getTypeConfigSchema(exists.storage_type);
309+
const secretFieldSet = new Set(getSecretFieldDefsFromSchema(schema).map((f) => f.name));
310+
218311
// 合并驱动 JSON 字段
219312
const boolKeys = new Set(["path_style", "tls_insecure_skip_verify"]);
220313
for (const [k, v] of Object.entries(updateData)) {
221314
if (["name", "storage_type", "is_public", "is_default", "status", "remark", "url_proxy"].includes(k)) continue;
222-
if (k === "access_key_id") {
223-
cfg.access_key_id = await encryptValue(v, encryptionSecret);
224-
} else if (k === "secret_access_key") {
225-
cfg.secret_access_key = await encryptValue(v, encryptionSecret);
226-
} else if (k === "password") {
227-
cfg.password = await encryptValue(v, encryptionSecret);
228-
} else if (k === "total_storage_bytes") {
315+
316+
// secret:空值/掩码占位符 -> 不提交(保留原值);真实值 -> 加密写入
317+
if (secretFieldSet.has(k)) {
318+
if (shouldSkipSecretWrite(v, { allowEmpty: true })) {
319+
continue;
320+
}
321+
// shouldSkipSecretWrite 已涵盖 masked,这里再次兜底
322+
if (isMaskedSecretPlaceholder(v)) {
323+
continue;
324+
}
325+
cfg[k] = await encryptValue(String(v), encryptionSecret);
326+
driverConfigChanged = true;
327+
continue;
328+
}
329+
330+
driverConfigChanged = true;
331+
if (k === "total_storage_bytes") {
229332
const val = parseInt(v, 10);
230333
cfg.total_storage_bytes = Number.isFinite(val) && val > 0 ? val : null;
231334
} else if (k === "signature_expires_in") {
@@ -264,6 +367,23 @@ export async function updateStorageConfig(db, id, updateData, adminId, encryptio
264367
const mountManager = new MountManager(db, encryptionSecret, factory);
265368
await mountManager.clearConfigCache(exists.storage_type, id);
266369
} catch {}
370+
371+
// 如果驱动配置变化:把依赖此存储配置的挂载点索引清掉 + 标记 not_ready
372+
if (driverConfigChanged) {
373+
try {
374+
const mountRepo = factory.getMountRepository();
375+
const mounts = await mountRepo.findByStorageConfig(id, exists.storage_type);
376+
const store = new FsSearchIndexStore(db);
377+
for (const m of mounts || []) {
378+
const mountId = String(m?.id || "").trim();
379+
if (!mountId) continue;
380+
await store.clearDerivedByMount(mountId, { keepState: true });
381+
}
382+
invalidateFsCache({ storageConfigId: id, reason: "storage-config-update", bumpMountsVersion: true, db });
383+
} catch (e) {
384+
console.warn("updateStorageConfig: 清理相关挂载点索引失败(将继续返回更新成功):", e);
385+
}
386+
}
267387
}
268388

269389
export async function deleteStorageConfig(db, id, adminId, repositoryFactory = null) {
@@ -283,7 +403,25 @@ export async function deleteStorageConfig(db, id, adminId, repositoryFactory = n
283403
}
284404
}
285405

406+
// 删除存储配置前:先把依赖它的挂载点与索引派生数据清掉
407+
// 否则会留下“挂载点指向不存在的存储配置”+ “索引残留”的脏数据
408+
try {
409+
const mountRepo = factory.getMountRepository();
410+
const mounts = await mountRepo.findByStorageConfig(id, exists.storage_type);
411+
const store = new FsSearchIndexStore(db);
412+
for (const m of mounts || []) {
413+
const mountId = String(m?.id || "").trim();
414+
if (!mountId) continue;
415+
await store.clearDerivedByMount(mountId, { keepState: false });
416+
await mountRepo.deleteMount(mountId);
417+
invalidateFsCache({ mountId, storageConfigId: id, reason: "mount-delete-by-storage-config", bumpMountsVersion: true, db });
418+
}
419+
} catch (error) {
420+
console.warn("deleteStorageConfig: 清理挂载点/索引失败,将继续删除存储配置本身:", error);
421+
}
422+
286423
await repo.deleteConfig(id);
424+
invalidateFsCache({ storageConfigId: id, reason: "storage-config-delete", bumpMountsVersion: true, db });
287425
}
288426

289427
export async function setDefaultStorageConfig(db, id, adminId, repositoryFactory = null) {

backend/src/services/storageMountService.js

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import { ValidationError, ConflictError, NotFoundError, AuthorizationError } fro
66
import { generateUUID } from "../utils/common.js";
77
import { MountRepository, StorageConfigRepository } from "../repositories/index.js";
88
import { invalidateFsCache } from "../cache/invalidation.js";
9+
import { FsSearchIndexStore } from "../storage/fs/search/FsSearchIndexStore.js";
910

1011
const emitMountCacheInvalidation = ({ mountId, storageConfigId = null, reason, db = null }) => {
1112
if (!mountId && !storageConfigId) {
@@ -205,6 +206,19 @@ class MountService {
205206
await this.mountRepository.updateMount(mountId, updateData);
206207

207208
const updatedMount = await this.mountRepository.findById(mountId);
209+
210+
// 如果“挂载实际指向的内容”发生变化,旧索引就一定不可信了:
211+
// - 改 mount_path:索引里存的是旧路径前缀,会导致搜索结果路径错乱
212+
// - 改 storage_config_id / storage_type:等于换了一个“数据源”,必须清掉旧索引
213+
const shouldResetIndex =
214+
(updateData.mount_path && updateData.mount_path !== existingMount.mount_path) ||
215+
(updateData.storage_config_id && updateData.storage_config_id !== existingMount.storage_config_id) ||
216+
(updateData.storage_type && updateData.storage_type !== existingMount.storage_type);
217+
if (shouldResetIndex) {
218+
const store = new FsSearchIndexStore(this.db);
219+
await store.clearDerivedByMount(mountId, { keepState: true });
220+
}
221+
208222
emitMountCacheInvalidation({ mountId, storageConfigId: updatedMount?.storage_config_id || null, reason: "mount-update", db: this.db });
209223

210224
// 返回更新后的挂载点信息
@@ -231,7 +245,12 @@ class MountService {
231245
throw new AuthorizationError("没有权限删除此挂载点");
232246
}
233247

234-
// 删除挂载点
248+
// 先清理索引派生数据,再删挂载点(避免删成功但索引残留)
249+
// - entries/dirty/state 都属于派生数据,不影响真实文件数据
250+
const store = new FsSearchIndexStore(this.db);
251+
await store.clearDerivedByMount(mountId, { keepState: false });
252+
253+
// 删除挂载点(真实业务数据)
235254
const result = await this.mountRepository.deleteMount(mountId);
236255

237256
emitMountCacheInvalidation({ mountId, storageConfigId: existingMount.storage_config_id || null, reason: "mount-delete", db: this.db });

0 commit comments

Comments
 (0)