Skip to content

Commit f37312f

Browse files
authored
Merge branch 'ling-drag0n:main' into main
2 parents 6184a97 + bd653d7 commit f37312f

File tree

100 files changed

+7260
-3924
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

100 files changed

+7260
-3924
lines changed

README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@
6161
### Multi-Storage Support
6262

6363
- **S3 Compatible**: Cloudflare R2, Backblaze B2, AWS S3, Alibaba Cloud OSS, Tencent Cloud COS, MinIO, etc.
64-
- **Cloud Storage Integration**: WebDAV, OneDrive, Google Drive, Telegram, HuggingFace Database, GitHub API/Releases (read-only), etc.
64+
- **Cloud Storage Integration**: WebDAV, OneDrive, Google Drive, Telegram, Discord Bot, HuggingFace Database, GitHub API/Releases (read-only), etc.
6565
- **Local Storage**: Docker deployment supports local file system
6666
- **Smart Upload**: Frontend pre-signed direct upload + streaming upload + chunked resumable upload, with real-time progress display, minimizing CF limitations
6767
- **File Preview**: Direct preview support for 30+ formats (images, videos, audio, PDF, Office, code, e-books, etc.), others can be previewed through external IFrame embedding [KKFileview](https://github.com/kekingcn/kkFileView)

README_CN.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@
6161
### 多存储支持
6262

6363
- **S3 兼容**:Cloudflare R2、Backblaze B2、AWS S3、阿里云 OSS、腾讯云 COS、MinIO 等
64-
- **网盘集成**:WebDAV、OneDrive、Google Drive、Telegram、HuggingFace Database、GitHub API/Releases(只读)等等
64+
- **网盘集成**:WebDAV、OneDrive、Google Drive、Telegram、Discord Bot、HuggingFace Database、GitHub API/Releases(只读)等等
6565
- **本地存储**:Docker 部署支持本地文件系统
6666
- **智能上传**:前端预签名直传 + 流式上传 +分片断点续传,进度实时显示,最大限度摆脱cf限制
6767
- **文件预览**:支持30+种格式直接预览(图片、视频、音频、PDF、Office、代码、电子书等),其余可通过外部IFrame嵌入[KKFileview](https://github.com/kekingcn/kkFileView)预览

backend/package.json

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
{
22
"name": "cloudpaste-api",
3-
"version": "1.9.0",
3+
"version": "1.9.1",
44
"description": "CloudPaste API基于Cloudflare Workers和D1数据库",
55
"main": "unified-entry.js",
66
"scripts": {
7+
"test": "node --test --test-concurrency=1",
8+
"test:parallel": "node --test",
79
"dev": "wrangler dev --local",
810
"dev:scheduled": "wrangler dev --local --test-scheduled",
911
"deploy": "wrangler publish",

backend/schema.sql

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ DROP TABLE IF EXISTS fs_search_index_entries;
2525
DROP TABLE IF EXISTS fs_search_index_state;
2626
DROP TABLE IF EXISTS fs_search_index_dirty;
2727
DROP TABLE IF EXISTS fs_search_index_fts;
28+
DROP TABLE IF EXISTS metrics_cache;
2829

2930
-- 创建pastes表 - 存储文本分享数据
3031
CREATE TABLE pastes (
@@ -541,3 +542,24 @@ CREATE TABLE scheduled_job_runs (
541542

542543
CREATE INDEX idx_scheduled_job_runs_task_started
543544
ON scheduled_job_runs (task_id, started_at DESC);
545+
546+
-- 通用指标缓存表:存储各种统计指标的缓存数据
547+
CREATE TABLE metrics_cache (
548+
scope_type TEXT NOT NULL, -- 作用域类型,如 'global'、'storage'、'user' 等
549+
scope_id TEXT NOT NULL, -- 作用域标识,如具体的存储ID或用户ID
550+
metric_key TEXT NOT NULL, -- 指标键名
551+
552+
value_num INTEGER, -- 数值类型的指标值
553+
value_text TEXT, -- 文本类型的指标值
554+
value_json_text TEXT, -- JSON格式的复杂指标值
555+
556+
snapshot_at_ms INTEGER, -- 快照时间戳(毫秒)
557+
updated_at_ms INTEGER NOT NULL, -- 最后更新时间戳(毫秒)
558+
559+
error_message TEXT, -- 错误信息(如果指标计算失败)
560+
561+
PRIMARY KEY (scope_type, scope_id, metric_key)
562+
);
563+
564+
-- 通用指标缓存索引:按作用域类型和ID查询
565+
CREATE INDEX idx_metrics_cache_scope ON metrics_cache(scope_type, scope_id);

backend/src/constants/index.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ export const DbTables = {
2020
FS_SEARCH_INDEX_STATE: "fs_search_index_state", // FS 搜索索引(状态表,派生数据)
2121
FS_SEARCH_INDEX_FTS: "fs_search_index_fts", // FS 搜索索引(FTS5 虚表,派生数据)
2222
FS_SEARCH_INDEX_DIRTY: "fs_search_index_dirty", // FS 搜索索引(dirty 队列表,派生数据)
23+
METRICS_CACHE: "metrics_cache", // 通用指标缓存表(快照/用量/配额等派生数据)
2324
TASKS: "tasks", // 任务编排表
2425
SCHEMA_MIGRATIONS: "schema_migrations", // 迁移历史表(用于记录 schema 版本)
2526
SCHEDULED_JOBS: "scheduled_jobs", // 后台调度作业表

backend/src/db/migrations/sqlite/engine/initDatabase.js

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ import {
1111
createStorageTables,
1212
createSystemTables,
1313
createTasksTables,
14+
createMetricsCacheTables,
1415
createUploadSessionsTables,
1516
createUploadPartsTables,
1617
createVfsTables,
@@ -46,6 +47,7 @@ export async function initDatabase(db) {
4647
await createScheduledJobRunsTables(db);
4748
await createUploadSessionsTables(db);
4849
await createVfsTables(db);
50+
await createMetricsCacheTables(db);
4951
await createUploadPartsTables(db);
5052

5153
await createIndexes(db);

backend/src/db/migrations/sqlite/engine/migrations.js

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
import { DbTables } from "../../../../constants/index.js";
22
import { LegacyDbTables } from "./legacyKeys.js";
3+
import { StorageFactory } from "../../../../storage/factory/StorageFactory.js";
4+
import { toBool } from "../../../../utils/environmentUtils.js";
35
import {
46
createFsMetaTables,
57
createFsSearchIndexTables,
@@ -9,6 +11,7 @@ import {
911
createUploadPartsTables,
1012
createUploadSessionsTables,
1113
createVfsTables,
14+
createMetricsCacheTables,
1215
} from "./schema.js";
1316
import {
1417
addCustomContentSettings,
@@ -24,6 +27,98 @@ import {
2427
*
2528
*/
2629

30+
function getBooleanFieldNamesFromStorageSchema(storageType) {
31+
if (!storageType) return [];
32+
const meta = StorageFactory.getTypeMetadata(storageType);
33+
const fields = meta?.configSchema?.fields;
34+
if (!Array.isArray(fields)) return [];
35+
return fields
36+
.filter((f) => f && typeof f === "object" && typeof f.name === "string" && (f.type === "boolean" || f.type === "bool"))
37+
.map((f) => f.name)
38+
.filter(Boolean);
39+
}
40+
41+
function coerceConfigJsonBooleans(storageType, configJsonObj) {
42+
if (!configJsonObj || typeof configJsonObj !== "object") {
43+
return { changed: false, next: configJsonObj };
44+
}
45+
const boolKeys = getBooleanFieldNamesFromStorageSchema(storageType);
46+
if (!boolKeys.length) {
47+
return { changed: false, next: configJsonObj };
48+
}
49+
50+
let changed = false;
51+
for (const key of boolKeys) {
52+
if (!Object.prototype.hasOwnProperty.call(configJsonObj, key)) continue;
53+
const raw = configJsonObj[key];
54+
if (raw === undefined || raw === null) continue;
55+
const nextVal = toBool(raw, false) ? 1 : 0;
56+
if (configJsonObj[key] !== nextVal) {
57+
configJsonObj[key] = nextVal;
58+
changed = true;
59+
}
60+
}
61+
62+
return { changed, next: configJsonObj };
63+
}
64+
65+
async function normalizeStorageConfigsBooleanFields(db) {
66+
console.log("版本34:开始归一化 storage_configs.config_json 中的布尔字段(统一为 0/1)...");
67+
68+
let rows = [];
69+
try {
70+
const res = await db
71+
.prepare(`SELECT id, storage_type, config_json FROM ${DbTables.STORAGE_CONFIGS} ORDER BY updated_at DESC`)
72+
.all();
73+
rows = Array.isArray(res?.results) ? res.results : [];
74+
} catch (e) {
75+
console.warn("版本34:读取 storage_configs 失败,跳过布尔字段归一化:", e?.message || e);
76+
return { total: 0, updated: 0, failed: 0, skipped: 0 };
77+
}
78+
79+
let updated = 0;
80+
let skipped = 0;
81+
let failed = 0;
82+
83+
for (const row of rows) {
84+
const id = row?.id ? String(row.id) : "";
85+
const storageType = row?.storage_type ? String(row.storage_type) : "";
86+
const rawJson = row?.config_json;
87+
if (!id || !storageType || !rawJson) {
88+
skipped += 1;
89+
continue;
90+
}
91+
92+
let cfgObj;
93+
try {
94+
cfgObj = typeof rawJson === "string" ? JSON.parse(rawJson) : rawJson;
95+
} catch {
96+
failed += 1;
97+
continue;
98+
}
99+
100+
const { changed, next } = coerceConfigJsonBooleans(storageType, cfgObj);
101+
if (!changed) {
102+
skipped += 1;
103+
continue;
104+
}
105+
106+
try {
107+
await db
108+
.prepare(`UPDATE ${DbTables.STORAGE_CONFIGS} SET config_json = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?`)
109+
.bind(JSON.stringify(next), id)
110+
.run();
111+
updated += 1;
112+
} catch (e) {
113+
failed += 1;
114+
console.warn("版本34:归一化 storage_config 失败:", { id, storageType, error: e?.message || e });
115+
}
116+
}
117+
118+
console.log("版本34:storage_configs 布尔字段归一化完成:", { total: rows.length, updated, skipped, failed });
119+
return { total: rows.length, updated, skipped, failed };
120+
}
121+
27122
export async function addTableField(db, tableName, fieldName, fieldDefinition) {
28123
try {
29124
const columnInfo = await db.prepare(`PRAGMA table_info(${tableName})`).all();
@@ -689,6 +784,51 @@ export async function runLegacyMigrationByVersion(db, version) {
689784
}
690785
break;
691786

787+
case 34: {
788+
console.log("版本34:新增 metrics_cache(用量快照缓存)+ 默认快照刷新任务...");
789+
try {
790+
await createMetricsCacheTables(db);
791+
} catch (e) {
792+
console.warn("版本34:创建 metrics_cache 失败(可忽略,后续会再次尝试):", e?.message || e);
793+
}
794+
try {
795+
const intervalSec = 6 * 60 * 60;
796+
const nextRunAfterIso = new Date(Date.now() + intervalSec * 1000).toISOString();
797+
await db
798+
.prepare(
799+
`
800+
INSERT INTO ${DbTables.SCHEDULED_JOBS} (task_id, handler_id, name, description, enabled, schedule_type, interval_sec, next_run_after, config_json)
801+
SELECT ?, ?, ?, ?, 1, 'interval', ?, ?, ?
802+
WHERE NOT EXISTS (
803+
SELECT 1 FROM ${DbTables.SCHEDULED_JOBS} WHERE task_id = ?
804+
)
805+
`,
806+
)
807+
.bind(
808+
"refresh_storage_usage_snapshots",
809+
"refresh_storage_usage_snapshots",
810+
"刷新存储用量快照(默认)",
811+
"定期刷新存储用量数据(已用/总量)。用于上传容量限制判断与管理端展示。",
812+
intervalSec,
813+
nextRunAfterIso,
814+
JSON.stringify({ maxItems: 50, maxConcurrency: 1 }),
815+
"refresh_storage_usage_snapshots",
816+
)
817+
.run();
818+
} catch (e) {
819+
console.warn("版本34:写入默认 refresh_storage_usage_snapshots 任务失败(可忽略):", e?.message || e);
820+
}
821+
822+
// 归一化 storage_configs.config_json 中的布尔字段(统一为 0/1)
823+
try {
824+
await normalizeStorageConfigsBooleanFields(db);
825+
} catch (e) {
826+
console.warn("版本34:归一化 storage_configs 布尔字段失败(可忽略,将由后续保存配置逐步修复):", e?.message || e);
827+
}
828+
829+
break;
830+
}
831+
692832
default:
693833
console.log(`未知的迁移版本: ${version}`);
694834
break;

backend/src/db/migrations/sqlite/engine/schema.js

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -642,6 +642,38 @@ export async function createVfsTables(db) {
642642
console.log("vfs_nodes 表检查/创建完成");
643643
}
644644

645+
export async function createMetricsCacheTables(db) {
646+
console.log("创建通用指标缓存表(metrics_cache)...");
647+
648+
await db
649+
.prepare(
650+
`
651+
CREATE TABLE IF NOT EXISTS ${DbTables.METRICS_CACHE} (
652+
scope_type TEXT NOT NULL,
653+
scope_id TEXT NOT NULL,
654+
metric_key TEXT NOT NULL,
655+
656+
value_num INTEGER,
657+
value_text TEXT,
658+
value_json_text TEXT,
659+
660+
snapshot_at_ms INTEGER,
661+
updated_at_ms INTEGER NOT NULL,
662+
663+
error_message TEXT,
664+
665+
PRIMARY KEY (scope_type, scope_id, metric_key)
666+
)
667+
`,
668+
)
669+
.run();
670+
671+
// 索引
672+
await db.prepare(`CREATE INDEX IF NOT EXISTS idx_metrics_cache_scope ON ${DbTables.METRICS_CACHE}(scope_type, scope_id)`).run();
673+
674+
console.log("metrics_cache 表检查/创建完成");
675+
}
676+
645677
export async function createUploadPartsTables(db) {
646678
console.log("创建上传分片明细表(upload_parts)...");
647679

backend/src/db/migrations/sqlite/engine/seed.js

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,31 @@ export async function initDefaultSettings(db) {
3434
)
3535
.run();
3636

37+
// 为 refresh_storage_usage_snapshots 写入默认调度配置(若不存在)
38+
const refreshIntervalSec = 6 * 60 * 60;
39+
const firstRefreshNextRunIso = new Date(Date.now() + refreshIntervalSec * 1000).toISOString();
40+
await db
41+
.prepare(
42+
`
43+
INSERT INTO ${DbTables.SCHEDULED_JOBS} (task_id, handler_id, name, description, enabled, schedule_type, interval_sec, next_run_after, config_json)
44+
SELECT ?, ?, ?, ?, 1, 'interval', ?, ?, ?
45+
WHERE NOT EXISTS (
46+
SELECT 1 FROM ${DbTables.SCHEDULED_JOBS} WHERE task_id = ?
47+
)
48+
`,
49+
)
50+
.bind(
51+
"refresh_storage_usage_snapshots",
52+
"refresh_storage_usage_snapshots",
53+
"刷新存储用量快照(默认)",
54+
"定期刷新存储用量数据(已用/总量)。用于上传容量限制判断与管理端展示。",
55+
refreshIntervalSec,
56+
firstRefreshNextRunIso,
57+
JSON.stringify({ maxItems: 50, maxConcurrency: 1 }),
58+
"refresh_storage_usage_snapshots",
59+
)
60+
.run();
61+
3762
const defaultSettings = [
3863
{
3964
key: "max_upload_size",

backend/src/db/migrations/sqlite/engine/version.js

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
// - `app-v01..app-vNN` 的 NN 上限来自这里
44
// - 这里的版本号用于“迁移编排”,不与具体数据库方言强绑定
55

6-
export const APP_SCHEMA_VERSION = 33;
6+
export const APP_SCHEMA_VERSION = 34;
77

88
// 兼容命名:历史代码中使用 DB_SCHEMA_VERSION
99
export const DB_SCHEMA_VERSION = APP_SCHEMA_VERSION;

0 commit comments

Comments
 (0)