Skip to content

Commit 16d17b8

Browse files
authored
feat(server): fallback store media locally (#544)
1 parent 94230f5 commit 16d17b8

File tree

11 files changed

+131
-14
lines changed

11 files changed

+131
-14
lines changed

README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,8 @@ docker compose -f docker-compose.yml up -d
110110

111111
### 使用 Docker Image
112112

113+
若未配置 MinIO 相关参数,媒体文件将默认保存至本地的 `data/media` 目录。
114+
113115
```bash
114116
docker run -d --name telegram-search -p 3333:3333 ghcr.io/groupultra/telegram-search:latest
115117
```
@@ -119,8 +121,6 @@ docker run -d --name telegram-search -p 3333:3333 ghcr.io/groupultra/telegram-se
119121
> [!IMPORTANT]
120122
> AI Embedding & LLM 设置现在在应用内**按账户**配置(设置 → API)。
121123
>
122-
> PGLite 因为性能原因,未来会被弃用,推荐使用 PostgreSQL。
123-
>
124124
> 请在修改完成 `.env` 文件后,再次执行 `docker compose -f docker-compose.yml up -d` 启动服务。
125125
126126
以下环境变量全部为可选,如果不填写,则会使用默认值。

apps/server/src/account.ts

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import { createCoreInstance } from '@tg-search/core'
77
import { coreMessageBatchesProcessedTotal, coreMessagesProcessedTotal, coreMetrics, withSpan } from '@tg-search/observability'
88

99
import { getDB } from './storage/drizzle'
10-
import { getMinioMediaStorage } from './storage/minio'
10+
import { getMediaStorage } from './storage/media'
1111

1212
/**
1313
* Account state - one per Telegram account
@@ -133,7 +133,7 @@ export function getOrCreateAccount(accountId: string, config: Config): AccountSt
133133
if (!accountStates.has(accountId)) {
134134
logger.withFields({ accountId }).log('Creating new account state')
135135

136-
const ctx = createCoreInstance(getDB, config, getMinioMediaStorage(), logger, coreMetrics)
136+
const ctx = createCoreInstance(getDB, config, getMediaStorage(), logger, coreMetrics)
137137

138138
bindTracingMetaToSpan(ctx.emitter)
139139

apps/server/src/index.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import pkg from '../package.json' with { type: 'json' }
1818
import { v1api } from './apis/v1'
1919
import { setupWsRoutes } from './app'
2020
import { getDB, initDrizzle } from './storage/drizzle'
21-
import { getMinioMediaStorage, initMinioMediaStorage } from './storage/minio'
21+
import { getMediaStorage, initMediaStorage } from './storage/media'
2222
import { removeHyperLinks, toSnakeCaseFields } from './utils/fields'
2323

2424
function setupErrorHandlers(logger: Logger): void {
@@ -63,7 +63,7 @@ function configureServer(logger: Logger, flags: RuntimeFlags, config: Config) {
6363
return Response.json({ success: true })
6464
}))
6565

66-
app.mount('/v1', v1api(getDB(), models, getMinioMediaStorage()))
66+
app.mount('/v1', v1api(getDB(), models, getMediaStorage()))
6767

6868
setupWsRoutes(app, config)
6969

@@ -106,7 +106,7 @@ async function bootstrap() {
106106

107107
await initDrizzle(logger, config, flags)
108108

109-
await initMinioMediaStorage(logger, config.minio)
109+
await initMediaStorage(logger, config)
110110

111111
setupErrorHandlers(logger)
112112

apps/server/src/storage/local.ts

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
import type { Logger } from '@guiiai/logg'
2+
import type { MediaConfig } from '@tg-search/common'
3+
import type { MediaBinaryDescriptor, MediaBinaryLocation, MediaBinaryProvider } from '@tg-search/core'
4+
5+
import { mkdir, readFile, writeFile } from 'node:fs/promises'
6+
7+
import { useDataPath } from '@tg-search/common/node'
8+
import { dirname, join, posix, resolve, sep } from 'pathe'
9+
10+
const DEFAULT_MEDIA_DIR = resolve(useDataPath(), 'media')
11+
12+
let localMediaStorage: MediaBinaryProvider | undefined
13+
14+
function buildRelativePath(descriptor: MediaBinaryDescriptor): string {
15+
return posix.join(descriptor.kind, descriptor.uuid)
16+
}
17+
18+
function resolveLocationPath(baseDir: string, locationPath: string): string | null {
19+
const root = resolve(baseDir)
20+
const resolved = resolve(baseDir, locationPath)
21+
22+
if (resolved === root || resolved.startsWith(`${root}${sep}`)) {
23+
return resolved
24+
}
25+
26+
return null
27+
}
28+
29+
export async function initLocalMediaStorage(logger: Logger, config?: MediaConfig): Promise<MediaBinaryProvider | undefined> {
30+
const baseDir = config?.dir ? resolve(config.dir) : DEFAULT_MEDIA_DIR
31+
32+
try {
33+
await mkdir(baseDir, { recursive: true })
34+
}
35+
catch (error) {
36+
logger.withError(error).warn('Failed to ensure local media directory; falling back to database storage for media')
37+
return undefined
38+
}
39+
40+
const provider: MediaBinaryProvider = {
41+
async save(descriptor: MediaBinaryDescriptor, bytes: Uint8Array, _mimeType?: string): Promise<MediaBinaryLocation> {
42+
const relativePath = buildRelativePath(descriptor)
43+
const absolutePath = join(baseDir, relativePath)
44+
45+
await mkdir(dirname(absolutePath), { recursive: true })
46+
await writeFile(absolutePath, bytes)
47+
48+
return {
49+
kind: descriptor.kind,
50+
path: relativePath,
51+
}
52+
},
53+
54+
async load(location: MediaBinaryLocation): Promise<Uint8Array | null> {
55+
const resolvedPath = resolveLocationPath(baseDir, location.path)
56+
if (!resolvedPath) {
57+
logger.withFields({ path: location.path }).warn('Rejected local media path outside base directory')
58+
return null
59+
}
60+
61+
try {
62+
const buffer = await readFile(resolvedPath)
63+
return new Uint8Array(buffer)
64+
}
65+
catch (error) {
66+
if (error instanceof Error && 'code' in error && (error as NodeJS.ErrnoException).code === 'ENOENT') {
67+
return null
68+
}
69+
70+
logger.withError(error).warn('Failed to load media from local storage; returning null')
71+
return null
72+
}
73+
},
74+
}
75+
76+
localMediaStorage = provider
77+
logger.withFields({ baseDir }).log('Local media storage provider registered')
78+
return provider
79+
}
80+
81+
export function getLocalMediaStorage(): MediaBinaryProvider | undefined {
82+
return localMediaStorage
83+
}

apps/server/src/storage/media.ts

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
import type { Logger } from '@guiiai/logg'
2+
import type { Config } from '@tg-search/common'
3+
import type { MediaBinaryProvider } from '@tg-search/core'
4+
5+
import { initLocalMediaStorage } from './local'
6+
import { initMinioMediaStorage } from './minio'
7+
8+
let mediaStorage: MediaBinaryProvider | undefined
9+
10+
export async function initMediaStorage(logger: Logger, config: Config): Promise<MediaBinaryProvider | undefined> {
11+
mediaStorage = await initMinioMediaStorage(logger, config.minio)
12+
13+
if (!mediaStorage) {
14+
mediaStorage = await initLocalMediaStorage(logger, config.media)
15+
}
16+
17+
return mediaStorage
18+
}
19+
20+
export function getMediaStorage(): MediaBinaryProvider | undefined {
21+
return mediaStorage
22+
}

apps/server/src/storage/minio.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ export async function registerMinioMediaStorage(logger: Logger, dsn: string, acc
6060
}
6161
}
6262
catch (error) {
63-
logger.withError(error).warn('Failed to ensure MinIO bucket; falling back to DB bytea for media')
63+
logger.withError(error).warn('Failed to ensure MinIO bucket; falling back to local media storage')
6464
return
6565
}
6666

@@ -110,7 +110,7 @@ export async function registerMinioMediaStorage(logger: Logger, dsn: string, acc
110110
/**
111111
* Attempt to register MinIO-based media storage. When configuration is
112112
* incomplete or MinIO is unavailable we log a warning and gracefully
113-
* fall back to storing media bytes in the database.
113+
* fall back to storing media bytes on local disk.
114114
*/
115115
export async function initMinioMediaStorage(logger: Logger, config: MinioConfig): Promise<MediaBinaryProvider | undefined> {
116116
try {
@@ -122,7 +122,7 @@ export async function initMinioMediaStorage(logger: Logger, config: MinioConfig)
122122
return minioMediaStorage
123123
}
124124
catch (error) {
125-
logger.withError(error).warn('Failed to register MinIO media storage; falling back to DB bytea')
125+
logger.withError(error).warn('Failed to register MinIO media storage; falling back to local media storage')
126126
}
127127
}
128128

docs/README_EN.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,8 @@ docker compose -f docker-compose.yml up -d
115115

116116
### Deploy with Docker Image
117117

118+
If MinIO is not configured, media files will be stored in the local `data/media` directory.
119+
118120
```bash
119121
docker run -d --name telegram-search -p 3333:3333 ghcr.io/groupultra/telegram-search:latest
120122
```
@@ -123,8 +125,6 @@ docker run -d --name telegram-search -p 3333:3333 ghcr.io/groupultra/telegram-se
123125

124126
> [!IMPORTANT]
125127
> AI Embedding & LLM settings are now **per-account** in-app (Settings → API).
126-
>
127-
> PGLite will be deprecated in the future due to performance reasons. PostgreSQL is recommended.
128128
>
129129
> Please restart the service after modifying the `.env` file by running `docker compose -f docker-compose.yml up -d`.
130130

docs/README_JA.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,8 @@ docker compose -f docker-compose.yml up -d
106106

107107
### Docker Image を使用してデプロイ
108108

109+
MinIO の関連パラメータが未設定の場合、メディアファイルはデフォルトでローカルの `data/media` ディレクトリに保存されます。
110+
109111
```bash
110112
docker run -d --name telegram-search -p 3333:3333 ghcr.io/groupultra/telegram-search:latest
111113
```
@@ -114,8 +116,6 @@ docker run -d --name telegram-search -p 3333:3333 ghcr.io/groupultra/telegram-se
114116

115117
> [!IMPORTANT]
116118
> AI 埋め込み & LLM の設定は現在「アカウントごとに」アプリ内で設定します(設定 → API)。
117-
>
118-
> PGLite は性能の問題で将来廃止される予定です。PostgreSQL を推奨します。
119119
>
120120
> 変更が完了した `.env` ファイルを再度 `docker compose -f docker-compose.yml up -d` で起動してください。
121121

packages/common/src/__test__/env.test.ts

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,9 @@ describe('mergeConfigWithEnv', () => {
129129
apiId: 'from-config',
130130
},
131131
},
132+
media: {
133+
dir: 'data/media',
134+
},
132135
}
133136

134137
const env = createEnv({

packages/common/src/config-schema.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,10 @@ export const minioConfigSchema = object({
8888
useSSL: optional(boolean(), false),
8989
})
9090

91+
export const mediaConfigSchema = object({
92+
dir: optional(string()),
93+
})
94+
9195
export const apiConfigSchema = object({
9296
telegram: optional(telegramConfigSchema, {}),
9397
})
@@ -96,12 +100,14 @@ export const configSchema = object({
96100
database: optional(databaseConfigSchema, {}),
97101
api: optional(apiConfigSchema, {}),
98102
minio: optional(minioConfigSchema, {}),
103+
media: optional(mediaConfigSchema, {}),
99104
})
100105

101106
export type Config = InferOutput<typeof configSchema>
102107
export type ProxyConfig = InferOutput<typeof proxyConfigSchema>
103108
export type DatabaseConfig = InferOutput<typeof databaseConfigSchema>
104109
export type MinioConfig = InferOutput<typeof minioConfigSchema>
110+
export type MediaConfig = InferOutput<typeof mediaConfigSchema>
105111

106112
export function generateDefaultConfig(): Config {
107113
const defaultConfig = safeParse(configSchema, {})

0 commit comments

Comments
 (0)