Skip to content

Commit 15d06ec

Browse files
HenryHengZJHenryHengZJ
authored
Feature/Allow winston to stream to s3 (#3848)
allow winston to stream to s3
1 parent a2d5cf9 commit 15d06ec

File tree

3 files changed

+131
-33
lines changed

3 files changed

+131
-33
lines changed

‎packages/server/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,7 @@
9797
"posthog-node": "^3.5.0",
9898
"prom-client": "^15.1.3",
9999
"reflect-metadata": "^0.1.13",
100+
"s3-streamlogger": "^1.11.0",
100101
"sanitize-html": "^2.11.0",
101102
"socket.io": "^4.6.1",
102103
"sqlite3": "^5.1.6",

‎packages/server/src/utils/logger.ts‎

Lines changed: 102 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,51 @@
11
import * as path from 'path'
22
import * as fs from 'fs'
3+
import { hostname } from 'node:os'
34
import config from './config' // should be replaced by node-config or similar
45
import { createLogger, transports, format } from 'winston'
56
import { NextFunction, Request, Response } from 'express'
67

8+
const { S3StreamLogger } = require('s3-streamlogger')
9+
710
const { combine, timestamp, printf, errors } = format
811

12+
let s3ServerStream: any
13+
let s3ErrorStream: any
14+
let s3ServerReqStream: any
15+
if (process.env.STORAGE_TYPE === 's3') {
16+
const accessKeyId = process.env.S3_STORAGE_ACCESS_KEY_ID
17+
const secretAccessKey = process.env.S3_STORAGE_SECRET_ACCESS_KEY
18+
const region = process.env.S3_STORAGE_REGION
19+
const s3Bucket = process.env.S3_STORAGE_BUCKET_NAME
20+
21+
s3ServerStream = new S3StreamLogger({
22+
bucket: s3Bucket,
23+
folder: 'logs/server',
24+
region,
25+
access_key_id: accessKeyId,
26+
secret_access_key: secretAccessKey,
27+
name_format: `server-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log`
28+
})
29+
30+
s3ErrorStream = new S3StreamLogger({
31+
bucket: s3Bucket,
32+
folder: 'logs/error',
33+
region,
34+
access_key_id: accessKeyId,
35+
secret_access_key: secretAccessKey,
36+
name_format: `server-error-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log`
37+
})
38+
39+
s3ServerReqStream = new S3StreamLogger({
40+
bucket: s3Bucket,
41+
folder: 'logs/requests',
42+
region,
43+
access_key_id: accessKeyId,
44+
secret_access_key: secretAccessKey,
45+
name_format: `server-requests-%Y-%m-%d-%H-%M-%S-%L-${hostname()}.log.jsonl`
46+
})
47+
}
48+
949
// expect the log dir be relative to the projects root
1050
const logDir = config.logging.dir
1151

@@ -29,33 +69,60 @@ const logger = createLogger({
2969
},
3070
transports: [
3171
new transports.Console(),
32-
new transports.File({
33-
filename: path.join(logDir, config.logging.server.filename ?? 'server.log'),
34-
level: config.logging.server.level ?? 'info'
35-
}),
36-
new transports.File({
37-
filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log'),
38-
level: 'error' // Log only errors to this file
39-
})
72+
...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
73+
? [
74+
new transports.File({
75+
filename: path.join(logDir, config.logging.server.filename ?? 'server.log'),
76+
level: config.logging.server.level ?? 'info'
77+
}),
78+
new transports.File({
79+
filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log'),
80+
level: 'error' // Log only errors to this file
81+
})
82+
]
83+
: []),
84+
...(process.env.STORAGE_TYPE === 's3'
85+
? [
86+
new transports.Stream({
87+
stream: s3ServerStream
88+
})
89+
]
90+
: [])
4091
],
4192
exceptionHandlers: [
42-
new transports.File({
43-
filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log')
44-
})
93+
...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
94+
? [
95+
new transports.File({
96+
filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log')
97+
})
98+
]
99+
: []),
100+
...(process.env.STORAGE_TYPE === 's3'
101+
? [
102+
new transports.Stream({
103+
stream: s3ErrorStream
104+
})
105+
]
106+
: [])
45107
],
46108
rejectionHandlers: [
47-
new transports.File({
48-
filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log')
49-
})
109+
...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
110+
? [
111+
new transports.File({
112+
filename: path.join(logDir, config.logging.server.errorFilename ?? 'server-error.log')
113+
})
114+
]
115+
: []),
116+
...(process.env.STORAGE_TYPE === 's3'
117+
? [
118+
new transports.Stream({
119+
stream: s3ErrorStream
120+
})
121+
]
122+
: [])
50123
]
51124
})
52125

53-
/**
54-
* This function is used by express as a middleware.
55-
* @example
56-
* this.app = express()
57-
* this.app.use(expressRequestLogger)
58-
*/
59126
export function expressRequestLogger(req: Request, res: Response, next: NextFunction): void {
60127
const unwantedLogURLs = ['/api/v1/node-icon/', '/api/v1/components-credentials-icon/']
61128
if (/\/api\/v1\//i.test(req.url) && !unwantedLogURLs.some((url) => new RegExp(url, 'i').test(req.url))) {
@@ -73,10 +140,21 @@ export function expressRequestLogger(req: Request, res: Response, next: NextFunc
73140
}
74141
},
75142
transports: [
76-
new transports.File({
77-
filename: path.join(logDir, config.logging.express.filename ?? 'server-requests.log.jsonl'),
78-
level: config.logging.express.level ?? 'debug'
79-
})
143+
...(!process.env.STORAGE_TYPE || process.env.STORAGE_TYPE === 'local'
144+
? [
145+
new transports.File({
146+
filename: path.join(logDir, config.logging.express.filename ?? 'server-requests.log.jsonl'),
147+
level: config.logging.express.level ?? 'debug'
148+
})
149+
]
150+
: []),
151+
...(process.env.STORAGE_TYPE === 's3'
152+
? [
153+
new transports.Stream({
154+
stream: s3ServerReqStream
155+
})
156+
]
157+
: [])
80158
]
81159
})
82160

0 commit comments

Comments
 (0)