Skip to content

Commit e77dfb5

Browse files
SuyunmengSuyunmeng
committed
refactor(log):Refactor log filtering to use centralized configuration and add server-specific filtering (#798)
1 parent 9a631ef commit e77dfb5

File tree

4 files changed

+66
-139
lines changed

4 files changed

+66
-139
lines changed

‎cmd/server.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ the address is defined in config file`,
4848
gin.SetMode(gin.ReleaseMode)
4949
}
5050
r := gin.New()
51-
r.Use(middlewares.ConfigurableFilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out))
51+
r.Use(middlewares.HTTPFilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out))
5252
server.Init(r)
5353
var httpHandler http.Handler = r
5454
if conf.Conf.Scheme.EnableH2c {
@@ -103,7 +103,7 @@ the address is defined in config file`,
103103
}
104104
if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable {
105105
s3r := gin.New()
106-
s3r.Use(middlewares.FilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out))
106+
s3r.Use(middlewares.S3FilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out))
107107
server.InitS3(s3r)
108108
s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port)
109109
utils.Log.Infof("start S3 server @ %s", s3Base)

‎internal/conf/config.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,32 @@ type LogConfig struct {
4444
MaxBackups int `json:"max_backups" env:"MAX_BACKUPS"`
4545
MaxAge int `json:"max_age" env:"MAX_AGE"`
4646
Compress bool `json:"compress" env:"COMPRESS"`
47+
Filter LogFilterConfig `json:"filter"` // Log filtering configuration (config file only, no env support)
48+
}
49+
50+
// LogFilterConfig holds configuration for log filtering
51+
// Note: This configuration is only supported via config file, not environment variables
52+
type LogFilterConfig struct {
53+
// EnableFiltering controls whether log filtering is enabled
54+
EnableFiltering bool `json:"enable_filtering"`
55+
56+
// FilterHealthChecks controls whether to filter health check requests
57+
FilterHealthChecks bool `json:"filter_health_checks"`
58+
59+
// FilterWebDAV controls whether to filter WebDAV requests (only for HTTP server)
60+
FilterWebDAV bool `json:"filter_webdav"`
61+
62+
// FilterHEADRequests controls whether to filter HEAD requests
63+
FilterHEADRequests bool `json:"filter_head_requests"`
64+
65+
// CustomSkipPaths allows adding custom paths to skip
66+
CustomSkipPaths []string `json:"custom_skip_paths"`
67+
68+
// CustomSkipMethods allows adding custom methods to skip
69+
CustomSkipMethods []string `json:"custom_skip_methods"`
70+
71+
// CustomSkipPrefixes allows adding custom path prefixes to skip
72+
CustomSkipPrefixes []string `json:"custom_skip_prefixes"`
4773
}
4874

4975
type TaskConfig struct {
@@ -152,6 +178,15 @@ func DefaultConfig(dataDir string) *Config {
152178
MaxSize: 50,
153179
MaxBackups: 30,
154180
MaxAge: 28,
181+
Filter: LogFilterConfig{
182+
EnableFiltering: true,
183+
FilterHealthChecks: true,
184+
FilterWebDAV: true,
185+
FilterHEADRequests: true,
186+
CustomSkipPaths: []string{},
187+
CustomSkipMethods: []string{},
188+
CustomSkipPrefixes: []string{},
189+
},
155190
},
156191
MaxConnections: 0,
157192
MaxConcurrency: 64,

‎server/middlewares/config.go‎

Lines changed: 29 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -1,145 +1,55 @@
11
package middlewares
22

33
import (
4-
"os"
5-
"strings"
6-
4+
"github.com/OpenListTeam/OpenList/v4/internal/conf"
75
"github.com/gin-gonic/gin"
86
log "github.com/sirupsen/logrus"
97
)
108

11-
// LogFilterConfig holds configuration for log filtering
12-
type LogFilterConfig struct {
13-
// EnableFiltering controls whether log filtering is enabled
14-
EnableFiltering bool
15-
16-
// FilterHealthChecks controls whether to filter health check requests
17-
FilterHealthChecks bool
18-
19-
// FilterWebDAV controls whether to filter WebDAV requests
20-
FilterWebDAV bool
21-
22-
// FilterHEADRequests controls whether to filter HEAD requests
23-
FilterHEADRequests bool
24-
25-
// CustomSkipPaths allows adding custom paths to skip
26-
CustomSkipPaths []string
27-
28-
// CustomSkipMethods allows adding custom methods to skip
29-
CustomSkipMethods []string
9+
// UnifiedFilteredLogger returns a filtered logger using global configuration
10+
// serverType: "http" for main HTTP server, "s3" for S3 server
11+
func UnifiedFilteredLogger(serverType string) gin.HandlerFunc {
12+
config := conf.Conf.Log.Filter
3013

31-
// CustomSkipPrefixes allows adding custom path prefixes to skip
32-
CustomSkipPrefixes []string
33-
}
34-
35-
// DefaultLogFilterConfig returns the default configuration
36-
func DefaultLogFilterConfig() LogFilterConfig {
37-
return LogFilterConfig{
38-
EnableFiltering: true,
39-
FilterHealthChecks: true,
40-
FilterWebDAV: true,
41-
FilterHEADRequests: true,
42-
CustomSkipPaths: []string{},
43-
CustomSkipMethods: []string{},
44-
CustomSkipPrefixes: []string{},
45-
}
46-
}
47-
48-
// LoadLogFilterConfigFromEnv loads configuration from environment variables
49-
func LoadLogFilterConfigFromEnv() LogFilterConfig {
50-
config := DefaultLogFilterConfig()
51-
52-
// Check if filtering is enabled
53-
if env := os.Getenv("OPENLIST_LOG_FILTER_ENABLED"); env != "" {
54-
config.EnableFiltering = strings.ToLower(env) == "true"
55-
}
56-
57-
// Check individual filter options
58-
if env := os.Getenv("OPENLIST_LOG_FILTER_HEALTH_CHECKS"); env != "" {
59-
config.FilterHealthChecks = strings.ToLower(env) == "true"
60-
}
61-
62-
if env := os.Getenv("OPENLIST_LOG_FILTER_WEBDAV"); env != "" {
63-
config.FilterWebDAV = strings.ToLower(env) == "true"
64-
}
65-
66-
if env := os.Getenv("OPENLIST_LOG_FILTER_HEAD_REQUESTS"); env != "" {
67-
config.FilterHEADRequests = strings.ToLower(env) == "true"
68-
}
69-
70-
// Load custom skip paths
71-
if env := os.Getenv("OPENLIST_LOG_FILTER_SKIP_PATHS"); env != "" {
72-
config.CustomSkipPaths = strings.Split(env, ",")
73-
for i, path := range config.CustomSkipPaths {
74-
config.CustomSkipPaths[i] = strings.TrimSpace(path)
75-
}
76-
}
77-
78-
// Load custom skip methods
79-
if env := os.Getenv("OPENLIST_LOG_FILTER_SKIP_METHODS"); env != "" {
80-
config.CustomSkipMethods = strings.Split(env, ",")
81-
for i, method := range config.CustomSkipMethods {
82-
config.CustomSkipMethods[i] = strings.TrimSpace(strings.ToUpper(method))
83-
}
84-
}
85-
86-
// Load custom skip prefixes
87-
if env := os.Getenv("OPENLIST_LOG_FILTER_SKIP_PREFIXES"); env != "" {
88-
config.CustomSkipPrefixes = strings.Split(env, ",")
89-
for i, prefix := range config.CustomSkipPrefixes {
90-
config.CustomSkipPrefixes[i] = strings.TrimSpace(prefix)
91-
}
92-
}
93-
94-
return config
95-
}
96-
97-
// ToFilteredLoggerConfig converts LogFilterConfig to FilteredLoggerConfig
98-
func (c LogFilterConfig) ToFilteredLoggerConfig() FilteredLoggerConfig {
99-
if !c.EnableFiltering {
100-
// Return empty config to disable filtering
101-
return FilteredLoggerConfig{
102-
Output: log.StandardLogger().Out,
103-
}
14+
if !config.EnableFiltering {
15+
// Return standard Gin logger if filtering is disabled
16+
return gin.LoggerWithWriter(log.StandardLogger().Out)
10417
}
10518

106-
config := FilteredLoggerConfig{
19+
loggerConfig := FilteredLoggerConfig{
10720
Output: log.StandardLogger().Out,
10821
}
10922

11023
// Add health check paths
111-
if c.FilterHealthChecks {
112-
config.SkipPaths = append(config.SkipPaths, "/ping")
24+
if config.FilterHealthChecks {
25+
loggerConfig.SkipPaths = append(loggerConfig.SkipPaths, "/ping")
11326
}
11427

11528
// Add HEAD method filtering
116-
if c.FilterHEADRequests {
117-
config.SkipMethods = append(config.SkipMethods, "HEAD")
29+
if config.FilterHEADRequests {
30+
loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, "HEAD")
11831
}
11932

120-
// Add WebDAV filtering
121-
if c.FilterWebDAV {
122-
config.SkipPathPrefixes = append(config.SkipPathPrefixes, "/dav/")
123-
config.SkipMethods = append(config.SkipMethods, "PROPFIND")
33+
// Add WebDAV filtering only for HTTP server (not for S3)
34+
if config.FilterWebDAV && serverType == "http" {
35+
loggerConfig.SkipPathPrefixes = append(loggerConfig.SkipPathPrefixes, "/dav/")
36+
loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, "PROPFIND")
12437
}
12538

12639
// Add custom configurations
127-
config.SkipPaths = append(config.SkipPaths, c.CustomSkipPaths...)
128-
config.SkipMethods = append(config.SkipMethods, c.CustomSkipMethods...)
129-
config.SkipPathPrefixes = append(config.SkipPathPrefixes, c.CustomSkipPrefixes...)
40+
loggerConfig.SkipPaths = append(loggerConfig.SkipPaths, config.CustomSkipPaths...)
41+
loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, config.CustomSkipMethods...)
42+
loggerConfig.SkipPathPrefixes = append(loggerConfig.SkipPathPrefixes, config.CustomSkipPrefixes...)
13043

131-
return config
44+
return FilteredLoggerWithConfig(loggerConfig)
13245
}
13346

134-
// ConfigurableFilteredLogger returns a filtered logger with configuration loaded from environment
135-
func ConfigurableFilteredLogger() gin.HandlerFunc {
136-
config := LoadLogFilterConfigFromEnv()
137-
loggerConfig := config.ToFilteredLoggerConfig()
138-
139-
if !config.EnableFiltering {
140-
// Return standard Gin logger if filtering is disabled
141-
return gin.LoggerWithWriter(log.StandardLogger().Out)
142-
}
143-
144-
return FilteredLoggerWithConfig(loggerConfig)
47+
// HTTPFilteredLogger returns a filtered logger for the main HTTP server
48+
func HTTPFilteredLogger() gin.HandlerFunc {
49+
return UnifiedFilteredLogger("http")
50+
}
51+
52+
// S3FilteredLogger returns a filtered logger for the S3 server
53+
func S3FilteredLogger() gin.HandlerFunc {
54+
return UnifiedFilteredLogger("s3")
14555
}

‎server/middlewares/filtered_logger.go‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -44,24 +44,6 @@ func FilteredLoggerWithConfig(config FilteredLoggerConfig) gin.HandlerFunc {
4444
})
4545
}
4646

47-
// FilteredLogger returns a gin.HandlerFunc (middleware) that logs requests
48-
// but filters out health check and PROPFIND requests
49-
func FilteredLogger() gin.HandlerFunc {
50-
config := FilteredLoggerConfig{
51-
SkipPaths: []string{
52-
"/ping",
53-
},
54-
SkipMethods: []string{
55-
"HEAD", // Skip HEAD requests for health checks
56-
},
57-
SkipPathPrefixes: []string{
58-
"/dav/", // Skip WebDAV PROPFIND requests
59-
},
60-
Output: log.StandardLogger().Out,
61-
}
62-
63-
return FilteredLoggerWithConfig(config)
64-
}
6547

6648
// shouldSkipLogging determines if a request should be skipped from logging
6749
func shouldSkipLogging(path, method string, config FilteredLoggerConfig) bool {

0 commit comments

Comments
 (0)