Skip to content

Commit a9f02ec

Browse files
SuyunmengSuyunmeng
authored
refactor(log):Refactor log filtering to use centralized configuration and add server-specific filtering (#798)
* feat(log):Add configurable log filtering middleware for HTTP requests Implement a comprehensive log filtering system that allows selective suppression of HTTP request logs based on paths, methods, and prefixes. The system includes environment variable configuration support and filters health checks, WebDAV requests, and HEAD requests by default to reduce log noise. * fix(log):Replace gin.DefaultLogFormatter with custom implementation * Remove filtered logger test file * fix(log):Refactor log filtering to use centralized configuration and add server-specific filtering * fix(log):Add documentation comments for log filtering configuration
1 parent 93849a3 commit a9f02ec

File tree

4 files changed

+66
-139
lines changed

4 files changed

+66
-139
lines changed

‎cmd/server.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ the address is defined in config file`,
4848
gin.SetMode(gin.ReleaseMode)
4949
}
5050
r := gin.New()
51-
r.Use(middlewares.ConfigurableFilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out))
51+
r.Use(middlewares.HTTPFilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out))
5252
server.Init(r)
5353
var httpHandler http.Handler = r
5454
if conf.Conf.Scheme.EnableH2c {
@@ -103,7 +103,7 @@ the address is defined in config file`,
103103
}
104104
if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable {
105105
s3r := gin.New()
106-
s3r.Use(middlewares.FilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out))
106+
s3r.Use(middlewares.S3FilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out))
107107
server.InitS3(s3r)
108108
s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port)
109109
utils.Log.Infof("start S3 server @ %s", s3Base)

‎internal/conf/config.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,32 @@ type LogConfig struct {
4444
MaxBackups int `json:"max_backups" env:"MAX_BACKUPS"`
4545
MaxAge int `json:"max_age" env:"MAX_AGE"`
4646
Compress bool `json:"compress" env:"COMPRESS"`
47+
Filter LogFilterConfig `json:"filter"` // Log filtering configuration (config file only, no env support)
48+
}
49+
50+
// LogFilterConfig holds configuration for log filtering
51+
// Note: This configuration is only supported via config file, not environment variables
52+
type LogFilterConfig struct {
53+
// EnableFiltering controls whether log filtering is enabled
54+
EnableFiltering bool `json:"enable_filtering"`
55+
56+
// FilterHealthChecks controls whether to filter health check requests
57+
FilterHealthChecks bool `json:"filter_health_checks"`
58+
59+
// FilterWebDAV controls whether to filter WebDAV requests (only for HTTP server)
60+
FilterWebDAV bool `json:"filter_webdav"`
61+
62+
// FilterHEADRequests controls whether to filter HEAD requests
63+
FilterHEADRequests bool `json:"filter_head_requests"`
64+
65+
// CustomSkipPaths allows adding custom paths to skip
66+
CustomSkipPaths []string `json:"custom_skip_paths"`
67+
68+
// CustomSkipMethods allows adding custom methods to skip
69+
CustomSkipMethods []string `json:"custom_skip_methods"`
70+
71+
// CustomSkipPrefixes allows adding custom path prefixes to skip
72+
CustomSkipPrefixes []string `json:"custom_skip_prefixes"`
4773
}
4874

4975
type TaskConfig struct {
@@ -152,6 +178,15 @@ func DefaultConfig(dataDir string) *Config {
152178
MaxSize: 50,
153179
MaxBackups: 30,
154180
MaxAge: 28,
181+
Filter: LogFilterConfig{
182+
EnableFiltering: true,
183+
FilterHealthChecks: true,
184+
FilterWebDAV: true,
185+
FilterHEADRequests: true,
186+
CustomSkipPaths: []string{},
187+
CustomSkipMethods: []string{},
188+
CustomSkipPrefixes: []string{},
189+
},
155190
},
156191
MaxConnections: 0,
157192
MaxConcurrency: 64,

‎server/middlewares/config.go‎

Lines changed: 29 additions & 119 deletions
Original file line numberDiff line numberDiff line change
@@ -1,145 +1,55 @@
11
package middlewares
22

33
import (
4-
"os"
5-
"strings"
6-
4+
"github.com/OpenListTeam/OpenList/v4/internal/conf"
75
"github.com/gin-gonic/gin"
86
log "github.com/sirupsen/logrus"
97
)
108

11-
// LogFilterConfig holds configuration for log filtering
12-
type LogFilterConfig struct {
13-
// EnableFiltering controls whether log filtering is enabled
14-
EnableFiltering bool
15-
16-
// FilterHealthChecks controls whether to filter health check requests
17-
FilterHealthChecks bool
18-
19-
// FilterWebDAV controls whether to filter WebDAV requests
20-
FilterWebDAV bool
21-
22-
// FilterHEADRequests controls whether to filter HEAD requests
23-
FilterHEADRequests bool
24-
25-
// CustomSkipPaths allows adding custom paths to skip
26-
CustomSkipPaths []string
27-
28-
// CustomSkipMethods allows adding custom methods to skip
29-
CustomSkipMethods []string
9+
// UnifiedFilteredLogger returns a filtered logger using global configuration
10+
// serverType: "http" for main HTTP server, "s3" for S3 server
11+
func UnifiedFilteredLogger(serverType string) gin.HandlerFunc {
12+
config := conf.Conf.Log.Filter
3013

31-
// CustomSkipPrefixes allows adding custom path prefixes to skip
32-
CustomSkipPrefixes []string
33-
}
34-
35-
// DefaultLogFilterConfig returns the default configuration
36-
func DefaultLogFilterConfig() LogFilterConfig {
37-
return LogFilterConfig{
38-
EnableFiltering: true,
39-
FilterHealthChecks: true,
40-
FilterWebDAV: true,
41-
FilterHEADRequests: true,
42-
CustomSkipPaths: []string{},
43-
CustomSkipMethods: []string{},
44-
CustomSkipPrefixes: []string{},
45-
}
46-
}
47-
48-
// LoadLogFilterConfigFromEnv loads configuration from environment variables
49-
func LoadLogFilterConfigFromEnv() LogFilterConfig {
50-
config := DefaultLogFilterConfig()
51-
52-
// Check if filtering is enabled
53-
if env := os.Getenv("OPENLIST_LOG_FILTER_ENABLED"); env != "" {
54-
config.EnableFiltering = strings.ToLower(env) == "true"
55-
}
56-
57-
// Check individual filter options
58-
if env := os.Getenv("OPENLIST_LOG_FILTER_HEALTH_CHECKS"); env != "" {
59-
config.FilterHealthChecks = strings.ToLower(env) == "true"
60-
}
61-
62-
if env := os.Getenv("OPENLIST_LOG_FILTER_WEBDAV"); env != "" {
63-
config.FilterWebDAV = strings.ToLower(env) == "true"
64-
}
65-
66-
if env := os.Getenv("OPENLIST_LOG_FILTER_HEAD_REQUESTS"); env != "" {
67-
config.FilterHEADRequests = strings.ToLower(env) == "true"
68-
}
69-
70-
// Load custom skip paths
71-
if env := os.Getenv("OPENLIST_LOG_FILTER_SKIP_PATHS"); env != "" {
72-
config.CustomSkipPaths = strings.Split(env, ",")
73-
for i, path := range config.CustomSkipPaths {
74-
config.CustomSkipPaths[i] = strings.TrimSpace(path)
75-
}
76-
}
77-
78-
// Load custom skip methods
79-
if env := os.Getenv("OPENLIST_LOG_FILTER_SKIP_METHODS"); env != "" {
80-
config.CustomSkipMethods = strings.Split(env, ",")
81-
for i, method := range config.CustomSkipMethods {
82-
config.CustomSkipMethods[i] = strings.TrimSpace(strings.ToUpper(method))
83-
}
84-
}
85-
86-
// Load custom skip prefixes
87-
if env := os.Getenv("OPENLIST_LOG_FILTER_SKIP_PREFIXES"); env != "" {
88-
config.CustomSkipPrefixes = strings.Split(env, ",")
89-
for i, prefix := range config.CustomSkipPrefixes {
90-
config.CustomSkipPrefixes[i] = strings.TrimSpace(prefix)
91-
}
92-
}
93-
94-
return config
95-
}
96-
97-
// ToFilteredLoggerConfig converts LogFilterConfig to FilteredLoggerConfig
98-
func (c LogFilterConfig) ToFilteredLoggerConfig() FilteredLoggerConfig {
99-
if !c.EnableFiltering {
100-
// Return empty config to disable filtering
101-
return FilteredLoggerConfig{
102-
Output: log.StandardLogger().Out,
103-
}
14+
if !config.EnableFiltering {
15+
// Return standard Gin logger if filtering is disabled
16+
return gin.LoggerWithWriter(log.StandardLogger().Out)
10417
}
10518

106-
config := FilteredLoggerConfig{
19+
loggerConfig := FilteredLoggerConfig{
10720
Output: log.StandardLogger().Out,
10821
}
10922

11023
// Add health check paths
111-
if c.FilterHealthChecks {
112-
config.SkipPaths = append(config.SkipPaths, "/ping")
24+
if config.FilterHealthChecks {
25+
loggerConfig.SkipPaths = append(loggerConfig.SkipPaths, "/ping")
11326
}
11427

11528
// Add HEAD method filtering
116-
if c.FilterHEADRequests {
117-
config.SkipMethods = append(config.SkipMethods, "HEAD")
29+
if config.FilterHEADRequests {
30+
loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, "HEAD")
11831
}
11932

120-
// Add WebDAV filtering
121-
if c.FilterWebDAV {
122-
config.SkipPathPrefixes = append(config.SkipPathPrefixes, "/dav/")
123-
config.SkipMethods = append(config.SkipMethods, "PROPFIND")
33+
// Add WebDAV filtering only for HTTP server (not for S3)
34+
if config.FilterWebDAV && serverType == "http" {
35+
loggerConfig.SkipPathPrefixes = append(loggerConfig.SkipPathPrefixes, "/dav/")
36+
loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, "PROPFIND")
12437
}
12538

12639
// Add custom configurations
127-
config.SkipPaths = append(config.SkipPaths, c.CustomSkipPaths...)
128-
config.SkipMethods = append(config.SkipMethods, c.CustomSkipMethods...)
129-
config.SkipPathPrefixes = append(config.SkipPathPrefixes, c.CustomSkipPrefixes...)
40+
loggerConfig.SkipPaths = append(loggerConfig.SkipPaths, config.CustomSkipPaths...)
41+
loggerConfig.SkipMethods = append(loggerConfig.SkipMethods, config.CustomSkipMethods...)
42+
loggerConfig.SkipPathPrefixes = append(loggerConfig.SkipPathPrefixes, config.CustomSkipPrefixes...)
13043

131-
return config
44+
return FilteredLoggerWithConfig(loggerConfig)
13245
}
13346

134-
// ConfigurableFilteredLogger returns a filtered logger with configuration loaded from environment
135-
func ConfigurableFilteredLogger() gin.HandlerFunc {
136-
config := LoadLogFilterConfigFromEnv()
137-
loggerConfig := config.ToFilteredLoggerConfig()
138-
139-
if !config.EnableFiltering {
140-
// Return standard Gin logger if filtering is disabled
141-
return gin.LoggerWithWriter(log.StandardLogger().Out)
142-
}
143-
144-
return FilteredLoggerWithConfig(loggerConfig)
47+
// HTTPFilteredLogger returns a filtered logger for the main HTTP server
48+
func HTTPFilteredLogger() gin.HandlerFunc {
49+
return UnifiedFilteredLogger("http")
50+
}
51+
52+
// S3FilteredLogger returns a filtered logger for the S3 server
53+
func S3FilteredLogger() gin.HandlerFunc {
54+
return UnifiedFilteredLogger("s3")
14555
}

‎server/middlewares/filtered_logger.go‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -44,24 +44,6 @@ func FilteredLoggerWithConfig(config FilteredLoggerConfig) gin.HandlerFunc {
4444
})
4545
}
4646

47-
// FilteredLogger returns a gin.HandlerFunc (middleware) that logs requests
48-
// but filters out health check and PROPFIND requests
49-
func FilteredLogger() gin.HandlerFunc {
50-
config := FilteredLoggerConfig{
51-
SkipPaths: []string{
52-
"/ping",
53-
},
54-
SkipMethods: []string{
55-
"HEAD", // Skip HEAD requests for health checks
56-
},
57-
SkipPathPrefixes: []string{
58-
"/dav/", // Skip WebDAV PROPFIND requests
59-
},
60-
Output: log.StandardLogger().Out,
61-
}
62-
63-
return FilteredLoggerWithConfig(config)
64-
}
6547

6648
// shouldSkipLogging determines if a request should be skipped from logging
6749
func shouldSkipLogging(path, method string, config FilteredLoggerConfig) bool {

0 commit comments

Comments
 (0)