Skip to content

Commit 6bebee7

Browse files
ljcbabySuyunmeng
authored andcommitted
refactor(log): filter (#816)
1 parent 3e1e283 commit 6bebee7

File tree

5 files changed

+104
-165
lines changed

5 files changed

+104
-165
lines changed

cmd/server.go

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,15 @@ the address is defined in config file`,
4848
gin.SetMode(gin.ReleaseMode)
4949
}
5050
r := gin.New()
51-
r.Use(middlewares.HTTPFilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out))
51+
52+
// gin log
53+
if conf.Conf.Log.Filter.Enable {
54+
r.Use(middlewares.FilteredLogger())
55+
} else {
56+
r.Use(gin.LoggerWithWriter(log.StandardLogger().Out))
57+
}
58+
r.Use(gin.RecoveryWithWriter(log.StandardLogger().Out))
59+
5260
server.Init(r)
5361
var httpHandler http.Handler = r
5462
if conf.Conf.Scheme.EnableH2c {
@@ -103,7 +111,7 @@ the address is defined in config file`,
103111
}
104112
if conf.Conf.S3.Port != -1 && conf.Conf.S3.Enable {
105113
s3r := gin.New()
106-
s3r.Use(middlewares.S3FilteredLogger(), gin.RecoveryWithWriter(log.StandardLogger().Out))
114+
s3r.Use(gin.LoggerWithWriter(log.StandardLogger().Out), gin.RecoveryWithWriter(log.StandardLogger().Out))
107115
server.InitS3(s3r)
108116
s3Base := fmt.Sprintf("%s:%d", conf.Conf.Scheme.Address, conf.Conf.S3.Port)
109117
utils.Log.Infof("start S3 server @ %s", s3Base)

internal/bootstrap/config.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,9 @@ func InitConfig() {
8282
if !conf.Conf.Force {
8383
confFromEnv()
8484
}
85+
if len(conf.Conf.Log.Filter.Filters) == 0 {
86+
conf.Conf.Log.Filter.Enable = false
87+
}
8588
// convert abs path
8689
convertAbsPath := func(path *string) {
8790
if !filepath.IsAbs(*path) {

internal/conf/config.go

Lines changed: 22 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -38,38 +38,24 @@ type Scheme struct {
3838
}
3939

4040
type LogConfig struct {
41-
Enable bool `json:"enable" env:"LOG_ENABLE"`
42-
Name string `json:"name" env:"LOG_NAME"`
43-
MaxSize int `json:"max_size" env:"MAX_SIZE"`
44-
MaxBackups int `json:"max_backups" env:"MAX_BACKUPS"`
45-
MaxAge int `json:"max_age" env:"MAX_AGE"`
46-
Compress bool `json:"compress" env:"COMPRESS"`
47-
Filter LogFilterConfig `json:"filter"` // Log filtering configuration (config file only, no env support)
41+
Enable bool `json:"enable" env:"ENABLE"`
42+
Name string `json:"name" env:"NAME"`
43+
MaxSize int `json:"max_size" env:"MAX_SIZE"`
44+
MaxBackups int `json:"max_backups" env:"MAX_BACKUPS"`
45+
MaxAge int `json:"max_age" env:"MAX_AGE"`
46+
Compress bool `json:"compress" env:"COMPRESS"`
47+
Filter LogFilterConfig `json:"filter" envPrefix:"FILTER_"`
4848
}
4949

50-
// LogFilterConfig holds configuration for log filtering
51-
// Note: This configuration is only supported via config file, not environment variables
5250
type LogFilterConfig struct {
53-
// EnableFiltering controls whether log filtering is enabled
54-
EnableFiltering bool `json:"enable_filtering"`
55-
56-
// FilterHealthChecks controls whether to filter health check requests
57-
FilterHealthChecks bool `json:"filter_health_checks"`
58-
59-
// FilterWebDAV controls whether to filter WebDAV requests (only for HTTP server)
60-
FilterWebDAV bool `json:"filter_webdav"`
61-
62-
// FilterHEADRequests controls whether to filter HEAD requests
63-
FilterHEADRequests bool `json:"filter_head_requests"`
64-
65-
// CustomSkipPaths allows adding custom paths to skip
66-
CustomSkipPaths []string `json:"custom_skip_paths"`
67-
68-
// CustomSkipMethods allows adding custom methods to skip
69-
CustomSkipMethods []string `json:"custom_skip_methods"`
70-
71-
// CustomSkipPrefixes allows adding custom path prefixes to skip
72-
CustomSkipPrefixes []string `json:"custom_skip_prefixes"`
51+
Enable bool `json:"enable" env:"ENABLE"`
52+
Filters []Filter `json:"filters"`
53+
}
54+
55+
type Filter struct {
56+
CIDR string `json:"cidr"`
57+
Path string `json:"path"`
58+
Method string `json:"method"`
7359
}
7460

7561
type TaskConfig struct {
@@ -131,7 +117,7 @@ type Config struct {
131117
TempDir string `json:"temp_dir" env:"TEMP_DIR"`
132118
BleveDir string `json:"bleve_dir" env:"BLEVE_DIR"`
133119
DistDir string `json:"dist_dir"`
134-
Log LogConfig `json:"log"`
120+
Log LogConfig `json:"log" envPrefix:"LOG_"`
135121
DelayedStart int `json:"delayed_start" env:"DELAYED_START"`
136122
MaxConnections int `json:"max_connections" env:"MAX_CONNECTIONS"`
137123
MaxConcurrency int `json:"max_concurrency" env:"MAX_CONCURRENCY"`
@@ -179,13 +165,12 @@ func DefaultConfig(dataDir string) *Config {
179165
MaxBackups: 30,
180166
MaxAge: 28,
181167
Filter: LogFilterConfig{
182-
EnableFiltering: true,
183-
FilterHealthChecks: true,
184-
FilterWebDAV: true,
185-
FilterHEADRequests: true,
186-
CustomSkipPaths: []string{},
187-
CustomSkipMethods: []string{},
188-
CustomSkipPrefixes: []string{},
168+
Enable: false,
169+
Filters: []Filter{
170+
{Path: "/ping"},
171+
{Method: "HEAD"},
172+
{Path: "/dav/", Method: "PROPFIND"},
173+
},
189174
},
190175
},
191176
MaxConnections: 0,

server/middlewares/config.go

Lines changed: 0 additions & 55 deletions
This file was deleted.
Lines changed: 69 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -1,101 +1,99 @@
11
package middlewares
22

33
import (
4-
"fmt"
5-
"io"
4+
"net/netip"
65
"strings"
7-
"time"
86

7+
"github.com/OpenListTeam/OpenList/v4/internal/conf"
98
"github.com/gin-gonic/gin"
109
log "github.com/sirupsen/logrus"
1110
)
1211

13-
// FilteredLoggerConfig defines the configuration for the filtered logger
14-
type FilteredLoggerConfig struct {
15-
// SkipPaths is a list of URL paths to skip logging
16-
SkipPaths []string
17-
// SkipMethods is a list of HTTP methods to skip logging
18-
SkipMethods []string
19-
// SkipPathPrefixes is a list of URL path prefixes to skip logging
20-
SkipPathPrefixes []string
21-
// Output is the writer where logs will be written
22-
Output io.Writer
12+
type filter struct {
13+
CIDR *netip.Prefix `json:"cidr,omitempty"`
14+
Path *string `json:"path,omitempty"`
15+
Method *string `json:"method,omitempty"`
2316
}
2417

25-
// FilteredLoggerWithConfig returns a gin.HandlerFunc (middleware) that logs requests
26-
// but skips logging for specified paths, methods, or path prefixes
27-
func FilteredLoggerWithConfig(config FilteredLoggerConfig) gin.HandlerFunc {
28-
if config.Output == nil {
29-
config.Output = log.StandardLogger().Out
30-
}
18+
var filterList []*filter
3119

32-
return gin.LoggerWithConfig(gin.LoggerConfig{
33-
Output: config.Output,
34-
SkipPaths: config.SkipPaths,
35-
Formatter: func(param gin.LogFormatterParams) string {
36-
// Skip logging for health check endpoints
37-
if shouldSkipLogging(param.Path, param.Method, config) {
38-
return ""
20+
func initFilterList() {
21+
for _, s := range conf.Conf.Log.Filter.Filters {
22+
f := new(filter)
23+
24+
if s.CIDR != "" {
25+
cidr, err := netip.ParsePrefix(s.CIDR)
26+
if err != nil {
27+
log.Errorf("failed to parse CIDR %s: %v", s.CIDR, err)
28+
continue
3929
}
30+
f.CIDR = &cidr
31+
}
4032

41-
// Use a custom log format similar to Gin's default
42-
return defaultLogFormatter(param)
43-
},
44-
})
45-
}
33+
if s.Path != "" {
34+
f.Path = &s.Path
35+
}
4636

37+
if s.Method != "" {
38+
f.Method = &s.Method
39+
}
4740

48-
// shouldSkipLogging determines if a request should be skipped from logging
49-
func shouldSkipLogging(path, method string, config FilteredLoggerConfig) bool {
50-
// Check if path should be skipped
51-
for _, skipPath := range config.SkipPaths {
52-
if path == skipPath {
53-
return true
41+
if f.CIDR == nil && f.Path == nil && f.Method == nil {
42+
log.Warnf("filter %s is empty, skipping", s)
43+
continue
5444
}
45+
46+
filterList = append(filterList, f)
47+
log.Debugf("added filter: %+v", f)
5548
}
5649

57-
// Check if method should be skipped
58-
for _, skipMethod := range config.SkipMethods {
59-
if method == skipMethod {
60-
return true
50+
log.Infof("Loaded %d log filters.", len(filterList))
51+
}
52+
53+
func skiperDecider(c *gin.Context) bool {
54+
// every filter need metch all condithon as filter match
55+
// so if any condithon not metch, skip this filter
56+
// all filters misatch, log this request
57+
58+
for _, f := range filterList {
59+
if f.CIDR != nil {
60+
cip := netip.MustParseAddr(c.ClientIP())
61+
if !f.CIDR.Contains(cip) {
62+
continue
63+
}
6164
}
62-
}
6365

64-
// Check if path prefix should be skipped
65-
for _, skipPrefix := range config.SkipPathPrefixes {
66-
if strings.HasPrefix(path, skipPrefix) {
67-
return true
66+
if f.Path != nil {
67+
if (*f.Path)[0] == '/' {
68+
// match path as prefix/exact path
69+
if !strings.HasPrefix(c.Request.URL.Path, *f.Path) {
70+
continue
71+
}
72+
} else {
73+
// match path as relative path
74+
if !strings.Contains(c.Request.URL.Path, "/"+*f.Path) {
75+
continue
76+
}
77+
}
78+
}
79+
80+
if f.Method != nil {
81+
if *f.Method != c.Request.Method {
82+
continue
83+
}
6884
}
69-
}
7085

71-
// Special case: Skip PROPFIND requests (common in WebDAV)
72-
if method == "PROPFIND" {
7386
return true
7487
}
7588

7689
return false
7790
}
7891

79-
// defaultLogFormatter provides a default log format similar to Gin's built-in formatter
80-
func defaultLogFormatter(param gin.LogFormatterParams) string {
81-
var statusColor, methodColor, resetColor string
82-
if param.IsOutputColor() {
83-
statusColor = param.StatusCodeColor()
84-
methodColor = param.MethodColor()
85-
resetColor = param.ResetColor()
86-
}
87-
88-
if param.Latency > time.Minute {
89-
param.Latency = param.Latency.Truncate(time.Second)
90-
}
92+
func FilteredLogger() gin.HandlerFunc {
93+
initFilterList()
9194

92-
return fmt.Sprintf("[GIN] %v |%s %3d %s| %13v | %15s |%s %-7s %s %#v\n%s",
93-
param.TimeStamp.Format("2006/01/02 - 15:04:05"),
94-
statusColor, param.StatusCode, resetColor,
95-
param.Latency,
96-
param.ClientIP,
97-
methodColor, param.Method, resetColor,
98-
param.Path,
99-
param.ErrorMessage,
100-
)
101-
}
95+
return gin.LoggerWithConfig(gin.LoggerConfig{
96+
Output: log.StandardLogger().Out,
97+
Skip: skiperDecider,
98+
})
99+
}

0 commit comments

Comments
 (0)