Skip to content

Commit d32f0a9

Browse files
agneumagneum
committed
feat: mask sensitive data in the DLE logs output
1 parent d410d09 commit d32f0a9

File tree

6 files changed

+91
-43
lines changed

6 files changed

+91
-43
lines changed

‎engine/cmd/database-lab/main.go

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,9 @@ func main() {
5656
log.Fatal(errors.WithMessage(err, "failed to parse config"))
5757
}
5858

59+
logFilter := log.GetFilter()
60+
logFilter.ReloadLogRegExp([]string{cfg.Server.VerificationToken, cfg.Platform.AccessToken})
61+
5962
config.ApplyGlobals(cfg)
6063

6164
docker, err := client.NewClientWithOpts(client.FromEnv)
@@ -188,14 +191,16 @@ func main() {
188191
embeddedUI,
189192
server,
190193
logCleaner,
194+
logFilter,
191195
)
192196
}
193197

194198
server := srv.NewServer(&cfg.Server, &cfg.Global, engProps, docker, cloningSvc, provisioner, retrievalSvc, platformSvc,
195-
obs, est, pm, tm, tokenHolder, embeddedUI, reloadConfigFn)
199+
obs, est, pm, tm, tokenHolder, logFilter, embeddedUI, reloadConfigFn)
196200
shutdownCh := setShutdownListener()
197201

198-
go setReloadListener(ctx, provisioner, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, embeddedUI, server, logCleaner)
202+
go setReloadListener(ctx, provisioner, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, embeddedUI, server,
203+
logCleaner, logFilter)
199204

200205
server.InitHandlers()
201206

@@ -276,12 +281,14 @@ func getEngineProperties(ctx context.Context, dockerCLI *client.Client, cfg *con
276281

277282
func reloadConfig(ctx context.Context, provisionSvc *provision.Provisioner, tm *telemetry.Agent,
278283
retrievalSvc *retrieval.Retrieval, pm *pool.Manager, cloningSvc *cloning.Base, platformSvc *platform.Service,
279-
est *estimator.Estimator, embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner) error {
284+
est *estimator.Estimator, embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner,
285+
filtering *log.Filtering) error {
280286
cfg, err := config.LoadConfiguration()
281287
if err != nil {
282288
return err
283289
}
284290

291+
filtering.ReloadLogRegExp([]string{cfg.Server.VerificationToken, cfg.Platform.AccessToken})
285292
config.ApplyGlobals(cfg)
286293

287294
if err := provision.IsValidConfig(cfg.Provision); err != nil {
@@ -328,14 +335,16 @@ func reloadConfig(ctx context.Context, provisionSvc *provision.Provisioner, tm *
328335

329336
func setReloadListener(ctx context.Context, provisionSvc *provision.Provisioner, tm *telemetry.Agent,
330337
retrievalSvc *retrieval.Retrieval, pm *pool.Manager, cloningSvc *cloning.Base, platformSvc *platform.Service,
331-
est *estimator.Estimator, embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner) {
338+
est *estimator.Estimator, embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner,
339+
logFilter *log.Filtering) {
332340
reloadCh := make(chan os.Signal, 1)
333341
signal.Notify(reloadCh, syscall.SIGHUP)
334342

335343
for range reloadCh {
336344
log.Msg("Reloading configuration")
337345

338-
if err := reloadConfig(ctx, provisionSvc, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, embeddedUI, server, cleaner); err != nil {
346+
if err := reloadConfig(ctx, provisionSvc, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, embeddedUI, server,
347+
cleaner, logFilter); err != nil {
339348
log.Err("Failed to reload configuration", err)
340349
}
341350

‎engine/internal/srv/server.go

Lines changed: 4 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -9,10 +9,8 @@ import (
99
"context"
1010
"fmt"
1111
"net/http"
12-
"regexp"
1312
"strings"
1413
"time"
15-
"unicode"
1614

1715
"github.com/AlekSi/pointer"
1816
"github.com/docker/docker/client"
@@ -41,8 +39,6 @@ import (
4139
"gitlab.com/postgres-ai/database-lab/v3/version"
4240
)
4341

44-
const minTokenLength = 8
45-
4642
// Server defines an HTTP server of the Database Lab.
4743
type Server struct {
4844
validator validator.Service
@@ -61,7 +57,7 @@ type Server struct {
6157
pm *pool.Manager
6258
tm *telemetry.Agent
6359
startedAt *models.LocalTime
64-
re *regexp.Regexp
60+
filtering *log.Filtering
6561
reloadFn func(server *Server) error
6662
}
6763

@@ -77,7 +73,7 @@ func NewServer(cfg *srvCfg.Config, globalCfg *global.Config, engineProps global.
7773
dockerClient *client.Client, cloning *cloning.Base, provisioner *provision.Provisioner,
7874
retrievalSvc *retrieval.Retrieval, platform *platform.Service, observer *observer.Observer,
7975
estimator *estimator.Estimator, pm *pool.Manager, tm *telemetry.Agent, tokenKeeper *ws.TokenKeeper,
80-
uiManager *embeddedui.UIManager, reloadConfigFn func(server *Server) error) *Server {
76+
filtering *log.Filtering, uiManager *embeddedui.UIManager, reloadConfigFn func(server *Server) error) *Server {
8177
server := &Server{
8278
Config: cfg,
8379
Global: globalCfg,
@@ -96,10 +92,10 @@ func NewServer(cfg *srvCfg.Config, globalCfg *global.Config, engineProps global.
9692
docker: dockerClient,
9793
pm: pm,
9894
tm: tm,
95+
filtering: filtering,
9996
startedAt: &models.LocalTime{Time: time.Now().Truncate(time.Second)},
10097
reloadFn: reloadConfigFn,
10198
}
102-
server.initLogRegExp()
10399

104100
return server
105101
}
@@ -185,7 +181,6 @@ func attachAPI(r *mux.Router) error {
185181
// Reload reloads server configuration.
186182
func (s *Server) Reload(cfg srvCfg.Config) {
187183
*s.Config = cfg
188-
s.initLogRegExp()
189184
}
190185

191186
// InitHandlers initializes handler functions of the HTTP server.
@@ -266,31 +261,5 @@ func reportLaunching(cfg *srvCfg.Config) {
266261
}
267262

268263
func (s *Server) initLogRegExp() {
269-
secretPatterns := []string{
270-
"password:\\s?(\\S+)",
271-
"POSTGRES_PASSWORD=(\\S+)",
272-
"PGPASSWORD=(\\S+)",
273-
"accessToken:\\s?(\\S+)",
274-
"ACCESS_KEY(_ID)?:\\s?(\\S+)",
275-
}
276-
277-
if len(s.Config.VerificationToken) >= minTokenLength && !containsSpace(s.Config.VerificationToken) {
278-
secretPatterns = append(secretPatterns, s.Config.VerificationToken)
279-
}
280-
281-
if accessToken := s.Platform.AccessToken(); len(accessToken) >= minTokenLength && !containsSpace(accessToken) {
282-
secretPatterns = append(secretPatterns, accessToken)
283-
}
284-
285-
s.re = regexp.MustCompile("(?i)" + strings.Join(secretPatterns, "|"))
286-
}
287-
288-
func containsSpace(s string) bool {
289-
for _, v := range s {
290-
if unicode.IsSpace(v) {
291-
return true
292-
}
293-
}
294-
295-
return false
264+
s.filtering.ReloadLogRegExp([]string{s.Config.VerificationToken, s.Platform.AccessToken()})
296265
}

‎engine/internal/srv/ws.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,5 +109,5 @@ func (s *Server) instanceLogs(w http.ResponseWriter, r *http.Request) {
109109
}
110110

111111
func (s *Server) filterLogLine(inputLine []byte) []byte {
112-
return s.re.ReplaceAll(inputLine, []byte("********"))
112+
return s.filtering.ReplaceAll(inputLine)
113113
}

‎engine/internal/srv/ws_test.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,13 +9,18 @@ import (
99

1010
"gitlab.com/postgres-ai/database-lab/v3/internal/platform"
1111
"gitlab.com/postgres-ai/database-lab/v3/internal/srv/config"
12+
"gitlab.com/postgres-ai/database-lab/v3/pkg/log"
1213
)
1314

1415
func TestLogLineFiltering(t *testing.T) {
1516
pl, err := platform.New(context.Background(), platform.Config{AccessToken: "platformAccessToken"})
1617
require.NoError(t, err)
1718

18-
s := Server{Config: &config.Config{VerificationToken: "secretToken"}, Platform: pl}
19+
s := Server{
20+
Config: &config.Config{VerificationToken: "secretToken"},
21+
Platform: pl,
22+
filtering: log.GetFilter(),
23+
}
1924
s.initLogRegExp()
2025

2126
testCases := []struct {

‎engine/pkg/log/filtering.go

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
package log
2+
3+
import (
4+
"regexp"
5+
"strings"
6+
"unicode"
7+
)
8+
9+
const (
10+
minTokenLength = 8
11+
replacingMask = "********"
12+
)
13+
14+
var filter = newFiltering([]string{})
15+
16+
// Filtering represents a struct to filter secrets in logs output.
17+
type Filtering struct {
18+
re *regexp.Regexp
19+
}
20+
21+
// GetFilter gets an instance of Log Filtering.
22+
func GetFilter() *Filtering {
23+
return filter
24+
}
25+
26+
func newFiltering(tokens []string) *Filtering {
27+
f := &Filtering{}
28+
f.ReloadLogRegExp(tokens)
29+
30+
return f
31+
}
32+
33+
// ReloadLogRegExp updates secrets configuration.
34+
func (f *Filtering) ReloadLogRegExp(secretStings []string) {
35+
secretPatterns := []string{
36+
"password:\\s?(\\S+)",
37+
"POSTGRES_PASSWORD=(\\S+)",
38+
"PGPASSWORD=(\\S+)",
39+
"accessToken:\\s?(\\S+)",
40+
"ACCESS_KEY(_ID)?:\\s?(\\S+)",
41+
}
42+
43+
for _, secret := range secretStings {
44+
if len(secret) >= minTokenLength && !containsSpace(secret) {
45+
secretPatterns = append(secretPatterns, secret)
46+
}
47+
}
48+
49+
f.re = regexp.MustCompile("(?i)" + strings.Join(secretPatterns, "|"))
50+
}
51+
52+
// ReplaceAll replaces all secrets in the input line.
53+
func (f *Filtering) ReplaceAll(input []byte) []byte {
54+
return f.re.ReplaceAll(input, []byte(replacingMask))
55+
}
56+
57+
func containsSpace(s string) bool {
58+
for _, v := range s {
59+
if unicode.IsSpace(v) {
60+
return true
61+
}
62+
}
63+
64+
return false
65+
}

‎engine/pkg/log/log.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ func prepareMessage(v ...interface{}) string {
7070
builder := strings.Builder{}
7171

7272
for _, value := range v {
73-
builder.WriteString(" " + toString(value))
73+
builder.WriteString(" " + filter.re.ReplaceAllString(toString(value), replacingMask))
7474
}
7575

7676
return builder.String()

0 commit comments

Comments
 (0)