Merge branch 'filter-logs-output' into 'master'

feat: mask sensitive data in the DLE logs output

See merge request postgres-ai/database-lab!705

Author: agneum

engine/cmd/database-lab/main.go

@@ -56,6 +56,9 @@ func main() {
 		log.Fatal(errors.WithMessage(err, "failed to parse config"))
 	}
 
+	logFilter := log.GetFilter()
+	logFilter.ReloadLogRegExp([]string{cfg.Server.VerificationToken, cfg.Platform.AccessToken})
+
 	config.ApplyGlobals(cfg)
 
 	docker, err := client.NewClientWithOpts(client.FromEnv)
@@ -188,14 +191,16 @@ func main() {
 			embeddedUI,
 			server,
 			logCleaner,
+			logFilter,
 		)
 	}
 
 	server := srv.NewServer(&cfg.Server, &cfg.Global, engProps, docker, cloningSvc, provisioner, retrievalSvc, platformSvc,
-		obs, est, pm, tm, tokenHolder, embeddedUI, reloadConfigFn)
+		obs, est, pm, tm, tokenHolder, logFilter, embeddedUI, reloadConfigFn)
 	shutdownCh := setShutdownListener()
 
-	go setReloadListener(ctx, provisioner, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, embeddedUI, server, logCleaner)
+	go setReloadListener(ctx, provisioner, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, embeddedUI, server,
+		logCleaner, logFilter)
 
 	server.InitHandlers()
 
@@ -276,12 +281,14 @@ func getEngineProperties(ctx context.Context, dockerCLI *client.Client, cfg *con
 
 func reloadConfig(ctx context.Context, provisionSvc *provision.Provisioner, tm *telemetry.Agent,
 	retrievalSvc *retrieval.Retrieval, pm *pool.Manager, cloningSvc *cloning.Base, platformSvc *platform.Service,
-	est *estimator.Estimator, embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner) error {
+	est *estimator.Estimator, embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner,
+	filtering *log.Filtering) error {
 	cfg, err := config.LoadConfiguration()
 	if err != nil {
 		return err
 	}
 
+	filtering.ReloadLogRegExp([]string{cfg.Server.VerificationToken, cfg.Platform.AccessToken})
 	config.ApplyGlobals(cfg)
 
 	if err := provision.IsValidConfig(cfg.Provision); err != nil {
@@ -328,14 +335,16 @@ func reloadConfig(ctx context.Context, provisionSvc *provision.Provisioner, tm *
 
 func setReloadListener(ctx context.Context, provisionSvc *provision.Provisioner, tm *telemetry.Agent,
 	retrievalSvc *retrieval.Retrieval, pm *pool.Manager, cloningSvc *cloning.Base, platformSvc *platform.Service,
-	est *estimator.Estimator, embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner) {
+	est *estimator.Estimator, embeddedUI *embeddedui.UIManager, server *srv.Server, cleaner *diagnostic.Cleaner,
+	logFilter *log.Filtering) {
 	reloadCh := make(chan os.Signal, 1)
 	signal.Notify(reloadCh, syscall.SIGHUP)
 
 	for range reloadCh {
 		log.Msg("Reloading configuration")
 
-		if err := reloadConfig(ctx, provisionSvc, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, embeddedUI, server, cleaner); err != nil {
+		if err := reloadConfig(ctx, provisionSvc, tm, retrievalSvc, pm, cloningSvc, platformSvc, est, embeddedUI, server,
+			cleaner, logFilter); err != nil {
 			log.Err("Failed to reload configuration", err)
 		}
 

engine/internal/srv/server.go

@@ -9,10 +9,8 @@ import (
 	"context"
 	"fmt"
 	"net/http"
-	"regexp"
 	"strings"
 	"time"
-	"unicode"
 
 	"github.com/AlekSi/pointer"
 	"github.com/docker/docker/client"
@@ -41,8 +39,6 @@ import (
 	"gitlab.com/postgres-ai/database-lab/v3/version"
 )
 
-const minTokenLength = 8
-
 // Server defines an HTTP server of the Database Lab.
 type Server struct {
 	validator   validator.Service
@@ -61,7 +57,7 @@ type Server struct {
 	pm          *pool.Manager
 	tm          *telemetry.Agent
 	startedAt   *models.LocalTime
-	re          *regexp.Regexp
+	filtering   *log.Filtering
 	reloadFn    func(server *Server) error
 }
 
@@ -77,7 +73,7 @@ func NewServer(cfg *srvCfg.Config, globalCfg *global.Config, engineProps global.
 	dockerClient *client.Client, cloning *cloning.Base, provisioner *provision.Provisioner,
 	retrievalSvc *retrieval.Retrieval, platform *platform.Service, observer *observer.Observer,
 	estimator *estimator.Estimator, pm *pool.Manager, tm *telemetry.Agent, tokenKeeper *ws.TokenKeeper,
-	uiManager *embeddedui.UIManager, reloadConfigFn func(server *Server) error) *Server {
+	filtering *log.Filtering, uiManager *embeddedui.UIManager, reloadConfigFn func(server *Server) error) *Server {
 	server := &Server{
 		Config:      cfg,
 		Global:      globalCfg,
@@ -96,10 +92,10 @@ func NewServer(cfg *srvCfg.Config, globalCfg *global.Config, engineProps global.
 		docker:    dockerClient,
 		pm:        pm,
 		tm:        tm,
+		filtering: filtering,
 		startedAt: &models.LocalTime{Time: time.Now().Truncate(time.Second)},
 		reloadFn:  reloadConfigFn,
 	}
-	server.initLogRegExp()
 
 	return server
 }
@@ -185,7 +181,6 @@ func attachAPI(r *mux.Router) error {
 // Reload reloads server configuration.
 func (s *Server) Reload(cfg srvCfg.Config) {
 	*s.Config = cfg
-	s.initLogRegExp()
 }
 
 // InitHandlers initializes handler functions of the HTTP server.
@@ -266,31 +261,5 @@ func reportLaunching(cfg *srvCfg.Config) {
 }
 
 func (s *Server) initLogRegExp() {
-	secretPatterns := []string{
-		"password:\\s?(\\S+)",
-		"POSTGRES_PASSWORD=(\\S+)",
-		"PGPASSWORD=(\\S+)",
-		"accessToken:\\s?(\\S+)",
-		"ACCESS_KEY(_ID)?:\\s?(\\S+)",
-	}
-
-	if len(s.Config.VerificationToken) >= minTokenLength && !containsSpace(s.Config.VerificationToken) {
-		secretPatterns = append(secretPatterns, s.Config.VerificationToken)
-	}
-
-	if accessToken := s.Platform.AccessToken(); len(accessToken) >= minTokenLength && !containsSpace(accessToken) {
-		secretPatterns = append(secretPatterns, accessToken)
-	}
-
-	s.re = regexp.MustCompile("(?i)" + strings.Join(secretPatterns, "|"))
-}
-
-func containsSpace(s string) bool {
-	for _, v := range s {
-		if unicode.IsSpace(v) {
-			return true
-		}
-	}
-
-	return false
+	s.filtering.ReloadLogRegExp([]string{s.Config.VerificationToken, s.Platform.AccessToken()})
 }

engine/internal/srv/ws.go

@@ -109,5 +109,5 @@ func (s *Server) instanceLogs(w http.ResponseWriter, r *http.Request) {
 }
 
 func (s *Server) filterLogLine(inputLine []byte) []byte {
-	return s.re.ReplaceAll(inputLine, []byte("********"))
+	return s.filtering.ReplaceAll(inputLine)
 }

engine/internal/srv/ws_test.go

@@ -9,13 +9,18 @@ import (
 
 	"gitlab.com/postgres-ai/database-lab/v3/internal/platform"
 	"gitlab.com/postgres-ai/database-lab/v3/internal/srv/config"
+	"gitlab.com/postgres-ai/database-lab/v3/pkg/log"
 )
 
 func TestLogLineFiltering(t *testing.T) {
 	pl, err := platform.New(context.Background(), platform.Config{AccessToken: "platformAccessToken"})
 	require.NoError(t, err)
 
-	s := Server{Config: &config.Config{VerificationToken: "secretToken"}, Platform: pl}
+	s := Server{
+		Config:    &config.Config{VerificationToken: "secretToken"},
+		Platform:  pl,
+		filtering: log.GetFilter(),
+	}
 	s.initLogRegExp()
 
 	testCases := []struct {

engine/pkg/log/filtering.go

@@ -0,0 +1,65 @@
+package log
+
+import (
+	"regexp"
+	"strings"
+	"unicode"
+)
+
+const (
+	minTokenLength = 8
+	replacingMask  = "********"
+)
+
+var filter = newFiltering([]string{})
+
+// Filtering represents a struct to filter secrets in logs output.
+type Filtering struct {
+	re *regexp.Regexp
+}
+
+// GetFilter gets an instance of Log Filtering.
+func GetFilter() *Filtering {
+	return filter
+}
+
+func newFiltering(tokens []string) *Filtering {
+	f := &Filtering{}
+	f.ReloadLogRegExp(tokens)
+
+	return f
+}
+
+// ReloadLogRegExp updates secrets configuration.
+func (f *Filtering) ReloadLogRegExp(secretStings []string) {
+	secretPatterns := []string{
+		"password:\\s?(\\S+)",
+		"POSTGRES_PASSWORD=(\\S+)",
+		"PGPASSWORD=(\\S+)",
+		"accessToken:\\s?(\\S+)",
+		"ACCESS_KEY(_ID)?:\\s?(\\S+)",
+	}
+
+	for _, secret := range secretStings {
+		if len(secret) >= minTokenLength && !containsSpace(secret) {
+			secretPatterns = append(secretPatterns, secret)
+		}
+	}
+
+	f.re = regexp.MustCompile("(?i)" + strings.Join(secretPatterns, "|"))
+}
+
+// ReplaceAll replaces all secrets in the input line.
+func (f *Filtering) ReplaceAll(input []byte) []byte {
+	return f.re.ReplaceAll(input, []byte(replacingMask))
+}
+
+func containsSpace(s string) bool {
+	for _, v := range s {
+		if unicode.IsSpace(v) {
+			return true
+		}
+	}
+
+	return false
+}

engine/pkg/log/log.go

@@ -70,7 +70,7 @@ func prepareMessage(v ...interface{}) string {
 	builder := strings.Builder{}
 
 	for _, value := range v {
-		builder.WriteString(" " + toString(value))
+		builder.WriteString(" " + filter.re.ReplaceAllString(toString(value), replacingMask))
 	}
 
 	return builder.String()