chore: solving conflicts from release branch merging (#377)

Author: danlucian

pkg/analytics/analytics.go

@@ -2,6 +2,7 @@ package analytics
 
 import (
 	"bytes"
+	"github.com/snyk/go-application-framework/pkg/logging"
 
 	//nolint:gosec // insecure sha1 used for legacy identifier
 	"crypto/sha1"
@@ -103,22 +104,8 @@ type dataOutput struct {
 	Data analyticsOutput `json:"data"`
 }
 
-var (
-	// sensitiveFieldNames is a list of field names that should be sanitized.
-	// data sanitization is used to prevent sensitive data from being sent to the analytics server.
-	sensitiveFieldNames = []string{
-		"headers",
-		"user",
-		"passw",
-		"token",
-		"key",
-		"secret",
-	}
-)
-
 const (
-	sanitizeReplacementString string = "REDACTED"
-	apiEndpoint               string = "/v1/analytics/cli"
+	apiEndpoint string = "/v1/analytics/cli"
 )
 
 // New creates a new Analytics instance.
@@ -266,7 +253,7 @@ func (a *AnalyticsImpl) GetRequest() (*http.Request, error) {
 		return nil, err
 	}
 
-	outputJson, err = SanitizeValuesByKey(sensitiveFieldNames, sanitizeReplacementString, outputJson)
+	outputJson, err = SanitizeValuesByKey(logging.SENSITIVE_FIELD_NAMES, logging.SANITIZE_REPLACEMENT_STRING, outputJson)
 	if err != nil {
 		return nil, err
 	}
@@ -275,7 +262,7 @@ func (a *AnalyticsImpl) GetRequest() (*http.Request, error) {
 	if err != nil {
 		return nil, err
 	}
-	outputJson, err = SanitizeUsername(user.Username, user.HomeDir, sanitizeReplacementString, outputJson)
+	outputJson, err = SanitizeUsername(user.Username, user.HomeDir, logging.SANITIZE_REPLACEMENT_STRING, outputJson)
 	if err != nil {
 		return nil, err
 	}

pkg/analytics/analytics_test.go

@@ -3,6 +3,7 @@ package analytics
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/snyk/go-application-framework/pkg/logging"
 	"io"
 	"net/http"
 	"os/user"
@@ -74,7 +75,7 @@ func Test_Basic(t *testing.T) {
 			body, err := io.ReadAll(request.Body)
 			assert.Nil(t, err)
 			// expect no CLI args to be sent to analytics (CLI-586)
-			assert.Equal(t, 0, strings.Count(string(body), sanitizeReplacementString))
+			assert.Equal(t, 0, strings.Count(string(body), logging.SANITIZE_REPLACEMENT_STRING))
 
 			var requestBody dataOutput
 			err = json.Unmarshal(body, &requestBody)
@@ -118,11 +119,11 @@ func Test_SanitizeValuesByKey(t *testing.T) {
 	}
 
 	// test input
-	filter := sensitiveFieldNames
+	filter := logging.SENSITIVE_FIELD_NAMES
 	input, err := json.Marshal(inputStruct)
 	assert.NoError(t, err)
 
-	replacement := sanitizeReplacementString
+	replacement := logging.SANITIZE_REPLACEMENT_STRING
 
 	fmt.Println("Before: " + string(input))
 
@@ -171,7 +172,7 @@ func Test_SanitizeUsername(t *testing.T) {
 	// 2. with domain name
 	// 3. user name and path are different
 	// 4. current OS values
-	replacement := "REDACTED"
+	replacement := "***"
 	inputData := []input{
 		{
 			userName:     "some.user",

pkg/analytics/instrumentation_collector.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"github.com/snyk/go-application-framework/pkg/logging"
 	"os/user"
 	"time"
 
@@ -180,7 +181,7 @@ func (ic *instrumentationCollectorImpl) sanitizeExtensionData(logger *zerolog.Lo
 	}
 
 	var sanitized []byte
-	sanitized, err = SanitizeValuesByKey(sensitiveFieldNames, sanitizeReplacementString, extension)
+	sanitized, err = SanitizeValuesByKey(logging.SENSITIVE_FIELD_NAMES, logging.SANITIZE_REPLACEMENT_STRING, extension)
 	if err != nil {
 		logger.Printf("failed to sanitize extension, removing object from analytics payload as sanitzation was not possible: %v", err)
 		return result
@@ -192,7 +193,7 @@ func (ic *instrumentationCollectorImpl) sanitizeExtensionData(logger *zerolog.Lo
 		return result
 	}
 
-	sanitized, err = SanitizeUsername(u.Username, u.HomeDir, sanitizeReplacementString, sanitized)
+	sanitized, err = SanitizeUsername(u.Username, u.HomeDir, logging.SANITIZE_REPLACEMENT_STRING, sanitized)
 	if err != nil {
 		logger.Printf("failed to sanitize user information in extension payload, removing object from analytics payload as sanitzation was not possible: %v", err)
 		return result

pkg/analytics/instrumentation_collector_test.go

@@ -214,7 +214,7 @@ func Test_InstrumentationCollector(t *testing.T) {
 
 		mockExtension := map[string]interface{}{
 			"strings":  "hello world",
-			"password": "REDACTED",
+			"password": "***",
 		}
 
 		expectedV2InstrumentationObject.Data.Attributes.Interaction.Extension = &mockExtension

pkg/logging/scrubbingLogWriter.go

@@ -21,6 +21,7 @@ import (
 	"io"
 	"os/user"
 	"regexp"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -31,8 +32,18 @@ import (
 	"github.com/snyk/go-application-framework/pkg/configuration"
 )
 
-const redactMask string = "***"
 const MAX_WRITE_RETRIES = 10
+const SANITIZE_REPLACEMENT_STRING string = "***"
+
+// SENSITIVE_FIELD_NAMES is a list of field names that should be sanitized.
+var SENSITIVE_FIELD_NAMES = []string{
+	"headers",
+	"user",
+	"passw",
+	"token",
+	"key",
+	"secret",
+}
 
 type ScrubbingLogWriter interface {
 	AddTerm(term string, matchGroup int)
@@ -146,43 +157,51 @@ func addMandatoryMasking(dict ScrubbingDict) ScrubbingDict {
 		groupToRedact: 3,
 		regex:         regexp.MustCompile(s),
 	}
+
 	s = fmt.Sprintf(`([t|T]oken )(%s)`, charGroup)
 	dict[s] = scrubStruct{
 		groupToRedact: 2,
 		regex:         regexp.MustCompile(s),
 	}
+
 	s = fmt.Sprintf(`([b|B]earer )(%s)`, charGroup)
 	dict[s] = scrubStruct{
 		groupToRedact: 2,
 		regex:         regexp.MustCompile(s),
 	}
+
 	s = fmt.Sprintf(`([b|B]asic )(%s)`, charGroup)
 	dict[s] = scrubStruct{
 		groupToRedact: 2,
 		regex:         regexp.MustCompile(s),
 	}
+
 	s = fmt.Sprintf("(gh[ps])_(%s)", charGroup)
 	dict[s] = scrubStruct{
 		groupToRedact: 2,
 		regex:         regexp.MustCompile(s),
 	}
+
 	s = fmt.Sprintf("(github_pat_)(%s)", charGroup)
 	dict[s] = scrubStruct{
 		groupToRedact: 2,
 		regex:         regexp.MustCompile(s),
 	}
+
 	// github
-	s = fmt.Sprintf("(access_token=)(%s)&", charGroup)
+	s = fmt.Sprintf(`(access_token[\\="\s:]+)(%s)&?`, charGroup)
 	dict[s] = scrubStruct{
 		groupToRedact: 2,
 		regex:         regexp.MustCompile(s),
 	}
-	s = fmt.Sprintf("(refresh_token=)(%s)&", charGroup)
+
+	s = fmt.Sprintf(`(refresh_token[\\="\s:]+)(%s)&?`, charGroup)
 	dict[s] = scrubStruct{
 		groupToRedact: 2,
 		regex:         regexp.MustCompile(s),
 	}
-	s = fmt.Sprintf(`("token":)"(%s)"`, charGroup)
+
+	s = fmt.Sprintf(`(token[\\="\s:]+)(%s)&?`, charGroup)
 	dict[s] = scrubStruct{
 		groupToRedact: 2,
 		regex:         regexp.MustCompile(s),
@@ -194,12 +213,72 @@ func addMandatoryMasking(dict ScrubbingDict) ScrubbingDict {
 		regex:         regexp.MustCompile(s),
 	}
 
+	// Hide whatever is the current username
 	u, err := user.Current()
 	if err == nil {
 		s = fmt.Sprintf(`\b%s\b`, regexp.QuoteMeta(u.Username))
 		addTermToDict(s, 0, dict)
 	}
 
+	// The legacy CLI's snyk-config package prints the entire configuration in debug mode.
+	// It begins with some pseudo-JSON structure, which we can redact.
+	s = `(?s)_:\s*\[(?<everything_inside_hard_brackets>.*)\]`
+	dict[s] = scrubStruct{
+		groupToRedact: 1,
+		regex:         regexp.MustCompile(s),
+	}
+
+	// JSON-formatted data, in general
+	kws := strings.Join(SENSITIVE_FIELD_NAMES, "|")
+	s = fmt.Sprintf(`(?i)"[^"]*(?<json_key>%s)[^"]*"\s*:\s*"(?<json_value>[^"]*)"`, kws)
+	dict[s] = scrubStruct{
+		groupToRedact: 2,
+		regex:         regexp.MustCompile(s),
+	}
+
+	// CLI argument mapping from the snyk-config debug logging
+	// I.e., if --argument=value is passed, it will be logged as { 'argument=value': true }
+	s = fmt.Sprintf(`(?im)(%s)[^=]*=(?P<value>.*)['"]`, kws)
+	dict[s] = scrubStruct{
+		groupToRedact: 2,
+		regex:         regexp.MustCompile(s),
+	}
+
+	// Same as above, only with short form
+	shorts := []string{"p", "u"}
+	shortForm := strings.Join(shorts, "")
+	s = fmt.Sprintf(`(?im)'[%s]=(?<value>.*)'`, shortForm)
+	dict[s] = scrubStruct{
+		groupToRedact: 2,
+		regex:         regexp.MustCompile(s),
+	}
+
+	// Specific short-form scrubbing of the JSON-ish log structures
+	// Appear in the snyk-config debug logging as various constellations of { 'u': 'john.doe', } with or without quotes,
+	// and values can contain spaces, double and/or single quotes.
+
+	s = fmt.Sprintf(`(?i)(?<short_form_key>\b[%s]\b)[,'":]+\s*(?:['"](?<short_form_value>.*)['"]|([^,'"\s]+))[,}]?`, shortForm)
+	dict[s] = scrubStruct{
+		groupToRedact: 2,
+		regex:         regexp.MustCompile(s),
+	}
+
+	// CLI argument-style-specific scrubbing
+	// Many cases are already covered by the JSON scrubbing above, thus this might seem incomplete.
+	// Refer to the unit tests for the full set of covered cases.
+	s = fmt.Sprintf(`(?im)\-[%s][\s=](?<short_form_value>\S*)`, shortForm)
+	dict[s] = scrubStruct{
+		groupToRedact: 1,
+		regex:         regexp.MustCompile(s),
+	}
+
+	// Long-form, rest is covered by the JSON scrubbing above
+	s = fmt.Sprintf(`(?im)--(?<argument_key>[^=\s]*(?:%s)[^=\s]*)[\s=]['"]?(?<argument_value>\S*)['"]?`, kws)
+	dict[s] = scrubStruct{
+		groupToRedact: 2,
+		regex:         regexp.MustCompile(s),
+	}
+
 	return dict
 }
 
@@ -212,10 +291,22 @@ func (w *scrubbingLevelWriter) Write(p []byte) (int, error) {
 
 func scrub(p []byte, scrubDict ScrubbingDict) []byte {
 	s := string(p)
-	for _, entry := range scrubDict {
+
+	// The dictionary order is important here, as we want potentially overlapping regexes to be applied
+	// in a specific order every time. Since dictionaries are unordered, we sort the keys here.
+	keys := make([]string, 0, len(scrubDict))
+	for k := range scrubDict {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, key := range keys {
+		entry := scrubDict[key]
 		matches := entry.regex.FindAllStringSubmatch(s, -1)
 		for _, match := range matches {
-			s = strings.Replace(s, match[entry.groupToRedact], redactMask, -1)
+			if entry.groupToRedact >= len(match) || match[entry.groupToRedact] == "" {
+				continue
+			}
+			s = strings.Replace(s, match[entry.groupToRedact], SANITIZE_REPLACEMENT_STRING, -1)
 		}
 	}
 	return []byte(s)