Merge pull request #198 from replicatedhq/laverya/redaction-reports

redaction reports framework

Author: laverya

cmd/troubleshoot/cli/run.go

@@ -1,7 +1,9 @@
 package cli
 
 import (
+	"bytes"
 	"crypto/tls"
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -22,6 +24,7 @@ import (
 	"github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme"
 	troubleshootclientsetscheme "github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme"
 	"github.com/replicatedhq/troubleshoot/pkg/collect"
+	"github.com/replicatedhq/troubleshoot/pkg/redact"
 	"github.com/spf13/viper"
 	spin "github.com/tj/go-spin"
 )
@@ -370,6 +373,33 @@ func uploadSupportBundle(r *troubleshootv1beta1.ResultRequest, archivePath strin
 		return fmt.Errorf("unexpected status code %d", resp.StatusCode)
 	}
 
+	// send redaction report
+	if r.RedactURI != "" {
+		type PutSupportBundleRedactions struct {
+			Redactions redact.RedactionList `json:"redactions"`
+		}
+
+		redactBytes, err := json.Marshal(PutSupportBundleRedactions{Redactions: redact.GetRedactionList()})
+		if err != nil {
+			return errors.Wrap(err, "get redaction report")
+		}
+
+		req, err := http.NewRequest("PUT", r.RedactURI, bytes.NewReader(redactBytes))
+		if err != nil {
+			return errors.Wrap(err, "create redaction report request")
+		}
+		req.ContentLength = int64(len(redactBytes))
+
+		resp, err := httpClient.Do(req)
+		if err != nil {
+			return errors.Wrap(err, "execute redaction request")
+		}
+
+		if resp.StatusCode >= 300 {
+			return fmt.Errorf("unexpected redaction status code %d", resp.StatusCode)
+		}
+	}
+
 	return nil
 }
 

pkg/apis/troubleshoot/v1beta1/collector_types.go

@@ -21,8 +21,9 @@ import (
 )
 
 type ResultRequest struct {
-	URI    string `json:"uri" yaml:"uri"`
-	Method string `json:"method" yaml:"method"`
+	URI       string `json:"uri" yaml:"uri"`
+	Method    string `json:"method" yaml:"method"`
+	RedactURI string `json:"redactUri" yaml:"redactUri"` // the URI to POST redaction reports to
 }
 
 type AfterCollection struct {

pkg/redact/literal.go

@@ -9,10 +9,16 @@ import (
 
 type literalRedactor struct {
 	matchString string
+	filePath    string
+	redactName  string
 }
 
-func literalString(matchString string) Redactor {
-	return literalRedactor{matchString: matchString}
+func literalString(matchString, path, name string) Redactor {
+	return literalRedactor{
+		matchString: matchString,
+		filePath:    path,
+		redactName:  name,
+	}
 }
 
 func (r literalRedactor) Redact(input io.Reader) io.Reader {
@@ -29,18 +35,31 @@ func (r literalRedactor) Redact(input io.Reader) io.Reader {
 		}()
 
 		reader := bufio.NewReader(input)
+		lineNum := 0
 		for {
+			lineNum++
 			var line string
 			line, err = readLine(reader)
 			if err != nil {
 				return
 			}
 
+			clean := strings.ReplaceAll(line, r.matchString, MASK_TEXT)
+
 			// io.WriteString would be nicer, but scanner strips new lines
-			fmt.Fprintf(writer, "%s\n", strings.ReplaceAll(line, r.matchString, MASK_TEXT))
+			fmt.Fprintf(writer, "%s\n", clean)
 			if err != nil {
 				return
 			}
+
+			if clean != line {
+				addRedaction(Redaction{
+					RedactorName:      r.redactName,
+					CharactersRemoved: len(line) - len(clean),
+					Line:              lineNum,
+					File:              r.filePath,
+				})
+			}
 		}
 	}()
 	return out

pkg/redact/multi_line.go

@@ -8,12 +8,14 @@ import (
 )
 
 type MultiLineRedactor struct {
-	re1      *regexp.Regexp
-	re2      *regexp.Regexp
-	maskText string
+	re1        *regexp.Regexp
+	re2        *regexp.Regexp
+	maskText   string
+	filePath   string
+	redactName string
 }
 
-func NewMultiLineRedactor(re1, re2, maskText string) (*MultiLineRedactor, error) {
+func NewMultiLineRedactor(re1, re2, maskText, path, name string) (*MultiLineRedactor, error) {
 	compiled1, err := regexp.Compile(re1)
 	if err != nil {
 		return nil, err
@@ -22,7 +24,7 @@ func NewMultiLineRedactor(re1, re2, maskText string) (*MultiLineRedactor, error)
 	if err != nil {
 		return nil, err
 	}
-	return &MultiLineRedactor{re1: compiled1, re2: compiled2, maskText: maskText}, nil
+	return &MultiLineRedactor{re1: compiled1, re2: compiled2, maskText: maskText, filePath: path, redactName: name}, nil
 }
 
 func (r *MultiLineRedactor) Redact(input io.Reader) io.Reader {
@@ -45,7 +47,10 @@ func (r *MultiLineRedactor) Redact(input io.Reader) io.Reader {
 		}
 
 		flushLastLine := false
+		lineNum := 1
 		for err == nil {
+			lineNum++ // the first line that can be redacted is line 2
+
 			// If line1 matches re1, then transform line2 using re2
 			if !r.re1.MatchString(line1) {
 				fmt.Fprintf(writer, "%s\n", line1)
@@ -63,6 +68,16 @@ func (r *MultiLineRedactor) Redact(input io.Reader) io.Reader {
 				return
 			}
 
+			// if clean is not equal to line2, a redaction was performed
+			if clean != line2 {
+				addRedaction(Redaction{
+					RedactorName:      r.redactName,
+					CharactersRemoved: len(line2) - len(clean),
+					Line:              lineNum,
+					File:              r.filePath,
+				})
+			}
+
 			line1, line2, err = getNextTwoLines(reader, nil)
 		}
 

pkg/redact/redact.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"io/ioutil"
 	"regexp"
+	"sync"
 
 	"github.com/gobwas/glob"
 	"github.com/pkg/errors"
@@ -17,12 +18,36 @@ const (
 	MASK_TEXT = "***HIDDEN***"
 )
 
+var allRedactions RedactionList
+var redactionListMut sync.Mutex
+var pendingRedactions sync.WaitGroup
+
+func init() {
+	allRedactions = RedactionList{
+		ByRedactor: map[string][]Redaction{},
+		ByFile:     map[string][]Redaction{},
+	}
+}
+
 type Redactor interface {
 	Redact(input io.Reader) io.Reader
 }
 
+// Redactions are indexed both by the file affected and by the name of the redactor
+type RedactionList struct {
+	ByRedactor map[string][]Redaction
+	ByFile     map[string][]Redaction
+}
+
+type Redaction struct {
+	RedactorName      string
+	CharactersRemoved int
+	Line              int
+	File              string
+}
+
 func Redact(input []byte, path string, additionalRedactors []*troubleshootv1beta1.Redact) ([]byte, error) {
-	redactors, err := getRedactors()
+	redactors, err := getRedactors(path)
 	if err != nil {
 		return nil, err
 	}
@@ -46,9 +71,25 @@ func Redact(input []byte, path string, additionalRedactors []*troubleshootv1beta
 	return redacted, nil
 }
 
+func GetRedactionList() RedactionList {
+	pendingRedactions.Wait()
+	redactionListMut.Lock()
+	defer redactionListMut.Unlock()
+	return allRedactions
+}
+
+func ResetRedactionList() {
+	redactionListMut.Lock()
+	defer redactionListMut.Unlock()
+	allRedactions = RedactionList{
+		ByRedactor: map[string][]Redaction{},
+		ByFile:     map[string][]Redaction{},
+	}
+}
+
 func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta1.Redact) ([]Redactor, error) {
 	additionalRedactors := []Redactor{}
-	for _, redact := range redacts {
+	for i, redact := range redacts {
 		if redact == nil {
 			continue
 		}
@@ -62,28 +103,30 @@ func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta1.Redact
 			continue
 		}
 
+		withinRedactNum := 0 // give unique redaction names
+
 		for _, re := range redact.Regex {
-			r, err := NewSingleLineRedactor(re, MASK_TEXT)
+			r, err := NewSingleLineRedactor(re, MASK_TEXT, path, redactorName(i, withinRedactNum, redact.Name, "regex", ""))
 			if err != nil {
 				return nil, errors.Wrapf(err, "redactor %q", re)
 			}
 			additionalRedactors = append(additionalRedactors, r)
 		}
 
 		for _, literal := range redact.Values {
-			additionalRedactors = append(additionalRedactors, literalString(literal))
+			additionalRedactors = append(additionalRedactors, literalString(literal, path, redactorName(i, withinRedactNum, redact.Name, "literal", "")))
 		}
 
 		for _, re := range redact.MultiLine {
-			r, err := NewMultiLineRedactor(re.Selector, re.Redactor, MASK_TEXT)
+			r, err := NewMultiLineRedactor(re.Selector, re.Redactor, MASK_TEXT, path, redactorName(i, withinRedactNum, redact.Name, "multiLine", ""))
 			if err != nil {
 				return nil, errors.Wrapf(err, "multiline redactor %+v", re)
 			}
 			additionalRedactors = append(additionalRedactors, r)
 		}
 
 		for _, yaml := range redact.Yaml {
-			r := NewYamlRedactor(yaml)
+			r := NewYamlRedactor(yaml, path, redactorName(i, withinRedactNum, redact.Name, "yaml", ""))
 			additionalRedactors = append(additionalRedactors, r)
 		}
 	}
@@ -122,7 +165,7 @@ func redactMatchesPath(path string, redact *troubleshootv1beta1.Redact) (bool, e
 	return false, nil
 }
 
-func getRedactors() ([]Redactor, error) {
+func getRedactors(path string) ([]Redactor, error) {
 	// TODO: Make this configurable
 
 	// (?i) makes it case insensitive
@@ -159,8 +202,8 @@ func getRedactors() ([]Redactor, error) {
 	}
 
 	redactors := make([]Redactor, 0)
-	for _, re := range singleLines {
-		r, err := NewSingleLineRedactor(re, MASK_TEXT)
+	for i, re := range singleLines {
+		r, err := NewSingleLineRedactor(re, MASK_TEXT, path, redactorName(-1, i, "", "defaultRegex", re))
 		if err != nil {
 			return nil, err // maybe skip broken ones?
 		}
@@ -201,8 +244,8 @@ func getRedactors() ([]Redactor, error) {
 		},
 	}
 
-	for _, l := range doubleLines {
-		r, err := NewMultiLineRedactor(l.line1, l.line2, MASK_TEXT)
+	for i, l := range doubleLines {
+		r, err := NewMultiLineRedactor(l.line1, l.line2, MASK_TEXT, path, redactorName(-1, i, "", "defaultMultiLine", l.line1))
 		if err != nil {
 			return nil, err // maybe skip broken ones?
 		}
@@ -247,3 +290,24 @@ func readLine(r *bufio.Reader) (string, error) {
 	}
 	return string(completeLine), nil
 }
+
+func addRedaction(redaction Redaction) {
+	pendingRedactions.Add(1)
+	go func(redaction Redaction) {
+		redactionListMut.Lock()
+		defer redactionListMut.Unlock()
+		defer pendingRedactions.Done()
+		allRedactions.ByRedactor[redaction.RedactorName] = append(allRedactions.ByRedactor[redaction.RedactorName], redaction)
+		allRedactions.ByFile[redaction.File] = append(allRedactions.ByFile[redaction.File], redaction)
+	}(redaction)
+}
+
+func redactorName(redactorNum, withinRedactorNum int, redactorName, redactorType, redactorLiteral string) string {
+	if redactorName != "" {
+		return fmt.Sprintf("%s-%d", redactorName, withinRedactorNum)
+	}
+	if redactorLiteral == "" {
+		return fmt.Sprintf("unnamed-%d.%d-%s", redactorNum, withinRedactorNum, redactorType)
+	}
+	return fmt.Sprintf("%s.%d-%q", redactorType, withinRedactorNum, redactorLiteral)
+}

pkg/redact/redact_test.go

@@ -1620,11 +1620,14 @@ func Test_Redactors(t *testing.T) {
 		}
 	  ]`
 
+	wantRedactionsLen := 38
+	wantRedactionsCount := 25
+
 	t.Run("test default redactors", func(t *testing.T) {
 		scopetest := scopeagent.StartTest(t)
 		defer scopetest.End()
 		req := require.New(t)
-		redactors, err := getRedactors()
+		redactors, err := getRedactors("testpath")
 		req.NoError(err)
 
 		nextReader := io.Reader(strings.NewReader(original))
@@ -1636,6 +1639,11 @@ func Test_Redactors(t *testing.T) {
 		req.NoError(err)
 
 		req.JSONEq(expected, string(redacted))
+
+		actualRedactions := GetRedactionList()
+		ResetRedactionList()
+		req.Len(actualRedactions.ByFile["testpath"], wantRedactionsLen)
+		req.Len(actualRedactions.ByRedactor, wantRedactionsCount)
 	})
 }