Skip to content

Commit 94ccf28

Browse files
laveryalaverya
authored
Merge pull request #184 from replicatedhq/laverya/custom-multiline-and-yaml-redactors
custom multiline and yaml redactors
2 parents 7752a33 + 2f092ae commit 94ccf28

File tree

9 files changed

+394
-12
lines changed

9 files changed

+394
-12
lines changed

‎pkg/apis/troubleshoot/v1beta1/redact_shared.go‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,16 @@
11
package v1beta1
22

3+
type MultiLine struct {
4+
Selector string `json:"selector,omitempty" yaml:"selector,omitempty"`
5+
Redactor string `json:"redactor,omitempty" yaml:"redactor,omitempty"`
6+
}
7+
38
type Redact struct {
4-
Name string `json:"name,omitempty" yaml:"name,omitempty"`
5-
File string `json:"file,omitempty" yaml:"file,omitempty"`
6-
Files []string `json:"files,omitempty" yaml:"files,omitempty"`
7-
Values []string `json:"values,omitempty" yaml:"values,omitempty"`
8-
Regex []string `json:"regex,omitempty" yaml:"regex,omitempty"`
9+
Name string `json:"name,omitempty" yaml:"name,omitempty"`
10+
File string `json:"file,omitempty" yaml:"file,omitempty"`
11+
Files []string `json:"files,omitempty" yaml:"files,omitempty"`
12+
Values []string `json:"values,omitempty" yaml:"values,omitempty"`
13+
Regex []string `json:"regex,omitempty" yaml:"regex,omitempty"`
14+
MultiLine []MultiLine `json:"multiLine,omitempty" yaml:"multiLine,omitempty"`
15+
Yaml []string `json:"yaml,omitempty" yaml:"yaml,omitempty"`
916
}

‎pkg/apis/troubleshoot/v1beta1/zz_generated.deepcopy.go‎

Lines changed: 25 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/collect/collector_test.go‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,67 @@ pwd=somethinggoeshere;`,
176176
"data/data/collectorname": `***HIDDEN*** ***HIDDEN***
177177
***HIDDEN*** line here
178178
pwd=***HIDDEN***;
179+
`,
180+
},
181+
},
182+
{
183+
name: "data with custom yaml redactor",
184+
Collect: &troubleshootv1beta1.Collect{
185+
Data: &troubleshootv1beta1.Data{
186+
CollectorMeta: troubleshootv1beta1.CollectorMeta{
187+
CollectorName: "datacollectorname",
188+
Exclude: multitype.BoolOrString{},
189+
},
190+
Name: "data",
191+
Data: `abc 123
192+
another line here`,
193+
},
194+
},
195+
Redactors: []*troubleshootv1beta1.Redact{
196+
{
197+
Yaml: []string{
198+
`abc`,
199+
},
200+
},
201+
},
202+
want: map[string]string{
203+
"data/datacollectorname": `abc 123
204+
another line here
205+
`,
206+
},
207+
},
208+
{
209+
name: "custom multiline redactor",
210+
Collect: &troubleshootv1beta1.Collect{
211+
Data: &troubleshootv1beta1.Data{
212+
CollectorMeta: troubleshootv1beta1.CollectorMeta{
213+
CollectorName: "datacollectorname",
214+
Exclude: multitype.BoolOrString{},
215+
},
216+
Name: "data",
217+
Data: `xyz123
218+
abc
219+
xyz123
220+
xyz123
221+
abc`,
222+
},
223+
},
224+
Redactors: []*troubleshootv1beta1.Redact{
225+
{
226+
MultiLine: []troubleshootv1beta1.MultiLine{
227+
{
228+
Selector: "abc",
229+
Redactor: "xyz(123)",
230+
},
231+
},
232+
},
233+
},
234+
want: map[string]string{
235+
"data/datacollectorname": `xyz123
236+
abc
237+
123
238+
xyz123
239+
abc
179240
`,
180241
},
181242
},

‎pkg/redact/literal.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ func literalString(matchString string) Redactor {
1616
}
1717

1818
func (r literalRedactor) Redact(input io.Reader) io.Reader {
19-
reader, writer := io.Pipe()
19+
out, writer := io.Pipe()
2020

2121
go func() {
2222
var err error
@@ -43,5 +43,5 @@ func (r literalRedactor) Redact(input io.Reader) io.Reader {
4343
}
4444
}
4545
}()
46-
return reader
46+
return out
4747
}

‎pkg/redact/multi_line.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ func NewMultiLineRedactor(re1, re2, maskText string) (*MultiLineRedactor, error)
2626
}
2727

2828
func (r *MultiLineRedactor) Redact(input io.Reader) io.Reader {
29-
reader, writer := io.Pipe()
29+
out, writer := io.Pipe()
3030
go func() {
3131
var err error
3232
defer func() {
@@ -70,7 +70,7 @@ func (r *MultiLineRedactor) Redact(input io.Reader) io.Reader {
7070
fmt.Fprintf(writer, "%s\n", line1)
7171
}
7272
}()
73-
return reader
73+
return out
7474
}
7575

7676
func getNextTwoLines(reader *bufio.Reader, curLine2 *string) (line1 string, line2 string, err error) {

‎pkg/redact/redact.go‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,14 +65,27 @@ func buildAdditionalRedactors(path string, redacts []*troubleshootv1beta1.Redact
6565
for _, re := range redact.Regex {
6666
r, err := NewSingleLineRedactor(re, MASK_TEXT)
6767
if err != nil {
68-
return nil, err // maybe skip broken ones?
68+
return nil, errors.Wrapf(err, "redactor %q", re)
6969
}
7070
additionalRedactors = append(additionalRedactors, r)
7171
}
7272

7373
for _, literal := range redact.Values {
7474
additionalRedactors = append(additionalRedactors, literalString(literal))
7575
}
76+
77+
for _, re := range redact.MultiLine {
78+
r, err := NewMultiLineRedactor(re.Selector, re.Redactor, MASK_TEXT)
79+
if err != nil {
80+
return nil, errors.Wrapf(err, "multiline redactor %+v", re)
81+
}
82+
additionalRedactors = append(additionalRedactors, r)
83+
}
84+
85+
for _, yaml := range redact.Yaml {
86+
r := NewYamlRedactor(yaml)
87+
additionalRedactors = append(additionalRedactors, r)
88+
}
7689
}
7790
return additionalRedactors, nil
7891
}

‎pkg/redact/single_line.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ func NewSingleLineRedactor(re, maskText string) (*SingleLineRedactor, error) {
2121
}
2222

2323
func (r *SingleLineRedactor) Redact(input io.Reader) io.Reader {
24-
reader, writer := io.Pipe()
24+
out, writer := io.Pipe()
2525

2626
go func() {
2727
var err error
@@ -57,5 +57,5 @@ func (r *SingleLineRedactor) Redact(input io.Reader) io.Reader {
5757
}
5858
}
5959
}()
60-
return reader
60+
return out
6161
}

‎pkg/redact/yaml.go‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
package redact
2+
3+
import (
4+
"bufio"
5+
"bytes"
6+
"io"
7+
"io/ioutil"
8+
"strconv"
9+
"strings"
10+
11+
"gopkg.in/yaml.v2"
12+
)
13+
14+
type YamlRedactor struct {
15+
maskPath []string
16+
foundMatch bool
17+
}
18+
19+
func NewYamlRedactor(yamlPath string) *YamlRedactor {
20+
pathComponents := strings.Split(yamlPath, ".")
21+
return &YamlRedactor{maskPath: pathComponents}
22+
}
23+
24+
func (r *YamlRedactor) Redact(input io.Reader) io.Reader {
25+
reader, writer := io.Pipe()
26+
go func() {
27+
var err error
28+
defer func() {
29+
if err == io.EOF {
30+
writer.Close()
31+
} else {
32+
writer.CloseWithError(err)
33+
}
34+
}()
35+
reader := bufio.NewReader(input)
36+
37+
var doc []byte
38+
doc, err = ioutil.ReadAll(reader)
39+
var yamlInterface interface{}
40+
err = yaml.Unmarshal(doc, &yamlInterface)
41+
if err != nil {
42+
buf := bytes.NewBuffer(doc)
43+
buf.WriteTo(writer)
44+
err = nil // this is not a fatal error
45+
return
46+
}
47+
48+
newYaml := r.redactYaml(yamlInterface, r.maskPath)
49+
if !r.foundMatch {
50+
// no match found, so make no changes
51+
buf := bytes.NewBuffer(doc)
52+
buf.WriteTo(writer)
53+
return
54+
}
55+
56+
var newBytes []byte
57+
newBytes, err = yaml.Marshal(newYaml)
58+
if err != nil {
59+
return
60+
}
61+
62+
buf := bytes.NewBuffer(newBytes)
63+
buf.WriteTo(writer)
64+
return
65+
}()
66+
return reader
67+
}
68+
69+
func (r *YamlRedactor) redactYaml(in interface{}, path []string) interface{} {
70+
if len(path) == 0 {
71+
r.foundMatch = true
72+
return MASK_TEXT
73+
}
74+
switch typed := in.(type) {
75+
case []interface{}:
76+
// check if first path element is * - if it is, run redact on all children
77+
if path[0] == "*" {
78+
var newArr []interface{}
79+
for _, child := range typed {
80+
newChild := r.redactYaml(child, path[1:])
81+
newArr = append(newArr, newChild)
82+
}
83+
return newArr
84+
}
85+
// check if first path element is an integer - if it is, run redact on that child
86+
pathIdx, err := strconv.Atoi(path[0])
87+
if err != nil {
88+
return typed
89+
}
90+
if len(typed) > pathIdx {
91+
child := typed[pathIdx]
92+
typed[pathIdx] = r.redactYaml(child, path[1:])
93+
return typed
94+
}
95+
return typed
96+
case map[interface{}]interface{}:
97+
child, ok := typed[path[0]]
98+
if ok {
99+
newChild := r.redactYaml(child, path[1:])
100+
typed[path[0]] = newChild
101+
}
102+
return typed
103+
default:
104+
return typed
105+
}
106+
}

0 commit comments

Comments
 (0)