add global and per-collector redactors

add redact type, and begin wiring global redactors

use per-collector redactors

add a test of the 'data' collector and redaction

handle literal string replacements

remove redundant types and redact calls

add proper redactor type, foundations of global redactors

accept global redactors from the CLI, include sample redaction spec

Author: laverya

cmd/troubleshoot/cli/root.go

@@ -43,6 +43,7 @@ from a server that can be used to assist when troubleshooting a server.`,
 	cmd.Flags().String("collectors", "", "name of the collectors to use")
 	cmd.Flags().String("image", "", "the full name of the collector image to use")
 	cmd.Flags().String("pullpolicy", "", "the pull policy of the collector image")
+	cmd.Flags().String("redactors", "", "name of the additional redactors to use")
 	cmd.Flags().Bool("redact", true, "enable/disable default redactions")
 	cmd.Flags().Bool("collect-without-permissions", false, "always run troubleshoot collectors even if some require permissions that troubleshoot does not have")
 

cmd/troubleshoot/cli/run.go

@@ -56,6 +56,23 @@ func runTroubleshoot(v *viper.Viper, arg string) error {
 
 	collector := obj.(*troubleshootv1beta1.Collector)
 
+	var additionalRedactors *troubleshootv1beta1.Redactor
+	if v.GetString("redactors") != "" {
+		redactorContent, err := loadSpec(v, v.GetString("redactors"))
+		if err != nil {
+			return errors.Wrap(err, "failed to load redactor spec")
+		}
+		obj, _, err := decode([]byte(redactorContent), nil, nil)
+		if err != nil {
+			return errors.Wrapf(err, "failed to parse redactors %s", v.GetString("redactors"))
+		}
+		var ok bool
+		additionalRedactors, ok = obj.(*troubleshootv1beta1.Redactor)
+		if !ok {
+			return fmt.Errorf("%s is not a troubleshootv1beta1 redactor type", v.GetString("redactors"))
+		}
+	}
+
 	s := spin.New()
 	finishedCh := make(chan bool, 1)
 	progressChan := make(chan interface{}, 0) // non-zero buffer can result in missed messages
@@ -87,7 +104,7 @@ func runTroubleshoot(v *viper.Viper, arg string) error {
 		close(finishedCh)
 	}()
 
-	archivePath, err := runCollectors(v, *collector, progressChan)
+	archivePath, err := runCollectors(v, *collector, additionalRedactors, progressChan)
 	if err != nil {
 		return errors.Wrap(err, "run collectors")
 	}
@@ -193,7 +210,7 @@ func canTryInsecure(v *viper.Viper) bool {
 	return true
 }
 
-func runCollectors(v *viper.Viper, collector troubleshootv1beta1.Collector, progressChan chan interface{}) (string, error) {
+func runCollectors(v *viper.Viper, collector troubleshootv1beta1.Collector, additionalRedactors *troubleshootv1beta1.Redactor, progressChan chan interface{}) (string, error) {
 	bundlePath, err := ioutil.TempDir("", "troubleshoot")
 	if err != nil {
 		return "", errors.Wrap(err, "create temp dir")
@@ -241,6 +258,11 @@ func runCollectors(v *viper.Viper, collector troubleshootv1beta1.Collector, prog
 		return "", errors.New("insufficient permissions to run all collectors")
 	}
 
+	globalRedactors := []*troubleshootv1beta1.Redact{}
+	if additionalRedactors != nil {
+		globalRedactors = additionalRedactors.Spec.Redactors
+	}
+
 	// Run preflights collectors synchronously
 	for _, collector := range collectors {
 		if len(collector.RBACErrors) > 0 {
@@ -253,7 +275,7 @@ func runCollectors(v *viper.Viper, collector troubleshootv1beta1.Collector, prog
 
 		progressChan <- collector.GetDisplayName()
 
-		result, err := collector.RunCollectorSync()
+		result, err := collector.RunCollectorSync(globalRedactors)
 		if err != nil {
 			progressChan <- fmt.Errorf("failed to run collector %q: %v", collector.GetDisplayName(), err)
 			continue

pkg/apis/troubleshoot/v1beta1/collector_shared.go

@@ -9,7 +9,8 @@ import (
 )
 
 type CollectorMeta struct {
-	CollectorName string `json:"collectorName,omitempty" yaml:"collectorName,omitempty"`
+	CollectorName string    `json:"collectorName,omitempty" yaml:"collectorName,omitempty"`
+	Redactors     []*Redact `json:"redactors,omitempty" yaml:"redactors,omitempty"`
 	// +optional
 	Exclude multitype.BoolOrString `json:"exclude,omitempty" yaml:"exclude,omitempty"`
 }

pkg/apis/troubleshoot/v1beta1/collector_types.go

@@ -34,6 +34,7 @@ type AfterCollection struct {
 type CollectorSpec struct {
 	Collectors      []*Collect         `json:"collectors,omitempty" yaml:"collectors,omitempty"`
 	AfterCollection []*AfterCollection `json:"afterCollection,omitempty" yaml:"afterCollection,omitempty"`
+	GlobalRedactors []*Redact          `json:"globalRedactors,omitempty" yaml:"globalRedactors,omitempty"`
 }
 
 // CollectorStatus defines the observed state of Collector

pkg/apis/troubleshoot/v1beta1/redact_shared.go

@@ -0,0 +1,9 @@
+package v1beta1
+
+type Redact struct {
+	Name   string   `json:"name,omitempty" yaml:"name,omitempty"`
+	File   string   `json:"file,omitempty" yaml:"file,omitempty"`
+	Files  []string `json:"files,omitempty" yaml:"files,omitempty"`
+	Values []string `json:"values,omitempty" yaml:"values,omitempty"`
+	Regex  []string `json:"regex,omitempty" yaml:"regex,omitempty"`
+}

pkg/apis/troubleshoot/v1beta1/redact_types.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2019 Replicated, Inc..
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// RedactorSpec defines the desired state of Redactor
+type RedactorSpec struct {
+	Redactors []*Redact `json:"redacts,omitempty"`
+}
+
+// RedactorStatus defines the observed state of Redactor
+type RedactorStatus struct {
+}
+
+// +genclient
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// Redactor is the Schema for the redaction API
+// +k8s:openapi-gen=true
+type Redactor struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   RedactorSpec   `json:"spec,omitempty"`
+	Status RedactorStatus `json:"status,omitempty"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// RedactorList contains a list of Redactor
+type RedactorList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []Redactor `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&Redactor{}, &RedactorList{})
+}