feat: add loader APIs to load specs from raw troubleshoot spec (#1202)

* feat: add loader APIs to load specs from a list of yaml docs

The change introduces a loader package that will contain loader
public APIs. The aim of these APIs will be to, given any source of
troubleshoot specs, the loaders will fetch the specs and parse out
all troubleshoot objects that can be extracted.

* Some refactoring

* Some more changes

* More changes caught when testing vendor portal

* Add tests and rename Troubleshoot kinds struct

* Additional test

* Handle ConfigMap and Secrets with multiple specs in them

* Fix failing test

* Revert multidoc split implementation

* Fix merge conflict

* Change LoadFromXXX functions to a single LoadSpecs function

Author: banjoh

cmd/analyze/cli/run.go

@@ -7,7 +7,7 @@ import (
 	"os"
 
 	"github.com/pkg/errors"
-	"github.com/replicatedhq/troubleshoot/cmd/util"
+	"github.com/replicatedhq/troubleshoot/internal/util"
 	analyzer "github.com/replicatedhq/troubleshoot/pkg/analyze"
 	"github.com/spf13/viper"
 )

cmd/collect/cli/run.go

@@ -10,10 +10,9 @@ import (
 	"time"
 
 	"github.com/pkg/errors"
-	"github.com/replicatedhq/troubleshoot/cmd/util"
+	"github.com/replicatedhq/troubleshoot/internal/util"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 	"github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme"
-	troubleshootclientsetscheme "github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme"
 	"github.com/replicatedhq/troubleshoot/pkg/collect"
 	"github.com/replicatedhq/troubleshoot/pkg/docrewrite"
 	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
@@ -88,7 +87,6 @@ func runCollect(v *viper.Viper, arg string) error {
 
 	multidocs := strings.Split(string(collectorContent), "\n---\n")
 
-	troubleshootclientsetscheme.AddToScheme(scheme.Scheme)
 	decode := scheme.Codecs.UniversalDeserializer().Decode
 
 	redactors, err := supportbundle.GetRedactorsFromURIs(v.GetStringSlice("redactors"))

cmd/troubleshoot/cli/analyze.go

@@ -7,7 +7,7 @@ import (
 	"net/http"
 	"os"
 
-	"github.com/replicatedhq/troubleshoot/cmd/util"
+	"github.com/replicatedhq/troubleshoot/internal/util"
 	analyzer "github.com/replicatedhq/troubleshoot/pkg/analyze"
 	"github.com/replicatedhq/troubleshoot/pkg/convert"
 	"github.com/spf13/cobra"

cmd/troubleshoot/cli/interactive_results.go

@@ -10,7 +10,7 @@ import (
 	"github.com/pkg/errors"
 	ui "github.com/replicatedhq/termui/v3"
 	"github.com/replicatedhq/termui/v3/widgets"
-	"github.com/replicatedhq/troubleshoot/cmd/util"
+	"github.com/replicatedhq/troubleshoot/internal/util"
 	analyzerunner "github.com/replicatedhq/troubleshoot/pkg/analyze"
 )
 

cmd/troubleshoot/cli/run.go

@@ -19,8 +19,6 @@ import (
 	"github.com/pkg/errors"
 	analyzer "github.com/replicatedhq/troubleshoot/pkg/analyze"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
-	"github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme"
-	troubleshootclientsetscheme "github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme"
 	"github.com/replicatedhq/troubleshoot/pkg/convert"
 	"github.com/replicatedhq/troubleshoot/pkg/httputil"
 	"github.com/replicatedhq/troubleshoot/pkg/k8sutil"
@@ -79,7 +77,6 @@ func runTroubleshoot(v *viper.Viper, arg []string) error {
 
 	var mainBundle *troubleshootv1beta2.SupportBundle
 
-	troubleshootclientsetscheme.AddToScheme(scheme.Scheme)
 	additionalRedactors := &troubleshootv1beta2.Redactor{}
 
 	// Defining `v` below will render using `v` in reference to Viper unusable.

cmd/util/util.go renamed to internal/util/util.go

@@ -40,3 +40,7 @@ func AppName(name string) string {
 
 	return strings.Join(casedWords, " ")
 }
+
+func SplitYAML(doc string) []string {
+	return strings.Split(doc, "\n---\n")
+}

pkg/collect/load.go

@@ -4,20 +4,23 @@ import (
 	"github.com/pkg/errors"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 	"github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme"
-	troubleshootclientsetscheme "github.com/replicatedhq/troubleshoot/pkg/client/troubleshootclientset/scheme"
 	"github.com/replicatedhq/troubleshoot/pkg/docrewrite"
+	"k8s.io/apimachinery/pkg/runtime"
 )
 
+var decoder runtime.Decoder
+
+func init() {
+	decoder = scheme.Codecs.UniversalDeserializer()
+}
+
 func ParseCollectorFromDoc(doc []byte) (*troubleshootv1beta2.Collector, error) {
 	doc, err := docrewrite.ConvertToV1Beta2(doc)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to convert to v1beta2")
 	}
 
-	troubleshootclientsetscheme.AddToScheme(scheme.Scheme)
-	decode := scheme.Codecs.UniversalDeserializer().Decode
-
-	obj, _, err := decode(doc, nil, nil)
+	obj, _, err := decoder.Decode(doc, nil, nil)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to parse document")
 	}
@@ -36,10 +39,7 @@ func ParseHostCollectorFromDoc(doc []byte) (*troubleshootv1beta2.HostCollector,
 		return nil, errors.Wrap(err, "failed to convert to v1beta2")
 	}
 
-	troubleshootclientsetscheme.AddToScheme(scheme.Scheme)
-	decode := scheme.Codecs.UniversalDeserializer().Decode
-
-	obj, _, err := decode(doc, nil, nil)
+	obj, _, err := decoder.Decode(doc, nil, nil)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to parse document")
 	}
@@ -58,10 +58,7 @@ func ParseRemoteCollectorFromDoc(doc []byte) (*troubleshootv1beta2.RemoteCollect
 		return nil, errors.Wrap(err, "failed to convert to v1beta2")
 	}
 
-	troubleshootclientsetscheme.AddToScheme(scheme.Scheme)
-	decode := scheme.Codecs.UniversalDeserializer().Decode
-
-	obj, _, err := decode(doc, nil, nil)
+	obj, _, err := decoder.Decode(doc, nil, nil)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to parse document")
 	}

pkg/constants/constants.go

@@ -15,7 +15,6 @@ const (
 	DEFAULT_LOGS_COLLECTOR_TIMEOUT = 60 * time.Second
 
 	// Tracing constants
-
 	LIB_TRACER_NAME             = "github.com/replicatedhq/troubleshoot"
 	TROUBLESHOOT_ROOT_SPAN_NAME = "ReplicatedTroubleshootRootSpan"
 	EXCLUDED                    = "excluded"
@@ -58,4 +57,12 @@ const (
 	EXIT_CODE_SPEC_ISSUES = 2
 	EXIT_CODE_FAIL        = 3
 	EXIT_CODE_WARN        = 4
+
+	// Troubleshoot label constants
+	SupportBundleKey = "support-bundle-spec"
+	RedactorKey      = "redactor-spec"
+	PreflightKey     = "preflight.yaml" // Shouldn't this be "preflight-spec"?
+
+	// Troubleshoot spec constants
+	Troubleshootv1beta2Kind = "troubleshoot.sh/v1beta2"
 )