Use unstructured instead of interface

Signed-off-by: Richard Wall <[email protected]>

Author: wallrj

api/datareading.go

@@ -1,8 +1,11 @@
 package api
 
 import (
+	"bytes"
 	"encoding/json"
 	"time"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 )
 
 // DataReadingsPost is the payload in the upload request.
@@ -48,3 +51,20 @@ func (v GatheredResource) MarshalJSON() ([]byte, error) {
 
 	return json.Marshal(data)
 }
+
+func (v *GatheredResource) UnmarshalJSON(data []byte) error {
+	var tmpResource struct {
+		Resource  *unstructured.Unstructured `json:"resource"`
+		DeletedAt Time        `json:"deleted_at,omitempty"`
+	}
+
+	d := json.NewDecoder(bytes.NewReader(data))
+	d.DisallowUnknownFields()
+
+	if err := d.Decode(&tmpResource); err != nil {
+		return err
+	}
+	v.Resource = tmpResource.Resource
+	v.DeletedAt = tmpResource.DeletedAt
+	return nil
+}

pkg/internal/cyberark/dataupload/dataupload.go

@@ -12,6 +12,7 @@ import (
 	"net/http"
 	"net/url"
 
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/client-go/transport"
 
 	"github.com/jetstack/preflight/api"
@@ -30,43 +31,60 @@ const (
 	apiPathSnapshotLinks = "/api/ingestions/kubernetes/snapshot-links"
 )
 
-type ResourceData map[string][]interface{}
+type ResourceData map[string][]*unstructured.Unstructured
 
 // Snapshot is the JSON that the CyberArk Discovery and Context API expects to
 // be uploaded to the AWS presigned URL.
 type Snapshot struct {
-	AgentVersion    string        `json:"agent_version"`
-	ClusterID       string        `json:"cluster_id"`
-	K8SVersion      string        `json:"k8s_version"`
-	Secrets         []interface{} `json:"secrets"`
-	ServiceAccounts []interface{} `json:"service_accounts"`
-	Roles           []interface{} `json:"roles"`
-	RoleBindings    []interface{} `json:"role_bindings"`
+	AgentVersion    string                       `json:"agent_version"`
+	ClusterID       string                       `json:"cluster_id"`
+	K8SVersion      string                       `json:"k8s_version"`
+	Secrets         []*unstructured.Unstructured `json:"secrets"`
+	ServiceAccounts []*unstructured.Unstructured `json:"service_accounts"`
+	Roles           []*unstructured.Unstructured `json:"roles"`
+	RoleBindings    []*unstructured.Unstructured `json:"role_bindings"`
 }
 
 // The names of Datagatherers which have the data to populate the Cyberark Snapshot mapped to the key in the Cyberark snapshot.
 var gathererNameToresourceDataKeyMap = map[string]string{
-	"k8s/secrets":             "secrets",
-	"k8s/serviceaccounts":     "serviceaccounts",
-	"k8s/roles":               "roles",
-	"k8s/clusterroles":        "roles",
-	"k8s/rolebindings":        "rolebindings",
-	"k8s/clusterrolebindings": "rolebindings",
+	"ark/secrets":             "secrets",
+	"ark/serviceaccounts":     "serviceaccounts",
+	"ark/roles":               "roles",
+	"ark/clusterroles":        "roles",
+	"ark/rolebindings":        "rolebindings",
+	"ark/clusterrolebindings": "rolebindings",
 }
 
-func extractResourceListFromReading(reading *api.DataReading) ([]interface{}, error) {
+func extractResourceListFromReading(reading *api.DataReading) ([]*unstructured.Unstructured, error) {
 	data, ok := reading.Data.(*k8s.DynamicData)
 	if !ok {
 		return nil, fmt.Errorf("failed to convert data: %s", reading.DataGatherer)
 	}
 	items := data.Items
-	resources := make([]interface{}, len(items))
-	for i, resource := range items {
-		resources[i] = resource.Resource
+	resources := make([]*unstructured.Unstructured, len(items))
+	for i, item := range items {
+		if resource, ok := item.Resource.(*unstructured.Unstructured); ok {
+			resources[i] = resource
+		} else {
+			return nil, fmt.Errorf("failed to convert resource: %#v", item)
+		}
 	}
 	return resources, nil
 }
 
+func extractClusterUIDFromReading(reading *api.DataReading) (string, error) {
+	resources, err := extractResourceListFromReading(reading)
+	if err != nil {
+		return "", err
+	}
+	for _, resource := range resources {
+		if resource.GetName() == "kube-system" {
+			return string(resource.GetUID()), nil
+		}
+	}
+	return "", fmt.Errorf("kube-system namespace UID not found in data reading: %v", reading)
+}
+
 func extractServerVersionFromReading(reading *api.DataReading) (string, error) {
 	data, ok := reading.Data.(*k8s.DiscoveryData)
 	if !ok {
@@ -87,14 +105,20 @@ func ConvertDataReadingsToCyberarkSnapshot(
 	clusterID := ""
 	resourceData := ResourceData{}
 	for _, reading := range readings {
-		if reading.DataGatherer == "k8s-discovery" {
+		if reading.DataGatherer == "ark/discovery" {
 			k8sVersion, err = extractServerVersionFromReading(reading)
 			if err != nil {
 				return nil, fmt.Errorf("while extracting server version from data-reading: %s", err)
 			}
 		}
+		if reading.DataGatherer == "ark/namespaces" {
+			clusterID, err = extractClusterUIDFromReading(reading)
+			if err != nil {
+				return nil, fmt.Errorf("while extracting cluster UID from data-reading: %s", err)
+			}
+		}
 		if key, found := gathererNameToresourceDataKeyMap[reading.DataGatherer]; found {
-			var resources []interface{}
+			var resources []*unstructured.Unstructured
 			resources, err = extractResourceListFromReading(reading)
 			if err != nil {
 				return nil, fmt.Errorf("while extracting resource list from data-reading: %s", err)

pkg/internal/cyberark/dataupload/testdata/example-1/README.md

@@ -14,7 +14,7 @@ go run . agent \
     --install-namespace venafi \
     --log-level 6 \
     --one-shot \
-    --agent-config pkg/internal/cyberark/dataupload/testdata/example-1/agent.yaml \
+    --agent-config-file pkg/internal/cyberark/dataupload/testdata/example-1/agent.yaml \
     --output-path pkg/internal/cyberark/dataupload/testdata/example-1/datareadings.json
 gzip pkg/internal/cyberark/dataupload/testdata/example-1/datareadings.json
 ```

pkg/internal/cyberark/dataupload/testdata/example-1/agent.yaml

@@ -2,16 +2,22 @@ cluster_id: example-cluster-id
 organization_id: example-organization-id
 data-gatherers:
 # gather k8s apiserver version information
-- kind: "k8s-discovery"
-  name: "k8s-discovery"
-- kind: "k8s-dynamic"
-  name: "k8s/serviceaccounts"
+- kind: k8s-discovery
+  name: ark/discovery
+- kind: k8s-dynamic
+  name: ark/namespaces
+  config:
+    resource-type:
+      version: v1
+      resource: namespaces
+- kind: k8s-dynamic
+  name: ark/serviceaccounts
   config:
     resource-type:
       resource: serviceaccounts
       version: v1
-- kind: "k8s-dynamic"
-  name: "k8s/secrets"
+- kind: k8s-dynamic
+  name: ark/secrets
   config:
     resource-type:
       version: v1
@@ -24,29 +30,29 @@ data-gatherers:
     - type!=kubernetes.io/ssh-auth
     - type!=bootstrap.kubernetes.io/token
     - type!=helm.sh/release.v1
-- kind: "k8s-dynamic"
-  name: "k8s/roles"
+- kind: k8s-dynamic
+  name: ark/roles
   config:
     resource-type:
       version: v1
       group: rbac.authorization.k8s.io
       resource: roles
-- kind: "k8s-dynamic"
-  name: "k8s/clusterroles"
+- kind: k8s-dynamic
+  name: ark/clusterroles
   config:
     resource-type:
       version: v1
       group: rbac.authorization.k8s.io
       resource: clusterroles
-- kind: "k8s-dynamic"
-  name: "k8s/rolebindings"
+- kind: k8s-dynamic
+  name: ark/rolebindings
   config:
     resource-type:
       version: v1
       group: rbac.authorization.k8s.io
       resource: rolebindings
-- kind: "k8s-dynamic"
-  name: "k8s/clusterrolebindings"
+- kind: k8s-dynamic
+  name: ark/clusterrolebindings
   config:
     resource-type:
       version: v1