Convert data to the required format before uploading

Signed-off-by: Richard Wall <[email protected]>

Author: wallrj

agent.yaml

@@ -5,16 +5,43 @@ data-gatherers:
 # gather k8s apiserver version information
 - kind: "k8s-discovery"
   name: "k8s-discovery"
-# pods data is used in the pods and application_versions packages
 - kind: "k8s-dynamic"
-  name: "k8s/pods"
+  name: "k8s/serviceaccounts"
   config:
     resource-type:
-      resource: pods
+      resource: serviceaccounts
       version: v1
 - kind: "k8s-dynamic"
-  name: "k8s/namespaces"
+  name: "k8s/secrets"
   config:
     resource-type:
-      resource: namespaces
       version: v1
+      resource: secrets
+- kind: "k8s-dynamic"
+  name: "k8s/roles"
+  config:
+    resource-type:
+      version: v1
+      group: rbac.authorization.k8s.io
+      resource: roles
+- kind: "k8s-dynamic"
+  name: "k8s/clusterroles"
+  config:
+    resource-type:
+      version: v1
+      group: rbac.authorization.k8s.io
+      resource: clusterroles
+- kind: "k8s-dynamic"
+  name: "k8s/rolebindings"
+  config:
+    resource-type:
+      version: v1
+      group: rbac.authorization.k8s.io
+      resource: rolebindings
+- kind: "k8s-dynamic"
+  name: "k8s/clusterrolebindings"
+  config:
+    resource-type:
+      version: v1
+      group: rbac.authorization.k8s.io
+      resource: clusterrolebindings

api/datareading.go

@@ -28,8 +28,8 @@ type DataReading struct {
 type GatheredResource struct {
 	// Resource is a reference to a k8s object that was found by the informer
 	// should be of type unstructured.Unstructured, raw Object
-	Resource  interface{}
-	DeletedAt Time
+	Resource  interface{} `json: "resource"`
+	DeletedAt Time        `json:"deleted_at,omitempty"`
 }
 
 func (v GatheredResource) MarshalJSON() ([]byte, error) {

pkg/agent/run.go

@@ -30,13 +30,13 @@ import (
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 
-	"github.com/jetstack/preflight/pkg/internal/cyberark/identity"
-	"github.com/jetstack/preflight/pkg/internal/cyberark/servicediscovery"
 	"github.com/jetstack/preflight/api"
 	"github.com/jetstack/preflight/pkg/client"
 	"github.com/jetstack/preflight/pkg/clusteruid"
 	"github.com/jetstack/preflight/pkg/datagatherer"
 	"github.com/jetstack/preflight/pkg/datagatherer/k8s"
+	"github.com/jetstack/preflight/pkg/internal/cyberark/identity"
+	"github.com/jetstack/preflight/pkg/internal/cyberark/servicediscovery"
 	"github.com/jetstack/preflight/pkg/kubeconfig"
 	"github.com/jetstack/preflight/pkg/logs"
 	"github.com/jetstack/preflight/pkg/version"
@@ -90,7 +90,7 @@ func Run(cmd *cobra.Command, args []string) (returnErr error) {
 
 		const (
 			discoveryContextServiceName = "inventory"
-			separator = "."
+			separator                   = "."
 		)
 
 		// TODO(wallrj): Maybe get this URL via the service discovery API.
@@ -99,7 +99,7 @@ func Run(cmd *cobra.Command, args []string) (returnErr error) {
 
 		var (
 			identityClient *identity.Client
-			err error
+			err            error
 		)
 		if platformDomain == "cyberark.cloud" {
 			identityClient, err = identity.New(ctx, subdomain)
@@ -377,16 +377,16 @@ func gatherAndOutputData(ctx context.Context, eventf Eventf, config CombinedConf
 		}
 	}
 
+	clusterID := clusteruid.ClusterUIDFromContext(ctx)
+	payload := api.DataReadingsPost{
+		AgentMetadata: &api.AgentMetadata{
+			Version:   version.PreflightVersion,
+			ClusterID: clusterID,
+		},
+		DataGatherTime: time.Now(),
+		DataReadings:   readings,
+	}
 	if config.OutputPath != "" {
-		clusterID := clusteruid.ClusterUIDFromContext(ctx)
-		payload := api.DataReadingsPost{
-			AgentMetadata: &api.AgentMetadata{
-				Version: version.PreflightVersion,
-				ClusterID: clusterID,
-			},
-			DataGatherTime: time.Now(),
-			DataReadings: readings,
-		}
 
 		data, err := json.MarshalIndent(payload, "", "  ")
 		if err != nil {
@@ -409,14 +409,6 @@ func gatherAndOutputData(ctx context.Context, eventf Eventf, config CombinedConf
 			log.Info("Warning: PushingErr: retrying", "in", t, "reason", err)
 		})
 		if config.MachineHubMode {
-			clusterID := clusteruid.ClusterUIDFromContext(ctx)
-			payload := api.DataReadingsPost{
-				AgentMetadata: &api.AgentMetadata{
-					Version: "",
-					ClusterID: clusterID,
-				},
-				DataReadings: readings,
-			}
 			post := func() (any, error) {
 				return struct{}{}, caClient.PostDataReadingsWithOptions(ctx, payload, client.CyberArkClientOptions{
 					ClusterName: clusterID,

pkg/internal/cyberark/dataupload/dataupload.go

@@ -37,6 +37,78 @@ type Options struct {
 	ClusterDescription string
 }
 
+type CyberarkPayload struct {
+	AgentVersion    string                  `json:"agent_version"`
+	K8sVersion      string                  `json:"k8s_version"`
+	ClusterID       string                  `json:"cluster_id"`
+	Secrets         []*api.GatheredResource `json:"secrets"`          // kubectl output, sanitized
+	ServiceAccounts []*api.GatheredResource `json:"service_accounts"` // kubectl output
+	Roles           []*api.GatheredResource `json:"roles"`            // k8s native format
+	RoleBindings    []*api.GatheredResource `json:"role_bindings"`    // k8s native format
+}
+
+// You may want to define these constants based on your data-gatherer names
+const (
+	Discovery                   = "k8s-discovery"
+	SecretsGatherer             = "k8s/secrets"
+	ServiceAccountsGatherer     = "k8s/serviceaccounts"
+	RolesGatherer               = "k8s/roles"
+	RoleBindingsGatherer        = "k8s/rolebindings"
+	ClusterRolesGatherer        = "k8s/clusterroles"
+	ClusterRoleBindingsGatherer = "k8s/clusterrolebindings"
+)
+
+// ConvertDataReadingsToCyberarkPayload converts jetstack-secure DataReadings into CyberarkPayload
+func ConvertDataReadingsToCyberarkPayload(
+	input api.DataReadingsPost,
+) CyberarkPayload {
+	var (
+		k8sVersion                                    []byte
+		secrets, serviceAccounts, roles, roleBindings []*api.GatheredResource
+	)
+
+	for _, reading := range input.DataReadings {
+		switch reading.DataGatherer {
+		case Discovery:
+			k8sVersion, _ = json.Marshal(reading.Data)
+		case SecretsGatherer:
+			if data, ok := reading.Data.(map[string]interface{}); ok {
+				if items, ok := data["items"].([]*api.GatheredResource); ok {
+					secrets = append(secrets, items...)
+				}
+			}
+		case ServiceAccountsGatherer:
+			if data, ok := reading.Data.(map[string]interface{}); ok {
+				if items, ok := data["items"].([]*api.GatheredResource); ok {
+					serviceAccounts = append(serviceAccounts, items...)
+				}
+			}
+		case RolesGatherer, ClusterRoleBindingsGatherer:
+			if data, ok := reading.Data.(map[string]interface{}); ok {
+				if items, ok := data["items"].([]*api.GatheredResource); ok {
+					roles = append(roles, items...)
+				}
+			}
+		case RoleBindingsGatherer, ClusterRolesGatherer:
+			if data, ok := reading.Data.(map[string]interface{}); ok {
+				if items, ok := data["items"].([]*api.GatheredResource); ok {
+					roleBindings = append(roleBindings, items...)
+				}
+			}
+		}
+	}
+
+	return CyberarkPayload{
+		AgentVersion:    input.AgentMetadata.Version,
+		K8sVersion:      string(k8sVersion),
+		ClusterID:       input.AgentMetadata.ClusterID,
+		Secrets:         secrets,
+		ServiceAccounts: serviceAccounts,
+		Roles:           roles,
+		RoleBindings:    roleBindings,
+	}
+}
+
 func NewCyberArkClient(trustedCAs *x509.CertPool, baseURL string, authenticateRequest func(req *http.Request) error) (*CyberArkClient, error) {
 	cyberClient := &http.Client{}
 	tr := http.DefaultTransport.(*http.Transport).Clone()
@@ -59,7 +131,7 @@ func (c *CyberArkClient) PostDataReadingsWithOptions(ctx context.Context, payloa
 
 	encodedBody := &bytes.Buffer{}
 	checksum := sha256.New()
-	if err := json.NewEncoder(io.MultiWriter(encodedBody, checksum)).Encode(payload); err != nil {
+	if err := json.NewEncoder(io.MultiWriter(encodedBody, checksum)).Encode(ConvertDataReadingsToCyberarkPayload(payload)); err != nil {
 		return err
 	}