add 'report all images' functionality, enable by default in embedded cluster installations (#329)

* add 'report all images' functionality, enable by default in embedded cluster installations

* test report all images functionality

* f?

* Revert "f?"

This reverts commit 41251b4.

* use fully qualified images in test chart

* make image filtering more permissive - ignore domains

* retry image check

* tests f

* cross namespace pod image reporting test

* track images in all namespaces when appropriate

* mock fix

* only watch pods in namespaces we have permissions for

* Apply suggestion from @laverya

* fix image name generation

* make image name handling more robust

* detect image names with tags from pod specf

* remove digest from spec image if present to avoid doubling it

Author: laverya

chart/templates/replicated-secret.yaml

@@ -45,6 +45,9 @@ stringData:
     replicatedID: {{ .Values.replicatedID | default "" | quote }}
     appID: {{ .Values.appID | default "" | quote }}
     tlsCertSecretName: {{ .Values.tlsCertSecretName | default "" | quote }}
+    {{- if hasKey .Values "reportAllImages" }}
+    reportAllImages: {{ .Values.reportAllImages }}
+    {{- end }}
   {{- if (.Values.integration).licenseID }}
   integration-license-id: {{ .Values.integration.licenseID }}
   {{- end }}

chart/values.yaml

@@ -300,3 +300,8 @@ tlsCertSecretName: ""
 # Use a more restrictive RBAC policy for the Replicated SDK. This requires setting statusInformers directly
 # in order to generate the correct RBAC rules.
 minimalRBAC: false
+
+# Report all images running in the cluster, not just those in the releaseImages list
+# When true, the SDK will report every image+digest in the cluster regardless of the releaseImages filter
+# When false (default), only images matching the releaseImages list will be reported
+reportAllImages: false

cmd/replicated/api.go

@@ -75,6 +75,7 @@ func APICmd() *cobra.Command {
 				ReplicatedID:          replicatedConfig.ReplicatedID,
 				AppID:                 replicatedConfig.AppID,
 				TlsCertSecretName:     replicatedConfig.TlsCertSecretName,
+				ReportAllImages:       replicatedConfig.ReportAllImages,
 				Namespace:             namespace,
 			}
 			apiserver.Start(params)

dagger/e2e.go

@@ -475,16 +475,16 @@ spec:
 
 	fmt.Printf("E2E test for distribution %s and version %s passed\n", distribution, version)
 
-	// Validate running images via vendor API
+	// Validate running images via vendor API (with default filtering)
 	// 1. Call vendor API to get running images for this instance
 	imagesSet, err := getRunningImages(ctx, appID, customerID, instanceAppID, tokenPlaintext)
 	if err != nil {
 		return fmt.Errorf("failed to get running images: %w", err)
 	}
 
-	// 2. Validate expected images
+	// 2. Validate expected images (should only contain images from releaseImages list)
 	required := []string{"docker.io/library/nginx:latest", "docker.io/library/nginx:alpine", strings.TrimSpace(sdkImage)}
-	forbidden := []string{"docker.io/alpine/curl:latest"}
+	forbidden := []string{"docker.io/alpine/curl:latest"} // This is from replicated-ssl-test, should be filtered out
 	missing := []string{}
 	for _, img := range required {
 		if img == "" {
@@ -504,10 +504,186 @@ spec:
 	}
 	for _, img := range forbidden {
 		if _, ok := imagesSet[img]; ok {
-			return fmt.Errorf("running images contains forbidden entry: %s", img)
+			return fmt.Errorf("running images contains forbidden entry: %s (should be filtered out)", img)
 		}
 	}
 
+	fmt.Println("Default image filtering test passed - only releaseImages are reported")
+
+	// Test reportAllImages functionality
+	fmt.Println("Testing reportAllImages=true functionality...")
+
+	// Create a ClusterRole and ClusterRoleBinding to allow the SDK to read pods across all namespaces
+	fmt.Println("Creating ClusterRole for cross-namespace pod access...")
+	clusterRoleYaml := `apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: replicated-cross-namespace-reader
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - "namespaces"
+  verbs:
+  - "list"
+- apiGroups:
+  - ""
+  resources:
+  - "pods"
+  verbs:
+  - "list"
+  - "watch"`
+	clusterRoleSource := source.WithNewFile("/clusterrole.yaml", clusterRoleYaml)
+
+	ctr = dag.Container().From("bitnami/kubectl:latest").
+		WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
+		WithEnvVariable("KUBECONFIG", kubeconfigPath).
+		WithFile("/tmp/clusterrole.yaml", clusterRoleSource.File("/clusterrole.yaml")).
+		WithExec([]string{"kubectl", "apply", "-f", "/tmp/clusterrole.yaml"})
+	out, err = ctr.Stdout(ctx)
+	if err != nil {
+		stderr, _ := ctr.Stderr(ctx)
+		return fmt.Errorf("failed to apply clusterrole: %w\n\nStderr: %s\n\nStdout: %s", err, stderr, out)
+	}
+	fmt.Println(out)
+
+	// Create ClusterRoleBinding
+	clusterRoleBindingYaml := `apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: replicated-cross-namespace-reader-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: replicated-cross-namespace-reader
+subjects:
+- kind: ServiceAccount
+  name: replicated
+  namespace: default`
+	clusterRoleBindingSource := source.WithNewFile("/clusterrolebinding.yaml", clusterRoleBindingYaml)
+
+	ctr = dag.Container().From("bitnami/kubectl:latest").
+		WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
+		WithEnvVariable("KUBECONFIG", kubeconfigPath).
+		WithFile("/tmp/clusterrolebinding.yaml", clusterRoleBindingSource.File("/clusterrolebinding.yaml")).
+		WithExec([]string{"kubectl", "apply", "-f", "/tmp/clusterrolebinding.yaml"})
+	out, err = ctr.Stdout(ctx)
+	if err != nil {
+		stderr, _ := ctr.Stderr(ctx)
+		return fmt.Errorf("failed to apply clusterrolebinding: %w\n\nStderr: %s\n\nStdout: %s", err, stderr, out)
+	}
+	fmt.Println(out)
+
+	// Deploy a test pod in kube-system namespace to verify cross-namespace image reporting
+	fmt.Println("Deploying test pod in kube-system namespace...")
+	kubeSystemPodYaml := `apiVersion: v1
+kind: Pod
+metadata:
+  name: replicated-test-pod
+  namespace: kube-system
+  labels:
+    app: replicated-test
+spec:
+  containers:
+  - name: busybox
+    image: docker.io/library/busybox:latest
+    command: ["sleep", "500d"]`
+	kubeSystemPodSource := source.WithNewFile("/kube-system-pod.yaml", kubeSystemPodYaml)
+
+	ctr = dag.Container().From("bitnami/kubectl:latest").
+		WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
+		WithEnvVariable("KUBECONFIG", kubeconfigPath).
+		WithFile("/tmp/kube-system-pod.yaml", kubeSystemPodSource.File("/kube-system-pod.yaml")).
+		WithExec([]string{"kubectl", "apply", "-f", "/tmp/kube-system-pod.yaml"})
+	out, err = ctr.Stdout(ctx)
+	if err != nil {
+		stderr, _ := ctr.Stderr(ctx)
+		return fmt.Errorf("failed to apply kube-system pod: %w\n\nStderr: %s\n\nStdout: %s", err, stderr, out)
+	}
+	fmt.Println(out)
+
+	// Wait for the kube-system pod to be ready
+	ctr = dag.Container().From("bitnami/kubectl:latest").
+		WithFile(kubeconfigPath, kubeconfigSource.File("/kubeconfig")).
+		WithEnvVariable("KUBECONFIG", kubeconfigPath).
+		WithExec([]string{"kubectl", "wait", "--for=condition=ready", "pod/replicated-test-pod", "-n", "kube-system", "--timeout=1m"})
+	out, err = ctr.Stdout(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to wait for kube-system pod to be ready: %w", err)
+	}
+	fmt.Println(out)
+
+	// Upgrade the chart to enable reportAllImages
+	err = upgradeChartAndRestart(ctx, kubeconfigSource, licenseID, channelSlug, []string{
+		"--set", "replicated.tlsCertSecretName=test-tls",
+		"--set", "replicated.reportAllImages=true",
+	})
+	if err != nil {
+		return fmt.Errorf("failed to upgrade chart enabling reportAllImages: %w", err)
+	}
+
+	err = waitForResourcesReady(ctx, newResourceNames, 30, 5*time.Second, tokenPlaintext, instanceAppID, distribution)
+	if err != nil {
+		return fmt.Errorf("failed to wait for resources to be ready: %w", err)
+	}
+
+	// Get running images again with retries, to allow server to refresh image reporting
+	var allImagesSet map[string]struct{}
+	// Now we expect both the alpine/curl from default namespace AND busybox from kube-system
+	nowRequired := []string{
+		"docker.io/alpine/curl:latest",  // from replicated-ssl-test in default namespace
+		"docker.io/library/busybox:latest", // from kube-system namespace
+	}
+	maxAttempts := 5
+	retryDelay := 5 * time.Second
+	for attempt := 1; attempt <= maxAttempts; attempt++ {
+		var getErr error
+		allImagesSet, getErr = getRunningImages(ctx, appID, customerID, instanceAppID, tokenPlaintext)
+		if getErr != nil {
+			if attempt == maxAttempts {
+				return fmt.Errorf("failed to get running images after enabling reportAllImages (after %d attempts): %w", maxAttempts, getErr)
+			}
+			fmt.Printf("attempt %d/%d: failed to get running images: %v\n", attempt, maxAttempts, getErr)
+			time.Sleep(retryDelay)
+			continue
+		}
+
+		// Validate that previously excluded images are now present
+		// The alpine/curl image from replicated-ssl-test should now be reported
+		missing = []string{}
+		for _, img := range nowRequired {
+			if _, ok := allImagesSet[img]; !ok {
+				missing = append(missing, img)
+			}
+		}
+		if len(missing) == 0 {
+			break
+		}
+
+		if attempt == maxAttempts {
+			seen := make([]string, 0, len(allImagesSet))
+			for k := range allImagesSet {
+				seen = append(seen, k)
+			}
+			return fmt.Errorf("with reportAllImages=true, missing expected images after %d attempts: %v. Seen: %v", maxAttempts, missing, seen)
+		}
+
+		fmt.Printf("attempt %d/%d: still missing expected images with reportAllImages=true: %v (retrying)\n", attempt, maxAttempts, missing)
+		time.Sleep(retryDelay)
+	}
+
+	// Should still have the original required images
+	for _, img := range required {
+		if img == "" {
+			continue
+		}
+		if _, ok := allImagesSet[img]; !ok {
+			return fmt.Errorf("with reportAllImages=true, missing original required image: %s", img)
+		}
+	}
+
+	fmt.Printf("reportAllImages test passed - found %d total images (vs %d with filtering)\n", len(allImagesSet), len(imagesSet))
+
 	return nil
 }
 

pkg/apiserver/bootstrap.go

@@ -14,6 +14,8 @@ import (
 	"github.com/replicatedhq/replicated-sdk/pkg/k8sutil"
 	sdklicense "github.com/replicatedhq/replicated-sdk/pkg/license"
 	"github.com/replicatedhq/replicated-sdk/pkg/logger"
+	"github.com/replicatedhq/replicated-sdk/pkg/report"
+	reporttypes "github.com/replicatedhq/replicated-sdk/pkg/report/types"
 	"github.com/replicatedhq/replicated-sdk/pkg/store"
 	"github.com/replicatedhq/replicated-sdk/pkg/upstream"
 	upstreamtypes "github.com/replicatedhq/replicated-sdk/pkg/upstream/types"
@@ -44,6 +46,16 @@ func bootstrap(params APIServerParams) error {
 	log.Println("replicatedID:", replicatedID)
 	log.Println("appID:", appID)
 
+	// In Embedded Cluster installations, automatically enable reporting all images
+	reportAllImages := params.ReportAllImages
+	if !reportAllImages {
+		distribution := report.GetDistribution(clientset)
+		if distribution == reporttypes.EmbeddedCluster {
+			reportAllImages = true
+			log.Println("Detected Embedded Cluster installation, enabling reportAllImages")
+		}
+	}
+
 	var unverifiedLicense *kotsv1beta1.License
 	if len(params.LicenseBytes) > 0 {
 		l, err := sdklicense.LoadLicenseFromBytes(params.LicenseBytes)
@@ -111,6 +123,7 @@ func bootstrap(params APIServerParams) error {
 		Namespace:             params.Namespace,
 		ReplicatedID:          replicatedID,
 		AppID:                 appID,
+		ReportAllImages:       reportAllImages,
 	})
 
 	isIntegrationModeEnabled, err := integration.IsEnabled(params.Context, clientset, store.GetStore().GetNamespace(), store.GetStore().GetLicense())

pkg/apiserver/server.go

@@ -40,6 +40,7 @@ type APIServerParams struct {
 	AppID                 string
 	Namespace             string
 	TlsCertSecretName     string
+	ReportAllImages       bool
 }
 
 func Start(params APIServerParams) {