diff --git a/pkg/addons/seaweedfs/install.go b/pkg/addons/seaweedfs/install.go index d8b103720..25877b00b 100644 --- a/pkg/addons/seaweedfs/install.go +++ b/pkg/addons/seaweedfs/install.go @@ -1,6 +1,7 @@ package seaweedfs import ( + "bytes" "context" "encoding/json" "fmt" @@ -12,6 +13,7 @@ import ( "github.com/replicatedhq/embedded-cluster/pkg/helpers" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/metadata" @@ -32,12 +34,14 @@ func (s *SeaweedFS) Install( return errors.Wrap(err, "generate helm values") } - err = s.ensurePostInstallHooksDeleted(ctx, kcli) - if err != nil { - return errors.Wrap(err, "ensure hooks deleted") + if !s.DryRun { + err = s.ensurePostInstallHooksDeleted(ctx, kcli) + if err != nil { + return errors.Wrap(err, "ensure hooks deleted") + } } - _, err = hcli.Install(ctx, helm.InstallOptions{ + opts := helm.InstallOptions{ ReleaseName: s.ReleaseName(), ChartPath: s.ChartLocation(domains), ChartVersion: Metadata.Version, @@ -45,10 +49,21 @@ func (s *SeaweedFS) Install( Namespace: s.Namespace(), Labels: getBackupLabels(), LogFn: helm.LogFn(logf), - }) - if err != nil { - return errors.Wrap(err, "helm install") } + + if s.DryRun { + manifests, err := hcli.Render(ctx, opts) + if err != nil { + return errors.Wrap(err, "dry run values") + } + s.dryRunManifests = append(s.dryRunManifests, manifests...) + } else { + _, err = hcli.Install(ctx, opts) + if err != nil { + return errors.Wrap(err, "helm install") + } + } + return nil } @@ -69,13 +84,21 @@ func (s *SeaweedFS) ensurePreRequisites(ctx context.Context, kcli client.Client) } func (s *SeaweedFS) ensureNamespace(ctx context.Context, kcli client.Client) error { - ns := corev1.Namespace{ + obj := &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: s.Namespace(), }, } - if err := kcli.Create(ctx, &ns); client.IgnoreAlreadyExists(err) != nil { - return err + if s.DryRun { + b := bytes.NewBuffer(nil) + if err := serializer.Encode(obj, b); err != nil { + return errors.Wrap(err, "serialize") + } + s.dryRunManifests = append(s.dryRunManifests, b.Bytes()) + } else { + if err := kcli.Create(ctx, obj); err != nil && !k8serrors.IsAlreadyExists(err) { + return err + } } return nil } @@ -91,6 +114,10 @@ func (s *SeaweedFS) ensureService(ctx context.Context, kcli client.Client, servi } obj := &corev1.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: "v1", + }, ObjectMeta: metav1.ObjectMeta{Name: _s3SVCName, Namespace: s.Namespace()}, Spec: corev1.ServiceSpec{ ClusterIP: clusterIP, @@ -111,22 +138,30 @@ func (s *SeaweedFS) ensureService(ctx context.Context, kcli client.Client, servi obj.Labels = ApplyLabels(obj.Labels, "s3") - var existingObj corev1.Service - if err := kcli.Get(ctx, client.ObjectKey{Name: obj.Name, Namespace: obj.Namespace}, &existingObj); client.IgnoreNotFound(err) != nil { - return errors.Wrap(err, "get s3 service") - } else if err == nil { - // if the service already exists and has the correct cluster IP, do not recreate it - if existingObj.Spec.ClusterIP == clusterIP { - return nil + if s.DryRun { + b := bytes.NewBuffer(nil) + if err := serializer.Encode(obj, b); err != nil { + return errors.Wrap(err, "serialize") } - err := kcli.Delete(ctx, &existingObj) - if err != nil { - return errors.Wrap(err, "delete existing s3 service") + s.dryRunManifests = append(s.dryRunManifests, b.Bytes()) + } else { + var existingObj corev1.Service + if err := kcli.Get(ctx, client.ObjectKey{Name: obj.Name, Namespace: obj.Namespace}, &existingObj); client.IgnoreNotFound(err) != nil { + return errors.Wrap(err, "get s3 service") + } else if err == nil { + // if the service already exists and has the correct cluster IP, do not recreate it + if existingObj.Spec.ClusterIP == clusterIP { + return nil + } + err := kcli.Delete(ctx, &existingObj) + if err != nil { + return errors.Wrap(err, "delete existing s3 service") + } } - } - if err := kcli.Create(ctx, obj); err != nil { - return errors.Wrap(err, "create s3 service") + if err := kcli.Create(ctx, obj); err != nil { + return errors.Wrap(err, "create s3 service") + } } return nil @@ -157,7 +192,12 @@ func (s *SeaweedFS) ensureS3Secret(ctx context.Context, kcli client.Client) erro } obj := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, ObjectMeta: metav1.ObjectMeta{Name: _s3SecretName, Namespace: s.Namespace()}, + Type: "Opaque", Data: map[string][]byte{ "seaweedfs_s3_config": configData, }, @@ -165,8 +205,16 @@ func (s *SeaweedFS) ensureS3Secret(ctx context.Context, kcli client.Client) erro obj.Labels = ApplyLabels(obj.Labels, "s3") - if err := kcli.Create(ctx, obj); client.IgnoreAlreadyExists(err) != nil { - return errors.Wrap(err, "create s3 secret") + if s.DryRun { + b := bytes.NewBuffer(nil) + if err := serializer.Encode(obj, b); err != nil { + return errors.Wrap(err, "serialize") + } + s.dryRunManifests = append(s.dryRunManifests, b.Bytes()) + } else { + if err := kcli.Create(ctx, obj); err != nil && !k8serrors.IsAlreadyExists(err) { + return errors.Wrap(err, "create s3 secret") + } } return nil diff --git a/pkg/addons/seaweedfs/integration/images_test.go b/pkg/addons/seaweedfs/integration/images_test.go new file mode 100644 index 000000000..65f27c44f --- /dev/null +++ b/pkg/addons/seaweedfs/integration/images_test.go @@ -0,0 +1,130 @@ +package integration + +import ( + "context" + "fmt" + "strings" + "testing" + + ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" + "github.com/replicatedhq/embedded-cluster/pkg/addons/seaweedfs" + "github.com/replicatedhq/embedded-cluster/pkg/helm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "sigs.k8s.io/yaml" +) + +func TestImageSubstitution(t *testing.T) { + addon := &seaweedfs.SeaweedFS{ + DryRun: true, + ServiceCIDR: "10.96.0.0/16", + } + + hcli, err := helm.NewClient(helm.HelmOptions{}) + require.NoError(t, err, "NewClient should not return an error") + + err = addon.Install(context.Background(), t.Logf, nil, nil, hcli, ecv1beta1.Domains{}, nil) + require.NoError(t, err, "seaweedfs.Install should not return an error") + + manifests := addon.DryRunManifests() + require.NotEmpty(t, manifests, "DryRunManifests should not be empty") + + // Build set of allowed images from metadata + allowedImages := make(map[string]bool) + for _, img := range seaweedfs.Metadata.Images { + allowedImages[img.String()] = true + } + require.NotEmpty(t, allowedImages, "Metadata should contain at least one image") + + // Track all images found in manifests + foundImages := make(map[string][]string) // map[image][]locations + + // Parse all manifests and extract images from any workload + for _, manifest := range manifests { + // Skip empty manifests + if len(manifest) == 0 { + continue + } + + // Parse as unstructured to get Kind and Name + var obj unstructured.Unstructured + if err := yaml.Unmarshal(manifest, &obj); err != nil { + // Skip invalid manifests + continue + } + + kind := obj.GetKind() + name := obj.GetName() + + // Skip non-workload resources + if !isWorkloadKind(kind) { + continue + } + + // Extract pod template spec + podSpec, found, err := unstructured.NestedMap(obj.Object, "spec", "template", "spec") + if err != nil || !found { + continue + } + + // Convert to PodSpec for easier access + podSpecBytes, err := yaml.Marshal(podSpec) + if err != nil { + continue + } + var ps corev1.PodSpec + if err := yaml.Unmarshal(podSpecBytes, &ps); err != nil { + continue + } + + // Check all containers + location := fmt.Sprintf("%s/%s", kind, name) + for i, container := range ps.Containers { + if container.Image != "" { + containerLocation := fmt.Sprintf("%s.spec.containers[%d](%s)", location, i, container.Name) + foundImages[container.Image] = append(foundImages[container.Image], containerLocation) + } + } + + // Check all init containers + for i, container := range ps.InitContainers { + if container.Image != "" { + containerLocation := fmt.Sprintf("%s.spec.initContainers[%d](%s)", location, i, container.Name) + foundImages[container.Image] = append(foundImages[container.Image], containerLocation) + } + } + } + + require.NotEmpty(t, foundImages, "Should find at least one image in manifests") + + // Verify all found images are in the allowed list + var unauthorizedImages []string + for image, locations := range foundImages { + if !allowedImages[image] { + for _, loc := range locations { + unauthorizedImages = append(unauthorizedImages, fmt.Sprintf("%s uses unauthorized image: %s", loc, image)) + } + } + + // Additional checks for all images + assert.NotContains(t, image, ":latest", "Image should not use :latest tag: %s", image) + assert.Contains(t, image, "proxy.replicated.com/library", "Image should use proxy library registry: %s", image) + } + + // Fail if any unauthorized images were found + if len(unauthorizedImages) > 0 { + t.Errorf("Found %d unauthorized images:\n%s", len(unauthorizedImages), strings.Join(unauthorizedImages, "\n")) + } +} + +// isWorkloadKind returns true if the kind can have a pod spec +func isWorkloadKind(kind string) bool { + switch kind { + case "Deployment", "StatefulSet", "DaemonSet", "Job", "CronJob", "ReplicaSet": + return true + default: + return false + } +} diff --git a/pkg/addons/seaweedfs/seaweedfs.go b/pkg/addons/seaweedfs/seaweedfs.go index f0a288cc1..56bd9d1d2 100644 --- a/pkg/addons/seaweedfs/seaweedfs.go +++ b/pkg/addons/seaweedfs/seaweedfs.go @@ -6,6 +6,9 @@ import ( ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1" "github.com/replicatedhq/embedded-cluster/pkg-new/constants" "github.com/replicatedhq/embedded-cluster/pkg/addons/types" + "github.com/replicatedhq/embedded-cluster/pkg/kubeutils" + "k8s.io/apimachinery/pkg/runtime" + jsonserializer "k8s.io/apimachinery/pkg/runtime/serializer/json" ) const ( @@ -25,11 +28,29 @@ const ( _s3SecretName = "secret-seaweedfs-s3" ) +var ( + serializer runtime.Serializer +) + +func init() { + scheme := kubeutils.Scheme + serializer = jsonserializer.NewSerializerWithOptions(jsonserializer.DefaultMetaFactory, scheme, scheme, jsonserializer.SerializerOptions{ + Yaml: true, + }) +} + var _ types.AddOn = (*SeaweedFS)(nil) type SeaweedFS struct { ServiceCIDR string SeaweedFSDataDir string + + // DryRun is a flag to enable dry-run mode for SeaweedFS. + // If true, SeaweedFS will only render the helm template and additional manifests, but not install + // the release. + DryRun bool + + dryRunManifests [][]byte } func (s *SeaweedFS) Name() string { @@ -60,3 +81,7 @@ func (s *SeaweedFS) ChartLocation(domains ecv1beta1.Domains) string { } return strings.Replace(Metadata.Location, "proxy.replicated.com", domains.ProxyRegistryDomain, 1) } + +func (s *SeaweedFS) DryRunManifests() [][]byte { + return s.dryRunManifests +} diff --git a/pkg/addons/seaweedfs/static/metadata.yaml b/pkg/addons/seaweedfs/static/metadata.yaml index 923fe1c0f..77e970da4 100644 --- a/pkg/addons/seaweedfs/static/metadata.yaml +++ b/pkg/addons/seaweedfs/static/metadata.yaml @@ -5,11 +5,11 @@ # $ make buildtools # $ output/bin/buildtools update addon # -version: 4.0.393 +version: 4.0.398 location: oci://proxy.replicated.com/anonymous/registry.replicated.com/ec-charts/seaweedfs images: seaweedfs: repo: proxy.replicated.com/library/seaweedfs tag: - amd64: 3.93-amd64@sha256:4f8c93955e547bd64e44868cb9889f46379fda02903b4a990a905cb536c15ae6 - arm64: 3.93-arm64@sha256:7dc9b386c3fbd01933e4f0da094b17f1c30e7518e16686a82e2074a6b9423d90 + amd64: 3.98-amd64@sha256:7bc9923ad7992a6f103fbbe36aed5831bef2e30e9c2d22c382c2fcbbba8b6682 + arm64: 3.98-arm64@sha256:4955dfbd9c3530fdfebf0945fff5d6c5631ddf77ff0550b58697dc6635e149da diff --git a/pkg/addons/seaweedfs/static/values.tpl.yaml b/pkg/addons/seaweedfs/static/values.tpl.yaml index 97f1f29b2..9891ff780 100644 --- a/pkg/addons/seaweedfs/static/values.tpl.yaml +++ b/pkg/addons/seaweedfs/static/values.tpl.yaml @@ -1,9 +1,6 @@ global: enableReplication: true replicationPlacment: "001" -{{- if .ReplaceImages }} - registry: "proxy.replicated.com/anonymous/" -{{- end }} master: {{- if .ReplaceImages }} @@ -145,3 +142,6 @@ filer: createBuckets: - name: registry anonymousRead: false + +s3: + enabled: false diff --git a/tests/integration/kind/registry/ha_test.go b/tests/integration/kind/registry/ha_test.go index 74a8848fa..62d828f19 100644 --- a/tests/integration/kind/registry/ha_test.go +++ b/tests/integration/kind/registry/ha_test.go @@ -54,17 +54,18 @@ func TestRegistry_EnableHAAirgap(t *testing.T) { }) // data and k0s directories are required for the admin console addon - ecDataDirMount := kind.Mount{ - HostPath: util.TempDirForHostMount(t, "data-dir-*"), - ContainerPath: "/var/lib/embedded-cluster", - } - k0sDirMount := kind.Mount{ - HostPath: util.TempDirForHostMount(t, "k0s-dir-*"), - ContainerPath: "/var/lib/embedded-cluster/k0s", + // Each node needs its own separate data directory to avoid conflicts with local persistent volumes + for i := range kindConfig.Nodes { + ecDataDirMount := kind.Mount{ + HostPath: util.TempDirForHostMount(t, "data-dir-*"), + ContainerPath: "/var/lib/embedded-cluster", + } + k0sDirMount := kind.Mount{ + HostPath: util.TempDirForHostMount(t, "k0s-dir-*"), + ContainerPath: "/var/lib/embedded-cluster/k0s", + } + kindConfig.Nodes[i].ExtraMounts = append(kindConfig.Nodes[i].ExtraMounts, ecDataDirMount, k0sDirMount) } - kindConfig.Nodes[0].ExtraMounts = append(kindConfig.Nodes[0].ExtraMounts, ecDataDirMount, k0sDirMount) - kindConfig.Nodes[1].ExtraMounts = append(kindConfig.Nodes[1].ExtraMounts, ecDataDirMount, k0sDirMount) - kindConfig.Nodes[2].ExtraMounts = append(kindConfig.Nodes[2].ExtraMounts, ecDataDirMount, k0sDirMount) kubeconfig := util.SetupKindClusterFromConfig(t, kindConfig) @@ -215,17 +216,18 @@ func TestRegistry_DisableHashiRaft(t *testing.T) { }) // data and k0s directories are required for the admin console addon - ecDataDirMount := kind.Mount{ - HostPath: util.TempDirForHostMount(t, "data-dir-*"), - ContainerPath: "/var/lib/embedded-cluster", - } - k0sDirMount := kind.Mount{ - HostPath: util.TempDirForHostMount(t, "k0s-dir-*"), - ContainerPath: "/var/lib/embedded-cluster/k0s", + // Each node needs its own separate data directory to avoid conflicts with local persistent volumes + for i := range kindConfig.Nodes { + ecDataDirMount := kind.Mount{ + HostPath: util.TempDirForHostMount(t, "data-dir-*"), + ContainerPath: "/var/lib/embedded-cluster", + } + k0sDirMount := kind.Mount{ + HostPath: util.TempDirForHostMount(t, "k0s-dir-*"), + ContainerPath: "/var/lib/embedded-cluster/k0s", + } + kindConfig.Nodes[i].ExtraMounts = append(kindConfig.Nodes[i].ExtraMounts, ecDataDirMount, k0sDirMount) } - kindConfig.Nodes[0].ExtraMounts = append(kindConfig.Nodes[0].ExtraMounts, ecDataDirMount, k0sDirMount) - kindConfig.Nodes[1].ExtraMounts = append(kindConfig.Nodes[1].ExtraMounts, ecDataDirMount, k0sDirMount) - kindConfig.Nodes[2].ExtraMounts = append(kindConfig.Nodes[2].ExtraMounts, ecDataDirMount, k0sDirMount) kubeconfig := util.SetupKindClusterFromConfig(t, kindConfig)