From 74134116bf3330e38d53be114efd8c116044646f Mon Sep 17 00:00:00 2001 From: Baptiste Girard-Carrabin Date: Wed, 17 Sep 2025 11:54:03 +0200 Subject: [PATCH 1/2] [nydus] Add layer in OCI image when merging platforms Add an empty tar.gz layer at the beginning of the OCI image that's referrenced in the nydus merged-manifest compared to the original OCI one. This empty layer, being the first layer, will force the OCI image to have a completely different chainID from its original counterpart. This is done so that containerd and other runtimes don't try to reuse the OCI layers for other images that share layers with the original OCI variant. The drawback being that nydus-merged images can't share layers with pure OCI images when they are pulled on non-nydus clients. The choice to use an empty tar.gz layer as the new first layer is because: - this is a format understood by every runtime (on the other hand, docker doesn't understand layers that use `application/vnd.oci.empty.v1+json`) - an empty layer won't change the unpacked digest of the final image - it shouldn't be possible for a regular image (not built using nydus merge-manifest) to start wtih an empty tar.gzip. That's because they would need to be based on `scratch` and any layer they would add will necessary change something on the filesystem so it won't be an empty change Signed-off-by: Baptiste Girard-Carrabin --- pkg/driver/nydus/nydus.go | 140 +++++++++++++++ pkg/driver/nydus/nydus_test.go | 310 +++++++++++++++++++++++++++++++++ 2 files changed, 450 insertions(+) create mode 100644 pkg/driver/nydus/nydus_test.go diff --git a/pkg/driver/nydus/nydus.go b/pkg/driver/nydus/nydus.go index c1ed411..52016e2 100644 --- a/pkg/driver/nydus/nydus.go +++ b/pkg/driver/nydus/nydus.go @@ -15,8 +15,12 @@ package nydus import ( + "archive/tar" "bytes" + "compress/gzip" "context" + "encoding/json" + "time" "fmt" @@ -41,6 +45,7 @@ import ( nydusutils "github.com/goharbor/acceleration-service/pkg/driver/nydus/utils" "github.com/goharbor/acceleration-service/pkg/errdefs" "github.com/goharbor/acceleration-service/pkg/utils" + "github.com/opencontainers/go-digest" specs "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -56,6 +61,9 @@ const ( annotationFsVersion = "containerd.io/snapshot/nydus-fs-version" // annotationBuilderVersion indicates the nydus builder (nydus-image) version. annotationBuilderVersion = "containerd.io/snapshot/nydus-builder-version" + // emptyTarGzipUnpackedDigest is the canonical sha256 digest of empty tar file (1024 NULL bytes). + // Can be used as the diffID of an empty layer tar.gz layer. + emptyTarGzipUnpackedDigest = digest.Digest("sha256:5f70bf18a086007016e948b04aed3b82103a36bea41755b6cddfaf10ace3c6ef") ) var builderVersion string @@ -344,6 +352,14 @@ func (d *Driver) makeManifestIndex(ctx context.Context, cs content.Store, oci, n if err != nil { return nil, errors.Wrap(err, "get oci image manifest list") } + for idx, desc := range ociDescs { + // Modify initial OCI image to prevent layer reuse with non-nydus OCI images + desc, err = PrependEmptyLayer(ctx, cs, desc) + if err != nil { + return nil, errors.Wrap(err, "prepend empty layer") + } + ociDescs[idx] = desc + } nydusDescs, err := utils.GetManifests(ctx, cs, nydus, d.platformMC) if err != nil { @@ -426,3 +442,127 @@ func (d *Driver) getChunkDict(ctx context.Context, provider accelcontent.Provide return &chunkDict, nil } + +// PrependEmptyLayer modifies the original image manifest and config to prepend an empty layer +// This is done on purpose to force new shas for all the subsequent layers when unpacked by runtimes +// So that no layer reuse can be possible between stock OCI images and nydus-converted OCI images +// It returns the updated manifest descriptor +func PrependEmptyLayer(ctx context.Context, cs content.Store, manifestDesc ocispec.Descriptor) (ocispec.Descriptor, error) { + // Read existing OCI manifest + manifestBytes, err := content.ReadBlob(ctx, cs, manifestDesc) + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "read manifest") + } + + var manifest ocispec.Manifest + if err := json.Unmarshal(manifestBytes, &manifest); err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "unmarshal manifest") + } + + // Read existing OCI config + configBytes, err := content.ReadBlob(ctx, cs, manifest.Config) + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "read config") + } + + var config ocispec.Image + if err := json.Unmarshal(configBytes, &config); err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "unmarshal config") + } + + // Rebuild the layer list with an empty layer at the beginning + // This will force new shas for all the subsequent layers + var ( + emptyLayerMediaType string + configDescriptorMediaType string + ) + + switch manifest.MediaType { + case ocispec.MediaTypeImageManifest: + emptyLayerMediaType = ocispec.MediaTypeImageLayerGzip + configDescriptorMediaType = ocispec.MediaTypeImageConfig + case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema1Manifest: + emptyLayerMediaType = images.MediaTypeDockerSchema2LayerGzip + configDescriptorMediaType = images.MediaTypeDockerSchema2Config + } + emptyDescriptorBytes := generateDockerEmptyLayer() + emptyDescriptor := ocispec.Descriptor{ + MediaType: emptyLayerMediaType, + Digest: digest.FromBytes(emptyDescriptorBytes), + Size: int64(len(emptyDescriptorBytes)), + } + + manifest.Layers = append([]ocispec.Descriptor{emptyDescriptor}, manifest.Layers...) + if manifest.Annotations == nil { + manifest.Annotations = map[string]string{} + } + manifest.Annotations[annotationSourceDigest] = manifestDesc.Digest.String() + // Add an empty diff_id at the beginning of the config + config.RootFS.DiffIDs = append([]digest.Digest{emptyTarGzipUnpackedDigest}, config.RootFS.DiffIDs...) + // Rewrite history to add an entry for the empty layer + createdTime := time.Now() + emptyLayerHistory := ocispec.History{ + Created: &createdTime, + CreatedBy: "Nydus Converter", + Comment: "Nydus Empty Layer", + } + config.History = append([]ocispec.History{emptyLayerHistory}, config.History...) + + newConfigDesc, newConfigBytes, err := nydusutils.MarshalToDesc(config, configDescriptorMediaType) + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "marshal modified config") + } + if newConfigDesc.Annotations == nil { + newConfigDesc.Annotations = map[string]string{} + } + newConfigDesc.Annotations[annotationSourceDigest] = manifest.Config.Digest.String() + + manifest.Config = *newConfigDesc + newManifestDesc, newManifestBytes, err := nydusutils.MarshalToDesc(manifest, manifest.MediaType) + if err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "marshal modified manifest") + } + // Add back the original information of the manifest descriptor + newManifestDesc.Platform = manifestDesc.Platform + newManifestDesc.URLs = manifestDesc.URLs + newManifestDesc.ArtifactType = manifestDesc.ArtifactType + newManifestDesc.Annotations = manifestDesc.Annotations + + if newManifestDesc.Annotations == nil { + newManifestDesc.Annotations = map[string]string{} + } + newManifestDesc.Annotations[annotationSourceDigest] = manifestDesc.Digest.String() + + // Write modified config + if err := content.WriteBlob( + ctx, cs, newConfigDesc.Digest.String(), bytes.NewReader(newConfigBytes), *newConfigDesc, + ); err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "write modified config") + } + + // Write empty blob + if err := content.WriteBlob( + ctx, cs, emptyDescriptor.Digest.String(), bytes.NewReader(emptyDescriptorBytes), emptyDescriptor, + ); err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "write empty json blob") + } + + // Write modified manifest + if err := content.WriteBlob( + ctx, cs, newManifestDesc.Digest.String(), bytes.NewReader(newManifestBytes), *newManifestDesc, + ); err != nil { + return ocispec.Descriptor{}, errors.Wrap(err, "write modified manifest") + } + + return *newManifestDesc, nil +} + +// Empty gzip-compressed tar file that can be used as an empty layer content +func generateDockerEmptyLayer() []byte { + var buf bytes.Buffer + gzw := gzip.NewWriter(&buf) + tw := tar.NewWriter(gzw) + tw.Close() + gzw.Close() + return buf.Bytes() +} diff --git a/pkg/driver/nydus/nydus_test.go b/pkg/driver/nydus/nydus_test.go new file mode 100644 index 0000000..8c76822 --- /dev/null +++ b/pkg/driver/nydus/nydus_test.go @@ -0,0 +1,310 @@ +package nydus + +import ( + "bytes" + "context" + "encoding/json" + "os" + "testing" + + "github.com/containerd/containerd/v2/core/content" + "github.com/containerd/containerd/v2/core/images" + "github.com/containerd/containerd/v2/plugins/content/local" + nydusutils "github.com/goharbor/acceleration-service/pkg/driver/nydus/utils" + "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type testManifest struct { + name string + mediaType string + layers []ocispec.Descriptor + config ocispec.Image + configMediaType string + expectedLayerCount int +} + +func Test_PrependEmptyLayer(t *testing.T) { + tests := []testManifest{ + { + name: "oci_manifest_single_layer", + mediaType: ocispec.MediaTypeImageManifest, + layers: []ocispec.Descriptor{ + { + MediaType: ocispec.MediaTypeImageLayerGzip, + Digest: "sha256:existing-layer-digest", + Size: 1000, + }, + }, + config: ocispec.Image{ + RootFS: ocispec.RootFS{ + Type: "layers", + DiffIDs: []digest.Digest{"sha256:existing-diff-id"}, + }, + History: []ocispec.History{ + { + CreatedBy: "test layer", + }, + }, + }, + configMediaType: ocispec.MediaTypeImageConfig, + expectedLayerCount: 2, + }, + { + name: "oci_manifest_multiple_layers", + mediaType: ocispec.MediaTypeImageManifest, + layers: []ocispec.Descriptor{ + { + MediaType: ocispec.MediaTypeImageLayerGzip, + Digest: "sha256:layer1-digest", + Size: 1000, + }, + { + MediaType: ocispec.MediaTypeImageLayerGzip, + Digest: "sha256:layer2-digest", + Size: 2000, + }, + }, + config: ocispec.Image{ + RootFS: ocispec.RootFS{ + Type: "layers", + DiffIDs: []digest.Digest{"sha256:diff-id1", "sha256:diff-id2"}, + }, + History: []ocispec.History{ + { + CreatedBy: "layer 1", + }, + { + CreatedBy: "layer 2", + }, + }, + }, + configMediaType: ocispec.MediaTypeImageConfig, + expectedLayerCount: 3, + }, + { + name: "oci_manifest_no_layers", + mediaType: ocispec.MediaTypeImageManifest, + layers: []ocispec.Descriptor{}, + config: ocispec.Image{ + RootFS: ocispec.RootFS{ + Type: "layers", + DiffIDs: []digest.Digest{}, + }, + History: []ocispec.History{}, + }, + configMediaType: ocispec.MediaTypeImageConfig, + expectedLayerCount: 1, + }, + { + name: "docker_manifest_single_layer", + mediaType: images.MediaTypeDockerSchema2Manifest, + layers: []ocispec.Descriptor{ + { + MediaType: images.MediaTypeDockerSchema2LayerGzip, + Digest: "sha256:docker-layer-digest", + Size: 1500, + }, + }, + config: ocispec.Image{ + RootFS: ocispec.RootFS{ + Type: "layers", + DiffIDs: []digest.Digest{"sha256:docker-diff-id"}, + }, + History: []ocispec.History{ + { + CreatedBy: "docker layer", + }, + }, + }, + configMediaType: images.MediaTypeDockerSchema2Config, + expectedLayerCount: 2, + }, + { + name: "docker_manifest_multiple_layers", + mediaType: images.MediaTypeDockerSchema2Manifest, + layers: []ocispec.Descriptor{ + { + MediaType: images.MediaTypeDockerSchema2LayerGzip, + Digest: "sha256:docker-layer1-digest", + Size: 1000, + }, + { + MediaType: images.MediaTypeDockerSchema2LayerGzip, + Digest: "sha256:docker-layer2-digest", + Size: 2000, + }, + }, + config: ocispec.Image{ + RootFS: ocispec.RootFS{ + Type: "layers", + DiffIDs: []digest.Digest{"sha256:docker-diff-id1", "sha256:docker-diff-id2"}, + }, + History: []ocispec.History{ + { + CreatedBy: "docker layer 1", + }, + { + CreatedBy: "docker layer 2", + }, + }, + }, + configMediaType: images.MediaTypeDockerSchema2Config, + expectedLayerCount: 3, + }, + { + name: "docker_manifest_no_layers", + mediaType: images.MediaTypeDockerSchema2Manifest, + layers: []ocispec.Descriptor{}, + config: ocispec.Image{ + RootFS: ocispec.RootFS{ + Type: "layers", + DiffIDs: []digest.Digest{}, + }, + History: []ocispec.History{}, + }, + configMediaType: images.MediaTypeDockerSchema2Config, + expectedLayerCount: 1, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + tempDir := t.TempDir() + defer func() { + if err := os.RemoveAll(tempDir); err != nil { + t.Logf("failed to cleanup temp dir: %v", err) + } + }() + + // Create content store + cs, err := local.NewStore(tempDir) + require.NoError(t, err, "failed to create content store") + + // Setup test data + configDesc, err := createConfigBlob(ctx, cs, tt.configMediaType, tt.config) + require.NoError(t, err, "failed to create config blob") + + manifestDesc, err := createManifestBlob(ctx, cs, tt.mediaType, tt.layers, *configDesc) + require.NoError(t, err, "failed to create manifest blob") + + // Test PrependEmptyLayer + newManifestDesc, err := PrependEmptyLayer(ctx, cs, *manifestDesc) + require.NoError(t, err, "PrependEmptyLayer should succeed") + + // Verify results + verifyPrependResults(ctx, t, cs, *manifestDesc, newManifestDesc, tt.config, tt.expectedLayerCount) + }) + } +} + +func createConfigBlob(ctx context.Context, cs content.Store, mediaType string, config ocispec.Image) (*ocispec.Descriptor, error) { + configDesc, configBytes, err := nydusutils.MarshalToDesc(config, mediaType) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal config") + } + + err = content.WriteBlob(ctx, cs, configDesc.Digest.String(), bytes.NewReader(configBytes), *configDesc) + if err != nil { + return nil, errors.Wrap(err, "failed to write config blob") + } + return configDesc, nil +} + +func createManifestBlob(ctx context.Context, cs content.Store, mediaType string, layers []ocispec.Descriptor, configDesc ocispec.Descriptor) (*ocispec.Descriptor, error) { + manifest := ocispec.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, + }, + MediaType: mediaType, + Config: configDesc, + Layers: layers, + } + + manifestDesc, manifestBytes, err := nydusutils.MarshalToDesc(manifest, mediaType) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal manifest") + } + + err = content.WriteBlob(ctx, cs, manifestDesc.Digest.String(), bytes.NewReader(manifestBytes), *manifestDesc) + if err != nil { + return nil, errors.Wrap(err, "failed to write manifest blob") + } + return manifestDesc, nil +} + +func verifyPrependResults(ctx context.Context, t *testing.T, cs content.Store, originalManifestDesc, newManifestDesc ocispec.Descriptor, originalConfig ocispec.Image, expectedLayerCount int) { + // Verify manifest descriptor changes + assert.Equal(t, originalManifestDesc.MediaType, newManifestDesc.MediaType, "media type should remain unchanged") + assert.Equal(t, originalManifestDesc.Digest.String(), newManifestDesc.Annotations[annotationSourceDigest], "source digest annotation should match original manifest digest") + + // Read the new manifest + newManifestBytes, err := content.ReadBlob(ctx, cs, newManifestDesc) + require.NoError(t, err, "failed to read new manifest") + + var newManifest ocispec.Manifest + require.NoError(t, json.Unmarshal(newManifestBytes, &newManifest), "failed to unmarshal new manifest") + + assert.Len(t, newManifest.Layers, expectedLayerCount, "layer count should match expected") + + // Verify empty layer is first + emptyLayer := newManifest.Layers[0] + expectedEmptyDigest := "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1" + assert.Equal(t, expectedEmptyDigest, emptyLayer.Digest.String(), "empty layer digest should match expected") + assert.Equal(t, int64(32), emptyLayer.Size, "empty layer size should be 32 bytes") + + // Verify media types based on the original manifest type + var expectedEmptyLayerMediaType, expectedconfigDescriptorMediaType string + switch originalManifestDesc.MediaType { + case ocispec.MediaTypeImageManifest: + expectedEmptyLayerMediaType = ocispec.MediaTypeImageLayerGzip + expectedconfigDescriptorMediaType = ocispec.MediaTypeImageConfig + case images.MediaTypeDockerSchema2Manifest, images.MediaTypeDockerSchema1Manifest: + expectedEmptyLayerMediaType = images.MediaTypeDockerSchema2LayerGzip + expectedconfigDescriptorMediaType = images.MediaTypeDockerSchema2Config + } + assert.Equal(t, expectedEmptyLayerMediaType, emptyLayer.MediaType, "empty layer media type should match manifest type") + assert.Equal(t, expectedconfigDescriptorMediaType, newManifest.Config.MediaType, "config media type should match manifest type") + + assert.Equal(t, originalManifestDesc.Digest.String(), newManifest.Annotations[annotationSourceDigest], + "source digest annotation should match original manifest digest") + + // Read and verify new config + newConfigBytes, err := content.ReadBlob(ctx, cs, newManifest.Config) + require.NoError(t, err, "failed to read new config") + + var newConfig ocispec.Image + require.NoError(t, json.Unmarshal(newConfigBytes, &newConfig), "failed to unmarshal new config") + + expectedDiffIDCount := expectedLayerCount + assert.Len(t, newConfig.RootFS.DiffIDs, expectedDiffIDCount, "diff IDs count should increase by 1") + + // Verify empty diff ID is first and matches constant + assert.Equal(t, emptyTarGzipUnpackedDigest, newConfig.RootFS.DiffIDs[0], + "first diff ID should match empty tar gzip unpacked digest constant") + + expectedHistoryCount := len(originalConfig.History) + 1 + assert.Len(t, newConfig.History, expectedHistoryCount, "history count should increase by 1") + + // Verify empty layer history entry + emptyHistory := newConfig.History[0] + assert.Equal(t, "Nydus Converter", emptyHistory.CreatedBy, "empty layer should be created by Nydus Converter") + assert.Equal(t, "Nydus Empty Layer", emptyHistory.Comment, "empty layer comment should be correct") +} + +func Test_generateDockerEmptyLayer(t *testing.T) { + emptyLayer := generateDockerEmptyLayer() + + expectedSize := 32 + expectedDigest := "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1" + + assert.Len(t, emptyLayer, expectedSize, "empty layer should be 32 bytes") + + actualDigest := digest.FromBytes(emptyLayer).String() + assert.Equal(t, expectedDigest, actualDigest, "empty layer digest should match expected") +} From 7e7bbfd36f6ab5ba8feb4687ad62ea2f5462924d Mon Sep 17 00:00:00 2001 From: Baptiste Girard-Carrabin Date: Fri, 19 Sep 2025 13:44:19 +0200 Subject: [PATCH 2/2] [nydus] Add annotation to indicate empty layer Set the annotation`containerd.io/snapshot/nydus-empty-layer: "true"` on the empty layer that's added in the OCI image. Just to be able to find it easily. Signed-off-by: Baptiste Girard-Carrabin --- pkg/driver/nydus/nydus.go | 9 ++++++--- pkg/driver/nydus/nydus_test.go | 1 + 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/driver/nydus/nydus.go b/pkg/driver/nydus/nydus.go index 52016e2..1c1153c 100644 --- a/pkg/driver/nydus/nydus.go +++ b/pkg/driver/nydus/nydus.go @@ -57,6 +57,8 @@ const ( annotationSourceDigest = "containerd.io/snapshot/nydus-source-digest" // annotationSourceReference indicates the source OCI image reference name. annotationSourceReference = "containerd.io/snapshot/nydus-source-reference" + // annotationEmptyLayer indicates that the layer is an empty layer added by nydus that won't change the final image's content. + annotationEmptyLayer = "containerd.io/snapshot/nydus-empty-layer" // annotationFsVersion indicates the fs version (rafs v5/v6) of nydus image. annotationFsVersion = "containerd.io/snapshot/nydus-fs-version" // annotationBuilderVersion indicates the nydus builder (nydus-image) version. @@ -487,9 +489,10 @@ func PrependEmptyLayer(ctx context.Context, cs content.Store, manifestDesc ocisp } emptyDescriptorBytes := generateDockerEmptyLayer() emptyDescriptor := ocispec.Descriptor{ - MediaType: emptyLayerMediaType, - Digest: digest.FromBytes(emptyDescriptorBytes), - Size: int64(len(emptyDescriptorBytes)), + Annotations: map[string]string{annotationEmptyLayer: "true"}, + MediaType: emptyLayerMediaType, + Digest: digest.FromBytes(emptyDescriptorBytes), + Size: int64(len(emptyDescriptorBytes)), } manifest.Layers = append([]ocispec.Descriptor{emptyDescriptor}, manifest.Layers...) diff --git a/pkg/driver/nydus/nydus_test.go b/pkg/driver/nydus/nydus_test.go index 8c76822..0662f4b 100644 --- a/pkg/driver/nydus/nydus_test.go +++ b/pkg/driver/nydus/nydus_test.go @@ -257,6 +257,7 @@ func verifyPrependResults(ctx context.Context, t *testing.T, cs content.Store, o expectedEmptyDigest := "sha256:4f4fb700ef54461cfa02571ae0db9a0dc1e0cdb5577484a6d75e68dc38e8acc1" assert.Equal(t, expectedEmptyDigest, emptyLayer.Digest.String(), "empty layer digest should match expected") assert.Equal(t, int64(32), emptyLayer.Size, "empty layer size should be 32 bytes") + assert.Equal(t, "true", emptyLayer.Annotations[annotationEmptyLayer], "empty layer annotation should be set to true") // Verify media types based on the original manifest type var expectedEmptyLayerMediaType, expectedconfigDescriptorMediaType string