image/directory: assign version based on digest algorithm

Introduce version 1.2 and dynamically assign versions based on the digest
algorithms used:
- Version 1.1 for sha256-only images (backward compatibility)
- Version 1.2 for images using non-sha256 digest algorithms (e.g., sha512)

Add validation in both ImageDestination and ImageSource to:
- Assume 1.1 if no version file found in dir transport images
- Accept both version 1.1 and 1.2
- Refuse unsupported future versions

Signed-off-by: Lokesh Mandvekar <[email protected]>

Author: lsm5

image/directory/directory_dest.go

@@ -20,20 +20,30 @@ import (
 	"go.podman.io/storage/pkg/fileutils"
 )
 
-const version = "Directory Transport Version: 1.1\n"
+// Write version file based on digest algorithm used
+// 1.1 for sha256-only images, 1.2 otherwise.
+const (
+	versionPrefix = "Directory Transport Version: "
+	version       = versionPrefix + "1.2\n"
+	version1_1    = versionPrefix + "1.1\n"
+)
 
 // ErrNotContainerImageDir indicates that the directory doesn't match the expected contents of a directory created
 // using the 'dir' transport
 var ErrNotContainerImageDir = errors.New("not a containers image directory, don't want to overwrite important data")
 
+// ErrUnsupportedVersion indicates that the directory uses a version newer than we support
+var ErrUnsupportedVersion = errors.New("unsupported directory transport version")
+
 type dirImageDestination struct {
 	impl.Compat
 	impl.PropertyMethodsInitialize
 	stubs.IgnoresOriginalOCIConfig
 	stubs.NoPutBlobPartialInitialize
 	stubs.AlwaysSupportsSignatures
 
-	ref dirReference
+	ref                 dirReference
+	usesNonSHA256Digest bool
 }
 
 // newImageDestination returns an ImageDestination for writing to a directory.
@@ -76,7 +86,11 @@ func newImageDestination(sys *types.SystemContext, ref dirReference) (private.Im
 					return nil, err
 				}
 				// check if contents of version file is what we expect it to be
-				if string(contents) != version {
+				versionStr := string(contents)
+				if versionStr != version && versionStr != version1_1 {
+					if versionStr > version {
+						return nil, fmt.Errorf("%w: %q", ErrUnsupportedVersion, versionStr)
+					}
 					return nil, ErrNotContainerImageDir
 				}
 			} else {
@@ -94,11 +108,6 @@ func newImageDestination(sys *types.SystemContext, ref dirReference) (private.Im
 			return nil, fmt.Errorf("unable to create directory %q: %w", ref.resolvedPath, err)
 		}
 	}
-	// create version file
-	err = os.WriteFile(ref.versionPath(), []byte(version), 0o644)
-	if err != nil {
-		return nil, fmt.Errorf("creating version file %q: %w", ref.versionPath(), err)
-	}
 
 	d := &dirImageDestination{
 		PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{
@@ -166,6 +175,10 @@ func (d *dirImageDestination) PutBlobWithOptions(ctx context.Context, stream io.
 		return private.UploadedBlob{}, err
 	}
 
+	if blobDigest.Algorithm() != digest.Canonical {
+		d.usesNonSHA256Digest = true
+	}
+
 	// On POSIX systems, blobFile was created with mode 0600, so we need to make it readable.
 	// On Windows, the “permissions of newly created files” argument to syscall.Open is
 	// ignored and the file is already readable; besides, blobFile.Chmod, i.e. syscall.Fchmod,
@@ -258,6 +271,14 @@ func (d *dirImageDestination) PutSignaturesWithFormat(ctx context.Context, signa
 // - Uploaded data MAY be visible to others before CommitWithOptions() is called
 // - Uploaded data MAY be removed or MAY remain around if Close() is called without CommitWithOptions() (i.e. rollback is allowed but not guaranteed)
 func (d *dirImageDestination) CommitWithOptions(ctx context.Context, options private.CommitOptions) error {
+	versionToWrite := version1_1
+	if d.usesNonSHA256Digest {
+		versionToWrite = version
+	}
+	err := os.WriteFile(d.ref.versionPath(), []byte(versionToWrite), 0o644)
+	if err != nil {
+		return fmt.Errorf("writing version file %q: %w", d.ref.versionPath(), err)
+	}
 	return nil
 }
 

image/directory/directory_src.go

@@ -27,6 +27,23 @@ type dirImageSource struct {
 // newImageSource returns an ImageSource reading from an existing directory.
 // The caller must call .Close() on the returned ImageSource.
 func newImageSource(ref dirReference) (private.ImageSource, error) {
+	versionPath := ref.versionPath()
+	contents, err := os.ReadFile(versionPath)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return nil, fmt.Errorf("reading version file %q: %w", versionPath, err)
+		}
+	} else {
+		versionStr := string(contents)
+		if versionStr != version && versionStr != version1_1 {
+			// Check if it's a future version we don't support
+			if versionStr > version {
+				return nil, fmt.Errorf("%w: %q", ErrUnsupportedVersion, versionStr)
+			}
+			return nil, fmt.Errorf("invalid version file content: %q", versionStr)
+		}
+	}
+
 	s := &dirImageSource{
 		PropertyMethodsInitialize: impl.PropertyMethods(impl.Properties{
 			HasThreadSafeGetBlob: false,

image/directory/directory_test.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"io"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/opencontainers/go-digest"
@@ -201,8 +202,78 @@ func TestGetPutSignatures(t *testing.T) {
 	assert.Equal(t, signatures, sigs)
 }
 
+func TestVersionAssignment(t *testing.T) {
+	t.Run("SHA256 gets version 1.1", func(t *testing.T) {
+		ref, tmpDir := refToTempDir(t)
+		cache := memory.New()
+
+		dest, err := ref.NewImageDestination(context.Background(), nil)
+		require.NoError(t, err)
+		defer dest.Close()
+
+		blob := []byte("test-blob-sha256")
+		_, err = dest.PutBlob(context.Background(), bytes.NewReader(blob), types.BlobInfo{Digest: "", Size: int64(len(blob))}, cache, false)
+		require.NoError(t, err)
+
+		err = dest.Commit(context.Background(), nil)
+		require.NoError(t, err)
+
+		versionBytes, err := os.ReadFile(filepath.Join(tmpDir, "version"))
+		require.NoError(t, err)
+		assert.Equal(t, "Directory Transport Version: 1.1\n", string(versionBytes))
+	})
+
+	t.Run("Non-SHA256 gets version 1.2", func(t *testing.T) {
+		ref, tmpDir := refToTempDir(t)
+		cache := memory.New()
+
+		dest, err := ref.NewImageDestination(context.Background(), nil)
+		require.NoError(t, err)
+		defer dest.Close()
+
+		blob := []byte("test-blob-sha512")
+		sha512Digest := digest.SHA512.FromBytes(blob)
+		_, err = dest.PutBlob(context.Background(), bytes.NewReader(blob), types.BlobInfo{Digest: sha512Digest, Size: int64(len(blob))}, cache, false)
+		require.NoError(t, err)
+
+		err = dest.Commit(context.Background(), nil)
+		require.NoError(t, err)
+
+		versionBytes, err := os.ReadFile(filepath.Join(tmpDir, "version"))
+		require.NoError(t, err)
+		assert.Equal(t, "Directory Transport Version: 1.2\n", string(versionBytes))
+	})
+
+	t.Run("Mixed digests get version 1.2", func(t *testing.T) {
+		ref, tmpDir := refToTempDir(t)
+		cache := memory.New()
+
+		dest, err := ref.NewImageDestination(context.Background(), nil)
+		require.NoError(t, err)
+		defer dest.Close()
+
+		blob1 := []byte("test-blob-sha256")
+		_, err = dest.PutBlob(context.Background(), bytes.NewReader(blob1), types.BlobInfo{Digest: "", Size: int64(len(blob1))}, cache, false)
+		require.NoError(t, err)
+
+		blob2 := []byte("test-blob-sha512")
+		sha512Digest := digest.SHA512.FromBytes(blob2)
+		_, err = dest.PutBlob(context.Background(), bytes.NewReader(blob2), types.BlobInfo{Digest: sha512Digest, Size: int64(len(blob2))}, cache, false)
+		require.NoError(t, err)
+
+		err = dest.Commit(context.Background(), nil)
+		require.NoError(t, err)
+
+		versionBytes, err := os.ReadFile(filepath.Join(tmpDir, "version"))
+		require.NoError(t, err)
+		assert.Equal(t, "Directory Transport Version: 1.2\n", string(versionBytes))
+	})
+}
+
 func TestSourceReference(t *testing.T) {
 	ref, tmpDir := refToTempDir(t)
+	err := os.WriteFile(filepath.Join(tmpDir, "version"), []byte("Directory Transport Version: 1.1\n"), 0o644)
+	require.NoError(t, err)
 
 	src, err := ref.NewImageSource(context.Background(), nil)
 	require.NoError(t, err)

image/directory/directory_transport_test.go

@@ -172,7 +172,9 @@ func TestReferenceNewImageNoValidManifest(t *testing.T) {
 }
 
 func TestReferenceNewImageSource(t *testing.T) {
-	ref, _ := refToTempDir(t)
+	ref, tmpDir := refToTempDir(t)
+	err := os.WriteFile(filepath.Join(tmpDir, "version"), []byte("Directory Transport Version: 1.1\n"), 0o644)
+	require.NoError(t, err)
 	src, err := ref.NewImageSource(context.Background(), nil)
 	assert.NoError(t, err)
 	defer src.Close()