Merge pull request containerd#3490 from apostasie/bug-3489-ipfs-missing-layer

Fix IPFS missing layer issue

Author: AkihiroSuda

cmd/nerdctl/issues/issues_linux_test.go

@@ -20,8 +20,13 @@ package issues
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
 	"testing"
 
+	"gotest.tools/v3/assert"
+
+	"github.com/containerd/nerdctl/v2/pkg/rootlessutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil"
 	"github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest"
 	"github.com/containerd/nerdctl/v2/pkg/testutil/test"
@@ -37,6 +42,16 @@ func TestIssue3425(t *testing.T) {
 
 	var registry *testregistry.RegistryServer
 
+	var ipfsPath string
+	if rootlessutil.IsRootless() {
+		var err error
+		ipfsPath, err = rootlessutil.XDGDataHome()
+		ipfsPath = filepath.Join(ipfsPath, "ipfs")
+		assert.NilError(t, err)
+	} else {
+		ipfsPath = filepath.Join(os.Getenv("HOME"), ".ipfs")
+	}
+
 	testCase := &test.Case{
 		Description: "TestIssue3425",
 		Setup: func(data test.Data, helpers test.Helpers) {
@@ -126,6 +141,32 @@ func TestIssue3425(t *testing.T) {
 				},
 				Expected: test.Expects(0, nil, nil),
 			},
+			{
+				Description: "with ipfs",
+				Require: test.Require(
+					nerdtest.Private,
+					test.Not(test.Windows),
+					test.Not(nerdtest.Docker),
+					test.Binary("ipfs"),
+				),
+				Env: map[string]string{
+					"IPFS_PATH": ipfsPath,
+				},
+				Setup: func(data test.Data, helpers test.Helpers) {
+					helpers.Ensure("image", "pull", testutil.CommonImage)
+					helpers.Ensure("run", "-d", "--name", data.Identifier(), testutil.CommonImage)
+					helpers.Ensure("image", "rm", "-f", testutil.CommonImage)
+					helpers.Ensure("image", "pull", testutil.CommonImage)
+				},
+				Cleanup: func(data test.Data, helpers test.Helpers) {
+					helpers.Anyhow("rm", "-f", data.Identifier())
+					helpers.Anyhow("rmi", "-f", data.Identifier())
+				},
+				Command: func(data test.Data, helpers test.Helpers) test.Command {
+					return helpers.Command("image", "push", "ipfs://"+testutil.CommonImage)
+				},
+				Expected: test.Expects(0, nil, nil),
+			},
 		},
 	}
 

pkg/cmd/image/ensure.go

@@ -19,7 +19,6 @@ package image
 import (
 	"context"
 	"errors"
-	"fmt"
 	"net/http"
 	"os"
 
@@ -43,7 +42,6 @@ func EnsureAllContent(ctx context.Context, client *containerd.Client, srcName st
 	imageService := client.ImageService()
 	img, err := imageService.Get(ctx, srcName)
 	if err != nil {
-		fmt.Println("Failed getting imageservice")
 		return err
 	}
 

pkg/cmd/image/push.go

@@ -61,6 +61,17 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options
 		}
 		log.G(ctx).Infof("pushing image %q to IPFS", ref)
 
+		parsedRef, err := distributionref.ParseDockerRef(ref)
+		if err != nil {
+			return err
+		}
+
+		// Ensure all the layers are here: https://github.com/containerd/nerdctl/issues/3489
+		err = EnsureAllContent(ctx, client, parsedRef.String(), options.GOptions)
+		if err != nil {
+			return err
+		}
+
 		var ipfsPath string
 		if options.IpfsAddress != "" {
 			dir, err := os.MkdirTemp("", "apidirtmp")
@@ -78,7 +89,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options
 		if options.Estargz {
 			layerConvert = eStargzConvertFunc()
 		}
-		c, err := ipfs.Push(ctx, client, ref, layerConvert, options.AllPlatforms, options.Platforms, options.IpfsEnsureImage, ipfsPath)
+		c, err := ipfs.Push(ctx, client, parsedRef.String(), layerConvert, options.AllPlatforms, options.Platforms, options.IpfsEnsureImage, ipfsPath)
 		if err != nil {
 			log.G(ctx).WithError(err).Warnf("ipfs push failed")
 			return err

pkg/ipfs/image.go

@@ -37,7 +37,6 @@ import (
 	"github.com/containerd/nerdctl/v2/pkg/idutil/imagewalker"
 	"github.com/containerd/nerdctl/v2/pkg/imgutil"
 	"github.com/containerd/nerdctl/v2/pkg/platformutil"
-	"github.com/containerd/nerdctl/v2/pkg/referenceutil"
 )
 
 const ipfsPathEnv = "IPFS_PATH"
@@ -93,11 +92,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, layerCo
 			log.G(ctx).WithError(err).Warnf("failed to ensure the existence of image %q", rawRef)
 		}
 	}
-	ref, err := referenceutil.ParseAny(rawRef)
-	if err != nil {
-		return "", err
-	}
-	return ipfs.PushWithIPFSPath(ctx, client, ref.String(), layerConvert, platMC, &ipath)
+	return ipfs.PushWithIPFSPath(ctx, client, rawRef, layerConvert, platMC, &ipath)
 }
 
 // ensureContentsOfIPFSImage ensures that the entire contents of an existing IPFS image are fully downloaded to containerd.