diff --git a/cmd/nerdctl/image/image.go b/cmd/nerdctl/image/image.go index 47db856f069..a711e392797 100644 --- a/cmd/nerdctl/image/image.go +++ b/cmd/nerdctl/image/image.go @@ -41,6 +41,7 @@ func Command() *cobra.Command { PushCommand(), LoadCommand(), SaveCommand(), + ImportCommand(), TagCommand(), imageRemoveCommand(), convertCommand(), diff --git a/cmd/nerdctl/image/image_import.go b/cmd/nerdctl/image/image_import.go new file mode 100644 index 00000000000..555bbcf7e05 --- /dev/null +++ b/cmd/nerdctl/image/image_import.go @@ -0,0 +1,133 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package image + +import ( + "fmt" + "io" + "net/http" + "os" + "strings" + + "github.com/spf13/cobra" + + "github.com/containerd/nerdctl/v2/cmd/nerdctl/completion" + "github.com/containerd/nerdctl/v2/cmd/nerdctl/helpers" + "github.com/containerd/nerdctl/v2/pkg/api/types" + "github.com/containerd/nerdctl/v2/pkg/clientutil" + "github.com/containerd/nerdctl/v2/pkg/cmd/image" +) + +func ImportCommand() *cobra.Command { + var cmd = &cobra.Command{ + Use: "import [OPTIONS] file|URL|- [REPOSITORY[:TAG]]", + Short: "Import the contents from a tarball to create a filesystem image", + Args: cobra.MinimumNArgs(1), + RunE: importAction, + ValidArgsFunction: imageImportShellComplete, + SilenceUsage: true, + SilenceErrors: true, + } + + cmd.Flags().StringP("message", "m", "", "Set commit message for imported image") + cmd.Flags().String("platform", "", "Set platform for imported image (e.g., linux/amd64)") + return cmd +} + +func importOptions(cmd *cobra.Command, args []string) (types.ImageImportOptions, error) { + globalOptions, err := helpers.ProcessRootCmdFlags(cmd) + if err != nil { + return types.ImageImportOptions{}, err + } + message, err := cmd.Flags().GetString("message") + if err != nil { + return types.ImageImportOptions{}, err + } + platform, err := cmd.Flags().GetString("platform") + if err != nil { + return types.ImageImportOptions{}, err + } + var reference string + if len(args) > 1 { + reference = args[1] + } + + var in io.ReadCloser + src := args[0] + switch { + case src == "-": + in = io.NopCloser(cmd.InOrStdin()) + case hasHTTPPrefix(src): + resp, err := http.Get(src) + if err != nil { + return types.ImageImportOptions{}, err + } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + defer resp.Body.Close() + return types.ImageImportOptions{}, fmt.Errorf("failed to download %s: %s", src, resp.Status) + } + in = resp.Body + default: + f, err := os.Open(src) + if err != nil { + return types.ImageImportOptions{}, err + } + in = f + } + + return types.ImageImportOptions{ + Stdout: cmd.OutOrStdout(), + Stdin: in, + GOptions: globalOptions, + Source: args[0], + Reference: reference, + Message: message, + Platform: platform, + }, nil +} + +func importAction(cmd *cobra.Command, args []string) error { + opt, err := importOptions(cmd, args) + if err != nil { + return err + } + client, ctx, cancel, err := clientutil.NewClient(cmd.Context(), opt.GOptions.Namespace, opt.GOptions.Address) + if err != nil { + return err + } + defer cancel() + defer func() { + if rc, ok := opt.Stdin.(io.ReadCloser); ok { + _ = rc.Close() + } + }() + + name, err := image.Import(ctx, client, opt) + if err != nil { + return err + } + _, err = cmd.OutOrStdout().Write([]byte(name + "\n")) + return err +} + +func imageImportShellComplete(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + return completion.ImageNames(cmd) +} + +func hasHTTPPrefix(s string) bool { + return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://") +} diff --git a/cmd/nerdctl/image/image_import_linux_test.go b/cmd/nerdctl/image/image_import_linux_test.go new file mode 100644 index 00000000000..7052c101a8e --- /dev/null +++ b/cmd/nerdctl/image/image_import_linux_test.go @@ -0,0 +1,205 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package image + +import ( + "archive/tar" + "bytes" + "errors" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "gotest.tools/v3/assert" + + "github.com/containerd/nerdctl/mod/tigron/expect" + "github.com/containerd/nerdctl/mod/tigron/require" + "github.com/containerd/nerdctl/mod/tigron/test" + "github.com/containerd/nerdctl/mod/tigron/tig" + + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" +) + +// minimalRootfsTar returns a valid tar archive with no files. +func minimalRootfsTar(t *testing.T) *bytes.Buffer { + t.Helper() + buf := new(bytes.Buffer) + tw := tar.NewWriter(buf) + assert.NilError(t, tw.Close()) + return buf +} + +func TestImageImportErrors(t *testing.T) { + nerdtest.Setup() + + testCase := &test.Case{ + Description: "TestImageImportErrors", + Require: require.Linux, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + return helpers.Command("import", "", "image:tag") + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + return &test.Expected{ + ExitCode: 1, + Errors: []error{errors.New(data.Labels().Get("error"))}, + } + }, + Data: test.WithLabels(map[string]string{ + "error": "no such file or directory", + }), + } + + testCase.Run(t) +} + +func TestImageImport(t *testing.T) { + testCase := nerdtest.Setup() + + var stopServer func() + + testCase.SubTests = []*test.Case{ + { + Description: "image import from stdin", + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rmi", "-f", data.Identifier()) + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + cmd := helpers.Command("import", "-", data.Identifier()) + cmd.Feed(bytes.NewReader(minimalRootfsTar(t).Bytes())) + return cmd + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + identifier := data.Identifier() + return &test.Expected{ + Output: expect.All( + func(stdout string, t tig.T) { + imgs := helpers.Capture("images") + assert.Assert(t, strings.Contains(imgs, identifier)) + }, + ), + } + }, + }, + { + Description: "image import from file", + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rmi", "-f", data.Identifier()) + }, + Setup: func(data test.Data, helpers test.Helpers) { + p := filepath.Join(data.Temp().Path(), "rootfs.tar") + assert.NilError(t, os.WriteFile(p, minimalRootfsTar(t).Bytes(), 0644)) + data.Labels().Set("tar", p) + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + return helpers.Command("import", data.Labels().Get("tar"), data.Identifier()) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + identifier := data.Identifier() + return &test.Expected{ + Output: expect.All( + func(stdout string, t tig.T) { + imgs := helpers.Capture("images") + assert.Assert(t, strings.Contains(imgs, identifier)) + }, + ), + } + }, + }, + { + Description: "image import with message", + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rmi", "-f", data.Identifier()) + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + cmd := helpers.Command("import", "-m", "A message", "-", data.Identifier()) + cmd.Feed(bytes.NewReader(minimalRootfsTar(t).Bytes())) + return cmd + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + identifier := data.Identifier() + ":latest" + return &test.Expected{ + Output: expect.All( + func(stdout string, t tig.T) { + img := nerdtest.InspectImage(helpers, identifier) + assert.Equal(t, img.Comment, "A message") + }, + ), + } + }, + }, + { + Description: "image import with platform", + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rmi", "-f", data.Identifier()) + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + cmd := helpers.Command("import", "--platform", "linux/amd64", "-", data.Identifier()) + cmd.Feed(bytes.NewReader(minimalRootfsTar(t).Bytes())) + return cmd + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + identifier := data.Identifier() + ":latest" + return &test.Expected{ + Output: expect.All( + func(stdout string, t tig.T) { + img := nerdtest.InspectImage(helpers, identifier) + assert.Equal(t, img.Architecture, "amd64") + assert.Equal(t, img.Os, "linux") + }, + ), + } + }, + }, + { + Description: "image import from URL", + Cleanup: func(data test.Data, helpers test.Helpers) { + if stopServer != nil { + stopServer() + stopServer = nil + } + helpers.Anyhow("rmi", "-f", data.Identifier()) + }, + Setup: func(data test.Data, helpers test.Helpers) { + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/x-tar") + _, _ = w.Write(minimalRootfsTar(t).Bytes()) + }) + url, stop, err := nerdtest.StartHTTPServer(handler) + assert.NilError(t, err) + stopServer = stop + data.Labels().Set("url", url) + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + return helpers.Command("import", data.Labels().Get("url"), data.Identifier()) + }, + Expected: func(data test.Data, helpers test.Helpers) *test.Expected { + identifier := data.Identifier() + return &test.Expected{ + Output: expect.All( + func(stdout string, t tig.T) { + imgs := helpers.Capture("images") + assert.Assert(t, strings.Contains(imgs, identifier)) + }, + ), + } + }, + }, + } + testCase.Run(t) +} diff --git a/cmd/nerdctl/main.go b/cmd/nerdctl/main.go index 45f9f6dfaa6..c5abcc60a6c 100644 --- a/cmd/nerdctl/main.go +++ b/cmd/nerdctl/main.go @@ -304,6 +304,7 @@ Config file ($NERDCTL_TOML): %s image.PushCommand(), image.LoadCommand(), image.SaveCommand(), + image.ImportCommand(), image.TagCommand(), image.RmiCommand(), image.HistoryCommand(), diff --git a/docs/command-reference.md b/docs/command-reference.md index 084719763a5..f9744bbc9b5 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -44,6 +44,7 @@ It does not necessarily mean that the corresponding features are missing in cont - [:whale: nerdctl push](#whale-nerdctl-push) - [:whale: nerdctl load](#whale-nerdctl-load) - [:whale: nerdctl save](#whale-nerdctl-save) + - [:whale: nerdctl import](#whale-nerdctl-import) - [:whale: nerdctl tag](#whale-nerdctl-tag) - [:whale: nerdctl rmi](#whale-nerdctl-rmi) - [:whale: nerdctl image inspect](#whale-nerdctl-image-inspect) @@ -905,6 +906,19 @@ Flags: - :nerd_face: `--platform=(amd64|arm64|...)`: Export content for a specific platform - :nerd_face: `--all-platforms`: Export content for all platforms +### :whale: nerdctl import + +Import the contents from a tarball to create a filesystem image. + +Usage: `nerdctl import [OPTIONS] file|URL|- [REPOSITORY[:TAG]]` + +Flags: + +- :whale: `-m, --message`: Set commit message for imported image +- :nerd_face: `--platform=(linux/amd64|linux/arm64|...)`: Set platform for the imported image + +Unimplemented `docker import` flags: `--change` + ### :whale: nerdctl tag Create a tag TARGET\_IMAGE that refers to SOURCE\_IMAGE. @@ -1919,7 +1933,6 @@ Container management: Image: -- `docker import` - `docker trust *` (Instead, nerdctl supports `nerdctl pull --verify=cosign|notation` and `nerdctl push --sign=cosign|notation`. See [`./cosign.md`](./cosign.md) and [`./notation.md`](./notation.md).) Network management: diff --git a/pkg/api/types/import_types.go b/pkg/api/types/import_types.go new file mode 100644 index 00000000000..e78d03ae92d --- /dev/null +++ b/pkg/api/types/import_types.go @@ -0,0 +1,31 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package types + +import "io" + +// ImageImportOptions specifies options for `nerdctl (image) import`. +type ImageImportOptions struct { + Stdout io.Writer + Stdin io.Reader + GOptions GlobalCommandOptions + + Source string + Reference string + Message string + Platform string +} diff --git a/pkg/cmd/image/import.go b/pkg/cmd/image/import.go new file mode 100644 index 00000000000..a8e61eb6944 --- /dev/null +++ b/pkg/cmd/image/import.go @@ -0,0 +1,252 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package image + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/rand" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "time" + + "github.com/opencontainers/go-digest" + "github.com/opencontainers/image-spec/identity" + "github.com/opencontainers/image-spec/specs-go" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + + containerd "github.com/containerd/containerd/v2/client" + "github.com/containerd/containerd/v2/core/content" + "github.com/containerd/containerd/v2/core/images" + "github.com/containerd/containerd/v2/core/leases" + "github.com/containerd/containerd/v2/pkg/archive/compression" + "github.com/containerd/errdefs" + "github.com/containerd/platforms" + + "github.com/containerd/nerdctl/v2/pkg/api/types" + "github.com/containerd/nerdctl/v2/pkg/referenceutil" +) + +func Import(ctx context.Context, client *containerd.Client, options types.ImageImportOptions) (string, error) { + img, err := importRootfs(ctx, client, options.GOptions.Snapshotter, options) + if err != nil { + return "", err + } + return img.Name, nil +} + +func importRootfs(ctx context.Context, client *containerd.Client, snapshotter string, options types.ImageImportOptions) (images.Image, error) { + var zero images.Image + + ctx, done, err := client.WithLease(ctx, leases.WithRandomID(), leases.WithExpiration(1*time.Hour)) + if err != nil { + return zero, err + } + defer done(ctx) + + if options.Stdin == nil { + return zero, fmt.Errorf("no input stream provided") + } + decomp, err := compression.DecompressStream(options.Stdin) + if err != nil { + return zero, err + } + defer decomp.Close() + + cs := client.ContentStore() + + ref := randomRef("import-rootfs-") + w, err := content.OpenWriter(ctx, cs, content.WithRef(ref)) + if err != nil { + return zero, err + } + defer w.Close() + if err := w.Truncate(0); err != nil { + return zero, err + } + + digester := digest.Canonical.Digester() + tee := io.TeeReader(decomp, digester.Hash()) + pr, pw := io.Pipe() + gz := gzip.NewWriter(pw) + doneCh := make(chan error, 1) + go func() { + _, err := io.Copy(gz, tee) + if err != nil { + doneCh <- err + _ = gz.Close() + _ = pw.CloseWithError(err) + return + } + if err := gz.Close(); err != nil { + doneCh <- err + _ = pw.CloseWithError(err) + return + } + doneCh <- pw.Close() + }() + + n, err := io.Copy(w, pr) + if err != nil { + return zero, err + } + if err := <-doneCh; err != nil { + return zero, err + } + + diffID := digester.Digest() + labels := map[string]string{ + "containerd.io/uncompressed": diffID.String(), + } + if err := w.Commit(ctx, n, "", content.WithLabels(labels)); err != nil && !errdefs.IsAlreadyExists(err) { + return zero, err + } + layerDesc := ocispec.Descriptor{ + MediaType: images.MediaTypeDockerSchema2LayerGzip, + Digest: w.Digest(), + Size: n, + } + + ociplat := platforms.DefaultSpec() + if options.Platform != "" { + p, err := platforms.Parse(options.Platform) + if err != nil { + return zero, err + } + ociplat = p + } + + created := time.Now().UTC() + imgConfig := ocispec.Image{ + Platform: ocispec.Platform{ + Architecture: ociplat.Architecture, + OS: ociplat.OS, + OSVersion: ociplat.OSVersion, + Variant: ociplat.Variant, + }, + Created: &created, + Config: ocispec.ImageConfig{}, + RootFS: ocispec.RootFS{ + Type: "layers", + DiffIDs: []digest.Digest{diffID}, + }, + History: []ocispec.History{{ + Created: &created, + Comment: options.Message, + }}, + } + + manifestDesc, _, err := writeConfigAndManifest(ctx, cs, snapshotter, imgConfig, []ocispec.Descriptor{layerDesc}) + if err != nil { + return zero, err + } + + storedName := options.Reference + if storedName == "" { + storedName = manifestDesc.Digest.String() + } else if refParsed, err := referenceutil.Parse(storedName); err == nil { + if refParsed.ExplicitTag == "" { + storedName = refParsed.FamiliarName() + ":latest" + } + if p2, err := referenceutil.Parse(storedName); err == nil { + storedName = p2.String() + } + } + name := storedName + + img := images.Image{ + Name: name, + Target: manifestDesc, + CreatedAt: time.Now(), + } + if _, err := client.ImageService().Update(ctx, img); err != nil { + if !errdefs.IsNotFound(err) { + return zero, err + } + if _, err := client.ImageService().Create(ctx, img); err != nil { + return zero, err + } + } + + cimg := containerd.NewImage(client, img) + if err := cimg.Unpack(ctx, snapshotter); err != nil { + return zero, err + } + return img, nil +} + +func randomRef(prefix string) string { + var b [6]byte + _, _ = rand.Read(b[:]) + return prefix + base64.RawURLEncoding.EncodeToString(b[:]) +} + +func writeConfigAndManifest(ctx context.Context, cs content.Store, snapshotter string, config ocispec.Image, layers []ocispec.Descriptor) (ocispec.Descriptor, digest.Digest, error) { + configJSON, err := json.Marshal(config) + if err != nil { + return ocispec.Descriptor{}, "", err + } + configDesc := ocispec.Descriptor{ + MediaType: images.MediaTypeDockerSchema2Config, + Digest: digest.FromBytes(configJSON), + Size: int64(len(configJSON)), + } + + gcLabel := map[string]string{} + if len(config.RootFS.DiffIDs) > 0 && snapshotter != "" { + gcLabel[fmt.Sprintf("containerd.io/gc.ref.snapshot.%s", snapshotter)] = identity.ChainID(config.RootFS.DiffIDs).String() + } + if err := content.WriteBlob(ctx, cs, configDesc.Digest.String(), bytes.NewReader(configJSON), configDesc, content.WithLabels(gcLabel)); err != nil && !errdefs.IsAlreadyExists(err) { + return ocispec.Descriptor{}, "", err + } + + manifest := struct { + MediaType string `json:"mediaType,omitempty"` + ocispec.Manifest + }{ + MediaType: images.MediaTypeDockerSchema2Manifest, + Manifest: ocispec.Manifest{ + Versioned: specs.Versioned{SchemaVersion: 2}, + Config: configDesc, + Layers: layers, + }, + } + manifestJSON, err := json.Marshal(manifest) + if err != nil { + return ocispec.Descriptor{}, "", err + } + manifestDesc := ocispec.Descriptor{ + MediaType: images.MediaTypeDockerSchema2Manifest, + Digest: digest.FromBytes(manifestJSON), + Size: int64(len(manifestJSON)), + } + + refLabels := map[string]string{ + "containerd.io/gc.ref.content.0": configDesc.Digest.String(), + } + for i, l := range layers { + refLabels[fmt.Sprintf("containerd.io/gc.ref.content.%d", i+1)] = l.Digest.String() + } + if err := content.WriteBlob(ctx, cs, manifestDesc.Digest.String(), bytes.NewReader(manifestJSON), manifestDesc, content.WithLabels(refLabels)); err != nil && !errdefs.IsAlreadyExists(err) { + return ocispec.Descriptor{}, "", err + } + + return manifestDesc, configDesc.Digest, nil +} diff --git a/pkg/testutil/nerdtest/utilities_linux.go b/pkg/testutil/nerdtest/utilities_linux.go index 017dd1adb00..0c996d77ce9 100644 --- a/pkg/testutil/nerdtest/utilities_linux.go +++ b/pkg/testutil/nerdtest/utilities_linux.go @@ -17,6 +17,9 @@ package nerdtest import ( + "net" + "net/http" + "net/http/httptest" "os" "strconv" "strings" @@ -26,6 +29,7 @@ import ( "github.com/containerd/nerdctl/mod/tigron/test" "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nettestutil" ) const SignalCaught = "received" @@ -73,3 +77,27 @@ func RunSigProxyContainer(signal os.Signal, exitOnSignal bool, args []string, da return cmd } + +// StartHTTPServer starts an HTTP server bound to 0.0.0.0 and returns a URL reachable +// from processes that cannot access 127.0.0.1 due to namespace isolation. +// It also returns a cleanup function that stops the server. +func StartHTTPServer(handler http.Handler) (url string, stop func(), err error) { + l, err := net.Listen("tcp", "0.0.0.0:0") + if err != nil { + return "", nil, err + } + srv := &httptest.Server{Config: &http.Server{Handler: handler}} + srv.Listener = l + srv.Start() + hostIP, herr := nettestutil.NonLoopbackIPv4() + if herr != nil { + srv.Close() + return "", nil, herr + } + _, port, perr := net.SplitHostPort(l.Addr().String()) + if perr != nil { + srv.Close() + return "", nil, perr + } + return "http://" + hostIP.String() + ":" + port, func() { srv.Close() }, nil +}