diff --git a/.github/workflows/ghcr-image-build-and-publish.yml b/.github/workflows/ghcr-image-build-and-publish.yml index 300fd4574f5..63c971ca12b 100644 --- a/.github/workflows/ghcr-image-build-and-publish.yml +++ b/.github/workflows/ghcr-image-build-and-publish.yml @@ -61,10 +61,12 @@ jobs: # Build and push Docker image with Buildx (don't push on PR) # https://github.com/docker/build-push-action - name: Build and push Docker image - uses: docker/build-push-action@1dc73863535b631f98b2378be8619f83b136f4a0 # v6.17.0 + uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6.18.0 with: context: . platforms: linux/amd64,linux/arm64 push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} + secrets: | + github_token=${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/job-test-dependencies.yml b/.github/workflows/job-test-dependencies.yml index c4457bae1c7..625dca61fa8 100644 --- a/.github/workflows/job-test-dependencies.yml +++ b/.github/workflows/job-test-dependencies.yml @@ -39,6 +39,8 @@ jobs: uses: crazy-max/ghaction-github-runtime@3cb05d89e1f492524af3d41a1c98c83bc3025124 # v3.1.0 - name: "Run: build dependencies for the integration test environment image" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | # Cache is sharded per-architecture arch=${{ env.RUNNER_ARCH == 'ARM64' && 'arm64' || 'amd64' }} @@ -49,6 +51,7 @@ jobs: args=(--build-arg CONTAINERD_VERSION=${{ inputs.containerd-version }}) fi docker buildx build \ + --secret id=github_token,env=GITHUB_TOKEN \ --cache-to type=gha,compression=zstd,mode=max,scope=test-integration-dependencies-"$arch" \ --cache-from type=gha,scope=test-integration-dependencies-"$arch" \ --target build-dependencies "${args[@]}" . diff --git a/.github/workflows/job-test-in-container.yml b/.github/workflows/job-test-in-container.yml index 6c1b9bae492..2257de5197d 100644 --- a/.github/workflows/job-test-in-container.yml +++ b/.github/workflows/job-test-in-container.yml @@ -86,6 +86,8 @@ jobs: canary::build::integration - if: ${{ ! inputs.canary }} name: "Init: prepare test image" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | buildargs=() # If the runner is old, use old ubuntu inside the container as well @@ -104,6 +106,7 @@ jobs: arch=${{ env.RUNNER_ARCH == 'ARM64' && 'arm64' || 'amd64' }} docker buildx create --name with-gha --use docker buildx build \ + --secret id=github_token,env=GITHUB_TOKEN \ --output=type=docker \ --cache-from type=gha,scope=test-integration-dependencies-"$arch" \ -t "$target" --target "$target" \ diff --git a/.github/workflows/job-test-in-lima.yml b/.github/workflows/job-test-in-lima.yml index 22e2f3e9f8b..8a7567cb120 100644 --- a/.github/workflows/job-test-in-lima.yml +++ b/.github/workflows/job-test-in-lima.yml @@ -31,7 +31,7 @@ jobs: fetch-depth: 1 - name: "Init: lima" - uses: lima-vm/lima-actions/setup@be564a1408f84557d067b099a475652288074b2e # v1.0.0 + uses: lima-vm/lima-actions/setup@03b96d61959e83b2c737e44162c3088e81de0886 # v1.0.1 id: lima-actions-setup - name: "Init: Cache" @@ -79,6 +79,8 @@ jobs: uses: crazy-max/ghaction-github-runtime@3cb05d89e1f492524af3d41a1c98c83bc3025124 # v3.1.0 - name: "Init: prepare integration tests" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -eux @@ -88,6 +90,7 @@ jobs: [ "$TARGET" = "rootless" ] && TARGET=test-integration-rootless || TARGET=test-integration docker buildx create --name with-gha --use docker buildx build \ + --secret id=github_token,env=GITHUB_TOKEN \ --output=type=docker \ --cache-from type=gha,scope=test-integration-dependencies-amd64 \ -t test-integration --target "${TARGET}" \ diff --git a/Dockerfile b/Dockerfile index ef3b8d55282..e315706b0a2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -61,6 +61,7 @@ ENV DEBIAN_FRONTEND=noninteractive RUN apt-get update -qq && apt-get install -qq --no-install-recommends \ make \ git \ + jq \ curl \ dpkg-dev ARG TARGETARCH @@ -75,6 +76,7 @@ RUN xx-apt-get update -qq && xx-apt-get install -qq --no-install-recommends \ pkg-config RUN git config --global advice.detachedHead false ADD hack/git-checkout-tag-with-hash.sh /usr/local/bin/ +ADD hack/scripts/lib.sh /usr/local/bin/http::helper FROM build-base AS build-containerd ARG TARGETARCH @@ -174,10 +176,11 @@ RUN cd /out/lib/systemd/system && \ echo "" >> buildkit.service && \ echo "# This file was converted from containerd.service, with \`sed -E '${sedcomm}'\`" >> buildkit.service ARG STARGZ_SNAPSHOTTER_VERSION -RUN STARGZ_SNAPSHOTTER_VERSION=${STARGZ_SNAPSHOTTER_VERSION%%@*}; \ +RUN --mount=type=secret,id=github_token,env=GITHUB_TOKEN \ + STARGZ_SNAPSHOTTER_VERSION=${STARGZ_SNAPSHOTTER_VERSION%%@*}; \ fname="stargz-snapshotter-${STARGZ_SNAPSHOTTER_VERSION}-${TARGETOS:-linux}-${TARGETARCH:-amd64}.tar.gz" && \ curl -o "${fname}" -fsSL --proto '=https' --tlsv1.2 "https://github.com/containerd/stargz-snapshotter/releases/download/${STARGZ_SNAPSHOTTER_VERSION}/${fname}" && \ - curl -o "stargz-snapshotter.service" -fsSL --proto '=https' --tlsv1.2 "https://raw.githubusercontent.com/containerd/stargz-snapshotter/${STARGZ_SNAPSHOTTER_VERSION}/script/config/etc/systemd/system/stargz-snapshotter.service" && \ + http::helper github::file containerd/stargz-snapshotter script/config/etc/systemd/system/stargz-snapshotter.service "${STARGZ_SNAPSHOTTER_VERSION}" > "stargz-snapshotter.service" && \ grep "${fname}" "/SHA256SUMS.d/stargz-snapshotter-${STARGZ_SNAPSHOTTER_VERSION}" | sha256sum -c - && \ grep "stargz-snapshotter.service" "/SHA256SUMS.d/stargz-snapshotter-${STARGZ_SNAPSHOTTER_VERSION}" | sha256sum -c - && \ tar xzf "${fname}" -C /out/bin && \ @@ -245,6 +248,10 @@ RUN ROOTLESSKIT_VERSION=${ROOTLESSKIT_VERSION%%@*}; \ ARG GOMODJAIL_VERSION COPY --from=build-gomodjail /out/${TARGETARCH:-amd64}/* /out/bin/ RUN echo "- gomodjail: ${GOMODJAIL_VERSION}" >> /out/share/doc/nerdctl-full/README.md +ARG CONTAINERIZED_SYSTEMD_VERSION +RUN --mount=type=secret,id=github_token,env=GITHUB_TOKEN \ + http::helper github::file AkihiroSuda/containerized-systemd docker-entrypoint.sh "${CONTAINERIZED_SYSTEMD_VERSION}" > /docker-entrypoint.sh && \ + chmod +x /docker-entrypoint.sh RUN echo "" >> /out/share/doc/nerdctl-full/README.md && \ echo "## License" >> /out/share/doc/nerdctl-full/README.md && \ @@ -281,9 +288,7 @@ RUN apt-get update -qq && apt-get install -qq -y --no-install-recommends \ iproute2 iptables \ dbus dbus-user-session systemd systemd-sysv \ fuse3 -ARG CONTAINERIZED_SYSTEMD_VERSION -RUN curl -o /docker-entrypoint.sh -fsSL --proto '=https' --tlsv1.2 https://raw.githubusercontent.com/AkihiroSuda/containerized-systemd/${CONTAINERIZED_SYSTEMD_VERSION}/docker-entrypoint.sh && \ - chmod +x /docker-entrypoint.sh +COPY --from=build-full /docker-entrypoint.sh /docker-entrypoint.sh COPY --from=out-full / /usr/local/ RUN perl -pi -e 's/multi-user.target/docker-entrypoint.target/g' /usr/local/lib/systemd/system/*.service && \ systemctl enable containerd buildkit stargz-snapshotter && \ diff --git a/Makefile b/Makefile index 65ec10c1c9a..1026a744eb4 100644 --- a/Makefile +++ b/Makefile @@ -253,7 +253,7 @@ TAR_OWNER0_FLAGS=--owner=0 --group=0 TAR_FLATTEN_FLAGS=--transform 's/.*\///g' define make_artifact_full_linux - $(DOCKER) build --output type=tar,dest=$(CURDIR)/_output/nerdctl-full-$(VERSION_TRIMMED)-linux-$(1).tar --target out-full --platform $(1) --build-arg GO_VERSION -f $(MAKEFILE_DIR)/Dockerfile $(MAKEFILE_DIR) + $(DOCKER) build --secret id=github_token,env=GITHUB_TOKEN --output type=tar,dest=$(CURDIR)/_output/nerdctl-full-$(VERSION_TRIMMED)-linux-$(1).tar --target out-full --platform $(1) --build-arg GO_VERSION -f $(MAKEFILE_DIR)/Dockerfile $(MAKEFILE_DIR) gzip -9 $(CURDIR)/_output/nerdctl-full-$(VERSION_TRIMMED)-linux-$(1).tar endef diff --git a/cmd/nerdctl/container/container_create.go b/cmd/nerdctl/container/container_create.go index e8d7e6a4d33..62ceb96a299 100644 --- a/cmd/nerdctl/container/container_create.go +++ b/cmd/nerdctl/container/container_create.go @@ -371,7 +371,6 @@ func createOptions(cmd *cobra.Command) (types.ContainerCreateOptions, error) { // #endregion // #region for metadata flags - opt.NameChanged = cmd.Flags().Changed("name") opt.Name, err = cmd.Flags().GetString("name") if err != nil { return opt, err diff --git a/cmd/nerdctl/image/image_convert.go b/cmd/nerdctl/image/image_convert.go index 871d9c97d81..eb3585099b2 100644 --- a/cmd/nerdctl/image/image_convert.go +++ b/cmd/nerdctl/image/image_convert.go @@ -89,6 +89,12 @@ func convertCommand() *cobra.Command { cmd.Flags().String("overlaybd-dbstr", "", "Database config string for overlaybd") // #endregion + // #region soci flags + cmd.Flags().Bool("soci", false, "Convert image to SOCI Index V2 format.") + cmd.Flags().Int64("soci-min-layer-size", -1, "The minimum size of layers that will be converted to SOCI Index V2 format") + cmd.Flags().Int64("soci-span-size", -1, "The size of SOCI spans") + // #endregion + // #region generic flags cmd.Flags().Bool("uncompress", false, "Convert tar.gz layers to uncompressed tar layers") cmd.Flags().Bool("oci", false, "Convert Docker media types to OCI media types") @@ -213,6 +219,21 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) { } // #endregion + // #region soci flags + soci, err := cmd.Flags().GetBool("soci") + if err != nil { + return types.ImageConvertOptions{}, err + } + sociMinLayerSize, err := cmd.Flags().GetInt64("soci-min-layer-size") + if err != nil { + return types.ImageConvertOptions{}, err + } + sociSpanSize, err := cmd.Flags().GetInt64("soci-span-size") + if err != nil { + return types.ImageConvertOptions{}, err + } + // #endregion + // #region generic flags uncompress, err := cmd.Flags().GetBool("uncompress") if err != nil { @@ -268,6 +289,13 @@ func convertOptions(cmd *cobra.Command) (types.ImageConvertOptions, error) { OverlayFsType: overlaybdFsType, OverlaydbDBStr: overlaybdDbstr, // #endregion + // #region soci flags + Soci: soci, + SociOptions: types.SociOptions{ + SpanSize: sociSpanSize, + MinLayerSize: sociMinLayerSize, + }, + // #endregion // #region generic flags Uncompress: uncompress, Oci: oci, diff --git a/cmd/nerdctl/image/image_convert_linux_test.go b/cmd/nerdctl/image/image_convert_linux_test.go index b26358ec8b9..fd4a8606324 100644 --- a/cmd/nerdctl/image/image_convert_linux_test.go +++ b/cmd/nerdctl/image/image_convert_linux_test.go @@ -88,6 +88,24 @@ func TestImageConvert(t *testing.T) { }, Expected: test.Expects(0, nil, nil), }, + { + Description: "soci", + Require: require.All( + require.Not(nerdtest.Docker), + nerdtest.Soci, + nerdtest.SociVersion("0.10.0"), + ), + Cleanup: func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("rmi", "-f", data.Identifier("converted-image")) + }, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + return helpers.Command("image", "convert", "--soci", + "--soci-span-size", "2097152", + "--soci-min-layer-size", "20971520", + testutil.CommonImage, data.Identifier("converted-image")) + }, + Expected: test.Expects(0, nil, nil), + }, }, } diff --git a/docs/command-reference.md b/docs/command-reference.md index a4173d8f93a..f559a73d711 100644 --- a/docs/command-reference.md +++ b/docs/command-reference.md @@ -960,6 +960,11 @@ Flags: - `--oci` : convert Docker media types to OCI media types - `--platform=` : convert content for a specific platform - `--all-platforms` : convert content for all platforms (default: false) +- `--soci`: generate SOCI v2 Indices to oci images. +*[**Note**: content is converted for all platforms by default when using this flag, use the `--platorm` flag to limit this behavior]* +- `--soci-min-layer-size` : Span size in bytes that soci index uses to segment layer data. Default is 4 MiB. +- `--soci-min-layer-size`: Minimum layer size in bytes to build zTOC for. Smaller layers won't have zTOC and not lazy pulled. Default is 10 MiB. + ### :nerd_face: nerdctl image encrypt diff --git a/docs/soci.md b/docs/soci.md index 67fbe92f584..d2dc84645df 100644 --- a/docs/soci.md +++ b/docs/soci.md @@ -45,3 +45,18 @@ For images that already have SOCI indices, see https://gallery.ecr.aws/soci-work nerdctl push --snapshotter=soci --soci-span-size=2097152 --soci-min-layer-size=20971520 public.ecr.aws/my-registry/my-repo:latest ``` --soci-span-size and --soci-min-layer-size are two properties to customize the SOCI index. See [Command Reference](https://github.com/containerd/nerdctl/blob/377b2077bb616194a8ef1e19ccde32aa1ffd6c84/docs/command-reference.md?plain=1#L773) for further details. + + +## Enable SOCI for `nerdctl image convert` + +| :zap: Requirement | nerdctl >= 2.2.0 | +| ----------------- | ---------------- | + +| :zap: Requirement | soci-snapshotter >= 0.10.0 | +| ----------------- | ---------------- | + +- Convert an image to generate SOCI Index artifacts v2. Running the `nerdctl image convert` with the `--soci` flag and a `srcImg` and `dstImg`, `nerdctl` will create the SOCI v2 indices and the new image will be present in the `dstImg` address. +```console +nerdctl image convert --soci --soci-span-size=2097152 --soci-min-layer-size=20971520 public.ecr.aws/my-registry/my-repo:latest public.ecr.aws/my-registry/my-repo:soci +``` +--soci-span-size and --soci-min-layer-size are two properties to customize the SOCI index. See [Command Reference](https://github.com/containerd/nerdctl/blob/377b2077bb616194a8ef1e19ccde32aa1ffd6c84/docs/command-reference.md?plain=1#L773) for further details. \ No newline at end of file diff --git a/go.mod b/go.mod index 777eede6974..b5ea3011f2d 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/Masterminds/semver/v3 v3.3.1 github.com/Microsoft/go-winio v0.6.2 github.com/Microsoft/hcsshim v0.13.0 - github.com/compose-spec/compose-go/v2 v2.6.3 //gomodjail:unconfined + github.com/compose-spec/compose-go/v2 v2.6.4 //gomodjail:unconfined github.com/containerd/accelerated-container-image v1.3.0 github.com/containerd/cgroups/v3 v3.0.5 //gomodjail:unconfined github.com/containerd/console v1.0.5 //gomodjail:unconfined diff --git a/go.sum b/go.sum index d9d8f080b4a..0554452d966 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,8 @@ github.com/cilium/ebpf v0.16.0 h1:+BiEnHL6Z7lXnlGUsXQPPAE7+kenAd4ES8MQ5min0Ok= github.com/cilium/ebpf v0.16.0/go.mod h1:L7u2Blt2jMM/vLAVgjxluxtBKlz3/GWjB0dMOEngfwE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/compose-spec/compose-go/v2 v2.6.3 h1:zfW1Qp605ESySyth/zR+6yLr55XE0AiOAUlZLHKMoW0= -github.com/compose-spec/compose-go/v2 v2.6.3/go.mod h1:vPlkN0i+0LjLf9rv52lodNMUTJF5YHVfHVGLLIP67NA= +github.com/compose-spec/compose-go/v2 v2.6.4 h1:Gjv6x8eAhqwwWvoXIo0oZ4bDQBh0OMwdU7LUL9PDLiM= +github.com/compose-spec/compose-go/v2 v2.6.4/go.mod h1:vPlkN0i+0LjLf9rv52lodNMUTJF5YHVfHVGLLIP67NA= github.com/containerd/accelerated-container-image v1.3.0 h1:sFbTgSuMboeKHa9f7MY11hWF1XxVWjFoiTsXYtOtvdU= github.com/containerd/accelerated-container-image v1.3.0/go.mod h1:EvKVWor6ZQNUyYp0MZm5hw4k21ropuz7EegM+m/Jb/Q= github.com/containerd/cgroups/v3 v3.0.5 h1:44na7Ud+VwyE7LIoJ8JTNQOa549a8543BmzaJHo6Bzo= diff --git a/hack/scripts/lib.sh b/hack/scripts/lib.sh index 8eb93ca527a..7ce1da9a103 100755 --- a/hack/scripts/lib.sh +++ b/hack/scripts/lib.sh @@ -226,9 +226,10 @@ github::settoken(){ } github::request(){ - local endpoint="$1" + local accept="$1" + local endpoint="$2" local args=( - "Accept: application/vnd.github+json" + "Accept: $accept" "X-GitHub-Api-Version: 2022-11-28" ) @@ -237,21 +238,30 @@ github::request(){ http::get /dev/stdout https://api.github.com/"$endpoint" "${args[@]}" } +github::file(){ + local repo="$1" + local path="$2" + local ref="${3:-main}" + github::request "application/vnd.github.v3.raw" "repos/$repo/contents/$path?ref=$ref" +} + github::tags::latest(){ local repo="$1" - github::request "repos/$repo/tags" | jq -rc .[0].name + github::request "application/vnd.github+json" "repos/$repo/tags" | jq -rc .[0].name } github::releases(){ local repo="$1" - github::request "repos/$repo/releases" | + github::request "application/vnd.github+json" "repos/$repo/releases" | jq -rc .[] } github::releases::latest(){ local repo="$1" - github::request "repos/$repo/releases/latest" | jq -rc . + github::request "application/vnd.github+json" "repos/$repo/releases/latest" | jq -rc . } log::init host::require jq tar curl shasum + +[[ "${1:-}" != "github"* ]] || "$@" diff --git a/pkg/api/types/container_types.go b/pkg/api/types/container_types.go index 8f0d0137b02..7489d449672 100644 --- a/pkg/api/types/container_types.go +++ b/pkg/api/types/container_types.go @@ -237,8 +237,6 @@ type ContainerCreateOptions struct { // #endregion // #region for metadata flags - // NameChanged specifies whether the name has been changed - NameChanged bool // Name assign a name to the container Name string // Label set meta data on a container diff --git a/pkg/api/types/image_types.go b/pkg/api/types/image_types.go index d48e6318026..b2a8316c890 100644 --- a/pkg/api/types/image_types.go +++ b/pkg/api/types/image_types.go @@ -19,7 +19,7 @@ package types import ( "io" - "github.com/opencontainers/image-spec/specs-go/v1" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // ImageListOptions specifies options for `nerdctl image list`. @@ -124,6 +124,12 @@ type ImageConvertOptions struct { OverlaydbDBStr string // #endregion + // #region soci flags + // Soci convert image to SOCI format.eiifc + Soci bool + // SociOptions contains SOCI-specific options + SociOptions SociOptions + // #endregion } // ImageCryptOptions specifies options for `nerdctl image encrypt` and `nerdctl image decrypt`. @@ -200,7 +206,7 @@ type ImagePullOptions struct { // If nil, it will unpack automatically if only 1 platform is specified. Unpack *bool // Content for specific platforms. Empty if `--all-platforms` is true - OCISpecPlatform []v1.Platform + OCISpecPlatform []ocispec.Platform // Pull mode Mode string // Suppress verbose output diff --git a/pkg/cmd/container/create.go b/pkg/cmd/container/create.go index 6699f97c00d..deee011f343 100644 --- a/pkg/cmd/container/create.go +++ b/pkg/cmd/container/create.go @@ -339,8 +339,7 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa cOpts = append(cOpts, lCOpts...) var containerNameStore namestore.NameStore - if options.Name == "" && !options.NameChanged { - // Automatically set the container name, unless `--name=""` was explicitly specified. + if options.Name == "" { var imageRef string if ensuredImage != nil { imageRef = ensuredImage.Ref @@ -352,15 +351,15 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa } options.Name = parsedReference.SuggestContainerName(id) } - if options.Name != "" { - containerNameStore, err = namestore.New(dataStore, options.GOptions.Namespace) - if err != nil { - return nil, generateRemoveOrphanedDirsFunc(ctx, id, dataStore, internalLabels), err - } - if err := containerNameStore.Acquire(options.Name, id); err != nil { - return nil, generateRemoveOrphanedDirsFunc(ctx, id, dataStore, internalLabels), err - } + + containerNameStore, err = namestore.New(dataStore, options.GOptions.Namespace) + if err != nil { + return nil, generateRemoveOrphanedDirsFunc(ctx, id, dataStore, internalLabels), err } + if err := containerNameStore.Acquire(options.Name, id); err != nil { + return nil, generateRemoveOrphanedDirsFunc(ctx, id, dataStore, internalLabels), err + } + internalLabels.name = options.Name internalLabels.pidFile = options.PidFile @@ -714,9 +713,7 @@ func withInternalLabels(internalLabels internalLabels) (containerd.NewContainerO var hostConfigLabel dockercompat.HostConfigLabel var dnsSettings dockercompat.DNSSettings m[labels.Namespace] = internalLabels.namespace - if internalLabels.name != "" { - m[labels.Name] = internalLabels.name - } + m[labels.Name] = internalLabels.name m[labels.Hostname] = internalLabels.hostname m[labels.Domainname] = internalLabels.domainname extraHostsJSON, err := json.Marshal(internalLabels.extraHosts) @@ -1024,15 +1021,13 @@ func generateGcFunc(ctx context.Context, container containerd.Container, ns, id, log.G(ctx).WithError(rmErr).Warnf("failed to remove container %q state dir %q", id, internalLabels.stateDir) } - if name != "" { - var errE error - if containerNameStore, errE = namestore.New(dataStore, ns); errE != nil { - log.G(ctx).WithError(errE).Warnf("failed to instantiate container name store during cleanup for container %q", id) - } - // Double-releasing may happen with containers started with --rm, so, ignore NotFound errors - if errE := containerNameStore.Release(name, id); errE != nil && !errors.Is(errE, store.ErrNotFound) { - log.G(ctx).WithError(errE).Warnf("failed to release container name store for container %q (%s)", name, id) - } + var errE error + if containerNameStore, errE = namestore.New(dataStore, ns); errE != nil { + log.G(ctx).WithError(errE).Warnf("failed to instantiate container name store during cleanup for container %q", id) + } + // Double-releasing may happen with containers started with --rm, so, ignore NotFound errors + if errE := containerNameStore.Release(name, id); errE != nil && !errors.Is(errE, store.ErrNotFound) { + log.G(ctx).WithError(errE).Warnf("failed to release container name store for container %q (%s)", name, id) } } } diff --git a/pkg/cmd/image/convert.go b/pkg/cmd/image/convert.go index b7963ea2702..12a2040d598 100644 --- a/pkg/cmd/image/convert.go +++ b/pkg/cmd/image/convert.go @@ -47,6 +47,7 @@ import ( converterutil "github.com/containerd/nerdctl/v2/pkg/imgutil/converter" "github.com/containerd/nerdctl/v2/pkg/platformutil" "github.com/containerd/nerdctl/v2/pkg/referenceutil" + "github.com/containerd/nerdctl/v2/pkg/snapshotterutil" ) func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRawRef string, options types.ImageConvertOptions) error { @@ -86,8 +87,9 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa zstdchunked := options.ZstdChunked overlaybd := options.Overlaybd nydus := options.Nydus + soci := options.Soci var finalize func(ctx context.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error) - if estargz || zstd || zstdchunked || overlaybd || nydus { + if estargz || zstd || zstdchunked || overlaybd || nydus || soci { convertCount := 0 if estargz { convertCount++ @@ -104,9 +106,12 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa if nydus { convertCount++ } + if soci { + convertCount++ + } if convertCount > 1 { - return errors.New("options --estargz, --zstdchunked, --overlaybd and --nydus lead to conflict, only one of them can be used") + return errors.New("options --estargz, --zstdchunked, --overlaybd, --nydus and --soci lead to conflict, only one of them can be used") } var convertFunc converter.ConvertFunc @@ -164,6 +169,16 @@ func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRa )), ) convertType = "nydus" + case soci: + // Convert image to SOCI format + convertedRef, err := snapshotterutil.ConvertSociIndexV2(ctx, client, srcRef, targetRef, options.GOptions, options.Platforms, options.SociOptions) + if err != nil { + return fmt.Errorf("failed to convert image to SOCI format: %w", err) + } + res := converterutil.ConvertedImageInfo{ + Image: convertedRef, + } + return printConvertedImage(options.Stdout, options, res) } if convertType != "overlaybd" { diff --git a/pkg/cmd/image/push.go b/pkg/cmd/image/push.go index 0c463e76f02..0519d919282 100644 --- a/pkg/cmd/image/push.go +++ b/pkg/cmd/image/push.go @@ -209,7 +209,7 @@ func Push(ctx context.Context, client *containerd.Client, rawRef string, options return err } if options.GOptions.Snapshotter == "soci" { - if err = snapshotterutil.CreateSoci(ref, options.GOptions, options.AllPlatforms, options.Platforms, options.SociOptions); err != nil { + if err = snapshotterutil.CreateSociIndexV1(ref, options.GOptions, options.AllPlatforms, options.Platforms, options.SociOptions); err != nil { return err } if err = snapshotterutil.PushSoci(ref, options.GOptions, options.AllPlatforms, options.Platforms); err != nil { diff --git a/pkg/portutil/portutil_test.go b/pkg/portutil/portutil_test.go index 55930e07f3b..46b9eff7544 100644 --- a/pkg/portutil/portutil_test.go +++ b/pkg/portutil/portutil_test.go @@ -30,7 +30,7 @@ import ( "github.com/containerd/nerdctl/v2/pkg/rootlessutil" ) -func TestTestParseFlagPWithPlatformSpec(t *testing.T) { +func TestParseFlagPWithPlatformSpec(t *testing.T) { if runtime.GOOS != "linux" || rootlessutil.IsRootless() { t.Skip("no non-Linux platform or rootless mode in Linux are not supported yet") } @@ -232,10 +232,10 @@ func TestParseFlagP(t *testing.T) { s string } tests := []struct { - name string - args args - want []cni.PortMapping - wantErr bool + name string + args args + want []cni.PortMapping + wantErrMsg string }{ { name: "normal", @@ -250,7 +250,7 @@ func TestParseFlagP(t *testing.T) { HostIP: "127.0.0.1", }, }, - wantErr: false, + wantErrMsg: "", }, { name: "with port range", @@ -271,15 +271,15 @@ func TestParseFlagP(t *testing.T) { HostIP: "127.0.0.1", }, }, - wantErr: false, + wantErrMsg: "", }, { name: "with wrong port range", args: args{ s: "127.0.0.1:3000-3001:8080-8082/tcp", }, - want: nil, - wantErr: true, + want: nil, + wantErrMsg: "invalid ranges specified for container and host Ports: 8080-8082 and 3000-3001", }, { name: "without host ip", @@ -294,7 +294,7 @@ func TestParseFlagP(t *testing.T) { HostIP: "0.0.0.0", }, }, - wantErr: false, + wantErrMsg: "", }, { name: "without protocol", @@ -309,7 +309,7 @@ func TestParseFlagP(t *testing.T) { HostIP: "0.0.0.0", }, }, - wantErr: false, + wantErrMsg: "", }, { name: "with protocol udp", @@ -324,10 +324,10 @@ func TestParseFlagP(t *testing.T) { HostIP: "0.0.0.0", }, }, - wantErr: false, + wantErrMsg: "", }, { - name: "with protocol udp", + name: "with protocol sctp", args: args{ s: "3000:8080/sctp", }, @@ -339,7 +339,7 @@ func TestParseFlagP(t *testing.T) { HostIP: "0.0.0.0", }, }, - wantErr: false, + wantErrMsg: "", }, { name: "with ipv6 host ip", @@ -354,86 +354,82 @@ func TestParseFlagP(t *testing.T) { HostIP: "::0", }, }, - wantErr: false, + wantErrMsg: "", }, { name: "with invalid protocol", args: args{ s: "3000:8080/invalid", }, - want: nil, - wantErr: true, + want: nil, + wantErrMsg: `invalid protocol "invalid"`, }, { name: "multiple colon", args: args{ s: "127.0.0.1:3000:0.0.0.0:8080", }, - want: nil, - wantErr: true, + want: nil, + wantErrMsg: "invalid hostPort: 127.0.0.1:3000:0.0.0.0", }, { name: "multiple slash", args: args{ s: "127.0.0.1:3000:8080/tcp/", }, - want: nil, - wantErr: true, + want: nil, + wantErrMsg: `failed to parse "127.0.0.1:3000:8080/tcp/", unexpected slashes`, }, { name: "invalid ip", args: args{ s: "127.0.0.256:3000:8080/tcp", }, - want: nil, - wantErr: true, + want: nil, + wantErrMsg: "invalid ip address: 127.0.0.256", }, { name: "large port", args: args{ s: "3000:65536", }, - want: nil, - wantErr: true, + want: nil, + wantErrMsg: "invalid containerPort: 65536", }, { name: "blank", args: args{ s: "", }, - want: nil, - wantErr: true, + want: nil, + wantErrMsg: "no port specified: ", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, err := ParseFlagP(tt.args.s) - t.Log(err) - if (err != nil) != tt.wantErr { - t.Errorf("ParseFlagP() error = %v, wantErr %v", err, tt.wantErr) - return + if tt.wantErrMsg == "" { + assert.NilError(t, err) + } else { + assert.Error(t, err, tt.wantErrMsg) } if !reflect.DeepEqual(got, tt.want) { - if len(got) == len(tt.want) { - if len(got) > 1 { - var hostPorts []int32 - var containerPorts []int32 - for _, value := range got { - hostPorts = append(hostPorts, value.HostPort) - containerPorts = append(containerPorts, value.ContainerPort) - } - sort.Slice(hostPorts, func(i, j int) bool { - return i < j - }) - sort.Slice(containerPorts, func(i, j int) bool { - return i < j - }) - if (hostPorts[len(hostPorts)-1] - hostPorts[0]) != (containerPorts[len(hostPorts)-1] - containerPorts[0]) { - t.Errorf("ParseFlagP() = %v, want %v", got, tt.want) - } + assert.Equal(t, len(got), len(tt.want)) + if len(got) > 0 { + sort.Slice(got, func(i, j int) bool { + return got[i].HostPort < got[j].HostPort + }) + assert.Equal( + t, + got[len(got)-1].HostPort-got[0].HostPort, + got[len(got)-1].ContainerPort-got[0].ContainerPort, + ) + for i := range len(got) { + assert.Equal(t, got[i].HostPort, tt.want[i].HostPort) + assert.Equal(t, got[i].ContainerPort, tt.want[i].ContainerPort) + assert.Equal(t, got[i].Protocol, tt.want[i].Protocol) + assert.Equal(t, got[i].HostIP, tt.want[i].HostIP) } - } else { - t.Errorf("ParseFlagP() = %v, want %v", got, tt.want) } } }) diff --git a/pkg/snapshotterutil/sociutil.go b/pkg/snapshotterutil/sociutil.go index a2148de027c..6f4da8dbd31 100644 --- a/pkg/snapshotterutil/sociutil.go +++ b/pkg/snapshotterutil/sociutil.go @@ -18,23 +18,26 @@ package snapshotterutil import ( "bufio" + "context" + "fmt" "os" "os/exec" "strconv" "strings" + "github.com/containerd/containerd/v2/client" "github.com/containerd/log" "github.com/containerd/nerdctl/v2/pkg/api/types" ) -// CreateSoci creates a SOCI index(`rawRef`) -func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string, sOpts types.SociOptions) error { +// setupSociCommand creates and sets up a SOCI command with common configuration +func setupSociCommand(gOpts types.GlobalCommandOptions) (*exec.Cmd, error) { sociExecutable, err := exec.LookPath("soci") if err != nil { log.L.WithError(err).Error("soci executable not found in path $PATH") log.L.Info("you might consider installing soci from: https://github.com/awslabs/soci-snapshotter/blob/main/docs/install.md") - return err + return nil, err } sociCmd := exec.Command(sociExecutable) @@ -47,7 +50,65 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo if gOpts.Namespace != "" { sociCmd.Args = append(sociCmd.Args, "--namespace", gOpts.Namespace) } - // #endregion + + return sociCmd, nil +} + +// ConvertSociIndexV2 converts an image to SOCI format and returns the converted image reference with digest +func ConvertSociIndexV2(ctx context.Context, client *client.Client, srcRef string, destRef string, gOpts types.GlobalCommandOptions, platforms []string, sOpts types.SociOptions) (string, error) { + sociCmd, err := setupSociCommand(gOpts) + if err != nil { + return "", err + } + + // TODO: Implement conversion logic + sociCmd.Args = append(sociCmd.Args, "convert") + + if len(platforms) > 0 { + // multiple values need to be passed as separate, repeating flags in soci as it uses urfave + // https://github.com/urfave/cli/blob/main/docs/v2/examples/flags.md#multiple-values-per-single-flag + for _, p := range platforms { + sociCmd.Args = append(sociCmd.Args, "--platform", p) + } + } + + if sOpts.SpanSize != -1 { + sociCmd.Args = append(sociCmd.Args, "--span-size", strconv.FormatInt(sOpts.SpanSize, 10)) + } + + if sOpts.MinLayerSize != -1 { + sociCmd.Args = append(sociCmd.Args, "--min-layer-size", strconv.FormatInt(sOpts.MinLayerSize, 10)) + } + + sociCmd.Args = append(sociCmd.Args, srcRef, destRef) + + log.L.Infof("Converting image from %s to %s using SOCI format", srcRef, destRef) + + err = processSociIO(sociCmd) + if err != nil { + return "", err + } + err = sociCmd.Wait() + if err != nil { + return "", err + } + + // Get the converted image's digest + img, err := client.GetImage(ctx, destRef) + if err != nil { + return "", fmt.Errorf("failed to get converted image: %w", err) + } + + // Return the full reference with digest + return fmt.Sprintf("%s@%s", destRef, img.Target().Digest), nil +} + +// CreateSociIndexV1 creates a SOCI index(`rawRef`) +func CreateSociIndexV1(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string, sOpts types.SociOptions) error { + sociCmd, err := setupSociCommand(gOpts) + if err != nil { + return err + } // Global flags have to be put before subcommand before soci upgrades to urfave v3. // https://github.com/urfave/cli/issues/1113 @@ -73,7 +134,7 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo // --timeout, --debug, --content-store sociCmd.Args = append(sociCmd.Args, rawRef) - log.L.Debugf("running %s %v", sociExecutable, sociCmd.Args) + log.L.Debugf("running soci %v", sociCmd.Args) err = processSociIO(sociCmd) if err != nil { @@ -88,25 +149,11 @@ func CreateSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform boo func PushSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, platforms []string) error { log.L.Debugf("pushing SOCI index: %s", rawRef) - sociExecutable, err := exec.LookPath("soci") + sociCmd, err := setupSociCommand(gOpts) if err != nil { - log.L.WithError(err).Error("soci executable not found in path $PATH") - log.L.Info("you might consider installing soci from: https://github.com/awslabs/soci-snapshotter/blob/main/docs/install.md") return err } - sociCmd := exec.Command(sociExecutable) - sociCmd.Env = os.Environ() - - // #region for global flags. - if gOpts.Address != "" { - sociCmd.Args = append(sociCmd.Args, "--address", gOpts.Address) - } - if gOpts.Namespace != "" { - sociCmd.Args = append(sociCmd.Args, "--namespace", gOpts.Namespace) - } - // #endregion - // Global flags have to be put before subcommand before soci upgrades to urfave v3. // https://github.com/urfave/cli/issues/1113 sociCmd.Args = append(sociCmd.Args, "push") @@ -131,7 +178,7 @@ func PushSoci(rawRef string, gOpts types.GlobalCommandOptions, allPlatform bool, } sociCmd.Args = append(sociCmd.Args, rawRef) - log.L.Debugf("running %s %v", sociExecutable, sociCmd.Args) + log.L.Debugf("running soci %v", sociCmd.Args) err = processSociIO(sociCmd) if err != nil { diff --git a/pkg/testutil/nerdtest/requirements.go b/pkg/testutil/nerdtest/requirements.go index e0e621501ee..8a791df720f 100644 --- a/pkg/testutil/nerdtest/requirements.go +++ b/pkg/testutil/nerdtest/requirements.go @@ -419,6 +419,53 @@ var RemapIDs = &test.Requirement{ }, } +// SociVersion returns a requirement that checks if the installed SOCI version +// meets the minimum required version +func SociVersion(minVersion string) *test.Requirement { + return &test.Requirement{ + Check: func(data test.Data, helpers test.Helpers) (bool, string) { + sociExecutable, err := exec.LookPath("soci") + if err != nil { + return false, fmt.Sprintf("soci executable not found in path $PATH: %v", err) + } + + cmd := exec.Command(sociExecutable, "--version") + output, err := cmd.Output() + if err != nil { + return false, fmt.Sprintf("failed to get soci version: %v", err) + } + + // Parse version from output + // Example output format: "soci version v0.9.0 737f61a3db40c386f997c1f126344158aa3ad43c" + versionStr := strings.TrimSpace(string(output)) + parts := strings.Fields(versionStr) + if len(parts) < 3 { + return false, fmt.Sprintf("unexpected soci version output format: %s", versionStr) + } + + // Extract version number without 'v' prefix + installedVersion := strings.TrimPrefix(parts[2], "v") + + // Compare versions + v1, err := semver.NewVersion(installedVersion) + if err != nil { + return false, fmt.Sprintf("failed to parse installed version %s: %v", installedVersion, err) + } + + v2, err := semver.NewVersion(minVersion) + if err != nil { + return false, fmt.Sprintf("failed to parse minimum required version %s: %v", minVersion, err) + } + + if v1.LessThan(v2) { + return false, fmt.Sprintf("installed soci version %s is older than required version %s", installedVersion, minVersion) + } + + return true, fmt.Sprintf("soci version %s meets minimum requirement %s", installedVersion, minVersion) + }, + } +} + func ContainerdVersion(v string) *test.Requirement { return &test.Requirement{ Check: func(data test.Data, helpers test.Helpers) (bool, string) {