add oci pull to images

Dockerfile

@@ -1,5 +1,13 @@
-FROM golang:1.25-alpine AS builder
+FROM golang:1.25-alpine AS go-builder
 
+# This is needed in order for OCI pull tests to succeed
+FROM docker:29-dind-rootless AS builder
+# FROM docker:29-dind AS builder
+
+COPY --from=go-builder /usr/local/go /usr/local/go
+ENV PATH="$PATH:/usr/local/go/bin"
+
+USER root
 RUN apk add \
 	binutils \
 	coreutils \
@@ -11,6 +19,8 @@ RUN apk add \
 	libpcap-dev
 WORKDIR /work
 COPY . .
+
+USER rootless
 RUN make all
 # Install Intel Firmware for e800 based network cards
 ENV ICE_VERSION=1.14.13

cmd/image/image.go

@@ -1,13 +1,22 @@
 package image
 
 import (
+	"archive/tar"
+	"context"
+	"errors"
 	"fmt"
 	"log/slog"
+	"path/filepath"
 
 	pb "github.com/cheggaaa/pb/v3"
 	"github.com/mholt/archiver"
 	lz4 "github.com/pierrec/lz4/v4"
 
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/name"
+	"github.com/google/go-containerregistry/pkg/v1/mutate"
+	"github.com/google/go-containerregistry/pkg/v1/remote"
+
 	//nolint:gosec
 	"crypto/md5"
 	"io"
@@ -25,7 +34,43 @@ func NewImage(log *slog.Logger) *Image {
 	return &Image{log: log}
 }
 
-// Pull a image from s3
+func (i *Image) OciPull(ctx context.Context, imageRef, mountDir, username, password string) error {
+	imageRefWithoutOciPrefix := strings.TrimPrefix(imageRef, "oci://")
+	// Parse the image reference (e.g., docker.io/library/alpine:latest)
+	ref, err := name.ParseReference(imageRefWithoutOciPrefix)
+	if err != nil {
+		return fmt.Errorf("parsing image reference: %w", err)
+	}
+
+	// Choose authentication method
+	var auth = authn.Anonymous
+	if username != "" || password != "" {
+		auth = &authn.Basic{
+			Username: username,
+			Password: password,
+		}
+	}
+
+	i.log.Info("pull oci image", "image", imageRef)
+	img, err := remote.Image(ref, remote.WithAuth(auth))
+	if err != nil {
+		return fmt.Errorf("fetching remote image: %w", err)
+	}
+
+	// Flatten layers and create a tar stream
+	rc := mutate.Extract(img)
+	defer rc.Close()
+
+	i.log.Info(fmt.Sprintf("untar oci image into %s", mountDir), "image", imageRef)
+	if err := i.untar(rc, mountDir); err != nil {
+		return fmt.Errorf("extracting tar: %w", err)
+	}
+
+	i.log.Info("pull oci image done", "image", imageRef)
+	return nil
+}
+
+// Pull an image from s3
 func (i *Image) Pull(image, destination string) error {
 	i.log.Info("pull image", "image", image)
 	md5destination := destination + ".md5"
@@ -49,7 +94,7 @@ func (i *Image) Pull(image, destination string) error {
 	return nil
 }
 
-// Burn a image pulling a tarball and unpack to a specific directory
+// Burn an image pulling a tarball and unpack to a specific directory
 func (i *Image) Burn(prefix, image, source string) error {
 	i.log.Info("burn image", "image", image)
 	begin := time.Now()
@@ -171,3 +216,71 @@ func (i *Image) download(source, dest string) error {
 
 	return nil
 }
+
+func (i *Image) untar(r io.Reader, dest string) error {
+	tr := tar.NewReader(r)
+	for {
+		hdr, err := tr.Next()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("reading tar: %w", err)
+		}
+
+		target := filepath.Join(dest, hdr.Name)
+
+		fmt.Printf("extracting:%s\n", target)
+
+		if strings.HasSuffix(target, ".log") {
+			fmt.Printf("skipping:%s\n", target)
+			continue
+		}
+
+		switch hdr.Typeflag {
+		case tar.TypeDir:
+			if err := os.MkdirAll(target, os.FileMode(hdr.Mode)); err != nil {
+				return fmt.Errorf("creating dir: %w", err)
+			}
+			if err := os.Lchown(target, hdr.Uid, hdr.Gid); err != nil && !errors.Is(err, os.ErrPermission) {
+				return fmt.Errorf("chown dir: %w", err)
+			}
+
+		case tar.TypeReg:
+			if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
+				return fmt.Errorf("creating parent dir: %w", err)
+			}
+			f, err := os.OpenFile(target, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode))
+			if err != nil {
+				return fmt.Errorf("creating file: %w", err)
+			}
+			if _, err := io.Copy(f, tr); err != nil {
+				f.Close()
+				return fmt.Errorf("copying file: %w", err)
+			}
+			f.Close()
+
+			if err := os.Chmod(target, os.FileMode(hdr.Mode)); err != nil {
+				return fmt.Errorf("chmod: %w", err)
+			}
+			if err := os.Lchown(target, hdr.Uid, hdr.Gid); err != nil && !errors.Is(err, os.ErrPermission) {
+				return fmt.Errorf("chown file: %w", err)
+			}
+
+		case tar.TypeSymlink:
+			if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
+				return fmt.Errorf("creating parent dir: %w", err)
+			}
+			if err := os.Symlink(hdr.Linkname, target); err != nil {
+				return fmt.Errorf("creating symlink: %w", err)
+			}
+			if err := os.Lchown(target, hdr.Uid, hdr.Gid); err != nil && !errors.Is(err, os.ErrPermission) {
+				return fmt.Errorf("chown symlink: %w", err)
+			}
+
+		default:
+			// skip unsupported or special files
+		}
+	}
+	return nil
+}

cmd/image/image_test.go

@@ -1,10 +1,23 @@
 package image
 
 import (
+	"context"
+	"crypto/rand"
+	"fmt"
 	"log/slog"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"testing"
+
+	"github.com/foomo/htpasswd"
+	"github.com/google/go-containerregistry/pkg/authn"
+	"github.com/google/go-containerregistry/pkg/crane"
+	"github.com/metal-stack/metal-lib/pkg/pointer"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"github.com/testcontainers/testcontainers-go"
+	"github.com/testcontainers/testcontainers-go/wait"
 )
 
 func TestCheckMD5(t *testing.T) {
@@ -41,5 +54,170 @@ func TestCheckMD5(t *testing.T) {
 	if !matches {
 		t.Error("expected md5 matches, but didn't")
 	}
+}
+
+func TestOciPull(t *testing.T) {
+	var (
+		assert = assert.New(t)
+
+		ctx          = context.Background()
+		mountDir     = "/tmp/oci-pull-mount-dir"
+		extractedBin = "a"
+
+		anonymousUsername = ""
+		anonymousPassword = ""
+	)
+
+	t.Run("successful anonymous pull", func(t *testing.T) {
+		regIP, regPort, err := startRegistry(nil, nil, nil)
+		require.NoError(t, err)
+		registry := fmt.Sprintf("%s:%d", regIP, regPort)
+
+		imageRef := fmt.Sprintf("%s/library/image", registry)
+		err = createImage(imageRef, "", "")
+		require.NoError(t, err)
+
+		err = os.MkdirAll(mountDir, 0777)
+		require.NoError(t, err)
+		defer os.RemoveAll(mountDir)
+
+		i := NewImage(slog.Default())
+		if err = i.OciPull(ctx, imageRef, mountDir, anonymousUsername, anonymousPassword); err != nil {
+			t.Error(err)
+		}
+
+		extractedBinFullPath := filepath.Join(mountDir, extractedBin)
+		assert.FileExists(extractedBinFullPath)
+	})
+
+	t.Run("successful authenticated pull", func(t *testing.T) {
+		var (
+			username = "test-user"
+			password = "test-password"
+		)
+
+		f, err := os.CreateTemp("", "htpasswd")
+		require.NoError(t, err)
+		defer func() {
+			_ = os.Remove(f.Name())
+		}()
+
+		err = htpasswd.SetPassword(f.Name(), username, password, htpasswd.HashBCrypt)
+		require.NoError(t, err)
+
+		env := map[string]string{
+			"REGISTRY_AUTH":                "htpasswd",
+			"REGISTRY_AUTH_HTPASSWD_REALM": "registry-login",
+			"REGISTRY_AUTH_HTPASSWD_PATH":  "/htpasswd",
+		}
+		regIP, regPort, err := startRegistry(env, pointer.Pointer(f.Name()), pointer.Pointer("/htpasswd"))
+		require.NoError(t, err)
+		registry := fmt.Sprintf("%s:%d", regIP, regPort)
+
+		imageRefBehindAuth := fmt.Sprintf("%s/library/image", registry)
+		err = createImage(imageRefBehindAuth, username, password)
+		require.NoError(t, err)
+
+		err = os.MkdirAll(mountDir, 0777)
+		require.NoError(t, err)
+		defer os.RemoveAll(mountDir)
+
+		i := NewImage(slog.Default())
+		if err = i.OciPull(ctx, imageRefBehindAuth, mountDir, username, password); err != nil {
+			t.Error(err)
+		}
+
+		extractedBinFullPath := filepath.Join(mountDir, extractedBin)
+		assert.FileExists(extractedBinFullPath)
+	})
+
+	t.Run("parsing of image refs fails", func(t *testing.T) {
+		invalidImageRef := "invalid://"
+		i := NewImage(slog.Default())
+		err := i.OciPull(ctx, invalidImageRef, mountDir, anonymousUsername, anonymousPassword)
+		assert.EqualError(err, "parsing image reference: could not parse reference: invalid://")
+	})
+
+	t.Run("pulling remote image fails", func(t *testing.T) {
+		imageRefDoesNotExist := "oci://does/not/exist:tag"
+		i := NewImage(slog.Default())
+		err := i.OciPull(ctx, imageRefDoesNotExist, mountDir, anonymousUsername, anonymousPassword)
+		assert.Error(err)
+	})
+}
+
+// HELPER FUNCTIONS
+func startRegistry(env map[string]string, src, dst *string) (string, int, error) {
+	ctx := context.Background()
+	var (
+		c   testcontainers.Container
+		err error
+	)
+
+	req := testcontainers.ContainerRequest{
+		Image:        "registry:3",
+		ExposedPorts: []string{"5000/tcp"},
+		Env:          env,
+		WaitingFor: wait.ForAll(
+			wait.ForLog("listening on"),
+			wait.ForListeningPort("5000/tcp"),
+		),
+	}
+	c, err = testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+		ContainerRequest: req,
+		Started:          true,
+	})
+	if err != nil {
+		return "", 0, err
+	}
+	if src != nil && dst != nil {
+		err = c.CopyFileToContainer(ctx, *src, *dst, 0o777)
+		if err != nil {
+			return "", 0, err
+		}
+	}
+
+	ip, err := c.Host(ctx)
+	if err != nil {
+		return ip, 0, err
+	}
+	port, err := c.MappedPort(ctx, "5000")
+	if err != nil {
+		return ip, port.Int(), err
+	}
+
+	return ip, port.Int(), nil
+}
+
+func createImage(imageName, username, password string, tags ...string) error {
+	// ensure every image has distinct content
+	buf := make([]byte, 128)
+	_, err := rand.Read(buf)
+	if err != nil {
+		return err
+	}
+	img, err := crane.Image(map[string][]byte{"a": buf})
+	if err != nil {
+		return err
+	}
+
+	var auth = authn.Anonymous
+	if username != "" || password != "" {
+		auth = &authn.Basic{
+			Username: username,
+			Password: password,
+		}
+	}
+	err = crane.Push(img, imageName, crane.WithAuth(auth))
+	if err != nil {
+		return err
+	}
+	for _, tag := range tags {
+		err := crane.Push(img, imageName+":"+tag, crane.WithAuth(auth))
+		if err != nil {
+			return err
+		}
+	}
 
+	return nil
 }