feat(registry): Add OCI Distribution registry for docker push support (#19)

* feat(registry): add OCI Distribution registry for image push

Implement OCI Distribution Spec compliant registry endpoints that accept
pushed images and trigger conversion to hypeman's disk format.

Changes:
- Add lib/registry with BlobStore and Registry handlers
- Mount /v2 endpoints in API with JWT auth
- Add ImportLocalImage to image manager for local pushes
- Add OCI cache path helpers to lib/paths
- Add comprehensive tests for push, caching, and layer deduplication

The registry uses go-containerregistry for protocol handling with custom
blob storage. Layer caching is automatic - subsequent pushes skip already
stored blobs. Manifest pushes trigger async conversion to ext4 disk format.

* fix: convert Docker v2 manifests to OCI format when pushing images

Images pushed from the local Docker daemon use Docker v2 manifest format,
but umoci (used for unpacking) only handles OCI format. This caused
'manifest data is not v1.Manifest' errors during image conversion.

Changes:
- Add blobStoreImage wrapper that transparently converts Docker v2 to OCI
- Convert manifest, config, and layer media types to OCI equivalents
- Compute correct digest for converted OCI manifests
- Add TestRegistryDockerV2ManifestConversion to verify the fix
- Update existing tests to verify images appear in ListImages after push
- Update README to clarify docker push limitations

* go.sum

* readme updates

* verify /dev/kvm access on startup

* remove old-style (pre go 1.17) build constraint syntax

* clean up test

* final cleanup

Author: rgarcia

cmd/api/api/registry_test.go

@@ -37,7 +37,7 @@ func TestGetVolume_ByName(t *testing.T) {
 
 	// Create a volume
 	createResp, err := svc.CreateVolume(ctx(), oapi.CreateVolumeRequestObject{
-		Body: &oapi.CreateVolumeRequest{
+		JSONBody: &oapi.CreateVolumeRequest{
 			Name:   "my-data",
 			SizeGb: 1,
 		},
@@ -62,7 +62,7 @@ func TestDeleteVolume_ByName(t *testing.T) {
 
 	// Create a volume
 	_, err := svc.CreateVolume(ctx(), oapi.CreateVolumeRequestObject{
-		Body: &oapi.CreateVolumeRequest{
+		JSONBody: &oapi.CreateVolumeRequest{
 			Name:   "to-delete",
 			SizeGb: 1,
 		},

cmd/api/api/volumes_test.go

@@ -108,6 +108,12 @@ func run() error {
 		logger.Warn("JWT_SECRET not configured - API authentication will fail")
 	}
 
+	// Verify KVM access (required for VM creation)
+	if err := checkKVMAccess(); err != nil {
+		return fmt.Errorf("KVM access check failed: %w\n\nEnsure:\n  1. KVM is enabled (check /dev/kvm exists)\n  2. User is in 'kvm' group: sudo usermod -aG kvm $USER\n  3. Log out and back in, or use: newgrp kvm", err)
+	}
+	logger.Info("KVM access verified")
+
 	// Validate log rotation config
 	var logMaxSize datasize.ByteSize
 	if err := logMaxSize.UnmarshalText([]byte(app.Config.LogMaxSize)); err != nil {
@@ -178,6 +184,16 @@ func run() error {
 		mw.JwtAuth(app.Config.JwtSecret),
 	).Get("/instances/{id}/exec", app.ApiService.ExecHandler)
 
+	// OCI Distribution registry endpoints for image push (outside OpenAPI spec)
+	r.Route("/v2", func(r chi.Router) {
+		r.Use(middleware.RequestID)
+		r.Use(middleware.RealIP)
+		r.Use(middleware.Logger)
+		r.Use(middleware.Recoverer)
+		r.Use(mw.JwtAuth(app.Config.JwtSecret))
+		r.Mount("/", app.Registry.Handler())
+	})
+
 	// Authenticated API endpoints
 	r.Group(func(r chi.Router) {
 		// Common middleware
@@ -323,3 +339,19 @@ func getRunningInstanceIDs(app *application) []string {
 	}
 	return running
 }
+
+// checkKVMAccess verifies KVM is available and the user has permission to use it
+func checkKVMAccess() error {
+	f, err := os.OpenFile("/dev/kvm", os.O_RDWR, 0)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return fmt.Errorf("/dev/kvm not found - KVM not enabled or not supported")
+		}
+		if os.IsPermission(err) {
+			return fmt.Errorf("permission denied accessing /dev/kvm - user not in 'kvm' group")
+		}
+		return fmt.Errorf("cannot access /dev/kvm: %w", err)
+	}
+	f.Close()
+	return nil
+}

cmd/api/main.go

@@ -1,4 +1,4 @@
-// +build wireinject
+//go:build wireinject
 
 package main
 
@@ -13,6 +13,7 @@ import (
 	"github.com/onkernel/hypeman/lib/instances"
 	"github.com/onkernel/hypeman/lib/network"
 	"github.com/onkernel/hypeman/lib/providers"
+	"github.com/onkernel/hypeman/lib/registry"
 	"github.com/onkernel/hypeman/lib/system"
 	"github.com/onkernel/hypeman/lib/volumes"
 )
@@ -27,6 +28,7 @@ type application struct {
 	NetworkManager  network.Manager
 	InstanceManager instances.Manager
 	VolumeManager   volumes.Manager
+	Registry        *registry.Registry
 	ApiService      *api.ApiService
 }
 
@@ -42,8 +44,8 @@ func initializeApp() (*application, func(), error) {
 		providers.ProvideNetworkManager,
 		providers.ProvideInstanceManager,
 		providers.ProvideVolumeManager,
+		providers.ProvideRegistry,
 		api.New,
 		wire.Struct(new(application), "*"),
 	))
 }
-

cmd/api/wire.go

@@ -71,6 +71,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/go-containerregistry v0.20.6 h1:cvWX87UxxLgaH76b4hIvya6Dzz9qHB31qAwjAohdSTU=
 github.com/google/go-containerregistry v0.20.6/go.mod h1:T0x8MuoAoKX/873bkeSfLD2FAkwCDf9/HZgsFJ02E2Y=
+github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE=
+github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -232,6 +234,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
 golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
+golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U=
+golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -256,6 +260,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
 golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE=
+golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
 gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=

cmd/api/wire_gen.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -24,6 +25,9 @@ const (
 type Manager interface {
 	ListImages(ctx context.Context) ([]Image, error)
 	CreateImage(ctx context.Context, req CreateImageRequest) (*Image, error)
+	// ImportLocalImage imports an image that was pushed to the local OCI cache.
+	// Unlike CreateImage, it does not resolve from a remote registry.
+	ImportLocalImage(ctx context.Context, repo, reference, digest string) (*Image, error)
 	GetImage(ctx context.Context, name string) (*Image, error)
 	DeleteImage(ctx context.Context, name string) error
 	RecoverInterruptedBuilds()
@@ -120,6 +124,46 @@ func (m *manager) CreateImage(ctx context.Context, req CreateImageRequest) (*Ima
 	return m.createAndQueueImage(ref)
 }
 
+// ImportLocalImage imports an image from the local OCI cache without resolving from a remote registry.
+// This is used for images that were pushed directly to the hypeman registry.
+func (m *manager) ImportLocalImage(ctx context.Context, repo, reference, digest string) (*Image, error) {
+	// Build the image reference string
+	var imageRef string
+	if strings.HasPrefix(reference, "sha256:") {
+		imageRef = repo + "@" + reference
+	} else {
+		imageRef = repo + ":" + reference
+	}
+
+	// Parse and normalize
+	normalized, err := ParseNormalizedRef(imageRef)
+	if err != nil {
+		return nil, fmt.Errorf("%w: %s", ErrInvalidName, err.Error())
+	}
+
+	// Create a ResolvedRef directly with the provided digest
+	ref := NewResolvedRef(normalized, digest)
+
+	m.createMu.Lock()
+	defer m.createMu.Unlock()
+
+	// Check if we already have this digest (deduplication)
+	if meta, err := readMetadata(m.paths, ref.Repository(), ref.DigestHex()); err == nil {
+		// We have this digest already
+		if meta.Status == StatusReady && ref.Tag() != "" {
+			createTagSymlink(m.paths, ref.Repository(), ref.Tag(), ref.DigestHex())
+		}
+		img := meta.toImage()
+		if meta.Status == StatusPending {
+			img.QueuePosition = m.queue.GetPosition(meta.Digest)
+		}
+		return img, nil
+	}
+
+	// Don't have this digest yet, queue the build
+	return m.createAndQueueImage(ref)
+}
+
 func (m *manager) createAndQueueImage(ref *ResolvedRef) (*Image, error) {
 	meta := &imageMetadata{
 		Name:      ref.String(),

go.sum

@@ -9,6 +9,9 @@
 //	    initrd/{arch}/latest -> {timestamp}
 //	    binaries/{version}/{arch}/cloud-hypervisor
 //	    oci-cache/
+//	      oci-layout
+//	      index.json
+//	      blobs/sha256/{digestHex}
 //	    builds/{ref}/
 //	  images/
 //	    {repository}/{digest}/
@@ -78,6 +81,26 @@ func (p *Paths) SystemOCICache() string {
 	return filepath.Join(p.dataDir, "system", "oci-cache")
 }
 
+// OCICacheBlobDir returns the path to the OCI cache blobs directory.
+func (p *Paths) OCICacheBlobDir() string {
+	return filepath.Join(p.SystemOCICache(), "blobs", "sha256")
+}
+
+// OCICacheBlob returns the path to a specific blob in the OCI cache.
+func (p *Paths) OCICacheBlob(digestHex string) string {
+	return filepath.Join(p.OCICacheBlobDir(), digestHex)
+}
+
+// OCICacheIndex returns the path to the OCI cache index.json.
+func (p *Paths) OCICacheIndex() string {
+	return filepath.Join(p.SystemOCICache(), "index.json")
+}
+
+// OCICacheLayout returns the path to the OCI cache oci-layout file.
+func (p *Paths) OCICacheLayout() string {
+	return filepath.Join(p.SystemOCICache(), "oci-layout")
+}
+
 // SystemBuild returns the path to a system build directory.
 func (p *Paths) SystemBuild(ref string) string {
 	return filepath.Join(p.dataDir, "system", "builds", ref)

lib/images/manager.go

@@ -13,6 +13,7 @@ import (
 	"github.com/onkernel/hypeman/lib/network"
 	hypemanotel "github.com/onkernel/hypeman/lib/otel"
 	"github.com/onkernel/hypeman/lib/paths"
+	"github.com/onkernel/hypeman/lib/registry"
 	"github.com/onkernel/hypeman/lib/system"
 	"github.com/onkernel/hypeman/lib/volumes"
 	"go.opentelemetry.io/otel"
@@ -113,3 +114,8 @@ func ProvideVolumeManager(p *paths.Paths, cfg *config.Config) (volumes.Manager,
 	meter := otel.GetMeterProvider().Meter("hypeman")
 	return volumes.NewManager(p, maxTotalVolumeStorage, meter), nil
 }
+
+// ProvideRegistry provides the OCI registry for image push
+func ProvideRegistry(p *paths.Paths, imageManager images.Manager) (*registry.Registry, error) {
+	return registry.New(p, imageManager)
+}