feat(registry): add OCI Distribution registry for image push

Implement OCI Distribution Spec compliant registry endpoints that accept
pushed images and trigger conversion to hypeman's disk format.

Changes:
- Add lib/registry with BlobStore and Registry handlers
- Mount /v2 endpoints in API with JWT auth
- Add ImportLocalImage to image manager for local pushes
- Add OCI cache path helpers to lib/paths
- Add comprehensive tests for push, caching, and layer deduplication

The registry uses go-containerregistry for protocol handling with custom
blob storage. Layer caching is automatic - subsequent pushes skip already
stored blobs. Manifest pushes trigger async conversion to ext4 disk format.

Author: rgarcia

cmd/api/api/registry_test.go

@@ -103,6 +103,16 @@ func run() error {
 		mw.JwtAuth(app.Config.JwtSecret),
 	).Get("/instances/{id}/exec", app.ApiService.ExecHandler)
 
+	// OCI Distribution registry endpoints for image push (outside OpenAPI spec)
+	r.Route("/v2", func(r chi.Router) {
+		r.Use(middleware.RequestID)
+		r.Use(middleware.RealIP)
+		r.Use(middleware.Logger)
+		r.Use(middleware.Recoverer)
+		r.Use(mw.JwtAuth(app.Config.JwtSecret))
+		r.Mount("/", app.Registry.Handler())
+	})
+
 	// Authenticated API endpoints
 	r.Group(func(r chi.Router) {
 		// Common middleware
@@ -225,4 +235,3 @@ func getRunningInstanceIDs(app *application) []string {
 	}
 	return running
 }
-

cmd/api/main.go

@@ -1,3 +1,4 @@
+//go:build wireinject
 // +build wireinject
 
 package main
@@ -13,6 +14,7 @@ import (
 	"github.com/onkernel/hypeman/lib/instances"
 	"github.com/onkernel/hypeman/lib/network"
 	"github.com/onkernel/hypeman/lib/providers"
+	"github.com/onkernel/hypeman/lib/registry"
 	"github.com/onkernel/hypeman/lib/system"
 	"github.com/onkernel/hypeman/lib/volumes"
 )
@@ -27,6 +29,7 @@ type application struct {
 	NetworkManager  network.Manager
 	InstanceManager instances.Manager
 	VolumeManager   volumes.Manager
+	Registry        *registry.Registry
 	ApiService      *api.ApiService
 }
 
@@ -42,8 +45,8 @@ func initializeApp() (*application, func(), error) {
 		providers.ProvideNetworkManager,
 		providers.ProvideInstanceManager,
 		providers.ProvideVolumeManager,
+		providers.ProvideRegistry,
 		api.New,
 		wire.Struct(new(application), "*"),
 	))
 }
-

cmd/api/wire.go

@@ -66,6 +66,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
 github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
 github.com/google/go-containerregistry v0.20.6 h1:cvWX87UxxLgaH76b4hIvya6Dzz9qHB31qAwjAohdSTU=
 github.com/google/go-containerregistry v0.20.6/go.mod h1:T0x8MuoAoKX/873bkeSfLD2FAkwCDf9/HZgsFJ02E2Y=
+github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE=
+github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -203,6 +205,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
 golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
+golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U=
+golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -227,6 +231,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
 golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE=
+golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
 gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=

cmd/api/wire_gen.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"sort"
+	"strings"
 	"sync"
 	"time"
 
@@ -23,6 +24,9 @@ const (
 type Manager interface {
 	ListImages(ctx context.Context) ([]Image, error)
 	CreateImage(ctx context.Context, req CreateImageRequest) (*Image, error)
+	// ImportLocalImage imports an image that was pushed to the local OCI cache.
+	// Unlike CreateImage, it does not resolve from a remote registry.
+	ImportLocalImage(ctx context.Context, repo, reference, digest string) (*Image, error)
 	GetImage(ctx context.Context, name string) (*Image, error)
 	DeleteImage(ctx context.Context, name string) error
 	RecoverInterruptedBuilds()
@@ -78,7 +82,7 @@ func (m *manager) CreateImage(ctx context.Context, req CreateImageRequest) (*Ima
 	// Add a 2-second timeout to ensure fast failure on rate limits or errors
 	resolveCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
 	defer cancel()
-	
+
 	ref, err := normalized.Resolve(resolveCtx, m.ociClient)
 	if err != nil {
 		return nil, fmt.Errorf("resolve manifest: %w", err)
@@ -107,6 +111,46 @@ func (m *manager) CreateImage(ctx context.Context, req CreateImageRequest) (*Ima
 	return m.createAndQueueImage(ref)
 }
 
+// ImportLocalImage imports an image from the local OCI cache without resolving from a remote registry.
+// This is used for images that were pushed directly to the hypeman registry.
+func (m *manager) ImportLocalImage(ctx context.Context, repo, reference, digest string) (*Image, error) {
+	// Build the image reference string
+	var imageRef string
+	if strings.HasPrefix(reference, "sha256:") {
+		imageRef = repo + "@" + reference
+	} else {
+		imageRef = repo + ":" + reference
+	}
+
+	// Parse and normalize
+	normalized, err := ParseNormalizedRef(imageRef)
+	if err != nil {
+		return nil, fmt.Errorf("%w: %s", ErrInvalidName, err.Error())
+	}
+
+	// Create a ResolvedRef directly with the provided digest
+	ref := NewResolvedRef(normalized, digest)
+
+	m.createMu.Lock()
+	defer m.createMu.Unlock()
+
+	// Check if we already have this digest (deduplication)
+	if meta, err := readMetadata(m.paths, ref.Repository(), ref.DigestHex()); err == nil {
+		// We have this digest already
+		if meta.Status == StatusReady && ref.Tag() != "" {
+			createTagSymlink(m.paths, ref.Repository(), ref.Tag(), ref.DigestHex())
+		}
+		img := meta.toImage()
+		if meta.Status == StatusPending {
+			img.QueuePosition = m.queue.GetPosition(meta.Digest)
+		}
+		return img, nil
+	}
+
+	// Don't have this digest yet, queue the build
+	return m.createAndQueueImage(ref)
+}
+
 func (m *manager) createAndQueueImage(ref *ResolvedRef) (*Image, error) {
 	meta := &imageMetadata{
 		Name:      ref.String(),

go.sum

@@ -9,6 +9,9 @@
 //	    initrd/{arch}/latest -> {timestamp}
 //	    binaries/{version}/{arch}/cloud-hypervisor
 //	    oci-cache/
+//	      oci-layout
+//	      index.json
+//	      blobs/sha256/{digestHex}
 //	    builds/{ref}/
 //	  images/
 //	    {repository}/{digest}/
@@ -72,6 +75,26 @@ func (p *Paths) SystemOCICache() string {
 	return filepath.Join(p.dataDir, "system", "oci-cache")
 }
 
+// OCICacheBlobDir returns the path to the OCI cache blobs directory.
+func (p *Paths) OCICacheBlobDir() string {
+	return filepath.Join(p.SystemOCICache(), "blobs", "sha256")
+}
+
+// OCICacheBlob returns the path to a specific blob in the OCI cache.
+func (p *Paths) OCICacheBlob(digestHex string) string {
+	return filepath.Join(p.OCICacheBlobDir(), digestHex)
+}
+
+// OCICacheIndex returns the path to the OCI cache index.json.
+func (p *Paths) OCICacheIndex() string {
+	return filepath.Join(p.SystemOCICache(), "index.json")
+}
+
+// OCICacheLayout returns the path to the OCI cache oci-layout file.
+func (p *Paths) OCICacheLayout() string {
+	return filepath.Join(p.SystemOCICache(), "oci-layout")
+}
+
 // SystemBuild returns the path to a system build directory.
 func (p *Paths) SystemBuild(ref string) string {
 	return filepath.Join(p.dataDir, "system", "builds", ref)

lib/images/manager.go

@@ -13,6 +13,7 @@ import (
 	"github.com/onkernel/hypeman/lib/logger"
 	"github.com/onkernel/hypeman/lib/network"
 	"github.com/onkernel/hypeman/lib/paths"
+	"github.com/onkernel/hypeman/lib/registry"
 	"github.com/onkernel/hypeman/lib/system"
 	"github.com/onkernel/hypeman/lib/volumes"
 )
@@ -68,3 +69,8 @@ func ProvideInstanceManager(p *paths.Paths, cfg *config.Config, imageManager ima
 func ProvideVolumeManager(p *paths.Paths) volumes.Manager {
 	return volumes.NewManager(p)
 }
+
+// ProvideRegistry provides the OCI registry for image push
+func ProvideRegistry(p *paths.Paths, imageManager images.Manager) (*registry.Registry, error) {
+	return registry.New(p, imageManager)
+}

lib/paths/paths.go

@@ -0,0 +1,151 @@
+# OCI Distribution Registry
+
+Implements an OCI Distribution Spec compliant registry that accepts pushed images and triggers conversion to hypeman's disk format.
+
+## Architecture
+
+```mermaid
+sequenceDiagram
+    participant Client as Docker Client
+    participant Registry as Hypeman Registry
+    participant BlobStore as Blob Store
+    participant ImageMgr as Image Manager
+
+    Client->>Registry: PUT /v2/.../blobs/{digest}
+    Registry->>BlobStore: Store blob
+    BlobStore-->>Registry: OK
+    Registry-->>Client: 201 Created
+
+    Client->>Registry: PUT /v2/.../manifests/{ref}
+    Registry->>BlobStore: Store manifest blob
+    Registry->>Registry: Update index.json
+    Registry-->>Client: 201 Created
+    
+    Registry--)ImageMgr: ImportLocalImage() (async)
+    ImageMgr->>ImageMgr: Queue conversion
+    ImageMgr->>ImageMgr: Unpack layers
+    ImageMgr->>ImageMgr: Create disk image
+```
+
+## How It Works
+
+### Push Flow
+
+1. **Version Check**: Client hits `GET /v2/` to verify registry compatibility
+2. **Blob Check**: Client does `HEAD /v2/{name}/blobs/{digest}` to check if layers exist
+3. **Blob Upload**: Missing blobs uploaded via `POST/PATCH/PUT` sequence
+4. **Manifest Upload**: Final `PUT /v2/{name}/manifests/{reference}` triggers conversion
+
+### Layer Caching
+
+Blobs are stored content-addressably in `system/oci-cache/blobs/sha256/`:
+
+```go
+// BlobStore.Stat() - Returns size if exists, ErrNotFound otherwise
+func (s *BlobStore) Stat(ctx context.Context, repo string, h v1.Hash) (int64, error) {
+    path := s.blobPath(h.String())
+    info, err := os.Stat(path)
+    if os.IsNotExist(err) {
+        return 0, ErrNotFound  // Client will upload
+    }
+    return info.Size(), nil    // Client skips upload
+}
+```
+
+When a client pushes:
+- First push: HEAD returns 404 → uploads all blobs
+- Second push: HEAD returns 200 with size → skips upload entirely
+
+### Manifest Handling
+
+go-containerregistry stores manifests in-memory, but we need them on disk for conversion. The registry intercepts manifest PUTs:
+
+```go
+// Read manifest body before passing to underlying handler
+body, _ := io.ReadAll(req.Body)
+
+// Store in blob store (for image manager to read later)
+r.storeManifestBlob(reference, body)
+
+// Reconstruct body for underlying handler
+req.Body = io.NopCloser(bytes.NewReader(body))
+r.handler.ServeHTTP(wrapper, req)
+
+// Trigger async conversion
+if wrapper.statusCode == http.StatusCreated {
+    go r.triggerConversion(repo, reference)
+}
+```
+
+### Conversion Trigger
+
+After a successful manifest push:
+
+1. Updates `index.json` with manifest entry (correct mediaType detected from content)
+2. Calls `ImageManager.ImportLocalImage()` to queue conversion
+3. Image manager reads from shared OCI cache and creates disk image
+
+## Files
+
+- **`blob_store.go`** - Filesystem-backed blob storage implementing `registry.BlobHandler`
+- **`registry.go`** - Registry handler wrapping go-containerregistry with manifest interception
+
+## Storage Layout
+
+```
+/var/lib/hypeman/system/oci-cache/
+  oci-layout           # {"imageLayoutVersion": "1.0.0"}
+  index.json           # Manifest index with annotations
+  blobs/sha256/
+    2d35eb...          # Layer blob (shared across all images)
+    706db5...          # Config blob
+    85f2b7...          # Manifest blob
+```
+
+## CLI Usage
+
+```bash
+# Push from local Docker daemon
+hypeman push myimage:latest
+
+# Push with custom target name
+hypeman push myimage:latest my-custom-name
+```
+
+## Authentication
+
+The registry endpoints use the same JWT authentication as the rest of the API. The CLI reads `HYPEMAN_API_KEY` or `HYPEMAN_BEARER_TOKEN` and passes it as a registry token.
+
+## Limitations
+
+- **Docker v2 manifests**: Images from local Docker daemon use Docker v2 manifest format which may need additional handling for conversion. Images pulled directly from registries (OCI format) work seamlessly.
+- **Digest references only**: Conversion is triggered only for digest references (`@sha256:...`), not tag references (`:latest`). The CLI automatically converts to digest references.
+
+## Design Decisions
+
+### Why wrap go-containerregistry/pkg/registry?
+
+**What:** Use the existing registry implementation from go-containerregistry with custom blob storage.
+
+**Why:**
+- Battle-tested OCI Distribution Spec compliance
+- Handles chunked uploads, content negotiation, error responses
+- We only need to customize storage, not protocol handling
+
+### Why store manifests separately?
+
+**What:** Intercept manifest PUT and store in blob store.
+
+**Why:**
+- go-containerregistry stores manifests in-memory by default
+- Our image manager needs to read manifests from disk
+- Enables content-addressable manifest storage consistent with layers
+
+### Why detect mediaType from manifest content?
+
+**What:** Parse manifest JSON to extract actual mediaType rather than hardcoding.
+
+**Why:**
+- Docker v2 (`application/vnd.docker.distribution.manifest.v2+json`) vs OCI (`application/vnd.oci.image.manifest.v1+json`)
+- Wrong mediaType in index.json causes "malicious manifest" errors during conversion
+- Content-based detection ensures compatibility with all sources