Image manager (#3)

* Export images to rootfs with docker

* docker client dependency injection

* Fix permissions during extraction

* Don't skip docker in tests

* extraction test passing but seems too complicated

* Use umoci for image to rootfs conversion

* Update docs

* Update README

* Move OCI cache dir

* Tweak disk settings

* Async API

* Simplify async API: remove sse for now

* Avoid rate limit in CI

* Add API level image test

* Check queuing works

* Test failure on invalid tag

* Fix image filesystem layout

* image name validation

* Update README

* Clean up error

* Use erofs

* API layer depends on domain

cmd/api/api (handlers)
    ↓ depends on
lib/images, lib/instances, lib/volumes (domain)
    ↓ independent of
lib/oapi (API types)

* Idempotency and race conditions

* Add erofs to ci

* OCI client internal to image manager

* Disambiguate digests and tags

* WIP: figuring out error handling registry errors

* Switch to more lightweight library for interacting with registry

* Handle 404

* Use docker auth config file

* Add timeout on resolve

* Update README

* Poll by digest

* Tweak readme

* caching test

* More comprehensive basic test

* 404 error mapping

Author: sjmiller609

.air.toml

@@ -5,7 +5,7 @@ tmp_dir = "tmp"
 [build]
   args_bin = []
   bin = "./tmp/main"
-  cmd = "go build -o ./tmp/main ./cmd/api"
+  cmd = "go build -tags containers_image_openpgp -o ./tmp/main ./cmd/api"
   delay = 1000
   exclude_dir = ["assets", "tmp", "vendor", "testdata", "bin", "scripts", "data", "kernel"]
   exclude_file = []

.github/workflows/test.yml

@@ -15,7 +15,18 @@ jobs:
           go-version: '1.25'
 
       - name: Install dependencies
-        run: go mod download
+        run: |
+          set -xe
+          sudo apt-get install -y erofs-utils
+          go mod download
+      
+      # Avoids rate limits when running the tests
+      # Tests includes pulling, then converting to disk images
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Run tests
         run: make test

Makefile

@@ -43,15 +43,15 @@ generate-all: oapi-generate generate-wire
 
 # Build the binary
 build: | $(BIN_DIR)
-	go build -o $(BIN_DIR)/hypeman ./cmd/api
+	go build -tags containers_image_openpgp -o $(BIN_DIR)/hypeman ./cmd/api
 
 # Run in development mode with hot reload
 dev: $(AIR)
 	$(AIR) -c .air.toml
 
 # Run tests
 test:
-	go test -v -timeout 30s ./...
+	go test -tags containers_image_openpgp -v -timeout 30s ./...
 
 # Clean generated files and binaries
 clean:

README.md

@@ -8,26 +8,40 @@ Run containerized workloads in VMs, powered by [Cloud Hypervisor](https://github
 
 ### Prerequisites
 
-**Cloud Hypervisor** - [Installation guide](https://www.cloudhypervisor.org/docs/prologue/quick-start/#use-pre-built-binaries)
-```bash
-cloud-hypervisor --version  # Verify
-ch-remote --version
-```
+**Go 1.25.4+**, **Cloud Hypervisor**, **KVM**, **erofs-utils**
 
-**containerd** - [Installation guide](https://github.com/containerd/containerd/blob/main/docs/getting-started.md)
 ```bash
-containerd --version  # Verify
+cloud-hypervisor --version
+mkfs.erofs --version
 ```
 
-**Go 1.25.4+** and **KVM**
-
 ### Configuration
 
+#### Environment variables
+
 ```bash
 cp .env.example .env
 # Edit .env and set JWT_SECRET
 ```
 
+#### Data directory
+
+Hypeman stores data in a configurable directory. Configure permissions for this directory.
+
+```bash
+sudo mkdir /var/lib/hypeman
+sudo chown $USER:$USER /var/lib/hypeman
+```
+
+#### Dockerhub login
+
+Requires Docker Hub authentication to avoid rate limits when running the tests:
+```bash
+docker login
+```
+
+Docker itself isn't required to be installed. `~/.docker/config.json` is a standard used for handling registry authentication.
+
 ### Build
 
 ```bash

cmd/api/api/api_test.go

@@ -16,9 +16,14 @@ func newTestService(t *testing.T) *ApiService {
 		DataDir: t.TempDir(),
 	}
 
+	imageMgr, err := images.NewManager(cfg.DataDir, 1)
+	if err != nil {
+		t.Fatalf("failed to create image manager: %v", err)
+	}
+
 	return &ApiService{
 		Config:          cfg,
-		ImageManager:    images.NewManager(cfg.DataDir),
+		ImageManager:    imageMgr,
 		InstanceManager: instances.NewManager(cfg.DataDir),
 		VolumeManager:   volumes.NewManager(cfg.DataDir),
 	}
@@ -27,4 +32,3 @@ func newTestService(t *testing.T) *ApiService {
 func ctx() context.Context {
 	return context.Background()
 }
-

cmd/api/api/images.go

@@ -9,32 +9,45 @@ import (
 	"github.com/onkernel/hypeman/lib/oapi"
 )
 
-// ListImages lists all images
 func (s *ApiService) ListImages(ctx context.Context, request oapi.ListImagesRequestObject) (oapi.ListImagesResponseObject, error) {
 	log := logger.FromContext(ctx)
 
-	imgs, err := s.ImageManager.ListImages(ctx)
+	domainImages, err := s.ImageManager.ListImages(ctx)
 	if err != nil {
 		log.Error("failed to list images", "error", err)
 		return oapi.ListImages500JSONResponse{
 			Code:    "internal_error",
 			Message: "failed to list images",
 		}, nil
 	}
-	return oapi.ListImages200JSONResponse(imgs), nil
+	
+	oapiImages := make([]oapi.Image, len(domainImages))
+	for i, img := range domainImages {
+		oapiImages[i] = imageToOAPI(img)
+	}
+	
+	return oapi.ListImages200JSONResponse(oapiImages), nil
 }
 
-// CreateImage creates a new image from an OCI reference
 func (s *ApiService) CreateImage(ctx context.Context, request oapi.CreateImageRequestObject) (oapi.CreateImageResponseObject, error) {
 	log := logger.FromContext(ctx)
 
-	img, err := s.ImageManager.CreateImage(ctx, *request.Body)
+	domainReq := images.CreateImageRequest{
+		Name: request.Body.Name,
+	}
+
+	img, err := s.ImageManager.CreateImage(ctx, domainReq)
 	if err != nil {
 		switch {
-		case errors.Is(err, images.ErrAlreadyExists):
+		case errors.Is(err, images.ErrInvalidName):
 			return oapi.CreateImage400JSONResponse{
-				Code:    "already_exists",
-				Message: "image already exists",
+				Code:    "invalid_name",
+				Message: err.Error(),
+			}, nil
+		case errors.Is(err, images.ErrNotFound):
+			return oapi.CreateImage404JSONResponse{
+				Code:    "not_found",
+				Message: "image not found",
 			}, nil
 		default:
 			log.Error("failed to create image", "error", err, "name", request.Body.Name)
@@ -44,46 +57,44 @@ func (s *ApiService) CreateImage(ctx context.Context, request oapi.CreateImageRe
 			}, nil
 		}
 	}
-	return oapi.CreateImage201JSONResponse(*img), nil
+	return oapi.CreateImage202JSONResponse(imageToOAPI(*img)), nil
 }
 
-// GetImage gets image details
 func (s *ApiService) GetImage(ctx context.Context, request oapi.GetImageRequestObject) (oapi.GetImageResponseObject, error) {
 	log := logger.FromContext(ctx)
 
-	img, err := s.ImageManager.GetImage(ctx, request.Id)
+	img, err := s.ImageManager.GetImage(ctx, request.Name)
 	if err != nil {
 		switch {
-		case errors.Is(err, images.ErrNotFound):
+		case errors.Is(err, images.ErrInvalidName), errors.Is(err, images.ErrNotFound):
 			return oapi.GetImage404JSONResponse{
 				Code:    "not_found",
 				Message: "image not found",
 			}, nil
 		default:
-			log.Error("failed to get image", "error", err, "id", request.Id)
+			log.Error("failed to get image", "error", err, "name", request.Name)
 			return oapi.GetImage500JSONResponse{
 				Code:    "internal_error",
 				Message: "failed to get image",
 			}, nil
 		}
 	}
-	return oapi.GetImage200JSONResponse(*img), nil
+	return oapi.GetImage200JSONResponse(imageToOAPI(*img)), nil
 }
 
-// DeleteImage deletes an image
 func (s *ApiService) DeleteImage(ctx context.Context, request oapi.DeleteImageRequestObject) (oapi.DeleteImageResponseObject, error) {
 	log := logger.FromContext(ctx)
 
-	err := s.ImageManager.DeleteImage(ctx, request.Id)
+	err := s.ImageManager.DeleteImage(ctx, request.Name)
 	if err != nil {
 		switch {
-		case errors.Is(err, images.ErrNotFound):
+		case errors.Is(err, images.ErrInvalidName), errors.Is(err, images.ErrNotFound):
 			return oapi.DeleteImage404JSONResponse{
 				Code:    "not_found",
 				Message: "image not found",
 			}, nil
 		default:
-			log.Error("failed to delete image", "error", err, "id", request.Id)
+			log.Error("failed to delete image", "error", err, "name", request.Name)
 			return oapi.DeleteImage500JSONResponse{
 				Code:    "internal_error",
 				Message: "failed to delete image",
@@ -93,3 +104,29 @@ func (s *ApiService) DeleteImage(ctx context.Context, request oapi.DeleteImageRe
 	return oapi.DeleteImage204Response{}, nil
 }
 
+func imageToOAPI(img images.Image) oapi.Image {
+	oapiImg := oapi.Image{
+		Name:          img.Name,
+		Digest:        img.Digest,
+		Status:        oapi.ImageStatus(img.Status),
+		QueuePosition: img.QueuePosition,
+		Error:         img.Error,
+		SizeBytes:     img.SizeBytes,
+		CreatedAt:     img.CreatedAt,
+	}
+
+	if len(img.Entrypoint) > 0 {
+		oapiImg.Entrypoint = &img.Entrypoint
+	}
+	if len(img.Cmd) > 0 {
+		oapiImg.Cmd = &img.Cmd
+	}
+	if len(img.Env) > 0 {
+		oapiImg.Env = &img.Env
+	}
+	if img.WorkingDir != "" {
+		oapiImg.WorkingDir = &img.WorkingDir
+	}
+
+	return oapiImg
+}