feat: add build system for source-to-image builds

Implements a secure build system that runs rootless BuildKit inside
ephemeral Cloud Hypervisor microVMs for multi-tenant isolation.

Components:
- lib/builds/types.go: Build, CreateBuildRequest, BuildConfig, etc.
- lib/builds/queue.go: In-memory build queue with concurrency control
- lib/builds/storage.go: Persistent storage for build metadata
- lib/builds/manager.go: Orchestrates builder VM lifecycle
- lib/builds/vsock_handler.go: Host-side vsock communication
- lib/builds/cache.go: Registry-based caching with tenant isolation
- lib/builds/templates/: Dockerfile generation for Node.js/Python
- lib/builds/builder_agent/: Guest binary for running BuildKit
- lib/builds/images/: Builder image Dockerfiles

API endpoints:
- POST /v1/builds: Submit build job
- GET /v1/builds: List builds
- GET /v1/builds/{id}: Get build details
- DELETE /v1/builds/{id}: Cancel build
- GET /v1/builds/{id}/logs: Stream logs (SSE)

Features:
- Rootless BuildKit for unprivileged builds
- Per-tenant cache isolation
- Build provenance for reproducibility
- Automatic Dockerfile generation from lockfiles
- Network isolation options (isolated/egress)

Author: hiroTamada

cmd/api/api/api.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"github.com/onkernel/hypeman/cmd/api/config"
+	"github.com/onkernel/hypeman/lib/builds"
 	"github.com/onkernel/hypeman/lib/devices"
 	"github.com/onkernel/hypeman/lib/images"
 	"github.com/onkernel/hypeman/lib/ingress"
@@ -20,6 +21,7 @@ type ApiService struct {
 	NetworkManager  network.Manager
 	DeviceManager   devices.Manager
 	IngressManager  ingress.Manager
+	BuildManager    builds.Manager
 }
 
 var _ oapi.StrictServerInterface = (*ApiService)(nil)
@@ -33,6 +35,7 @@ func New(
 	networkManager network.Manager,
 	deviceManager devices.Manager,
 	ingressManager ingress.Manager,
+	buildManager builds.Manager,
 ) *ApiService {
 	return &ApiService{
 		Config:          config,
@@ -42,5 +45,6 @@ func New(
 		NetworkManager:  networkManager,
 		DeviceManager:   deviceManager,
 		IngressManager:  ingressManager,
+		BuildManager:    buildManager,
 	}
 }

cmd/api/api/builds.go

@@ -0,0 +1,284 @@
+package api
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"io"
+	"strconv"
+
+	"github.com/onkernel/hypeman/lib/builds"
+	"github.com/onkernel/hypeman/lib/logger"
+	"github.com/onkernel/hypeman/lib/oapi"
+)
+
+// ListBuilds returns all builds
+func (s *ApiService) ListBuilds(ctx context.Context, request oapi.ListBuildsRequestObject) (oapi.ListBuildsResponseObject, error) {
+	log := logger.FromContext(ctx)
+
+	domainBuilds, err := s.BuildManager.ListBuilds(ctx)
+	if err != nil {
+		log.ErrorContext(ctx, "failed to list builds", "error", err)
+		return oapi.ListBuilds500JSONResponse{
+			Code:    "internal_error",
+			Message: "failed to list builds",
+		}, nil
+	}
+
+	oapiBuilds := make([]oapi.Build, len(domainBuilds))
+	for i, b := range domainBuilds {
+		oapiBuilds[i] = buildToOAPI(b)
+	}
+
+	return oapi.ListBuilds200JSONResponse(oapiBuilds), nil
+}
+
+// CreateBuild creates a new build job
+func (s *ApiService) CreateBuild(ctx context.Context, request oapi.CreateBuildRequestObject) (oapi.CreateBuildResponseObject, error) {
+	log := logger.FromContext(ctx)
+
+	// Parse multipart form fields
+	var sourceData []byte
+	var runtime string
+	var baseImageDigest, cacheScope, dockerfile string
+	var timeoutSeconds int
+
+	for {
+		part, err := request.Body.NextPart()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return oapi.CreateBuild400JSONResponse{
+				Code:    "invalid_request",
+				Message: "failed to parse multipart form",
+			}, nil
+		}
+
+		switch part.FormName() {
+		case "source":
+			sourceData, err = io.ReadAll(part)
+			if err != nil {
+				return oapi.CreateBuild400JSONResponse{
+					Code:    "invalid_source",
+					Message: "failed to read source data",
+				}, nil
+			}
+		case "runtime":
+			var buf bytes.Buffer
+			io.Copy(&buf, part)
+			runtime = buf.String()
+		case "base_image_digest":
+			var buf bytes.Buffer
+			io.Copy(&buf, part)
+			baseImageDigest = buf.String()
+		case "cache_scope":
+			var buf bytes.Buffer
+			io.Copy(&buf, part)
+			cacheScope = buf.String()
+		case "dockerfile":
+			var buf bytes.Buffer
+			io.Copy(&buf, part)
+			dockerfile = buf.String()
+		case "timeout_seconds":
+			var buf bytes.Buffer
+			io.Copy(&buf, part)
+			if v, err := strconv.Atoi(buf.String()); err == nil {
+				timeoutSeconds = v
+			}
+		}
+		part.Close()
+	}
+
+	if runtime == "" {
+		return oapi.CreateBuild400JSONResponse{
+			Code:    "invalid_request",
+			Message: "runtime is required",
+		}, nil
+	}
+
+	if len(sourceData) == 0 {
+		return oapi.CreateBuild400JSONResponse{
+			Code:    "invalid_request",
+			Message: "source is required",
+		}, nil
+	}
+
+	// Build domain request
+	domainReq := builds.CreateBuildRequest{
+		Runtime:         runtime,
+		BaseImageDigest: baseImageDigest,
+		CacheScope:      cacheScope,
+		Dockerfile:      dockerfile,
+	}
+
+	// Apply timeout if provided
+	if timeoutSeconds > 0 {
+		domainReq.BuildPolicy = &builds.BuildPolicy{
+			TimeoutSeconds: timeoutSeconds,
+		}
+	}
+
+	build, err := s.BuildManager.CreateBuild(ctx, domainReq, sourceData)
+	if err != nil {
+		switch {
+		case errors.Is(err, builds.ErrInvalidRuntime):
+			return oapi.CreateBuild400JSONResponse{
+				Code:    "invalid_runtime",
+				Message: err.Error(),
+			}, nil
+		case errors.Is(err, builds.ErrInvalidSource):
+			return oapi.CreateBuild400JSONResponse{
+				Code:    "invalid_source",
+				Message: err.Error(),
+			}, nil
+		default:
+			log.ErrorContext(ctx, "failed to create build", "error", err)
+			return oapi.CreateBuild500JSONResponse{
+				Code:    "internal_error",
+				Message: "failed to create build",
+			}, nil
+		}
+	}
+
+	return oapi.CreateBuild202JSONResponse(buildToOAPI(build)), nil
+}
+
+// GetBuild gets build details
+func (s *ApiService) GetBuild(ctx context.Context, request oapi.GetBuildRequestObject) (oapi.GetBuildResponseObject, error) {
+	log := logger.FromContext(ctx)
+
+	build, err := s.BuildManager.GetBuild(ctx, request.Id)
+	if err != nil {
+		if errors.Is(err, builds.ErrNotFound) {
+			return oapi.GetBuild404JSONResponse{
+				Code:    "not_found",
+				Message: "build not found",
+			}, nil
+		}
+		log.ErrorContext(ctx, "failed to get build", "error", err, "id", request.Id)
+		return oapi.GetBuild500JSONResponse{
+			Code:    "internal_error",
+			Message: "failed to get build",
+		}, nil
+	}
+
+	return oapi.GetBuild200JSONResponse(buildToOAPI(build)), nil
+}
+
+// CancelBuild cancels a build
+func (s *ApiService) CancelBuild(ctx context.Context, request oapi.CancelBuildRequestObject) (oapi.CancelBuildResponseObject, error) {
+	log := logger.FromContext(ctx)
+
+	err := s.BuildManager.CancelBuild(ctx, request.Id)
+	if err != nil {
+		switch {
+		case errors.Is(err, builds.ErrNotFound):
+			return oapi.CancelBuild404JSONResponse{
+				Code:    "not_found",
+				Message: "build not found",
+			}, nil
+		case errors.Is(err, builds.ErrBuildInProgress):
+			return oapi.CancelBuild409JSONResponse{
+				Code:    "conflict",
+				Message: "build already in progress",
+			}, nil
+		default:
+			log.ErrorContext(ctx, "failed to cancel build", "error", err, "id", request.Id)
+			return oapi.CancelBuild500JSONResponse{
+				Code:    "internal_error",
+				Message: "failed to cancel build",
+			}, nil
+		}
+	}
+
+	return oapi.CancelBuild204Response{}, nil
+}
+
+// GetBuildLogs streams build logs
+func (s *ApiService) GetBuildLogs(ctx context.Context, request oapi.GetBuildLogsRequestObject) (oapi.GetBuildLogsResponseObject, error) {
+	log := logger.FromContext(ctx)
+
+	logs, err := s.BuildManager.GetBuildLogs(ctx, request.Id)
+	if err != nil {
+		if errors.Is(err, builds.ErrNotFound) {
+			return oapi.GetBuildLogs404JSONResponse{
+				Code:    "not_found",
+				Message: "build not found",
+			}, nil
+		}
+		log.ErrorContext(ctx, "failed to get build logs", "error", err, "id", request.Id)
+		return oapi.GetBuildLogs500JSONResponse{
+			Code:    "internal_error",
+			Message: "failed to get build logs",
+		}, nil
+	}
+
+	// Return logs as SSE
+	// For simplicity, return all logs at once
+	// TODO: Implement proper SSE streaming with follow support
+	return oapi.GetBuildLogs200TexteventStreamResponse{
+		Body:          stringReader(string(logs)),
+		ContentLength: int64(len(logs)),
+	}, nil
+}
+
+// buildToOAPI converts a domain Build to OAPI Build
+func buildToOAPI(b *builds.Build) oapi.Build {
+	oapiBuild := oapi.Build{
+		Id:            b.ID,
+		Status:        oapi.BuildStatus(b.Status),
+		Runtime:       b.Runtime,
+		QueuePosition: b.QueuePosition,
+		ImageDigest:   b.ImageDigest,
+		ImageRef:      b.ImageRef,
+		Error:         b.Error,
+		CreatedAt:     b.CreatedAt,
+		StartedAt:     b.StartedAt,
+		CompletedAt:   b.CompletedAt,
+		DurationMs:    b.DurationMS,
+	}
+
+	if b.Provenance != nil {
+		oapiBuild.Provenance = &oapi.BuildProvenance{
+			BaseImageDigest:  &b.Provenance.BaseImageDigest,
+			SourceHash:       &b.Provenance.SourceHash,
+			ToolchainVersion: &b.Provenance.ToolchainVersion,
+			BuildkitVersion:  &b.Provenance.BuildkitVersion,
+			Timestamp:        &b.Provenance.Timestamp,
+		}
+		if len(b.Provenance.LockfileHashes) > 0 {
+			oapiBuild.Provenance.LockfileHashes = &b.Provenance.LockfileHashes
+		}
+	}
+
+	return oapiBuild
+}
+
+// deref safely dereferences a pointer, returning empty string if nil
+func deref(s *string) string {
+	if s == nil {
+		return ""
+	}
+	return *s
+}
+
+// stringReader wraps a string as an io.Reader
+type stringReaderImpl struct {
+	s string
+	i int
+}
+
+func stringReader(s string) io.Reader {
+	return &stringReaderImpl{s: s}
+}
+
+func (r *stringReaderImpl) Read(p []byte) (n int, err error) {
+	if r.i >= len(r.s) {
+		return 0, io.EOF
+	}
+	n = copy(p, r.s[r.i:])
+	r.i += n
+	return n, nil
+}
+

cmd/api/config/config.go

@@ -101,6 +101,12 @@ type Config struct {
 
 	// Cloudflare configuration (if AcmeDnsProvider=cloudflare)
 	CloudflareApiToken string // Cloudflare API token
+
+	// Build system configuration
+	MaxConcurrentSourceBuilds int    // Max concurrent source-to-image builds
+	BuilderImage              string // OCI image for builder VMs
+	RegistryURL               string // URL of registry for built images
+	BuildTimeout              int    // Default build timeout in seconds
 }
 
 // Load loads configuration from environment variables
@@ -163,6 +169,12 @@ func Load() *Config {
 
 		// Cloudflare configuration
 		CloudflareApiToken: getEnv("CLOUDFLARE_API_TOKEN", ""),
+
+		// Build system configuration
+		MaxConcurrentSourceBuilds: getEnvInt("MAX_CONCURRENT_SOURCE_BUILDS", 2),
+		BuilderImage:              getEnv("BUILDER_IMAGE", "hypeman/builder:latest"),
+		RegistryURL:               getEnv("REGISTRY_URL", "localhost:8080"),
+		BuildTimeout:              getEnvInt("BUILD_TIMEOUT", 600),
 	}
 
 	return cfg

cmd/api/wire.go

@@ -9,6 +9,7 @@ import (
 	"github.com/google/wire"
 	"github.com/onkernel/hypeman/cmd/api/api"
 	"github.com/onkernel/hypeman/cmd/api/config"
+	"github.com/onkernel/hypeman/lib/builds"
 	"github.com/onkernel/hypeman/lib/devices"
 	"github.com/onkernel/hypeman/lib/images"
 	"github.com/onkernel/hypeman/lib/ingress"
@@ -32,6 +33,7 @@ type application struct {
 	InstanceManager instances.Manager
 	VolumeManager   volumes.Manager
 	IngressManager  ingress.Manager
+	BuildManager    builds.Manager
 	Registry        *registry.Registry
 	ApiService      *api.ApiService
 }
@@ -50,6 +52,7 @@ func initializeApp() (*application, func(), error) {
 		providers.ProvideInstanceManager,
 		providers.ProvideVolumeManager,
 		providers.ProvideIngressManager,
+		providers.ProvideBuildManager,
 		providers.ProvideRegistry,
 		api.New,
 		wire.Struct(new(application), "*"),