Ingress (#24)

* Ingress feature

* fixes

- Fix ID type
- latest envoy
- configurable if envoy stops or not
- dedupe instance lookup by id, name, id prefix to inside instance
  manager

* Host port configurable by ingress resource

* Auto download envoy when missing

* envoy starting

* Avoid shared memory conflicts between envoys on same host

* Use hostname for routing

* Update default config

* review tweaks

* context logger

* Config validation and OTEL

* Fix otel

* Envoy provides stats to otel, not per-request tracing

* Update stainless

* Update stainless again

* naming things is hard

* Address pr review comments

* Use file-based simple xDS instead of static config+reload

* Fix config validation for dynamic resources, add tests, fix import cycle

* Otel cluster should be static

* e2e test should actually do a http request

* Give ingress 5s to be ready

* Up to 30s

* Delete comment

* Fix dynamic config update race condition

---------

Co-authored-by: Rafael Garcia <[email protected]>

Author: sjmiller609

.gitignore

@@ -20,3 +20,6 @@ lib/vmm/binaries/cloud-hypervisor/*/*/cloud-hypervisor
 cloud-hypervisor
 cloud-hypervisor/**
 lib/system/exec_agent/exec-agent
+
+# Envoy binaries
+lib/ingress/binaries/envoy/*/*/envoy

Makefile

@@ -1,5 +1,5 @@
 SHELL := /bin/bash
-.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build test install-tools gen-jwt download-ch-binaries download-ch-spec ensure-ch-binaries
+.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build test install-tools gen-jwt download-ch-binaries download-ch-spec ensure-ch-binaries download-envoy-binaries ensure-envoy-binaries
 
 # Directory where local binaries will be installed
 BIN_DIR ?= $(CURDIR)/bin
@@ -49,6 +49,19 @@ download-ch-binaries:
 	@chmod +x lib/vmm/binaries/cloud-hypervisor/v*/*/cloud-hypervisor
 	@echo "Binaries downloaded successfully"
 
+# Download Envoy binaries
+download-envoy-binaries:
+	@echo "Downloading Envoy binaries..."
+	@mkdir -p lib/ingress/binaries/envoy/v1.36/{x86_64,aarch64}
+	@echo "Downloading Envoy v1.36.3 for x86_64..."
+	@curl -L -o lib/ingress/binaries/envoy/v1.36/x86_64/envoy \
+		https://github.com/envoyproxy/envoy/releases/download/v1.36.3/envoy-1.36.3-linux-x86_64
+	@echo "Downloading Envoy v1.36.3 for aarch64..."
+	@curl -L -o lib/ingress/binaries/envoy/v1.36/aarch64/envoy \
+		https://github.com/envoyproxy/envoy/releases/download/v1.36.3/envoy-1.36.3-linux-aarch_64
+	@chmod +x lib/ingress/binaries/envoy/v1.36/*/envoy
+	@echo "Envoy binaries downloaded successfully"
+
 # Download Cloud Hypervisor API spec
 download-ch-spec:
 	@echo "Downloading Cloud Hypervisor API spec..."
@@ -86,21 +99,29 @@ generate-grpc:
 # Generate all code
 generate-all: oapi-generate generate-vmm-client generate-wire generate-grpc
 
-# Check if binaries exist, download if missing
+# Check if CH binaries exist, download if missing
 .PHONY: ensure-ch-binaries
 ensure-ch-binaries:
 	@if [ ! -f lib/vmm/binaries/cloud-hypervisor/v48.0/x86_64/cloud-hypervisor ]; then \
 		echo "Cloud Hypervisor binaries not found, downloading..."; \
 		$(MAKE) download-ch-binaries; \
 	fi
 
+# Check if Envoy binaries exist, download if missing
+.PHONY: ensure-envoy-binaries
+ensure-envoy-binaries:
+	@if [ ! -f lib/ingress/binaries/envoy/v1.36/x86_64/envoy ]; then \
+		echo "Envoy binaries not found, downloading..."; \
+		$(MAKE) download-envoy-binaries; \
+	fi
+
 # Build exec-agent (guest binary) into its own directory for embedding
 lib/system/exec_agent/exec-agent: lib/system/exec_agent/main.go
 	@echo "Building exec-agent..."
 	cd lib/system/exec_agent && CGO_ENABLED=0 go build -ldflags="-s -w" -o exec-agent .
 
 # Build the binary
-build: ensure-ch-binaries lib/system/exec_agent/exec-agent | $(BIN_DIR)
+build: ensure-ch-binaries ensure-envoy-binaries lib/system/exec_agent/exec-agent | $(BIN_DIR)
 	go build -tags containers_image_openpgp -o $(BIN_DIR)/hypeman ./cmd/api
 
 # Build exec CLI
@@ -118,7 +139,7 @@ dev: $(AIR)
 # Compile test binaries and grant network capabilities (runs as user, not root)
 # Usage: make test                              - runs all tests
 #        make test TEST=TestCreateInstanceWithNetwork  - runs specific test
-test: ensure-ch-binaries lib/system/exec_agent/exec-agent
+test: ensure-ch-binaries ensure-envoy-binaries lib/system/exec_agent/exec-agent
 	@echo "Building test binaries..."
 	@mkdir -p $(BIN_DIR)/tests
 	@for pkg in $$(go list -tags containers_image_openpgp ./...); do \

README.md

@@ -58,6 +58,24 @@ getcap ./bin/hypeman
 
 **Note:** These capabilities must be reapplied after each rebuild. For production deployments, set capabilities on the installed binary. For local testing, this is handled automatically in `make test`.
 
+**File Descriptor Limits:**
+
+Envoy (used for ingress) requires a higher file descriptor limit than the default on some systems (root defaults to 1024 on many systems). If you see "Too many open files" errors, increase the limit:
+
+```bash
+# Check current limit (also check with: sudo bash -c 'ulimit -n')
+ulimit -n
+
+# Increase temporarily (current session)
+ulimit -n 65536
+
+# For persistent changes, add to /etc/security/limits.conf:
+*  soft  nofile  65536
+*  hard  nofile  65536
+root  soft  nofile  65536
+root  hard  nofile  65536
+```
+
 ### Configuration
 
 #### Environment variables
@@ -81,6 +99,10 @@ Hypeman can be configured using the following environment variables:
 | `OTEL_SERVICE_INSTANCE_ID` | Instance ID for telemetry (differentiates multiple servers) | hostname |
 | `LOG_LEVEL` | Default log level (debug, info, warn, error) | `info` |
 | `LOG_LEVEL_<SUBSYSTEM>` | Per-subsystem log level (API, IMAGES, INSTANCES, NETWORK, VOLUMES, VMM, SYSTEM, EXEC) | inherits default |
+| `ENVOY_LISTEN_ADDRESS` | Address for Envoy ingress listeners | `0.0.0.0` |
+| `ENVOY_ADMIN_ADDRESS` | Address for Envoy admin API | `127.0.0.1` |
+| `ENVOY_ADMIN_PORT` | Port for Envoy admin API | `9901` |
+| `ENVOY_STOP_ON_SHUTDOWN` | Stop Envoy when hypeman shuts down (if false, Envoy continues running) | `false` |
 
 **Important: Subnet Configuration**
 

cmd/api/api/api.go

@@ -3,6 +3,7 @@ package api
 import (
 	"github.com/onkernel/hypeman/cmd/api/config"
 	"github.com/onkernel/hypeman/lib/images"
+	"github.com/onkernel/hypeman/lib/ingress"
 	"github.com/onkernel/hypeman/lib/instances"
 	"github.com/onkernel/hypeman/lib/network"
 	"github.com/onkernel/hypeman/lib/oapi"
@@ -16,6 +17,7 @@ type ApiService struct {
 	InstanceManager instances.Manager
 	VolumeManager   volumes.Manager
 	NetworkManager  network.Manager
+	IngressManager  ingress.Manager
 }
 
 var _ oapi.StrictServerInterface = (*ApiService)(nil)
@@ -27,13 +29,14 @@ func New(
 	instanceManager instances.Manager,
 	volumeManager volumes.Manager,
 	networkManager network.Manager,
+	ingressManager ingress.Manager,
 ) *ApiService {
 	return &ApiService{
 		Config:          config,
 		ImageManager:    imageManager,
 		InstanceManager: instanceManager,
 		VolumeManager:   volumeManager,
 		NetworkManager:  networkManager,
+		IngressManager:  ingressManager,
 	}
 }
-

cmd/api/api/ingress.go

@@ -0,0 +1,162 @@
+package api
+
+import (
+	"context"
+	"errors"
+
+	"github.com/onkernel/hypeman/lib/ingress"
+	"github.com/onkernel/hypeman/lib/logger"
+	"github.com/onkernel/hypeman/lib/oapi"
+)
+
+// ListIngresses lists all ingress resources
+func (s *ApiService) ListIngresses(ctx context.Context, request oapi.ListIngressesRequestObject) (oapi.ListIngressesResponseObject, error) {
+	log := logger.FromContext(ctx)
+
+	ingresses, err := s.IngressManager.List(ctx)
+	if err != nil {
+		log.ErrorContext(ctx, "failed to list ingresses", "error", err)
+		return oapi.ListIngresses500JSONResponse{
+			Code:    "internal_error",
+			Message: "failed to list ingresses",
+		}, nil
+	}
+
+	oapiIngresses := make([]oapi.Ingress, len(ingresses))
+	for i, ing := range ingresses {
+		oapiIngresses[i] = ingressToOAPI(ing)
+	}
+
+	return oapi.ListIngresses200JSONResponse(oapiIngresses), nil
+}
+
+// CreateIngress creates a new ingress resource
+func (s *ApiService) CreateIngress(ctx context.Context, request oapi.CreateIngressRequestObject) (oapi.CreateIngressResponseObject, error) {
+	log := logger.FromContext(ctx)
+
+	// Convert OAPI request to domain request
+	domainReq := ingress.CreateIngressRequest{
+		Name:  request.Body.Name,
+		Rules: make([]ingress.IngressRule, len(request.Body.Rules)),
+	}
+
+	for i, rule := range request.Body.Rules {
+		matchPort := 80
+		if rule.Match.Port != nil {
+			matchPort = *rule.Match.Port
+		}
+		domainReq.Rules[i] = ingress.IngressRule{
+			Match: ingress.IngressMatch{
+				Hostname: rule.Match.Hostname,
+				Port:     matchPort,
+			},
+			Target: ingress.IngressTarget{
+				Instance: rule.Target.Instance,
+				Port:     rule.Target.Port,
+			},
+		}
+	}
+
+	ing, err := s.IngressManager.Create(ctx, domainReq)
+	if err != nil {
+		switch {
+		case errors.Is(err, ingress.ErrInvalidRequest):
+			return oapi.CreateIngress400JSONResponse{
+				Code:    "bad_request",
+				Message: err.Error(),
+			}, nil
+		case errors.Is(err, ingress.ErrAlreadyExists):
+			return oapi.CreateIngress409JSONResponse{
+				Code:    "already_exists",
+				Message: err.Error(),
+			}, nil
+		case errors.Is(err, ingress.ErrHostnameInUse):
+			return oapi.CreateIngress409JSONResponse{
+				Code:    "hostname_in_use",
+				Message: err.Error(),
+			}, nil
+		case errors.Is(err, ingress.ErrInstanceNotFound):
+			return oapi.CreateIngress400JSONResponse{
+				Code:    "instance_not_found",
+				Message: err.Error(),
+			}, nil
+		default:
+			log.ErrorContext(ctx, "failed to create ingress", "error", err, "name", request.Body.Name)
+			return oapi.CreateIngress500JSONResponse{
+				Code:    "internal_error",
+				Message: "failed to create ingress",
+			}, nil
+		}
+	}
+
+	return oapi.CreateIngress201JSONResponse(ingressToOAPI(*ing)), nil
+}
+
+// GetIngress gets ingress details by ID or name
+func (s *ApiService) GetIngress(ctx context.Context, request oapi.GetIngressRequestObject) (oapi.GetIngressResponseObject, error) {
+	log := logger.FromContext(ctx)
+
+	ing, err := s.IngressManager.Get(ctx, request.Id)
+	if err != nil {
+		if errors.Is(err, ingress.ErrNotFound) {
+			return oapi.GetIngress404JSONResponse{
+				Code:    "not_found",
+				Message: "ingress not found",
+			}, nil
+		}
+		log.ErrorContext(ctx, "failed to get ingress", "error", err, "id", request.Id)
+		return oapi.GetIngress500JSONResponse{
+			Code:    "internal_error",
+			Message: "failed to get ingress",
+		}, nil
+	}
+
+	return oapi.GetIngress200JSONResponse(ingressToOAPI(*ing)), nil
+}
+
+// DeleteIngress deletes an ingress by ID or name
+func (s *ApiService) DeleteIngress(ctx context.Context, request oapi.DeleteIngressRequestObject) (oapi.DeleteIngressResponseObject, error) {
+	log := logger.FromContext(ctx)
+
+	err := s.IngressManager.Delete(ctx, request.Id)
+	if err != nil {
+		if errors.Is(err, ingress.ErrNotFound) {
+			return oapi.DeleteIngress404JSONResponse{
+				Code:    "not_found",
+				Message: "ingress not found",
+			}, nil
+		}
+		log.ErrorContext(ctx, "failed to delete ingress", "error", err, "id", request.Id)
+		return oapi.DeleteIngress500JSONResponse{
+			Code:    "internal_error",
+			Message: "failed to delete ingress",
+		}, nil
+	}
+
+	return oapi.DeleteIngress204Response{}, nil
+}
+
+// ingressToOAPI converts a domain Ingress to the OAPI type
+func ingressToOAPI(ing ingress.Ingress) oapi.Ingress {
+	rules := make([]oapi.IngressRule, len(ing.Rules))
+	for i, rule := range ing.Rules {
+		port := rule.Match.GetPort()
+		rules[i] = oapi.IngressRule{
+			Match: oapi.IngressMatch{
+				Hostname: rule.Match.Hostname,
+				Port:     &port,
+			},
+			Target: oapi.IngressTarget{
+				Instance: rule.Target.Instance,
+				Port:     rule.Target.Port,
+			},
+		}
+	}
+
+	return oapi.Ingress{
+		Id:        ing.ID,
+		Name:      ing.Name,
+		Rules:     rules,
+		CreatedAt: ing.CreatedAt,
+	}
+}