Merge pull request #62 from diggerhq/tls2

ALB-based TLS termination: proxy data-plane through control plane

Author: motatoes

cmd/server/main.go

@@ -137,6 +137,12 @@ func main() {
 		defer redisRegistry.Stop()
 		opts.WorkerRegistry = redisRegistry
 		log.Println("opensandbox: Redis worker registry started")
+
+		// Create sandbox API proxy for routing data-plane requests to workers
+		if opts.Store != nil && opts.JWTIssuer != nil {
+			opts.SandboxAPIProxy = proxy.NewSandboxAPIProxy(opts.Store, redisRegistry, opts.JWTIssuer)
+			log.Println("opensandbox: sandbox API proxy enabled (data-plane requests proxied to workers)")
+		}
 	}
 
 	// Initialize EC2 compute pool + autoscaler (server mode with AWS configured)

deploy/ec2/Caddyfile

@@ -11,7 +11,7 @@ set -euo pipefail
 # Prerequisites:
 #   - Ubuntu 24.04 LTS AMI
 #   - r7gd.metal (ARM64) or c7i.metal (x86_64) for production
-#   - Security group: 443 (HTTPS), 8080 (HTTP), 9090 (gRPC), 9091 (metrics) open inbound
+#   - Security group: 8080 (HTTP), 9090 (gRPC), 9091 (metrics) open from VPC only (workers are internal)
 #   - SSH access
 
 # Detect architecture
@@ -73,32 +73,6 @@ sudo apt-get install -y e2fsprogs
 echo "==> Installing Redis..."
 sudo apt-get install -y redis-server
 
-# -------------------------------------------------------------------
-# Caddy (custom build with Route53 DNS module for wildcard certs)
-# -------------------------------------------------------------------
-echo "==> Installing Go (needed for xcaddy)..."
-GO_VERSION="1.23.6"
-curl -sL "https://go.dev/dl/go${GO_VERSION}.linux-${GOARCH}.tar.gz" | sudo tar -C /usr/local -xzf -
-export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin
-
-echo "==> Building Caddy with Route53 DNS module..."
-go install github.com/caddyserver/xcaddy/cmd/xcaddy@latest
-xcaddy build --with github.com/caddy-dns/route53 --output /tmp/caddy-custom
-sudo mv /tmp/caddy-custom /usr/local/bin/caddy
-sudo chmod +x /usr/local/bin/caddy
-
-echo "==> Verifying Caddy has Route53 module..."
-caddy list-modules | grep route53 || { echo "ERROR: Caddy missing route53 module"; exit 1; }
-
-echo "==> Installing Caddy config..."
-sudo mkdir -p /etc/caddy
-sudo cp /tmp/deploy-ec2/Caddyfile /etc/caddy/Caddyfile 2>/dev/null || \
-  echo "    NOTE: Copy deploy/ec2/Caddyfile to /etc/caddy/Caddyfile manually"
-
-echo "==> Installing Caddy systemd unit..."
-sudo cp /tmp/deploy-ec2/caddy.service /etc/systemd/system/caddy.service 2>/dev/null || \
-  echo "    NOTE: Copy deploy/ec2/caddy.service to /etc/systemd/system/ manually"
-
 # -------------------------------------------------------------------
 # NVMe instance storage (XFS with reflink for instant rootfs copies)
 # -------------------------------------------------------------------
@@ -233,7 +207,7 @@ WORKER_ID="w-use2-${SHORT_ID}"
 mkdir -p /etc/opensandbox
 cat > /etc/opensandbox/worker-identity.env << EOF
 OPENSANDBOX_WORKER_ID=${WORKER_ID}
-OPENSANDBOX_HTTP_ADDR=http://${PUBLIC_IP:-$PRIVATE_IP}:8080
+OPENSANDBOX_HTTP_ADDR=http://${PRIVATE_IP}:8080
 OPENSANDBOX_GRPC_ADVERTISE=${PRIVATE_IP}:9090
 EOF
 echo "opensandbox-identity: ${WORKER_ID} private=${PRIVATE_IP} public=${PUBLIC_IP:-none}"
@@ -305,14 +279,12 @@ sudo systemctl daemon-reload
 sudo systemctl enable opensandbox-nvme
 sudo systemctl enable opensandbox-identity
 sudo systemctl enable opensandbox-worker
-sudo systemctl enable caddy 2>/dev/null || true
 
 # -------------------------------------------------------------------
 # Cleanup
 # -------------------------------------------------------------------
-echo "==> Cleaning up build tools..."
+echo "==> Cleaning up..."
 sudo apt-get clean
-sudo rm -rf /usr/local/go $HOME/go
 
 echo ""
 echo "============================================"

deploy/ec2/caddy.service

@@ -46,6 +46,7 @@ type Server struct {
 	ecrConfig       *ecr.Config                       // nil if ECR not configured
 	cfClient        *cloudflare.Client                // nil if Cloudflare not configured
 	pendingCreates  sync.Map                          // map[sandboxID]*pendingCreate — async sandbox creation tracking
+	sandboxAPIProxy *proxy.SandboxAPIProxy            // nil except in server mode (proxies data-plane to workers)
 }
 
 // pendingCreate tracks an async sandbox creation.
@@ -73,6 +74,7 @@ type ServerOpts struct {
 	CheckpointStore *storage.CheckpointStore           // nil if hibernation not configured
 	ECRConfig       *ecr.Config                        // nil if ECR not configured
 	CFClient        *cloudflare.Client                 // nil if Cloudflare not configured
+	SandboxAPIProxy *proxy.SandboxAPIProxy             // nil except in server mode (proxies data-plane to workers)
 }
 
 // NewServer creates a new API server with all routes configured.
@@ -102,6 +104,7 @@ func NewServer(mgr sandbox.Manager, ptyMgr *sandbox.PTYManager, apiKey string, o
 		s.sandboxDomain = opts.SandboxDomain
 		s.ecrConfig = opts.ECRConfig
 		s.cfClient = opts.CFClient
+		s.sandboxAPIProxy = opts.SandboxAPIProxy
 	}
 
 	// Global middleware
@@ -132,7 +135,6 @@ func NewServer(mgr sandbox.Manager, ptyMgr *sandbox.PTYManager, apiKey string, o
 	api.GET("/sandboxes", s.listSandboxes)
 	api.GET("/sandboxes/:id", s.getSandbox)
 	api.DELETE("/sandboxes/:id", s.killSandbox)
-	api.POST("/sandboxes/:id/timeout", s.setTimeout)
 
 	// Hibernation
 	api.POST("/sandboxes/:id/hibernate", s.hibernateSandbox)
@@ -155,32 +157,70 @@ func NewServer(mgr sandbox.Manager, ptyMgr *sandbox.PTYManager, apiKey string, o
 	api.GET("/sandboxes/:id/preview", s.listPreviewURLs)
 	api.DELETE("/sandboxes/:id/preview/:port", s.deletePreviewURL)
 
-	// Exec sessions (replaces old /commands)
-	api.POST("/sandboxes/:id/exec", s.createExecSession)
-	api.GET("/sandboxes/:id/exec", s.listExecSessions)
-	api.GET("/sandboxes/:id/exec/:sessionID", s.execSessionWebSocket)
-	api.POST("/sandboxes/:id/exec/:sessionID/kill", s.killExecSession)
-	api.POST("/sandboxes/:id/exec/run", s.execRun)
-
-	// Agent sessions (Claude Agent SDK)
-	api.POST("/sandboxes/:id/agent", s.createAgentSession)
-	api.GET("/sandboxes/:id/agent", s.listAgentSessions)
-	api.POST("/sandboxes/:id/agent/:sid/prompt", s.sendAgentPrompt)
-	api.POST("/sandboxes/:id/agent/:sid/interrupt", s.interruptAgent)
-	api.POST("/sandboxes/:id/agent/:sid/kill", s.killAgentSession)
-
-	// Filesystem
-	api.GET("/sandboxes/:id/files", s.readFile)
-	api.PUT("/sandboxes/:id/files", s.writeFile)
-	api.GET("/sandboxes/:id/files/list", s.listDir)
-	api.POST("/sandboxes/:id/files/mkdir", s.makeDir)
-	api.DELETE("/sandboxes/:id/files", s.removeFile)
-
-	// PTY
-	api.POST("/sandboxes/:id/pty", s.createPTY)
-	api.GET("/sandboxes/:id/pty/:sessionID", s.ptyWebSocket)
-	api.POST("/sandboxes/:id/pty/:sessionID/resize", s.resizePTY)
-	api.DELETE("/sandboxes/:id/pty/:sessionID", s.killPTY)
+	// Data-plane routes: in server mode, proxy to workers; otherwise handle locally
+	if s.sandboxAPIProxy != nil {
+		// Server mode: proxy all data-plane requests to the worker that owns the sandbox
+		pxy := s.sandboxAPIProxy.ProxyHandler
+
+		// Exec
+		api.POST("/sandboxes/:id/exec", pxy)
+		api.GET("/sandboxes/:id/exec", pxy)
+		api.GET("/sandboxes/:id/exec/:sessionID", pxy)
+		api.POST("/sandboxes/:id/exec/:sessionID/kill", pxy)
+		api.POST("/sandboxes/:id/exec/run", pxy)
+
+		// Agent
+		api.POST("/sandboxes/:id/agent", pxy)
+		api.GET("/sandboxes/:id/agent", pxy)
+		api.POST("/sandboxes/:id/agent/:sid/prompt", pxy)
+		api.POST("/sandboxes/:id/agent/:sid/interrupt", pxy)
+		api.POST("/sandboxes/:id/agent/:sid/kill", pxy)
+
+		// Filesystem
+		api.GET("/sandboxes/:id/files", pxy)
+		api.PUT("/sandboxes/:id/files", pxy)
+		api.GET("/sandboxes/:id/files/list", pxy)
+		api.POST("/sandboxes/:id/files/mkdir", pxy)
+		api.DELETE("/sandboxes/:id/files", pxy)
+
+		// PTY
+		api.POST("/sandboxes/:id/pty", pxy)
+		api.GET("/sandboxes/:id/pty/:sessionID", pxy)
+		api.POST("/sandboxes/:id/pty/:sessionID/resize", pxy)
+		api.DELETE("/sandboxes/:id/pty/:sessionID", pxy)
+
+		// Timeout
+		api.POST("/sandboxes/:id/timeout", pxy)
+
+		// Token refresh
+		api.POST("/sandboxes/:id/token/refresh", pxy)
+	} else {
+		// Combined/worker mode: handle locally
+		api.POST("/sandboxes/:id/exec", s.createExecSession)
+		api.GET("/sandboxes/:id/exec", s.listExecSessions)
+		api.GET("/sandboxes/:id/exec/:sessionID", s.execSessionWebSocket)
+		api.POST("/sandboxes/:id/exec/:sessionID/kill", s.killExecSession)
+		api.POST("/sandboxes/:id/exec/run", s.execRun)
+
+		api.POST("/sandboxes/:id/agent", s.createAgentSession)
+		api.GET("/sandboxes/:id/agent", s.listAgentSessions)
+		api.POST("/sandboxes/:id/agent/:sid/prompt", s.sendAgentPrompt)
+		api.POST("/sandboxes/:id/agent/:sid/interrupt", s.interruptAgent)
+		api.POST("/sandboxes/:id/agent/:sid/kill", s.killAgentSession)
+
+		api.GET("/sandboxes/:id/files", s.readFile)
+		api.PUT("/sandboxes/:id/files", s.writeFile)
+		api.GET("/sandboxes/:id/files/list", s.listDir)
+		api.POST("/sandboxes/:id/files/mkdir", s.makeDir)
+		api.DELETE("/sandboxes/:id/files", s.removeFile)
+
+		api.POST("/sandboxes/:id/pty", s.createPTY)
+		api.GET("/sandboxes/:id/pty/:sessionID", s.ptyWebSocket)
+		api.POST("/sandboxes/:id/pty/:sessionID/resize", s.resizePTY)
+		api.DELETE("/sandboxes/:id/pty/:sessionID", s.killPTY)
+
+		api.POST("/sandboxes/:id/timeout", s.setTimeout)
+	}
 
 	// Snapshots (pre-built declarative images)
 	api.POST("/snapshots", s.createSnapshot)

deploy/ec2/setup-instance.sh

@@ -311,12 +311,11 @@ func (s *Server) createSandboxRemote(c echo.Context, ctx context.Context, cfg ty
 	}
 
 	resp := map[string]interface{}{
-		"sandboxID":  grpcResp.SandboxId,
-		"connectURL": worker.HTTPAddr,
-		"token":      token,
-		"status":     grpcResp.Status,
-		"region":     region,
-		"workerID":   worker.ID,
+		"sandboxID": grpcResp.SandboxId,
+		"token":     token,
+		"status":    grpcResp.Status,
+		"region":    region,
+		"workerID":  worker.ID,
 	}
 
 	return c.JSON(http.StatusCreated, resp)
@@ -385,13 +384,6 @@ func (s *Server) getSandboxRemote(c echo.Context, sandboxID string) error {
 		return c.JSON(http.StatusOK, resp)
 	}
 
-	// Look up worker address
-	worker := s.workerRegistry.GetWorker(session.WorkerID)
-	connectURL := ""
-	if worker != nil {
-		connectURL = worker.HTTPAddr
-	}
-
 	// Issue a fresh token
 	var token string
 	if s.jwtIssuer != nil {
@@ -402,14 +394,13 @@ func (s *Server) getSandboxRemote(c echo.Context, sandboxID string) error {
 	}
 
 	resp := map[string]interface{}{
-		"sandboxID":  sandboxID,
-		"connectURL": connectURL,
-		"token":      token,
-		"status":     session.Status,
-		"region":     session.Region,
-		"workerID":   session.WorkerID,
-		"startedAt":  session.StartedAt,
-		"template":   session.Template,
+		"sandboxID": sandboxID,
+		"token":     token,
+		"status":    session.Status,
+		"region":    session.Region,
+		"workerID":  session.WorkerID,
+		"startedAt": session.StartedAt,
+		"template":  session.Template,
 	}
 
 	return c.JSON(http.StatusOK, resp)
@@ -544,12 +535,6 @@ func (s *Server) listSandboxesRemote(c echo.Context) error {
 			"startedAt": sess.StartedAt,
 		}
 
-		// Attach connectURL from registry
-		worker := s.workerRegistry.GetWorker(sess.WorkerID)
-		if worker != nil {
-			entry["connectURL"] = worker.HTTPAddr
-		}
-
 		// Issue fresh JWT
 		if s.jwtIssuer != nil {
 			token, err := s.jwtIssuer.IssueSandboxToken(orgID, sess.SandboxID, sess.WorkerID, 24*time.Hour)
@@ -565,13 +550,6 @@ func (s *Server) listSandboxesRemote(c echo.Context) error {
 }
 
 func (s *Server) setTimeout(c echo.Context) error {
-	// In server mode, timeout must be set directly on the worker via connectURL
-	if s.workerRegistry != nil {
-		return c.JSON(http.StatusBadRequest, map[string]string{
-			"error": "timeout must be set directly on the worker via connectURL",
-		})
-	}
-
 	if s.router == nil {
 		return c.JSON(http.StatusServiceUnavailable, errSandboxNotAvailable)
 	}