tweaks

Author: rgarcia

.cursor/plans/systemd_vm_with_go_init_c0a9c010.plan.md

@@ -92,6 +92,8 @@ flowchart TB
     end
 ```
 
+
+
 ## Shared vs Mode-Specific Behavior
 
 | Step | Exec Mode | Systemd Mode ||------|-----------|--------------|| Mount proc/sys/dev | Shared | Shared || Mount rootfs overlay | Shared | Shared || Read config disk | Shared | Shared || Configure network | Init configures it | Init configures it (before pivot) || Load GPU drivers | Shared | Shared || Mount volumes | Shared | Shared || Copy guest-agent | To `/opt/hypeman/` | To `/opt/hypeman/` || Start guest-agent | Background process | Systemd service || PID 1 | Go init binary | Systemd || App lifecycle | Managed by init | Managed by systemd |
@@ -143,6 +145,8 @@ func (l *Logger) Error(phase, msg string, err error) {
 // 2024-12-23T10:15:32Z [INFO] [systemd] exec /sbin/init
 ```
 
+
+
 ## Go-based Init Binary
 
 Package structure at `lib/system/init/`:
@@ -160,6 +164,8 @@ lib/system/init/
     logger.go         # Human-readable logging to hypeman operations log
 ```
 
+
+
 ### Main Orchestration
 
 ```go
@@ -216,6 +222,8 @@ func main() {
 }
 ```
 
+
+
 ### Systemd Mode
 
 ```go
@@ -277,6 +285,8 @@ WantedBy=multi-user.target
 }
 ```
 
+
+
 ## Detection Logic
 
 Auto-detect systemd mode by inspecting the image's CMD. No override flag - if CMD is a systemd init, always use systemd mode.
@@ -320,6 +330,8 @@ func IsSystemdImage(entrypoint, cmd []string) bool {
 }
 ```
 
+
+
 ## E2E Test
 
 Custom Dockerfile in repository at `integration/testdata/systemd/Dockerfile`:
@@ -400,7 +412,7 @@ func TestExecModeUnchanged(t *testing.T) {
     result = execInVM(t, inst, "cat", "/proc/1/comm")
     assert.Equal(t, "init", strings.TrimSpace(result.Stdout))
 }
-```
 
 
-## Files to Modify/Create
+
+```

Makefile

@@ -1,5 +1,5 @@
 SHELL := /bin/bash
-.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build test install-tools gen-jwt download-ch-binaries download-ch-spec ensure-ch-binaries build-caddy-binaries build-caddy ensure-caddy-binaries build-preview-cli release-prep clean
+.PHONY: oapi-generate generate-vmm-client generate-wire generate-all dev build test install-tools gen-jwt download-ch-binaries download-ch-spec ensure-ch-binaries build-caddy-binaries build-caddy ensure-caddy-binaries  release-prep clean build-embedded
 
 # Directory where local binaries will be installed
 BIN_DIR ?= $(CURDIR)/bin
@@ -176,7 +176,6 @@ lib/system/init/init: lib/system/init/*.go
 	cd lib/system/init && CGO_ENABLED=0 go build -ldflags="-s -w" -o init .
 
 # Build all embedded binaries
-.PHONY: build-embedded
 build-embedded: lib/system/guest_agent/guest-agent lib/system/init/init
 
 # Build the binary

cmd/api/api/exec.go

@@ -29,11 +29,12 @@ var upgrader = websocket.Upgrader{
 
 // ExecRequest represents the JSON body for exec requests
 type ExecRequest struct {
-	Command []string          `json:"command"`
-	TTY     bool              `json:"tty"`
-	Env     map[string]string `json:"env,omitempty"`
-	Cwd     string            `json:"cwd,omitempty"`
-	Timeout int32             `json:"timeout,omitempty"` // seconds
+	Command      []string          `json:"command"`
+	TTY          bool              `json:"tty"`
+	Env          map[string]string `json:"env,omitempty"`
+	Cwd          string            `json:"cwd,omitempty"`
+	Timeout      int32             `json:"timeout,omitempty"`       // seconds
+	WaitForAgent int32             `json:"wait_for_agent,omitempty"` // seconds to wait for guest agent to be ready
 }
 
 // ExecHandler handles exec requests via WebSocket for bidirectional streaming
@@ -106,6 +107,7 @@ func (s *ApiService) ExecHandler(w http.ResponseWriter, r *http.Request) {
 		"tty", execReq.TTY,
 		"cwd", execReq.Cwd,
 		"timeout", execReq.Timeout,
+		"wait_for_agent", execReq.WaitForAgent,
 	)
 
 	// Create WebSocket read/writer wrapper
@@ -122,14 +124,15 @@ func (s *ApiService) ExecHandler(w http.ResponseWriter, r *http.Request) {
 
 	// Execute via vsock
 	exit, err := guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{
-		Command: execReq.Command,
-		Stdin:   wsConn,
-		Stdout:  wsConn,
-		Stderr:  wsConn,
-		TTY:     execReq.TTY,
-		Env:     execReq.Env,
-		Cwd:     execReq.Cwd,
-		Timeout: execReq.Timeout,
+		Command:      execReq.Command,
+		Stdin:        wsConn,
+		Stdout:       wsConn,
+		Stderr:       wsConn,
+		TTY:          execReq.TTY,
+		Env:          execReq.Env,
+		Cwd:          execReq.Cwd,
+		Timeout:      execReq.Timeout,
+		WaitForAgent: time.Duration(execReq.WaitForAgent) * time.Second,
 	})
 
 	duration := time.Since(startTime)

cmd/api/api/exec_test.go

@@ -115,38 +115,24 @@ func TestExecInstanceNonTTY(t *testing.T) {
 		t.Logf("vsock socket exists: %s", actualInst.VsockSocket)
 	}
 
-	// Wait for exec agent to be ready (retry a few times)
-	var exit *guest.ExitStatus
+	// Wait for exec agent to be ready using WaitForAgent
 	var stdout, stderr outputBuffer
-	var execErr error
 
 	dialer, err := hypervisor.NewVsockDialer(actualInst.HypervisorType, actualInst.VsockSocket, actualInst.VsockCID)
 	require.NoError(t, err)
 
 	t.Log("Testing exec command: whoami")
-	maxRetries := 10
-	for i := 0; i < maxRetries; i++ {
-		stdout = outputBuffer{}
-		stderr = outputBuffer{}
-
-		exit, execErr = guest.ExecIntoInstance(ctx(), dialer, guest.ExecOptions{
-			Command: []string{"/bin/sh", "-c", "whoami"},
-			Stdin:   nil,
-			Stdout:  &stdout,
-			Stderr:  &stderr,
-			TTY:     false,
-		})
-
-		if execErr == nil {
-			break
-		}
-
-		t.Logf("Exec attempt %d/%d failed, retrying: %v", i+1, maxRetries, execErr)
-		time.Sleep(1 * time.Second)
-	}
+	exit, execErr := guest.ExecIntoInstance(ctx(), dialer, guest.ExecOptions{
+		Command:      []string{"/bin/sh", "-c", "whoami"},
+		Stdin:        nil,
+		Stdout:       &stdout,
+		Stderr:       &stderr,
+		TTY:          false,
+		WaitForAgent: 10 * time.Second, // Wait up to 10s for guest agent to be ready
+	})
 
 	// Assert exec worked
-	require.NoError(t, execErr, "exec should succeed after retries")
+	require.NoError(t, execErr, "exec should succeed")
 	require.NotNil(t, exit, "exit status should be returned")
 	require.Equal(t, 0, exit.Code, "whoami should exit with code 0")
 

integration/systemd_test.go

@@ -170,35 +170,25 @@ func TestSystemdMode(t *testing.T) {
 
 // waitForGuestAgent polls until the guest agent is ready
 func waitForGuestAgent(ctx context.Context, mgr instances.Manager, instanceID string, timeout time.Duration) error {
-	deadline := time.Now().Add(timeout)
-	for time.Now().Before(deadline) {
-		inst, err := mgr.GetInstance(ctx, instanceID)
-		if err != nil {
-			time.Sleep(500 * time.Millisecond)
-			continue
-		}
-
-		// Try to connect to the guest agent
-		dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID)
-		if err != nil {
-			time.Sleep(500 * time.Millisecond)
-			continue
-		}
-
-		// Try a simple exec to verify agent is responding
-		var stdout bytes.Buffer
-		_, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{
-			Command: []string{"echo", "ready"},
-			Stdout:  &stdout,
-			TTY:     false,
-		})
-		if err == nil {
-			return nil
-		}
+	inst, err := mgr.GetInstance(ctx, instanceID)
+	if err != nil {
+		return err
+	}
 
-		time.Sleep(500 * time.Millisecond)
+	dialer, err := hypervisor.NewVsockDialer(inst.HypervisorType, inst.VsockSocket, inst.VsockCID)
+	if err != nil {
+		return err
 	}
-	return context.DeadlineExceeded
+
+	// Use WaitForAgent to wait for the agent to be ready
+	var stdout bytes.Buffer
+	_, err = guest.ExecIntoInstance(ctx, dialer, guest.ExecOptions{
+		Command:      []string{"echo", "ready"},
+		Stdout:       &stdout,
+		TTY:          false,
+		WaitForAgent: timeout,
+	})
+	return err
 }
 
 // execInInstance executes a command in the instance

lib/guest/client.go

@@ -30,7 +30,6 @@ const (
 // This can happen if:
 // - The VM is still booting
 // - The guest agent was stopped or deleted
-// - The VM is in systemd mode and the agent service failed to start
 type AgentConnectionError struct {
 	Err error
 }
@@ -43,14 +42,6 @@ func (e *AgentConnectionError) Unwrap() error {
 	return e.Err
 }
 
-// IsAgentConnectionError checks if an error is due to the guest agent not responding.
-func IsAgentConnectionError(err error) bool {
-	var agentErr *AgentConnectionError
-	return err != nil && (strings.Contains(err.Error(), "guest agent not responding") ||
-		strings.Contains(err.Error(), "connection refused") ||
-		errors.As(err, &agentErr))
-}
-
 // connPool manages reusable gRPC connections per vsock dialer key
 // This avoids the overhead and potential issues of rapidly creating/closing connections
 var connPool = struct {
@@ -122,19 +113,54 @@ type ExitStatus struct {
 
 // ExecOptions configures command execution
 type ExecOptions struct {
-	Command []string
-	Stdin   io.Reader
-	Stdout  io.Writer
-	Stderr  io.Writer
-	TTY     bool
-	Env     map[string]string // Environment variables
-	Cwd     string            // Working directory (optional)
-	Timeout int32             // Execution timeout in seconds (0 = no timeout)
+	Command      []string
+	Stdin        io.Reader
+	Stdout       io.Writer
+	Stderr       io.Writer
+	TTY          bool
+	Env          map[string]string // Environment variables
+	Cwd          string            // Working directory (optional)
+	Timeout      int32             // Execution timeout in seconds (0 = no timeout)
+	WaitForAgent time.Duration     // Max time to wait for agent to be ready (0 = no wait, fail immediately)
 }
 
 // ExecIntoInstance executes command in instance via vsock using gRPC.
 // The dialer is a hypervisor-specific VsockDialer that knows how to connect to the guest.
+// If WaitForAgent is set, it will retry on AgentConnectionError until the timeout.
 func ExecIntoInstance(ctx context.Context, dialer hypervisor.VsockDialer, opts ExecOptions) (*ExitStatus, error) {
+	// If no wait requested, execute immediately
+	if opts.WaitForAgent == 0 {
+		return execIntoInstanceOnce(ctx, dialer, opts)
+	}
+
+	deadline := time.Now().Add(opts.WaitForAgent)
+
+	for {
+		exit, err := execIntoInstanceOnce(ctx, dialer, opts)
+
+		// Success or non-connection error - return immediately
+		var connErr *AgentConnectionError
+		if err == nil || !errors.As(err, &connErr) {
+			return exit, err
+		}
+
+		// Connection error - check if we should retry
+		if time.Now().After(deadline) {
+			return nil, err
+		}
+
+		// Wait before retrying, but respect context cancellation
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-time.After(500 * time.Millisecond):
+			// Continue to retry
+		}
+	}
+}
+
+// execIntoInstanceOnce executes command in instance via vsock using gRPC (single attempt).
+func execIntoInstanceOnce(ctx context.Context, dialer hypervisor.VsockDialer, opts ExecOptions) (*ExitStatus, error) {
 	start := time.Now()
 	var bytesSent int64
 

lib/system/init/mode_exec.go

@@ -97,4 +97,3 @@ func buildEnv(env map[string]string) []string {
 
 	return result
 }
-

lib/system/init/network.go

@@ -55,4 +55,3 @@ func runIP(args ...string) error {
 	}
 	return nil
 }
-