feat(v3): headless install orchestrator stub and dryrun test (#3158)

* feat(v3): headless install orchestrator stub and dryrun test

* f

Author: emosbaugh

.github/workflows/ci.yaml

@@ -293,7 +293,9 @@ jobs:
           path: |
             ./dev/.gocache
             ./dev/.gomodcache
-          key: dryrun-tests-go-cache
+          key: dryrun-tests-go-cache-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            dryrun-tests-go-cache-
       - name: Run tests
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -436,7 +438,9 @@ jobs:
         with:
           path: |
             output/bins
-          key: bins-cache
+          key: bins-cache-${{ hashFiles('Makefile', 'versions.mk') }}
+          restore-keys: |
+            bins-cache-
 
       - name: Setup go
         uses: actions/setup-go@v6
@@ -587,7 +591,9 @@ jobs:
         with:
           path: |
             output/bins
-          key: bins-cache
+          key: bins-cache-${{ hashFiles('Makefile', 'versions.mk') }}
+          restore-keys: |
+            bins-cache-
 
       - name: Setup go
         uses: actions/setup-go@v6

.github/workflows/release-prod.yaml

@@ -158,7 +158,9 @@ jobs:
         with:
           path: |
             output/bins
-          key: bins-cache
+          key: bins-cache-${{ hashFiles('Makefile', 'versions.mk') }}
+          restore-keys: |
+            bins-cache-
 
       - name: Setup Go
         uses: actions/setup-go@v6

cmd/installer/cli/api.go

@@ -3,7 +3,6 @@ package cli
 import (
 	"context"
 	"crypto/tls"
-	"errors"
 	"fmt"
 	"io/fs"
 	"log"
@@ -37,23 +36,15 @@ type apiOptions struct {
 	WebAssetsFS     fs.FS
 }
 
-func startAPI(ctx context.Context, cert tls.Certificate, opts apiOptions, cancel context.CancelFunc) error {
+// startAPI starts the API server and returns a channel that will receive an error if the API exits unexpectedly.
+// The returned channel will be closed when the API exits normally (via context cancellation).
+func startAPI(ctx context.Context, cert tls.Certificate, opts apiOptions) (<-chan error, error) {
 	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", opts.ManagerPort))
 	if err != nil {
-		return fmt.Errorf("unable to create listener: %w", err)
+		return nil, fmt.Errorf("unable to create listener: %w", err)
 	}
 	logrus.Debugf("API server listening on port: %d", opts.ManagerPort)
 
-	go func() {
-		// If the api exits, we want to exit the entire process
-		defer cancel()
-		if err := serveAPI(ctx, listener, cert, opts); err != nil {
-			if !errors.Is(err, http.ErrServerClosed) {
-				logrus.Errorf("API server exited with error: %v", err)
-			}
-		}
-	}()
-
 	addr := fmt.Sprintf("https://localhost:%d", opts.ManagerPort)
 	httpClient := &http.Client{
 		Timeout: 2 * time.Second,
@@ -64,11 +55,30 @@ func startAPI(ctx context.Context, cert tls.Certificate, opts apiOptions, cancel
 			},
 		},
 	}
-	if err := waitForAPI(ctx, httpClient, addr); err != nil {
-		return fmt.Errorf("unable to wait for api: %w", err)
+
+	apiExitCh := make(chan error, 1)
+	waitErrCh := make(chan error, 1)
+
+	go func() {
+		apiExitCh <- serveAPI(ctx, listener, cert, opts)
+	}()
+
+	go func() {
+		waitErrCh <- waitForAPI(ctx, httpClient, addr)
+	}()
+
+	select {
+	case err := <-apiExitCh:
+		if err != nil {
+			return nil, fmt.Errorf("serve api: %w", err)
+		}
+	case err := <-waitErrCh:
+		if err != nil {
+			return apiExitCh, fmt.Errorf("wait for api: %w", err)
+		}
 	}
 
-	return nil
+	return apiExitCh, nil
 }
 
 func serveAPI(ctx context.Context, listener net.Listener, cert tls.Certificate, opts apiOptions) error {

cmd/installer/cli/headless/install/mock_orchestrator.go

@@ -0,0 +1,23 @@
+package install
+
+import (
+	"context"
+
+	"github.com/stretchr/testify/mock"
+)
+
+// MockOrchestrator is a mock implementation of the Orchestrator interface for testing.
+// It records method calls and allows configuring return values.
+type MockOrchestrator struct {
+	mock.Mock
+}
+
+// NewMockOrchestrator creates a new MockOrchestrator instance
+func NewMockOrchestrator() *MockOrchestrator {
+	return &MockOrchestrator{}
+}
+
+// RunHeadlessInstall implements the Orchestrator interface
+func (m *MockOrchestrator) RunHeadlessInstall(ctx context.Context, opts HeadlessInstallOptions) error {
+	return m.Called(ctx, opts).Error(0)
+}

cmd/installer/cli/headless/install/orchestrator.go

@@ -0,0 +1,62 @@
+package install
+
+import (
+	"context"
+	"fmt"
+
+	apitypes "github.com/replicatedhq/embedded-cluster/api/types"
+)
+
+// HeadlessInstallOptions contains the configuration options for a headless installation
+type HeadlessInstallOptions struct {
+	// ConfigValues are the application config values to use for installation
+	ConfigValues apitypes.AppConfigValues
+
+	// LinuxInstallationConfig contains the installation settings for the Linux target
+	LinuxInstallationConfig apitypes.LinuxInstallationConfig
+
+	// IgnoreHostPreflights indicates whether to bypass host preflight check failures
+	IgnoreHostPreflights bool
+
+	// IgnoreAppPreflights indicates whether to bypass app preflight check failures
+	IgnoreAppPreflights bool
+
+	// AirgapBundle is the path to the airgap bundle file (empty string for online installs)
+	AirgapBundle string
+}
+
+// Orchestrator defines the interface for headless installation operations.
+// It orchestrates the installation process by interacting with the v3 API server
+// running in-process via HTTP calls to localhost.
+type Orchestrator interface {
+	// RunHeadlessInstall executes a complete headless installation workflow.
+	// It performs the following steps in order:
+	//   1. Configure application with config values
+	//   2. Configure installation settings
+	//   3. Run host preflights (with optional bypass)
+	//   4. Setup infrastructure
+	//   5. Process airgap bundle (if provided)
+	//   6. Run app preflights (with optional bypass)
+	//   7. Install application
+	//
+	// The installation cannot be resumed if it fails after infrastructure setup begins.
+	// Any failure after that point requires running 'embedded-cluster reset' and retrying.
+	//
+	// Returns an error if any step fails, with a descriptive message for recovery.
+	RunHeadlessInstall(ctx context.Context, opts HeadlessInstallOptions) error
+}
+
+// NewOrchestrator creates a new Orchestrator instance
+func NewOrchestrator() Orchestrator {
+	return &orchestrator{}
+}
+
+// orchestrator is the default implementation of the Orchestrator interface
+type orchestrator struct {
+}
+
+// RunHeadlessInstall executes a complete headless installation workflow
+func (o *orchestrator) RunHeadlessInstall(ctx context.Context, opts HeadlessInstallOptions) error {
+	// TODO(PR2): Implement real headless installation orchestration
+	return fmt.Errorf("headless installation is not yet fully implemented - coming in a future release")
+}

cmd/installer/cli/install.go

@@ -543,21 +543,50 @@ func runManagerExperienceInstall(
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	if err := startAPI(ctx, installCfg.tlsCert, apiConfig, cancel); err != nil {
+	apiExitCh, err := startAPI(ctx, installCfg.tlsCert, apiConfig)
+	if err != nil {
 		return fmt.Errorf("failed to start api: %w", err)
 	}
 
 	if flags.headless {
+		// Setup signal handler with the metrics reporter cleanup function
+		signalHandler(ctx, cancel, func(ctx context.Context, sig os.Signal) {
+			metricsReporter.ReportSignalAborted(ctx, sig)
+		})
+
 		// TODO(PR2): Implement headless installation orchestration
-		return fmt.Errorf("headless installation is not yet fully implemented - coming in a future release")
+		err := fmt.Errorf("headless installation is not yet fully implemented - coming in a future release")
+
+		if err != nil {
+			// Check if this is an interrupt error from the terminal
+			if errors.Is(err, terminal.InterruptErr) {
+				metricsReporter.ReportSignalAborted(ctx, syscall.SIGINT)
+			} else {
+				metricsReporter.ReportInstallationFailed(ctx, err)
+			}
+			return err
+		}
+		metricsReporter.ReportInstallationSucceeded(ctx)
+
+		return nil
 	}
 
 	logrus.Infof("\nVisit the %s manager to continue: %s\n",
 		appTitle,
 		getManagerURL(flags.hostname, flags.managerPort))
-	<-ctx.Done()
 
-	return nil
+	// Wait for either user cancellation or API unexpected exit
+	select {
+	case <-ctx.Done():
+		// Normal exit (user interrupted)
+		return nil
+	case err := <-apiExitCh:
+		// API exited unexpectedly
+		if err != nil {
+			return err
+		}
+		return fmt.Errorf("api server exited unexpectedly")
+	}
 }
 
 func runInstall(ctx context.Context, flags installFlags, installCfg *installConfig, rc runtimeconfig.RuntimeConfig, metricsReporter *installReporter) (finalErr error) {

cmd/installer/cli/upgrade.go

@@ -496,16 +496,27 @@ func runManagerExperienceUpgrade(
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	if err := startAPI(ctx, upgradeConfig.tlsCert, apiConfig, cancel); err != nil {
+	apiExitCh, err := startAPI(ctx, upgradeConfig.tlsCert, apiConfig)
+	if err != nil {
 		return fmt.Errorf("failed to start api: %w", err)
 	}
 
 	logrus.Infof("\nVisit the %s manager to continue the upgrade: %s\n",
 		appTitle,
 		getManagerURL(upgradeConfig.tlsConfig.Hostname, upgradeConfig.managerPort))
-	<-ctx.Done()
 
-	return nil
+	// Wait for either user cancellation or API unexpected exit
+	select {
+	case <-ctx.Done():
+		// Normal exit (user interrupted)
+		return nil
+	case err := <-apiExitCh:
+		// API exited unexpectedly
+		if err != nil {
+			return err
+		}
+		return fmt.Errorf("api server exited unexpectedly")
+	}
 }
 
 // checkRequiresInfraUpgrade determines if an infrastructure upgrade is required by comparing

pkg-new/replicatedapi/client.go

@@ -17,6 +17,16 @@ import (
 
 var defaultHTTPClient = newRetryableHTTPClient()
 
+// ClientFactory is a function type for creating replicatedapi clients
+type ClientFactory func(replicatedAppURL string, license *kotsv1beta1.License, releaseData *release.ReleaseData, opts ...ClientOption) (Client, error)
+
+var clientFactory ClientFactory = defaultNewClient
+
+// SetClientFactory sets a custom factory for creating clients (used for testing/mocking)
+func SetClientFactory(factory ClientFactory) {
+	clientFactory = factory
+}
+
 type Client interface {
 	SyncLicense(ctx context.Context) (*kotsv1beta1.License, []byte, error)
 }
@@ -43,7 +53,13 @@ func WithHTTPClient(httpClient *retryablehttp.Client) ClientOption {
 	}
 }
 
+// NewClient creates a new replicatedapi client using the configured factory
 func NewClient(replicatedAppURL string, license *kotsv1beta1.License, releaseData *release.ReleaseData, opts ...ClientOption) (Client, error) {
+	return clientFactory(replicatedAppURL, license, releaseData, opts...)
+}
+
+// defaultNewClient is the default implementation of NewClient
+func defaultNewClient(replicatedAppURL string, license *kotsv1beta1.License, releaseData *release.ReleaseData, opts ...ClientOption) (Client, error) {
 	c := &client{
 		replicatedAppURL: replicatedAppURL,
 		license:          license,

pkg/dryrun/dryrun.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/replicatedhq/embedded-cluster/pkg-new/config"
 	"github.com/replicatedhq/embedded-cluster/pkg-new/k0s"
+	"github.com/replicatedhq/embedded-cluster/pkg-new/replicatedapi"
 	"github.com/replicatedhq/embedded-cluster/pkg/dryrun/types"
 	"github.com/replicatedhq/embedded-cluster/pkg/helm"
 	"github.com/replicatedhq/embedded-cluster/pkg/helpers"
@@ -18,6 +19,8 @@ import (
 	"github.com/replicatedhq/embedded-cluster/pkg/kubeutils"
 	"github.com/replicatedhq/embedded-cluster/pkg/metrics"
 	"github.com/replicatedhq/embedded-cluster/pkg/netutils"
+	"github.com/replicatedhq/embedded-cluster/pkg/release"
+	kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1"
 	troubleshootv1beta2 "github.com/replicatedhq/troubleshoot/pkg/apis/troubleshoot/v1beta2"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
@@ -42,6 +45,7 @@ type Client struct {
 	K0sClient                *K0s
 	HelmClient               helm.Client
 	Kotsadm                  *Kotsadm
+	ReplicatedAPIClient      *ReplicatedAPIClient
 	NetworkInterfaceProvider netutils.NetworkInterfaceProvider
 	ChooseHostInterfaceImpl  *ChooseInterfaceImpl
 }
@@ -78,6 +82,11 @@ func Init(outputFile string, client *Client) {
 			return client.HelmClient, nil
 		})
 	}
+	if client.ReplicatedAPIClient != nil {
+		replicatedapi.SetClientFactory(func(replicatedAppURL string, license *kotsv1beta1.License, releaseData *release.ReleaseData, opts ...replicatedapi.ClientOption) (replicatedapi.Client, error) {
+			return client.ReplicatedAPIClient, nil
+		})
+	}
 	if client.NetworkInterfaceProvider != nil {
 		config.NetworkInterfaceProvider = client.NetworkInterfaceProvider
 		netutils.DefaultNetworkInterfaceProvider = client.NetworkInterfaceProvider

pkg/dryrun/replicatedapi.go

@@ -0,0 +1,32 @@
+package dryrun
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/replicatedhq/embedded-cluster/pkg-new/replicatedapi"
+	kotsv1beta1 "github.com/replicatedhq/kotskinds/apis/kots/v1beta1"
+	"sigs.k8s.io/yaml"
+)
+
+var _ replicatedapi.Client = (*ReplicatedAPIClient)(nil)
+
+// ReplicatedAPIClient is a mockable implementation of the replicatedapi.Client interface.
+type ReplicatedAPIClient struct {
+	License      *kotsv1beta1.License
+	LicenseBytes []byte
+}
+
+// SyncLicense returns the mocked license data.
+func (c *ReplicatedAPIClient) SyncLicense(ctx context.Context) (*kotsv1beta1.License, []byte, error) {
+	// If License is not set but LicenseBytes is, parse the license from bytes
+	if c.License == nil && len(c.LicenseBytes) > 0 {
+		var license kotsv1beta1.License
+		if err := yaml.Unmarshal(c.LicenseBytes, &license); err != nil {
+			return nil, nil, fmt.Errorf("failed to parse license from bytes: %w", err)
+		}
+		c.License = &license
+	}
+
+	return c.License, c.LicenseBytes, nil
+}