Get charts and k0s images from KOTS and deprecate airgap bundle flag for node joins (#2131)

* Get charts and k0s images from KOTS and deprecate airgap bundle flag for node joins

Author: sgalsaleh

cmd/installer/cli/join.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/AlecAivazis/survey/v2/terminal"
 	k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
+	"github.com/replicatedhq/embedded-cluster/cmd/installer/goods"
 	ecv1beta1 "github.com/replicatedhq/embedded-cluster/kinds/apis/v1beta1"
 	"github.com/replicatedhq/embedded-cluster/kinds/types/join"
 	"github.com/replicatedhq/embedded-cluster/pkg/addons"
@@ -27,6 +28,7 @@ import (
 	"github.com/replicatedhq/embedded-cluster/pkg/release"
 	"github.com/replicatedhq/embedded-cluster/pkg/runtimeconfig"
 	"github.com/replicatedhq/embedded-cluster/pkg/spinner"
+	"github.com/replicatedhq/embedded-cluster/pkg/support"
 	"github.com/replicatedhq/embedded-cluster/pkg/versions"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -36,8 +38,6 @@ import (
 )
 
 type JoinCmdFlags struct {
-	airgapBundle         string
-	isAirgap             bool
 	noHA                 bool
 	networkInterface     string
 	assumeYes            bool
@@ -60,8 +60,6 @@ func JoinCmd(ctx context.Context, name string) *cobra.Command {
 				return err
 			}
 
-			flags.isAirgap = flags.airgapBundle != ""
-
 			return nil
 		},
 		PostRun: func(cmd *cobra.Command, args []string) {
@@ -84,7 +82,7 @@ func JoinCmd(ctx context.Context, name string) *cobra.Command {
 				metricsReporter.ReportSignalAborted(ctx, sig)
 			})
 
-			if err := runJoin(cmd.Context(), name, flags, jcmd, metricsReporter); err != nil {
+			if err := runJoin(cmd.Context(), name, flags, jcmd, args[0], metricsReporter); err != nil {
 				// Check if this is an interrupt error from the terminal
 				if errors.Is(err, terminal.InterruptErr) {
 					metricsReporter.ReportSignalAborted(ctx, syscall.SIGINT)
@@ -114,8 +112,6 @@ func preRunJoin(flags *JoinCmdFlags) error {
 		return fmt.Errorf("join command must be run as root")
 	}
 
-	flags.isAirgap = flags.airgapBundle != ""
-
 	// if a network interface flag was not provided, attempt to discover it
 	if flags.networkInterface == "" {
 		autoInterface, err := determineBestNetworkInterface()
@@ -128,7 +124,11 @@ func preRunJoin(flags *JoinCmdFlags) error {
 }
 
 func addJoinFlags(cmd *cobra.Command, flags *JoinCmdFlags) error {
-	cmd.Flags().StringVar(&flags.airgapBundle, "airgap-bundle", "", "Path to the air gap bundle. If set, the installation will complete without internet access.")
+	cmd.Flags().String("airgap-bundle", "", "Path to the air gap bundle. If set, the installation will complete without internet access.")
+	if err := cmd.Flags().MarkDeprecated("airgap-bundle", "This flag is deprecated (ignored) and will be removed in a future version. The cluster will automatically determine if it's in airgap mode and fetch the necessary artifacts from other nodes."); err != nil {
+		return err
+	}
+
 	cmd.Flags().StringVar(&flags.networkInterface, "network-interface", "", "The network interface to use for the cluster")
 	cmd.Flags().BoolVar(&flags.ignoreHostPreflights, "ignore-host-preflights", false, "Run host preflight checks, but prompt the user to continue if they fail instead of exiting.")
 	cmd.Flags().BoolVar(&flags.noHA, "no-ha", false, "Do not prompt for or enable high availability.")
@@ -147,7 +147,7 @@ func addJoinFlags(cmd *cobra.Command, flags *JoinCmdFlags) error {
 	return nil
 }
 
-func runJoin(ctx context.Context, name string, flags JoinCmdFlags, jcmd *join.JoinCommandResponse, metricsReporter preflights.MetricsReporter) error {
+func runJoin(ctx context.Context, name string, flags JoinCmdFlags, jcmd *join.JoinCommandResponse, kotsAPIAddress string, metricsReporter preflights.MetricsReporter) error {
 	// both controller and worker nodes will have 'worker' in the join command
 	isWorker := !strings.Contains(jcmd.K0sJoinCommand, "controller")
 	if !isWorker {
@@ -158,7 +158,7 @@ func runJoin(ctx context.Context, name string, flags JoinCmdFlags, jcmd *join.Jo
 		return err
 	}
 
-	cidrCfg, err := initializeJoin(ctx, name, flags, jcmd)
+	cidrCfg, err := initializeJoin(ctx, name, jcmd, kotsAPIAddress)
 	if err != nil {
 		return fmt.Errorf("unable to initialize join: %w", err)
 	}
@@ -221,18 +221,6 @@ func runJoinVerifyAndPrompt(name string, flags JoinCmdFlags, jcmd *join.JoinComm
 		return err
 	}
 
-	err = verifyChannelRelease("join", flags.isAirgap, flags.assumeYes)
-	if err != nil {
-		return err
-	}
-
-	if flags.isAirgap {
-		logrus.Debugf("checking airgap bundle matches binary")
-		if err := checkAirgapMatches(flags.airgapBundle); err != nil {
-			return err // we want the user to see the error message without a prefix
-		}
-	}
-
 	runtimeconfig.Set(jcmd.InstallationSpec.RuntimeConfig)
 	isWorker := !strings.Contains(jcmd.K0sJoinCommand, "controller")
 	if isWorker {
@@ -282,7 +270,7 @@ func runJoinVerifyAndPrompt(name string, flags JoinCmdFlags, jcmd *join.JoinComm
 	return nil
 }
 
-func initializeJoin(ctx context.Context, name string, flags JoinCmdFlags, jcmd *join.JoinCommandResponse) (cidrCfg *CIDRConfig, err error) {
+func initializeJoin(ctx context.Context, name string, jcmd *join.JoinCommandResponse, kotsAPIAddress string) (cidrCfg *CIDRConfig, err error) {
 	logrus.Info("")
 	spinner := spinner.Start()
 	spinner.Infof("Initializing")
@@ -305,8 +293,8 @@ func initializeJoin(ctx context.Context, name string, flags JoinCmdFlags, jcmd *
 	}
 
 	logrus.Debugf("materializing %s binaries", name)
-	if err := materializeFiles(flags.airgapBundle); err != nil {
-		return nil, err
+	if err := materializeFilesForJoin(ctx, jcmd, kotsAPIAddress); err != nil {
+		return nil, fmt.Errorf("failed to materialize files: %w", err)
 	}
 
 	logrus.Debugf("configuring sysctl")
@@ -337,6 +325,24 @@ func initializeJoin(ctx context.Context, name string, flags JoinCmdFlags, jcmd *
 	return cidrCfg, nil
 }
 
+func materializeFilesForJoin(ctx context.Context, jcmd *join.JoinCommandResponse, kotsAPIAddress string) error {
+	materializer := goods.NewMaterializer()
+	if err := materializer.Materialize(); err != nil {
+		return fmt.Errorf("materialize binaries: %w", err)
+	}
+	if err := support.MaterializeSupportBundleSpec(); err != nil {
+		return fmt.Errorf("materialize support bundle spec: %w", err)
+	}
+
+	if jcmd.InstallationSpec.AirGap {
+		if err := airgap.FetchAndWriteArtifacts(ctx, kotsAPIAddress); err != nil {
+			return fmt.Errorf("failed to fetch artifacts: %w", err)
+		}
+	}
+
+	return nil
+}
+
 func getJoinCIDRConfig(jcmd *join.JoinCommandResponse) (*CIDRConfig, error) {
 	podCIDR, serviceCIDR, err := netutils.SplitNetworkCIDR(ecv1beta1.DefaultNetworkCIDR)
 	if err != nil {
@@ -400,7 +406,7 @@ func installAndJoinCluster(ctx context.Context, jcmd *join.JoinCommandResponse,
 		return fmt.Errorf("unable to join node to cluster: %w", err)
 	}
 
-	if err := startAndWaitForK0s(ctx, name, jcmd); err != nil {
+	if err := startAndWaitForK0s(name); err != nil {
 		return err
 	}
 
@@ -464,7 +470,7 @@ func applyNetworkConfiguration(networkInterface string, jcmd *join.JoinCommandRe
 }
 
 // startAndWaitForK0s starts the k0s service and waits for the node to be ready.
-func startAndWaitForK0s(ctx context.Context, name string, jcmd *join.JoinCommandResponse) error {
+func startAndWaitForK0s(name string) error {
 	logrus.Debugf("starting %s service", name)
 	if _, err := helpers.RunCommand(runtimeconfig.K0sBinaryPath(), "start"); err != nil {
 		return fmt.Errorf("unable to start service: %w", err)
@@ -600,7 +606,7 @@ func maybeEnableHA(ctx context.Context, kcli client.Client, flags JoinCmdFlags,
 	}
 
 	airgapChartsPath := ""
-	if flags.isAirgap {
+	if jcmd.InstallationSpec.AirGap {
 		airgapChartsPath = runtimeconfig.EmbeddedClusterChartsSubDir()
 	}
 	hcli, err := helm.NewClient(helm.HelmOptions{

cmd/installer/cli/join_runpreflights.go

@@ -39,7 +39,7 @@ func JoinRunPreflightsCmd(ctx context.Context, name string) *cobra.Command {
 			if err != nil {
 				return fmt.Errorf("unable to get join token: %w", err)
 			}
-			if err := runJoinRunPreflights(cmd.Context(), name, flags, jcmd); err != nil {
+			if err := runJoinRunPreflights(cmd.Context(), name, flags, jcmd, args[0]); err != nil {
 				return err
 			}
 
@@ -54,14 +54,14 @@ func JoinRunPreflightsCmd(ctx context.Context, name string) *cobra.Command {
 	return cmd
 }
 
-func runJoinRunPreflights(ctx context.Context, name string, flags JoinCmdFlags, jcmd *join.JoinCommandResponse) error {
+func runJoinRunPreflights(ctx context.Context, name string, flags JoinCmdFlags, jcmd *join.JoinCommandResponse, kotsAPIAddress string) error {
 	if err := runJoinVerifyAndPrompt(name, flags, jcmd); err != nil {
 		return err
 	}
 
 	logrus.Debugf("materializing %s binaries", name)
-	if err := materializeFiles(flags.airgapBundle); err != nil {
-		return err
+	if err := materializeFilesForJoin(ctx, jcmd, kotsAPIAddress); err != nil {
+		return fmt.Errorf("failed to materialize files: %w", err)
 	}
 
 	logrus.Debugf("configuring sysctl")
@@ -107,7 +107,7 @@ func runJoinPreflights(ctx context.Context, jcmd *join.JoinCommandResponse, flag
 		PodCIDR:                cidrCfg.PodCIDR,
 		ServiceCIDR:            cidrCfg.ServiceCIDR,
 		NodeIP:                 nodeIP,
-		IsAirgap:               flags.isAirgap,
+		IsAirgap:               jcmd.InstallationSpec.AirGap,
 		SkipHostPreflights:     flags.skipHostPreflights,
 		IgnoreHostPreflights:   flags.ignoreHostPreflights,
 		AssumeYes:              flags.assumeYes,

dev/dockerfiles/local-artifact-mirror/Dockerfile.ttlsh

@@ -1,4 +1,4 @@
-FROM golang:1.24 AS build
+FROM golang:1.24.2 AS build
 
 WORKDIR /app
 

dev/dockerfiles/operator/Dockerfile.local

@@ -1,4 +1,4 @@
-FROM golang:1.24-alpine AS build
+FROM golang:1.24.2-alpine AS build
 
 RUN apk add --no-cache ca-certificates curl git make bash
 

dev/dockerfiles/operator/Dockerfile.ttlsh

@@ -1,4 +1,4 @@
-FROM golang:1.24 AS build
+FROM golang:1.24.2 AS build
 
 WORKDIR /app
 

pkg/addons/adminconsole/static/metadata.yaml

@@ -5,24 +5,24 @@
 # $ make buildtools
 # $ output/bin/buildtools update addon <addon name>
 #
-version: 1.124.15-ec.2
+version: 1.124.16-ec.0
 location: oci://proxy.replicated.com/anonymous/registry.replicated.com/library/admin-console
 images:
     kotsadm:
         repo: proxy.replicated.com/anonymous/kotsadm/kotsadm
         tag:
-            amd64: v1.124.15-ec.2-amd64@sha256:55647c859506b1462cac40078d2279bb5ea284a39d7b8ab5c8caceea8cf57382
-            arm64: v1.124.15-ec.2-arm64@sha256:bc4608169f30250d2b997a28fbd59711cdf975766332110776c52b4d87563db5
+            amd64: v1.124.16-ec.0-amd64@sha256:30bf71416ac4e28f343fc31b72e19fead80e3c833135b6175f6bf85f8a72916a
+            arm64: v1.124.16-ec.0-arm64@sha256:8b99803ed1ae94889f93646b61e991988293e5dbc253f76d7adf3185403c51af
     kotsadm-migrations:
         repo: proxy.replicated.com/anonymous/kotsadm/kotsadm-migrations
         tag:
-            amd64: v1.124.15-ec.2-amd64@sha256:8563ebe149f917e6030737e5e8b58b8a7796f47638ff87546d1d33c26bb27c80
-            arm64: v1.124.15-ec.2-arm64@sha256:4d72708542edfc5956aa815fe106e7172aee98550cc51e90dad8eb34300b55f5
+            amd64: v1.124.16-ec.0-amd64@sha256:a9f0a7313e43579f33a2cf776d4a6096e4f942545cda4b0fe2a4fd48de81ffcc
+            arm64: v1.124.16-ec.0-arm64@sha256:0cdab3221eeb9aedf7a70e915b2adfad72d484b4dd147401ee563fc8959cd530
     kurl-proxy:
         repo: proxy.replicated.com/anonymous/kotsadm/kurl-proxy
         tag:
-            amd64: v1.124.15-ec.2-amd64@sha256:0dc979da23442b7e4ae53fa46fbbbb1835c4f186b2e9d4d1f11000431a6dd6ad
-            arm64: v1.124.15-ec.2-arm64@sha256:4d369bee17c270991f5daee2b01762be7caee1dd2882b8a320902e910cbd05cc
+            amd64: v1.124.16-ec.0-amd64@sha256:67555f18b467d96c886aee8b868acb3ba719ad0ff9562d3fb87aaa093338adcd
+            arm64: v1.124.16-ec.0-arm64@sha256:677d752ce90e1831f58868c618322720affe21443c5c638d2d4b416860828402
     rqlite:
         repo: proxy.replicated.com/anonymous/kotsadm/rqlite
         tag:

pkg/airgap/materialize.go

@@ -3,15 +3,17 @@ package airgap
 import (
 	"archive/tar"
 	"compress/gzip"
+	"context"
 	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 
+	"github.com/replicatedhq/embedded-cluster/pkg/kotsadm"
 	"github.com/replicatedhq/embedded-cluster/pkg/runtimeconfig"
 )
 
-const K0sImagePath = "images/images-amd64.tar"
+const K0sImagePath = "images/ec-images-amd64.tar"
 
 // MaterializeAirgap places the airgap image bundle for k0s and the embedded cluster charts on disk.
 // - image bundle should be located at 'images-amd64.tar' within the embedded-cluster directory within the airgap bundle.
@@ -58,6 +60,33 @@ func MaterializeAirgap(airgapReader io.Reader) error {
 	}
 }
 
+// FetchAndWriteArtifacts fetches the k0s images and Helm charts from the KOTS API
+// and writes them to the appropriate directories
+func FetchAndWriteArtifacts(ctx context.Context, kotsAPIAddress string) error {
+	// Fetch and write k0s images
+	imagesFile, err := kotsadm.GetK0sImagesFile(ctx, kotsAPIAddress)
+	if err != nil {
+		return fmt.Errorf("failed to get k0s images file: %w", err)
+	}
+	defer imagesFile.Close()
+
+	if err := writeOneFile(imagesFile, filepath.Join(runtimeconfig.EmbeddedClusterK0sSubDir(), K0sImagePath), 0644); err != nil {
+		return fmt.Errorf("failed to write k0s images file: %w", err)
+	}
+
+	// Fetch and write Helm charts
+	chartsTGZ, err := kotsadm.GetECCharts(ctx, kotsAPIAddress)
+	if err != nil {
+		return fmt.Errorf("failed to get ec charts: %w", err)
+	}
+	defer chartsTGZ.Close()
+
+	if err := writeChartFiles(chartsTGZ); err != nil {
+		return fmt.Errorf("failed to write chart files: %w", err)
+	}
+	return nil
+}
+
 func writeOneFile(reader io.Reader, path string, mode int64) error {
 	// setup destination
 	if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {

pkg/dryrun/kotsadm.go

@@ -3,6 +3,7 @@ package dryrun
 import (
 	"context"
 	"fmt"
+	"io"
 	"reflect"
 	"strings"
 
@@ -45,21 +46,57 @@ func (c *Kotsadm) setResponse(resp interface{}, err error, methodName string, ar
 	return nil
 }
 
-// SetGetJoinTokenResponse sets the response for the GetJoinToken method, based on the provided address and shortToken.
-func (c *Kotsadm) SetGetJoinTokenResponse(address, shortToken string, resp *join.JoinCommandResponse, err error) {
-	mockErr := c.setResponse(resp, err, "GetJoinToken", address, shortToken)
+// SetGetJoinTokenResponse sets the response for the GetJoinToken method, based on the provided kotsAPIAddress and shortToken.
+func (c *Kotsadm) SetGetJoinTokenResponse(kotsAPIAddress, shortToken string, resp *join.JoinCommandResponse, err error) {
+	mockErr := c.setResponse(resp, err, "GetJoinToken", kotsAPIAddress, shortToken)
 	if mockErr != nil {
 		panic(mockErr)
 	}
 }
 
 // GetJoinToken issues a request to the kots api to get the actual join command
 // based on the short token provided by the user.
-func (c *Kotsadm) GetJoinToken(ctx context.Context, address, shortToken string) (*join.JoinCommandResponse, error) {
-	key := strings.Join([]string{"GetJoinToken", address, shortToken}, ":")
+func (c *Kotsadm) GetJoinToken(ctx context.Context, kotsAPIAddress, shortToken string) (*join.JoinCommandResponse, error) {
+	key := strings.Join([]string{"GetJoinToken", kotsAPIAddress, shortToken}, ":")
 	if handler, ok := c.mockHandlers[key]; ok {
 		return handler.resp.(*join.JoinCommandResponse), handler.err
 	} else {
-		return nil, fmt.Errorf("no response set for GetJoinToken, address: %s, shortToken: %s", address, shortToken)
+		return nil, fmt.Errorf("no response set for GetJoinToken, kotsAPIAddress: %s, shortToken: %s", kotsAPIAddress, shortToken)
+	}
+}
+
+func (c *Kotsadm) SetGetK0sImagesFileResponse(kotsAPIAddress string, resp io.ReadCloser, err error) {
+	mockErr := c.setResponse(resp, err, "GetK0sImagesFile", kotsAPIAddress)
+	if mockErr != nil {
+		panic(mockErr)
+	}
+}
+
+// GetK0sImagesFile fetches the k0s images file from the KOTS API.
+// caller is responsible for closing the response body.
+func (c *Kotsadm) GetK0sImagesFile(ctx context.Context, kotsAPIAddress string) (io.ReadCloser, error) {
+	key := strings.Join([]string{"GetK0sImagesFile", kotsAPIAddress}, ":")
+	if handler, ok := c.mockHandlers[key]; ok {
+		return handler.resp.(io.ReadCloser), handler.err
+	} else {
+		return nil, fmt.Errorf("no response set for GetK0sImagesFile, kotsAPIAddress: %s", kotsAPIAddress)
+	}
+}
+
+func (c *Kotsadm) SetGetECChartsResponse(kotsAPIAddress string, resp io.ReadCloser, err error) {
+	mockErr := c.setResponse(resp, err, "GetECCharts", kotsAPIAddress)
+	if mockErr != nil {
+		panic(mockErr)
+	}
+}
+
+// GetECCharts fetches the Helm charts file from the KOTS API.
+// caller is responsible for closing the response body.
+func (c *Kotsadm) GetECCharts(ctx context.Context, kotsAPIAddress string) (io.ReadCloser, error) {
+	key := strings.Join([]string{"GetECCharts", kotsAPIAddress}, ":")
+	if handler, ok := c.mockHandlers[key]; ok {
+		return handler.resp.(io.ReadCloser), handler.err
+	} else {
+		return nil, fmt.Errorf("no response set for GetECCharts, kotsAPIAddress: %s", kotsAPIAddress)
 	}
 }