add airgap support to installer (#435)

* add airgap flag. disable metrics when airgap flag is passed

* add 'materialize airgap files' function

* tell k0s to find helm charts on the host

* wip create configmaps for airgap installation

* ensure automated install gets run

* flesh out configmap creation code

* call configmap creation before waiting for charts

* begin app configmap test

* finish app configmap test

* implement app airgap bundle version checker

* lint fix

* f

* check that the airgap bundle matches the binary before installing

* check airgap bundle version on join, and ensure version matches too

* set airgap values in helm chart

* implement MaterializeAirgapImages

* wip registry and seaweed

* install seaweedfs

* wait for ns first

* add chart to expected metadata

* fix seaweedfs storageclass

* disable filer, fix total seaweedfs wait count, fix s3 config

* reenable filer

* disable unsupported overrides test until I have all the new charts worked out

* use hostpaths

* expose seaweed creds to other addons

* no master/filer storage?

* kepe master hostpath

* add registry?

* fixups

* give filer a pvc again

* registry chart registry name

* fix secret

* override the name

* airgap chart metadata

* remove tarball

here you go Chris

* fix unit test

* f

* airgap test file

* a testing change

* update materialize code

* materialize proper airgap file format

* all pvcs for seaweed

* fixes

* materialize both the docker images and helm charts

* include the app release semver in the helm chart path

* reenable unsupported overrides test

* automatically update charts

* lolgic

* include chart versions in filenames, don't use app version dirs

* create kotsadm namespace in configmap upload

* hostpath registry

* fix version test

* run airgap outros too

* update registry config

* fix configmap creation log message

* real registry user/pw

* rework airgap bundle format

* update registry name

* improve spinners

* due to name change registry comes before operator

* remove seaweed labels

* f

* fix version

* typo

* use real registry IP

* labels

* simplify registry address code

* add insecure registry config, fix registry secret type

* fix admin console code

* reorder imports

* fix registry config to allow writing

* run registry as root instead of chmoding the directory

* rename constants

* rename 'airgap' flag to 'airgap-bundle', and hide it

* import external embedded cluster release metadata type

* rename functions

* require embedded release when installing airgap

Author: laverya

.github/workflows/dependencies.yaml

@@ -27,6 +27,18 @@ jobs:
       run: |
         export VERSION=`curl https://api.github.com/repos/openebs/charts/releases/latest | jq -r .name | tr -d openebs-`
         sed -i "/^OPENEBS_CHART_VERSION/c\OPENEBS_CHART_VERSION = $VERSION" Makefile
+    - name: SeaweedFS
+      run: |
+        export VERSION=`curl https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/k8s/charts/seaweedfs/Chart.yaml | grep version | tr -d 'version: '`
+        sed -i "/^SEAWEEDFS_CHART_VERSION/c\SEAWEEDFS_CHART_VERSION = $VERSION" Makefile
+    - name: Registry Chart
+      run: |
+        export VERSION=`curl https://api.github.com/repos/twuni/docker-registry.helm/tags | jq -r .[].name | head -1 | tr -d v`
+        sed -i "/^REGISTRY_CHART_VERSION/c\REGISTRY_CHART_VERSION = $VERSION" Makefile
+    - name: Registry Image
+      run: |
+        export VERSION=`curl https://api.github.com/repos/distribution/distribution/tags | jq -r '.[].name'  | tr -d v | grep -v alpha | grep -v beta | head -n 1`
+        sed -i "/^REGISTRY_IMAGE_VERSION/c\REGISTRY_IMAGE_VERSION = $VERSION" Makefile
     - name: Kubectl
       run: |
         export VERSION=`curl -L -s https://dl.k8s.io/release/stable.txt`

Makefile

@@ -13,6 +13,10 @@ EMBEDDED_OPERATOR_CHART_VERSION = 0.24.1
 OPENEBS_CHART_URL = https://openebs.github.io/charts
 OPENEBS_CHART_NAME = openebs/openebs
 OPENEBS_CHART_VERSION = 3.10.0
+REGISTRY_CHART_URL = https://helm.twun.io
+REGISTRY_CHART_NAME = twuni/docker-registry
+REGISTRY_CHART_VERSION = 2.2.2
+REGISTRY_IMAGE_VERSION = 2.8.3
 KUBECTL_VERSION = v1.29.2
 K0S_VERSION = v1.29.2+k0s.0
 K0S_BINARY_SOURCE_OVERRIDE =
@@ -32,7 +36,11 @@ LD_FLAGS = -X github.com/replicatedhq/embedded-cluster/pkg/defaults.K0sVersion=$
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/embeddedclusteroperator.Version=$(EMBEDDED_OPERATOR_CHART_VERSION) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/openebs.ChartURL=$(OPENEBS_CHART_URL) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/openebs.ChartName=$(OPENEBS_CHART_NAME) \
-	-X github.com/replicatedhq/embedded-cluster/pkg/addons/openebs.Version=$(OPENEBS_CHART_VERSION)
+	-X github.com/replicatedhq/embedded-cluster/pkg/addons/openebs.Version=$(OPENEBS_CHART_VERSION) \
+	-X github.com/replicatedhq/embedded-cluster/pkg/addons/registry.ChartURL=$(REGISTRY_CHART_URL) \
+	-X github.com/replicatedhq/embedded-cluster/pkg/addons/registry.ChartName=$(REGISTRY_CHART_NAME) \
+	-X github.com/replicatedhq/embedded-cluster/pkg/addons/registry.Version=$(REGISTRY_CHART_VERSION) \
+	-X github.com/replicatedhq/embedded-cluster/pkg/addons/registry.ImageVersion=$(REGISTRY_IMAGE_VERSION)
 
 .DEFAULT_GOAL := default
 default: embedded-cluster-linux-amd64

cmd/embedded-cluster/install.go

@@ -14,10 +14,12 @@ import (
 	k8syaml "sigs.k8s.io/yaml"
 
 	"github.com/replicatedhq/embedded-cluster/pkg/addons"
+	"github.com/replicatedhq/embedded-cluster/pkg/airgap"
 	"github.com/replicatedhq/embedded-cluster/pkg/config"
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
 	"github.com/replicatedhq/embedded-cluster/pkg/goods"
 	"github.com/replicatedhq/embedded-cluster/pkg/helpers"
+	"github.com/replicatedhq/embedded-cluster/pkg/kubeutils"
 	"github.com/replicatedhq/embedded-cluster/pkg/metrics"
 	"github.com/replicatedhq/embedded-cluster/pkg/preflights"
 	"github.com/replicatedhq/embedded-cluster/pkg/prompts"
@@ -52,7 +54,7 @@ func runCommand(bin string, args ...string) (string, error) {
 }
 
 // installAndEnableLocalArtifactMirror installs and enables the local artifact mirror. This
-// service is reponsible for serving on localhost, through http, all files that are used
+// service is responsible for serving on localhost, through http, all files that are used
 // during a cluster upgrade.
 func installAndEnableLocalArtifactMirror() error {
 	ourbin := defaults.PathToEmbeddedClusterBinary("local-artifact-mirror")
@@ -179,7 +181,73 @@ func checkLicenseMatches(c *cli.Context) error {
 	}
 
 	return nil
+}
+
+func checkAirgapMatches(c *cli.Context) error {
+	rel, err := release.GetChannelRelease()
+	if err != nil {
+		return fmt.Errorf("failed to get release from binary: %w", err) // this should only be if the release is malformed
+	}
+	if rel == nil {
+		return fmt.Errorf("airgap bundle provided but no release was found in binary, please rerun without the airgap-bundle flag")
+	}
+
+	// read file from path
+	rawfile, err := os.Open(c.String("airgap-bundle"))
+	if err != nil {
+		return fmt.Errorf("failed to open airgap file: %w", err)
+	}
+	defer rawfile.Close()
+
+	appSlug, channelID, airgapVersion, err := airgap.ChannelReleaseMetadata(rawfile)
+	if err != nil {
+		return fmt.Errorf("failed to get airgap bundle versions: %w", err)
+	}
+
+	// Check if the airgap bundle matches the application version data
+	if rel.AppSlug != appSlug {
+		// if the app is different, we will not be able to provide the correct vendor supplied charts and k0s overrides
+		return fmt.Errorf("airgap bundle app %s does not match binary app %s, please provide the correct bundle", appSlug, rel.AppSlug)
+	}
+	if rel.ChannelID != channelID {
+		// if the channel is different, we will not be able to install the pinned vendor application version within kots
+		return fmt.Errorf("airgap bundle channel %s does not match binary channel %s, please provide the correct bundle", channelID, rel.ChannelID)
+	}
+	if rel.VersionLabel != airgapVersion {
+		// if the version is different, who knows what might be different
+		return fmt.Errorf("airgap bundle version %s does not match binary version %s, please provide the correct bundle", airgapVersion, rel.VersionLabel)
+	}
+
+	return nil
+}
+
+func materializeFiles(c *cli.Context) error {
+	mat := spinner.Start()
+	defer mat.Close()
+	mat.Infof("Materializing files")
+
+	if err := goods.Materialize(); err != nil {
+		return fmt.Errorf("unable to materialize binaries: %w", err)
+	}
+	if c.String("airgap-bundle") != "" {
+		mat.Infof("Materializing airgap installation files")
+
+		// read file from path
+		rawfile, err := os.Open(c.String("airgap-bundle"))
+		if err != nil {
+			return fmt.Errorf("failed to open airgap file: %w", err)
+		}
+		defer rawfile.Close()
 
+		if err := airgap.MaterializeAirgap(rawfile); err != nil {
+			err = fmt.Errorf("unable to materialize airgap files: %w", err)
+			return err
+		}
+	}
+
+	mat.Infof("Host files materialized")
+
+	return nil
 }
 
 // createK0sConfig creates a new k0s.yaml configuration file. The file is saved in the
@@ -205,13 +273,21 @@ func ensureK0sConfig(c *cli.Context) error {
 		}
 		opts = append(opts, addons.WithLicense(license))
 	}
+	if c.String("airgap-bundle") != "" {
+		opts = append(opts, addons.Airgap())
+	}
 	if err := config.UpdateHelmConfigs(cfg, opts...); err != nil {
 		return fmt.Errorf("unable to update helm configs: %w", err)
 	}
 	var err error
 	if cfg, err = applyUnsupportedOverrides(c, cfg); err != nil {
 		return fmt.Errorf("unable to apply unsupported overrides: %w", err)
 	}
+	if c.String("airgap-bundle") != "" {
+		// update the k0s config to install with airgap
+		airgap.RemapHelm(cfg)
+		airgap.SetAirgapConfig(cfg)
+	}
 	data, err := k8syaml.Marshal(cfg)
 	if err != nil {
 		return fmt.Errorf("unable to marshal config: %w", err)
@@ -296,6 +372,32 @@ func waitForK0s() error {
 	return nil
 }
 
+// createAirgapConfigMaps creates the airgap configmaps in the k8s cluster from the airgap file.
+func createAirgapConfigMaps(c *cli.Context) error {
+	loading := spinner.Start()
+	defer loading.Close()
+	loading.Infof("Creating airgap configmaps")
+	// create k8s client
+	os.Setenv("KUBECONFIG", defaults.PathToKubeConfig())
+	cli, err := kubeutils.KubeClient()
+	if err != nil {
+		return fmt.Errorf("failed to create k8s client: %w", err)
+	}
+
+	// read file from path
+	rawfile, err := os.Open(c.String("airgap-bundle"))
+	if err != nil {
+		return fmt.Errorf("failed to open airgap file: %w", err)
+	}
+	defer rawfile.Close()
+
+	if err = airgap.CreateAppConfigMaps(c.Context, cli, rawfile); err != nil {
+		return fmt.Errorf("unable to create airgap configmaps: %w", err)
+	}
+	loading.Infof("Airgap configmaps created")
+	return nil
+}
+
 // runOutro calls Outro() in all enabled addons by means of Applier.
 func runOutro(c *cli.Context) error {
 	os.Setenv("KUBECONFIG", defaults.PathToKubeConfig())
@@ -314,6 +416,9 @@ func runOutro(c *cli.Context) error {
 		}
 		opts = append(opts, addons.WithEndUserConfig(eucfg))
 	}
+	if c.String("airgap-bundle") != "" {
+		opts = append(opts, addons.Airgap())
+	}
 	return addons.NewApplier(opts...).Outro(c.Context)
 }
 
@@ -327,6 +432,10 @@ var installCommand = &cli.Command{
 		if os.Getuid() != 0 {
 			return fmt.Errorf("install command must be run as root")
 		}
+
+		if c.String("airgap-bundle") != "" {
+			metrics.DisableMetrics()
+		}
 		return nil
 	},
 	Flags: []cli.Flag{
@@ -346,6 +455,11 @@ var installCommand = &cli.Command{
 			Usage:   "Path to the application license file",
 			Hidden:  false,
 		},
+		&cli.StringFlag{
+			Name:   "airgap-bundle",
+			Usage:  "Path to the airgap bundle. If set, the installation will be completed without internet access.",
+			Hidden: false,
+		},
 	},
 	Action: func(c *cli.Context) error {
 		logrus.Debugf("checking if %s is already installed", binName)
@@ -365,9 +479,14 @@ var installCommand = &cli.Command{
 			metrics.ReportApplyFinished(c, metricErr)
 			return err // do not return the metricErr, as we want the user to see the error message without a prefix
 		}
+		if c.String("airgap-bundle") != "" {
+			logrus.Debugf("checking airgap bundle matches binary")
+			if err := checkAirgapMatches(c); err != nil {
+				return err // we want the user to see the error message without a prefix
+			}
+		}
 		logrus.Debugf("materializing binaries")
-		if err := goods.Materialize(); err != nil {
-			err := fmt.Errorf("unable to materialize binaries: %w", err)
+		if err := materializeFiles(c); err != nil {
 			metrics.ReportApplyFinished(c, err)
 			return err
 		}
@@ -401,6 +520,13 @@ var installCommand = &cli.Command{
 			metrics.ReportApplyFinished(c, err)
 			return err
 		}
+		if c.String("airgap-bundle") != "" {
+			err := createAirgapConfigMaps(c)
+			if err != nil {
+				err = fmt.Errorf("unable to create airgap configmaps: %w", err)
+				return err
+			}
+		}
 		logrus.Debugf("running outro")
 		if err := runOutro(c); err != nil {
 			metrics.ReportApplyFinished(c, err)

cmd/embedded-cluster/join.go

@@ -19,9 +19,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8syaml "sigs.k8s.io/yaml"
 
+	"github.com/replicatedhq/embedded-cluster/pkg/airgap"
 	"github.com/replicatedhq/embedded-cluster/pkg/config"
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
-	"github.com/replicatedhq/embedded-cluster/pkg/goods"
 	"github.com/replicatedhq/embedded-cluster/pkg/helpers"
 	"github.com/replicatedhq/embedded-cluster/pkg/metrics"
 )
@@ -34,6 +34,7 @@ type JoinCommandResponse struct {
 	K0sUnsupportedOverrides   string    `json:"k0sUnsupportedOverrides"`
 	EndUserK0sConfigOverrides string    `json:"endUserK0sConfigOverrides"`
 	MetricsBaseURL            string    `json:"metricsBaseURL"`
+	AirgapRegistryAddress     string    `json:"airgapRegistryAddress"`
 }
 
 // extractK0sConfigOverridePatch parses the provided override and returns a dig.Mapping that
@@ -98,10 +99,21 @@ var joinCommand = &cli.Command{
 	Name:      "join",
 	Usage:     fmt.Sprintf("Join the current node to a %s cluster", binName),
 	ArgsUsage: "<url> <token>",
+	Flags: []cli.Flag{
+		&cli.StringFlag{
+			Name:   "airgap-bundle",
+			Usage:  "Path to the airgap bundle. If set, the installation will be completed without internet access.",
+			Hidden: true,
+		},
+	},
 	Before: func(c *cli.Context) error {
 		if os.Getuid() != 0 {
 			return fmt.Errorf("node join command must be run as root")
 		}
+
+		if c.String("airgap-bundle") != "" {
+			metrics.DisableMetrics()
+		}
 		return nil
 	},
 	Action: func(c *cli.Context) error {
@@ -126,10 +138,16 @@ var joinCommand = &cli.Command{
 			return fmt.Errorf("unable to get join token: %w", err)
 		}
 
+		if c.String("airgap-bundle") != "" {
+			logrus.Debugf("checking airgap bundle matches binary")
+			if err := checkAirgapMatches(c); err != nil {
+				return err // we want the user to see the error message without a prefix
+			}
+		}
+
 		metrics.ReportJoinStarted(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID)
 		logrus.Infof("Materializing %s binaries", binName)
-		if err := goods.Materialize(); err != nil {
-			err := fmt.Errorf("unable to materialize binaries: %w", err)
+		if err := materializeFiles(c); err != nil {
 			metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
 			return err
 		}
@@ -154,6 +172,13 @@ var joinCommand = &cli.Command{
 			return err
 		}
 
+		if jcmd.AirgapRegistryAddress != "" {
+			if err := airgap.AddInsecureRegistry(jcmd.AirgapRegistryAddress); err != nil {
+				err := fmt.Errorf("unable to add insecure registry: %w", err)
+				metrics.ReportJoinFailed(c.Context, jcmd.MetricsBaseURL, jcmd.ClusterID, err)
+			}
+		}
+
 		logrus.Infof("Joining node to cluster")
 		if err := runK0sInstallCommand(jcmd.K0sJoinCommand); err != nil {
 			err := fmt.Errorf("unable to join node to cluster: %w", err)

cmd/embedded-cluster/types/types.go

@@ -9,9 +9,9 @@ import (
 	"github.com/jedib0t/go-pretty/table"
 	"github.com/k0sproject/k0s/pkg/airgap"
 	k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
+	"github.com/replicatedhq/embedded-cluster-kinds/types"
 	"github.com/urfave/cli/v2"
 
-	"github.com/replicatedhq/embedded-cluster/cmd/embedded-cluster/types"
 	"github.com/replicatedhq/embedded-cluster/pkg/addons"
 	"github.com/replicatedhq/embedded-cluster/pkg/config"
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
@@ -98,6 +98,17 @@ var metadataCommand = &cli.Command{
 		}
 		meta.Protected = protectedFields
 
+		// Airgap
+		airgapCht, airgapRepo, err := applier.GetAirgapCharts()
+		if err != nil {
+			return fmt.Errorf("unable to get airgap charts: %w", err)
+		}
+		meta.AirgapConfigs = k0sconfig.HelmExtensions{
+			ConcurrencyLevel: 1,
+			Charts:           airgapCht,
+			Repositories:     airgapRepo,
+		}
+
 		// Render k0s config to get the images contained within
 		k0sConfig := config.RenderK0sConfig()
 		if err != nil {

cmd/embedded-cluster/version.go

@@ -59,6 +59,7 @@ type Input struct {
 	LicensePath                       string
 	EmbeddedClusterPath               string
 	EmbeddedClusterReleaseBuilderPath string // used to replace the release in the binary
+	AirgapBundlePath                  string
 	Image                             string
 	network                           string
 	T                                 *testing.T
@@ -397,6 +398,11 @@ func CopyFilesToNode(in *Input, node string) {
 			DestPath:   "/usr/local/bin/embedded-cluster-release-builder",
 			Mode:       0755,
 		},
+		{
+			SourcePath: in.AirgapBundlePath,
+			DestPath:   "/tmp/release.airgap",
+			Mode:       0755,
+		},
 	}
 	scriptFiles, err := scripts.FS.ReadDir(".")
 	if err != nil {