Merge remote-tracking branch 'origin/main' into k0s-1-28-7

Author: laverya

.github/workflows/dependencies.yaml

@@ -11,6 +11,10 @@ jobs:
     steps:
     - name: Check out repo
       uses: actions/checkout@v4
+    - name: Set up Go
+      uses: actions/setup-go@v5
+      with:
+        go-version: "1.21"
     - name: AdminConsole
       run: |
         export VERSION=`curl https://api.github.com/repos/replicatedhq/kots-helm/tags | jq -r .[].name | grep -v alpha | head -1 | tr -d v`
@@ -35,6 +39,7 @@ jobs:
         if [ "$CURVERSION" != "$VERSION" ]; then
           sed -i "/^K0S_VERSION/c\K0S_VERSION = $VERSION" Makefile
           sed -i "/^K0S_BINARY_SOURCE_OVERRIDE/c\K0S_BINARY_SOURCE_OVERRIDE =" Makefile
+          go get github.com/k0sproject/k0s@${VERSION}
         fi
     - name: Troubleshoot
       run: |

.github/workflows/pull-request.yaml

@@ -1,6 +1,9 @@
 name: Pull request
 on:
 - pull_request
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
 jobs:
   sanitize:
     name: Sanitize
@@ -50,6 +53,8 @@ jobs:
     - name: Build Linux AMD64
       run: |
         make embedded-cluster-linux-amd64 VERSION=dev-$SHORT_SHA
+    - name: Output Metadata
+      run: |
         ./output/bin/embedded-cluster version metadata > metadata.json
     - name: Install Replicated CLI
       run: |
@@ -61,7 +66,6 @@ jobs:
         tar xf replicated.tar.gz replicated && rm replicated.tar.gz
         mv replicated /usr/local/bin/replicated
     - name: Create CI Releases
-      if: github.actor != 'dependabot[bot]'
       env:
         REPLICATED_APP: "embedded-cluster-smoke-test-staging-app"
         REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
@@ -72,10 +76,12 @@ jobs:
         sed -i "s/__version_string__/${SHORT_SHA}/g" e2e/kots-release-install/cluster-config.yaml
         sed -i "s/__version_string__/${SHORT_SHA}-upgrade/g" e2e/kots-release-upgrade/cluster-config.yaml
 
+        # re-promote a release containing an old version of embedded-cluster to test upgrades
+        replicated release promote 807 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${SHORT_SHA}-pre-minio-removal"
+
         replicated release create --yaml-dir e2e/kots-release-install --promote CI --version "${SHORT_SHA}"
         replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI --version "${SHORT_SHA}-upgrade"
     - name: Build CI binary
-      if: github.actor != 'dependabot[bot]'
       run: |
         export SHORT_SHA=$(git rev-parse --short=7 HEAD)
         echo "# channel release object" > e2e/kots-release-install/release.yaml
@@ -86,7 +92,6 @@ jobs:
         cp output/bin/embedded-cluster output/bin/embedded-cluster-original
         make embedded-release # this is done after the metadata.json is generated so as to not include additional charts
     - name: Cache files for integration test
-      if: github.actor != 'dependabot[bot]'
       env:
         S3_BUCKET: "tf-staging-embedded-cluster-bin"
         AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }}
@@ -128,6 +133,7 @@ jobs:
           - TestInstallWithoutEmbed
           - TestResetAndReinstall
           - TestCollectSupportBundle
+          - TestOldVersionUpgrade
     steps:
     - name: Checkout
       uses: actions/checkout@v4
@@ -137,7 +143,6 @@ jobs:
         name: embedded-release
         path: output/bin
     - uses: ./.github/actions/e2e
-      if: github.actor != 'dependabot[bot]'
       with:
         test-name: '${{ matrix.tests }}'
         license-id: ${{ secrets.STAGING_EMBEDDED_CLUSTER_LICENSE_ID }}

.github/workflows/release-dev.yaml

@@ -21,6 +21,8 @@ jobs:
         run: |
           make embedded-cluster-linux-amd64
           tar -C output/bin -czvf embedded-cluster-linux-amd64.tgz embedded-cluster
+      - name: Output Metadata
+        run: |
           ./output/bin/embedded-cluster version metadata > metadata.json
       - name: Publish development release
         uses: marvinpinto/action-automatic-releases@latest
@@ -42,7 +44,6 @@ jobs:
           tar xf replicated.tar.gz replicated && rm replicated.tar.gz
           mv replicated /usr/local/bin/replicated
       - name: Create CI Releases
-        if: github.actor != 'dependabot[bot]'
         env:
           REPLICATED_APP: "embedded-cluster-smoke-test-staging-app"
           REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }}
@@ -52,11 +53,13 @@ jobs:
           echo "${SHORT_SHA}"
           sed -i "s/__version_string__/${SHORT_SHA}/g" e2e/kots-release-install/cluster-config.yaml
           sed -i "s/__version_string__/${SHORT_SHA}-upgrade/g" e2e/kots-release-upgrade/cluster-config.yaml
+
+          # re-promote a release containing an old version of embedded-cluster to test upgrades
+          replicated release promote 807 2cHXb1RCttzpR0xvnNWyaZCgDBP --version "${SHORT_SHA}-pre-minio-removal"
           
           replicated release create --yaml-dir e2e/kots-release-install --promote CI --version "${SHORT_SHA}"
           replicated release create --yaml-dir e2e/kots-release-upgrade --promote CI --version "${SHORT_SHA}-upgrade"
       - name: Build CI binary
-        if: github.actor != 'dependabot[bot]'
         run: |
           export SHORT_SHA=$(git rev-parse --short=7 HEAD)
           echo "# channel release object" > e2e/kots-release-install/release.yaml
@@ -67,7 +70,6 @@ jobs:
           cp output/bin/embedded-cluster output/bin/embedded-cluster-original
           make embedded-release # this is done after the metadata.json is generated so as to not include additional charts
       - name: Cache files for integration test
-        if: github.actor != 'dependabot[bot]'
         env:
           S3_BUCKET: "tf-staging-embedded-cluster-bin"
           AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }}
@@ -109,6 +111,7 @@ jobs:
           - TestInstallFromReplicatedApp
           - TestResetAndReinstall
           - TestCollectSupportBundle
+          - TestOldVersionUpgrade
     steps:
       - name: Checkout
         uses: actions/checkout@v4

.github/workflows/release-prod.yaml

@@ -23,6 +23,8 @@ jobs:
         run: |
           make embedded-cluster-linux-amd64 VERSION=$TAG_NAME
           tar -C output/bin -czvf embedded-cluster-linux-amd64.tgz embedded-cluster
+      - name: Output Metadata
+        run: |
           ./output/bin/embedded-cluster version metadata > metadata.json
       - name: Cache Staging Files
         env:

Makefile

@@ -4,7 +4,7 @@ ARCH := $(shell uname -m)
 APP_NAME = embedded-cluster
 ADMIN_CONSOLE_CHART_URL = oci://registry.replicated.com/library
 ADMIN_CONSOLE_CHART_NAME = admin-console
-ADMIN_CONSOLE_CHART_VERSION = 1.107.8
+ADMIN_CONSOLE_CHART_VERSION = 1.108.0-build.1
 ADMIN_CONSOLE_IMAGE_OVERRIDE =
 ADMIN_CONSOLE_MIGRATIONS_IMAGE_OVERRIDE =
 EMBEDDED_OPERATOR_CHART_URL = oci://registry.replicated.com/library
@@ -20,6 +20,8 @@ TROUBLESHOOT_VERSION = v0.83.0
 LD_FLAGS = -X github.com/replicatedhq/embedded-cluster/pkg/defaults.K0sVersion=$(K0S_VERSION) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/defaults.Version=$(VERSION) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/defaults.K0sBinaryURL=$(K0S_BINARY_SOURCE_OVERRIDE) \
+	-X github.com/replicatedhq/embedded-cluster/pkg/defaults.TroubleshootVersion=$(TROUBLESHOOT_VERSION) \
+	-X github.com/replicatedhq/embedded-cluster/pkg/defaults.KubectlVersion=$(KUBECTL_VERSION) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.ChartURL=$(ADMIN_CONSOLE_CHART_URL) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.ChartName=$(ADMIN_CONSOLE_CHART_NAME) \
 	-X github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole.Version=$(ADMIN_CONSOLE_CHART_VERSION) \
@@ -79,20 +81,24 @@ output/bin/embedded-cluster-release-builder:
 embedded-release: embedded-cluster-linux-amd64 output/tmp/release.tar.gz output/bin/embedded-cluster-release-builder
 	./output/bin/embedded-cluster-release-builder output/bin/embedded-cluster output/tmp/release.tar.gz output/bin/embedded-cluster
 
+go.mod: Makefile
+	go get github.com/k0sproject/k0s@$(K0S_VERSION)
+	go mod tidy
+
 .PHONY: static
 static: pkg/goods/bins/k0s \
 	pkg/goods/bins/kubectl-preflight \
 	pkg/goods/bins/kubectl \
 	pkg/goods/bins/kubectl-support_bundle
 
 .PHONY: embedded-cluster-linux-amd64
-embedded-cluster-linux-amd64: static
+embedded-cluster-linux-amd64: static go.mod
 	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags "$(LD_FLAGS)" -o ./output/bin/$(APP_NAME) ./cmd/embedded-cluster
 
 # for testing
-.PHONY: embedded-cluster-darwin-amd64
-embedded-cluster-darwin-amd64:
-	CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags "$(LD_FLAGS)" -o ./output/bin/$(APP_NAME) ./cmd/embedded-cluster
+.PHONY: embedded-cluster-darwin-arm64
+embedded-cluster-darwin-arm64: go.mod
+	CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -ldflags "$(LD_FLAGS)" -o ./output/bin/$(APP_NAME) ./cmd/embedded-cluster
 
 .PHONY: unit-tests
 unit-tests:

cmd/embedded-cluster/install.go

@@ -162,18 +162,15 @@ func checkLicenseMatches(c *cli.Context) error {
 // createK0sConfig creates a new k0s.yaml configuration file. The file is saved in the
 // global location (as returned by defaults.PathToK0sConfig()). If a file already sits
 // there, this function returns an error.
-func ensureK0sConfig(c *cli.Context, useprompt bool) error {
+func ensureK0sConfig(c *cli.Context) error {
 	cfgpath := defaults.PathToK0sConfig()
 	if _, err := os.Stat(cfgpath); err == nil {
 		return fmt.Errorf("configuration file already exists")
 	}
 	if err := os.MkdirAll(filepath.Dir(cfgpath), 0755); err != nil {
 		return fmt.Errorf("unable to create directory: %w", err)
 	}
-	cfg, err := config.RenderK0sConfig(c.Context)
-	if err != nil {
-		return fmt.Errorf("unable to render config: %w", err)
-	}
+	cfg := config.RenderK0sConfig()
 	opts := []addons.Option{}
 	if c.Bool("no-prompt") {
 		opts = append(opts, addons.WithoutPrompt())
@@ -188,6 +185,7 @@ func ensureK0sConfig(c *cli.Context, useprompt bool) error {
 	if err := config.UpdateHelmConfigs(cfg, opts...); err != nil {
 		return fmt.Errorf("unable to update helm configs: %w", err)
 	}
+	var err error
 	if cfg, err = applyUnsupportedOverrides(c, cfg); err != nil {
 		return fmt.Errorf("unable to apply unsupported overrides: %w", err)
 	}
@@ -320,9 +318,10 @@ var installCommand = &cli.Command{
 			Hidden: true,
 		},
 		&cli.StringFlag{
-			Name:   "license",
-			Usage:  "Path to the application license file",
-			Hidden: false,
+			Name:    "license",
+			Aliases: []string{"l"},
+			Usage:   "Path to the application license file",
+			Hidden:  false,
 		},
 	},
 	Action: func(c *cli.Context) error {
@@ -356,7 +355,7 @@ var installCommand = &cli.Command{
 			return err
 		}
 		logrus.Debugf("creating k0s configuration file")
-		if err := ensureK0sConfig(c, !c.Bool("no-prompt")); err != nil {
+		if err := ensureK0sConfig(c); err != nil {
 			err := fmt.Errorf("unable to create config file: %w", err)
 			metrics.ReportApplyFinished(c, err)
 			return err

cmd/embedded-cluster/types/types.go

@@ -0,0 +1,14 @@
+package types
+
+import "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
+
+// ReleaseMetadata holds the metadata about a specific release, including addons and
+// their versions.
+type ReleaseMetadata struct {
+	Versions     map[string]string
+	K0sSHA       string
+	K0sBinaryURL string
+	K0sImages    []string
+	Configs      v1beta1.HelmExtensions
+	Protected    map[string][]string
+}

cmd/embedded-cluster/version.go

@@ -7,9 +7,11 @@ import (
 	"strings"
 
 	"github.com/jedib0t/go-pretty/table"
+	"github.com/k0sproject/k0s/pkg/airgap"
+	k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
 	"github.com/urfave/cli/v2"
 
-	k0sconfig "github.com/k0sproject/k0s/pkg/apis/k0s/v1beta1"
+	"github.com/replicatedhq/embedded-cluster/cmd/embedded-cluster/types"
 	"github.com/replicatedhq/embedded-cluster/pkg/addons"
 	"github.com/replicatedhq/embedded-cluster/pkg/config"
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
@@ -53,16 +55,6 @@ var versionCommand = &cli.Command{
 	},
 }
 
-// ReleaseMetadata holds the metadata about a specific release, including addons and
-// their versions.
-type ReleaseMetadata struct {
-	Versions     map[string]string
-	K0sSHA       string
-	K0sBinaryURL string
-	Configs      k0sconfig.HelmExtensions
-	Protected    map[string][]string
-}
-
 var metadataCommand = &cli.Command{
 	Name:   "metadata",
 	Usage:  "Print metadata about this release",
@@ -75,6 +67,8 @@ var metadataCommand = &cli.Command{
 		}
 		versions["Kubernetes"] = defaults.K0sVersion
 		versions["Installer"] = defaults.Version
+		versions["Troubleshoot"] = defaults.TroubleshootVersion
+		versions["Kubectl"] = defaults.KubectlVersion
 		channelRelease, err := release.GetChannelRelease()
 		if err == nil && channelRelease != nil {
 			versions[defaults.BinaryName()] = channelRelease.VersionLabel
@@ -83,7 +77,7 @@ var metadataCommand = &cli.Command{
 		if err != nil {
 			return fmt.Errorf("unable to get k0s binary sha256: %w", err)
 		}
-		meta := ReleaseMetadata{
+		meta := types.ReleaseMetadata{
 			Versions:     versions,
 			K0sSHA:       sha,
 			K0sBinaryURL: defaults.K0sBinaryURL,
@@ -103,6 +97,14 @@ var metadataCommand = &cli.Command{
 			return fmt.Errorf("unable to get protected fields: %w", err)
 		}
 		meta.Protected = protectedFields
+
+		// Render k0s config to get the images contained within
+		k0sConfig := config.RenderK0sConfig()
+		if err != nil {
+			return fmt.Errorf("unable to render k0s config: %w", err)
+		}
+		meta.K0sImages = airgap.GetImageURIs(k0sConfig.Spec, true)
+
 		data, err := json.MarshalIndent(meta, "", "\t")
 		if err != nil {
 			return fmt.Errorf("unable to marshal versions: %w", err)

docs/resources.md

@@ -0,0 +1,42 @@
+# Embedded Cluster Custom Resources
+Controlling resources utilized inside embedded cluster
+
+## ClusterConfig
+- **Owner**: k0s
+- **Namespace**: kube-system
+
+The `ClusterConfig` object contains the ingested k0s config from `/etc/k0s/k0s.yaml`
+
+This ingestion happens at k0s daemon startup on controller nodes.
+
+It can be dynamically updated, and the k0s config reconciliation process will apply any changes to the cluster except from `spec.api` and `spec.storage`.
+
+The [embedded cluster operator](https://github.com/replicatedhq/embedded-cluster-operator/) reconciles helm chart updates against the `ClusterConfig` object to initiate helm chart upgrades via the k0s helm reconciler.
+
+## Chart
+- **Owner**: k0s
+- **Namespace**: kube-system
+
+The `Chart` object contains the spec, values and tracking information for helm charts installed by the [k0s helm reconciler](https://docs.k0sproject.io/head/helm-charts/); they can be created, deleted and updated. Deleting a `Chart` object will uninstall the related helm chart from the cluster, however if the helm chart configuration is still present in the `ClusterConfig` the k0s reconciliation process will recreate/reinstall it.
+
+`Chart` Objects are currently only managed by the k0s helm reconciler, and it’s best to leave it that way for now as the API / schema for these resources is not documented.
+
+`Chart` Objects are monitored by the Embedded Cluster Operator in order to track and surface helm installation processes and errors.
+
+## Plan
+- **Owner**: k0s
+- **Cluster scoped object**
+
+`Plan` objects are used to configure the k0s autopilot operator, the autopilot operator controls cluster version upgrades via distributing and installing new k0s binaries and airgap bundles.
+
+The `Plan` resource is created by the Embedded Cluster Operator using details from the `Installation` object.
+
+## Installation
+- **Owner**: Replicated
+- **Cluster scoped object**
+
+The `Installation` object is used by the Embedded Cluster Operator to both initiate and track cluster and helm chart upgrades. they are created by [KOTS](https://github.com/replicatedhq/kots), and are marked as `Obsolete` when superseded by a newer `Installation` object.
+
+The `Installation` object can contain errors surfaced from the `Plan` and `Chart` resources.
+
+Possible Installation statuses can be found here: https://github.com/replicatedhq/embedded-cluster-operator/blob/e4fbb42919ad3b58cdc563dca77471cf76099393/api/v1beta1/installation_types.go#L24