feat: create library for embedding kots release (#279)

ricardomaraschini · web-flow · commit 70085da19d0f · 2024-01-18T20:00:40.000+01:00
* feat: create library for embedding kots release

created a library so we can use it on airgap project. this library
allows for embedding kots releases into an embedded cluster binary.

* chore: moving customization package to embed package

move the logic from the customization package into the embed package.

* chore: generating the binary using the library

generate the embedded-cluster binary used during our e2e tests using our
library of embedded kots releases into embedded-cluster binaries.
diff --git a/.gitignore b/.gitignore
@@ -3,6 +3,4 @@ bundle
 pkg/goods/bins
 pkg/goods/images
 *tgz
-release.tar.gz
-release.o
 .pre-commit-config.yaml
diff --git a/Makefile b/Makefile
@@ -73,12 +73,13 @@ output/tmp/release.tar.gz:
 	mkdir -p output/tmp
 	tar -czf output/tmp/release.tar.gz -C e2e/kots-release .
 
+output/bin/embedded-cluster-release-builder:
+	mkdir -p output/bin
+	go build -o output/bin/embedded-cluster-release-builder e2e/embedded-cluster-release-builder/main.go
+
 .PHONY: embedded-release
-embedded-release: embedded-cluster-linux-amd64 output/tmp/release.tar.gz
-	objcopy --input-target binary --output-target binary --rename-section .data=sec_bundle output/tmp/release.tar.gz output/tmp/release.o
-	@if ! objcopy --update-section sec_bundle=output/tmp/release.o output/bin/embedded-cluster 2>/dev/null; then \
-		objcopy --add-section sec_bundle=output/tmp/release.o output/bin/embedded-cluster ; \
-	fi
+embedded-release: embedded-cluster-linux-amd64 output/tmp/release.tar.gz output/bin/embedded-cluster-release-builder
+	./output/bin/embedded-cluster-release-builder output/bin/embedded-cluster output/tmp/release.tar.gz output/bin/embedded-cluster
 
 .PHONY: static
 static: pkg/goods/bins/embedded-cluster/kubectl-preflight \
diff --git a/cmd/embedded-cluster/install.go b/cmd/embedded-cluster/install.go
@@ -25,8 +25,8 @@ import (
 
 	"github.com/replicatedhq/embedded-cluster/pkg/addons"
 	"github.com/replicatedhq/embedded-cluster/pkg/config"
-	"github.com/replicatedhq/embedded-cluster/pkg/customization"
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
+	"github.com/replicatedhq/embedded-cluster/pkg/embed"
 	"github.com/replicatedhq/embedded-cluster/pkg/goods"
 	"github.com/replicatedhq/embedded-cluster/pkg/metrics"
 	"github.com/replicatedhq/embedded-cluster/pkg/preflights"
@@ -183,7 +183,7 @@ func updateConfig(c *cli.Context) error {
 // applyUnsupportedOverrides applies overrides to the k0s configuration. Applies first the
 // overrides embedded into the binary and after the ones provided by the user (--overrides).
 func applyUnsupportedOverrides(c *cli.Context, cfg *k0sv1beta1.Cluster) error {
-	if embcfg, err := customization.GetEmbeddedClusterConfig(); err != nil {
+	if embcfg, err := embed.GetEmbeddedClusterConfig(); err != nil {
 		return fmt.Errorf("unable to get embedded cluster config: %w", err)
 	} else if embcfg != nil {
 		overrides := embcfg.Spec.UnsupportedOverrides.K0s
diff --git a/cmd/embedded-cluster/join_test.go b/cmd/embedded-cluster/join_test.go
@@ -10,7 +10,7 @@ import (
 
 	"github.com/k0sproject/dig"
 	embeddedclusterv1beta1 "github.com/replicatedhq/embedded-cluster-operator/api/v1beta1"
-	"github.com/replicatedhq/embedded-cluster/pkg/customization"
+	embedded "github.com/replicatedhq/embedded-cluster/pkg/embed"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v3"
@@ -76,7 +76,7 @@ func Test_patchK0sConfig(t *testing.T) {
 			req.NoError(err, "unable to unmarshal expected file")
 
 			assert.Equal(t, expected, original)
-			customization.DefaultProvider = &customization.AdminConsole{}
+			embedded.DefaultProvider = &embedded.AdminConsole{}
 		})
 	}
 }
diff --git a/e2e/embedded-cluster-release-builder/main.go b/e2e/embedded-cluster-release-builder/main.go
@@ -0,0 +1,43 @@
+package main
+
+import (
+	"io"
+	"log"
+	"os"
+
+	"github.com/replicatedhq/embedded-cluster/pkg/embed"
+)
+
+func main() {
+	os.Exit(run())
+}
+
+func run() int {
+	if len(os.Args) != 4 {
+		log.Printf("Usage: %s <binary> <release.tar.gz> <output>", os.Args[0])
+		return 1
+	}
+	builder, err := embed.NewElfBuilder("")
+	if err != nil {
+		log.Printf("failed to create builder: %v", err)
+		return 1
+	}
+	defer builder.Close()
+	src, err := builder.Build(os.Args[1], os.Args[2])
+	if err != nil {
+		log.Printf("failed to build: %v", err)
+		return 1
+	}
+	defer src.Close()
+	dst, err := os.OpenFile(os.Args[3], os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755)
+	if err != nil {
+		log.Printf("failed to open destination: %v", err)
+		return 1
+	}
+	defer dst.Close()
+	if _, err := io.Copy(dst, src); err != nil {
+		log.Printf("failed to copy: %v", err)
+		return 1
+	}
+	return 0
+}
diff --git a/go.mod b/go.mod
@@ -22,7 +22,6 @@ require (
 	golang.org/x/crypto v0.18.0
 	golang.org/x/term v0.16.0
 	gopkg.in/yaml.v2 v2.4.0
-	helm.sh/helm/v3 v3.13.3
 	k8s.io/api v0.29.0
 	k8s.io/apimachinery v0.29.0
 	sigs.k8s.io/controller-runtime v0.16.3
@@ -50,6 +49,7 @@ require (
 	go.uber.org/goleak v1.3.0 // indirect
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
 	golang.org/x/mod v0.14.0 // indirect
+	helm.sh/helm/v3 v3.13.3 // indirect
 )
 
 require (
diff --git a/pkg/addons/adminconsole/adminconsole.go b/pkg/addons/adminconsole/adminconsole.go
@@ -15,8 +15,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	k8syaml "sigs.k8s.io/yaml"
 
-	"github.com/replicatedhq/embedded-cluster/pkg/customization"
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
+	"github.com/replicatedhq/embedded-cluster/pkg/embed"
 	"github.com/replicatedhq/embedded-cluster/pkg/kubeutils"
 	"github.com/replicatedhq/embedded-cluster/pkg/metrics"
 	pb "github.com/replicatedhq/embedded-cluster/pkg/progressbar"
@@ -107,14 +107,14 @@ func (a *AdminConsole) GetProtectedFields() map[string][]string {
 }
 
 // HostPreflights returns the host preflight objects found inside the adminconsole
-// or as part of the embedded kots release (customization).
+// or as part of the embedded kots release.
 func (a *AdminConsole) HostPreflights() (*v1beta2.HostPreflightSpec, error) {
-	return customization.GetHostPreflights()
+	return embed.GetHostPreflights()
 }
 
 // addLicenseAndVersionToHelmValues adds the embedded license to the helm values.
 func (a *AdminConsole) addLicenseAndVersionToHelmValues() error {
-	license, err := customization.GetLicense()
+	license, err := embed.GetLicense()
 	if err != nil {
 		return fmt.Errorf("unable to get license: %w", err)
 	}
@@ -126,7 +126,7 @@ func (a *AdminConsole) addLicenseAndVersionToHelmValues() error {
 		return fmt.Errorf("unable to marshal license: %w", err)
 	}
 	var appVersion string
-	if release, err := customization.GetChannelRelease(); err != nil {
+	if release, err := embed.GetChannelRelease(); err != nil {
 		return fmt.Errorf("unable to get channel release: %w", err)
 	} else if release != nil {
 		appVersion = release.VersionLabel
@@ -195,7 +195,7 @@ func (a *AdminConsole) addPasswordToHelmValues() error {
 // addKotsApplicationToHelmValues extracts the embed application struct found in this binary
 // and adds it to the helm values.
 func (a *AdminConsole) addKotsApplicationToHelmValues() error {
-	app, err := customization.GetApplication()
+	app, err := embed.GetApplication()
 	if err != nil {
 		return fmt.Errorf("unable to get application: %w", err)
 	} else if app == nil {
diff --git a/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go b/pkg/addons/embeddedclusteroperator/embeddedclusteroperator.go
@@ -15,8 +15,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/replicatedhq/embedded-cluster/pkg/addons/adminconsole"
-	"github.com/replicatedhq/embedded-cluster/pkg/customization"
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
+	"github.com/replicatedhq/embedded-cluster/pkg/embed"
 	"github.com/replicatedhq/embedded-cluster/pkg/kubeutils"
 	"github.com/replicatedhq/embedded-cluster/pkg/metrics"
 	pb "github.com/replicatedhq/embedded-cluster/pkg/progressbar"
@@ -93,7 +93,7 @@ func (e *EmbeddedClusterOperator) Outro(ctx context.Context, cli client.Client)
 		return err
 	}
 	loading.Closef("Embedded Cluster Operator is ready!")
-	cfg, err := customization.GetEmbeddedClusterConfig()
+	cfg, err := embed.GetEmbeddedClusterConfig()
 	if err != nil {
 		return err
 	}
diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -24,8 +24,8 @@ import (
 	k8syaml "sigs.k8s.io/yaml"
 
 	"github.com/replicatedhq/embedded-cluster/pkg/addons"
-	"github.com/replicatedhq/embedded-cluster/pkg/customization"
 	"github.com/replicatedhq/embedded-cluster/pkg/defaults"
+	"github.com/replicatedhq/embedded-cluster/pkg/embed"
 	"github.com/replicatedhq/embedded-cluster/pkg/prompts"
 	"github.com/replicatedhq/embedded-cluster/pkg/ssh"
 )
@@ -365,7 +365,7 @@ func ApplyEmbeddedUnsupportedOverrides(config *v1beta1.Cluster, embconfig string
 }
 
 func getControllerRoleName() string {
-	clusterConfig, err := customization.GetEmbeddedClusterConfig()
+	clusterConfig, err := embed.GetEmbeddedClusterConfig()
 
 	controllerRoleName := "controller"
 	if err == nil {
@@ -379,7 +379,7 @@ func getControllerRoleName() string {
 }
 
 func additionalControllerLabels() map[string]string {
-	clusterConfig, err := customization.GetEmbeddedClusterConfig()
+	clusterConfig, err := embed.GetEmbeddedClusterConfig()
 
 	if err == nil {
 		if clusterConfig != nil {
diff --git a/pkg/embed/customization.go b/pkg/embed/customization.go
@@ -1,4 +1,4 @@
-package customization
+package embed
 
 import (
 	"archive/tar"
diff --git a/pkg/embed/elf.go b/pkg/embed/elf.go
@@ -0,0 +1,124 @@
+package embed
+
+import (
+	"debug/elf"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"path"
+)
+
+// ElfBuilder is an implementation of the Embedded Cluster Builder using elf sections. This
+// is deprecated and exists for backward compatibility. Used in old Embedded Cluster versions.
+type ElfBuilder struct{ baseDir string }
+
+// NewElfBuilder returns a new instance of the ElfBuilder. Callers need to Close the builder
+// when done with it, this ensure temporary files are removed.
+func NewElfBuilder(tmpdir string) (*ElfBuilder, error) {
+	if _, err := exec.LookPath("objcopy"); err != nil {
+		return nil, fmt.Errorf("unable to find objcopy binary in path: %w", err)
+	}
+	tmp, err := os.MkdirTemp(tmpdir, "embedded-cluster-builder-*")
+	if err != nil {
+		return nil, fmt.Errorf("unable to create temp dir: %w", err)
+	}
+	return &ElfBuilder{tmp}, nil
+}
+
+// Here we look for the sec_bundle section.
+func (e *ElfBuilder) hasEmbeddedRelease() (bool, error) {
+	binary := path.Join(e.baseDir, "embedded-cluster")
+	bin, err := elf.Open(binary)
+	if err != nil {
+		return false, fmt.Errorf("unable to open binary: %v", err)
+	}
+	section := bin.Section("sec_bundle")
+	return section != nil, nil
+}
+
+// Build builds a new version of the Embedded Cluster binary pointed by bin with the release
+// pointed by release embedded into it. Returns a ReadCloser from where the new binary can be
+// read from. Callers need to Close the returned value when done with it.
+func (e *ElfBuilder) Build(bin, release string) (io.ReadCloser, error) {
+	if err := e.withBinary(bin); err != nil {
+		return nil, fmt.Errorf("unable to add binary to builder: %w", err)
+	}
+	if err := e.withRelease(release); err != nil {
+		return nil, fmt.Errorf("unable to add release to builder: %w", err)
+	}
+	if err := e.embed(); err != nil {
+		return nil, fmt.Errorf("unable to build binary: %w", err)
+	}
+	binpath := path.Join(e.baseDir, "embedded-cluster")
+	return os.Open(binpath)
+}
+
+// Size returns the size of the embedded cluster binary once it is built (includes the kots
+// release data).
+func (e *ElfBuilder) Size() (int64, error) {
+	binpath := path.Join(e.baseDir, "embedded-cluster")
+	fi, err := os.Stat(binpath)
+	if err != nil {
+		return 0, fmt.Errorf("unable to stat binary: %w", err)
+	}
+	return fi.Size(), nil
+}
+
+// Close closes the builder. Removes the temporary directory.
+func (e *ElfBuilder) Close() error {
+	return os.RemoveAll(e.baseDir)
+}
+
+// embed embeds the release object file into the embedded cluster binary in a section called
+// sec_bundle.
+func (e *ElfBuilder) embed() error {
+	binary := path.Join(e.baseDir, "embedded-cluster")
+	release := path.Join(e.baseDir, "release.o")
+	bundleArg := fmt.Sprintf("sec_bundle=%s", release)
+	cmdflag := "--add-section"
+	if hasEmbedded, err := e.hasEmbeddedRelease(); err != nil {
+		return fmt.Errorf("unable to check if binary has embedded release: %v", err)
+	} else if hasEmbedded {
+		cmdflag = "--update-section"
+	}
+	command := exec.Command("objcopy", cmdflag, bundleArg, binary)
+	if out, err := command.CombinedOutput(); err != nil {
+		return fmt.Errorf("unable to embed release: %v: %s", err, string(out))
+	}
+	return nil
+}
+
+// withBinary adds the given binary to the builder directory. This copies the binary to the
+// temporary directory as the elf builder needs to modify it.
+func (e *ElfBuilder) withBinary(bin string) error {
+	src, err := os.Open(bin)
+	if err != nil {
+		return fmt.Errorf("unable to open binary file: %w", err)
+	}
+	defer src.Close()
+	dstpath := path.Join(e.baseDir, "embedded-cluster")
+	dst, err := os.Create(dstpath)
+	if err != nil {
+		return fmt.Errorf("unable to create destination file: %w", err)
+	}
+	defer dst.Close()
+	_, err = io.Copy(dst, src)
+	return err
+}
+
+// WithRelease adds the given release to the builder. Turns the release into an object file.
+func (c *ElfBuilder) withRelease(release string) error {
+	dstpath := path.Join(c.baseDir, "release.o")
+	command := exec.Command(
+		"objcopy",
+		"--input-target", "binary",
+		"--output-target", "binary",
+		"--rename-section", ".data=sec_bundle",
+		release, dstpath,
+	)
+	if out, err := command.CombinedOutput(); err != nil {
+		return fmt.Errorf("unable to create release object: %v: %s", err, string(out))
+	}
+	return nil
+}
diff --git a/pkg/embed/elf_test.go b/pkg/embed/elf_test.go
diff --git a/pkg/embed/helpers.go b/pkg/embed/helpers.go
diff --git a/pkg/embed/testdata/release.tar.gz b/pkg/embed/testdata/release.tar.gz
diff --git a/pkg/metrics/reporter.go b/pkg/metrics/reporter.go
diff --git a/pkg/metrics/sender.go b/pkg/metrics/sender.go

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-package customization`
	`1`	`+package embed`
`2`	`2`
`3`	`3`	`import (`
`4`	`4`	`"archive/tar"`