Set ETCD_UNSUPPORTED_ARCH on ARM controller nodes (#184)

erdii · kke · web-flow · commit d97cc1f50730 · 2021-08-23T15:15:29.000+03:00
* set ETCD_UNSUPPORTED_ARCH on arm nodes

Signed-off-by: erdii &lt;me@erdii.engineering&gt;

* use h.Configurer.WriteFile to upload arm override file

Signed-off-by: erdii &lt;me@erdii.engineering&gt;

* Finetuning

* Use rig 0.3.24 for service env management

* Whoops, left out the arm

* Missing return

* Update go.mod

* Try trickery

* Ok the file cant be called _arm

* ineffassign

* Clean up service environment during reset even if k0s not running

* More cleaning up

* Ok, after hooks were interntionally even if it errored

Co-authored-by: Kimmo Lehto &lt;kimmo.lehto@gmail.com&gt;
diff --git a/cmd/apply.go b/cmd/apply.go
@@ -76,6 +76,7 @@ var applyCommand = &cli.Command{
 			&phase.UploadBinaries{},
 			&phase.DownloadK0s{},
 			&phase.RunHooks{Stage: "before", Action: "apply"},
+			&phase.PrepareArm{},
 			&phase.ConfigureK0s{},
 			&phase.Restore{
 				RestoreFrom: ctx.String("restore-from"),
diff --git a/config/cluster/host.go b/config/cluster/host.go
@@ -71,6 +71,8 @@ type configurer interface {
 	PrivateInterface(os.Host) (string, error)
 	PrivateAddress(os.Host, string, string) (string, error)
 	TempDir(os.Host) (string, error)
+	UpdateServiceEnvironment(os.Host, string, map[string]string) error
+	CleanupServiceEnvironment(os.Host, string) error
 }
 
 // HostMetadata resolved metadata for host
diff --git a/phase/arm_prepare.go b/phase/arm_prepare.go
@@ -0,0 +1,59 @@
+package phase
+
+import (
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/k0sproject/k0sctl/config"
+	"github.com/k0sproject/k0sctl/config/cluster"
+)
+
+// PrepareArm implements a phase which fixes arm quirks
+type PrepareArm struct {
+	GenericPhase
+
+	hosts cluster.Hosts
+}
+
+// Title for the phase
+func (p *PrepareArm) Title() string {
+	return "Prepare ARM nodes"
+}
+
+// Prepare the phase
+func (p *PrepareArm) Prepare(config *config.Cluster) error {
+	p.Config = config
+
+	p.hosts = p.Config.Spec.Hosts.Filter(func(h *cluster.Host) bool {
+		arch := h.Metadata.Arch
+		return h.Role != "worker" && (strings.HasPrefix(arch, "arm") || strings.HasPrefix(arch, "aarch"))
+	})
+
+	return nil
+}
+
+// ShouldRun is true when there are arm controllers
+func (p *PrepareArm) ShouldRun() bool {
+	return len(p.hosts) > 0
+}
+
+// Run the phase
+func (p *PrepareArm) Run() error {
+	return p.hosts.ParallelEach(p.etcdUnsupportedArch)
+}
+
+func (p *PrepareArm) etcdUnsupportedArch(h *cluster.Host) error {
+	var arch string
+	switch h.Metadata.Arch {
+	case "aarch32", "arm32", "armv7l", "armhfp", "arm-32":
+		arch = "arm32"
+	default:
+		arch = "arm64"
+	}
+
+	log.Warnf("%s: enabling ETCD_UNSUPPORTED_ARCH=%s override - you may encounter problems with etcd", h, arch)
+	h.Environment["ETCD_UNSUPPORTED_ARCH"] = arch
+
+	return nil
+}
diff --git a/phase/initialize_k0s.go b/phase/initialize_k0s.go
@@ -32,6 +32,16 @@ func (p *InitializeK0s) ShouldRun() bool {
 	return p.leader != nil
 }
 
+// CleanUp cleans up the environment override file
+func (p *InitializeK0s) CleanUp() {
+	h := p.leader
+	if len(h.Environment) > 0 {
+		if err := h.Configurer.CleanupServiceEnvironment(h, h.K0sServiceName()); err != nil {
+			log.Warnf("%s: failed to clean up service environment: %s", h, err.Error())
+		}
+	}
+}
+
 // Run the phase
 func (p *InitializeK0s) Run() error {
 	h := p.leader
@@ -42,6 +52,13 @@ func (p *InitializeK0s) Run() error {
 		return err
 	}
 
+	if len(h.Environment) > 0 {
+		log.Infof("%s: updating service environment", h)
+		if err := h.Configurer.UpdateServiceEnvironment(h, h.K0sServiceName(), h.Environment); err != nil {
+			return err
+		}
+	}
+
 	if err := h.Configurer.StartService(h, h.K0sServiceName()); err != nil {
 		return err
 	}
diff --git a/phase/install_controllers.go b/phase/install_controllers.go
@@ -37,6 +37,17 @@ func (p *InstallControllers) ShouldRun() bool {
 	return len(p.hosts) > 0
 }
 
+// CleanUp cleans up the environment override files on hosts
+func (p *InstallControllers) CleanUp() {
+	for _, h := range p.hosts {
+		if len(h.Environment) > 0 {
+			if err := h.Configurer.CleanupServiceEnvironment(h, h.K0sServiceName()); err != nil {
+				log.Warnf("%s: failed to clean up service environment: %s", h, err.Error())
+			}
+		}
+	}
+}
+
 // Run the phase
 func (p *InstallControllers) Run() error {
 	for _, h := range p.hosts {
@@ -60,6 +71,13 @@ func (p *InstallControllers) Run() error {
 			return err
 		}
 
+		if len(h.Environment) > 0 {
+			log.Infof("%s: updating service environment", h)
+			if err := h.Configurer.UpdateServiceEnvironment(h, h.K0sServiceName(), h.Environment); err != nil {
+				return err
+			}
+		}
+
 		log.Infof("%s: starting service", h)
 		if err := h.Configurer.StartService(h, h.K0sServiceName()); err != nil {
 			return err
diff --git a/phase/install_workers.go b/phase/install_workers.go
@@ -37,6 +37,17 @@ func (p *InstallWorkers) ShouldRun() bool {
 	return len(p.hosts) > 0
 }
 
+// CleanUp cleans up the environment override files on hosts
+func (p *InstallWorkers) CleanUp() {
+	for _, h := range p.hosts {
+		if len(h.Environment) > 0 {
+			if err := h.Configurer.CleanupServiceEnvironment(h, h.K0sServiceName()); err != nil {
+				log.Warnf("%s: failed to clean up service environment: %s", h, err.Error())
+			}
+		}
+	}
+}
+
 // Run the phase
 func (p *InstallWorkers) Run() error {
 	log.Infof("%s: generating token", p.leader)
@@ -75,6 +86,13 @@ func (p *InstallWorkers) Run() error {
 			return err
 		}
 
+		if len(h.Environment) > 0 {
+			log.Infof("%s: updating service environment", h)
+			if err := h.Configurer.UpdateServiceEnvironment(h, h.K0sServiceName(), h.Environment); err != nil {
+				return err
+			}
+		}
+
 		log.Infof("%s: starting service", h)
 		if err := h.Configurer.StartService(h, h.K0sServiceName()); err != nil {
 			return err
diff --git a/phase/manager.go b/phase/manager.go
@@ -37,6 +37,10 @@ type propsetter interface {
 	SetProp(string, interface{})
 }
 
+type withcleanup interface {
+	CleanUp()
+}
+
 // Manager executes phases to construct the cluster
 type Manager struct {
 	phases []phase
@@ -50,6 +54,20 @@ func (m *Manager) AddPhase(p ...phase) {
 
 // Run executes all the added Phases in order
 func (m *Manager) Run() error {
+	var ran []phase
+	var result error
+
+	defer func() {
+		if result != nil {
+			for _, p := range ran {
+				if c, ok := p.(withcleanup); ok {
+					log.Infof(Colorize.Red("* Running clean-up for phase: %s").String(), p.Title())
+					c.CleanUp()
+				}
+			}
+		}
+	}()
+
 	for _, p := range m.phases {
 		title := p.Title()
 
@@ -81,7 +99,8 @@ func (m *Manager) Run() error {
 
 		text := Colorize.Green("==> Running phase: %s").String()
 		log.Infof(text, title)
-		result := p.Run()
+		result = p.Run()
+		ran = append(ran, p)
 
 		if p, ok := p.(afterhook); ok {
 			if err := p.After(result); err != nil {
diff --git a/phase/reset.go b/phase/reset.go
@@ -48,6 +48,11 @@ func (p *Reset) Prepare(config *config.Cluster) error {
 // Run the phase
 func (p *Reset) Run() error {
 	return p.hosts.ParallelEach(func(h *cluster.Host) error {
+		log.Infof("%s: cleaning up service environment", h)
+		if err := h.Configurer.CleanupServiceEnvironment(h, h.K0sServiceName()); err != nil {
+			return err
+		}
+
 		if h.Configurer.ServiceIsRunning(h, h.K0sServiceName()) {
 			log.Infof("%s: stopping k0s", h)
 			if err := h.Configurer.StopService(h, h.K0sServiceName()); err != nil {
diff --git a/phase/upgrade_controllers.go b/phase/upgrade_controllers.go
@@ -40,6 +40,17 @@ func (p *UpgradeControllers) ShouldRun() bool {
 	return len(p.hosts) > 0
 }
 
+// CleanUp cleans up the environment override files on hosts
+func (p *UpgradeControllers) CleanUp() {
+	for _, h := range p.hosts {
+		if len(h.Environment) > 0 {
+			if err := h.Configurer.CleanupServiceEnvironment(h, h.K0sServiceName()); err != nil {
+				log.Warnf("%s: failed to clean up service environment: %s", h, err.Error())
+			}
+		}
+	}
+}
+
 // Run the phase
 func (p *UpgradeControllers) Run() error {
 	for _, h := range p.hosts {
@@ -59,6 +70,14 @@ func (p *UpgradeControllers) Run() error {
 		if err := h.UpdateK0sBinary(p.Config.Spec.K0s.Version); err != nil {
 			return err
 		}
+
+		if len(h.Environment) > 0 {
+			log.Infof("%s: updating service environment", h)
+			if err := h.Configurer.UpdateServiceEnvironment(h, h.K0sServiceName(), h.Environment); err != nil {
+				return err
+			}
+		}
+
 		if err := h.Configurer.StartService(h, h.K0sServiceName()); err != nil {
 			return err
 		}
diff --git a/phase/upgrade_workers.go b/phase/upgrade_workers.go
@@ -44,6 +44,17 @@ func (p *UpgradeWorkers) ShouldRun() bool {
 	return len(p.hosts) > 0
 }
 
+// CleanUp cleans up the environment override files on hosts
+func (p *UpgradeWorkers) CleanUp() {
+	for _, h := range p.hosts {
+		if len(h.Environment) > 0 {
+			if err := h.Configurer.CleanupServiceEnvironment(h, h.K0sServiceName()); err != nil {
+				log.Warnf("%s: failed to clean up service environment: %s", h, err.Error())
+			}
+		}
+	}
+}
+
 // Run the phase
 func (p *UpgradeWorkers) Run() error {
 	// Upgrade worker hosts parallelly in 10% chunks
@@ -93,6 +104,14 @@ func (p *UpgradeWorkers) upgradeWorker(h *cluster.Host) error {
 	if err := h.UpdateK0sBinary(p.Config.Spec.K0s.Version); err != nil {
 		return err
 	}
+
+	if len(h.Environment) > 0 {
+		log.Infof("%s: updating service environment", h)
+		if err := h.Configurer.UpdateServiceEnvironment(h, h.K0sServiceName(), h.Environment); err != nil {
+			return err
+		}
+	}
+
 	if err := h.Configurer.StartService(h, h.K0sServiceName()); err != nil {
 		return err
 	}

Original file line number	Diff line number	Diff line change
`@@ -71,6 +71,8 @@ type configurer interface {`
`71`	`71`	`PrivateInterface(os.Host) (string, error)`
`72`	`72`	`PrivateAddress(os.Host, string, string) (string, error)`
`73`	`73`	`TempDir(os.Host) (string, error)`
	`74`	`+ UpdateServiceEnvironment(os.Host, string, map[string]string) error`
	`75`	`+ CleanupServiceEnvironment(os.Host, string) error`
`74`	`76`	`}`
`75`	`77`
`76`	`78`	`// HostMetadata resolved metadata for host`