Additional hook points (#864)

* Define additional hook points

Signed-off-by: Kimmo Lehto <[email protected]>

* Make it pass

Signed-off-by: Kimmo Lehto <[email protected]>

---------

Signed-off-by: Kimmo Lehto <[email protected]>

Author: kke

README.md

@@ -384,6 +384,14 @@ Example:
 
 ```yaml
 hooks:
+  connect:
+    after:
+      - echo "connected and detected" >> k0sctl-connect.log
+  upgrade:
+    before:
+      - echo "about to upgrade" >> k0sctl-upgrade.log
+    after:
+      - echo "upgraded" >> k0sctl-upgrade.log
   apply:
     before:
       - date >> k0sctl-apply.log
@@ -393,16 +401,30 @@ hooks:
 
 The currently available "hook points" are:
 
+* `connect`: 
+    - `after`: Runs immediately after OS detection completes
 * `apply`: Runs during `k0sctl apply`
     - `before`: Runs after configuration and host validation, right before configuring k0s on the host
     - `after`: Runs before disconnecting from the host after a successful apply operation
+* `upgrade`: Runs during `k0sctl apply`
+    - `before`: Runs for each host that is going to be upgraded, before the upgrade begins
+    - `after`: Runs for each host that was upgraded, after the upgrade completes
+* `install`: Runs during `k0sctl apply`
+    - `before`: Runs on each host just before installing its k0s components. This includes the first controller (Initialize the k0s cluster), additional controllers, and workers.
+    - `after`: Runs on each host immediately after installing its k0s components (service started and ready checks done).
 * `backup`: Runs during `k0s backup`
     - `before`: Runs before k0sctl runs the `k0s backup` command
     - `after`: Runs before disconnecting from the host after successfully taking a backup
-* `reset`: Runs during `k0sctl reset`
+* `reset`: Runs during `k0sctl reset` or when `k0sctl apply` resets a host.
     - `before`: Runs after gathering information about the cluster, right before starting to remove the k0s installation.
     - `after`: Runs before disconnecting from the host after a successful reset operation
 
+Notes:
+
+- Hooks run on each host that defines them, using the same remote user as the connection. If elevated privileges are required, prefix commands with `sudo`.
+- In dry-run mode, hooks are not executed; k0sctl prints what would run on each host.
+- Hooks execute only on hosts targeted by the related phase. For example, `upgrade` hooks run only for hosts that need upgrade.
+
 ##### `spec.hosts[*].os` <string> (optional) (default: ``)
 
 Override OS distribution auto-detection. By default `k0sctl` detects the OS by reading `/etc/os-release` or `/usr/lib/os-release` files. In case your system is based on e.g. Debian but the OS release info has something else configured you can override `k0sctl` to use Debian based functionality for the node with:
@@ -741,7 +763,7 @@ The following tokens can be used in the `k0sDownloadURL` and `files.[*].src` fie
 - `%%` - literal `%`
 - `%p` - host architecture (arm, arm64, amd64)
 - `%v` - k0s version (v1.21.0+k0s.0)
-- `%x` - k0s binary extension (currently always empty)
+- `%x` - k0s binary extension (.exe on Windows, empty elsewhere)
 
 Any other tokens will be output as-is including the `%` character.
 
@@ -752,8 +774,5 @@ Example:
     k0sDownloadURL: https://files.example.com/k0s%20files/k0s-%v-%p%x
     # Expands to https://files.example.com/k0s%20files/k0s-v1.21.0+k0s.0-amd64
 ```
-
-
-
 ## License
 [![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fk0sproject%2Fk0sctl.svg?type=large)](https://app.fossa.com/projects/git%2Bgithub.com%2Fk0sproject%2Fk0sctl?ref=badge_large)

action/apply.go

@@ -55,8 +55,8 @@ func NewApply(opts ApplyOptions) *Apply {
 		ApplyOptions: opts,
 		Phases: phase.Phases{
 			&phase.DefaultK0sVersion{},
-			&phase.Connect{},
-			&phase.DetectOS{},
+            &phase.Connect{},
+            &phase.DetectOS{},
 			lockPhase,
 			&phase.PrepareHosts{},
 			&phase.GatherFacts{},

action/backup.go

@@ -13,24 +13,22 @@ import (
 type Backup struct {
 	// Manager is the phase manager
 	Manager *phase.Manager
-	Out  io.Writer
+	Out     io.Writer
 }
 
 func (b Backup) Run(ctx context.Context) error {
 	start := time.Now()
 
 	lockPhase := &phase.Lock{}
 
-	b.Manager.AddPhase(
-		&phase.Connect{},
-		&phase.DetectOS{},
-		lockPhase,
-		&phase.PrepareHosts{},
+    b.Manager.AddPhase(
+        &phase.Connect{},
+        &phase.DetectOS{},
+        lockPhase,
+        &phase.PrepareHosts{},
 		&phase.GatherFacts{SkipMachineIDs: true},
 		&phase.GatherK0sFacts{},
-		&phase.RunHooks{Stage: "before", Action: "backup"},
 		&phase.Backup{Out: b.Out},
-		&phase.RunHooks{Stage: "after", Action: "backup"},
 		&phase.Unlock{Cancel: lockPhase.Cancel},
 		&phase.Disconnect{},
 	)

action/kubeconfig.go

@@ -22,12 +22,12 @@ func (k *Kubeconfig) Run(ctx context.Context) error {
 	// do not need to connect to all nodes
 	k.Manager.Config.Spec.Hosts = cluster.Hosts{k.Manager.Config.Spec.K0sLeader()}
 
-	k.Manager.AddPhase(
-		&phase.Connect{},
-		&phase.DetectOS{},
-		&phase.GetKubeconfig{APIAddress: k.KubeconfigAPIAddress, User: k.KubeconfigUser, Cluster: k.KubeconfigCluster},
-		&phase.Disconnect{},
-	)
+    k.Manager.AddPhase(
+        &phase.Connect{},
+        &phase.DetectOS{},
+        &phase.GetKubeconfig{APIAddress: k.KubeconfigAPIAddress, User: k.KubeconfigUser, Cluster: k.KubeconfigCluster},
+        &phase.Disconnect{},
+    )
 
 	return k.Manager.Run(ctx)
 }

action/reset.go

@@ -42,14 +42,13 @@ func (r Reset) Run(ctx context.Context) error {
 	}
 
 	lockPhase := &phase.Lock{}
-	r.Manager.AddPhase(
-		&phase.Connect{},
-		&phase.DetectOS{},
-		lockPhase,
+    r.Manager.AddPhase(
+        &phase.Connect{},
+        &phase.DetectOS{},
+        lockPhase,
 		&phase.PrepareHosts{},
 		&phase.GatherFacts{SkipMachineIDs: true},
 		&phase.GatherK0sFacts{},
-		&phase.RunHooks{Stage: "before", Action: "reset"},
 		&phase.ResetWorkers{
 			NoDrain:  true,
 			NoDelete: true,
@@ -61,7 +60,6 @@ func (r Reset) Run(ctx context.Context) error {
 		},
 		&phase.ResetLeader{},
 		&phase.DaemonReload{},
-		&phase.RunHooks{Stage: "after", Action: "reset"},
 		&phase.Unlock{Cancel: lockPhase.Cancel},
 		&phase.Disconnect{},
 	)

phase/backup.go

@@ -31,6 +31,22 @@ func (p *Backup) Title() string {
 	return "Take backup"
 }
 
+// Before runs "before backup" hooks
+func (p *Backup) Before() error {
+    if err := p.runHooks(context.Background(), "backup", "before", p.leader); err != nil {
+        return fmt.Errorf("running hooks failed: %w", err)
+    }
+    return nil
+}
+
+// After runs "after backup" hooks
+func (p *Backup) After() error {
+    if err := p.runHooks(context.Background(), "backup", "after", p.leader); err != nil {
+        return fmt.Errorf("running hooks failed: %w", err)
+    }
+    return nil
+}
+
 // Prepare the phase
 func (p *Backup) Prepare(config *v1beta1.Cluster) error {
 	p.Config = config

phase/detect_os.go

@@ -18,7 +18,7 @@ import (
 
 // DetectOS performs remote OS detection
 type DetectOS struct {
-	GenericPhase
+    GenericPhase
 }
 
 // Title for the phase
@@ -55,3 +55,8 @@ func (p *DetectOS) Run(ctx context.Context) error {
 		return nil
 	})
 }
+
+// After runs the per-host "connect: after" hooks once OS detection has succeeded.
+func (p *DetectOS) After() error {
+    return p.runHooks(context.Background(), "connect", "after", p.Config.Spec.Hosts...)
+}

phase/generic_phase.go

@@ -1,11 +1,11 @@
 package phase
 
 import (
-	"context"
-	"fmt"
+    "context"
+    "fmt"
 
-	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1"
-	"github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster"
+    "github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1"
+    "github.com/k0sproject/k0sctl/pkg/apis/k0sctl.k0sproject.io/v1beta1/cluster"
 )
 
 // GenericPhase is a basic phase which gets a config via prepare, sets it into p.Config
@@ -28,12 +28,12 @@ func (p *GenericPhase) Prepare(c *v1beta1.Cluster) error {
 
 // Wet is a shorthand for manager.Wet
 func (p *GenericPhase) Wet(host fmt.Stringer, msg string, funcs ...errorfunc) error {
-	return p.manager.Wet(host, msg, funcs...)
+    return p.manager.Wet(host, msg, funcs...)
 }
 
-// IsWet returns true if manager is in dry-run mode
+// IsWet returns true when not in dry-run mode (i.e., wet mode)
 func (p *GenericPhase) IsWet() bool {
-	return !p.manager.DryRun
+    return !p.manager.DryRun
 }
 
 // DryMsg is a shorthand for manager.DryMsg
@@ -59,8 +59,28 @@ func (p *GenericPhase) parallelDo(ctx context.Context, hosts cluster.Hosts, func
 }
 
 func (p *GenericPhase) parallelDoUpload(ctx context.Context, hosts cluster.Hosts, funcs ...func(context.Context, *cluster.Host) error) error {
-	if p.manager.Concurrency == 0 {
-		return hosts.ParallelEach(ctx, funcs...)
-	}
-	return hosts.BatchedParallelEach(ctx, p.manager.ConcurrentUploads, funcs...)
+    if p.manager.Concurrency == 0 {
+        return hosts.ParallelEach(ctx, funcs...)
+    }
+    return hosts.BatchedParallelEach(ctx, p.manager.ConcurrentUploads, funcs...)
+}
+
+// runHooks executes hooks for the provided hosts honoring the given context.
+func (p *GenericPhase) runHooks(ctx context.Context, action, stage string, hosts ...*cluster.Host) error {
+    return p.parallelDo(ctx, hosts, func(_ context.Context, h *cluster.Host) error {
+        if !p.IsWet() {
+            // In dry-run, list each hook command that would be executed.
+            cmds := h.Hooks.ForActionAndStage(action, stage)
+            for _, cmd := range cmds {
+                p.DryMsgf(h, "run %s %s hook: %q", stage, action, cmd)
+            }
+            return nil
+        }
+
+        if err := h.RunHooks(ctx, action, stage); err != nil {
+            return fmt.Errorf("running hooks failed: %w", err)
+        }
+
+        return nil
+    })
 }

phase/initialize_k0s.go

@@ -15,8 +15,8 @@ import (
 
 // InitializeK0s sets up the "initial" k0s controller
 type InitializeK0s struct {
-	GenericPhase
-	leader *cluster.Host
+    GenericPhase
+    leader *cluster.Host
 }
 
 // Title for the phase
@@ -34,6 +34,22 @@ func (p *InitializeK0s) Prepare(config *v1beta1.Cluster) error {
 	return nil
 }
 
+// Before runs "before install" hooks for the leader controller
+func (p *InitializeK0s) Before() error {
+    if p.leader == nil || p.leader.Reset {
+        return nil
+    }
+    return p.runHooks(context.Background(), "install", "before", p.leader)
+}
+
+// After runs "after install" hooks for the leader controller
+func (p *InitializeK0s) After() error {
+    if p.leader == nil || p.leader.Reset {
+        return nil
+    }
+    return p.runHooks(context.Background(), "install", "after", p.leader)
+}
+
 // ShouldRun is true when there is a leader host
 func (p *InitializeK0s) ShouldRun() bool {
 	return p.leader != nil && !p.leader.Reset

phase/install_controllers.go

@@ -48,9 +48,16 @@ func (p *InstallControllers) ShouldRun() bool {
 	return len(p.hosts) > 0
 }
 
+// Before runs "before install" hooks for controller hosts
+func (p *InstallControllers) Before() error {
+    if len(p.hosts) == 0 {
+        return nil
+    }
+    return p.runHooks(context.Background(), "install", "before", p.hosts...)
+}
+
 // CleanUp cleans up the environment override files on hosts
 func (p *InstallControllers) CleanUp() {
-	_ = p.After()
 	_ = p.hosts.Filter(func(h *cluster.Host) bool {
 		return !h.Metadata.Ready
 	}).ParallelEach(context.Background(), func(_ context.Context, h *cluster.Host) error {
@@ -69,7 +76,12 @@ func (p *InstallControllers) CleanUp() {
 	})
 }
 
+// After runs "after install" hooks for controller hosts and cleans up tokens
 func (p *InstallControllers) After() error {
+    // Run "after install" hooks for controllers first
+    if err := p.runHooks(context.Background(), "install", "after", p.hosts...); err != nil {
+        return err
+    }
 	for i, h := range p.hosts {
 		if h.Metadata.K0sTokenData.Token == "" {
 			continue
@@ -80,7 +92,7 @@ func (p *InstallControllers) After() error {
 			return p.leader.Exec(p.leader.Configurer.K0sCmdf("token invalidate --data-dir=%s %s", p.leader.K0sDataDir(), h.Metadata.K0sTokenData.ID), exec.Sudo(p.leader))
 		})
 		if err != nil {
-			log.Warnf("%s: failed to invalidate worker join token: %v", p.leader, err)
+			log.Warnf("%s: failed to invalidate controller join token: %v", p.leader, err)
 		}
 		_ = p.Wet(h, "overwrite k0s join token file", func() error {
 			if err := h.Configurer.WriteFile(h, h.K0sJoinTokenPath(), "# overwritten by k0sctl after join\n", "0600"); err != nil {
@@ -222,6 +234,7 @@ func (p *InstallControllers) installK0s(ctx context.Context, h *cluster.Host) er
 		return err
 	}
 	log.Infof("%s: installing k0s controller", h)
+
 	err = p.Wet(h, fmt.Sprintf("install k0s controller using `%s", strings.ReplaceAll(cmd, h.K0sInstallLocation(), "k0s")), func() error {
 		var stdout, stderr bytes.Buffer
 		runner, err := h.ExecStreams(cmd, nil, &stdout, &stderr, exec.Sudo(h))