Merge pull request #165 from thin-edge/fix-verify-container-engine-on-startup

reubenmiller · web-flow · commit 9822290dcd67 · 2025-09-10T12:17:26.000Z
fix: verify container engine connection on startup and support retries
diff --git a/cli/container/finalize.go b/cli/container/finalize.go
@@ -24,7 +24,7 @@ func NewFinalizeCommand(ctx cli.Cli) *cobra.Command {
 			if !pruneImages {
 				return nil
 			}
-			cli, err := container.NewContainerClient()
+			cli, err := container.NewContainerClient(context.TODO())
 			if err != nil {
 				return err
 			}
diff --git a/cli/container/install.go b/cli/container/install.go
@@ -75,7 +75,7 @@ func (c *InstallCommand) RunE(cmd *cobra.Command, args []string) error {
 	// Only enable pulling if the user is providing a file
 	disablePull := c.File != ""
 
-	cli, err := container.NewContainerClient()
+	cli, err := container.NewContainerClient(context.TODO())
 	if err != nil {
 		return err
 	}
diff --git a/cli/container/list.go b/cli/container/list.go
@@ -22,7 +22,7 @@ func NewListCommand(cliContext cli.Cli) *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			slog.Info("Executing", "cmd", cmd.CalledAs(), "args", args)
 			ctx := context.Background()
-			cli, err := container.NewContainerClient()
+			cli, err := container.NewContainerClient(ctx)
 			if err != nil {
 				return err
 			}
diff --git a/cli/container/remove.go b/cli/container/remove.go
@@ -35,7 +35,7 @@ Example 1: Remove a container
 			ctx := context.Background()
 			containerName := args[0]
 
-			cli, err := container.NewContainerClient()
+			cli, err := container.NewContainerClient(ctx)
 			if err != nil {
 				return err
 			}
diff --git a/cli/container_group/finalize.go b/cli/container_group/finalize.go
@@ -24,7 +24,7 @@ func NewFinalizeCommand(ctx cli.Cli) *cobra.Command {
 			if !pruneImages {
 				return nil
 			}
-			cli, err := container.NewContainerClient()
+			cli, err := container.NewContainerClient(context.TODO())
 			if err != nil {
 				return err
 			}
diff --git a/cli/container_group/install.go b/cli/container_group/install.go
@@ -52,7 +52,7 @@ func (c *InstallCommand) RunE(cmd *cobra.Command, args []string) error {
 	projectName := args[0]
 	stderr := cmd.ErrOrStderr()
 
-	cli, err := container.NewContainerClient()
+	cli, err := container.NewContainerClient(context.TODO())
 	if err != nil {
 		return err
 	}
diff --git a/cli/container_group/list.go b/cli/container_group/list.go
@@ -42,7 +42,7 @@ func NewListCommand(cliContext cli.Cli) *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			slog.Info("Executing", "cmd", cmd.CalledAs(), "args", args)
 			ctx := context.Background()
-			cli, err := container.NewContainerClient()
+			cli, err := container.NewContainerClient(context.TODO())
 			if err != nil {
 				return err
 			}
diff --git a/cli/container_group/remove.go b/cli/container_group/remove.go
@@ -40,7 +40,7 @@ func (c *RemoveCommand) RunE(cmd *cobra.Command, args []string) error {
 	ctx := context.Background()
 	projectName := args[0]
 
-	cli, err := container.NewContainerClient()
+	cli, err := container.NewContainerClient(context.TODO())
 	if err != nil {
 		return err
 	}
diff --git a/cli/engine/docker.go b/cli/engine/docker.go
@@ -4,6 +4,7 @@ Copyright © 2024 thin-edge.io <info@thin-edge.io>
 package engine
 
 import (
+	"context"
 	"log/slog"
 	"os/exec"
 
@@ -33,7 +34,7 @@ func NewRunCommand(ctx cli.Cli) *cobra.Command {
 
 func (c *DockerCommand) RunE(cmd *cobra.Command, args []string) error {
 	slog.Debug("Executing", "cmd", cmd.CalledAs(), "args", args)
-	containerCli, err := container.NewContainerClient()
+	containerCli, err := container.NewContainerClient(context.TODO())
 	if err != nil {
 		return err
 	}
diff --git a/cli/self/check.go b/cli/self/check.go
@@ -78,7 +78,7 @@ func (c *CheckCommand) RunE(cmd *cobra.Command, args []string) error {
 	}
 
 	// Check container self
-	containerCLI, err := container.NewContainerClient()
+	containerCLI, err := container.NewContainerClient(context.TODO())
 	if err != nil {
 		return err
 	}
diff --git a/cli/self/list.go b/cli/self/list.go
@@ -36,7 +36,7 @@ func NewListCommand(cliContext cli.Cli) *cobra.Command {
 			}
 
 			ctx := context.Background()
-			containerCli, err := container.NewContainerClient()
+			containerCli, err := container.NewContainerClient(context.TODO())
 			if err != nil {
 				return newNotSupportedErr(err)
 			}
diff --git a/cli/tools/container_clone.go b/cli/tools/container_clone.go
@@ -87,7 +87,7 @@ func (c *ContainerCloneCommand) RunE(cmd *cobra.Command, args []string) error {
 		slog.Info("Custom args.", "args", containerCmd)
 	}
 
-	containerCli, err := container.NewContainerClient()
+	containerCli, err := container.NewContainerClient(context.TODO())
 	if err != nil {
 		return err
 	}
diff --git a/cli/tools/container_logs.go b/cli/tools/container_logs.go
@@ -54,7 +54,7 @@ func NewContainerLogsCommand(ctx cli.Cli) *cobra.Command {
 func (c *ContainerLogsCommand) RunE(cmd *cobra.Command, args []string) error {
 	slog.Debug("Executing", "cmd", cmd.CalledAs(), "args", args)
 
-	containerCli, err := container.NewContainerClient()
+	containerCli, err := container.NewContainerClient(context.TODO())
 	if err != nil {
 		return err
 	}
diff --git a/cli/tools/container_remove.go b/cli/tools/container_remove.go
@@ -46,7 +46,7 @@ func NewContainerRemoveCommand(ctx cli.Cli) *cobra.Command {
 func (c *ContainerRemoveCommand) RunE(cmd *cobra.Command, args []string) error {
 	slog.Debug("Executing", "cmd", cmd.CalledAs(), "args", args)
 
-	containerCli, err := container.NewContainerClient()
+	containerCli, err := container.NewContainerClient(context.TODO())
 	if err != nil {
 		return err
 	}
diff --git a/cli/tools/container_restart.go b/cli/tools/container_restart.go
@@ -40,7 +40,7 @@ func NewContainerRestartCommand(ctx cli.Cli) *cobra.Command {
 func (c *ContainerRestartCommand) RunE(cmd *cobra.Command, args []string) error {
 	slog.Debug("Executing", "cmd", cmd.CalledAs(), "args", args)
 
-	containerCli, err := container.NewContainerClient()
+	containerCli, err := container.NewContainerClient(context.TODO())
 	if err != nil {
 		return err
 	}
diff --git a/cli/tools/run_context.go b/cli/tools/run_context.go
@@ -59,7 +59,7 @@ func NewContainerRunInContextCommand(ctx cli.Cli) *cobra.Command {
 func (c *ContainerRunInContextCommand) RunE(cmd *cobra.Command, args []string) error {
 	slog.Debug("Executing", "cmd", cmd.CalledAs(), "args", args)
 
-	containerCli, err := container.NewContainerClient()
+	containerCli, err := container.NewContainerClient(context.TODO())
 	if err != nil {
 		return err
 	}
diff --git a/pkg/app/app.go b/pkg/app/app.go
@@ -96,7 +96,14 @@ func NewApp(device tedge.Target, config Config) (*App, error) {
 	}
 	tedgeClient := tedge.NewClient(device, *serviceTarget, config.ServiceName, tedgeOpts)
 
-	containerClient, err := container.NewContainerClient()
+	ctx, ctxCancel := context.WithTimeout(context.TODO(), 300*time.Second)
+	defer ctxCancel()
+
+	containerClient, err := container.NewContainerClient(
+		ctx,
+		// Use a time-based timeout instead of limiting number of retries
+		container.WithInfiniteRetries(),
+	)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/container/container.go b/pkg/container/container.go
@@ -290,29 +290,122 @@ func findContainerEngineSocket() (socketAddr string) {
 	return socketAddr
 }
 
-func NewContainerClient() (*ContainerClient, error) {
+type Opt func(o *ClientOptions) error
+
+type ClientOptions struct {
+	Attempts      int
+	Host          string
+	RetryInterval time.Duration
+}
+
+func WithAttempts(total int) Opt {
+	return func(o *ClientOptions) error {
+		o.Attempts = total
+		return nil
+	}
+}
+
+func WithInfiniteRetries() Opt {
+	return func(o *ClientOptions) error {
+		o.Attempts = -1
+		return nil
+	}
+}
+
+func WithHost(host string) Opt {
+	return func(o *ClientOptions) error {
+		o.Host = host
+		return nil
+	}
+}
+func WithRetryInterval(v time.Duration) Opt {
+	return func(o *ClientOptions) error {
+		o.RetryInterval = v
+		return nil
+	}
+}
+
+func newContainerClient(options *ClientOptions) (*ContainerClient, error) {
 	// Find container socket
-	addr := findContainerEngineSocket()
-	if addr != "" {
-		if err := os.Setenv("DOCKER_HOST", addr); err != nil {
-			return nil, err
-		}
-		// Used by podman and podman-remote
-		if err := os.Setenv("CONTAINER_HOST", addr); err != nil {
-			return nil, err
-		}
-		slog.Info("Using container engine socket.", "value", addr)
+	if options.Host == "" {
+		options.Host = findContainerEngineSocket()
+	}
+	if options.Host == "" {
+		return nil, fmt.Errorf("container engine socket was not found")
+	}
+	slog.Info("Using container engine socket.", "value", options.Host)
+	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithHost(options.Host), client.WithAPIVersionNegotiation())
+	if err != nil {
+		return nil, err
 	}
 
-	cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+	// Check if engine is functional
+	engineInfo, err := cli.Info(context.Background())
 	if err != nil {
 		return nil, err
 	}
+	slog.Info("Found container engine", "name", engineInfo.Name, "version", engineInfo.ServerVersion, "os", engineInfo.OperatingSystem, "osVersion", engineInfo.OSVersion)
+
+	// Only set env variables after the engine has been verified
+	if err := os.Setenv("DOCKER_HOST", options.Host); err != nil {
+		return nil, err
+	}
+	// Used by podman and podman-remote
+	if err := os.Setenv("CONTAINER_HOST", options.Host); err != nil {
+		return nil, err
+	}
+
 	return &ContainerClient{
 		Client: cli,
 	}, nil
 }
 
+func NewContainerClient(ctx context.Context, opts ...Opt) (*ContainerClient, error) {
+	options := &ClientOptions{
+		Attempts:      5,
+		RetryInterval: 5 * time.Second,
+	}
+	for _, opt := range opts {
+		if err := opt(options); err != nil {
+			return nil, err
+		}
+	}
+
+	attempts := 0
+	var client *ContainerClient
+	var err error
+
+	for {
+		client, err = newContainerClient(options)
+		attempts += 1
+
+		if err == nil {
+			return client, nil
+		}
+		slog.Info("Could not create container engine client", "err", err)
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+
+		if options.Attempts != -1 && attempts >= options.Attempts {
+			return nil, fmt.Errorf("failed to create container engine client after %d attempts", attempts)
+		}
+
+		t := time.NewTimer(options.RetryInterval)
+		select {
+		case <-ctx.Done():
+			slog.Info("context expired")
+			t.Stop()
+			if ctx.Err() == nil {
+				return nil, fmt.Errorf("failed to create container engine client")
+			}
+			return nil, ctx.Err()
+		case <-t.C:
+			slog.Info("Retrying to create container engine client")
+		}
+	}
+}
+
 type ContainerTelemetryMessage struct {
 	Container ContainerStats `json:"container"`
 }
diff --git a/pkg/container/container_test.go b/pkg/container/container_test.go
@@ -61,8 +61,8 @@ func Test_FilterEnvVariables(t *testing.T) {
 	assert.Equal(t, out[1], "BAR=2")
 }
 
-func Test_PruneIMages(t *testing.T) {
-	client, err := NewContainerClient()
+func Test_PruneImages(t *testing.T) {
+	client, err := NewContainerClient(context.TODO())
 	if err != nil {
 		t.Fatal(err)
 	}

Original file line number	Diff line number	Diff line change
`@@ -24,7 +24,7 @@ func NewFinalizeCommand(ctx cli.Cli) *cobra.Command {`
`24`	`24`	`if !pruneImages {`
`25`	`25`	`return nil`
`26`	`26`	`}`
`27`		`- cli, err := container.NewContainerClient()`
	`27`	`+ cli, err := container.NewContainerClient(context.TODO())`
`28`	`28`	`if err != nil {`
`29`	`29`	`return err`
`30`	`30`	`}`
Original file line number	Diff line number	Diff line change
`@@ -75,7 +75,7 @@ func (c InstallCommand) RunE(cmd cobra.Command, args []string) error {`
`75`	`75`	`// Only enable pulling if the user is providing a file`
`76`	`76`	`disablePull := c.File != ""`
`77`	`77`
`78`		`- cli, err := container.NewContainerClient()`
	`78`	`+ cli, err := container.NewContainerClient(context.TODO())`
`79`	`79`	`if err != nil {`
`80`	`80`	`return err`
`81`	`81`	`}`
Original file line number	Diff line number	Diff line change
`@@ -22,7 +22,7 @@ func NewListCommand(cliContext cli.Cli) *cobra.Command {`
`22`	`22`	`RunE: func(cmd *cobra.Command, args []string) error {`
`23`	`23`	`slog.Info("Executing", "cmd", cmd.CalledAs(), "args", args)`
`24`	`24`	`ctx := context.Background()`
`25`		`- cli, err := container.NewContainerClient()`
	`25`	`+ cli, err := container.NewContainerClient(ctx)`
`26`	`26`	`if err != nil {`
`27`	`27`	`return err`
`28`	`28`	`}`
Original file line number	Diff line number	Diff line change
`@@ -35,7 +35,7 @@ Example 1: Remove a container`
`35`	`35`	`ctx := context.Background()`
`36`	`36`	`containerName := args[0]`
`37`	`37`
`38`		`- cli, err := container.NewContainerClient()`
	`38`	`+ cli, err := container.NewContainerClient(ctx)`
`39`	`39`	`if err != nil {`
`40`	`40`	`return err`
`41`	`41`	`}`
Original file line number	Diff line number	Diff line change
`@@ -52,7 +52,7 @@ func (c InstallCommand) RunE(cmd cobra.Command, args []string) error {`
`52`	`52`	`projectName := args[0]`
`53`	`53`	`stderr := cmd.ErrOrStderr()`
`54`	`54`
`55`		`- cli, err := container.NewContainerClient()`
	`55`	`+ cli, err := container.NewContainerClient(context.TODO())`
`56`	`56`	`if err != nil {`
`57`	`57`	`return err`
`58`	`58`	`}`
Original file line number	Diff line number	Diff line change
`@@ -42,7 +42,7 @@ func NewListCommand(cliContext cli.Cli) *cobra.Command {`
`42`	`42`	`RunE: func(cmd *cobra.Command, args []string) error {`
`43`	`43`	`slog.Info("Executing", "cmd", cmd.CalledAs(), "args", args)`
`44`	`44`	`ctx := context.Background()`
`45`		`- cli, err := container.NewContainerClient()`
	`45`	`+ cli, err := container.NewContainerClient(context.TODO())`
`46`	`46`	`if err != nil {`
`47`	`47`	`return err`
`48`	`48`	`}`
Original file line number	Diff line number	Diff line change
`@@ -40,7 +40,7 @@ func (c RemoveCommand) RunE(cmd cobra.Command, args []string) error {`
`40`	`40`	`ctx := context.Background()`
`41`	`41`	`projectName := args[0]`
`42`	`42`
`43`		`- cli, err := container.NewContainerClient()`
	`43`	`+ cli, err := container.NewContainerClient(context.TODO())`
`44`	`44`	`if err != nil {`
`45`	`45`	`return err`
`46`	`46`	`}`
Original file line number	Diff line number	Diff line change
`@@ -78,7 +78,7 @@ func (c CheckCommand) RunE(cmd cobra.Command, args []string) error {`
`78`	`78`	`}`
`79`	`79`
`80`	`80`	`// Check container self`
`81`		`- containerCLI, err := container.NewContainerClient()`
	`81`	`+ containerCLI, err := container.NewContainerClient(context.TODO())`
`82`	`82`	`if err != nil {`
`83`	`83`	`return err`
`84`	`84`	`}`