fix(inputs.docker): use startup-error-behavior framework for connection failures

Implement the startup-error-behavior framework (TSD-006) to handle
Docker daemon unavailability during startup.

This allows users to configure retry behavior via the
startup_error_behavior option (error, retry, ignore, probe) instead of
silently logging warnings and deferring connection to the first Gather.

Author: skartikey

plugins/inputs/docker/client.go

@@ -12,10 +12,15 @@ import (
 	"github.com/docker/docker/client"
 )
 
+// IsErrConnectionFailed returns true if the error is caused by connection failure.
+// This is a passthrough to the docker client library function.
+var IsErrConnectionFailed = client.IsErrConnectionFailed
+
 var (
 	defaultHeaders = map[string]string{"User-Agent": "engine-api-cli-1.0"}
 )
 
+//nolint:interfacebloat // wrapping upstream docker client which has many methods
 type dockerClient interface {
 	// Info retrieves system-wide information about the Docker server.
 	Info(ctx context.Context) (system.Info, error)
@@ -35,6 +40,8 @@ type dockerClient interface {
 	DiskUsage(ctx context.Context, options types.DiskUsageOptions) (types.DiskUsage, error)
 	// ClientVersion retrieves the version of the Docker client.
 	ClientVersion() string
+	// Ping pings the server and returns information about the server.
+	Ping(ctx context.Context) (types.Ping, error)
 	// Close releases any resources held by the client.
 	Close() error
 }
@@ -114,6 +121,11 @@ func (c *socketClient) ClientVersion() string {
 	return c.client.ClientVersion()
 }
 
+// Ping pings the server and returns information about the server.
+func (c *socketClient) Ping(ctx context.Context) (types.Ping, error) {
+	return c.client.Ping(ctx)
+}
+
 // Close releases any resources held by the client.
 func (c *socketClient) Close() error {
 	return c.client.Close()

plugins/inputs/docker/docker.go

@@ -25,6 +25,7 @@ import (
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/config"
 	"github.com/influxdata/telegraf/filter"
+	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/internal/choice"
 	"github.com/influxdata/telegraf/internal/docker"
 	docker_stats "github.com/influxdata/telegraf/plugins/common/docker"
@@ -143,25 +144,29 @@ func (d *Docker) Init() error {
 }
 
 func (d *Docker) Start(telegraf.Accumulator) error {
-	// Attempt initial connection but don't fail if Docker is unavailable.
-	// This preserves backwards compatibility where Telegraf starts even when
-	// Docker daemon is not running.
-	if err := d.initClient(); err != nil {
-		d.Log.Warnf("Failed to connect to Docker daemon during startup: %v. Will retry on first gather.", err)
-	}
-	return nil
-}
-
-// initClient initializes the Docker client and performs Podman detection.
-// Returns an error if the connection fails, but does not prevent Telegraf from starting.
-func (d *Docker) initClient() error {
 	// Get client
 	c, err := d.getNewClient()
 	if err != nil {
-		return err
+		return &internal.StartupError{
+			Err:   fmt.Errorf("failed to create Docker client: %w", err),
+			Retry: IsErrConnectionFailed(err),
+		}
 	}
 	d.client = c
 
+	// Use Ping to check connectivity, this is a lightweight check
+	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(d.Timeout))
+	_, err = d.client.Ping(ctx)
+	cancel()
+	if err != nil {
+		d.client.Close()
+		d.client = nil
+		return &internal.StartupError{
+			Err:   fmt.Errorf("failed to ping Docker daemon: %w", err),
+			Retry: IsErrConnectionFailed(err),
+		}
+	}
+
 	// Check API version compatibility
 	version, err := semver.NewVersion(d.client.ClientVersion())
 	if err != nil {
@@ -179,14 +184,17 @@ func (d *Docker) initClient() error {
 	}
 
 	// Get info from docker daemon for Podman detection
-	ctx, cancel := context.WithTimeout(context.Background(), time.Duration(d.Timeout))
+	ctx, cancel = context.WithTimeout(context.Background(), time.Duration(d.Timeout))
 	defer cancel()
 
 	info, err := d.client.Info(ctx)
 	if err != nil {
 		d.client.Close()
 		d.client = nil
-		return fmt.Errorf("failed to get Docker info: %w", err)
+		return &internal.StartupError{
+			Err:   fmt.Errorf("failed to get Docker info: %w", err),
+			Retry: IsErrConnectionFailed(err),
+		}
 	}
 
 	d.engineHost = info.Name
@@ -213,11 +221,8 @@ func (d *Docker) Stop() {
 }
 
 func (d *Docker) Gather(acc telegraf.Accumulator) error {
-	// If client is not initialized, try to connect now
 	if d.client == nil {
-		if err := d.initClient(); err != nil {
-			return fmt.Errorf("failed to connect to Docker daemon: %w", err)
-		}
+		return errors.New("docker client not initialized")
 	}
 
 	// Create label filters if not already created

plugins/inputs/docker/docker_test.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/config"
+	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/internal/choice"
 	"github.com/influxdata/telegraf/testutil"
 )
@@ -33,6 +34,7 @@ type mockClient struct {
 	NodeListF         func() ([]swarm.Node, error)
 	DiskUsageF        func() (types.DiskUsage, error)
 	ClientVersionF    func() string
+	PingF             func() (types.Ping, error)
 	CloseF            func() error
 }
 
@@ -72,6 +74,10 @@ func (c *mockClient) ClientVersion() string {
 	return c.ClientVersionF()
 }
 
+func (c *mockClient) Ping(context.Context) (types.Ping, error) {
+	return c.PingF()
+}
+
 func (c *mockClient) Close() error {
 	return c.CloseF()
 }
@@ -104,6 +110,9 @@ var baseClient = mockClient{
 	ClientVersionF: func() string {
 		return version
 	},
+	PingF: func() (types.Ping, error) {
+		return types.Ping{}, nil
+	},
 	CloseF: func() error {
 		return nil
 	},
@@ -422,7 +431,8 @@ func TestDocker_WindowsMemoryContainerStats(t *testing.T) {
 	var acc testutil.Accumulator
 
 	d := Docker{
-		Log: testutil.Logger{},
+		Log:     testutil.Logger{},
+		Timeout: config.Duration(5 * time.Second),
 		newClient: func(string, *tls.Config) (dockerClient, error) {
 			return &mockClient{
 				InfoF: func() (system.Info, error) {
@@ -452,6 +462,9 @@ func TestDocker_WindowsMemoryContainerStats(t *testing.T) {
 				ClientVersionF: func() string {
 					return version
 				},
+				PingF: func() (types.Ping, error) {
+					return types.Ping{}, nil
+				},
 				CloseF: func() error {
 					return nil
 				},
@@ -1694,6 +1707,7 @@ func TestPodmanDetection(t *testing.T) {
 			var acc testutil.Accumulator
 			d := Docker{
 				Endpoint: tt.endpoint,
+				Timeout:  config.Duration(5 * time.Second),
 				newClient: func(string, *tls.Config) (dockerClient, error) {
 					return &mockClient{
 						InfoF: func() (system.Info, error) {
@@ -1712,6 +1726,9 @@ func TestPodmanDetection(t *testing.T) {
 						ClientVersionF: func() string {
 							return "1.24.0"
 						},
+						PingF: func() (types.Ping, error) {
+							return types.Ping{}, nil
+						},
 						CloseF: func() error {
 							return nil
 						},
@@ -1777,8 +1794,7 @@ func TestPodmanStatsCache(t *testing.T) {
 }
 
 func TestStartWithUnavailableDocker(t *testing.T) {
-	// Test that Telegraf starts successfully even when Docker is unavailable
-	// This is a regression test for https://github.com/influxdata/telegraf/issues/18089
+	// Test that Start returns a retryable StartupError when Docker is unavailable
 	var acc testutil.Accumulator
 	d := Docker{
 		Log: testutil.Logger{},
@@ -1791,42 +1807,70 @@ func TestStartWithUnavailableDocker(t *testing.T) {
 	}
 
 	require.NoError(t, d.Init())
-	// Start should NOT return an error even when Docker is unavailable
-	require.NoError(t, d.Start(&acc))
-	// Client should be nil since connection failed
-	require.Nil(t, d.client)
 
-	// Gather should return an error since Docker is still unavailable
-	err := d.Gather(&acc)
+	// Start should return a StartupError when Docker is unavailable
+	err := d.Start(&acc)
 	require.Error(t, err)
-	require.Contains(t, err.Error(), "failed to connect to Docker daemon")
+
+	var startupErr *internal.StartupError
+	require.ErrorAs(t, err, &startupErr)
+	require.Contains(t, startupErr.Error(), "failed to create Docker client")
 }
 
-func TestLazyClientInitialization(t *testing.T) {
-	// Test that client is initialized lazily on first Gather if Start failed to connect
+func TestStartWithPingFailure(t *testing.T) {
+	// Test that Start returns a retryable StartupError when Ping fails due to connection issues
 	var acc testutil.Accumulator
 
-	// Track connection attempts
-	connectionAttempts := 0
+	// Create a mock client that succeeds on creation but fails on Ping
+	d := Docker{
+		Log:     testutil.Logger{},
+		Timeout: config.Duration(5 * time.Second),
+		newClient: func(string, *tls.Config) (dockerClient, error) {
+			return &mockClient{
+				PingF: func() (types.Ping, error) {
+					return types.Ping{}, errors.New("connection refused")
+				},
+				CloseF: func() error {
+					return nil
+				},
+			}, nil
+		},
+		newEnvClient: func() (dockerClient, error) {
+			return nil, errors.New("not using env client")
+		},
+	}
+
+	require.NoError(t, d.Init())
+
+	// Start should return a StartupError when Ping fails
+	err := d.Start(&acc)
+	require.Error(t, err)
+
+	var startupErr *internal.StartupError
+	require.ErrorAs(t, err, &startupErr)
+	require.Contains(t, startupErr.Error(), "failed to ping Docker daemon")
+	// Client should be nil since we clean up on failure
+	require.Nil(t, d.client)
+}
+
+func TestStartSuccess(t *testing.T) {
+	// Test that Start succeeds when Docker is available
+	var acc testutil.Accumulator
 
 	d := Docker{
-		Log: testutil.Logger{},
+		Log:     testutil.Logger{},
+		Timeout: config.Duration(5 * time.Second),
 		newClient: func(string, *tls.Config) (dockerClient, error) {
-			connectionAttempts++
-			// First attempt fails, subsequent attempts succeed
-			if connectionAttempts == 1 {
-				return nil, errors.New("docker daemon not ready")
-			}
 			return &mockClient{
+				PingF: func() (types.Ping, error) {
+					return types.Ping{}, nil
+				},
 				InfoF: func() (system.Info, error) {
 					return system.Info{
 						Name:          "docker-desktop",
 						ServerVersion: "20.10.0",
 					}, nil
 				},
-				ContainerListF: func(container.ListOptions) ([]container.Summary, error) {
-					return nil, nil
-				},
 				ClientVersionF: func() string {
 					return "1.24.0"
 				},
@@ -1841,18 +1885,6 @@ func TestLazyClientInitialization(t *testing.T) {
 	}
 
 	require.NoError(t, d.Init())
-	// Start should succeed even though connection fails
 	require.NoError(t, d.Start(&acc))
-	require.Equal(t, 1, connectionAttempts)
-	require.Nil(t, d.client)
-
-	// First Gather fails because Docker is still unavailable (same mock returns error on attempt 1)
-	// Reset connection attempts to simulate Docker becoming available
-	connectionAttempts = 1 // Set to 1 so next attempt (2) will succeed
-
-	// Second Gather should succeed after lazy initialization
-	err := d.Gather(&acc)
-	require.NoError(t, err)
-	require.Equal(t, 2, connectionAttempts)
 	require.NotNil(t, d.client)
 }