kubernetes.go

//go:build !no_workceptor
// +build !no_workceptor

package workceptor

import (
	"bufio"
	"context"
	"errors"
	"fmt"
	"io"
	"math"
	"net"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/ghjm/cmdline"
	"github.com/google/shlex"
	"github.com/spf13/viper"
	batchv1 "k8s.io/api/batch/v1"
	corev1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/version"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/tools/remotecommand"
	watch2 "k8s.io/client-go/tools/watch"
	"k8s.io/client-go/util/flowcontrol"
)

// KubeUnit implements the WorkUnit interface.
type KubeUnit struct {
	BaseWorkUnitForWorkUnit
	KubeAPIWrapperInstance KubeAPIer
	authMethod             string
	streamMethod           string
	baseParams             string
	allowRuntimeAuth       bool
	allowRuntimeCommand    bool
	allowRuntimeParams     bool
	allowRuntimePod        bool
	deletePodOnRestart     bool
	namePrefix             string
	config                 *rest.Config
	clientset              kubernetes.Interface
	Pod                    *corev1.Pod
	Job                    *batchv1.Job
	isJobSpec              bool
	podPendingTimeout      time.Duration
}

// kubeExtraData is the content of the ExtraData JSON field for a Kubernetes worker.
type KubeExtraData struct {
	Image         string
	Command       string
	Params        string
	KubeNamespace string
	KubeConfig    string
	KubePod       string
	PodName       string
}

type KubeAPIer interface {
	NewNotFound(schema.GroupResource, string) *apierrors.StatusError
	OneTermEqualSelector(string, string) fields.Selector
	NewForConfig(*rest.Config) (kubernetes.Interface, error)
	GetLogs(kubernetes.Interface, string, string, *corev1.PodLogOptions) *rest.Request
	Get(context.Context, kubernetes.Interface, string, string, metav1.GetOptions) (*corev1.Pod, error)
	Create(context.Context, kubernetes.Interface, string, *corev1.Pod, metav1.CreateOptions) (*corev1.Pod, error)
	List(context.Context, kubernetes.Interface, string, metav1.ListOptions) (*corev1.PodList, error)
	Watch(context.Context, kubernetes.Interface, string, metav1.ListOptions) (watch.Interface, error)
	Delete(context.Context, kubernetes.Interface, string, string, metav1.DeleteOptions) error
	CreateJob(context.Context, kubernetes.Interface, string, *batchv1.Job, metav1.CreateOptions) (*batchv1.Job, error)
	GetJob(context.Context, kubernetes.Interface, string, string, metav1.GetOptions) (*batchv1.Job, error)
	DeleteJob(context.Context, kubernetes.Interface, string, string, metav1.DeleteOptions) error
	SubResource(kubernetes.Interface, string, string) *rest.Request
	InClusterConfig() (*rest.Config, error)
	NewDefaultClientConfigLoadingRules() *clientcmd.ClientConfigLoadingRules
	BuildConfigFromFlags(string, string) (*rest.Config, error)
	NewClientConfigFromBytes([]byte) (clientcmd.ClientConfig, error)
	NewSPDYExecutor(*rest.Config, string, *url.URL) (remotecommand.Executor, error)
	StreamWithContext(context.Context, remotecommand.Executor, remotecommand.StreamOptions) error
	UntilWithSync(context.Context, cache.ListerWatcher, runtime.Object, watch2.PreconditionFunc, ...watch2.ConditionFunc) (*watch.Event, error)
	NewFakeNeverRateLimiter() flowcontrol.RateLimiter
	NewFakeAlwaysRateLimiter() flowcontrol.RateLimiter
}

type KubeAPIWrapper struct{}

func (ku KubeAPIWrapper) NewNotFound(qualifiedResource schema.GroupResource, name string) *apierrors.StatusError {
	return apierrors.NewNotFound(qualifiedResource, name)
}

func (ku KubeAPIWrapper) OneTermEqualSelector(k string, v string) fields.Selector {
	return fields.OneTermEqualSelector(k, v)
}

func (ku KubeAPIWrapper) NewForConfig(c *rest.Config) (kubernetes.Interface, error) {
	return kubernetes.NewForConfig(c)
}

func (ku KubeAPIWrapper) GetLogs(clientset kubernetes.Interface, namespace string, name string, opts *corev1.PodLogOptions) *rest.Request {
	return clientset.CoreV1().Pods(namespace).GetLogs(name, opts)
}

func (ku KubeAPIWrapper) Get(ctx context.Context, clientset kubernetes.Interface, namespace string, name string, opts metav1.GetOptions) (*corev1.Pod, error) {
	return clientset.CoreV1().Pods(namespace).Get(ctx, name, opts)
}

func (ku KubeAPIWrapper) Create(ctx context.Context, clientset kubernetes.Interface, namespace string, pod *corev1.Pod, opts metav1.CreateOptions) (*corev1.Pod, error) {
	return clientset.CoreV1().Pods(namespace).Create(ctx, pod, opts)
}

func (ku KubeAPIWrapper) List(ctx context.Context, clientset kubernetes.Interface, namespace string, opts metav1.ListOptions) (*corev1.PodList, error) {
	return clientset.CoreV1().Pods(namespace).List(ctx, opts)
}

func (ku KubeAPIWrapper) Watch(ctx context.Context, clientset kubernetes.Interface, namespace string, opts metav1.ListOptions) (watch.Interface, error) {
	return clientset.CoreV1().Pods(namespace).Watch(ctx, opts)
}

func (ku KubeAPIWrapper) Delete(ctx context.Context, clientset kubernetes.Interface, namespace string, name string, opts metav1.DeleteOptions) error {
	return clientset.CoreV1().Pods(namespace).Delete(ctx, name, opts)
}

func (ku KubeAPIWrapper) CreateJob(ctx context.Context, clientset kubernetes.Interface, namespace string, job *batchv1.Job, opts metav1.CreateOptions) (*batchv1.Job, error) {
	return clientset.BatchV1().Jobs(namespace).Create(ctx, job, opts)
}

func (ku KubeAPIWrapper) GetJob(ctx context.Context, clientset kubernetes.Interface, namespace string, name string, opts metav1.GetOptions) (*batchv1.Job, error) {
	return clientset.BatchV1().Jobs(namespace).Get(ctx, name, opts)
}

func (ku KubeAPIWrapper) DeleteJob(ctx context.Context, clientset kubernetes.Interface, namespace string, name string, opts metav1.DeleteOptions) error {
	return clientset.BatchV1().Jobs(namespace).Delete(ctx, name, opts)
}

func (ku KubeAPIWrapper) SubResource(clientset kubernetes.Interface, podName string, podNamespace string) *rest.Request {
	return clientset.CoreV1().RESTClient().Post().Resource("pods").Name(podName).Namespace(podNamespace).SubResource("attach")
}

func (ku KubeAPIWrapper) InClusterConfig() (*rest.Config, error) {
	return rest.InClusterConfig()
}

func (ku KubeAPIWrapper) NewDefaultClientConfigLoadingRules() *clientcmd.ClientConfigLoadingRules {
	return clientcmd.NewDefaultClientConfigLoadingRules()
}

func (ku KubeAPIWrapper) BuildConfigFromFlags(masterURL string, kubeconfigPath string) (*rest.Config, error) {
	return clientcmd.BuildConfigFromFlags(masterURL, kubeconfigPath)
}

func (ku KubeAPIWrapper) NewClientConfigFromBytes(configBytes []byte) (clientcmd.ClientConfig, error) {
	return clientcmd.NewClientConfigFromBytes(configBytes)
}

func (ku KubeAPIWrapper) NewSPDYExecutor(config *rest.Config, method string, url *url.URL) (remotecommand.Executor, error) {
	return remotecommand.NewSPDYExecutor(config, method, url)
}

func (ku KubeAPIWrapper) StreamWithContext(ctx context.Context, exec remotecommand.Executor, options remotecommand.StreamOptions) error {
	return exec.StreamWithContext(ctx, options)
}

func (ku KubeAPIWrapper) UntilWithSync(ctx context.Context, lw cache.ListerWatcher, objType runtime.Object, precondition watch2.PreconditionFunc, conditions ...watch2.ConditionFunc) (*watch.Event, error) {
	return watch2.UntilWithSync(ctx, lw, objType, precondition, conditions...)
}

func (ku KubeAPIWrapper) NewFakeNeverRateLimiter() flowcontrol.RateLimiter {
	return flowcontrol.NewFakeNeverRateLimiter()
}

func (ku KubeAPIWrapper) NewFakeAlwaysRateLimiter() flowcontrol.RateLimiter {
	return flowcontrol.NewFakeAlwaysRateLimiter()
}

// ErrPodCompleted is returned when pod has already completed before we could attach.
var ErrPodCompleted = fmt.Errorf("pod ran to completion")

// ErrPodFailed is returned when pod has failed before we could attach.
var ErrPodFailed = fmt.Errorf("pod failed to start")

// ErrImagePullBackOff is returned when the image for the container in the Pod cannot be pulled.
var ErrImagePullBackOff = fmt.Errorf("container failed to start")

const WorkerContainerName = "worker"

// podRunningAndReady is a completion criterion for pod ready to be attached to.
func podRunningAndReady(kw KubeUnit) func(event watch.Event) (bool, error) {
	imagePullBackOffRetries := 3
	inner := func(event watch.Event) (bool, error) {
		if event.Type == watch.Deleted {
			return false, kw.KubeAPIWrapperInstance.NewNotFound(schema.GroupResource{Resource: "pods"}, "")
		}
		if t, ok := event.Object.(*corev1.Pod); ok {
			switch t.Status.Phase {
			case corev1.PodFailed:
				return false, ErrPodFailed
			case corev1.PodSucceeded:
				return false, ErrPodCompleted
			case corev1.PodRunning, corev1.PodPending:
				conditions := t.Status.Conditions
				if conditions == nil {
					return false, nil
				}
				for i := range conditions {
					if conditions[i].Type == corev1.PodReady &&
						conditions[i].Status == corev1.ConditionTrue {
						return true, nil
					}
					if conditions[i].Type == corev1.ContainersReady &&
						conditions[i].Status == corev1.ConditionFalse {
						statuses := t.Status.ContainerStatuses
						for j := range statuses {
							if statuses[j].State.Waiting != nil {
								if statuses[j].State.Waiting.Reason == "ImagePullBackOff" {
									if imagePullBackOffRetries == 0 {
										return false, ErrImagePullBackOff
									}
									imagePullBackOffRetries--
								}
							}
						}
					}
				}
			}
		}

		return false, nil
	}

	return inner
}

func (kw *KubeUnit) waitForJobPod() error {
	jobName := kw.Job.Name
	namespace := kw.Job.Namespace

	for i := 0; i < 60; i++ {
		pods, err := kw.KubeAPIWrapperInstance.List(kw.GetContext(), kw.clientset, namespace, metav1.ListOptions{
			LabelSelector: fmt.Sprintf("job-name=%s", jobName),
		})
		if err != nil {
			return fmt.Errorf("failed to list pods for job %s: %w", jobName, err)
		}

		if len(pods.Items) > 0 {
			kw.Pod = &pods.Items[0]
			kw.UpdateFullStatus(func(status *StatusFileData) {
				status.State = WorkStatePending
				status.Detail = "Job created, pod found"
				status.StdoutSize = 0
				status.ExtraData.(*KubeExtraData).PodName = kw.Pod.Name
			})

			return nil
		}

		time.Sleep(500 * time.Millisecond)
	}

	return fmt.Errorf("job %s did not create any pods within 30 seconds", jobName)
}

func (kw *KubeUnit) validateAndSetupWorkerContainer(spec *corev1.PodSpec) error {
	foundWorker := false
	for i := range spec.Containers {
		if spec.Containers[i].Name == WorkerContainerName {
			spec.Containers[i].Stdin = true
			spec.Containers[i].StdinOnce = true
			foundWorker = true

			break
		}
	}
	if !foundWorker {
		return fmt.Errorf("at least one container must be named worker")
	}
	spec.RestartPolicy = corev1.RestartPolicyNever

	return nil
}

func (kw *KubeUnit) extractMetadata(objectMeta *metav1.ObjectMeta, ked *KubeExtraData) {
	if objectMeta.Namespace != "" {
		ked.KubeNamespace = objectMeta.Namespace
	}
	if objectMeta.Name != "" {
		kw.namePrefix = objectMeta.Name + "-"
	}
}

func (kw *KubeUnit) GetKubeTimeoutStart() time.Duration {
	// RECEPTOR_KUBE_TIMEOUT_START
	// default: 1 second
	kubeTimeoutStart := 1 * time.Second
	envTimeout := os.Getenv("RECEPTOR_KUBE_TIMEOUT_START")
	if envTimeout != "" {
		var err error
		kubeTimeoutStart, err = time.ParseDuration(envTimeout)
		if err != nil || kubeTimeoutStart <= 0 {
			// ignore error, use default
			kw.GetWorkceptor().nc.GetLogger().Warning("Invalid value for RECEPTOR_KUBE_TIMEOUT_START: %s. Ignoring", envTimeout)
			kubeTimeoutStart = 1 * time.Second
		}
		// ignore if exceeds limit, use max
		if kubeTimeoutStart > time.Minute*1 {
			kw.GetWorkceptor().nc.GetLogger().Warning("RECEPTOR_KUBE_TIMEOUT_START of: %d is larger than the max timeout of 1m. Max of 1m will be used", kubeTimeoutStart)
			kubeTimeoutStart = time.Minute * 1
		}
	}
	kw.GetWorkceptor().nc.GetLogger().Debug("RECEPTOR_KUBE_TIMEOUT_START: %s", kubeTimeoutStart)

	return kubeTimeoutStart
}

func (kw *KubeUnit) GetKubeRetryCount() int {
	// RECEPTOR_KUBE_RETRY_COUNT
	// default: 5
	kubeRetryCount := 5
	envRetryCount := os.Getenv("RECEPTOR_KUBE_RETRY_COUNT")
	if envRetryCount != "" {
		var err error
		kubeRetryCount, err = strconv.Atoi(envRetryCount)
		if err != nil || kubeRetryCount < 1 {
			// ignore error, use default
			kw.GetWorkceptor().nc.GetLogger().Warning("Invalid value for RECEPTOR_KUBE_RETRY_COUNT: %s. Default of 5 will be used", envRetryCount)
			kubeRetryCount = 5
		}
		// ignore if exceeds limit, use max retry
		if kubeRetryCount > 100 {
			kw.GetWorkceptor().nc.GetLogger().Warning("RECEPTOR_KUBE_RETRY_COUNT of: %d is larger than the max retry count of 100. Retry count of 100 will be used", kubeRetryCount)
			kubeRetryCount = 100
		}
	}
	kw.GetWorkceptor().nc.GetLogger().Debug("RECEPTOR_KUBE_RETRY_COUNT: %d", kubeRetryCount)

	return kubeRetryCount
}

func (kw *KubeUnit) GetSleepDuration(multipler int) time.Duration {
	maxSleepDuration := time.Minute * 5
	baseTimeout := int64(kw.GetKubeTimeoutStart())

	if baseTimeout > 0 && int64(multipler) > math.MaxInt64/baseTimeout {
		return maxSleepDuration
	}

	sleepDuration := kw.GetKubeTimeoutStart() * time.Duration(multipler)
	if sleepDuration > maxSleepDuration {
		return maxSleepDuration
	}

	return sleepDuration
}

func (kw *KubeUnit) kubeLoggingConnectionHandler(timestamps bool, sinceTime time.Time) (io.ReadCloser, error) {
	var logStream io.ReadCloser
	var err error
	podNamespace := kw.Pod.Namespace
	podName := kw.Pod.Name
	podOptions := &corev1.PodLogOptions{
		Container: WorkerContainerName,
		Follow:    true,
	}
	if timestamps {
		podOptions.Timestamps = true
		podOptions.SinceTime = &metav1.Time{Time: sinceTime}
	}

	logReq := kw.KubeAPIWrapperInstance.GetLogs(kw.clientset, podNamespace, podName, podOptions)
	// get logstream, with retry
	for retries := kw.GetKubeRetryCount(); retries > 0; retries-- {
		logStream, err = logReq.Stream(kw.GetContext())
		if err == nil {
			break
		}
		kw.GetWorkceptor().nc.GetLogger().Warning(
			"Error opening log stream for pod %s/%s. Will retry %d more times. Error: %s",
			podNamespace,
			podName,
			retries,
			err,
		)
		time.Sleep(kw.GetKubeTimeoutStart())
	}
	if err != nil {
		errMsg := fmt.Sprintf("Error opening log stream for pod %s/%s. Error: %s", podNamespace, podName, err)
		kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet
		kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)

		return nil, err
	}

	return logStream, nil
}

func (kw *KubeUnit) kubeLoggingNoReconnect(streamWait *sync.WaitGroup, stdout *STDoutWriter, stdoutErr *error) {
	// Legacy method, for use on k8s < v1.23.14
	// uses io.Copy to stream data from pod to stdout file
	// known issues around this, as logstream can terminate due to log rotation
	// or 4 hr timeout
	defer streamWait.Done()
	podNamespace := kw.Pod.Namespace
	podName := kw.Pod.Name
	logStream, err := kw.kubeLoggingConnectionHandler(false, time.Time{})
	if err != nil {
		return
	}

	_, *stdoutErr = io.Copy(stdout, logStream)
	if *stdoutErr != nil {
		kw.GetWorkceptor().nc.GetLogger().Error(
			"Error streaming pod logs to stdout for pod %s/%s. Error: %s",
			podNamespace,
			podName,
			*stdoutErr,
		)
	}
}

func (kw *KubeUnit) KubeLoggingWithReconnect(streamWait *sync.WaitGroup, stdout *STDoutWriter, stdinErr *error, stdoutErr *error) {
	// preferred method for k8s >= 1.23.14
	defer streamWait.Done()
	var sinceTime time.Time
	var err error
	var retryGetLogStream int
	var successfulWrite bool
	podNamespace := kw.Pod.Namespace
	podName := kw.Pod.Name

	retries := kw.GetKubeRetryCount()
	prevDelay, curDelay := 0, 1
	prevPodDelay, curPodDelay := 0, 1
	prevContainerDelay, curContainerDelay := 0, 1
	retryGetLogStream = retries

mainLoop:
	for {
		if *stdinErr != nil {
			// fail to send stdin to pod, no need to continue
			return
		}

		// get pod, with retry
		for retryGetPod := retries; retryGetPod > 0; retryGetPod-- {
			kw.Pod, err = kw.KubeAPIWrapperInstance.Get(kw.GetContext(), kw.clientset, podNamespace, podName, metav1.GetOptions{})
			if err == nil {
				break
			}
			kw.GetWorkceptor().nc.GetLogger().Warning(
				"Error getting pod %s/%s. Will retry %d more times. Error: %s",
				podNamespace,
				podName,
				retryGetPod,
				err,
			)
			time.Sleep(kw.GetSleepDuration(curPodDelay))
			prevPodDelay, curPodDelay = curPodDelay, prevPodDelay+curPodDelay
		}
		if err != nil {
			errMsg := fmt.Errorf("Error getting pod %s/%s. Error: %s", podNamespace, podName, err)
			kw.GetWorkceptor().nc.GetLogger().Error("%s", errMsg.Error())
			*stdoutErr = errMsg

			// fail to get pod, no need to continue
			return
		}
		prevPodDelay, curPodDelay = 1, 1

		// Reset successfulWrite on each reconnection attempt to ensure proper duplicate detection
		successfulWrite = false

		logStream, err := kw.kubeLoggingConnectionHandler(true, sinceTime)
		if err != nil {
			// fail to get log stream, no need to continue
			return
		}
		defer logStream.Close()

		// read from logstream
		streamReader := bufio.NewReader(logStream)
		for { // check between every line read to see if we need to stop reading
			line, err := streamReader.ReadString('\n')
			if err != nil {
				// Check if the context was canceled and the work state isn't "Succeeded".
				// If so, set the error and mark the job as failed.
				if kw.GetContext().Err() == context.Canceled {
					if kw.Status().State != WorkStateSucceeded &&
						kw.Status().State != WorkStateFailed &&
						kw.Status().State != WorkStateCanceled {
						errMsg := fmt.Sprintf("Context was canceled while reading logs for pod %s/%s. This is unrecoverable. Marking the job as failed and exiting. Error: %s",
							podNamespace,
							podName,
							err.Error(),
						)
						*stdoutErr = fmt.Errorf("%s", errMsg)
						kw.GetWorkceptor().nc.GetLogger().Error("%s", errMsg)
					}

					return
				}

				// Check if the error is not EOF, if error is not EOF retry 5 times if error persists set error and mark the job as failed.
				if err != io.EOF {
					retryGetLogStream--
					if retryGetLogStream > 0 {
						kw.GetWorkceptor().nc.GetLogger().Info(
							"Detected non-EOF Error: %s for pod %s/%s. Will retry %d more times.",
							err,
							podNamespace,
							podName,
							retryGetLogStream,
						)

						time.Sleep(kw.GetSleepDuration(curDelay))
						prevDelay, curDelay = curDelay, prevDelay+curDelay

						continue mainLoop
					}

					*stdoutErr = err
					kw.GetWorkceptor().nc.GetLogger().Error(
						"Unexpected non-EOF error while reading logs for pod %s/%s, retries exhausted. Error: %s",
						podNamespace,
						podName,
						err.Error(),
					)

					return
				}

				// EOF errors are expected in two cases.
				// 1. When the job is finished and the last line is sent.
				// In this case we monitor the container status to ensure we move out of Running,
				// and we make sure we get the last line of output.
				// 2. When the job lasts longer than 4 hours and kube api closes the log stream.
				// This is a recoverable EOF, so we attempt to reconnect with a back-off.
				// In BOTH cases, we have a simular approach, wait 1-2 seconds and check if the container status has changed.

				podDetails, kubeErr := kw.KubeAPIWrapperInstance.Get(kw.GetContext(), kw.clientset, podNamespace, podName, metav1.GetOptions{})
				if kubeErr != nil {
					// There are many reasons why the kube api might not be able to get the pod,
					// This does not mean there is a problem just yet.
					// Let's try to get the pod again, max 5 times, and decide.
					kw.GetWorkceptor().nc.GetLogger().Info("Error getting pod after reading stream: '%s' , continuing try to get pod up to 5 more times.", kubeErr)

					continue mainLoop
				}

				var containerState corev1.ContainerState
				foundContainer := false
				for _, containerStatus := range podDetails.Status.ContainerStatuses {
					if containerStatus.Name == WorkerContainerName {
						containerState = containerStatus.State
						foundContainer = true
					}
				}

				if !foundContainer {
					kw.GetWorkceptor().nc.GetLogger().Error("Unable to find the container %s for pod %s. This is unrecoverable. Marking the job as failed and exiting", WorkerContainerName, podName)
					*stdoutErr = fmt.Errorf("unable to find the container %s for pod %s. This is unrecoverable. Marking the job as failed and exiting", WorkerContainerName, podName)

					return
				}

				switch {
				case containerState.Running != nil:
					// We got EOF but pod is running, is this because we checked too fast? Will it turn into a terminated state soon or are we hitting the 4 hour log stream kube error. We will attempt to reconnect a max of 5 times in order to cover both cases
					// If we can't get reconnect without an EOF we will error and mark the job as failed.
					retryGetLogStream--
					if retryGetLogStream > 0 {
						kw.GetWorkceptor().nc.GetLogger().Info(
							"Detected EOF Error: %s for pod %s/%s in with container state: Running. Job may not be complete. Will retry %d more times.",
							err,
							podNamespace,
							podName,
							retryGetLogStream,
						)

						time.Sleep(kw.GetSleepDuration(curContainerDelay))
						prevContainerDelay, curContainerDelay = curContainerDelay, prevContainerDelay+curContainerDelay

						continue mainLoop
					}
					// Retrying hasn't worked we will error and mark the job as failed
					kw.GetWorkceptor().nc.GetLogger().Error("Container in %s pod is running but is continuing to stream EOF after retries exhausted", WorkerContainerName)
					*stdoutErr = fmt.Errorf("detected Error: %s for pod %s/%s. Pod is running but is continuing to stream EOF after retries exhausted", err,
						podNamespace,
						podName,
					)

					return
				case containerState.Terminated != nil:
					// We got EOF and the pod terminated, we will log the terminated information
					if containerState.Terminated.ExitCode != 0 {
						kw.GetWorkceptor().nc.GetLogger().Info("Container in %s pod has terminated, with nonzero exit code: %v, terminated reason: %v and terminated message: %v", WorkerContainerName, containerState.Terminated.ExitCode, containerState.Terminated.Reason, containerState.Terminated.Message)
					}

					// We need to check if last line has data
					if line != "" {
						msg, _, _ := kw.ProcessLogLine(line, sinceTime, successfulWrite)
						if msg != "" {
							_, err = stdout.Write([]byte(msg + "\n"))
							if err != nil {
								*stdoutErr = fmt.Errorf("error writing last line to stdout: %s", err)
								kw.GetWorkceptor().nc.GetLogger().Error("Error writing last line to stdout: %s", err)

								return
							}
						}
					}
					// Got EOF, terminated and ensured we captured last line then return
					return
				default:
					// We dont expect to ever get here, However, beinging in an unknown state will not have a negative effect so we will log and ignore.
					kw.GetWorkceptor().nc.GetLogger().Debug("%s is in an unexpected container state %s. This is unexpected. We will continue.", podName, containerState)
				}

				// Something has gone very wrong if we are here. EOF is true and we can get the container state, but it is not running or terminated.
				// At this stage something has gone very wrong with our interactions with the container.
				// We will fail, and mark the job as failed due to an unknown kube container state.

				kw.GetWorkceptor().nc.GetLogger().Error("received EOF on log stream for pod %s and container state is not valid %s, failing and marking the job as failed", podName, containerState)
				*stdoutErr = fmt.Errorf("received EOF on log stream for pod %s and container state is not valid %s, failing and marking the job as failed", podName, containerState)

				return
			}

			msg, newSinceTime, shouldSkip := kw.ProcessLogLine(line, sinceTime, successfulWrite)
			sinceTime = newSinceTime

			// shouldSkip is a variable that is used to represent if a line has already be read from the container, if true we already have the line, move to the next iteration
			if shouldSkip {
				continue
			}

			_, err = stdout.Write([]byte(msg))
			if err != nil {
				*stdoutErr = fmt.Errorf("writing to stdout: %s", err)
				kw.GetWorkceptor().nc.GetLogger().Error("Error writing to stdout: %s", err)

				return
			}

			// Set successfulWrite = true after writing to stdout to track that we've successfully written during this connection session
			successfulWrite = true
			retryGetLogStream = retries
		}
	}
}

func (kw *KubeUnit) CreatePod(env map[string]string) error {
	ked := kw.UnredactedStatus().ExtraData.(*KubeExtraData)
	command, err := shlex.Split(ked.Command)
	if err != nil {
		return err
	}
	params, err := shlex.Split(ked.Params)
	if err != nil {
		return err
	}

	pod := &corev1.Pod{}
	var spec *corev1.PodSpec
	var objectMeta *metav1.ObjectMeta
	if ked.KubePod != "" {
		decode := scheme.Codecs.UniversalDeserializer().Decode

		// Try Job first
		job := &batchv1.Job{}
		_, _, err := decode([]byte(ked.KubePod), nil, job)
		if err == nil && job.Kind == "Job" {
			kw.isJobSpec = true
			spec = &job.Spec.Template.Spec
			kw.extractMetadata(&job.ObjectMeta, ked)

			err = kw.validateAndSetupWorkerContainer(spec)
			if err != nil {
				return err
			}
			
			// Create Job
			job.ObjectMeta.Name = ""
			job.ObjectMeta.GenerateName = kw.namePrefix
			job.ObjectMeta.Namespace = ked.KubeNamespace
			
			kw.Job, err = kw.KubeAPIWrapperInstance.CreateJob(kw.GetContext(), kw.clientset, ked.KubeNamespace, job, metav1.CreateOptions{})
			if err != nil {
				return err
			}
			
			// Wait for Job to create Pod
			err = kw.waitForJobPod()
			if err != nil {
				return err
			}
		} else {
			// Fall back to Pod spec
			_, _, err := decode([]byte(ked.KubePod), nil, pod)
			if err != nil {
				return err
			}
			
			spec = &pod.Spec
			kw.extractMetadata(&pod.ObjectMeta, ked)
			
			err = kw.validateAndSetupWorkerContainer(spec)
			if err != nil {
				return err
			}
			
			objectMeta = &pod.ObjectMeta
			objectMeta.Name = ""
			objectMeta.GenerateName = kw.namePrefix
			objectMeta.Namespace = ked.KubeNamespace
		}
	} else {
		objectMeta = &metav1.ObjectMeta{
			GenerateName: kw.namePrefix,
			Namespace:    ked.KubeNamespace,
		}
		spec = &corev1.PodSpec{
			Containers: []corev1.Container{{
				Name:      WorkerContainerName,
				Image:     ked.Image,
				Command:   command,
				Args:      params,
				Stdin:     true,
				StdinOnce: true,
				TTY:       false,
			}},
			RestartPolicy: corev1.RestartPolicyNever,
		}
	}

	// Only create Pod directly if it's not a Job spec (Job already created Pod)
	if !kw.isJobSpec {
		pod = &corev1.Pod{
			ObjectMeta: *objectMeta,
			Spec:       *spec,
		}

		if env != nil {
			evs := make([]corev1.EnvVar, 0)
			for k, v := range env {
				evs = append(evs, corev1.EnvVar{
					Name:  k,
					Value: v,
				})
			}
			pod.Spec.Containers[0].Env = evs
		}

		// get pod and store to kw.Pod
		kw.Pod, err = kw.KubeAPIWrapperInstance.Create(kw.GetContext(), kw.clientset, ked.KubeNamespace, pod, metav1.CreateOptions{})
		if err != nil {
			return err
		}
	}

	select {
	case <-kw.GetContext().Done():
		return fmt.Errorf("cancelled")
	default:
	}

	kw.UpdateFullStatus(func(status *StatusFileData) {
		status.State = WorkStatePending
		status.Detail = "Pod created"
		status.StdoutSize = 0
		status.ExtraData.(*KubeExtraData).PodName = kw.Pod.Name
	})

	// Wait for the pod to be running
	fieldSelector := kw.KubeAPIWrapperInstance.OneTermEqualSelector("metadata.name", kw.Pod.Name).String()
	lw := &cache.ListWatch{
		ListFunc: func(options metav1.ListOptions) (runtime.Object, error) {
			options.FieldSelector = fieldSelector

			return kw.KubeAPIWrapperInstance.List(kw.GetContext(), kw.clientset, ked.KubeNamespace, options)
		},
		WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
			options.FieldSelector = fieldSelector

			return kw.KubeAPIWrapperInstance.Watch(kw.GetContext(), kw.clientset, ked.KubeNamespace, options)
		},
	}

	ctxPodReady := kw.GetContext()
	if kw.podPendingTimeout != time.Duration(0) {
		var ctxPodCancel context.CancelFunc
		ctxPodReady, ctxPodCancel = context.WithTimeout(kw.GetContext(), kw.podPendingTimeout)
		defer ctxPodCancel()
	}

	time.Sleep(2 * time.Second)
	ev, err := kw.KubeAPIWrapperInstance.UntilWithSync(ctxPodReady, lw, &corev1.Pod{}, nil, podRunningAndReady(*kw))
	if ev == nil || ev.Object == nil {
		return fmt.Errorf("did not return an event while watching pod for work unit %s", kw.ID())
	}

	var ok bool
	kw.Pod, ok = ev.Object.(*corev1.Pod)
	if !ok {
		return fmt.Errorf("watch did not return a pod")
	}

	if err == ErrPodCompleted {
		// Hao: shouldn't we also call kw.Cancel() in these cases?
		for _, cstat := range kw.Pod.Status.ContainerStatuses {
			if cstat.Name == WorkerContainerName {
				if cstat.State.Terminated != nil && cstat.State.Terminated.ExitCode != 0 {
					return fmt.Errorf("container failed with exit code %d: %s", cstat.State.Terminated.ExitCode, cstat.State.Terminated.Message)
				}

				break
			}
		}

		return err
	} else if err != nil { // any other error besides ErrPodCompleted
		stdout, err2 := NewStdoutWriter(FileSystem{}, kw.UnitDir())
		if err2 != nil {
			errMsg := fmt.Sprintf("Error opening stdout file: %s", err2)
			kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet
			kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)

			return fmt.Errorf(errMsg) //nolint:govet,staticcheck
		}
		var stdoutErr error
		var streamWait sync.WaitGroup
		streamWait.Add(1)
		go kw.kubeLoggingNoReconnect(&streamWait, stdout, &stdoutErr)
		streamWait.Wait()
		kw.Cancel()
		if len(kw.Pod.Status.ContainerStatuses) == 1 {
			if kw.Pod.Status.ContainerStatuses[0].State.Waiting != nil {
				return fmt.Errorf("%s, %s", err.Error(), kw.Pod.Status.ContainerStatuses[0].State.Waiting.Reason)
			}

			for _, cstat := range kw.Pod.Status.ContainerStatuses {
				if cstat.Name == WorkerContainerName {
					if cstat.State.Waiting != nil {
						return fmt.Errorf("%s, %s", err.Error(), cstat.State.Waiting.Reason)
					}

					if cstat.State.Terminated != nil && cstat.State.Terminated.ExitCode != 0 {
						return fmt.Errorf("%s, exit code %d: %s", err.Error(), cstat.State.Terminated.ExitCode, cstat.State.Terminated.Message)
					}

					break
				}
			}
		}

		return err
	}

	return nil
}

// runWorkUsingLogger is a private wrapper that calls the public RunWorkUsingLogger method.
// This maintains backward compatibility while enabling direct testing.
func (kw *KubeUnit) runWorkUsingLogger() {
	kw.RunWorkUsingLogger()
}

// RunWorkUsingLogger orchestrates the complete workflow for running work in a Kubernetes pod
// using logger-based streaming. This method is exposed publicly to enable comprehensive testing
// of the complex pod lifecycle, stdin/stdout streaming, and error handling logic.
//
// The method handles:
// - Creating new pods or resuming existing ones
// - Setting up SPDY executors for stdin streaming
// - Managing goroutines for stdin/stdout coordination
// - Error propagation and status transitions
// - Proper cleanup and resource management.
func (kw *KubeUnit) RunWorkUsingLogger() {
	skipStdin := true

	status := kw.Status()
	ked := status.ExtraData.(*KubeExtraData)

	podName := ked.PodName
	podNamespace := ked.KubeNamespace

	if podName == "" {
		// create new pod if ked.PodName is empty
		// TODO: add retry logic to make this more resilient to transient errors
		if err := kw.CreatePod(nil); err != nil {
			if err != ErrPodCompleted {
				errMsg := fmt.Sprintf("Error creating pod: %s", err)
				kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet
				kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)

				return
			}
		} else {
			// for newly created pod we need to stream stdin
			skipStdin = false
		}

		podName = kw.Pod.Name
		podNamespace = kw.Pod.Namespace
	} else {
		if podNamespace == "" {
			errMsg := fmt.Sprintf("Error creating pod: pod namespace is empty for pod %s",
				podName,
			)
			kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet
			kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)

			return
		}

		// resuming from a previously created pod
		var err error
		for retries := 5; retries > 0; retries-- {
			// check if the kw.ctx is already cancel
			select {
			case <-kw.GetContext().Done():
				errMsg := fmt.Sprintf("Context Done while getting pod %s/%s. Error: %s", podNamespace, podName, kw.GetContext().Err())
				kw.GetWorkceptor().nc.GetLogger().Warning(errMsg) //nolint:govet

				return
			default:
			}

			kw.Pod, err = kw.KubeAPIWrapperInstance.Get(kw.GetContext(), kw.clientset, podNamespace, podName, metav1.GetOptions{})
			if err == nil {
				break
			}
			kw.GetWorkceptor().nc.GetLogger().Warning(
				"Error getting pod %s/%s. Will retry %d more times. Retrying: %s",
				podNamespace,
				podName,
				retries,
				err,
			)
			time.Sleep(200 * time.Millisecond)
		}
		if err != nil {
			errMsg := fmt.Sprintf("Error getting pod %s/%s. Error: %s", podNamespace, podName, err)
			kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet
			kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)

			return
		}
	}

	// Attach stdin stream to the pod
	var exec remotecommand.Executor
	if !skipStdin {
		req := kw.KubeAPIWrapperInstance.SubResource(kw.clientset, podName, podNamespace)

		req.VersionedParams(
			&corev1.PodExecOptions{
				Container: WorkerContainerName,
				Stdin:     true,
				Stdout:    false,
				Stderr:    false,
				TTY:       false,
			},
			scheme.ParameterCodec,
		)
		var err error
		exec, err = kw.KubeAPIWrapperInstance.NewSPDYExecutor(kw.config, "POST", req.URL())
		if err != nil {
			errMsg := fmt.Sprintf("Error creating SPDY executor: %s", err)
			kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)

			return
		}
	}

	var stdinErr error
	var stdoutErr error

	// finishedChan signal the stdin and stdout monitoring goroutine to stop
	finishedChan := make(chan struct{})

	// this will signal the stdin and stdout monitoring goroutine to stop when this function returns
	defer close(finishedChan)

	stdinErrChan := make(chan struct{}) // signal that stdin goroutine have errored and stop stdout goroutine

	// open stdin reader that reads from the work unit's data directory
	var stdin *STDinReader
	if !skipStdin {
		var err error
		stdin, err = NewStdinReader(FileSystem{}, kw.UnitDir())
		if err != nil {
			if errors.Is(err, errFileSizeZero) {
				skipStdin = true
			} else {
				errMsg := fmt.Sprintf("Error opening stdin file: %s", err)
				kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet
				kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)

				return
			}
		} else {
			// goroutine to cancel stdin reader
			go func() {
				select {
				case <-kw.GetContext().Done():
					stdin.reader.Close()

					return
				case <-finishedChan:
				case <-stdin.Done():
					return
				}
			}()
		}
	}

	// open stdout writer that writes to work unit's data directory
	stdout, err := NewStdoutWriter(FileSystem{}, kw.UnitDir())
	if err != nil {
		errMsg := fmt.Sprintf("Error opening stdout file: %s", err)
		kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet
		kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)

		return
	}

	// goroutine to cancel stdout stream
	go func() {
		select {
		case <-kw.GetContext().Done():
			stdout.writer.Close()

			return
		case <-stdinErrChan:
			stdout.writer.Close()

			return
		case <-finishedChan:
			return
		}
	}()

	streamWait := sync.WaitGroup{}
	streamWait.Add(2)

	if skipStdin {
		kw.UpdateBasicStatus(WorkStateRunning, "Pod Running", stdout.Size())
		streamWait.Done()
	} else {
		retryCount := kw.GetKubeRetryCount()
		prevPodDelay, curPodDelay := 1, 1
		prevContainerDelay, curContainerDelay := 1, 1
	podLoop:
		for {
			podDetails, kubeErr := kw.KubeAPIWrapperInstance.Get(kw.GetContext(), kw.clientset, podNamespace, podName, metav1.GetOptions{})
			if kubeErr != nil {
				// There are many reasons why the kube api might not be able to get the pod,
				// This does not mean there is a problem just yet.
				// Let's try to get the pod again, max 5 times, and decide.
				retryCount--
				if retryCount > 0 {
					kw.GetWorkceptor().nc.GetLogger().Debug("Error getting pod while trying to attach stdin: '%s' , continuing try to get pod up to %v more times.", kubeErr, retryCount)

					time.Sleep(kw.GetSleepDuration(curPodDelay))
					prevPodDelay, curPodDelay = curPodDelay, prevPodDelay+curPodDelay

					continue
				}
				errMsg := fmt.Sprintf("Error getting pod %s/%s, after retries exhausted. Error: %s", podNamespace, podName, kubeErr)
				kw.GetWorkceptor().nc.GetLogger().Error("%s", errMsg)
				kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)

				return
			}
			retryCount = kw.GetKubeRetryCount()

			var containerState corev1.ContainerState
			foundContainer := false
			for _, containerStatus := range podDetails.Status.ContainerStatuses {
				if containerStatus.Name == WorkerContainerName {
					containerState = containerStatus.State
					foundContainer = true
				}
			}

			if !foundContainer {
				errMsg := fmt.Sprintf("Unable to find the container %s for pod %s. This is unrecoverable. Marking the job as failed and exiting", WorkerContainerName, podName)
				kw.GetWorkceptor().nc.GetLogger().Error("%s", errMsg)
				kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)

				return
			}

			// If container state not running retry or fail job.
			switch {
			case containerState.Running != nil:
				kw.GetWorkceptor().nc.GetLogger().Debug("Container in %s pod is running.", podName)

				break podLoop
			case containerState.Waiting != nil:
				retryCount--
				if retryCount > 0 {
					kw.GetWorkceptor().nc.GetLogger().Debug("Container in %s pod is waiting, will retry %v more times.", podName, retryCount)

					time.Sleep(kw.GetSleepDuration(curContainerDelay))
					prevContainerDelay, curContainerDelay = curContainerDelay, prevContainerDelay+curContainerDelay

					continue podLoop
				}
				errMsg := fmt.Sprintf("Container in %s pod is not running, retries exhausted", podName)
				kw.GetWorkceptor().nc.GetLogger().Error("%s", errMsg)
				kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)

				return
			case containerState.Terminated != nil:
				errMsg := fmt.Sprintf("Container in %s pod has terminated, with exit code: %v, terminated reason: %v and terminated message: %v", podName, containerState.Terminated.ExitCode, containerState.Terminated.Reason, containerState.Terminated.Message)
				kw.GetWorkceptor().nc.GetLogger().Error("%s", errMsg)
				kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)

				return
			default:
				retryCount--
				if retryCount > 0 {
					kw.GetWorkceptor().nc.GetLogger().Debug("%s is in an unexpected container state %s. This is unexpected. Will retry %v more times.", podName, containerState, retryCount)

					time.Sleep(kw.GetSleepDuration(curContainerDelay))
					prevContainerDelay, curContainerDelay = curContainerDelay, prevContainerDelay+curContainerDelay

					continue podLoop
				} else {
					errMsg := fmt.Sprintf("Container in %s pod is not running container state unknown, retries exhausted", podName)
					kw.GetWorkceptor().nc.GetLogger().Error("%s", errMsg)
					kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)

					return
				}
			}
		}

		go func() {
			defer streamWait.Done()

			kw.UpdateFullStatus(func(status *StatusFileData) {
				status.State = WorkStatePending
				status.Detail = "Sending stdin to pod"
			})

			var err error
			for retries := kw.GetKubeRetryCount(); retries > 0; retries-- {
				err = kw.KubeAPIWrapperInstance.StreamWithContext(kw.GetContext(), exec, remotecommand.StreamOptions{
					Stdin: stdin,
					Tty:   false,
				})
				if err != nil {
					// NOTE: io.EOF for stdin is handled by remotecommand and will not trigger this
					kw.GetWorkceptor().nc.GetLogger().Warning(
						"Error streaming stdin to pod %s/%s. Will retry %d more times. Error: %s",
						podNamespace,
						podName,
						retries,
						err,
					)
					time.Sleep(200 * time.Millisecond)
				} else {
					break
				}
			}

			if err != nil {
				stdinErr = err
				errMsg := fmt.Sprintf(
					"Error streaming stdin to pod %s/%s. Error: %s",
					podNamespace,
					podName,
					err,
				)
				kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet
				kw.UpdateBasicStatus(WorkStateFailed, errMsg, stdout.Size())

				close(stdinErrChan) // signal STDOUT goroutine to stop
			} else {
				if stdin.Error() == io.EOF {
					kw.UpdateBasicStatus(WorkStateRunning, "Pod Running", stdout.Size())
				} else {
					// this is probably not possible...
					errMsg := fmt.Sprintf("Error reading stdin: %s", stdin.Error())
					kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet
					kw.GetWorkceptor().nc.GetLogger().Error("Pod status at time of error %s", kw.Pod.Status.String())
					kw.UpdateBasicStatus(WorkStateFailed, errMsg, stdout.Size())

					close(stdinErrChan) // signal STDOUT goroutine to stop
				}
			}
		}()
	}

	stdoutWithReconnect := ShouldUseReconnect(kw)
	if stdoutWithReconnect && stdoutErr == nil {
		kw.GetWorkceptor().nc.GetLogger().Debug("streaming stdout with reconnect support")
		go kw.KubeLoggingWithReconnect(&streamWait, stdout, &stdinErr, &stdoutErr)
	} else {
		kw.GetWorkceptor().nc.GetLogger().Debug("streaming stdout with no reconnect support")
		go kw.kubeLoggingNoReconnect(&streamWait, stdout, &stdoutErr)
	}

	streamWait.Wait()

	if stdinErr != nil || stdoutErr != nil {
		var errDetail string
		switch {
		case stdinErr == nil:
			errDetail = fmt.Sprintf("Error with pod's stdout: %s", stdoutErr)
		case stdoutErr == nil:

			errDetail = fmt.Sprintf("Error with pod's stdin: %s", stdinErr)
		default:
			errDetail = fmt.Sprintf("Error running pod. stdin: %s, stdout: %s", stdinErr, stdoutErr)
		}

		if kw.GetContext().Err() != context.Canceled {
			kw.UpdateBasicStatus(WorkStateFailed, errDetail, stdout.Size())
		}

		return
	}

	// only transition from WorkStateRunning to WorkStateSucceeded if WorkStateFailed is set we do not override
	if kw.GetContext().Err() != context.Canceled && kw.Status().State == WorkStateRunning {
		kw.UpdateBasicStatus(WorkStateSucceeded, "Finished", stdout.Size())
	}
}

func IsCompatibleK8S(kw *KubeUnit, versionStr string) bool {
	semver, err := version.ParseSemantic(versionStr)
	if err != nil {
		kw.GetWorkceptor().nc.GetLogger().Warning("could parse Kubernetes server version %s, will not use reconnect support", versionStr)

		return false
	}

	// ignore pre-release in version comparison
	semver = semver.WithPreRelease("")

	// The patch was backported to minor version 23, 24 and 25
	// We check z stream based on the minor version
	// if minor versions < 23, set to high value (e.g. v1.22.9999)
	// if minor versions == 23, compare with v1.23.14
	// if minor version == 24, compare with v1.24.8
	// if minor version == 25, compare with v1.25.4
	// if minor versions > 23, compare with low value (e.g. v1.26.0)
	var compatibleVer string
	switch {
	case semver.Minor() == 23:
		compatibleVer = "v1.23.14"
	case semver.Minor() == 24:
		compatibleVer = "v1.24.8"
	case semver.Minor() == 25:
		compatibleVer = "v1.25.4"
	case semver.Minor() > 25:
		compatibleVer = fmt.Sprintf("%d.%d.0", semver.Major(), semver.Minor())
	default:
		compatibleVer = fmt.Sprintf("%d.%d.9999", semver.Major(), semver.Minor())
	}

	if semver.AtLeast(version.MustParseSemantic(compatibleVer)) {
		kw.GetWorkceptor().nc.GetLogger().Debug("Kubernetes version %s is at least %s, using reconnect support", semver, compatibleVer)

		return true
	}

	kw.GetWorkceptor().nc.GetLogger().Debug("Kubernetes version %s not at least %s, not using reconnect support", semver, compatibleVer)

	return false
}

func ShouldUseReconnect(kw *KubeUnit) bool {
	// Support for streaming from pod with timestamps using reconnect method is in all current versions
	// Can override the detection by setting the RECEPTOR_KUBE_SUPPORT_RECONNECT
	// accepted values: "enabled", "disabled", "auto".  The default is "enabled"
	// all invalid values will assume to be "disabled"

	version := viper.GetInt("version")
	var env string
	ok := false
	switch version {
	case 2:
		env = viper.GetString("node.ReceptorKubeSupportReconnect")
		if env != "" {
			ok = true
		}
	default:
		env, ok = os.LookupEnv("RECEPTOR_KUBE_SUPPORT_RECONNECT")
	}
	if ok {
		switch env {
		case "enabled":
			return true
		case "disabled":
			return false
		case "auto":
			return true
		default:
			return false
		}
	}

	serverVerInfo, err := kw.clientset.Discovery().ServerVersion()
	if err != nil {
		kw.GetWorkceptor().nc.GetLogger().Warning("could not detect Kubernetes server version, will not use reconnect support")

		return false
	}

	return IsCompatibleK8S(kw, serverVerInfo.String())
}

func ParseTime(s string) *time.Time {
	t, err := time.Parse(time.RFC3339, s)
	if err == nil {
		return &t
	}

	t, err = time.Parse(time.RFC3339Nano, s)
	if err == nil {
		return &t
	}

	return nil
}

func getDefaultInterface() (string, error) {
	nifs, err := net.Interfaces()
	if err != nil {
		return "", err
	}
	for i := range nifs {
		nif := nifs[i]
		if nif.Flags&net.FlagUp != 0 && nif.Flags&net.FlagLoopback == 0 {
			ads, err := nif.Addrs()
			if err == nil && len(ads) > 0 {
				for j := range ads {
					ad := ads[j]
					ip, ok := ad.(*net.IPNet)
					if ok {
						if !ip.IP.IsLoopback() && !ip.IP.IsMulticast() {
							return ip.IP.String(), nil
						}
					}
				}
			}
		}
	}

	return "", fmt.Errorf("could not determine local address")
}

func (kw *KubeUnit) runWorkUsingTCP() {
	// Create local cancellable context
	ctx, cancel := kw.GetContext(), kw.GetCancel()
	defer cancel()

	// Create the TCP listener
	lc := net.ListenConfig{}
	defaultInterfaceIP, err := getDefaultInterface()
	var li net.Listener
	if err == nil {
		li, err = lc.Listen(ctx, "tcp", fmt.Sprintf("%s:", defaultInterfaceIP))
	}
	if ctx.Err() != nil {
		return
	}
	var listenHost, listenPort string
	if err == nil {
		listenHost, listenPort, err = net.SplitHostPort(li.Addr().String())
	}
	if err != nil {
		errMsg := fmt.Sprintf("Error listening: %s", err)
		kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)
		kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet

		return
	}

	// Wait for a single incoming connection
	connChan := make(chan *net.TCPConn)
	go func() {
		conn, err := li.Accept()
		lcerr := li.Close()
		if lcerr != nil {
			errMsg := fmt.Sprintf("Error closing listener: %+v", lcerr)
			kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)
			kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet
			cancel()

			return
		}
		if ctx.Err() != nil {
			return
		}
		var tcpConn *net.TCPConn
		if err == nil {
			var ok bool
			tcpConn, ok = conn.(*net.TCPConn)
			if !ok {
				err = fmt.Errorf("connection was not a TCPConn")
			}
		}
		if err != nil {
			errMsg := fmt.Sprintf("Error accepting: %s", err)
			kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)
			kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet
			cancel()

			return
		}
		connChan <- tcpConn
	}()

	// Create the pod
	err = kw.CreatePod(map[string]string{"RECEPTOR_HOST": listenHost, "RECEPTOR_PORT": listenPort})
	if err != nil {
		errMsg := fmt.Sprintf("Error creating pod: %s", err)
		kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)
		kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet
		cancel()

		return
	}

	// Wait for the pod to connect back to us
	var conn *net.TCPConn
	select {
	case <-ctx.Done():
		return
	case conn = <-connChan:
	}

	// Open stdin reader
	var stdin *STDinReader
	stdin, err = NewStdinReader(FileSystem{}, kw.UnitDir())
	if err != nil {
		errMsg := fmt.Sprintf("Error opening stdin file: %s", err)
		kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet
		kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)
		cancel()

		return
	}

	// Open stdout writer
	stdout, err := NewStdoutWriter(FileSystem{}, kw.UnitDir())
	if err != nil {
		errMsg := fmt.Sprintf("Error opening stdout file: %s", err)
		kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet
		kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)
		cancel()

		return
	}

	kw.UpdateBasicStatus(WorkStatePending, "Sending stdin to pod", 0)

	// Write stdin to pod
	go func() {
		_, err := io.Copy(conn, stdin)
		if ctx.Err() != nil {
			return
		}
		cwerr := conn.CloseWrite()
		if cwerr != nil {
			errMsg := fmt.Sprintf("Error closing writing side: %+v", cwerr)
			kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet
			kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)
			cancel()

			return
		}
		if err != nil {
			errMsg := fmt.Sprintf("Error sending stdin to pod: %s", err)
			kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet
			kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)
			cancel()

			return
		}
	}()

	// Goroutine to update status when stdin is fully sent to the pod, which is when we
	// update from WorkStatePending to WorkStateRunning.
	go func() {
		select {
		case <-ctx.Done():
			return
		case <-stdin.Done():
			err := stdin.Error()
			if err == io.EOF {
				kw.UpdateBasicStatus(WorkStateRunning, "Pod Running", stdout.Size())
			} else {
				kw.UpdateBasicStatus(WorkStateFailed, fmt.Sprintf("Error reading stdin: %s", err), stdout.Size())
				cancel()
			}
		}
	}()

	// Read stdout from pod
	_, err = io.Copy(stdout, conn)
	if ctx.Err() != nil {
		return
	}
	if err != nil {
		errMsg := fmt.Sprintf("Error reading stdout from pod: %s", err)
		kw.GetWorkceptor().nc.GetLogger().Error(errMsg) //nolint:govet
		kw.UpdateBasicStatus(WorkStateFailed, errMsg, 0)
		cancel()

		return
	}

	if ctx.Err() == nil {
		kw.UpdateBasicStatus(WorkStateSucceeded, "Finished", stdout.Size())
	}
}

func (kw *KubeUnit) connectUsingKubeconfig() error {
	var err error
	ked := kw.UnredactedStatus().ExtraData.(*KubeExtraData)
	if ked.KubeConfig == "" {
		clr := kw.KubeAPIWrapperInstance.NewDefaultClientConfigLoadingRules()
		kw.config, err = kw.KubeAPIWrapperInstance.BuildConfigFromFlags("", clr.GetDefaultFilename())
		if ked.KubeNamespace == "" {
			c, err := clr.Load()
			if err != nil {
				return err
			}
			curContext, ok := c.Contexts[c.CurrentContext]
			if ok && curContext != nil {
				kw.UpdateFullStatus(func(sfd *StatusFileData) {
					sfd.ExtraData.(*KubeExtraData).KubeNamespace = curContext.Namespace
				})
			} else {
				return fmt.Errorf("could not determine namespace")
			}
		}
	} else {
		cfg, err := kw.KubeAPIWrapperInstance.NewClientConfigFromBytes([]byte(ked.KubeConfig))
		if err != nil {
			return err
		}
		if ked.KubeNamespace == "" {
			namespace, _, err := cfg.Namespace()
			if err != nil {
				return err
			}
			kw.UpdateFullStatus(func(sfd *StatusFileData) {
				sfd.ExtraData.(*KubeExtraData).KubeNamespace = namespace
			})
		}
		kw.config, err = cfg.ClientConfig()
		if err != nil {
			return err
		}
	}
	if err != nil {
		return err
	}

	return nil
}

func (kw *KubeUnit) connectUsingIncluster() error {
	var err error
	kw.config, err = kw.KubeAPIWrapperInstance.InClusterConfig()
	if err != nil {
		return err
	}

	return nil
}

func (kw *KubeUnit) connectToKube() error {
	var err error
	switch {
	case kw.authMethod == "kubeconfig" || kw.authMethod == "runtime":
		err = kw.connectUsingKubeconfig()
	case kw.authMethod == "incluster":
		err = kw.connectUsingIncluster()
	default:
		return fmt.Errorf("unknown auth method %s", kw.authMethod)
	}
	if err != nil {
		return err
	}

	kw.config.QPS = float32(100)
	kw.config.Burst = 1000

	// RECEPTOR_KUBE_CLIENTSET_QPS
	// default: 100
	version := viper.GetInt("version")
	var envQPS string
	ok := false
	switch version {
	case 2:
		envQPS = viper.GetString("node.ReceptorKubeClientsetQPS")
		if envQPS != "" {
			ok = true
		}
	default:
		envQPS, ok = os.LookupEnv("RECEPTOR_KUBE_CLIENTSET_QPS")
	}
	if ok {
		qps, err := strconv.Atoi(envQPS)
		if err != nil {
			// ignore error, use default
			kw.GetWorkceptor().nc.GetLogger().Warning("Invalid value for RECEPTOR_KUBE_CLIENTSET_QPS: %s. Ignoring", envQPS)
		} else {
			kw.config.QPS = float32(qps)
			kw.config.Burst = qps * 10
		}
	}

	kw.GetWorkceptor().nc.GetLogger().Debug("RECEPTOR_KUBE_CLIENTSET_QPS: %s", envQPS)

	// RECEPTOR_KUBE_CLIENTSET_BURST
	// default: 10 x QPS
	var envBurst string
	switch version {
	case 2:
		envBurst = viper.GetString("node.ReceptorKubeClientsetBurst")
		if envBurst != "" {
			ok = true
		}
	default:
		envBurst, ok = os.LookupEnv("RECEPTOR_KUBE_CLIENTSET_BURST")
	}
	if ok {
		burst, err := strconv.Atoi(envBurst)
		if err != nil {
			kw.GetWorkceptor().nc.GetLogger().Warning("Invalid value for RECEPTOR_KUBE_CLIENTSET_BURST: %s. Ignoring", envQPS)
		} else {
			kw.config.Burst = burst
		}
	}

	kw.GetWorkceptor().nc.GetLogger().Debug("RECEPTOR_KUBE_CLIENTSET_BURST: %s", envBurst)

	kw.GetWorkceptor().nc.GetLogger().Debug("Initializing Kubernetes clientset")
	// RECEPTOR_KUBE_CLIENTSET_RATE_LIMITER
	// default: tokenbucket
	// options: never, always, tokenbucket
	var envRateLimiter string
	switch version {
	case 2:
		envRateLimiter = viper.GetString("node.ReceptorKubeClientsetRateLimiter")
		if envRateLimiter != "" {
			ok = true
		}
	default:
		envRateLimiter, ok = os.LookupEnv("RECEPTOR_KUBE_CLIENTSET_RATE_LIMITER")
	}
	if ok {
		switch envRateLimiter {
		case "never":
			kw.config.RateLimiter = kw.KubeAPIWrapperInstance.NewFakeNeverRateLimiter()
		case "always":
			kw.config.RateLimiter = kw.KubeAPIWrapperInstance.NewFakeAlwaysRateLimiter()
		default:
		}
		kw.GetWorkceptor().nc.GetLogger().Debug("RateLimiter: %s", envRateLimiter)
	}

	kw.GetWorkceptor().nc.GetLogger().Debug("QPS: %f, Burst: %d", kw.config.QPS, kw.config.Burst)
	kw.clientset, err = kw.KubeAPIWrapperInstance.NewForConfig(kw.config)
	if err != nil {
		return err
	}

	return nil
}

// ReadFileToString reads a file and returns its contents as a string.
// If filename is empty, it returns an empty string.
func ReadFileToString(filename string) (string, error) {
	// If filename is "", the function returns ""
	if filename == "" {
		return "", nil
	}
	content, err := os.ReadFile(filename)
	if err != nil {
		return "", err
	}

	return string(content), nil
}

// SetFromParams sets the in-memory state from parameters.
func (kw *KubeUnit) SetFromParams(params map[string]string) error {
	ked := kw.GetStatusCopy().ExtraData.(*KubeExtraData)
	type value struct {
		name       string
		permission bool
		setter     func(string) error
	}
	setString := func(target *string) func(string) error {
		ssf := func(value string) error {
			*target = value

			return nil
		}

		return ssf
	}
	var err error
	ked.KubePod, err = ReadFileToString(ked.KubePod)
	if err != nil {
		return fmt.Errorf("could not read pod: %s", err)
	}
	ked.KubeConfig, err = ReadFileToString(ked.KubeConfig)
	if err != nil {
		return fmt.Errorf("could not read kubeconfig: %s", err)
	}
	userParams := ""
	userCommand := ""
	userImage := ""
	userPod := ""
	podPendingTimeoutString := ""
	values := []value{
		{name: "kube_command", permission: kw.allowRuntimeCommand, setter: setString(&userCommand)},
		{name: "kube_image", permission: kw.allowRuntimeCommand, setter: setString(&userImage)},
		{name: "kube_params", permission: kw.allowRuntimeParams, setter: setString(&userParams)},
		{name: "kube_namespace", permission: kw.allowRuntimeAuth, setter: setString(&ked.KubeNamespace)},
		{name: "secret_kube_config", permission: kw.allowRuntimeAuth, setter: setString(&ked.KubeConfig)},
		{name: "secret_kube_pod", permission: kw.allowRuntimePod, setter: setString(&userPod)},
		{name: "pod_pending_timeout", permission: kw.allowRuntimeParams, setter: setString(&podPendingTimeoutString)},
	}
	for i := range values {
		v := values[i]
		value, ok := params[v.name]
		if ok && value != "" {
			if !v.permission {
				return fmt.Errorf("%s provided but not allowed", v.name)
			}
			err := v.setter(value)
			if err != nil {
				return fmt.Errorf("error setting value for %s: %s", v.name, err)
			}
		}
	}
	if kw.authMethod == "runtime" && ked.KubeConfig == "" {
		return fmt.Errorf("param secret_kube_config must be provided if AuthMethod=runtime")
	}
	if userPod != "" && (userParams != "" || userCommand != "" || userImage != "") {
		return fmt.Errorf("params kube_command, kube_image, kube_params not compatible with secret_kube_pod")
	}

	if podPendingTimeoutString != "" {
		podPendingTimeout, err := time.ParseDuration(podPendingTimeoutString)
		if err != nil {
			kw.GetWorkceptor().nc.GetLogger().Error("Failed to parse pod_pending_timeout -- valid examples include '1.5h', '30m', '30m10s'")

			return err
		}
		kw.podPendingTimeout = podPendingTimeout
	}

	if userCommand != "" {
		ked.Command = userCommand
	}
	if userImage != "" {
		ked.Image = userImage
	}
	if userPod != "" {
		ked.KubePod = userPod
		ked.Image = ""
		ked.Command = ""
		kw.baseParams = ""
	} else {
		ked.Params = combineParams(kw.baseParams, userParams)
	}

	return nil
}

// Status returns a copy of the status currently loaded in memory.
func (kw *KubeUnit) Status() *StatusFileData {
	status := kw.UnredactedStatus()
	ed, ok := status.ExtraData.(*KubeExtraData)
	if ok {
		ed.KubeConfig = ""
		ed.KubePod = ""
	}

	return status
}

// SetClientset sets the clientset for testing purposes.
func (kw *KubeUnit) SetClientset(clientset kubernetes.Interface) {
	kw.clientset = clientset
}

// Status returns a copy of the status currently loaded in memory.
func (kw *KubeUnit) UnredactedStatus() *StatusFileData {
	kw.GetStatusLock().RLock()
	status := kw.GetStatusWithoutExtraData()
	ked, ok := kw.GetStatusCopy().ExtraData.(*KubeExtraData)
	if ok {
		kedCopy := *ked
		status.ExtraData = &kedCopy
	}
	kw.GetStatusLock().RUnlock()

	return status
}

// startOrRestart is a shared implementation of Start() and Restart().
func (kw *KubeUnit) startOrRestart() error {
	// Connect to the Kubernetes API
	if err := kw.connectToKube(); err != nil {
		return err
	}
	// Launch runner process
	if kw.streamMethod == "tcp" {
		go kw.runWorkUsingTCP()
	} else {
		go kw.runWorkUsingLogger()
	}
	go kw.MonitorLocalStatus()

	return nil
}

// Restart resumes monitoring a job after a Receptor restart.
func (kw *KubeUnit) Restart() error {
	status := kw.Status()
	ked := status.ExtraData.(*KubeExtraData)
	if IsComplete(status.State) {
		return nil
	}
	isTCP := kw.streamMethod == "tcp"
	if status.State == WorkStateRunning && !isTCP {
		return kw.startOrRestart()
	}
	// Work unit is in Pending state
	if kw.deletePodOnRestart {
		err := kw.connectToKube()
		if err != nil {
			kw.GetWorkceptor().nc.GetLogger().Warning("Pod %s could not be deleted: %s", ked.PodName, err.Error())
		} else {
			err := kw.KubeAPIWrapperInstance.Delete(context.Background(), kw.clientset, ked.KubeNamespace, ked.PodName, metav1.DeleteOptions{})
			if err != nil {
				kw.GetWorkceptor().nc.GetLogger().Warning("Pod %s could not be deleted: %s", ked.PodName, err.Error())
			}
		}
	}
	if isTCP {
		return fmt.Errorf("restart not implemented for streammethod tcp")
	}

	return fmt.Errorf("work unit is not in running state, cannot be restarted")
}

// Start launches a job with given parameters.
func (kw *KubeUnit) Start() error {
	kw.UpdateBasicStatus(WorkStatePending, "Connecting to Kubernetes", 0)

	return kw.startOrRestart()
}

// Cancel releases resources associated with a job, including cancelling it if running.
func (kw *KubeUnit) Cancel() error {
	kw.CancelContext()
	kw.UpdateBasicStatus(WorkStateCanceled, "Canceled", -1)
	if kw.Job != nil {
		err := kw.KubeAPIWrapperInstance.DeleteJob(context.Background(), kw.clientset, kw.Job.Namespace, kw.Job.Name, metav1.DeleteOptions{})
		if err != nil {
			kw.GetWorkceptor().nc.GetLogger().Error("Error deleting job %s: %s", kw.Job.Name, err)
		}
	}
	if kw.Pod != nil {
		err := kw.KubeAPIWrapperInstance.Delete(context.Background(), kw.clientset, kw.Pod.Namespace, kw.Pod.Name, metav1.DeleteOptions{})
		if err != nil {
			kw.GetWorkceptor().nc.GetLogger().Error("Error deleting pod %s: %s", kw.Pod.Name, err)
		}
	}
	if kw.GetCancel() != nil {
		kw.CancelContext()
	}

	return nil
}

// Release releases resources associated with a job.  Implies Cancel.
func (kw *KubeUnit) Release(force bool) error {
	err := kw.Cancel()
	if err != nil && !force {
		return err
	}

	return kw.BaseWorkUnitForWorkUnit.Release(force)
}

// ProcessLogLine handles timestamp parsing and stripping from log lines.
func (kw *KubeUnit) ProcessLogLine(line string, sinceTime time.Time, successfulWrite bool) (msg string, newSinceTime time.Time, shouldSkip bool) {
	split := strings.SplitN(line, " ", 2)
	msg = line
	newSinceTime = sinceTime

	timestamp := ParseTime(split[0])
	if timestamp != nil {
		if !timestamp.After(sinceTime) && !successfulWrite {
			return "", sinceTime, true
		}
		newSinceTime = *timestamp
		if len(split) > 1 {
			msg = split[1]
		} else {
			msg = ""
		}
	} else {
		kw.GetWorkceptor().nc.GetLogger().Debug("No timestamp received, log line: '%s'", line)
	}

	return msg, newSinceTime, false
}

// **************************************************************************
// Command line
// **************************************************************************

// KubeWorkerCfg is the cmdline configuration object for a Kubernetes worker plugin.
type KubeWorkerCfg struct {
	WorkType            string `required:"true" description:"Name for this worker type"`
	Namespace           string `description:"Kubernetes namespace to create pods in"`
	Image               string `description:"Container image to use for the worker pod"`
	Command             string `description:"Command to run in the container (overrides entrypoint)"`
	Params              string `description:"Command-line parameters to pass to the entrypoint"`
	AuthMethod          string `description:"One of: kubeconfig, incluster" default:"incluster"`
	KubeConfig          string `description:"Kubeconfig filename (for authmethod=kubeconfig)"`
	Pod                 string `description:"Pod definition filename, in json or yaml format"`
	AllowRuntimeAuth    bool   `description:"Allow passing API parameters at runtime" default:"false"`
	AllowRuntimeCommand bool   `description:"Allow specifying image & command at runtime" default:"false"`
	AllowRuntimeParams  bool   `description:"Allow adding command parameters at runtime" default:"false"`
	AllowRuntimePod     bool   `description:"Allow passing Pod at runtime" default:"false"`
	DeletePodOnRestart  bool   `description:"On restart, delete the pod if in pending state" default:"true"`
	StreamMethod        string `description:"Method for connecting to worker pods: logger or tcp" default:"logger"`
	VerifySignature     bool   `description:"Verify a signed work submission" default:"false"`
}

// NewWorker is a factory to produce worker instances.
func (cfg KubeWorkerCfg) NewWorker(bwu BaseWorkUnitForWorkUnit, w *Workceptor, unitID string, workType string) WorkUnit {
	return cfg.NewkubeWorker(bwu, w, unitID, workType, nil)
}

func (cfg KubeWorkerCfg) NewkubeWorker(bwu BaseWorkUnitForWorkUnit, w *Workceptor, unitID string, workType string, kawi KubeAPIer) WorkUnit {
	var kubeAPIWrapperInstance KubeAPIer
	if bwu == nil {
		bwu = &BaseWorkUnit{
			status: StatusFileData{
				ExtraData: &KubeExtraData{
					Image:         cfg.Image,
					Command:       cfg.Command,
					KubeNamespace: cfg.Namespace,
					KubePod:       cfg.Pod,
					KubeConfig:    cfg.KubeConfig,
				},
			},
		}
	}

	if kawi != nil {
		kubeAPIWrapperInstance = kawi
	} else {
		kubeAPIWrapperInstance = KubeAPIWrapper{}
	}

	ku := &KubeUnit{
		BaseWorkUnitForWorkUnit: bwu,
		KubeAPIWrapperInstance:  kubeAPIWrapperInstance,
		authMethod:              strings.ToLower(cfg.AuthMethod),
		streamMethod:            strings.ToLower(cfg.StreamMethod),
		baseParams:              cfg.Params,
		allowRuntimeAuth:        cfg.AllowRuntimeAuth,
		allowRuntimeCommand:     cfg.AllowRuntimeCommand,
		allowRuntimeParams:      cfg.AllowRuntimeParams,
		allowRuntimePod:         cfg.AllowRuntimePod,
		deletePodOnRestart:      cfg.DeletePodOnRestart,
		namePrefix:              fmt.Sprintf("%s-", strings.ToLower(cfg.WorkType)),
	}
	ku.BaseWorkUnitForWorkUnit.Init(w, unitID, workType, FileSystem{})

	return ku
}

// Prepare inspects the configuration for validity.
func (cfg KubeWorkerCfg) Prepare() error {
	lcAuth := strings.ToLower(cfg.AuthMethod)
	if lcAuth != "kubeconfig" && lcAuth != "incluster" && lcAuth != "runtime" {
		return fmt.Errorf("invalid AuthMethod: %s", cfg.AuthMethod)
	}
	if cfg.Namespace == "" && !(lcAuth == "kubeconfig" || cfg.AllowRuntimeAuth) {
		return fmt.Errorf("must provide namespace when AuthMethod is not kubeconfig")
	}
	if cfg.KubeConfig != "" {
		if lcAuth != "kubeconfig" {
			return fmt.Errorf("can only provide KubeConfig when AuthMethod=kubeconfig")
		}
		_, err := os.Stat(cfg.KubeConfig)
		if err != nil {
			return fmt.Errorf("error accessing kubeconfig file: %s", err)
		}
	}
	if cfg.Pod != "" && (cfg.Image != "" || cfg.Command != "" || cfg.Params != "") {
		return fmt.Errorf("can only provide Pod when Image, Command, and Params are empty")
	}
	if cfg.Pod == "" && cfg.Image == "" && !cfg.AllowRuntimeCommand && !cfg.AllowRuntimePod {
		return fmt.Errorf("must specify a container image to run")
	}
	method := strings.ToLower(cfg.StreamMethod)
	if method != "logger" && method != "tcp" {
		return fmt.Errorf("stream mode must be logger or tcp")
	}

	return nil
}

func (cfg KubeWorkerCfg) GetWorkType() string {
	return cfg.WorkType
}

func (cfg KubeWorkerCfg) GetVerifySignature() bool {
	return cfg.VerifySignature
}

// Run runs the action.
func (cfg KubeWorkerCfg) Run() error {
	err := MainInstance.RegisterWorker(cfg.WorkType, cfg.NewWorker, cfg.VerifySignature)

	return err
}

func init() {
	version := viper.GetInt("version")
	if version > 1 {
		return
	}
	cmdline.RegisterConfigTypeForApp("receptor-workers",
		"work-kubernetes", "Run a worker using Kubernetes", KubeWorkerCfg{}, cmdline.Section(workersSection))
}