Skip to content

Commit 7e63cc8

Browse files
verbverb
committed
kubectl debug: add support for debugging nodes
When called with a node target, `kubectl debug` will create a run-once pod in the target node's namespaces.
1 parent 896da22 commit 7e63cc8

File tree

2 files changed

+271
-8
lines changed

2 files changed

+271
-8
lines changed

‎staging/src/k8s.io/kubectl/pkg/cmd/debug/debug.go‎

Lines changed: 112 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,12 +51,28 @@ import (
5151
)
5252

5353
var (
54-
debugLong = templates.LongDesc(i18n.T(`Tools for debugging Kubernetes resources`))
54+
debugLong = templates.LongDesc(i18n.T(`
55+
Debug cluster resources using interactive debugging containers.
56+
57+
'debug' provides automation for common debugging tasks for cluster objects identified by
58+
resource and name. Pods will be used by default if resource is not specified.
59+
60+
The action taken by 'debug' varies depending on what resource is specified. Supported
61+
actions include:
62+
63+
* Workload: Create a copy of an existing pod with certain attributes changed,
64+
for example changing the image tag to a new version.
65+
* Workload: Add an ephemeral container to an already running pod, for example to add
66+
debugging utilities without restarting the pod.
67+
* Node: Create a new pod that runs in the node's host namespaces and can access
68+
the node's filesystem.
69+
70+
Alpha disclaimer: command line flags may change`))
5571

5672
debugExample = templates.Examples(i18n.T(`
5773
# Create an interactive debugging session in pod mypod and immediately attach to it.
5874
# (requires the EphemeralContainers feature to be enabled in the cluster)
59-
kubectl alpha debug mypod -i --image=busybox
75+
kubectl alpha debug mypod -it --image=busybox
6076
6177
# Create a debug container named debugger using a custom automated debugging image.
6278
# (requires the EphemeralContainers feature to be enabled in the cluster)
@@ -67,6 +83,10 @@ var (
6783
6884
# Create a copy of mypod named my-debugger with my-container's image changed to busybox
6985
kubectl alpha debug mypod --image=busybox --container=my-container --copy-to=my-debugger -- sleep 1d
86+
87+
# Create an interactive debugging session on a node and immediately attach to it.
88+
# The container will run in the host namespaces and the host's filesystem will be mounted at /host
89+
kubectl alpha debug node/mynode -it --image=busybox
7090
`))
7191
)
7292

@@ -133,7 +153,7 @@ func NewCmdDebug(f cmdutil.Factory, streams genericclioptions.IOStreams) *cobra.
133153

134154
func addDebugFlags(cmd *cobra.Command, opt *DebugOptions) {
135155
cmd.Flags().BoolVar(&opt.ArgsOnly, "arguments-only", opt.ArgsOnly, i18n.T("If specified, everything after -- will be passed to the new container as Args instead of Command."))
136-
cmd.Flags().BoolVar(&opt.Attach, "attach", opt.Attach, i18n.T("If true, wait for the Pod to start running, and then attach to the Pod as if 'kubectl attach ...' were called. Default false, unless '-i/--stdin' is set, in which case the default is true."))
156+
cmd.Flags().BoolVar(&opt.Attach, "attach", opt.Attach, i18n.T("If true, wait for the container to start running, and then attach as if 'kubectl attach ...' were called. Default false, unless '-i/--stdin' is set, in which case the default is true."))
137157
cmd.Flags().StringVarP(&opt.Container, "container", "c", opt.Container, i18n.T("Container name to use for debug container."))
138158
cmd.Flags().StringVar(&opt.CopyTo, "copy-to", opt.CopyTo, i18n.T("Create a copy of the target Pod with this name."))
139159
cmd.Flags().BoolVar(&opt.Replace, "replace", opt.Replace, i18n.T("When used with '--copy-to', delete the original Pod"))
@@ -142,11 +162,11 @@ func addDebugFlags(cmd *cobra.Command, opt *DebugOptions) {
142162
cmd.MarkFlagRequired("image")
143163
cmd.Flags().String("image-pull-policy", string(corev1.PullIfNotPresent), i18n.T("The image pull policy for the container."))
144164
cmd.Flags().BoolVarP(&opt.Interactive, "stdin", "i", opt.Interactive, i18n.T("Keep stdin open on the container(s) in the pod, even if nothing is attached."))
145-
cmd.Flags().BoolVar(&opt.Quiet, "quiet", opt.Quiet, i18n.T("If true, suppress prompt messages."))
146-
cmd.Flags().BoolVar(&opt.SameNode, "same-node", opt.SameNode, i18n.T("Schedule the copy of target Pod on the same node."))
165+
cmd.Flags().BoolVar(&opt.Quiet, "quiet", opt.Quiet, i18n.T("If true, suppress informational messages."))
166+
cmd.Flags().BoolVar(&opt.SameNode, "same-node", opt.SameNode, i18n.T("When used with '--copy-to', schedule the copy of target Pod on the same node."))
147167
cmd.Flags().BoolVar(&opt.ShareProcesses, "share-processes", opt.ShareProcesses, i18n.T("When used with '--copy-to', enable process namespace sharing in the copy."))
148-
cmd.Flags().StringVar(&opt.Target, "target", "", i18n.T("Target processes in this container name."))
149-
cmd.Flags().BoolVarP(&opt.TTY, "tty", "t", opt.TTY, i18n.T("Allocated a TTY for each container in the pod."))
168+
cmd.Flags().StringVar(&opt.Target, "target", "", i18n.T("When debugging a pod, target processes in this container name."))
169+
cmd.Flags().BoolVarP(&opt.TTY, "tty", "t", opt.TTY, i18n.T("Allocate a TTY for the debugging container."))
150170
}
151171

152172
// Complete finishes run-time initialization of debug.DebugOptions.
@@ -221,6 +241,11 @@ func (o *DebugOptions) Validate(cmd *cobra.Command) error {
221241
return fmt.Errorf("invalid image pull policy: %s", o.PullPolicy)
222242
}
223243

244+
// Target
245+
if len(o.Target) > 0 && len(o.CopyTo) > 0 {
246+
return fmt.Errorf("--target is incompatible with --copy-to. Use --share-processes instead.")
247+
}
248+
224249
// TTY
225250
if o.TTY && !o.Interactive {
226251
return fmt.Errorf("-i/--stdin is required for containers with -t/--tty=true")
@@ -253,6 +278,8 @@ func (o *DebugOptions) Run(f cmdutil.Factory, cmd *cobra.Command) error {
253278
visitErr error
254279
)
255280
switch obj := info.Object.(type) {
281+
case *corev1.Node:
282+
debugPod, containerName, visitErr = o.visitNode(ctx, obj)
256283
case *corev1.Pod:
257284
debugPod, containerName, visitErr = o.visitPod(ctx, obj)
258285
default:
@@ -293,6 +320,18 @@ func (o *DebugOptions) Run(f cmdutil.Factory, cmd *cobra.Command) error {
293320
return err
294321
}
295322

323+
// visitNode handles debugging for node targets by creating a privileged pod running in the host namespaces.
324+
// Returns an already created pod and container name for subsequent attach, if applicable.
325+
func (o *DebugOptions) visitNode(ctx context.Context, node *corev1.Node) (*corev1.Pod, string, error) {
326+
pods := o.podClient.Pods(o.Namespace)
327+
newPod, err := pods.Create(ctx, o.generateNodeDebugPod(node.Name), metav1.CreateOptions{})
328+
if err != nil {
329+
return nil, "", err
330+
}
331+
332+
return newPod, newPod.Spec.Containers[0].Name, nil
333+
}
334+
296335
// visitPod handles debugging for pod targets by (depending on options):
297336
// 1. Creating an ephemeral debug container in an existing pod, OR
298337
// 2. Making a copy of pod with certain attributes changed
@@ -371,6 +410,71 @@ func (o *DebugOptions) generateDebugContainer(pod *corev1.Pod) *corev1.Ephemeral
371410
return ec
372411
}
373412

413+
// generateNodeDebugPod generates a debugging pod that schedules on the specified node.
414+
// The generated pod will run in the host PID, Network & IPC namespaces, and it will have the node's filesystem mounted at /host.
415+
func (o *DebugOptions) generateNodeDebugPod(node string) *corev1.Pod {
416+
cn := "debugger"
417+
// Setting a user-specified container name doesn't make much difference when there's only one container,
418+
// but the argument exists for pod debugging so it might be confusing if it didn't work here.
419+
if len(o.Container) > 0 {
420+
cn = o.Container
421+
}
422+
423+
// The name of the debugging pod is based on the target node, and it's not configurable to
424+
// limit the number of command line flags. There may be a collision on the name, but this
425+
// should be rare enough that it's not worth the API round trip to check.
426+
pn := fmt.Sprintf("node-debugger-%s-%s", node, nameSuffixFunc(5))
427+
if !o.Quiet {
428+
fmt.Fprintf(o.Out, "Creating debugging pod %s with container %s on node %s.\n", pn, cn, node)
429+
}
430+
431+
p := &corev1.Pod{
432+
ObjectMeta: metav1.ObjectMeta{
433+
Name: pn,
434+
},
435+
Spec: corev1.PodSpec{
436+
Containers: []corev1.Container{
437+
{
438+
Name: cn,
439+
Env: o.Env,
440+
Image: o.Image,
441+
ImagePullPolicy: o.PullPolicy,
442+
Stdin: o.Interactive,
443+
TerminationMessagePolicy: corev1.TerminationMessageReadFile,
444+
TTY: o.TTY,
445+
VolumeMounts: []corev1.VolumeMount{
446+
{
447+
MountPath: "/host",
448+
Name: "host-root",
449+
},
450+
},
451+
},
452+
},
453+
HostIPC: true,
454+
HostNetwork: true,
455+
HostPID: true,
456+
NodeName: node,
457+
RestartPolicy: corev1.RestartPolicyNever,
458+
Volumes: []corev1.Volume{
459+
{
460+
Name: "host-root",
461+
VolumeSource: corev1.VolumeSource{
462+
HostPath: &corev1.HostPathVolumeSource{Path: "/"},
463+
},
464+
},
465+
},
466+
},
467+
}
468+
469+
if o.ArgsOnly {
470+
p.Spec.Containers[0].Args = o.Args
471+
} else {
472+
p.Spec.Containers[0].Command = o.Args
473+
}
474+
475+
return p
476+
}
477+
374478
// generatePodCopy takes a Pod and returns a copy and the debug container name of that copy
375479
func (o *DebugOptions) generatePodCopyWithDebugContainer(pod *corev1.Pod) (*corev1.Pod, string) {
376480
copied := &corev1.Pod{
@@ -426,7 +530,7 @@ func (o *DebugOptions) computeDebugContainerName(pod *corev1.Pod) string {
426530
cn = fmt.Sprintf("debugger-%s", nameSuffixFunc(5))
427531
}
428532
if !o.Quiet {
429-
fmt.Fprintf(o.ErrOut, "Defaulting debug container name to %s.\n", cn)
533+
fmt.Fprintf(o.Out, "Defaulting debug container name to %s.\n", cn)
430534
}
431535
name = cn
432536
}

‎staging/src/k8s.io/kubectl/pkg/cmd/debug/debug_test.go‎

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -645,3 +645,162 @@ func TestGeneratePodCopyWithDebugContainer(t *testing.T) {
645645
})
646646
}
647647
}
648+
649+
func TestGenerateNodeDebugPod(t *testing.T) {
650+
defer func(old func(int) string) { nameSuffixFunc = old }(nameSuffixFunc)
651+
var suffixCounter int
652+
nameSuffixFunc = func(int) string {
653+
suffixCounter++
654+
return fmt.Sprint(suffixCounter)
655+
}
656+
657+
for _, tc := range []struct {
658+
name, nodeName string
659+
opts *DebugOptions
660+
expected *corev1.Pod
661+
}{
662+
{
663+
name: "minimum options",
664+
nodeName: "node-XXX",
665+
opts: &DebugOptions{
666+
Image: "busybox",
667+
PullPolicy: corev1.PullIfNotPresent,
668+
},
669+
expected: &corev1.Pod{
670+
ObjectMeta: metav1.ObjectMeta{
671+
Name: "node-debugger-node-XXX-1",
672+
},
673+
Spec: corev1.PodSpec{
674+
Containers: []corev1.Container{
675+
{
676+
Name: "debugger",
677+
Image: "busybox",
678+
ImagePullPolicy: corev1.PullIfNotPresent,
679+
TerminationMessagePolicy: corev1.TerminationMessageReadFile,
680+
VolumeMounts: []corev1.VolumeMount{
681+
{
682+
MountPath: "/host",
683+
Name: "host-root",
684+
},
685+
},
686+
},
687+
},
688+
HostIPC: true,
689+
HostNetwork: true,
690+
HostPID: true,
691+
NodeName: "node-XXX",
692+
RestartPolicy: corev1.RestartPolicyNever,
693+
Volumes: []corev1.Volume{
694+
{
695+
Name: "host-root",
696+
VolumeSource: corev1.VolumeSource{
697+
HostPath: &corev1.HostPathVolumeSource{Path: "/"},
698+
},
699+
},
700+
},
701+
},
702+
},
703+
},
704+
{
705+
name: "debug args as container command",
706+
nodeName: "node-XXX",
707+
opts: &DebugOptions{
708+
Args: []string{"/bin/echo", "one", "two", "three"},
709+
Container: "custom-debugger",
710+
Image: "busybox",
711+
PullPolicy: corev1.PullIfNotPresent,
712+
},
713+
expected: &corev1.Pod{
714+
ObjectMeta: metav1.ObjectMeta{
715+
Name: "node-debugger-node-XXX-1",
716+
},
717+
Spec: corev1.PodSpec{
718+
Containers: []corev1.Container{
719+
{
720+
Name: "custom-debugger",
721+
Command: []string{"/bin/echo", "one", "two", "three"},
722+
Image: "busybox",
723+
ImagePullPolicy: corev1.PullIfNotPresent,
724+
TerminationMessagePolicy: corev1.TerminationMessageReadFile,
725+
VolumeMounts: []corev1.VolumeMount{
726+
{
727+
MountPath: "/host",
728+
Name: "host-root",
729+
},
730+
},
731+
},
732+
},
733+
HostIPC: true,
734+
HostNetwork: true,
735+
HostPID: true,
736+
NodeName: "node-XXX",
737+
RestartPolicy: corev1.RestartPolicyNever,
738+
Volumes: []corev1.Volume{
739+
{
740+
Name: "host-root",
741+
VolumeSource: corev1.VolumeSource{
742+
HostPath: &corev1.HostPathVolumeSource{Path: "/"},
743+
},
744+
},
745+
},
746+
},
747+
},
748+
},
749+
{
750+
name: "debug args as container args",
751+
nodeName: "node-XXX",
752+
opts: &DebugOptions{
753+
ArgsOnly: true,
754+
Container: "custom-debugger",
755+
Args: []string{"echo", "one", "two", "three"},
756+
Image: "busybox",
757+
PullPolicy: corev1.PullIfNotPresent,
758+
},
759+
expected: &corev1.Pod{
760+
ObjectMeta: metav1.ObjectMeta{
761+
Name: "node-debugger-node-XXX-1",
762+
},
763+
Spec: corev1.PodSpec{
764+
Containers: []corev1.Container{
765+
{
766+
Name: "custom-debugger",
767+
Args: []string{"echo", "one", "two", "three"},
768+
Image: "busybox",
769+
ImagePullPolicy: corev1.PullIfNotPresent,
770+
TerminationMessagePolicy: corev1.TerminationMessageReadFile,
771+
VolumeMounts: []corev1.VolumeMount{
772+
{
773+
MountPath: "/host",
774+
Name: "host-root",
775+
},
776+
},
777+
},
778+
},
779+
HostIPC: true,
780+
HostNetwork: true,
781+
HostPID: true,
782+
NodeName: "node-XXX",
783+
RestartPolicy: corev1.RestartPolicyNever,
784+
Volumes: []corev1.Volume{
785+
{
786+
Name: "host-root",
787+
VolumeSource: corev1.VolumeSource{
788+
HostPath: &corev1.HostPathVolumeSource{Path: "/"},
789+
},
790+
},
791+
},
792+
},
793+
},
794+
},
795+
} {
796+
t.Run(tc.name, func(t *testing.T) {
797+
tc.opts.IOStreams = genericclioptions.NewTestIOStreamsDiscard()
798+
suffixCounter = 0
799+
800+
pod := tc.opts.generateNodeDebugPod(tc.nodeName)
801+
if diff := cmp.Diff(tc.expected, pod); diff != "" {
802+
t.Error("unexpected diff in generated object: (-want +got):\n", diff)
803+
}
804+
})
805+
}
806+
}

0 commit comments

Comments
 (0)