Skip to content

Commit 9a6d50c

Browse files
committed
Add namespace targeting to the kubelet
1 parent 8971422 commit 9a6d50c

File tree

9 files changed

+242
-30
lines changed

9 files changed

+242
-30
lines changed

pkg/kubelet/kuberuntime/kuberuntime_container.go

Lines changed: 52 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,9 @@ import (
4141
kubetypes "k8s.io/apimachinery/pkg/types"
4242
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
4343
"k8s.io/apimachinery/pkg/util/sets"
44+
utilfeature "k8s.io/apiserver/pkg/util/feature"
4445
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
46+
"k8s.io/kubernetes/pkg/features"
4547
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
4648
"k8s.io/kubernetes/pkg/kubelet/events"
4749
"k8s.io/kubernetes/pkg/kubelet/types"
@@ -86,13 +88,52 @@ func (m *kubeGenericRuntimeManager) recordContainerEvent(pod *v1.Pod, container
8688
m.recorder.Event(ref, eventType, reason, eventMessage)
8789
}
8890

91+
// startSpec wraps the spec required to start a container, either a regular/init container
92+
// or an ephemeral container. Ephemeral containers contain all the fields of regular/init
93+
// containers, plus some additional fields. In both cases startSpec.container will be set.
94+
type startSpec struct {
95+
container *v1.Container
96+
ephemeralContainer *v1.EphemeralContainer
97+
}
98+
99+
func containerStartSpec(c *v1.Container) *startSpec {
100+
return &startSpec{container: c}
101+
}
102+
103+
func ephemeralContainerStartSpec(ec *v1.EphemeralContainer) *startSpec {
104+
return &startSpec{
105+
container: (*v1.Container)(&ec.EphemeralContainerCommon),
106+
ephemeralContainer: ec,
107+
}
108+
}
109+
110+
// getTargetID returns the kubecontainer.ContainerID for ephemeral container namespace
111+
// targeting. The target is stored as EphemeralContainer.TargetContainerName, which must be
112+
// resolved to a ContainerID using podStatus. The target container must already exist, which
113+
// usually isn't a problem since ephemeral containers aren't allowed at pod creation time.
114+
// This always returns nil when the EphemeralContainers feature is disabled.
115+
func (s *startSpec) getTargetID(podStatus *kubecontainer.PodStatus) (*kubecontainer.ContainerID, error) {
116+
if s.ephemeralContainer == nil || s.ephemeralContainer.TargetContainerName == "" || !utilfeature.DefaultFeatureGate.Enabled(features.EphemeralContainers) {
117+
return nil, nil
118+
}
119+
120+
targetStatus := podStatus.FindContainerStatusByName(s.ephemeralContainer.TargetContainerName)
121+
if targetStatus == nil {
122+
return nil, fmt.Errorf("unable to find target container %v", s.ephemeralContainer.TargetContainerName)
123+
}
124+
125+
return &targetStatus.ID, nil
126+
}
127+
89128
// startContainer starts a container and returns a message indicates why it is failed on error.
90129
// It starts the container through the following steps:
91130
// * pull the image
92131
// * create the container
93132
// * start the container
94133
// * run the post start lifecycle hooks (if applicable)
95-
func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandboxConfig *runtimeapi.PodSandboxConfig, container *v1.Container, pod *v1.Pod, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, podIP string, podIPs []string) (string, error) {
134+
func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandboxConfig *runtimeapi.PodSandboxConfig, spec *startSpec, pod *v1.Pod, podStatus *kubecontainer.PodStatus, pullSecrets []v1.Secret, podIP string, podIPs []string) (string, error) {
135+
container := spec.container
136+
96137
// Step 1: pull the image.
97138
imageRef, msg, err := m.imagePuller.EnsureImageExists(pod, container, pullSecrets, podSandboxConfig)
98139
if err != nil {
@@ -115,7 +156,14 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb
115156
restartCount = containerStatus.RestartCount + 1
116157
}
117158

118-
containerConfig, cleanupAction, err := m.generateContainerConfig(container, pod, restartCount, podIP, imageRef, podIPs)
159+
target, err := spec.getTargetID(podStatus)
160+
if err != nil {
161+
s, _ := grpcstatus.FromError(err)
162+
m.recordContainerEvent(pod, container, "", v1.EventTypeWarning, events.FailedToCreateContainer, "Error: %v", s.Message())
163+
return s.Message(), ErrCreateContainerConfig
164+
}
165+
166+
containerConfig, cleanupAction, err := m.generateContainerConfig(container, pod, restartCount, podIP, imageRef, podIPs, target)
119167
if cleanupAction != nil {
120168
defer cleanupAction()
121169
}
@@ -195,7 +243,7 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb
195243
}
196244

197245
// generateContainerConfig generates container config for kubelet runtime v1.
198-
func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Container, pod *v1.Pod, restartCount int, podIP, imageRef string, podIPs []string) (*runtimeapi.ContainerConfig, func(), error) {
246+
func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Container, pod *v1.Pod, restartCount int, podIP, imageRef string, podIPs []string, nsTarget *kubecontainer.ContainerID) (*runtimeapi.ContainerConfig, func(), error) {
199247
opts, cleanupAction, err := m.runtimeHelper.GenerateRunContainerOptions(pod, container, podIP, podIPs)
200248
if err != nil {
201249
return nil, nil, err
@@ -239,7 +287,7 @@ func (m *kubeGenericRuntimeManager) generateContainerConfig(container *v1.Contai
239287
}
240288

241289
// set platform specific configurations.
242-
if err := m.applyPlatformSpecificContainerConfig(config, container, pod, uid, username); err != nil {
290+
if err := m.applyPlatformSpecificContainerConfig(config, container, pod, uid, username, nsTarget); err != nil {
243291
return nil, cleanupAction, err
244292
}
245293

pkg/kubelet/kuberuntime/kuberuntime_container_linux.go

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -28,22 +28,28 @@ import (
2828
"k8s.io/klog"
2929
v1helper "k8s.io/kubernetes/pkg/apis/core/v1/helper"
3030
kubefeatures "k8s.io/kubernetes/pkg/features"
31+
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
3132
"k8s.io/kubernetes/pkg/kubelet/qos"
3233
)
3334

3435
// applyPlatformSpecificContainerConfig applies platform specific configurations to runtimeapi.ContainerConfig.
35-
func (m *kubeGenericRuntimeManager) applyPlatformSpecificContainerConfig(config *runtimeapi.ContainerConfig, container *v1.Container, pod *v1.Pod, uid *int64, username string) error {
36-
config.Linux = m.generateLinuxContainerConfig(container, pod, uid, username)
36+
func (m *kubeGenericRuntimeManager) applyPlatformSpecificContainerConfig(config *runtimeapi.ContainerConfig, container *v1.Container, pod *v1.Pod, uid *int64, username string, nsTarget *kubecontainer.ContainerID) error {
37+
config.Linux = m.generateLinuxContainerConfig(container, pod, uid, username, nsTarget)
3738
return nil
3839
}
3940

4041
// generateLinuxContainerConfig generates linux container config for kubelet runtime v1.
41-
func (m *kubeGenericRuntimeManager) generateLinuxContainerConfig(container *v1.Container, pod *v1.Pod, uid *int64, username string) *runtimeapi.LinuxContainerConfig {
42+
func (m *kubeGenericRuntimeManager) generateLinuxContainerConfig(container *v1.Container, pod *v1.Pod, uid *int64, username string, nsTarget *kubecontainer.ContainerID) *runtimeapi.LinuxContainerConfig {
4243
lc := &runtimeapi.LinuxContainerConfig{
4344
Resources: &runtimeapi.LinuxContainerResources{},
4445
SecurityContext: m.determineEffectiveSecurityContext(pod, container, uid, username),
4546
}
4647

48+
if nsTarget != nil && lc.SecurityContext.NamespaceOptions.Pid == runtimeapi.NamespaceMode_CONTAINER {
49+
lc.SecurityContext.NamespaceOptions.Pid = runtimeapi.NamespaceMode_TARGET
50+
lc.SecurityContext.NamespaceOptions.TargetId = nsTarget.ID
51+
}
52+
4753
// set linux container resources
4854
var cpuShares int64
4955
cpuRequest := container.Resources.Requests.Cpu()

pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go

Lines changed: 77 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,13 +22,17 @@ import (
2222
"reflect"
2323
"testing"
2424

25+
"github.com/google/go-cmp/cmp"
2526
cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
2627
"github.com/stretchr/testify/assert"
2728
v1 "k8s.io/api/core/v1"
29+
"k8s.io/apimachinery/pkg/api/resource"
2830
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
31+
utilfeature "k8s.io/apiserver/pkg/util/feature"
32+
featuregatetesting "k8s.io/component-base/featuregate/testing"
2933
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
30-
31-
"k8s.io/apimachinery/pkg/api/resource"
34+
"k8s.io/kubernetes/pkg/features"
35+
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
3236
)
3337

3438
func makeExpectedConfig(m *kubeGenericRuntimeManager, pod *v1.Pod, containerIndex int) *runtimeapi.ContainerConfig {
@@ -57,7 +61,7 @@ func makeExpectedConfig(m *kubeGenericRuntimeManager, pod *v1.Pod, containerInde
5761
Stdin: container.Stdin,
5862
StdinOnce: container.StdinOnce,
5963
Tty: container.TTY,
60-
Linux: m.generateLinuxContainerConfig(container, pod, new(int64), ""),
64+
Linux: m.generateLinuxContainerConfig(container, pod, new(int64), "", nil),
6165
Envs: envs,
6266
}
6367
return expectedConfig
@@ -93,7 +97,7 @@ func TestGenerateContainerConfig(t *testing.T) {
9397
}
9498

9599
expectedConfig := makeExpectedConfig(m, pod, 0)
96-
containerConfig, _, err := m.generateContainerConfig(&pod.Spec.Containers[0], pod, 0, "", pod.Spec.Containers[0].Image, []string{})
100+
containerConfig, _, err := m.generateContainerConfig(&pod.Spec.Containers[0], pod, 0, "", pod.Spec.Containers[0].Image, []string{}, nil)
97101
assert.NoError(t, err)
98102
assert.Equal(t, expectedConfig, containerConfig, "generate container config for kubelet runtime v1.")
99103
assert.Equal(t, runAsUser, containerConfig.GetLinux().GetSecurityContext().GetRunAsUser().GetValue(), "RunAsUser should be set")
@@ -124,7 +128,7 @@ func TestGenerateContainerConfig(t *testing.T) {
124128
},
125129
}
126130

127-
_, _, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image, []string{})
131+
_, _, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image, []string{}, nil)
128132
assert.Error(t, err)
129133

130134
imageID, _ := imageService.PullImage(&runtimeapi.ImageSpec{Image: "busybox"}, nil, nil)
@@ -136,7 +140,7 @@ func TestGenerateContainerConfig(t *testing.T) {
136140
podWithContainerSecurityContext.Spec.Containers[0].SecurityContext.RunAsUser = nil
137141
podWithContainerSecurityContext.Spec.Containers[0].SecurityContext.RunAsNonRoot = &runAsNonRootTrue
138142

139-
_, _, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image, []string{})
143+
_, _, err = m.generateContainerConfig(&podWithContainerSecurityContext.Spec.Containers[0], podWithContainerSecurityContext, 0, "", podWithContainerSecurityContext.Spec.Containers[0].Image, []string{}, nil)
140144
assert.Error(t, err, "RunAsNonRoot should fail for non-numeric username")
141145
}
142146

@@ -296,3 +300,70 @@ func TestGetHugepageLimitsFromResources(t *testing.T) {
296300
}
297301
}
298302
}
303+
304+
func TestGenerateLinuxContainerConfigNamespaces(t *testing.T) {
305+
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EphemeralContainers, true)()
306+
_, _, m, err := createTestRuntimeManager()
307+
if err != nil {
308+
t.Fatalf("error creating test RuntimeManager: %v", err)
309+
}
310+
311+
for _, tc := range []struct {
312+
name string
313+
pod *v1.Pod
314+
target *kubecontainer.ContainerID
315+
want *runtimeapi.NamespaceOption
316+
}{
317+
{
318+
"Default namespaces",
319+
&v1.Pod{
320+
Spec: v1.PodSpec{
321+
Containers: []v1.Container{
322+
{Name: "test"},
323+
},
324+
},
325+
},
326+
nil,
327+
&runtimeapi.NamespaceOption{
328+
Pid: runtimeapi.NamespaceMode_CONTAINER,
329+
},
330+
},
331+
{
332+
"PID Namespace POD",
333+
&v1.Pod{
334+
Spec: v1.PodSpec{
335+
Containers: []v1.Container{
336+
{Name: "test"},
337+
},
338+
ShareProcessNamespace: &[]bool{true}[0],
339+
},
340+
},
341+
nil,
342+
&runtimeapi.NamespaceOption{
343+
Pid: runtimeapi.NamespaceMode_POD,
344+
},
345+
},
346+
{
347+
"PID Namespace TARGET",
348+
&v1.Pod{
349+
Spec: v1.PodSpec{
350+
Containers: []v1.Container{
351+
{Name: "test"},
352+
},
353+
},
354+
},
355+
&kubecontainer.ContainerID{Type: "docker", ID: "really-long-id-string"},
356+
&runtimeapi.NamespaceOption{
357+
Pid: runtimeapi.NamespaceMode_TARGET,
358+
TargetId: "really-long-id-string",
359+
},
360+
},
361+
} {
362+
t.Run(tc.name, func(t *testing.T) {
363+
got := m.generateLinuxContainerConfig(&tc.pod.Spec.Containers[0], tc.pod, nil, "", tc.target)
364+
if diff := cmp.Diff(tc.want, got.SecurityContext.NamespaceOptions); diff != "" {
365+
t.Errorf("%v: diff (-want +got):\n%v", t.Name(), diff)
366+
}
367+
})
368+
}
369+
}

pkg/kubelet/kuberuntime/kuberuntime_container_test.go

Lines changed: 76 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,22 @@ limitations under the License.
1717
package kuberuntime
1818

1919
import (
20+
"fmt"
2021
"path/filepath"
2122
"strings"
2223
"testing"
2324
"time"
2425

26+
"github.com/google/go-cmp/cmp"
2527
"github.com/stretchr/testify/assert"
2628
"github.com/stretchr/testify/require"
2729
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2830

29-
"k8s.io/api/core/v1"
31+
v1 "k8s.io/api/core/v1"
32+
utilfeature "k8s.io/apiserver/pkg/util/feature"
33+
featuregatetesting "k8s.io/component-base/featuregate/testing"
3034
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
35+
"k8s.io/kubernetes/pkg/features"
3136
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
3237
containertest "k8s.io/kubernetes/pkg/kubelet/container/testing"
3338
"k8s.io/kubernetes/pkg/kubelet/lifecycle"
@@ -325,7 +330,7 @@ func TestLifeCycleHook(t *testing.T) {
325330
}
326331

327332
// Now try to create a container, which should in turn invoke PostStart Hook
328-
_, err := m.startContainer(fakeSandBox.Id, fakeSandBoxConfig, testContainer, testPod, fakePodStatus, nil, "", []string{})
333+
_, err := m.startContainer(fakeSandBox.Id, fakeSandBoxConfig, containerStartSpec(testContainer), testPod, fakePodStatus, nil, "", []string{})
329334
if err != nil {
330335
t.Errorf("startContainer error =%v", err)
331336
}
@@ -334,3 +339,72 @@ func TestLifeCycleHook(t *testing.T) {
334339
}
335340
})
336341
}
342+
343+
func TestStartSpec(t *testing.T) {
344+
podStatus := &kubecontainer.PodStatus{
345+
ContainerStatuses: []*kubecontainer.ContainerStatus{
346+
{
347+
ID: kubecontainer.ContainerID{
348+
Type: "docker",
349+
ID: "docker-something-something",
350+
},
351+
Name: "target",
352+
},
353+
},
354+
}
355+
356+
for _, tc := range []struct {
357+
name string
358+
spec *startSpec
359+
want *kubecontainer.ContainerID
360+
}{
361+
{
362+
"Regular Container",
363+
containerStartSpec(&v1.Container{
364+
Name: "test",
365+
}),
366+
nil,
367+
},
368+
{
369+
"Ephemeral Container w/o Target",
370+
ephemeralContainerStartSpec(&v1.EphemeralContainer{
371+
EphemeralContainerCommon: v1.EphemeralContainerCommon{
372+
Name: "test",
373+
},
374+
}),
375+
nil,
376+
},
377+
{
378+
"Ephemeral Container w/ Target",
379+
ephemeralContainerStartSpec(&v1.EphemeralContainer{
380+
EphemeralContainerCommon: v1.EphemeralContainerCommon{
381+
Name: "test",
382+
},
383+
TargetContainerName: "target",
384+
}),
385+
&kubecontainer.ContainerID{
386+
Type: "docker",
387+
ID: "docker-something-something",
388+
},
389+
},
390+
} {
391+
t.Run(tc.name, func(t *testing.T) {
392+
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EphemeralContainers, true)()
393+
if got, err := tc.spec.getTargetID(podStatus); err != nil {
394+
t.Fatalf("%v: getTargetID got unexpected error: %v", t.Name(), err)
395+
} else if diff := cmp.Diff(tc.want, got); diff != "" {
396+
t.Errorf("%v: getTargetID got unexpected result. diff:\n%v", t.Name(), diff)
397+
}
398+
})
399+
400+
// Test with feature disabled in self-contained section which can be removed when feature flag is removed.
401+
t.Run(fmt.Sprintf("%s (disabled)", tc.name), func(t *testing.T) {
402+
defer featuregatetesting.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.EphemeralContainers, false)()
403+
if got, err := tc.spec.getTargetID(podStatus); err != nil {
404+
t.Fatalf("%v: getTargetID got unexpected error: %v", t.Name(), err)
405+
} else if got != nil {
406+
t.Errorf("%v: getTargetID got: %v, wanted nil", t.Name(), got)
407+
}
408+
})
409+
}
410+
}

pkg/kubelet/kuberuntime/kuberuntime_container_unsupported.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,10 @@ package kuberuntime
2121
import (
2222
"k8s.io/api/core/v1"
2323
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
24+
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
2425
)
2526

2627
// applyPlatformSpecificContainerConfig applies platform specific configurations to runtimeapi.ContainerConfig.
27-
func (m *kubeGenericRuntimeManager) applyPlatformSpecificContainerConfig(config *runtimeapi.ContainerConfig, container *v1.Container, pod *v1.Pod, uid *int64, username string) error {
28+
func (m *kubeGenericRuntimeManager) applyPlatformSpecificContainerConfig(config *runtimeapi.ContainerConfig, container *v1.Container, pod *v1.Pod, uid *int64, username string, nsTarget *kubecontainer.ContainerID) error {
2829
return nil
2930
}

pkg/kubelet/kuberuntime/kuberuntime_container_windows.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,18 +20,20 @@ package kuberuntime
2020

2121
import (
2222
"fmt"
23+
2324
"github.com/docker/docker/pkg/sysinfo"
2425

2526
"k8s.io/api/core/v1"
2627
utilfeature "k8s.io/apiserver/pkg/util/feature"
2728
runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1alpha2"
2829
kubefeatures "k8s.io/kubernetes/pkg/features"
2930
kubeletapis "k8s.io/kubernetes/pkg/kubelet/apis"
31+
kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
3032
"k8s.io/kubernetes/pkg/securitycontext"
3133
)
3234

3335
// applyPlatformSpecificContainerConfig applies platform specific configurations to runtimeapi.ContainerConfig.
34-
func (m *kubeGenericRuntimeManager) applyPlatformSpecificContainerConfig(config *runtimeapi.ContainerConfig, container *v1.Container, pod *v1.Pod, uid *int64, username string) error {
36+
func (m *kubeGenericRuntimeManager) applyPlatformSpecificContainerConfig(config *runtimeapi.ContainerConfig, container *v1.Container, pod *v1.Pod, uid *int64, username string, _ *kubecontainer.ContainerID) error {
3537
windowsConfig, err := m.generateWindowsContainerConfig(container, pod, uid, username)
3638
if err != nil {
3739
return err

0 commit comments

Comments
 (0)