Skip to content

Commit 254fef7

Browse files
andyzhangxandyzhangx
authored
Merge pull request #2679 from kubernetes-sigs/kata-extensible
chore: make runtimeClassHandler and confidentialContainerLabel configurable
2 parents 4d9c909 + 1937e19 commit 254fef7

File tree

8 files changed

+77
-62
lines changed

8 files changed

+77
-62
lines changed

‎pkg/azurefile/azurefile.go

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,10 @@ const (
162162
premium = "premium"
163163
selectRandomMatchingAccountField = "selectrandommatchingaccount"
164164
accountQuotaField = "accountquota"
165-
defaultKataCCLabel = "kubernetes.azure.com/kata-cc-isolation"
165+
confidentialContainerLabelField = "confidentialcontainerlabel"
166+
defaultConfidentialContainerLabel = "kubernetes.azure.com/kata-cc-isolation"
167+
runtimeClassHandlerField = "runtimeclasshandler"
168+
defaultRuntimeClassHandler = "kata-cc"
166169

167170
accountNotProvisioned = "StorageAccountIsNotProvisioned"
168171
// this is a workaround fix for 429 throttling issue, will update cloud provider for better fix later
@@ -470,7 +473,7 @@ func (d *Driver) Run(ctx context.Context) error {
470473
csi.RegisterControllerServer(server, d)
471474
csi.RegisterNodeServer(server, d)
472475
d.server = server
473-
d.isKataNode = isKataNode(ctx, d.NodeID, d.kubeClient)
476+
d.isKataNode = isKataNode(ctx, d.NodeID, defaultConfidentialContainerLabel, d.kubeClient)
474477

475478
listener, err := csicommon.ListenEndpoint(ctx, d.endpoint)
476479
if err != nil {
@@ -1343,7 +1346,7 @@ func (d *Driver) getFileShareClientForSub(subscriptionID string) (fileshareclien
13431346
return d.cloud.ComputeClientFactory.GetFileShareClientForSub(subscriptionID)
13441347
}
13451348

1346-
func isKataNode(ctx context.Context, nodeID string, kubeClient clientset.Interface) bool {
1349+
func isKataNode(ctx context.Context, nodeID, confidentialContainerLabel string, kubeClient clientset.Interface) bool {
13471350
if nodeID == "" {
13481351
return false
13491352
}
@@ -1364,7 +1367,7 @@ func isKataNode(ctx context.Context, nodeID string, kubeClient clientset.Interfa
13641367
}
13651368

13661369
// Check for the kata isolation labels
1367-
if _, ok := node.Labels[defaultKataCCLabel]; !ok {
1370+
if _, ok := node.Labels[confidentialContainerLabel]; !ok {
13681371
return false
13691372
}
13701373
klog.V(4).Infof("node(%s) is a kata node with labels: %v", nodeID, node.Labels)

‎pkg/azurefile/azurefile_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1675,7 +1675,7 @@ func TestIsKataNode(t *testing.T) {
16751675
nodeName: "test-node",
16761676
setupClient: true,
16771677
labels: map[string]string{
1678-
defaultKataCCLabel: "",
1678+
defaultConfidentialContainerLabel: "",
16791679
},
16801680
expected: true,
16811681
},
@@ -1684,7 +1684,7 @@ func TestIsKataNode(t *testing.T) {
16841684
nodeName: "test-node",
16851685
setupClient: true,
16861686
labels: map[string]string{
1687-
defaultKataCCLabel: "test",
1687+
defaultConfidentialContainerLabel: "test",
16881688
"kubernetes.azure.com/kata-mshv-vm-isolation": "true",
16891689
"katacontainers.io/kata-runtime": "true",
16901690
},
@@ -1711,7 +1711,7 @@ func TestIsKataNode(t *testing.T) {
17111711
_, err := clientset.CoreV1().Nodes().Create(ctx, node, metav1.CreateOptions{})
17121712
assert.NoError(t, err)
17131713
}
1714-
result := isKataNode(ctx, tc.nodeName, clientset)
1714+
result := isKataNode(ctx, tc.nodeName, defaultConfidentialContainerLabel, clientset)
17151715
assert.Equal(t, tc.expected, result)
17161716
})
17171717
}

‎pkg/azurefile/controllerserver.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -226,10 +226,10 @@ func (d *Driver) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest)
226226
case pvNameKey:
227227
fileShareNameReplaceMap[pvNameMetadata] = v
228228
case serverNameField:
229-
// no op, only used in NodeStageVolume
230229
case folderNameField:
231-
// no op, only used in NodeStageVolume
232230
case clientIDField:
231+
case confidentialContainerLabelField:
232+
case runtimeClassHandlerField:
233233
// no op, only used in NodeStageVolume
234234
case fsGroupChangePolicyField:
235235
fsGroupChangePolicy = v

‎pkg/azurefile/controllerserver_test.go

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -906,22 +906,24 @@ var _ = ginkgo.Describe("TestCreateVolume", func() {
906906
}
907907

908908
allParam := map[string]string{
909-
skuNameField: "premium",
910-
storageAccountTypeField: "stoacctype",
911-
locationField: "loc",
912-
storageAccountField: "stoacc",
913-
resourceGroupField: "rg",
914-
shareNameField: "",
915-
diskNameField: "diskname.vhd",
916-
fsTypeField: "",
917-
storeAccountKeyField: "storeaccountkey",
918-
secretNamespaceField: "default",
919-
mountPermissionsField: "0755",
920-
accountQuotaField: "1000",
921-
useDataPlaneAPIField: "oauth",
922-
clientIDField: "client-id",
923-
provisionedBandwidthField: "100",
924-
provisionedIopsField: "800",
909+
skuNameField: "premium",
910+
storageAccountTypeField: "stoacctype",
911+
locationField: "loc",
912+
storageAccountField: "stoacc",
913+
resourceGroupField: "rg",
914+
shareNameField: "",
915+
diskNameField: "diskname.vhd",
916+
fsTypeField: "",
917+
storeAccountKeyField: "storeaccountkey",
918+
secretNamespaceField: "default",
919+
mountPermissionsField: "0755",
920+
accountQuotaField: "1000",
921+
useDataPlaneAPIField: "oauth",
922+
clientIDField: "client-id",
923+
provisionedBandwidthField: "100",
924+
provisionedIopsField: "800",
925+
runtimeClassHandlerField: "runtime-handler",
926+
confidentialContainerLabelField: "confidential-container-label",
925927
}
926928

927929
req := &csi.CreateVolumeRequest{

‎pkg/azurefile/nodeserver.go

Lines changed: 39 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -113,40 +113,52 @@ func (d *Driver) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolu
113113
}
114114
}
115115

116-
enableKataCCMount := d.isKataNode && d.enableKataCCMount
117-
if enableKataCCMount && context[podNameField] != "" && context[podNamespaceField] != "" {
118-
runtimeClass, err := getRuntimeClassForPodFunc(ctx, d.kubeClient, context[podNameField], context[podNamespaceField])
119-
if err != nil {
120-
return nil, status.Errorf(codes.Internal, "failed to get runtime class for pod %s/%s: %v", context[podNamespaceField], context[podNameField], err)
121-
}
122-
klog.V(2).Infof("NodePublishVolume: volume(%s) mount on %s with runtimeClass %s", volumeID, target, runtimeClass)
123-
isConfidentialRuntimeClass, err := isConfidentialRuntimeClassFunc(ctx, d.kubeClient, runtimeClass)
124-
if err != nil {
125-
return nil, status.Errorf(codes.Internal, "failed to check if runtime class %s is confidential: %v", runtimeClass, err)
116+
if d.enableKataCCMount && context[podNameField] != "" && context[podNamespaceField] != "" {
117+
enableKataCCMount := d.isKataNode
118+
confidentialContainerLabel := getValueInMap(context, confidentialContainerLabelField)
119+
if !enableKataCCMount && confidentialContainerLabel != "" {
120+
klog.V(2).Infof("NodePublishVolume: checking if node %s is a kata node with confidential container label %s", d.NodeID, confidentialContainerLabel)
121+
enableKataCCMount = isKataNode(ctx, d.NodeID, confidentialContainerLabel, d.kubeClient)
126122
}
127-
if isConfidentialRuntimeClass {
128-
klog.V(2).Infof("NodePublishVolume for volume(%s) where runtimeClass is %s", volumeID, runtimeClass)
129-
source := req.GetStagingTargetPath()
130-
if len(source) == 0 {
131-
return nil, status.Error(codes.InvalidArgument, "Staging target not provided")
132-
}
133-
// Load the mount info from staging area
134-
mountInfo, err := d.directVolume.VolumeMountInfo(source)
123+
124+
if enableKataCCMount {
125+
runtimeClass, err := getRuntimeClassForPodFunc(ctx, d.kubeClient, context[podNameField], context[podNamespaceField])
135126
if err != nil {
136-
return nil, status.Errorf(codes.Internal, "failed to load mount info from %s: %v", source, err)
127+
return nil, status.Errorf(codes.Internal, "failed to get runtime class for pod %s/%s: %v", context[podNamespaceField], context[podNameField], err)
137128
}
138-
if mountInfo == nil {
139-
return nil, status.Errorf(codes.Internal, "mount info is nil for volume %s", volumeID)
129+
klog.V(2).Infof("NodePublishVolume: volume(%s) mount on %s with runtimeClass %s", volumeID, target, runtimeClass)
130+
runtimeClassHandler := getValueInMap(context, runtimeClassHandlerField)
131+
if runtimeClassHandler == "" {
132+
runtimeClassHandler = defaultRuntimeClassHandler
140133
}
141-
data, err := json.Marshal(mountInfo)
134+
isConfidentialRuntimeClass, err := isConfidentialRuntimeClassFunc(ctx, d.kubeClient, runtimeClass, runtimeClassHandler)
142135
if err != nil {
143-
return nil, status.Errorf(codes.Internal, "failed to marshal mount info %s: %v", source, err)
136+
return nil, status.Errorf(codes.Internal, "failed to check if runtime class %s is confidential: %v", runtimeClass, err)
144137
}
145-
if err = d.directVolume.Add(target, string(data)); err != nil {
146-
return nil, status.Errorf(codes.Internal, "failed to save mount info %s: %v", target, err)
138+
if isConfidentialRuntimeClass {
139+
klog.V(2).Infof("NodePublishVolume for volume(%s) where runtimeClass is %s", volumeID, runtimeClass)
140+
source := req.GetStagingTargetPath()
141+
if len(source) == 0 {
142+
return nil, status.Error(codes.InvalidArgument, "Staging target not provided")
143+
}
144+
// Load the mount info from staging area
145+
mountInfo, err := d.directVolume.VolumeMountInfo(source)
146+
if err != nil {
147+
return nil, status.Errorf(codes.Internal, "failed to load mount info from %s: %v", source, err)
148+
}
149+
if mountInfo == nil {
150+
return nil, status.Errorf(codes.Internal, "mount info is nil for volume %s", volumeID)
151+
}
152+
data, err := json.Marshal(mountInfo)
153+
if err != nil {
154+
return nil, status.Errorf(codes.Internal, "failed to marshal mount info %s: %v", source, err)
155+
}
156+
if err = d.directVolume.Add(target, string(data)); err != nil {
157+
return nil, status.Errorf(codes.Internal, "failed to save mount info %s: %v", target, err)
158+
}
159+
klog.V(2).Infof("NodePublishVolume: direct volume mount %s at %s successfully", source, target)
160+
return &csi.NodePublishVolumeResponse{}, nil
147161
}
148-
klog.V(2).Infof("NodePublishVolume: direct volume mount %s at %s successfully", source, target)
149-
return &csi.NodePublishVolumeResponse{}, nil
150162
}
151163
}
152164
}

‎pkg/azurefile/nodeserver_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -117,7 +117,7 @@ func mockGetRuntimeClassForPod(_ context.Context, _ clientset.Interface, _, _ st
117117
return "mockRuntimeClass", nil
118118
}
119119

120-
func mockIsConfidentialRuntimeClass(_ context.Context, _ clientset.Interface, _ string) (bool, error) {
120+
func mockIsConfidentialRuntimeClass(_ context.Context, _ clientset.Interface, _ string, _ string) (bool, error) {
121121
return true, nil
122122
}
123123

‎pkg/azurefile/utils.go

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -319,10 +319,8 @@ func isReadOnlyFromCapability(vc *csi.VolumeCapability) bool {
319319
mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY)
320320
}
321321

322-
const confidentialRuntimeClassHandler = "kata-cc"
323-
324322
// check if runtimeClass is confidential
325-
func isConfidentialRuntimeClass(ctx context.Context, kubeClient clientset.Interface, runtimeClassName string) (bool, error) {
323+
func isConfidentialRuntimeClass(ctx context.Context, kubeClient clientset.Interface, runtimeClassName, runtimeClassHandler string) (bool, error) {
326324
// if runtimeClassName is empty, return false
327325
if runtimeClassName == "" {
328326
return false, nil
@@ -336,7 +334,7 @@ func isConfidentialRuntimeClass(ctx context.Context, kubeClient clientset.Interf
336334
return false, err
337335
}
338336
klog.V(4).Infof("runtimeClass %s handler: %s", runtimeClassName, runtimeClass.Handler)
339-
return runtimeClass.Handler == confidentialRuntimeClassHandler, nil
337+
return runtimeClass.Handler == runtimeClassHandler, nil
340338
}
341339

342340
// getBackOff returns a backoff object based on the config

‎pkg/azurefile/utils_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -806,7 +806,7 @@ func TestIsConfidentialRuntimeClass(t *testing.T) {
806806
ctx := context.TODO()
807807

808808
// Test the case where kubeClient is nil
809-
_, err := isConfidentialRuntimeClass(ctx, nil, "test-runtime-class")
809+
_, err := isConfidentialRuntimeClass(ctx, nil, "test-runtime-class", defaultRuntimeClassHandler)
810810
if err == nil || err.Error() != "kubeClient is nil" {
811811
t.Fatalf("expected error 'kubeClient is nil', got %v", err)
812812
}
@@ -819,14 +819,14 @@ func TestIsConfidentialRuntimeClass(t *testing.T) {
819819
ObjectMeta: metav1.ObjectMeta{
820820
Name: "test-runtime-class",
821821
},
822-
Handler: confidentialRuntimeClassHandler,
822+
Handler: defaultRuntimeClassHandler,
823823
}
824824
_, err = clientset.NodeV1().RuntimeClasses().Create(ctx, runtimeClass, metav1.CreateOptions{})
825825
if err != nil {
826826
t.Fatalf("expected no error, got %v", err)
827827
}
828828

829-
isConfidential, err := isConfidentialRuntimeClass(ctx, clientset, "test-runtime-class")
829+
isConfidential, err := isConfidentialRuntimeClass(ctx, clientset, "test-runtime-class", defaultRuntimeClassHandler)
830830
if err != nil {
831831
t.Fatalf("expected no error, got %v", err)
832832
}
@@ -847,7 +847,7 @@ func TestIsConfidentialRuntimeClass(t *testing.T) {
847847
t.Fatalf("expected no error, got %v", err)
848848
}
849849

850-
isConfidential, err = isConfidentialRuntimeClass(ctx, clientset, "test-runtime-class-non-confidential")
850+
isConfidential, err = isConfidentialRuntimeClass(ctx, clientset, "test-runtime-class-non-confidential", defaultRuntimeClassHandler)
851851
if err != nil {
852852
t.Fatalf("expected no error, got %v", err)
853853
}
@@ -857,7 +857,7 @@ func TestIsConfidentialRuntimeClass(t *testing.T) {
857857
}
858858

859859
// Test the case where the runtime class does not exist
860-
_, err = isConfidentialRuntimeClass(ctx, clientset, "nonexistent-runtime-class")
860+
_, err = isConfidentialRuntimeClass(ctx, clientset, "nonexistent-runtime-class", defaultRuntimeClassHandler)
861861
if err == nil {
862862
t.Fatalf("expected an error, got nil")
863863
}

0 commit comments

Comments
 (0)