Skip to content

Commit 8613f0f

Browse files
sunnylovestiramisuSneha-at
authored andcommitted
Add GKE Data Cache Feature Support
1 parent 586f6c8 commit 8613f0f

File tree

16 files changed

+469
-36
lines changed

16 files changed

+469
-36
lines changed

Dockerfile

Lines changed: 44 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ RUN GOARCH=$(echo $TARGETPLATFORM | cut -f2 -d '/') GCE_PD_CSI_STAGING_VERSION=$
2525
FROM gke.gcr.io/debian-base:bullseye-v1.4.3-gke.5 as debian
2626
# Install necessary dependencies
2727
# google_nvme_id script depends on the following packages: nvme-cli, xxd, bash
28-
RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs nvme-cli xxd bash
28+
RUN clean-install util-linux e2fsprogs mount ca-certificates udev xfsprogs nvme-cli xxd bash kmod lvm2
2929

3030
# Since we're leveraging apt to pull in dependencies, we use `gcr.io/distroless/base` because it includes glibc.
3131
FROM gcr.io/distroless/base-debian11 as distroless-base
@@ -54,6 +54,33 @@ COPY --from=debian /sbin/e2fsck /sbin/e2fsck
5454
COPY --from=debian /sbin/fsck /sbin/fsck
5555
COPY --from=debian /sbin/fsck* /sbin/
5656
COPY --from=debian /sbin/fsck.xfs /sbin/fsck.xfs
57+
# Add dependencies for LVM
58+
COPY --from=debian /etc/lvm /etc/lvm
59+
COPY --from=debian /etc/lvm* /etc/
60+
COPY --from=debian /lib/systemd/system/blk-availability.service /lib/systemd/system/blk-availability.service
61+
COPY --from=debian /lib/systemd/system/lvm2-lvmpolld.service /lib/systemd/system/lvm2-lvmpolld.service
62+
COPY --from=debian /lib/systemd/system/lvm2-lvmpolld.socket /lib/systemd/system/lvm2-lvmpolld.socket
63+
COPY --from=debian /lib/systemd/system/lvm2-monitor.service /lib/systemd/system/lvm2-monitor.service
64+
COPY --from=debian /lib/udev/rules.d/56-lvm.rules /lib/udev/rules.d/56-lvm.rules
65+
COPY --from=debian /sbin/fsadm /sbin/fsadm
66+
COPY --from=debian /sbin/lvm /sbin/lvm
67+
COPY --from=debian /sbin/lvmdump /sbin/lvmdump
68+
COPY --from=debian /sbin/lvmpolld /sbin/lvmpolld
69+
COPY --from=debian /usr/lib/tmpfiles.d /usr/lib/tmpfiles.d
70+
COPY --from=debian /usr/lib/tmpfiles.d/lvm2.conf /usr/lib/tmpfiles.d/lvm2.conf
71+
COPY --from=debian /sbin/lv* /sbin/
72+
COPY --from=debian /sbin/pv* /sbin/
73+
COPY --from=debian /sbin/vg* /sbin/
74+
COPY --from=debian /sbin/modprobe /sbin/modprobe
75+
COPY --from=debian /lib/udev /lib/udev
76+
COPY --from=debian /lib/udev/rules.d /lib/udev/rules.d
77+
COPY --from=debian /lib/udev/rules.d/55-dm.rules /lib/udev/rules.d/55-dm.rules
78+
COPY --from=debian /lib/udev/rules.d/60-persistent-storage-dm.rules /lib/udev/rules.d/60-persistent-storage-dm.rules
79+
COPY --from=debian /lib/udev/rules.d/95-dm-notify.rules /lib/udev/rules.d/95-dm-notify.rules
80+
COPY --from=debian /sbin/blkdeactivate /sbin/blkdeactivate
81+
COPY --from=debian /sbin/dmsetup /sbin/dmsetup
82+
COPY --from=debian /sbin/dmstats /sbin/dmstats
83+
# End of dependencies for LVM
5784
COPY --from=debian /sbin/mke2fs /sbin/mke2fs
5885
COPY --from=debian /sbin/mkfs* /sbin/
5986
COPY --from=debian /sbin/resize2fs /sbin/resize2fs
@@ -76,12 +103,18 @@ COPY --from=debian /lib/${LIB_DIR_PREFIX}-linux-gnu/libpcre.so.3 \
76103
/lib/${LIB_DIR_PREFIX}-linux-gnu/libselinux.so.1 \
77104
/lib/${LIB_DIR_PREFIX}-linux-gnu/libtinfo.so.6 \
78105
/lib/${LIB_DIR_PREFIX}-linux-gnu/libe2p.so.2 \
106+
# The following does not exist in either lib or usr/lib
107+
# /lib/${LIB_DIR_PREFIX}-linux-gnu/libcap.so.2 \
79108
/lib/${LIB_DIR_PREFIX}-linux-gnu/libcom_err.so.2 \
80109
/lib/${LIB_DIR_PREFIX}-linux-gnu/libdevmapper.so.1.02.1 \
110+
/lib/${LIB_DIR_PREFIX}-linux-gnu/libm.so.6 \
111+
/lib/${LIB_DIR_PREFIX}-linux-gnu/libc.so.6 \
112+
/lib/${LIB_DIR_PREFIX}-linux-gnu/libdevmapper-event.so.1.02.1 \
81113
/lib/${LIB_DIR_PREFIX}-linux-gnu/libext2fs.so.2 \
82114
/lib/${LIB_DIR_PREFIX}-linux-gnu/libgcc_s.so.1 \
83115
/lib/${LIB_DIR_PREFIX}-linux-gnu/liblzma.so.5 \
84116
/lib/${LIB_DIR_PREFIX}-linux-gnu/libreadline.so.8 \
117+
/lib/${LIB_DIR_PREFIX}-linux-gnu/libgpg-error.so.0 \
85118
/lib/${LIB_DIR_PREFIX}-linux-gnu/libz.so.1 /lib/${LIB_DIR_PREFIX}-linux-gnu/
86119

87120
COPY --from=debian /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libblkid.so.1 \
@@ -90,6 +123,11 @@ COPY --from=debian /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libblkid.so.1 \
90123
/usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libmount.so.1 \
91124
/usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libudev.so.1 \
92125
/usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libuuid.so.1 \
126+
/usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libzstd.so.1 \
127+
/usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libaio.so.1 \
128+
/usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libgcrypt.so.20 \
129+
/usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libsystemd.so.0 \
130+
/usr/lib/${LIB_DIR_PREFIX}-linux-gnu/liblz4.so.1 \
93131
/usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libacl.so.1 \
94132
/usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libattr.so.1 \
95133
/usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libedit.so.2 \
@@ -104,6 +142,11 @@ COPY --from=debian /usr/lib/${LIB_DIR_PREFIX}-linux-gnu/libblkid.so.1 \
104142
# Copy NVME support required script and rules into distroless base.
105143
COPY deploy/kubernetes/udev/google_nvme_id /lib/udev_containerized/google_nvme_id
106144

145+
SHELL ["/bin/bash", "-c"]
146+
RUN /bin/sed -i -e "s/.*allow_mixed_block_sizes = 0.*/ allow_mixed_block_sizes = 1/" /etc/lvm/lvm.conf
147+
RUN /bin/sed -i -e "s/.*udev_sync = 1.*/ udev_sync = 0/" /etc/lvm/lvm.conf
148+
RUN /bin/sed -i -e "s/.*udev_rules = 1.*/ udev_rules = 0/" /etc/lvm/lvm.conf
149+
107150
# Build stage used for validation of the output-image
108151
# See validate-container-linux-* targets in Makefile
109152
FROM output-image as validation-image

Makefile

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ else
4848
endif
4949

5050
build-container: require-GCE_PD_CSI_STAGING_IMAGE require-GCE_PD_CSI_STAGING_VERSION init-buildx
51-
$(DOCKER) buildx build --platform=linux --progress=plain \
51+
$(DOCKER) buildx build --progress=plain --platform=linux --progress=plain \
5252
-t $(STAGINGIMAGE):$(STAGINGVERSION) \
5353
--build-arg BUILDPLATFORM=linux \
5454
--build-arg STAGINGVERSION=$(STAGINGVERSION) \

cmd/gce-pd-csi-driver/main.go

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ var (
7171
formatAndMountTimeout = flag.Duration("format-and-mount-timeout", 1*time.Minute, "The maximum duration of a format and mount operation before another such operation will be started. Used only if --serialize-format-and-mount")
7272
fallbackRequisiteZonesFlag = flag.String("fallback-requisite-zones", "", "Comma separated list of requisite zones that will be used if there are not sufficient zones present in requisite topologies when provisioning a disk")
7373
enableStoragePoolsFlag = flag.Bool("enable-storage-pools", false, "If set to true, the CSI Driver will allow volumes to be provisioned in Storage Pools")
74+
// TODO: set enableDataCacheFlag default to false after testing
75+
enableDataCacheFlag = flag.Bool("enable-data-cache", true, "If set to true, the CSI Driver will allow volumes to be provisioned with data cache configuration")
7476

7577
multiZoneVolumeHandleDiskTypesFlag = flag.String("multi-zone-volume-handle-disk-types", "", "Comma separated list of allowed disk types that can use the multi-zone volumeHandle. Used only if --multi-zone-volume-handle-enable")
7678
multiZoneVolumeHandleEnableFlag = flag.Bool("multi-zone-volume-handle-enable", false, "If set to true, the multi-zone volumeHandle feature will be enabled")
@@ -208,7 +210,7 @@ func handle() {
208210
}
209211
initialBackoffDuration := time.Duration(*errorBackoffInitialDurationMs) * time.Millisecond
210212
maxBackoffDuration := time.Duration(*errorBackoffMaxDurationMs) * time.Millisecond
211-
controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, multiZoneVolumeHandleConfig, listVolumesConfig)
213+
controllerServer = driver.NewControllerServer(gceDriver, cloudProvider, initialBackoffDuration, maxBackoffDuration, fallbackRequisiteZones, *enableStoragePoolsFlag, *enableDataCacheFlag, multiZoneVolumeHandleConfig, listVolumesConfig)
212214
} else if *cloudConfigFilePath != "" {
213215
klog.Warningf("controller service is disabled but cloud config given - it has no effect")
214216
}
@@ -226,7 +228,7 @@ func handle() {
226228
if err != nil {
227229
klog.Fatalf("Failed to set up metadata service: %v", err.Error())
228230
}
229-
nodeServer = driver.NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter)
231+
nodeServer = driver.NewNodeServer(gceDriver, mounter, deviceUtils, meta, statter, *enableDataCacheFlag)
230232
if *maxConcurrentFormatAndMount > 0 {
231233
nodeServer = nodeServer.WithSerializedFormatAndMount(*formatAndMountTimeout, *maxConcurrentFormatAndMount)
232234
}

deploy/kubernetes/base/node_linux/node.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,8 @@ spec:
6666
mountPath: /run/udev
6767
- name: sys
6868
mountPath: /sys
69+
- name: lib-modules
70+
mountPath: /lib/modules
6971
volumes:
7072
- name: registration-dir
7173
hostPath:
@@ -101,6 +103,10 @@ spec:
101103
hostPath:
102104
path: /sys
103105
type: Directory
106+
- name: lib-modules
107+
hostPath:
108+
path: /lib/modules
109+
type: Directory
104110
# https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
105111
# See "special case". This will tolerate everything. Node component should
106112
# be scheduled on all nodes.

deploy/kubernetes/images/stable-master/image.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,6 @@ imageTag:
5151
name: gke.gcr.io/gcp-compute-persistent-disk-csi-driver
5252
# Don't change stable image without changing pdImagePlaceholder in
5353
# test/k8s-integration/main.go
54-
newName: registry.k8s.io/cloud-provider-gcp/gcp-compute-persistent-disk-csi-driver
55-
newTag: "v1.13.2"
54+
newName: gcr.io/songsunny-joonix/gcp-compute-persistent-disk-csi-driver
55+
newTag: "datacache"
5656
---

pkg/common/parameters.go

Lines changed: 48 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ package common
1919
import (
2020
"fmt"
2121
"strings"
22+
23+
"k8s.io/klog/v2"
2224
)
2325

2426
const (
@@ -32,7 +34,11 @@ const (
3234
ParameterAvailabilityClass = "availability-class"
3335
ParameterKeyEnableConfidentialCompute = "enable-confidential-storage"
3436
ParameterKeyStoragePools = "storage-pools"
35-
ParameterKeyResourceTags = "resource-tags"
37+
38+
// Parameters for Data Cache
39+
ParameterKeyDataCacheSize = "data-cache-size"
40+
ParameterKeyDataCacheMode = "data-cache-mode"
41+
ParameterKeyResourceTags = "resource-tags"
3642

3743
// Parameters for VolumeSnapshotClass
3844
ParameterKeyStorageLocations = "storage-locations"
@@ -68,6 +74,16 @@ const (
6874
tagKeyCreatedForSnapshotContentName = "kubernetes.io/created-for/volumesnapshotcontent/name"
6975
)
7076

77+
type DataCacheParameters struct {
78+
// Values: {string}
79+
// Default: ""
80+
// Example: "25Gi"
81+
DataCacheSize string
82+
// Values: writethrough, writeback
83+
// Default: writethrough
84+
DataCacheMode string
85+
}
86+
7187
// DiskParameters contains normalized and defaulted disk parameters
7288
type DiskParameters struct {
7389
// Values: pd-standard, pd-balanced, pd-ssd, or any other PD disk type. Not validated.
@@ -125,7 +141,7 @@ type StoragePool struct {
125141
// put them into a well defined struct making sure to default unspecified fields.
126142
// extraVolumeLabels are added as labels; if there are also labels specified in
127143
// parameters, any matching extraVolumeLabels will be overridden.
128-
func ExtractAndDefaultParameters(parameters map[string]string, driverName string, extraVolumeLabels map[string]string, enableStoragePools bool, extraTags map[string]string) (DiskParameters, error) {
144+
func ExtractAndDefaultParameters(parameters map[string]string, driverName string, extraVolumeLabels map[string]string, enableStoragePools bool, enableDataCache bool, extraTags map[string]string) (DiskParameters, DataCacheParameters, error) {
129145
p := DiskParameters{
130146
DiskType: "pd-standard", // Default
131147
ReplicationType: replicationTypeNone, // Default
@@ -135,6 +151,12 @@ func ExtractAndDefaultParameters(parameters map[string]string, driverName string
135151
ResourceTags: make(map[string]string), // Default
136152
}
137153

154+
// Set data cache feature default
155+
d := DataCacheParameters{}
156+
if enableDataCache {
157+
d.DataCacheMode = "writethrough"
158+
}
159+
138160
for k, v := range extraVolumeLabels {
139161
p.Labels[k] = v
140162
}
@@ -169,7 +191,7 @@ func ExtractAndDefaultParameters(parameters map[string]string, driverName string
169191
case ParameterKeyLabels:
170192
paramLabels, err := ConvertLabelsStringToMap(v)
171193
if err != nil {
172-
return p, fmt.Errorf("parameters contain invalid labels parameter: %w", err)
194+
return p, d, fmt.Errorf("parameters contain invalid labels parameter: %w", err)
173195
}
174196
// Override any existing labels with those from this parameter.
175197
for labelKey, labelValue := range paramLabels {
@@ -178,58 +200,71 @@ func ExtractAndDefaultParameters(parameters map[string]string, driverName string
178200
case ParameterKeyProvisionedIOPSOnCreate:
179201
paramProvisionedIOPSOnCreate, err := ConvertStringToInt64(v)
180202
if err != nil {
181-
return p, fmt.Errorf("parameters contain invalid provisionedIOPSOnCreate parameter: %w", err)
203+
return p, d, fmt.Errorf("parameters contain invalid provisionedIOPSOnCreate parameter: %w", err)
182204
}
183205
p.ProvisionedIOPSOnCreate = paramProvisionedIOPSOnCreate
184206
case ParameterKeyProvisionedThroughputOnCreate:
185207
paramProvisionedThroughputOnCreate, err := ConvertMiStringToInt64(v)
186208
if err != nil {
187-
return p, fmt.Errorf("parameters contain invalid provisionedThroughputOnCreate parameter: %w", err)
209+
return p, d, fmt.Errorf("parameters contain invalid provisionedThroughputOnCreate parameter: %w", err)
188210
}
189211
p.ProvisionedThroughputOnCreate = paramProvisionedThroughputOnCreate
190212
case ParameterAvailabilityClass:
191213
paramAvailabilityClass, err := ConvertStringToAvailabilityClass(v)
192214
if err != nil {
193-
return p, fmt.Errorf("parameters contain invalid availability class parameter: %w", err)
215+
return p, d, fmt.Errorf("parameters contain invalid availability class parameter: %w", err)
194216
}
195217
if paramAvailabilityClass == ParameterRegionalHardFailoverClass {
196218
p.ForceAttach = true
197219
}
198220
case ParameterKeyEnableConfidentialCompute:
199221
paramEnableConfidentialCompute, err := ConvertStringToBool(v)
200222
if err != nil {
201-
return p, fmt.Errorf("parameters contain invalid value for enable-confidential-storage parameter: %w", err)
223+
return p, d, fmt.Errorf("parameters contain invalid value for enable-confidential-storage parameter: %w", err)
202224
}
203225

204226
if paramEnableConfidentialCompute {
205227
// DiskEncryptionKmsKey is needed to enable confidentialStorage
206228
if val, ok := parameters[ParameterKeyDiskEncryptionKmsKey]; !ok || !isValidDiskEncryptionKmsKey(val) {
207-
return p, fmt.Errorf("Valid %v is required to enable ConfidentialStorage", ParameterKeyDiskEncryptionKmsKey)
229+
return p, d, fmt.Errorf("Valid %v is required to enable ConfidentialStorage", ParameterKeyDiskEncryptionKmsKey)
208230
}
209231
}
210232

211233
p.EnableConfidentialCompute = paramEnableConfidentialCompute
212234
case ParameterKeyStoragePools:
213235
if !enableStoragePools {
214-
return p, fmt.Errorf("parameters contains invalid option %q", ParameterKeyStoragePools)
236+
return p, d, fmt.Errorf("parameters contains invalid option %q", ParameterKeyStoragePools)
215237
}
216238
storagePools, err := ParseStoragePools(v)
217239
if err != nil {
218-
return p, fmt.Errorf("parameters contain invalid value for %s parameter: %w", ParameterKeyStoragePools, err)
240+
return p, d, fmt.Errorf("parameters contain invalid value for %s parameter: %w", ParameterKeyStoragePools, err)
219241
}
220242
p.StoragePools = storagePools
243+
case ParameterKeyDataCacheSize:
244+
if !enableDataCache {
245+
return p, d, fmt.Errorf("parameters contains invalid option %q", ParameterKeyDataCacheSize)
246+
}
247+
// TODO: need to parse or validate the string
248+
d.DataCacheSize = v
249+
klog.V(2).Infof("====== Data cache size is %v ======", v)
250+
case ParameterKeyDataCacheMode:
251+
if !enableDataCache {
252+
return p, d, fmt.Errorf("parameters contains invalid option %q", ParameterKeyDataCacheSize)
253+
}
254+
d.DataCacheMode = v
255+
klog.V(2).Infof("====== Data cache mode is %v ======", v)
221256
case ParameterKeyResourceTags:
222257
if err := extractResourceTagsParameter(v, p.ResourceTags); err != nil {
223-
return p, err
258+
return p, d, err
224259
}
225260
default:
226-
return p, fmt.Errorf("parameters contains invalid option %q", k)
261+
return p, d, fmt.Errorf("parameters contains invalid option %q", k)
227262
}
228263
}
229264
if len(p.Tags) > 0 {
230265
p.Tags[tagKeyCreatedBy] = driverName
231266
}
232-
return p, nil
267+
return p, d, nil
233268
}
234269

235270
func ExtractAndDefaultSnapshotParameters(parameters map[string]string, driverName string, extraTags map[string]string) (SnapshotParameters, error) {

pkg/common/parameters_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -354,7 +354,7 @@ func TestExtractAndDefaultParameters(t *testing.T) {
354354

355355
for _, tc := range tests {
356356
t.Run(tc.name, func(t *testing.T) {
357-
p, err := ExtractAndDefaultParameters(tc.parameters, "testDriver", tc.labels, tc.enableStoragePools, tc.extraTags)
357+
p, _, err := ExtractAndDefaultParameters(tc.parameters, "testDriver", tc.labels, tc.enableStoragePools, false, tc.extraTags)
358358
if gotErr := err != nil; gotErr != tc.expectErr {
359359
t.Fatalf("ExtractAndDefaultParameters(%+v) = %v; expectedErr: %v", tc.parameters, err, tc.expectErr)
360360
}

pkg/common/runcmd.go

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
package common
2+
3+
import (
4+
"fmt"
5+
"os/exec"
6+
"strings"
7+
/*
8+
"bufio"
9+
"io"
10+
"io/ioutil"
11+
*/
12+
13+
"k8s.io/klog/v2"
14+
)
15+
16+
const (
17+
// Error thrown by exec cmd.Run() when process spawned by cmd.Start() completes before cmd.Wait() is called (see - k/k issue #103753)
18+
errNoChildProcesses = "wait: no child processes"
19+
)
20+
21+
// RunCommand wraps a k8s exec to deal with the no child process error. Same as exec.CombinedOutput.
22+
// On error, the output is included so callers don't need to echo it again.
23+
func RunCommand(cmd string, args ...string) ([]byte, error) {
24+
klog.V(2).Infof("====== Start RunCommand ======")
25+
execCmd := exec.Command(cmd, args...)
26+
output, err := execCmd.CombinedOutput()
27+
if err != nil {
28+
if err.Error() == errNoChildProcesses {
29+
if execCmd.ProcessState.Success() {
30+
// If the process succeeded, this can be ignored, see k/k issue #103753
31+
return output, nil
32+
}
33+
// Get actual error
34+
err = &exec.ExitError{ProcessState: execCmd.ProcessState}
35+
}
36+
return output, fmt.Errorf("%s %s failed: %w; output: %s", cmd, strings.Join(args, " "), err, string(output))
37+
}
38+
return output, nil
39+
/*
40+
pipe, _ := execCmd.StdoutPipe()
41+
if err := execCmd.Start(); err != nil {
42+
klog.Errorf("====== Failed execCmd.Start() ======")
43+
}
44+
outStr := ""
45+
go func(p io.ReadCloser) {
46+
reader := bufio.NewReader(pipe)
47+
klog.V(2).Infof("====== Start ioutil.ReadAll ======")
48+
b, err := ioutil.ReadAll(reader)
49+
if err != nil {
50+
klog.Errorf("====== Failed ioutil.ReadAll(reader) %v ======", err)
51+
}
52+
klog.V(2).Infof("%v \n", string(b))
53+
klog.V(2).Infof("====== End ioutil.ReadAll ======")
54+
}(pipe)
55+
56+
if err := execCmd.Wait(); err != nil {
57+
klog.Errorf("====== Failed execCmd.Wait(): %v ======", err)
58+
}
59+
return []byte(outStr), nil*/
60+
}

0 commit comments

Comments
 (0)