Skip to content

Commit 3317389

Browse files
cvvzcvvz
authored
feat: support aznfs mount helper (#987)
* Squashed commit of the following: commit 408353f Merge: d57e587 3465228 Author: weizhichen <[email protected]> Date: Thu Aug 17 09:55:54 2023 +0000 Merge branch 'master' of https://github.com/kubernetes-sigs/blob-csi-driver into support-aznfs commit d57e587 Author: weizhichen <[email protected]> Date: Thu Aug 17 09:23:31 2023 +0000 fix commit 3148e86 Author: weizhichen <[email protected]> Date: Thu Aug 17 09:18:54 2023 +0000 fix commit 8036bf2 Author: weizhichen <[email protected]> Date: Thu Aug 17 09:13:14 2023 +0000 Revert "support arm64" This reverts commit b5c9bfd. commit b3f9a6c Author: weizhichen <[email protected]> Date: Wed Aug 16 15:06:48 2023 +0000 fix commit b5c9bfd Author: weizhichen <[email protected]> Date: Wed Aug 16 03:33:38 2023 +0000 support arm64 commit 917c1f6 Author: weizhichen <[email protected]> Date: Wed Aug 9 09:10:27 2023 +0000 official release commit 783cdaf Author: weizhichen <[email protected]> Date: Fri Aug 4 04:08:12 2023 +0000 fix commit 9c1d08a Author: weizhichen <[email protected]> Date: Fri Aug 4 03:07:13 2023 +0000 fix commit c8e5015 Author: weizhichen <[email protected]> Date: Thu Aug 3 08:33:16 2023 +0000 fix commit 9747128 Merge: 57a1407 cc884d5 Author: weizhichen <[email protected]> Date: Thu Aug 3 08:31:48 2023 +0000 Merge branch 'master' of https://github.com/kubernetes-sigs/blob-csi-driver into support-aznfs commit 57a1407 Author: weizhichen <[email protected]> Date: Thu Aug 3 08:05:38 2023 +0000 fix commit 8f262b2 Author: weizhichen <[email protected]> Date: Wed Aug 2 08:07:54 2023 +0000 fix commit 78808c3 Author: weizhichen <[email protected]> Date: Wed Aug 2 08:06:04 2023 +0000 fix commit 833dde7 Author: weizhichen <[email protected]> Date: Wed Aug 2 07:56:17 2023 +0000 fix commit d6d6de2 Author: weizhichen <[email protected]> Date: Wed Aug 2 02:39:04 2023 +0000 POC * fix by review * helm * fix * helm * fix log panic * fix * fix * fix
1 parent d406a1f commit 3317389

File tree

11 files changed

+148
-84
lines changed

11 files changed

+148
-84
lines changed

‎Makefile

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ blob: blobfuse-proxy
111111
blob-windows:
112112
CGO_ENABLED=0 GOOS=windows go build -a -ldflags ${LDFLAGS} -mod vendor -o _output/blobplugin.exe ./pkg/blobplugin
113113

114-
.PHONT: blob-darwin
114+
.PHONY: blob-darwin
115115
blob-darwin:
116116
CGO_ENABLED=0 GOOS=darwin go build -a -ldflags ${LDFLAGS} -mod vendor -o _output/blobplugin ./pkg/blobplugin
117117

@@ -182,4 +182,4 @@ delete-metrics-svc:
182182

183183
.PHONY: blobfuse-proxy
184184
blobfuse-proxy:
185-
CGO_ENABLED=0 GOOS=linux go build -mod vendor -ldflags="-s -w" -o _output/${ARCH}/blobfuse-proxy ./pkg/blobfuse-proxy
185+
CGO_ENABLED=0 GOOS=linux GOARCH=$(ARCH) go build -mod vendor -ldflags="-s -w" -o _output/${ARCH}/blobfuse-proxy ./pkg/blobfuse-proxy

‎charts/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@ The following table lists the configurable parameters of the latest Azure Blob S
148148
| `linux.distro` | configure ssl certificates for different Linux distribution(available values: `debian`, `fedora`) | `debian`
149149
| `workloadIdentity.clientID` | client ID of workload identity | ''
150150
| `workloadIdentity.tenantID` | [optional] If the AAD application or user-assigned managed identity is not in the same tenant as the cluster then set tenantID with the AAD application or user-assigned managed identity tenant ID | ''
151+
| `node.enableAznfsMount` | enable [AZNFS mount helper](https://github.com/Azure/AZNFS-mount/) for NFS protocol | true
151152

152153
## troubleshooting
153154
- Add `--wait -v=5 --debug` in `helm install` command to get detailed error

‎charts/latest/blob-csi-driver-v0.0.0.tgz

146 Bytes
Binary file not shown.

‎charts/latest/blob-csi-driver/templates/csi-blob-node.yaml

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,9 +35,9 @@ spec:
3535
imagePullSecrets:
3636
{{ toYaml .Values.imagePullSecrets | indent 8 }}
3737
{{- end }}
38-
{{- if .Values.node.enableBlobfuseProxy }}
38+
{{- if or .Values.node.enableBlobfuseProxy .Values.node.enableAznfsMount }}
3939
hostPID: true
40-
{{- end }}
40+
{{- end }}
4141
hostNetwork: true
4242
dnsPolicy: Default
4343
serviceAccountName: {{ .Values.serviceAccount.node }}
@@ -170,6 +170,7 @@ spec:
170170
- "--append-timestamp-cache-dir={{ .Values.node.appendTimeStampInCacheDir }}"
171171
- "--mount-permissions={{ .Values.node.mountPermissions }}"
172172
- "--allow-inline-volume-key-access-with-idenitity={{ .Values.node.allowInlineVolumeKeyAccessWithIdentity }}"
173+
- "--enable-aznfs-mount={{ .Values.node.enableAznfsMount }}"
173174
ports:
174175
- containerPort: {{ .Values.node.livenessProbe.healthPort }}
175176
name: healthz
@@ -238,7 +239,31 @@ spec:
238239
mountPath: /etc/pki/ca-trust/extracted
239240
readOnly: true
240241
{{- end }}
242+
{{- if .Values.node.enableAznfsMount }}
243+
- mountPath: /opt/microsoft/aznfs/data
244+
name: aznfs-data
245+
{{- end }}
241246
resources: {{- toYaml .Values.node.resources.blob | nindent 12 }}
247+
{{- if .Values.node.enableAznfsMount }}
248+
- name: aznfswatchdog
249+
{{- if hasPrefix "/" .Values.image.blob.repository }}
250+
image: "{{ .Values.image.baseRepo }}{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}"
251+
{{- else }}
252+
image: "{{ .Values.image.blob.repository }}:{{ .Values.image.blob.tag }}"
253+
{{- end }}
254+
command:
255+
- "aznfswatchdog"
256+
imagePullPolicy: {{ .Values.image.blob.pullPolicy }}
257+
securityContext:
258+
privileged: true
259+
resources: {{- toYaml .Values.node.resources.aznfswatchdog | nindent 12 }}
260+
volumeMounts:
261+
- mountPath: /opt/microsoft/aznfs/data
262+
name: aznfs-data
263+
- mountPath: {{ .Values.linux.kubelet }}/
264+
mountPropagation: Bidirectional
265+
name: mountpoint-dir
266+
{{- end }}
242267
volumes:
243268
{{- if .Values.node.enableBlobfuseProxy }}
244269
- name: host-usr
@@ -280,6 +305,12 @@ spec:
280305
hostPath:
281306
path: /etc/pki/ca-trust/extracted
282307
{{- end }}
308+
{{- if .Values.node.enableAznfsMount }}
309+
- hostPath:
310+
path: /opt/microsoft/aznfs/data
311+
type: DirectoryOrCreate
312+
name: aznfs-data
313+
{{- end }}
283314
{{- if .Values.securityContext }}
284315
securityContext: {{- toYaml .Values.securityContext | nindent 8 }}
285316
{{- end }}

‎charts/latest/blob-csi-driver/values.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,10 +145,17 @@ node:
145145
requests:
146146
cpu: 10m
147147
memory: 20Mi
148+
aznfswatchdog:
149+
limits:
150+
memory: 100Mi
151+
requests:
152+
cpu: 10m
153+
memory: 20Mi
148154
affinity: {}
149155
nodeSelector: {}
150156
tolerations:
151157
- operator: "Exists"
158+
enableAznfsMount: true
152159

153160
feature:
154161
fsGroupPolicy: ReadWriteOnceWithFSType

‎pkg/blob/blob.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,7 @@ const (
104104
Fuse = "fuse"
105105
Fuse2 = "fuse2"
106106
NFS = "nfs"
107+
AZNFS = "aznfs"
107108
vnetResourceGroupField = "vnetresourcegroup"
108109
vnetNameField = "vnetname"
109110
subnetNameField = "subnetname"
@@ -168,6 +169,7 @@ type DriverOptions struct {
168169
MountPermissions uint64
169170
KubeAPIQPS float64
170171
KubeAPIBurst int
172+
EnableAznfsMount bool
171173
}
172174

173175
// Driver implements all interfaces of CSI drivers
@@ -192,6 +194,7 @@ type Driver struct {
192194
mountPermissions uint64
193195
kubeAPIQPS float64
194196
kubeAPIBurst int
197+
enableAznfsMount bool
195198
mounter *mount.SafeFormatAndMount
196199
volLockMap *util.LockMap
197200
// A map storing all volumes with ongoing operations so that additional operations
@@ -229,6 +232,7 @@ func NewDriver(options *DriverOptions) *Driver {
229232
mountPermissions: options.MountPermissions,
230233
kubeAPIQPS: options.KubeAPIQPS,
231234
kubeAPIBurst: options.KubeAPIBurst,
235+
enableAznfsMount: options.EnableAznfsMount,
232236
}
233237
d.Name = options.DriverName
234238
d.Version = driverVersion

‎pkg/blob/nodeserver.go

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -314,10 +314,15 @@ func (d *Driver) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRe
314314
klog.V(2).Infof("target %v\nprotocol %v\n\nvolumeId %v\ncontext %v\nmountflags %v\nserverAddress %v",
315315
targetPath, protocol, volumeID, attrib, mountFlags, serverAddress)
316316

317+
mountType := AZNFS
318+
if !d.enableAznfsMount {
319+
mountType = NFS
320+
}
321+
317322
source := fmt.Sprintf("%s:/%s/%s", serverAddress, accountName, containerName)
318323
mountOptions := util.JoinMountOptions(mountFlags, []string{"sec=sys,vers=3,nolock"})
319324
if err := wait.PollImmediate(1*time.Second, 2*time.Minute, func() (bool, error) {
320-
return true, d.mounter.MountSensitive(source, targetPath, NFS, mountOptions, []string{})
325+
return true, d.mounter.MountSensitive(source, targetPath, mountType, mountOptions, []string{})
321326
}); err != nil {
322327
var helpLinkMsg string
323328
if d.appendMountErrorHelpLink {

‎pkg/blobfuse-proxy/init.sh

Lines changed: 81 additions & 76 deletions
Original file line numberDiff line numberDiff line change
@@ -26,99 +26,104 @@ HOST_CMD="nsenter --mount=/proc/1/ns/mnt"
2626

2727
DISTRIBUTION=$($HOST_CMD cat /etc/os-release | grep ^ID= | cut -d'=' -f2 | tr -d '"')
2828
echo "Linux distribution: $DISTRIBUTION"
29+
ARCH=$($HOST_CMD uname -m)
30+
echo "Linux Arch is $(uname -m)"
2931

30-
if [ "${DISTRIBUTION}" = "ubuntu" ] && { [ "${INSTALL_BLOBFUSE}" = "true" ] || [ "${INSTALL_BLOBFUSE2}" = "true" ]; }
32+
if [ "${ARCH}" != "aarch64" ]
3133
then
32-
release=$($HOST_CMD lsb_release -rs)
33-
echo "Ubuntu release: $release"
34-
35-
if [ "$(expr "$release" \< "22.04")" -eq 1 ]
34+
if [ "${DISTRIBUTION}" = "ubuntu" ] && { [ "${INSTALL_BLOBFUSE}" = "true" ] || [ "${INSTALL_BLOBFUSE2}" = "true" ]; }
3635
then
37-
cp /blobfuse-proxy/packages-microsoft-prod-18.04.deb /host/etc/packages-microsoft-prod.deb
38-
else
39-
cp /blobfuse-proxy/packages-microsoft-prod-22.04.deb /host/etc/packages-microsoft-prod.deb
40-
fi
41-
42-
# when running dpkg -i /etc/packages-microsoft-prod.deb, need to enter y to continue.
43-
# refer to https://stackoverflow.com/questions/45349571/how-to-install-deb-with-dpkg-non-interactively
44-
yes | $HOST_CMD dpkg -i /etc/packages-microsoft-prod.deb && $HOST_CMD apt update
45-
46-
pkg_list=""
47-
if [ "${INSTALL_BLOBFUSE}" = "true" ] && [ "$(expr "$release" \< "22.04")" -eq 1 ]
48-
then
49-
pkg_list="${pkg_list} fuse"
50-
# install blobfuse with latest version or specific version
51-
if [ -z "${BLOBFUSE_VERSION}" ]; then
52-
echo "install blobfuse with latest version"
53-
pkg_list="${pkg_list} blobfuse"
36+
release=$($HOST_CMD lsb_release -rs)
37+
echo "Ubuntu release: $release"
38+
39+
if [ "$(expr "$release" \< "22.04")" -eq 1 ]
40+
then
41+
cp /blobfuse-proxy/packages-microsoft-prod-18.04.deb /host/etc/packages-microsoft-prod.deb
5442
else
55-
pkg_list="${pkg_list} blobfuse=${BLOBFUSE_VERSION}"
43+
cp /blobfuse-proxy/packages-microsoft-prod-22.04.deb /host/etc/packages-microsoft-prod.deb
5644
fi
57-
fi
58-
59-
if [ "${INSTALL_BLOBFUSE2}" = "true" ]
60-
then
61-
if [ "$(expr "$release" \< "22.04")" -eq 1 ]; then
62-
echo "install fuse for blobfuse2"
45+
46+
# when running dpkg -i /etc/packages-microsoft-prod.deb, need to enter y to continue.
47+
# refer to https://stackoverflow.com/questions/45349571/how-to-install-deb-with-dpkg-non-interactively
48+
yes | $HOST_CMD dpkg -i /etc/packages-microsoft-prod.deb && $HOST_CMD apt update
49+
50+
pkg_list=""
51+
if [ "${INSTALL_BLOBFUSE}" = "true" ] && [ "$(expr "$release" \< "22.04")" -eq 1 ]
52+
then
6353
pkg_list="${pkg_list} fuse"
64-
else
65-
echo "install fuse3 for blobfuse2, current release is $release"
66-
pkg_list="${pkg_list} fuse3"
54+
# install blobfuse with latest version or specific version
55+
if [ -z "${BLOBFUSE_VERSION}" ]; then
56+
echo "install blobfuse with latest version"
57+
pkg_list="${pkg_list} blobfuse"
58+
else
59+
pkg_list="${pkg_list} blobfuse=${BLOBFUSE_VERSION}"
60+
fi
6761
fi
6862

69-
# install blobfuse2 with latest version or specific version
70-
if [ -z "${BLOBFUSE2_VERSION}" ]; then
71-
echo "install blobfuse2 with latest version"
72-
pkg_list="${pkg_list} blobfuse2"
73-
else
74-
pkg_list="${pkg_list} blobfuse2=${BLOBFUSE2_VERSION}"
63+
if [ "${INSTALL_BLOBFUSE2}" = "true" ]
64+
then
65+
if [ "$(expr "$release" \< "22.04")" -eq 1 ]; then
66+
echo "install fuse for blobfuse2"
67+
pkg_list="${pkg_list} fuse"
68+
else
69+
echo "install fuse3 for blobfuse2, current release is $release"
70+
pkg_list="${pkg_list} fuse3"
71+
fi
72+
73+
# install blobfuse2 with latest version or specific version
74+
if [ -z "${BLOBFUSE2_VERSION}" ]; then
75+
echo "install blobfuse2 with latest version"
76+
pkg_list="${pkg_list} blobfuse2"
77+
else
78+
pkg_list="${pkg_list} blobfuse2=${BLOBFUSE2_VERSION}"
79+
fi
7580
fi
81+
echo "begin to install ${pkg_list}"
82+
$HOST_CMD apt-get install -y $pkg_list
83+
$HOST_CMD rm -f /etc/packages-microsoft-prod.deb
7684
fi
77-
echo "begin to install ${pkg_list}"
78-
$HOST_CMD apt-get install -y $pkg_list
79-
$HOST_CMD rm -f /etc/packages-microsoft-prod.deb
80-
fi
8185

82-
updateBlobfuseProxy="true"
83-
if [ -f "/host/usr/bin/blobfuse-proxy" ];then
84-
old=$(sha256sum /host/usr/bin/blobfuse-proxy | awk '{print $1}')
85-
new=$(sha256sum /blobfuse-proxy/blobfuse-proxy | awk '{print $1}')
86-
if [ "$old" = "$new" ];then
87-
updateBlobfuseProxy="false"
88-
echo "no need to update blobfuse-proxy"
86+
updateBlobfuseProxy="true"
87+
if [ -f "/host/usr/bin/blobfuse-proxy" ];then
88+
old=$(sha256sum /host/usr/bin/blobfuse-proxy | awk '{print $1}')
89+
new=$(sha256sum /blobfuse-proxy/blobfuse-proxy | awk '{print $1}')
90+
if [ "$old" = "$new" ];then
91+
updateBlobfuseProxy="false"
92+
echo "no need to update blobfuse-proxy"
93+
fi
8994
fi
90-
fi
9195

92-
if [ "$updateBlobfuseProxy" = "true" ];then
93-
echo "copy blobfuse-proxy...."
94-
rm -rf /host/var/lib/kubelet/plugins/blob.csi.azure.com/blobfuse-proxy.sock
95-
rm -rf /host/usr/bin/blobfuse-proxy
96-
cp /blobfuse-proxy/blobfuse-proxy /host/usr/bin/blobfuse-proxy
97-
chmod 755 /host/usr/bin/blobfuse-proxy
98-
fi
96+
if [ "$updateBlobfuseProxy" = "true" ];then
97+
echo "copy blobfuse-proxy...."
98+
rm -rf /host/var/lib/kubelet/plugins/blob.csi.azure.com/blobfuse-proxy.sock
99+
rm -rf /host/usr/bin/blobfuse-proxy
100+
cp /blobfuse-proxy/blobfuse-proxy /host/usr/bin/blobfuse-proxy
101+
chmod 755 /host/usr/bin/blobfuse-proxy
102+
fi
99103

100-
updateService="true"
101-
if [ -f "/host/usr/lib/systemd/system/blobfuse-proxy.service" ];then
102-
old=$(sha256sum /host/usr/lib/systemd/system/blobfuse-proxy.service | awk '{print $1}')
103-
new=$(sha256sum /blobfuse-proxy/blobfuse-proxy.service | awk '{print $1}')
104-
if [ "$old" = "$new" ];then
105-
updateService="false"
106-
echo "no need to update blobfuse-proxy.service"
104+
updateService="true"
105+
if [ -f "/host/usr/lib/systemd/system/blobfuse-proxy.service" ];then
106+
old=$(sha256sum /host/usr/lib/systemd/system/blobfuse-proxy.service | awk '{print $1}')
107+
new=$(sha256sum /blobfuse-proxy/blobfuse-proxy.service | awk '{print $1}')
108+
if [ "$old" = "$new" ];then
109+
updateService="false"
110+
echo "no need to update blobfuse-proxy.service"
111+
fi
107112
fi
108-
fi
109113

110-
if [ "$updateService" = "true" ];then
111-
echo "copy blobfuse-proxy.service...."
112-
mkdir -p /host/usr/lib/systemd/system
113-
cp /blobfuse-proxy/blobfuse-proxy.service /host/usr/lib/systemd/system/blobfuse-proxy.service
114-
fi
114+
if [ "$updateService" = "true" ];then
115+
echo "copy blobfuse-proxy.service...."
116+
mkdir -p /host/usr/lib/systemd/system
117+
cp /blobfuse-proxy/blobfuse-proxy.service /host/usr/lib/systemd/system/blobfuse-proxy.service
118+
fi
115119

116-
if [ "${INSTALL_BLOBFUSE_PROXY}" = "true" ];then
117-
if [ "$updateBlobfuseProxy" = "true" ] || [ "$updateService" = "true" ];then
118-
echo "start blobfuse-proxy...."
119-
$HOST_CMD systemctl daemon-reload
120-
$HOST_CMD systemctl enable blobfuse-proxy.service
121-
$HOST_CMD systemctl restart blobfuse-proxy.service
120+
if [ "${INSTALL_BLOBFUSE_PROXY}" = "true" ];then
121+
if [ "$updateBlobfuseProxy" = "true" ] || [ "$updateService" = "true" ];then
122+
echo "start blobfuse-proxy...."
123+
$HOST_CMD systemctl daemon-reload
124+
$HOST_CMD systemctl enable blobfuse-proxy.service
125+
$HOST_CMD systemctl restart blobfuse-proxy.service
126+
fi
122127
fi
123128
fi
124129

‎pkg/blobplugin/Dockerfile

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,16 @@ RUN chmod +x /blobfuse-proxy/init.sh && \
2828
chmod +x /blobfuse-proxy/blobfuse-proxy.service && \
2929
chmod +x /blobfuse-proxy/blobfuse-proxy
3030

31-
RUN apt update && apt upgrade -y && apt-mark unhold libcap2 && clean-install ca-certificates uuid-dev util-linux mount udev wget e2fsprogs nfs-common netbase
31+
# packages that are only needed by aznfs: procps conntrack iptables bind9-host iproute2 bash netcat sysvinit-utils.
32+
RUN apt update && apt upgrade -y && apt-mark unhold libcap2 && clean-install ca-certificates uuid-dev util-linux mount udev wget e2fsprogs nfs-common netbase procps conntrack iptables bind9-host iproute2 bash netcat sysvinit-utils
33+
34+
# install aznfs
35+
RUN if [ "$ARCH" = "amd64" ] ; then \
36+
wget -O aznfs.tar.gz https://github.com/Azure/AZNFS-mount/releases/download/1.0.8/aznfs-1.0.8-1.x86_64.tar.gz; \
37+
else \
38+
wget -O aznfs.tar.gz https://github.com/Azure/AZNFS-mount/releases/download/1.0.8/aznfs-1.0.8-1.arm64.tar.gz;fi
39+
RUN tar xvzf aznfs.tar.gz -C / && rm aznfs.tar.gz
3240

33-
ARG ARCH=amd64
3441
RUN if [ "$ARCH" = "amd64" ] ; then \
3542
clean-install libcurl4-gnutls-dev && \
3643
wget -O /blobfuse-proxy/packages-microsoft-prod-22.04.deb https://packages.microsoft.com/config/ubuntu/22.04/packages-microsoft-prod.deb && \

‎pkg/blobplugin/main.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ var (
5454
kubeAPIQPS = flag.Float64("kube-api-qps", 25.0, "QPS to use while communicating with the kubernetes apiserver.")
5555
kubeAPIBurst = flag.Int("kube-api-burst", 50, "Burst to use while communicating with the kubernetes apiserver.")
5656
appendMountErrorHelpLink = flag.Bool("append-mount-error-help-link", true, "Whether to include a link for help with mount errors when a mount error occurs.")
57+
enableAznfsMount = flag.Bool("enable-aznfs-mount", true, "replace nfs mount with aznfs mount")
5758
)
5859

5960
func main() {
@@ -94,6 +95,7 @@ func handle() {
9495
AppendMountErrorHelpLink: *appendMountErrorHelpLink,
9596
KubeAPIQPS: *kubeAPIQPS,
9697
KubeAPIBurst: *kubeAPIBurst,
98+
EnableAznfsMount: *enableAznfsMount,
9799
}
98100
driver := blob.NewDriver(&driverOptions)
99101
if driver == nil {

0 commit comments

Comments
 (0)