Skip to content

Commit a808c6f

Browse files
andyzhangxandyzhangx
authored
Merge pull request #367 from boddumanohar/blobfuse-proxy-ds
install blobfuse proxy on the host using dameonset
2 parents 38d7e72 + 5652b7d commit a808c6f

File tree

8 files changed

+65
-51
lines changed

8 files changed

+65
-51
lines changed

‎deploy/blobfuse-proxy/blobfuse-proxy.yaml

Lines changed: 50 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
1-
# This is to be used in e2e test, while for production, we suggest install this proxy as a standalone process
21
---
3-
kind: DaemonSet
42
apiVersion: apps/v1
3+
kind: DaemonSet
54
metadata:
65
name: csi-blobfuse-proxy
76
namespace: kube-system
@@ -14,10 +13,6 @@ spec:
1413
labels:
1514
app: csi-blobfuse-proxy
1615
spec:
17-
hostNetwork: true
18-
dnsPolicy: ClusterFirstWithHostNet
19-
nodeSelector:
20-
kubernetes.io/os: linux
2116
affinity:
2217
nodeAffinity:
2318
requiredDuringSchedulingIgnoredDuringExecution:
@@ -27,45 +22,54 @@ spec:
2722
operator: NotIn
2823
values:
2924
- virtual-kubelet
25+
containers:
26+
- command:
27+
- nsenter
28+
- '--target'
29+
- '1'
30+
- '--mount'
31+
- '--uts'
32+
- '--ipc'
33+
- '--net'
34+
- '--pid'
35+
- '--'
36+
- sh
37+
- '-c'
38+
- |
39+
set -xe
40+
apt update
41+
# update blobfuse version on the host to 1.3.6
42+
wget https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb
43+
dpkg -i packages-microsoft-prod.deb
44+
apt-get update
45+
apt-get install -y blobfuse=1.3.6
46+
# download blobfuse-proxy .deb package
47+
wget https://github.com/boddumanohar/blob-csi-driver/raw/blobfuse-proxy-ds/deploy/blobfuse-proxy/v0.1.0/blobfuse-proxy-v0.1.0.deb -O /tmp/blobfuse-proxy-v0.1.0.deb
48+
dpkg -i /tmp/blobfuse-proxy-v0.1.0.deb
49+
mkdir -p /var/lib/kubelet/plugins/blob.csi.azure.com
50+
echo "Enabling blobfuse proxy systemctl service"
51+
systemctl daemon-reload
52+
systemctl enable blobfuse-proxy
53+
systemctl stop blobfuse-proxy
54+
systemctl start blobfuse-proxy
55+
echo "removing the file: /tmp/blobfuse-proxy-v0.1.0.deb"
56+
rm /tmp/blobfuse-proxy-v0.1.0.deb
57+
echo "sleeping for 19 years"
58+
sleep 600000000s
59+
image: alpine
60+
imagePullPolicy: IfNotPresent
61+
name: sysctl-install-blobfuse-proxy
62+
resources:
63+
requests:
64+
cpu: 10m
65+
securityContext:
66+
privileged: true
67+
dnsPolicy: ClusterFirstWithHostNet
68+
hostNetwork: true
69+
hostPID: true
70+
nodeSelector:
71+
kubernetes.io/os: linux
3072
priorityClassName: system-node-critical
73+
restartPolicy: Always
3174
tolerations:
32-
- operator: "Exists"
33-
containers:
34-
- name: proxy
35-
image: bmanu199/blobfuse-proxy@sha256:65659c9effa9df94c42a5f8388cef9656343c8fdd47a200ab69dc53913af7a42
36-
imagePullPolicy: IfNotPresent
37-
args:
38-
- "--blobfuse-proxy-endpoint=$(BLOBFUSE_PROXY_ENDPOINT)"
39-
env:
40-
- name: BLOBFUSE_PROXY_ENDPOINT
41-
value: unix://csi/blobfuse-proxy.sock
42-
securityContext:
43-
privileged: true
44-
volumeMounts:
45-
- mountPath: /csi
46-
name: socket-dir
47-
- mountPath: /var/lib/kubelet/
48-
mountPropagation: Bidirectional
49-
name: mountpoint-dir
50-
- mountPath: /mnt
51-
name: blob-cache
52-
resources:
53-
limits:
54-
cpu: 2
55-
memory: 2100Mi
56-
requests:
57-
cpu: 10m
58-
memory: 20Mi
59-
volumes:
60-
- hostPath:
61-
path: /var/lib/kubelet/plugins/blob.csi.azure.com
62-
type: DirectoryOrCreate
63-
name: socket-dir
64-
- hostPath:
65-
path: /var/lib/kubelet/
66-
type: DirectoryOrCreate
67-
name: mountpoint-dir
68-
- hostPath:
69-
path: /mnt
70-
name: blob-cache
71-
---
75+
- operator: Exists

‎deploy/blobfuse-proxy/v0.1.0/blobfuse-proxy-v0.1.0.deb

1.27 KB
Binary file not shown.

‎deploy/blobfuse-proxy/v0.1/blobfuse-proxy-v0.1.deb

-2.38 MB
Binary file not shown.

‎pkg/blob/nodeserver.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -318,7 +318,7 @@ func (d *Driver) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolu
318318
klog.V(2).Infof("NodeUnstageVolume: volume %s unmounting on %s", volumeID, stagingTargetPath)
319319
err := mount.CleanupMountPoint(stagingTargetPath, d.mounter, false)
320320
if err != nil {
321-
return nil, status.Errorf(codes.Internal, "failed to unmount staing target %q: %v", stagingTargetPath, err)
321+
return nil, status.Errorf(codes.Internal, "failed to unmount staging target %q: %v", stagingTargetPath, err)
322322
}
323323
klog.V(2).Infof("NodeUnstageVolume: volume %s unmount on %s successfully", volumeID, stagingTargetPath)
324324

‎pkg/blobfuse-proxy/debpackage/etc/systemd/system/blobfuse-proxy.service

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
Description=Blobfuse proxy service
33

44
[Service]
5-
ExecStart=/usr/bin/blobfuse-proxy -blobfuse-proxy-endpoint=unix://var/lib/kubelet/blobfuse-proxy.sock
5+
ExecStart=/usr/bin/blobfuse-proxy --v=5 --blobfuse-proxy-endpoint=unix://var/lib/kubelet/plugins/blob.csi.azure.com/blobfuse-proxy.sock
66

77
[Install]
88
WantedBy=multi-user.target

‎pkg/blobfuse-proxy/main.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ package main
1818

1919
import (
2020
"flag"
21-
"log"
2221
"net"
2322
"os"
2423

@@ -37,6 +36,7 @@ var (
3736
)
3837

3938
func main() {
39+
klog.InitFlags(nil)
4040
flag.Parse()
4141
proto, addr, err := csicommon.ParseEndpoint(*blobfuseProxyEndpoint)
4242
if err != nil {
@@ -57,8 +57,8 @@ func main() {
5757

5858
mountServer := server.NewMountServiceServer()
5959

60-
log.Printf("Listening for connections on address: %#v\n", listener.Addr())
60+
klog.V(2).Info("Listening for connections on address: %v\n", listener.Addr())
6161
if err = server.RunGRPCServer(mountServer, false, listener); err != nil {
62-
klog.Fatalf("Listening for connections on address: %#v, error: %v", listener.Addr(), err)
62+
klog.Fatalf("Error running grpc server. Error: %v", listener.Addr(), err)
6363
}
6464
}

‎pkg/blobfuse-proxy/server/server.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,7 @@ func (server *MountServer) MountAzureBlob(ctx context.Context,
6363
klog.V(2).Infof("successfully mounted")
6464
}
6565
result.Output = string(output)
66+
klog.V(2).Infof("blobfuse output: %s\n", result.Output)
6667
return &result, err
6768
}
6869

‎test/utils/blob_log.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,3 +49,12 @@ echo "print out cloudprovider_azure metrics ..."
4949
echo "======================================================================================"
5050
ip=`kubectl get svc csi-blob-controller -n kube-system | grep blob | awk '{print $4}'`
5151
curl http://$ip:29634/metrics
52+
53+
54+
echo "print out sysctl-install-blobfuseproxy logs ..."
55+
echo "======================================================================================"
56+
LABEL='app=csi-blobfuse-proxy'
57+
PROXY=sysctl-install-blobfuse-proxy
58+
kubectl get pods -n${NS} -l${LABEL} \
59+
| awk 'NR>1 {print $1}' \
60+
| xargs -I {} kubectl logs {} --prefix -c${PROXY} -n${NS}

0 commit comments

Comments
 (0)