Merge pull request #182 from xing-yang/new_beta_split_controller

Split snapshot controller using beta APIs

Author: k8s-ci-robot

Makefile

@@ -12,8 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-.PHONY: all csi-snapshotter clean test
+.PHONY: all snapshot-controller csi-snapshotter clean test
 
-CMDS=csi-snapshotter
+CMDS=snapshot-controller csi-snapshotter
 all: build
 include release-tools/build.make

Dockerfile renamed to cmd/csi-snapshotter/Dockerfile

@@ -1,6 +1,6 @@
 FROM gcr.io/distroless/static:latest
 LABEL maintainers="Kubernetes Authors"
-LABEL description="CSI External Snapshotter"
+LABEL description="CSI External Snapshotter Sidecar"
 
 COPY ./bin/csi-snapshotter csi-snapshotter
 ENTRYPOINT ["/csi-snapshotter"]

cmd/csi-snapshotter/main.go

@@ -37,7 +37,7 @@ import (
 	"github.com/kubernetes-csi/csi-lib-utils/connection"
 	"github.com/kubernetes-csi/csi-lib-utils/leaderelection"
 	csirpc "github.com/kubernetes-csi/csi-lib-utils/rpc"
-	"github.com/kubernetes-csi/external-snapshotter/pkg/controller"
+	controller "github.com/kubernetes-csi/external-snapshotter/pkg/sidecar-controller"
 	"github.com/kubernetes-csi/external-snapshotter/pkg/snapshotter"
 
 	clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned"
@@ -56,17 +56,13 @@ const (
 
 // Command line flags
 var (
-	snapshotterName                 = flag.String("snapshotter", "", "This option is deprecated.")
-	kubeconfig                      = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Required only when running out of cluster.")
-	connectionTimeout               = flag.Duration("connection-timeout", 0, "The --connection-timeout flag is deprecated")
-	csiAddress                      = flag.String("csi-address", "/run/csi/socket", "Address of the CSI driver socket.")
-	createSnapshotContentRetryCount = flag.Int("create-snapshotcontent-retrycount", 5, "Number of retries when we create a snapshot content object for a snapshot.")
-	createSnapshotContentInterval   = flag.Duration("create-snapshotcontent-interval", 10*time.Second, "Interval between retries when we create a snapshot content object for a snapshot.")
-	resyncPeriod                    = flag.Duration("resync-period", 60*time.Second, "Resync interval of the controller.")
-	snapshotNamePrefix              = flag.String("snapshot-name-prefix", "snapshot", "Prefix to apply to the name of a created snapshot")
-	snapshotNameUUIDLength          = flag.Int("snapshot-name-uuid-length", -1, "Length in characters for the generated uuid of a created snapshot. Defaults behavior is to NOT truncate.")
-	showVersion                     = flag.Bool("version", false, "Show version.")
-	csiTimeout                      = flag.Duration("timeout", defaultCSITimeout, "The timeout for any RPCs to the CSI driver. Default is 1 minute.")
+	kubeconfig             = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Required only when running out of cluster.")
+	csiAddress             = flag.String("csi-address", "/run/csi/socket", "Address of the CSI driver socket.")
+	resyncPeriod           = flag.Duration("resync-period", 60*time.Second, "Resync interval of the controller.")
+	snapshotNamePrefix     = flag.String("snapshot-name-prefix", "snapshot", "Prefix to apply to the name of a created snapshot")
+	snapshotNameUUIDLength = flag.Int("snapshot-name-uuid-length", -1, "Length in characters for the generated uuid of a created snapshot. Defaults behavior is to NOT truncate.")
+	showVersion            = flag.Bool("version", false, "Show version.")
+	csiTimeout             = flag.Duration("timeout", defaultCSITimeout, "The timeout for any RPCs to the CSI driver. Default is 1 minute.")
 
 	leaderElection          = flag.Bool("leader-election", false, "Enables leader election.")
 	leaderElectionNamespace = flag.String("leader-election-namespace", "", "The namespace where the leader election resource exists. Defaults to the pod namespace if not set.")
@@ -88,14 +84,6 @@ func main() {
 	}
 	klog.Infof("Version: %s", version)
 
-	if *connectionTimeout != 0 {
-		klog.Warning("--connection-timeout is deprecated and will have no effect")
-	}
-
-	if *snapshotterName != "" {
-		klog.Warning("--snapshotter is deprecated and will have no effect")
-	}
-
 	// Create the client config. Use kubeconfig if given, otherwise assume in-cluster.
 	config, err := buildConfig(*kubeconfig)
 	if err != nil {
@@ -133,13 +121,13 @@ func main() {
 	defer cancel()
 
 	// Find driver name
-	*snapshotterName, err = csirpc.GetDriverName(ctx, csiConn)
+	driverName, err := csirpc.GetDriverName(ctx, csiConn)
 	if err != nil {
 		klog.Errorf("error getting CSI driver name: %v", err)
 		os.Exit(1)
 	}
 
-	klog.V(2).Infof("CSI driver name: %q", *snapshotterName)
+	klog.V(2).Infof("CSI driver name: %q", driverName)
 
 	// Check it's ready
 	if err = csirpc.ProbeForever(csiConn, *csiTimeout); err != nil {
@@ -154,7 +142,7 @@ func main() {
 		os.Exit(1)
 	}
 	if !supportsCreateSnapshot {
-		klog.Errorf("CSI driver %s does not support ControllerCreateSnapshot", *snapshotterName)
+		klog.Errorf("CSI driver %s does not support ControllerCreateSnapshot", driverName)
 		os.Exit(1)
 	}
 
@@ -163,19 +151,15 @@ func main() {
 		os.Exit(1)
 	}
 
-	klog.V(2).Infof("Start NewCSISnapshotController with snapshotter [%s] kubeconfig [%s] csiTimeout [%+v] csiAddress [%s] createSnapshotContentRetryCount [%d] createSnapshotContentInterval [%+v] resyncPeriod [%+v] snapshotNamePrefix [%s] snapshotNameUUIDLength [%d]", *snapshotterName, *kubeconfig, *csiTimeout, *csiAddress, createSnapshotContentRetryCount, *createSnapshotContentInterval, *resyncPeriod, *snapshotNamePrefix, snapshotNameUUIDLength)
+	klog.V(2).Infof("Start NewCSISnapshotSideCarController with snapshotter [%s] kubeconfig [%s] csiTimeout [%+v] csiAddress [%s] resyncPeriod [%+v] snapshotNamePrefix [%s] snapshotNameUUIDLength [%d]", driverName, *kubeconfig, *csiTimeout, *csiAddress, *resyncPeriod, *snapshotNamePrefix, snapshotNameUUIDLength)
 
 	snapShotter := snapshotter.NewSnapshotter(csiConn)
-	ctrl := controller.NewCSISnapshotController(
+	ctrl := controller.NewCSISnapshotSideCarController(
 		snapClient,
 		kubeClient,
-		*snapshotterName,
-		factory.Snapshot().V1beta1().VolumeSnapshots(),
+		driverName,
 		factory.Snapshot().V1beta1().VolumeSnapshotContents(),
 		factory.Snapshot().V1beta1().VolumeSnapshotClasses(),
-		coreFactory.Core().V1().PersistentVolumeClaims(),
-		*createSnapshotContentRetryCount,
-		*createSnapshotContentInterval,
 		snapShotter,
 		*csiTimeout,
 		*resyncPeriod,
@@ -200,7 +184,7 @@ func main() {
 	if !*leaderElection {
 		run(context.TODO())
 	} else {
-		lockName := fmt.Sprintf("%s-%s", prefix, strings.Replace(*snapshotterName, "/", "-", -1))
+		lockName := fmt.Sprintf("%s-%s", prefix, strings.Replace(driverName, "/", "-", -1))
 		le := leaderelection.NewLeaderElection(kubeClient, lockName, run)
 		if *leaderElectionNamespace != "" {
 			le.WithNamespace(*leaderElectionNamespace)

cmd/snapshot-controller/Dockerfile

@@ -0,0 +1,6 @@
+FROM gcr.io/distroless/static:latest
+LABEL maintainers="Kubernetes Authors"
+LABEL description="Snapshot Controller"
+
+COPY ./bin/snapshot-controller snapshot-controller
+ENTRYPOINT ["/snapshot-controller"]

cmd/snapshot-controller/main.go

@@ -0,0 +1,146 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"os/signal"
+	"time"
+
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog"
+
+	"github.com/kubernetes-csi/csi-lib-utils/leaderelection"
+	controller "github.com/kubernetes-csi/external-snapshotter/pkg/common-controller"
+
+	clientset "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned"
+	snapshotscheme "github.com/kubernetes-csi/external-snapshotter/pkg/client/clientset/versioned/scheme"
+	informers "github.com/kubernetes-csi/external-snapshotter/pkg/client/informers/externalversions"
+	coreinformers "k8s.io/client-go/informers"
+)
+
+const (
+	// Number of worker threads
+	threads = 10
+)
+
+// Command line flags
+var (
+	kubeconfig                      = flag.String("kubeconfig", "", "Absolute path to the kubeconfig file. Required only when running out of cluster.")
+	createSnapshotContentRetryCount = flag.Int("create-snapshotcontent-retrycount", 5, "Number of retries when we create a snapshot content object for a snapshot.")
+	createSnapshotContentInterval   = flag.Duration("create-snapshotcontent-interval", 10*time.Second, "Interval between retries when we create a snapshot content object for a snapshot.")
+	resyncPeriod                    = flag.Duration("resync-period", 60*time.Second, "Resync interval of the controller.")
+	showVersion                     = flag.Bool("version", false, "Show version.")
+
+	leaderElection          = flag.Bool("leader-election", false, "Enables leader election.")
+	leaderElectionNamespace = flag.String("leader-election-namespace", "", "The namespace where the leader election resource exists. Defaults to the pod namespace if not set.")
+)
+
+var (
+	version = "unknown"
+)
+
+func main() {
+	klog.InitFlags(nil)
+	flag.Set("logtostderr", "true")
+	flag.Parse()
+
+	if *showVersion {
+		fmt.Println(os.Args[0], version)
+		os.Exit(0)
+	}
+	klog.Infof("Version: %s", version)
+
+	// Create the client config. Use kubeconfig if given, otherwise assume in-cluster.
+	config, err := buildConfig(*kubeconfig)
+	if err != nil {
+		klog.Error(err.Error())
+		os.Exit(1)
+	}
+
+	kubeClient, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		klog.Error(err.Error())
+		os.Exit(1)
+	}
+
+	snapClient, err := clientset.NewForConfig(config)
+	if err != nil {
+		klog.Errorf("Error building snapshot clientset: %s", err.Error())
+		os.Exit(1)
+	}
+
+	factory := informers.NewSharedInformerFactory(snapClient, *resyncPeriod)
+	coreFactory := coreinformers.NewSharedInformerFactory(kubeClient, *resyncPeriod)
+
+	// Add Snapshot types to the defualt Kubernetes so events can be logged for them
+	snapshotscheme.AddToScheme(scheme.Scheme)
+
+	klog.V(2).Infof("Start NewCSISnapshotController with kubeconfig [%s] createSnapshotContentRetryCount [%d] createSnapshotContentInterval [%d] resyncPeriod [%+v]", *kubeconfig, *createSnapshotContentRetryCount, *createSnapshotContentInterval, *resyncPeriod)
+
+	ctrl := controller.NewCSISnapshotCommonController(
+		snapClient,
+		kubeClient,
+		factory.Snapshot().V1beta1().VolumeSnapshots(),
+		factory.Snapshot().V1beta1().VolumeSnapshotContents(),
+		factory.Snapshot().V1beta1().VolumeSnapshotClasses(),
+		coreFactory.Core().V1().PersistentVolumeClaims(),
+		*createSnapshotContentRetryCount,
+		*createSnapshotContentInterval,
+		*resyncPeriod,
+	)
+
+	run := func(context.Context) {
+		// run...
+		stopCh := make(chan struct{})
+		factory.Start(stopCh)
+		coreFactory.Start(stopCh)
+		go ctrl.Run(threads, stopCh)
+
+		// ...until SIGINT
+		c := make(chan os.Signal, 1)
+		signal.Notify(c, os.Interrupt)
+		<-c
+		close(stopCh)
+	}
+
+	if !*leaderElection {
+		run(context.TODO())
+	} else {
+		lockName := "snapshot-controller-leader"
+		le := leaderelection.NewLeaderElection(kubeClient, lockName, run)
+		if *leaderElectionNamespace != "" {
+			le.WithNamespace(*leaderElectionNamespace)
+		}
+		if err := le.Run(); err != nil {
+			klog.Fatalf("failed to initialize leader election: %v", err)
+		}
+	}
+}
+
+func buildConfig(kubeconfig string) (*rest.Config, error) {
+	if kubeconfig != "" {
+		return clientcmd.BuildConfigFromFlags("", kubeconfig)
+	}
+	return rest.InClusterConfig()
+}

deploy/kubernetes/README.md renamed to deploy/kubernetes/csi-snapshotter/README.md

@@ -20,15 +20,6 @@ metadata:
   # rename if there are conflicts
   name: external-snapshotter-runner
 rules:
-  - apiGroups: [""]
-    resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch"]
-  - apiGroups: [""]
-    resources: ["persistentvolumeclaims"]
-    verbs: ["get", "list", "watch", "update"]
-  - apiGroups: ["storage.k8s.io"]
-    resources: ["storageclasses"]
-    verbs: ["get", "list", "watch"]
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["list", "watch", "create", "update", "patch"]
@@ -48,15 +39,6 @@ rules:
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: ["volumesnapshotcontents/status"]
     verbs: ["update"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshots"]
-    verbs: ["get", "list", "watch", "update"]
-  - apiGroups: ["snapshot.storage.k8s.io"]
-    resources: ["volumesnapshots/status"]
-    verbs: ["update"]
-  - apiGroups: ["apiextensions.k8s.io"]
-    resources: ["customresourcedefinitions"]
-    verbs: ["create", "list", "watch", "delete", "get", "update"]
 
 ---
 kind: ClusterRoleBinding

deploy/kubernetes/rbac.yaml renamed to deploy/kubernetes/csi-snapshotter/rbac-csi-snapshotter.yaml

@@ -1,6 +1,6 @@
 # This YAML file shows how to deploy the CSI snapshotter together
 # with the hostpath CSI driver. It depends on the RBAC rules
-# from rbac.yaml and rbac-external-provisioner.yaml.
+# from rbac-csi-snapshotter.yaml and rbac-external-provisioner.yaml.
 #
 # Because external-snapshotter and external-provisioner get
 # deployed in the same pod, we have to merge the permissions
@@ -72,11 +72,10 @@ spec:
       serviceAccount: csi-snapshotter
       containers:
         - name: csi-provisioner
-          image: quay.io/k8scsi/csi-provisioner:v1.3.0
+          image: quay.io/k8scsi/csi-provisioner:v1.5.0-rc1
           args:
-            - "--provisioner=csi-hostpath"
+            - "--v=5"
             - "--csi-address=$(ADDRESS)"
-            - "--connection-timeout=15s"
           env:
             - name: ADDRESS
               value: /csi/csi.sock
@@ -85,20 +84,21 @@ spec:
             - name: socket-dir
               mountPath: /csi
         - name: csi-snapshotter
-          image: quay.io/k8scsi/csi-snapshotter:v1.2.0
+          # NOTE: replace with official image when released: quay.io/k8scsi/csi-snapshotter:v2.0.0
+          image: csi-snapshotter:testbeta #quay.io/k8scsi/csi-snapshotter:testbeta
           args:
+            - "--v=5"
             - "--csi-address=$(ADDRESS)"
-            - "--connection-timeout=15s"
             - "--leader-election=false"
           env:
             - name: ADDRESS
               value: /csi/csi.sock
-          imagePullPolicy: Always
+          imagePullPolicy: IfNotPresent #Always
           volumeMounts:
             - name: socket-dir
               mountPath: /csi
         - name: hostpath
-          image: quay.io/k8scsi/hostpathplugin:v1.1.0
+          image: quay.io/k8scsi/hostpathplugin:v1.2.0
           args:
             - "--v=5"
             - "--endpoint=$(CSI_ENDPOINT)"