Consolidate multiple commits into one: Add PVC manager functionality and related fixes

Author: laik

Makefile

@@ -62,12 +62,15 @@ endif
 
 # Specify the name for the binaries
 PROVISIONER_LOCALPV=provisioner-localpv
+PVC_MANAGER=pvc-manager
 
 # Specify the name of the image
 PROVISIONER_LOCALPV_IMAGE?=provisioner-localpv
+PVC_MANAGER_IMAGE?=pvc-manager
 
 # Final variable with image org, name and tag
 PROVISIONER_LOCALPV_IMAGE_TAG=${IMAGE_ORG}/${PROVISIONER_LOCALPV_IMAGE}:${TAG}
+PVC_MANAGER_IMAGE_TAG=${IMAGE_ORG}/${PVC_MANAGER_IMAGE}:${TAG}
 
 # Specify the date of build
 DBUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ')
@@ -97,7 +100,7 @@ EXTERNAL_TOOLS=\
 export DBUILD_ARGS=--build-arg DBUILD_DATE=${DBUILD_DATE} --build-arg DBUILD_REPO_URL=${DBUILD_REPO_URL} --build-arg DBUILD_SITE_URL=${DBUILD_SITE_URL} --build-arg BRANCH=${BRANCH} --build-arg RELEASE_TAG=${RELEASE_TAG}
 
 .PHONY: all
-all: test provisioner-localpv-image
+all: test provisioner-localpv-image pvc-manager-image
 
 .PHONY: deps
 deps:
@@ -126,8 +129,10 @@ clean:
 	rm -rf bin
 	./ci/ci-test.sh clean
 	chmod -R u+w ${GOPATH}/bin/${PROVISIONER_LOCALPV} 2>/dev/null || true
+	chmod -R u+w ${GOPATH}/bin/${PVC_MANAGER} 2>/dev/null || true
 	chmod -R u+w ${GOPATH}/pkg/* 2>/dev/null || true
 	rm -rf ${GOPATH}/bin/${PROVISIONER_LOCALPV}
+	rm -rf ${GOPATH}/bin/${PVC_MANAGER}
 	rm -rf ${GOPATH}/pkg/*
 
 .PHONY: test
@@ -173,6 +178,14 @@ provisioner-localpv:
 	@echo "----------------------------"
 	@PNAME=${PROVISIONER_LOCALPV} CTLNAME=${PROVISIONER_LOCALPV} sh -c "'./buildscripts/build.sh'"
 
+#Use this to build pvc-manager
+.PHONY: pvc-manager
+pvc-manager:
+	@echo "----------------------------"
+	@echo "--> pvc-manager            "
+	@echo "----------------------------"
+	@PNAME=${PVC_MANAGER} CTLNAME=${PVC_MANAGER} sh -c "'./buildscripts/build.sh'"
+
 .PHONY: provisioner-localpv-image
 provisioner-localpv-image: provisioner-localpv
 	@echo "-------------------------------"
@@ -182,6 +195,15 @@ provisioner-localpv-image: provisioner-localpv
 	@cd buildscripts/provisioner-localpv && docker build -t ${PROVISIONER_LOCALPV_IMAGE_TAG} ${DBUILD_ARGS} . --no-cache
 	@rm buildscripts/provisioner-localpv/${PROVISIONER_LOCALPV}
 
+.PHONY: pvc-manager-image
+pvc-manager-image: pvc-manager
+	@echo "-------------------------------"
+	@echo "--> pvc-manager image         "
+	@echo "-------------------------------"
+	@cp bin/pvc-manager/${PVC_MANAGER} buildscripts/pvc-manager/
+	@cd buildscripts/pvc-manager && docker build -t ${PVC_MANAGER_IMAGE_TAG} ${DBUILD_ARGS} . --no-cache
+	@rm buildscripts/pvc-manager/${PVC_MANAGER}
+
 .PHONY: image-tag
 image-tag:
 	@echo ${TAG}
@@ -197,6 +219,7 @@ image-ref:
 .PHONY: push
 push:
 	DIMAGE=${IMAGE_ORG}/${PROVISIONER_LOCALPV_IMAGE} ./buildscripts/push.sh
+	DIMAGE=${IMAGE_ORG}/${PVC_MANAGER_IMAGE} ./buildscripts/push.sh
 
 # include the buildx recipes
 include Makefile.buildx.mk

README.md

@@ -23,6 +23,8 @@ OpenEBS Dynamic LocalPV Provisioner is an open‐source Kubernetes component tha
 
 ## Architecture
 
+### Traditional Architecture (Helper Pods)
+
 ```mermaid
 
 graph TD
@@ -66,6 +68,51 @@ graph TD
 
 ```
 
+### New Architecture (PVC Manager DaemonSet)
+
+```mermaid
+
+graph TD
+    %% Define Styles with Black Text  
+    style kubelet fill:#ffcc00,stroke:#d4a017,stroke-width:2px,color:#000
+    style Provisioner fill:#66ccff,stroke:#3388cc,stroke-width:2px,color:#000
+    style PVCManager fill:#99ff99,stroke:#44aa44,stroke-width:2px,color:#000
+    style App fill:#ff9999,stroke:#cc6666,stroke-width:2px,color:#000
+    style Hostpath fill:#ffdd99,stroke:#d4a017,stroke-width:2px,color:#000
+    style PVC fill:#ffdd99,stroke:#d4a017,stroke-width:2px,color:#000
+    style PV fill:#d9b3ff,stroke:#9955cc,stroke-width:2px,color:#000
+
+    subgraph "Kubernetes Cluster"
+        subgraph "Kubelet"
+            kubelet["Kubelet"]
+        end
+        subgraph "OpenEBS LocalPV"
+            Provisioner["LocalPV Provisioner"]
+        end
+        subgraph "Worker Node (Every Node)"
+            PVCManager["PVC Manager (DaemonSet)"]
+            App["Application Pod"]
+            PVC["Persistent Volume Claim"]
+            PV["Persistent Volume"]
+            Hostpath["[User-defined path on host]"]
+        end
+    end
+
+    %% Storage Flow
+    App -->|Requests Storage| PVC
+    PVC -->|Binds to| PV
+    PV -->|Mounted on| Hostpath
+    kubelet -->|Mounts Path| Hostpath
+
+    %% Provisioning Flow (New)
+    Provisioner -->|HTTP Request| PVCManager
+    Provisioner -->|Watches PVC Requests| Provisioner
+    Provisioner -->|Creates PV| PV
+    PVCManager -->|Creates/Deletes Directory| Hostpath
+    PV -->|Bound to| PVC
+
+```
+
 Please check [here](./design/hostpath_localpv_provisioner.md) for complete design and architecture.
 
 ## Kubernetes Compatibility Matrix
@@ -90,6 +137,9 @@ Please check [here](./design/hostpath_localpv_provisioner.md) for complete desig
 
 ## Features
 
+- [x] **Two Architecture Modes**:
+    - [x] **Helper Pod Mode** (Traditional): Creates ephemeral pods for volume operations
+    - [x] **PVC Manager Mode** (New): Uses DaemonSet with HTTP API for faster operations
 - [x] Access Modes
     - [x] ReadWriteOnce
     - ~~ReadOnlyMany~~
@@ -99,6 +149,25 @@ Please check [here](./design/hostpath_localpv_provisioner.md) for complete desig
     - [ ] `Block` mode
 - [x] [Volume Resize(Via Quotas)](./docs/tutorials/hostpath/xfs_quota/)
 
+### PVC Manager Architecture Benefits
+
+The new PVC Manager architecture provides:
+
+- **Better Performance**: Eliminates pod creation overhead for volume operations
+- **Reduced API Server Load**: Fewer ephemeral pod objects created/destroyed
+- **Improved Reliability**: Persistent service always available for volume operations
+- **Enhanced Observability**: HTTP API with health checks and metrics
+- **Backward Compatibility**: Can be disabled to use traditional helper pod mode
+
+To enable PVC Manager mode, set the environment variable:
+```yaml
+env:
+- name: OPENEBS_IO_ENABLE_PVC_MANAGER
+  value: "true"
+```
+
+[Learn more about PVC Manager architecture](./docs/pvc-manager-architecture.md)
+
 ## Inspiration/Credit
 
 OpenEBS Local PV has been inspired by the prior work done by the following the Kubernetes projects:

buildscripts/pvc-manager/Dockerfile

@@ -0,0 +1,28 @@
+FROM alpine:3.20.1
+
+RUN apk add --no-cache \
+  iproute2 \
+  bash \
+  util-linux \
+  e2fsprogs \
+  xfsprogs \
+  xfsprogs-extra \
+  blkid \
+  findmnt \
+  ca-certificates \
+  quota-tools
+
+COPY pvc-manager /usr/local/bin/
+
+ARG DBUILD_DATE
+ARG DBUILD_REPO_URL
+ARG DBUILD_SITE_URL
+LABEL org.label-schema.name="pvc-manager"
+LABEL org.label-schema.description="OpenEBS LocalPV PVC Manager"
+LABEL org.label-schema.url="$DBUILD_SITE_URL"
+LABEL org.label-schema.vcs-url="$DBUILD_REPO_URL"
+LABEL org.label-schema.schema-version="1.0"
+LABEL org.label-schema.build-date="$DBUILD_DATE"
+
+ENTRYPOINT ["/usr/local/bin/pvc-manager"]
+CMD ["--help"]

cmd/provisioner-localpv/app/env.go

@@ -31,11 +31,18 @@ const (
 	// ProvisionerImagePullSecrets is the environment variable that provides the
 	// init pod to use as authentication when pulling helper image, it is used in the scene where authentication is required
 	ProvisionerImagePullSecrets menv.ENVKey = "OPENEBS_IO_IMAGE_PULL_SECRETS"
+
+	// ProvisionerEnablePVCManager enables the PVC Manager mode instead of helper pods
+	ProvisionerEnablePVCManager string = "OPENEBS_IO_ENABLE_PVC_MANAGER"
+
+	// ProvisionerPVCManagerPort is the port where PVC Manager service listens
+	ProvisionerPVCManagerPort string = "OPENEBS_IO_PVC_MANAGER_PORT"
 )
 
 var (
-	defaultHelperImage = "openebs/linux-utils:latest"
-	defaultBasePath    = "/var/openebs/local"
+	defaultHelperImage    = "openebs/linux-utils:latest"
+	defaultBasePath       = "/var/openebs/local"
+	defaultPVCManagerPort = "8080"
 )
 
 func getOpenEBSNamespace() string {
@@ -59,3 +66,19 @@ func getOpenEBSServiceAccountName() string {
 func getOpenEBSImagePullSecrets() string {
 	return menv.Get(ProvisionerImagePullSecrets)
 }
+
+// GetEnv gets an environment variable value
+func GetEnv(key string) string {
+	return menv.Get(menv.ENVKey(key))
+}
+
+// getPVCManagerEnabled returns whether PVC Manager mode is enabled
+func getPVCManagerEnabled() bool {
+	val, _ := k8sEnv.GetBool(ProvisionerEnablePVCManager, true) // Default to true for new architecture
+	return val
+}
+
+// getPVCManagerPort returns the port for PVC Manager service
+func getPVCManagerPort() string {
+	return k8sEnv.GetString(ProvisionerPVCManagerPort, defaultPVCManagerPort)
+}

cmd/provisioner-localpv/app/helper_hostpath.go

@@ -9,16 +9,16 @@ import (
 	"strings"
 	"time"
 
+	"github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/container"
+	"github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/pod"
+	"github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/volume"
 	hostpath "github.com/openebs/maya/pkg/hostpath/v1alpha1"
 	errors "github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	k8serror "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/klog/v2"
-
-	"github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/container"
-	"github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/pod"
-	"github.com/openebs/dynamic-localpv-provisioner/pkg/kubernetes/api/core/v1/volume"
+	"sigs.k8s.io/yaml"
 )
 
 type podConfig struct {
@@ -213,7 +213,21 @@ func (p *Provisioner) createCleanupPod(ctx context.Context, pOpts *HelperPodOpti
 
 	config.taints = pOpts.selectedNodeTaints
 
-	config.pOpts.cmdsForPath = append(config.pOpts.cmdsForPath, filepath.Join("/data/", config.volumeDir))
+	path := filepath.Join("/data/", config.volumeDir)
+
+	scripts := "" +
+		"FS=`stat -f -c %T " + path + "` ; " +
+		"if [[ \"$FS\" == \"xfs\" ]]; then " +
+		"id=`xfs_io -c stat " + path + " 2>/dev/null | grep projid | head -1 | awk -F '=' '{print $2}' | tr -d ' '` ; " +
+		"echo \"projid=$id\" ; " +
+		"if [[ -n \"$id\" && \"$id\" != \"0\" ]]; then " +
+		"xfs_io -c 'chproj -R 0' " + path + " 2>/dev/null || true ; " +
+		"xfs_quota -x -c \"limit -p bsoft=0 bhard=0 $id\" /data 2>/dev/null || true ; " +
+		"fi ; " +
+		"fi ; " +
+		"rm -rf " + path
+
+	config.pOpts.cmdsForPath = []string{"sh", "-c", scripts}
 
 	_, err := p.launchPod(ctx, config)
 	if err != nil && !k8serror.IsAlreadyExists(err) {
@@ -349,6 +363,13 @@ func (p *Provisioner) launchPod(ctx context.Context, config podConfig) (*corev1.
 		return nil, err
 	}
 
+	helperPodYAML, err := yaml.Marshal(helperPod)
+	if err != nil {
+		klog.Errorf("failed to marshal helper pod to YAML: %v", err)
+	} else {
+		klog.Infof("launching helper pod: %s", string(helperPodYAML))
+	}
+
 	var hPod *corev1.Pod
 
 	//Launch the helper pod.