build: add build-time protobuf generation infrastructure

Signed-off-by: Alexander Indenbaum <aindenba@redhat.com>

Author: Alexander Indenbaum

.github/workflows/check-proto-changes.yml

@@ -0,0 +1,67 @@
+# yamllint disable rule:truthy
+---
+name: Check Proto Changes
+
+on:
+  pull_request:
+    paths:
+      - 'internal/nvme/gateway/proto/gateway.proto'
+      - 'build.env'
+  push:
+    branches:
+      - devel
+    paths:
+      - 'internal/nvme/gateway/proto/gateway.proto'
+      - 'build.env'
+
+jobs:
+  check-proto-changes:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683
+        with:
+          fetch-depth: 0
+
+      - name: Verify committed proto file matches build.env specification
+        # yamllint disable rule:line-length
+        run: |
+          # Source build.env to get version information
+          source build.env
+
+          # Fetch the specific version from ceph-nvmeof based on build.env
+          echo "Fetching proto file from ${PROTO_SOURCE_REPO} branch " \
+            "${PROTO_SOURCE_BRANCH} SHA ${PROTO_SOURCE_SHA}..."
+
+          # Create fetch URL, handle 'latest' case
+          if [[ "${PROTO_SOURCE_SHA}" == "latest" ]]; then
+            # Fetch from branch tip
+            FETCH_URL="${PROTO_SOURCE_REPO}/raw/refs/heads/${PROTO_SOURCE_BRANCH}/control/proto/gateway.proto"
+          else
+            # Fetch from specific SHA
+            FETCH_URL="${PROTO_SOURCE_REPO}/raw/${PROTO_SOURCE_SHA}/control/proto/gateway.proto"
+          fi
+
+          # Fetch proto file
+          curl -fsSL "${FETCH_URL}" -o /tmp/gateway.proto.expected
+
+          # Normalize the committed proto file by removing go_package option
+          echo "Normalizing committed proto file (removing go_package option)..."
+          grep -v "option go_package" internal/nvme/gateway/proto/gateway.proto > \
+            /tmp/gateway.proto.normalized
+
+          # Compare normalized committed file with fetched file
+          if diff -u /tmp/gateway.proto.normalized /tmp/gateway.proto.expected; then
+            echo "✅ Committed proto file matches build.env specification"
+            echo "  Repository: ${PROTO_SOURCE_REPO}"
+            echo "  Branch: ${PROTO_SOURCE_BRANCH}"
+            echo "  SHA: ${PROTO_SOURCE_SHA}"
+            echo "  Note: go_package option is added by the build process"
+          else
+            echo "❌ Committed proto file does not match build.env specification"
+            echo "  Expected: ${PROTO_SOURCE_REPO} branch ${PROTO_SOURCE_BRANCH} SHA " \
+              "${PROTO_SOURCE_SHA}"
+            echo "  Please update the committed proto file to match the build.env specification"
+            exit 1
+          fi

Makefile

@@ -168,11 +168,23 @@ commitlint:
 	@test $(REBASE) -eq 0 || git -c user.name=commitlint -c user.email=commitline@localhost rebase FETCH_HEAD
 	commitlint --verbose --from $(GIT_SINCE)
 
-.PHONY: cephcsi
-cephcsi: check-env
+.PHONY: cephcsi generate-proto clean-proto force-generate-proto
+cephcsi: check-env generate-proto
 	if [ ! -d ./vendor ]; then (go mod tidy && go mod vendor); fi
 	GOOS=linux go build $(GO_TAGS) -mod vendor -a -ldflags '$(LDFLAGS)' -o _output/cephcsi ./cmd/
 
+generate-proto: check-env
+	@echo "Generating protobuf stubs..."
+	./scripts/generate-proto.sh
+
+clean-proto: check-env
+	@echo "Cleaning protobuf generated files..."
+	./scripts/generate-proto.sh --clean
+
+force-generate-proto: check-env
+	@echo "Force regenerating protobuf stubs..."
+	./scripts/generate-proto.sh --force
+
 e2e.test: check-env
 	cd e2e && go test $(GO_TAGS) -mod=vendor -c -o ../e2e.test ./
 
@@ -217,6 +229,9 @@ containerized-build: TARGET = cephcsi
 containerized-build: .container-cmd .devel-container-id
 	$(CONTAINER_CMD) run --rm -v $(CURDIR):/go/src/github.com/ceph/ceph-csi$(SELINUX_VOL_FLAG) $(CSI_IMAGE_NAME):devel make $(TARGET) CONTAINERIZED=yes
 
+containerized-generate-proto: .container-cmd .devel-container-id
+	$(CONTAINER_CMD) run --rm -v $(CURDIR):/go/src/github.com/ceph/ceph-csi$(SELINUX_VOL_FLAG) $(CSI_IMAGE_NAME):devel make generate-proto CONTAINERIZED=yes
+
 containerized-test: TARGET = test
 containerized-test: REBASE ?= 0
 containerized-test: .container-cmd .test-container-id
@@ -272,6 +287,7 @@ clean:
 	rm -f _output/cephcsi
 	$(RM) scripts/golangci.yml
 	$(RM) e2e.test
+	rm -f internal/nvme/gateway/proto/*.pb.go
 	[ ! -f .devel-container-id ] || $(CONTAINER_CMD) rmi $(CSI_IMAGE_NAME):devel
 	$(RM) .devel-container-id
 	[ ! -f .test-container-id ] || $(CONTAINER_CMD) rmi $(CSI_IMAGE_NAME):test

build.env

@@ -35,6 +35,19 @@ COMMITLINT_VERSION=latest
 # static checks and linters
 GOLANGCI_VERSION=v2.1.5
 
+# protobuf tool versions
+PROTOC_VERSION=3.14.0
+PROTOC_GEN_GO_VERSION=v1.36.6
+PROTOC_GEN_GO_GRPC_VERSION=v1.5.1
+
+# protobuf source configuration
+# Using committed gateway.proto file instead of fetching
+# The committed file should match the current devel branch tip
+PROTO_SOURCE_REPO=https://github.com/ceph/ceph-nvmeof
+PROTO_SOURCE_BRANCH=devel
+PROTO_SOURCE_SHA=latest
+# 'latest' for branch tip, or specific commit SHA for reproducible builds
+
 # external snapshotter version
 # Refer: https://github.com/kubernetes-csi/external-snapshotter/releases
 SNAPSHOT_VERSION=v8.2.0

deploy/cephcsi/image/Dockerfile

@@ -40,14 +40,12 @@ RUN source /build.env && \
 # test if the downloaded version of Golang works (different arch?)
 RUN ${GOROOT}/bin/go version && ${GOROOT}/bin/go env
 
-# other/conflicting versions of protobuf get installed as dependency
-RUN dnf -y remove protobuf
-
 RUN dnf -y install --nodocs \
 	librados-devel librbd-devel libcephfs-devel \
 	/usr/bin/cc \
 	make \
 	git \
+	protobuf-compiler \
 	&& dnf clean all \
 	&& rm -rf /var/cache/yum \
     && true
@@ -60,6 +58,10 @@ ENV GOROOT=${GOROOT} \
     ENV_CSI_IMAGE_NAME="${CSI_IMAGE_NAME}" \
     PATH="${GOROOT}/bin:${GOPATH}/bin:${PATH}"
 
+# Install Go protobuf plugins using versions from build.env
+RUN source /build.env \
+    && ${GOROOT}/bin/go install google.golang.org/protobuf/cmd/protoc-gen-go@${PROTOC_GEN_GO_VERSION} \
+    && ${GOROOT}/bin/go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@${PROTOC_GEN_GO_GRPC_VERSION}
 
 WORKDIR ${SRC_DIR}