Skip to content

Commit 0ccbc06

Browse files
BlaineEXEBlaineEXE
committed
proto: update to latest grpc protobuf gen tools
Modeled after: container-storage-interface/spec#552 Update to the latest, non-deprecated versions of Google's tools for protobuf generation. Also update make tools to allow working with go modules outside of GOPATH, and to build on ARM-based Macbooks. New tools are significantly different from prior tools. Notably, the grpc plugin is no longer supported, so the `--go-gprc_out` flag is used, which moves some generated content from `cosi.pb.go` to a new `cosi_grpc.pb.go` file. We are still able to keep fake-gen by continuing to use deprecated tool versions for that purpose. There's a chance that it may not serve the COSI project long-term, but it should still work for now. Signed-off-by: Blaine Gardner <[email protected]>
1 parent 1bdaaba commit 0ccbc06

File tree

10 files changed

+794
-871
lines changed

10 files changed

+794
-871
lines changed

‎proto/.gitignore‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,6 @@
22
.DS_Store
33
.build
44
*.swp
5-
cosi.proto.tmp
6-
protoc*
7-
cosi.a
85
.protoc
96
.idea
107
.project

‎proto/Makefile‎

Lines changed: 58 additions & 168 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ export GOPATH
2121

2222
# Only set PROTOC_VER if it has an empty value.
2323
ifeq (,$(strip $(PROTOC_VER)))
24-
PROTOC_VER := 3.14.0
24+
PROTOC_VER := 27.2
2525
endif
2626

2727
PROTOC_OS := $(shell uname -s)
@@ -36,98 +36,30 @@ else ifeq (arm64,$(PROTOC_ARCH))
3636
PROTOC_ARCH := aarch_64
3737
endif
3838

39-
PROTOC := ./protoc
4039
PROTOC_ZIP := protoc-$(PROTOC_VER)-$(PROTOC_OS)-$(PROTOC_ARCH).zip
4140
PROTOC_URL := https://github.com/protocolbuffers/protobuf/releases/download/v$(PROTOC_VER)/$(PROTOC_ZIP)
4241
PROTOC_TMP_DIR := .protoc
43-
PROTOC_TMP_BIN := $(PROTOC_TMP_DIR)/bin/protoc
42+
PROTOC := $(PROTOC_TMP_DIR)/bin/protoc
4443
PROTOC_TMP_INC := $(PROTOC_TMP_DIR)/include
4544

4645
$(PROTOC):
4746
-mkdir -p "$(PROTOC_TMP_DIR)" && \
4847
curl -L $(PROTOC_URL) -o "$(PROTOC_TMP_DIR)/$(PROTOC_ZIP)" && \
4948
unzip -o "$(PROTOC_TMP_DIR)/$(PROTOC_ZIP)" -d "$(PROTOC_TMP_DIR)" && \
5049
unzip -o "$(PROTOC_TMP_DIR)/$(PROTOC_ZIP)" -d "$(PROTOC_TMP_INC)/*" && \
51-
chmod 0755 "$(PROTOC_TMP_BIN)" && \
52-
cp -f "$(PROTOC_TMP_BIN)" "$@"
50+
chmod 0755 "$@"
5351
stat "$@" > /dev/null 2>&1
5452

53+
$(GOBIN)/protoc-gen-go: go.mod
54+
go install google.golang.org/protobuf/cmd/protoc-gen-go
5555

56-
########################################################################
57-
## PROTOC-GEN-GO ##
58-
########################################################################
59-
60-
# This is the recipe for getting and installing the go plug-in
61-
# for protoc
62-
PROTOC_GEN_GO_PKG := github.com/golang/protobuf/protoc-gen-go
63-
PROTOC_GEN_GO := protoc-gen-go
64-
$(PROTOC_GEN_GO): PROTOBUF_PKG := $(dir $(PROTOC_GEN_GO_PKG))
65-
$(PROTOC_GEN_GO): PROTOBUF_VERSION := v1.3.2
66-
$(PROTOC_GEN_GO):
67-
mkdir -p $(dir $(GOPATH)/src/$(PROTOBUF_PKG))
68-
test -d $(GOPATH)/src/$(PROTOBUF_PKG)/.git || git clone https://$(PROTOBUF_PKG) $(GOPATH)/src/$(PROTOBUF_PKG)
69-
(cd $(GOPATH)/src/$(PROTOBUF_PKG) && \
70-
(test "$$(git describe --tags | head -1)" = "$(PROTOBUF_VERSION)" || \
71-
(git fetch && git checkout tags/$(PROTOBUF_VERSION))))
72-
(cd $(GOPATH)/src/$(PROTOBUF_PKG) && go get -v -d $$(go list -f '{{ .ImportPath }}' ./...)) && \
73-
go build -o "$@" $(PROTOC_GEN_GO_PKG)
56+
$(GOBIN)/protoc-gen-go-grpc:
57+
go install google.golang.org/grpc/cmd/[email protected]
7458

59+
$(GOBIN)/protoc-gen-go-json:
60+
go install github.com/mitchellh/[email protected]
7561

76-
########################################################################
77-
## PROTOC-GEN-GO-JSON ##
78-
########################################################################
79-
80-
# This is the recipe for getting and installing the json plug-in
81-
# for protoc-gen-go
82-
PROTOC_GEN_GO_JSON_PKG := github.com/mitchellh/protoc-gen-go-json
83-
PROTOC_GEN_GO_JSON := protoc-gen-go-json
84-
$(PROTOC_GEN_GO_JSON): PROTOC_GEN_GO_JSON_VERSION := v1.1.0
85-
$(PROTOC_GEN_GO_JSON):
86-
mkdir -p $(dir $(GOPATH)/src/$(PROTOC_GEN_GO_JSON_PKG))
87-
test -d $(GOPATH)/src/$(PROTOC_GEN_GO_JSON_PKG)/.git || git clone https://$(PROTOC_GEN_GO_JSON_PKG) $(GOPATH)/src/$(PROTOC_GEN_GO_JSON_PKG)
88-
(cd $(GOPATH)/src/$(PROTOC_GEN_GO_JSON_PKG) && \
89-
(test "$$(git describe --tags | head -1)" = "$(PROTOC_GEN_GO_JSON_VERSION)" || \
90-
(git fetch && git checkout tags/$(PROTOC_GEN_GO_JSON_VERSION))))
91-
(cd $(GOPATH)/src/$(PROTOC_GEN_GO_JSON_PKG) && go get -v -d $$(go list -f '{{ .ImportPath }}' ./...)) && \
92-
go build -o "$@" $(PROTOC_GEN_GO_JSON_PKG)
93-
94-
95-
########################################################################
96-
## GEN-PROTO-GO ##
97-
########################################################################
98-
99-
# This is the recipe for getting and installing the gen-proto pkg
100-
# This is a dependency of grpc-go and must be installed before
101-
# installing grpc-go.
102-
GENPROTO_GO_SRC := github.com/googleapis/go-genproto
103-
GENPROTO_GO_PKG := google.golang.org/genproto
104-
GENPROTO_BUILD_GO := genproto-build-go
105-
$(GENPROTO_BUILD_GO): GENPROTO_VERSION := 24fa4b261c55da65468f2abfdae2b024eef27dfb
106-
$(GENPROTO_BUILD_GO):
107-
mkdir -p $(dir $(GOPATH)/src/$(GENPROTO_GO_PKG))
108-
test -d $(GOPATH)/src/$(GENPROTO_GO_PKG)/.git || git clone https://$(GENPROTO_GO_SRC) $(GOPATH)/src/$(GENPROTO_GO_PKG)
109-
(cd $(GOPATH)/src/$(GENPROTO_GO_PKG) && \
110-
(git fetch && git checkout $(GENPROTO_VERSION)))
111-
(cd $(GOPATH)/src/$(GENPROTO_GO_PKG) && go get -v -d $$(go list -f '{{ .ImportPath }}' ./...))
112-
113-
114-
########################################################################
115-
## GRPC-GO ##
116-
########################################################################
117-
118-
# This is the recipe for getting and installing the grpc go
119-
GRPC_GO_SRC := github.com/grpc/grpc-go
120-
GRPC_GO_PKG := google.golang.org/grpc
121-
GRPC_BUILD_GO := grpc-build-go
122-
$(GRPC_BUILD_GO): GRPC_VERSION := v1.26.0
123-
$(GRPC_BUILD_GO):
124-
mkdir -p $(dir $(GOPATH)/src/$(GRPC_GO_PKG))
125-
test -d $(GOPATH)/src/$(GRPC_GO_PKG)/.git || git clone https://$(GRPC_GO_SRC) $(GOPATH)/src/$(GRPC_GO_PKG)
126-
(cd $(GOPATH)/src/$(GRPC_GO_PKG) && \
127-
(test "$$(git describe --tags | head -1)" = "$(GRPC_VERSION)" || \
128-
(git fetch && git checkout tags/$(GRPC_VERSION))))
129-
(cd $(GOPATH)/src/$(GRPC_GO_PKG) && go get -v -d $$(go list -f '{{ .ImportPath }}' ./...) && \
130-
go build -o "$@" $(GRPC_GO_PKG))
62+
PROTOC_ALL := $(GOBIN)/protoc-gen-go $(GOBIN)/protoc-gen-go-grpc $(GOBIN)/protoc-gen-go-json $(PROTOC)
13163

13264

13365
########################################################################
@@ -136,110 +68,68 @@ $(GRPC_BUILD_GO):
13668

13769
# This is the recipe for getting and installing the grpc go
13870
PROTOC_GEN_GO_FAKE_SRC := ./hack/fake-gen
139-
PROTOC_GEN_GO_FAKE := protoc-gen-gofake
71+
PROTOC_GEN_GO_FAKE := $(PROTOC_TMP_DIR)/bin/protoc-gen-gofake
14072
$(PROTOC_GEN_GO_FAKE):
14173
go build -o $(PROTOC_GEN_GO_FAKE) $(PROTOC_GEN_GO_FAKE_SRC)
14274

143-
144-
########################################################################
145-
## PATH ##
146-
########################################################################
147-
148-
# Update PATH with the current directory. This enables the protoc
149-
# binary to discover the protoc-gen-go binary, built inside this
150-
# directory.
151-
export PATH := $(shell pwd):$(PATH)
75+
# Update PATH to discover the protoc-gen-gofake binary
76+
export PATH := $(PROTOC_TMP_DIR)/bin:$(PATH)
15277

15378

15479
########################################################################
15580
## BUILD ##
15681
########################################################################
157-
COSI_PROTO := ./cosi.proto
15882
COSI_SPEC := spec.md
159-
COSI_PKG_ROOT := sigs.k8s.io/container-object-storage-interface-spec
160-
COSI_PKG_SUB := .
161-
COSI_BUILD := $(COSI_PKG_SUB)/.build
162-
COSI_GO := $(COSI_PKG_SUB)/cosi.pb.go
163-
COSI_GO_JSON := $(COSI_PKG_SUB)/cosi.pb.json.go
164-
COSI_GO_FAKE := $(COSI_PKG_SUB)/fake/cosi.pb.fake.go
165-
COSI_A := cosi.a
166-
COSI_GO_TMP := $(COSI_BUILD)/$(COSI_PKG_ROOT)/cosi.pb.go
167-
COSI_GO_JSON_TMP := $(COSI_BUILD)/$(COSI_PKG_ROOT)/cosi.pb.json.go
168-
COSI_GO_FAKE_TMP := $(COSI_BUILD)/fake/$(COSI_PKG_ROOT)/cosi.pb.fake.go
169-
170-
# This recipe generates the go language bindings to a temp area.
171-
$(COSI_GO_TMP): HERE := $(shell pwd)
172-
$(COSI_GO_TMP): PTYPES_PKG := github.com/golang/protobuf/ptypes
173-
$(COSI_GO_TMP): GO_OUT := plugins=grpc
174-
$(COSI_GO_TMP): GO_OUT := $(GO_OUT),Mgoogle/protobuf/descriptor.proto=github.com/golang/protobuf/protoc-gen-go/descriptor
175-
$(COSI_GO_TMP): GO_OUT := $(GO_OUT),Mgoogle/protobuf/wrappers.proto=$(PTYPES_PKG)/wrappers
176-
$(COSI_GO_TMP): GO_OUT := $(GO_OUT):"$(HERE)/$(COSI_BUILD)"
177-
$(COSI_GO_TMP): GO_JSON_OUT := emit_defaults=true
178-
$(COSI_GO_TMP): GO_JSON_OUT := $(GO_JSON_OUT):"$(HERE)/$(COSI_BUILD)"
179-
$(COSI_GO_TMP): GO_FAKE_OUT := emit_defaults
180-
$(COSI_GO_TMP): GO_FAKE_OUT := $(GO_FAKE_OUT),packagePath=sigs.k8s.io/container-object-storage-interface-spec
181-
$(COSI_GO_TMP): GO_FAKE_OUT := $(GO_FAKE_OUT):"$(HERE)/$(COSI_BUILD)"/fake
182-
$(COSI_GO_TMP): INCLUDE := -I$(GOPATH)/src -I$(HERE)/$(PROTOC_TMP_DIR)/include
183-
$(COSI_GO_TMP): $(COSI_PROTO) | $(PROTOC) $(PROTOC_GEN_GO) $(PROTOC_GEN_GO_JSON) $(PROTOC_GEN_GO_FAKE)
184-
@mkdir -p "$(@D)"
185-
@mkdir -p "$(COSI_BUILD)/fake"
186-
(cd "$(GOPATH)/src" && \
187-
$(HERE)/$(PROTOC) $(INCLUDE) --go_out=$(GO_OUT) --go-json_out=$(GO_JSON_OUT) --gofake_out=$(GO_FAKE_OUT) "$(COSI_PKG_ROOT)/$(<F)")
188-
189-
# The temp language bindings are compared to the ones that are
190-
# versioned. If they are different then it means the language
191-
# bindings were not updated prior to being committed.
192-
$(COSI_GO): $(COSI_GO_TMP)
193-
ifeq (true,$(TRAVIS))
194-
diff "$@" "$?"
195-
else
196-
@mkdir -p "$(@D)"
197-
diff "$@" "$?" > /dev/null 2>&1 || cp -f "$?" "$@"
198-
endif
19983

200-
# The temp language bindings are compared to the ones that are
201-
# versioned. If they are different then it means the language
202-
# bindings were not updated prior to being committed.
203-
$(COSI_GO_JSON): $(COSI_GO_JSON_TMP)
204-
ifeq (true,$(TRAVIS))
205-
diff "$@" "$?"
206-
else
207-
@mkdir -p "$(@D)"
208-
diff "$@" "$?" > /dev/null 2>&1 || cp -f "$?" "$@"
209-
endif
84+
COSI_PROTO := ./cosi.proto
85+
COSI_GO := ./cosi.pb.go
86+
COSI_GO_GRPC := ./cosi_grpc.pb.go
87+
COSI_GO_JSON := ./cosi.pb.json.go
88+
COSI_GO_FAKE := ./fake/cosi.pb.fake.go
21089

211-
# The temp language bindings are compared to the ones that are
212-
# versioned. If they are different then it means the language
213-
# bindings were not updated prior to being committed.
214-
$(COSI_GO_FAKE): $(COSI_GO_FAKE_TMP)
215-
ifeq (true,$(TRAVIS))
216-
diff "$@" "$?"
217-
else
218-
@mkdir -p "$(@D)"
219-
diff "$@" "$?" > /dev/null 2>&1 || cp -f "$?" "$@"
220-
endif
90+
BUILD_TMP_DIR := ./.build
91+
BUILD_TMP_COSI_A := $(BUILD_TMP_DIR)/cosi.a
92+
BUILD_PROTO_PATH := sigs.k8s.io/container-object-storage-interface-api/proto
93+
BUILD_COSI_GO_TMP := $(BUILD_TMP_DIR)/$(BUILD_PROTO_PATH)/cosi.pb.go
94+
BUILD_COSI_GO_GRPC_TMP := $(BUILD_TMP_DIR)/$(BUILD_PROTO_PATH)/cosi_grpc.pb.go
95+
BUILD_COSI_GO_JSON_TMP := $(BUILD_TMP_DIR)/$(BUILD_PROTO_PATH)/cosi.pb.json.go
96+
BUILD_COSI_GO_FAKE_TMP := $(BUILD_TMP_DIR)/fake/cosi.pb.fake.go
22197

222-
# This recipe builds the Go archive from the sources in three steps:
223-
#
224-
# 1. Go get any missing dependencies.
225-
# 2. Cache the packages.
226-
# 3. Build the archive file.
227-
$(COSI_A): $(COSI_GO) $(COSI_GO_JSON) $(COSI_GO_FAKE) $(GENPROTO_BUILD_GO) $(GRPC_BUILD_GO)
228-
go get -v -d ./...
229-
go install ./$(COSI_PKG_SUB)
230-
go build -o "$@" ./$(COSI_PKG_SUB)
231-
232-
generate:
98+
GO_JSON_OPTS := emit_defaults=true
99+
100+
GO_FAKE_OPTS := emit_defaults
101+
GO_FAKE_OPTS := $(GO_FAKE_OPTS),packagePath=sigs.k8s.io/container-object-storage-interface-api/proto
102+
103+
$(COSI_PROTO): $(COSI_SPEC)
233104
echo "// Code generated by make; DO NOT EDIT." > "$(COSI_PROTO)"
234105
cat $(COSI_SPEC) | sed -n -e '/```protobuf$$/,/^```$$/ p' | sed '/^```/d' >> "$(COSI_PROTO)"
235106

236-
build: generate $(COSI_A)
237-
238-
clean:
107+
$(COSI_GO) $(COSI_GO_GRPC) $(COSI_GO_JSON) $(COSI_GO_FAKE): $(COSI_PROTO) $(PROTOC_ALL) $(PROTOC_GEN_GO_FAKE)
108+
mkdir -p "$(BUILD_TMP_DIR)"
109+
mkdir -p "$(BUILD_TMP_DIR)/fake"
110+
$(PROTOC) -I. -I$(PROTOC_TMP_INC) \
111+
--go_out=$(BUILD_TMP_DIR) \
112+
--go-grpc_out=$(BUILD_TMP_DIR) \
113+
--go-json_out=$(GO_JSON_OPTS):$(BUILD_TMP_DIR) \
114+
--gofake_out=$(GO_FAKE_OPTS):$(BUILD_TMP_DIR)/fake \
115+
"$(<F)"
116+
cp -f $(BUILD_COSI_GO_TMP) $(COSI_GO)
117+
cp -f $(BUILD_COSI_GO_GRPC_TMP) $(COSI_GO_GRPC)
118+
cp -f $(BUILD_COSI_GO_JSON_TMP) $(COSI_GO_JSON)
119+
cp -f $(BUILD_COSI_GO_FAKE_TMP) $(COSI_GO_FAKE)
120+
121+
$(BUILD_TMP_COSI_A): $(COSI_GO) $(COSI_GO_GRPC) $(COSI_GO_JSON) $(COSI_GO_FAKE)
122+
go build -o "$@" ./.
123+
124+
generate: $(COSI_PROTO) ## Generate cosi.proto
125+
126+
build: generate $(BUILD_TMP_COSI_A) ## Generate protobuf files, and ensure they build
127+
128+
.PHONY: clean
129+
clean: ## Clean all builds and generated files
239130
go clean -i ./...
240-
rm -rf "$(COSI_PROTO)" "$(COSI_A)" "$(COSI_GO)" "$(COSI_GO_JSON)" "$(COSI_BUILD)"
241-
242-
clobber: clean
243-
rm -fr "$(PROTOC)" "$(PROTOC_TMP_DIR)" "$(PROTOC_GEN_GO)" "$(PROTOC_GEN_GO_JSON)" "$(PROTOC_GEN_GO_FAKE)"
131+
rm -rf "$(BUILD_TMP_DIR)" "$(COSI_PROTO)" "$(COSI_GO)" "$(COSI_GO_GRPC)" "$(COSI_GO_JSON)" "$(COSI_GO_FAKE)"
244132

245-
.PHONY: clean clobber $(GRPC_BUILD_GO) $(GENPROTO_BUILD_GO)
133+
.PHONY: clobber
134+
clobber: clean ## Clean, and remove all cached tooling
135+
rm -fr "$(PROTOC_TMP_DIR)"

0 commit comments

Comments
 (0)