Skip to content

Commit e7538bd

Browse files
aceforeverdaceforeverd
authored
build(proto): improve protobuf generation (#3636)
1. tracking all generated files (.cpp, .h, .java), instead of .cpp file only 2. auto clean all generated files on clean target, fix #891 for proto 3. simplify dependency in java modules, fix #1090 - proto generated files to openmldb-common only, instead of multiple projects - openmldb-native now depends on openmldb-common, no direct dependency to openmldb-common from other modules
1 parent d4f6346 commit e7538bd

File tree

8 files changed

+83
-123
lines changed

8 files changed

+83
-123
lines changed

‎hybridse/src/proto/CMakeLists.txt‎

Lines changed: 40 additions & 77 deletions
Original file line numberDiff line numberDiff line change
@@ -12,80 +12,43 @@
1212
# See the License for the specific language governing permissions and
1313
# limitations under the License.
1414

15-
include_directories(${INCLUDE_DIRECTORIES}
16-
${PROJECT_SOURCE_DIR}/src
17-
${PROJECT_BINARY_DIR}/src)
18-
19-
set(TYPE_PB_CC
20-
${CMAKE_CURRENT_BINARY_DIR}/fe_type.pb.cc)
21-
22-
set(COMMON_PB_CC
23-
${CMAKE_CURRENT_BINARY_DIR}/fe_common.pb.cc
24-
)
25-
26-
set(DBMS_PB_CC
27-
${CMAKE_CURRENT_BINARY_DIR}/dbms.pb.cc)
28-
29-
set(TABLET_PB_CC
30-
${CMAKE_CURRENT_BINARY_DIR}/fe_tablet.pb.cc)
31-
32-
set(BATCH_PB_CC
33-
${CMAKE_CURRENT_BINARY_DIR}/batch.pb.cc)
34-
35-
set(PLAN_PB_CC
36-
${CMAKE_CURRENT_BINARY_DIR}/plan.pb.cc)
37-
38-
add_custom_command(OUTPUT ${COMMON_PB_CC}
39-
COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto
40-
--cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/fe_common.proto
41-
DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/fe_common.proto
42-
)
43-
44-
add_custom_command(OUTPUT ${TYPE_PB_CC}
45-
COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/fe_type.proto
46-
DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/fe_type.proto
47-
)
48-
49-
add_custom_command(OUTPUT ${DBMS_PB_CC}
50-
COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/dbms.proto
51-
DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/dbms.proto
52-
)
53-
54-
add_custom_command(OUTPUT ${TABLET_PB_CC}
55-
COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/fe_tablet.proto
56-
DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/fe_tablet.proto
57-
)
58-
59-
add_custom_command(OUTPUT ${BATCH_PB_CC}
60-
COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto
61-
--cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/batch.proto
62-
DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/batch.proto
63-
)
64-
65-
add_custom_command(OUTPUT ${PLAN_PB_CC}
66-
COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/plan.proto
67-
DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/plan.proto
68-
)
69-
70-
set(HYBRIDSE_PROTO_SRC_FILES
71-
${TABLET_PB_CC}
72-
${DBMS_PB_CC}
73-
${COMMON_PB_CC}
74-
${TYPE_PB_CC}
75-
${BATCH_PB_CC}
76-
${PLAN_PB_CC})
77-
78-
ADD_CUSTOM_TARGET(run_gen_proto DEPENDS ${HYBRIDSE_PROTO_SRC_FILES})
79-
80-
add_library(hybridse_proto OBJECT ${HYBRIDSE_PROTO_SRC_FILES})
81-
add_dependencies(hybridse_proto run_gen_proto)
82-
83-
if (SQL_JAVASDK_ENABLE)
84-
add_custom_target(hybridse_proto_java ALL
85-
COMMAND ${Protobuf_PROTOC_EXECUTABLE}
86-
-I ${PROJECT_SOURCE_DIR}/src/proto
87-
--java_out=${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java
88-
${PROJECT_SOURCE_DIR}/src/proto/*.proto
89-
COMMENT "Generating protobuf java class into ${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java"
90-
)
91-
endif()
15+
include_directories(${INCLUDE_DIRECTORIES} ${PROJECT_SOURCE_DIR}/src
16+
${PROJECT_BINARY_DIR}/src)
17+
18+
set(PROTO_CPP_FILES "")
19+
set(PROTO_FILES "")
20+
function(compile_proto proto_name java_file_suffix_path)
21+
add_custom_command(
22+
OUTPUT
23+
${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.cc
24+
${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.h
25+
${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java/com/_4paradigm/hybridse/${java_file_suffix_path}
26+
COMMAND ${Protobuf_PROTOC_EXECUTABLE}
27+
-I ${PROJECT_SOURCE_DIR}/src/proto
28+
--cpp_out=${CMAKE_CURRENT_BINARY_DIR}
29+
--java_out=${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java
30+
${PROJECT_SOURCE_DIR}/src/proto/${proto_name}.proto
31+
DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/${proto_name}.proto)
32+
list(APPEND PROTO_CPP_FILES ${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.cc)
33+
list(APPEND PROTO_FILES
34+
${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.cc
35+
${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.h
36+
${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java/com/_4paradigm/hybridse/${java_file_suffix_path}
37+
)
38+
set(PROTO_CPP_FILES ${PROTO_CPP_FILES} PARENT_SCOPE)
39+
set(PROTO_FILES ${PROTO_FILES} PARENT_SCOPE)
40+
endfunction(compile_proto)
41+
42+
compile_proto(fe_type type/TypeOuterClass.java)
43+
compile_proto(fe_common common/Common.java)
44+
compile_proto(dbms dbms/DBMS.java)
45+
compile_proto(fe_tablet tablet/Tablet.java)
46+
compile_proto(batch batch/Batch.java)
47+
compile_proto(plan batch/Plan.java)
48+
49+
add_library(hybridse_proto OBJECT ${PROTO_CPP_FILES})
50+
set_property(
51+
TARGET hybridse_proto
52+
APPEND
53+
PROPERTY ADDITIONAL_CLEAN_FILES ${PROTO_FILES}
54+
)

‎hybridse/src/sdk/CMakeLists.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,5 +72,5 @@ if(SQL_JAVASDK_ENABLE)
7272
COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_SOURCE_DIR}/java/hybridse-native/src/main/resources/
7373
COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:hybridse_jsdk_core> ${CMAKE_SOURCE_DIR}/java/hybridse-native/src/main/resources/
7474
COMMENT "copy hybridse native so")
75-
add_dependencies(cp_hybridse_native_so hybridse_proto_java)
75+
add_dependencies(cp_hybridse_native_so hybridse_proto)
7676
endif()

‎java/openmldb-batch/pom.xml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -210,12 +210,6 @@
210210
</exclusions>
211211
</dependency>
212212

213-
<dependency>
214-
<groupId>com.4paradigm.openmldb</groupId>
215-
<artifactId>openmldb-common</artifactId>
216-
<version>${project.parent.version}</version>
217-
</dependency>
218-
219213
<dependency>
220214
<groupId>com.4paradigm.openmldb</groupId>
221215
<artifactId>openmldb-spark-connector</artifactId>

‎java/openmldb-jdbc/pom.xml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,6 @@
3838
<artifactId>openmldb-native</artifactId>
3939
<version>${variant.native.version}</version>
4040
</dependency>
41-
<dependency>
42-
<groupId>com.4paradigm.openmldb</groupId>
43-
<artifactId>openmldb-common</artifactId>
44-
<version>${project.parent.version}</version>
45-
</dependency>
4641

4742
<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
4843
<dependency>

‎java/openmldb-native/pom.xml‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,12 @@
2121
</properties>
2222

2323
<dependencies>
24-
<dependency>
24+
<dependency>
25+
<groupId>com.4paradigm.openmldb</groupId>
26+
<artifactId>openmldb-common</artifactId>
27+
<version>${project.parent.version}</version>
28+
</dependency>
29+
<dependency>
2530
<groupId>org.slf4j</groupId>
2631
<artifactId>slf4j-api</artifactId>
2732
<version>1.7.21</version>

‎java/openmldb-synctool/pom.xml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,6 @@
2828
<scope>test</scope>
2929
</dependency>
3030

31-
<dependency>
32-
<groupId>com.4paradigm.openmldb</groupId>
33-
<artifactId>openmldb-common</artifactId>
34-
<version>${project.parent.version}</version>
35-
</dependency>
3631
<dependency>
3732
<groupId>com.4paradigm.openmldb</groupId>
3833
<artifactId>openmldb-jdbc</artifactId>

‎java/openmldb-taskmanager/pom.xml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -159,12 +159,6 @@
159159
<version>${project.parent.version}</version>
160160
</dependency>
161161

162-
<dependency>
163-
<groupId>com.4paradigm.openmldb</groupId>
164-
<artifactId>openmldb-common</artifactId>
165-
<version>${project.parent.version}</version>
166-
</dependency>
167-
168162
<dependency>
169163
<groupId>com.4paradigm.openmldb</groupId>
170164
<artifactId>openmldb-batchjob</artifactId>

‎src/CMakeLists.txt‎

Lines changed: 36 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -23,19 +23,26 @@ add_subdirectory(statistics)
2323
file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/src/proto)
2424

2525
set(PROTO_FILES "")
26-
function(compile_proto proto_name project_dir)
27-
add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc
28-
COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto
29-
--cpp_out=${CMAKE_BINARY_DIR}/src/proto
30-
--java_out=${project_dir}/java/openmldb-native/src/main/java
31-
--java_out=${project_dir}/java/openmldb-import/src/main/java
32-
--java_out=${project_dir}/java/openmldb-taskmanager/src/main/java
33-
--java_out=${project_dir}/java/openmldb-common/src/main/java
34-
--java_out=${project_dir}/java/openmldb-synctool/src/main/java
35-
${project_dir}/src/proto/${proto_name}.proto
36-
DEPENDS ${project_dir}/src/proto/${proto_name}.proto)
37-
list(APPEND PROTO_FILES ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc)
38-
set(PROTO_FILES ${PROTO_FILES} PARENT_SCOPE)
26+
set(PROTO_CPP_FILES "")
27+
function(compile_proto proto_name java_out_file_name)
28+
# will rerun if DEPENDS changed or OUTPUT not exists
29+
add_custom_command(
30+
OUTPUT ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc
31+
${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.h
32+
${CMAKE_SOURCE_DIR}/java/openmldb-common/src/main/java/com/_4paradigm/openmldb/proto/${java_out_file_name}.java
33+
COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto
34+
--cpp_out=${CMAKE_BINARY_DIR}/src/proto
35+
--java_out=${CMAKE_SOURCE_DIR}/java/openmldb-common/src/main/java
36+
${CMAKE_SOURCE_DIR}/src/proto/${proto_name}.proto
37+
DEPENDS ${CMAKE_SOURCE_DIR}/src/proto/${proto_name}.proto)
38+
list(APPEND PROTO_CPP_FILES ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc)
39+
list(APPEND PROTO_FILES
40+
${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc
41+
${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.h
42+
${CMAKE_SOURCE_DIR}/java/openmldb-common/src/main/java/com/_4paradigm/openmldb/proto/${java_out_file_name}.java
43+
)
44+
set(PROTO_CPP_FILES ${PROTO_CPP_FILES} PARENT_SCOPE)
45+
set(PROTO_FILES ${PROTO_FILES} PARENT_SCOPE)
3946
endfunction(compile_proto)
4047

4148
function(compile_lib LIB_NAME DIR DEPEND_FILE_LIST)
@@ -98,16 +105,23 @@ function(compile_test DIR)
98105
set(test_list ${test_list} PARENT_SCOPE)
99106
endfunction(compile_test)
100107

101-
compile_proto(type ${PROJECT_SOURCE_DIR})
102-
compile_proto(common ${PROJECT_SOURCE_DIR})
103-
compile_proto(tablet ${PROJECT_SOURCE_DIR})
104-
compile_proto(name_server ${PROJECT_SOURCE_DIR})
105-
compile_proto(sql_procedure ${PROJECT_SOURCE_DIR})
106-
compile_proto(api_server ${PROJECT_SOURCE_DIR})
107-
compile_proto(taskmanager ${PROJECT_SOURCE_DIR})
108-
compile_proto(data_sync ${PROJECT_SOURCE_DIR})
108+
compile_proto(type Type)
109+
compile_proto(common Common)
110+
compile_proto(tablet Tablet)
111+
compile_proto(name_server NS)
112+
compile_proto(sql_procedure SQLProcedure)
113+
compile_proto(api_server ApiServer)
114+
compile_proto(taskmanager TaskManager)
115+
compile_proto(data_sync DataSync)
116+
117+
add_library(openmldb_proto STATIC ${PROTO_CPP_FILES})
118+
# clean generated proto file, including '{cmake_source}/java/openmldb-common'
119+
set_property(
120+
TARGET openmldb_proto
121+
APPEND
122+
PROPERTY ADDITIONAL_CLEAN_FILES ${PROTO_FILES}
123+
)
109124

110-
add_library(openmldb_proto STATIC ${PROTO_FILES})
111125
add_library(openmldb_flags STATIC flags.cc)
112126

113127
compile_lib(openmldb_codec codec "")

0 commit comments

Comments
 (0)