Skip to content

Commit fa2e650

Browse files
committed
added generation of python code for some of the protos and other supporting changes
Signed-off-by: William Woodall <[email protected]>
1 parent 8da5bcb commit fa2e650

File tree

4 files changed

+260
-13
lines changed

4 files changed

+260
-13
lines changed

intrinsic_sdk_cmake/CMakeLists.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@ include(cmake/sdk_imw_zenoh.cmake)
3434
# Build the sdk C++ sources into a library.
3535
include(cmake/sdk.cmake)
3636

37+
# Collect and install the sdk Python sources.
38+
include(cmake/sdk_python.cmake)
39+
3740
# Extract tool binaries from bazel build of sdk.
3841
include(cmake/sdk_tools.cmake)
3942

intrinsic_sdk_cmake/cmake/sdk_protos.cmake

Lines changed: 74 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/protos_gen)
2+
file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/protos_gen_py)
23

34
file(GLOB_RECURSE intrinsic_proto_SRCS "${intrinsic_sdk_SOURCE_DIR}/**/*.proto")
45
list(FILTER intrinsic_proto_SRCS EXCLUDE REGEX "_test\\.proto$")
56

67
set(exclude_SRCS
7-
intrinsic/icon/proto/service.proto
8+
intrinsic/icon/proto/service.proto
89
third_party/ros2/ros_interfaces/jazzy/diagnostic_msgs/srv/self_test.proto
910
third_party/ros2/ros_interfaces/jazzy/diagnostic_msgs/srv/add_diagnostics.proto
1011
third_party/ros2/ros_interfaces/jazzy/rcl_interfaces/srv/set_parameters.proto
@@ -77,22 +78,28 @@ set(grpc_gateway_SRCS
7778
${grpc_gateway_SOURCE_DIR}/protoc-gen-openapiv2/options/openapiv2.proto
7879
)
7980

80-
# Generate code from protos and build into a library.
81-
add_library(intrinsic_sdk_protos STATIC
81+
set(sdk_protos
8282
${intrinsic_proto_SRCS}
8383
${googleapis_SRCS}
8484
${grpc_gateway_SRCS}
8585
${grpc_SRCS}
8686
)
87+
set(sdk_proto_import_dirs
88+
${intrinsic_sdk_SOURCE_DIR}
89+
${googleapis_SOURCE_DIR}
90+
${grpc_gateway_SOURCE_DIR}
91+
${grpc_SOURCE_DIR}
92+
)
93+
94+
# Generate code from protos and build into a library.
95+
add_library(intrinsic_sdk_protos STATIC
96+
${sdk_protos}
97+
)
8798
set_property(TARGET intrinsic_sdk_protos PROPERTY POSITION_INDEPENDENT_CODE ON)
8899
protobuf_generate(
89100
TARGET intrinsic_sdk_protos
90101
LANGUAGE cpp
91-
IMPORT_DIRS
92-
${intrinsic_sdk_SOURCE_DIR}
93-
${googleapis_SOURCE_DIR}
94-
${grpc_gateway_SOURCE_DIR}
95-
${grpc_SOURCE_DIR}
102+
IMPORT_DIRS ${sdk_proto_import_dirs}
96103
PROTOC_OPTIONS --experimental_editions
97104
PROTOC_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/protos_gen
98105
)
@@ -114,11 +121,7 @@ protobuf_generate(
114121
TARGET intrinsic_sdk_services
115122
LANGUAGE grpc
116123
PLUGIN protoc-gen-grpc=$<TARGET_FILE:gRPC::grpc_cpp_plugin>
117-
IMPORT_DIRS
118-
${intrinsic_sdk_SOURCE_DIR}
119-
${googleapis_SOURCE_DIR}
120-
${grpc_gateway_SOURCE_DIR}
121-
${grpc_SOURCE_DIR}
124+
IMPORT_DIRS ${sdk_proto_import_dirs}
122125
PROTOC_OPTIONS --experimental_editions
123126
PROTOC_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/protos_gen
124127
GENERATE_EXTENSIONS .grpc.pb.h .grpc.pb.cc
@@ -131,6 +134,64 @@ target_include_directories(intrinsic_sdk_services
131134
target_link_libraries(intrinsic_sdk_services PUBLIC intrinsic_sdk_protos)
132135
set_property(TARGET intrinsic_sdk_services PROPERTY POSITION_INDEPENDENT_CODE ON)
133136

137+
# Generate Python code from the protos.
138+
protobuf_generate(
139+
LANGUAGE python
140+
PROTOS ${sdk_protos}
141+
IMPORT_DIRS ${sdk_proto_import_dirs}
142+
PROTOC_OUT_DIR ${CMAKE_CURRENT_BINARY_DIR}/protos_gen_py
143+
OUT_VAR sdk_protos_python_sources
144+
)
145+
# Create empty __init__.py files for each python package generated by protoc.
146+
set(python_package_directories "")
147+
foreach(generated_python_file_maybe_hyphen ${sdk_protos_python_sources})
148+
file(RELATIVE_PATH rel_generated_python_file_maybe_hyphen
149+
"${CMAKE_CURRENT_BINARY_DIR}/protos_gen_py"
150+
"${generated_python_file_maybe_hyphen}"
151+
)
152+
string(REPLACE "-" "_" rel_generated_python_file "${rel_generated_python_file_maybe_hyphen}")
153+
set(generated_python_file "${CMAKE_CURRENT_BINARY_DIR}/protos_gen_py/${rel_generated_python_file}")
154+
get_filename_component(generated_python_file_dir "${generated_python_file}" DIRECTORY)
155+
list(APPEND python_package_directories "${generated_python_file_dir}")
156+
endforeach()
157+
list(REMOVE_DUPLICATES python_package_directories)
158+
set(python_init_py_files "")
159+
# add __init__.py to "leaf" directories
160+
foreach(python_package_directory ${python_package_directories})
161+
file(MAKE_DIRECTORY "${python_package_directory}")
162+
set(python_init_py_file "${python_package_directory}/__init__.py")
163+
list(APPEND python_init_py_files "${python_init_py_file}")
164+
file(TOUCH "${python_init_py_file}")
165+
endforeach()
166+
# second pass to add __init__.py to intermediate directories now that the folder structure exists.
167+
file(GLOB_RECURSE leaf_python_init_files_and_dirs
168+
LIST_DIRECTORIES true
169+
RELATIVE "${CMAKE_CURRENT_BINARY_DIR}/protos_gen_py/"
170+
"${CMAKE_CURRENT_BINARY_DIR}/protos_gen_py/*/__init__.py"
171+
)
172+
foreach(leaf_python_init_file_or_dir ${leaf_python_init_files_and_dirs})
173+
if(IS_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/protos_gen_py/${leaf_python_init_file_or_dir}")
174+
set(python_init_py_file "${CMAKE_CURRENT_BINARY_DIR}/protos_gen_py/${leaf_python_init_file_or_dir}/__init__.py")
175+
list(APPEND python_init_py_files "${python_init_py_file}")
176+
file(TOUCH "${python_init_py_file}")
177+
endif()
178+
endforeach()
179+
# final pass because glob doesn't get the root folders correctly.
180+
file(GLOB root_directories
181+
LIST_DIRECTORIES true
182+
"${CMAKE_CURRENT_BINARY_DIR}/protos_gen_py/*"
183+
)
184+
foreach(root_directory ${root_directories})
185+
if(IS_DIRECTORY "${root_directory}")
186+
set(python_init_py_file "${root_directory}/__init__.py")
187+
list(APPEND python_init_py_files "${python_init_py_file}")
188+
file(TOUCH "${python_init_py_file}")
189+
endif()
190+
endforeach()
191+
list(REMOVE_DUPLICATES python_init_py_files)
192+
add_custom_target(intrinsic_sdk_protos_python ALL DEPENDS ${sdk_protos_python_sources} ${python_init_py_files})
193+
# Generated Python code is installed with the Python sdk files later.
194+
134195
# Generate a descriptor set.
135196
add_custom_command(
136197
OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/intrinsic_proto.desc
Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
# Glob the source files and then exclude files that don't make sense to be in the glob.
2+
# Note(wjwwood): we know this isn't the "right" way to do this, but it helps us catch
3+
# additions to the sdk when it is upgraded until we have a more structured approach
4+
# to extract build targets out of bazel.
5+
file(GLOB_RECURSE intrinsic_python_SRCS
6+
RELATIVE "${intrinsic_sdk_SOURCE_DIR}"
7+
"${intrinsic_sdk_SOURCE_DIR}/**/*.py"
8+
)
9+
list(FILTER intrinsic_python_SRCS EXCLUDE REGEX "/\\.github/")
10+
list(FILTER intrinsic_python_SRCS EXCLUDE REGEX "/examples/")
11+
list(FILTER intrinsic_python_SRCS EXCLUDE REGEX "_test\\.py$")
12+
13+
# Overlay sdk Python files on top of generated protobuf Python files.
14+
set(python_sdk_destination_dir "${CMAKE_CURRENT_BINARY_DIR}/protos_gen_py")
15+
set(copied_python_sdk_files "")
16+
foreach(sdk_python_file ${intrinsic_python_SRCS})
17+
# Ensure the target directory exists.
18+
get_filename_component(sdk_python_file_dir "${sdk_python_file}" DIRECTORY)
19+
set(python_sdk_file_dest_dir "${python_sdk_destination_dir}/${sdk_python_file_dir}")
20+
file(MAKE_DIRECTORY "${python_sdk_file_dest_dir}")
21+
# Ensure every python package (directory in the output) has at least an empty __init__.py.
22+
set(python_init_py_file "${python_sdk_file_dest_dir}/__init__.py")
23+
if(NOT EXISTS "${python_init_py_file}")
24+
file(TOUCH "${python_init_py_file}")
25+
endif()
26+
# Copy the file over.
27+
set(python_sdk_file_destination "${python_sdk_destination_dir}/${sdk_python_file}")
28+
file(COPY_FILE "${intrinsic_sdk_SOURCE_DIR}/${sdk_python_file}" "${python_sdk_file_destination}" ONLY_IF_DIFFERENT)
29+
list(APPEND copied_python_sdk_files "${python_sdk_file_destination}")
30+
endforeach()
31+
32+
# Check to see if there are any collisions between the protobuf generation and copied files.
33+
foreach(copied_python_sdk_file ${copied_python_sdk_files})
34+
if("${copied_python_sdk_file}" IN_LIST sdk_protos_python_sources)
35+
message(FATAL_ERROR "Python sdk file '${copied_python_sdk_file}' would be overwritten by protoc.")
36+
endif()
37+
endforeach()
38+
39+
# Install select (for now) python packages.
40+
# Note(wjwwood): More work needs to be done to wrangle all of the generated protobuf code into
41+
# a single python project and/or distrbute them separately by namespace.
42+
set(python_packages_to_install
43+
# "google" # This one is disabled because it collides with google.protobuf which comes from elsewhere.
44+
"intrinsic"
45+
"protoc_gen_openapiv2" # This one should be fixed to have a better python package name
46+
# "src" # This one is disabled because it isn't being used atm and is a weird layout
47+
# "third_party" # This one contains protobuf versions of ROS messages and is unused atm
48+
)
49+
50+
macro(_get_python_install_dir)
51+
if(NOT DEFINED PYTHON_INSTALL_DIR)
52+
# avoid storing backslash in cached variable since CMake will interpret it as escape character
53+
# This auto detection code uses the same logic as get_python_install_path() in colcon-core
54+
set(_python_code
55+
"\
56+
import os
57+
import sysconfig
58+
schemes = sysconfig.get_scheme_names()
59+
kwargs = {'vars': {'base': '${CMAKE_INSTALL_PREFIX}'}}
60+
if 'deb_system' in schemes or 'osx_framework_library' in schemes:
61+
kwargs['scheme'] = 'posix_prefix'
62+
elif 'rpm_prefix' in schemes:
63+
kwargs['scheme'] = 'rpm_prefix'
64+
print(os.path.relpath(sysconfig.get_path('purelib', **kwargs), start='${CMAKE_INSTALL_PREFIX}').replace(os.sep, '/'))"
65+
)
66+
get_executable_path(_python_interpreter Python3::Interpreter CONFIGURE)
67+
execute_process(
68+
COMMAND
69+
"${_python_interpreter}"
70+
"-c"
71+
"${_python_code}"
72+
OUTPUT_VARIABLE _output
73+
RESULT_VARIABLE _result
74+
OUTPUT_STRIP_TRAILING_WHITESPACE
75+
)
76+
if(NOT _result EQUAL 0)
77+
message(FATAL_ERROR
78+
"execute_process(${_python_interpreter} -c '${_python_code}') returned "
79+
"error code ${_result}")
80+
endif()
81+
82+
set(PYTHON_INSTALL_DIR
83+
"${_output}"
84+
CACHE INTERNAL
85+
"The directory for Python library installation. This needs to be in PYTHONPATH when 'setup.py install' is called.")
86+
endif()
87+
endmacro()
88+
89+
_get_python_install_dir()
90+
91+
function(_install_python_package package_name)
92+
cmake_parse_arguments(
93+
ARG "SKIP_COMPILE" "PACKAGE_DIR;DESTINATION" "" ${ARGN})
94+
if(ARG_UNPARSED_ARGUMENTS)
95+
message(FATAL_ERROR "_install_python_package() called with unused "
96+
"arguments: ${ARG_UNPARSED_ARGUMENTS}")
97+
endif()
98+
99+
if(NOT ARG_PACKAGE_DIR)
100+
message(FATAL_ERROR "ARG PACKAGE_DIR required")
101+
endif()
102+
103+
if(NOT ARG_DESTINATION)
104+
if(NOT PYTHON_INSTALL_DIR)
105+
message(FATAL_ERROR "_install_python_package() variable 'PYTHON_INSTALL_DIR' must not be empty")
106+
endif()
107+
set(ARG_DESTINATION ${PYTHON_INSTALL_DIR})
108+
endif()
109+
110+
set(build_dir "${CMAKE_CURRENT_BINARY_DIR}/_install_python_package/${package_name}")
111+
112+
string(CONFIGURE "\
113+
from setuptools import find_packages
114+
from setuptools import setup
115+
116+
setup(
117+
name='${package_name}',
118+
version='${sdk_version}',
119+
packages=find_packages(
120+
include=('${package_name}', '${package_name}.*')),
121+
)
122+
" setup_py_content)
123+
124+
file(GENERATE
125+
OUTPUT "${build_dir}/setup.py"
126+
CONTENT "${setup_py_content}"
127+
)
128+
129+
add_custom_target(
130+
_install_python_package_copy_${package_name}
131+
COMMAND ${CMAKE_COMMAND} -E copy_directory
132+
"${ARG_PACKAGE_DIR}" "${build_dir}/${package_name}"
133+
)
134+
set(egg_dependencies _install_python_package_copy_${package_name})
135+
136+
get_executable_path(python_interpreter Python3::Interpreter BUILD)
137+
138+
add_custom_target(
139+
_install_python_package_build_${package_name}_egg ALL
140+
COMMAND ${python_interpreter} setup.py egg_info
141+
WORKING_DIRECTORY "${build_dir}"
142+
DEPENDS ${egg_dependencies}
143+
)
144+
145+
set(python_version "py${Python3_VERSION_MAJOR}.${Python3_VERSION_MINOR}")
146+
147+
set(egg_name "${package_name}")
148+
set(egg_install_name "${egg_name}-${ARG_VERSION}")
149+
set(egg_install_name "${egg_install_name}-${python_version}")
150+
151+
install(
152+
DIRECTORY "${build_dir}/${egg_name}.egg-info/"
153+
DESTINATION "${ARG_DESTINATION}/${egg_install_name}.egg-info"
154+
)
155+
156+
install(
157+
DIRECTORY "${ARG_PACKAGE_DIR}/"
158+
DESTINATION "${ARG_DESTINATION}/${package_name}"
159+
PATTERN "*.pyc" EXCLUDE
160+
PATTERN "__pycache__" EXCLUDE
161+
)
162+
163+
if(NOT ARG_SKIP_COMPILE)
164+
get_executable_path(python_interpreter_config Python3::Interpreter CONFIGURE)
165+
# compile Python files
166+
install(CODE
167+
"execute_process(
168+
COMMAND
169+
\"${python_interpreter_config}\" \"-m\" \"compileall\"
170+
\"${CMAKE_INSTALL_PREFIX}/${ARG_DESTINATION}/${package_name}\"
171+
)"
172+
)
173+
endif()
174+
endfunction()
175+
176+
foreach(python_package_to_install ${python_packages_to_install})
177+
_install_python_package(
178+
${python_package_to_install}
179+
PACKAGE_DIR "${CMAKE_CURRENT_BINARY_DIR}/protos_gen_py/${python_package_to_install}"
180+
)
181+
endforeach()

intrinsic_sdk_cmake/package.xml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@
3131
<depend>pybind11_protobuf_vendor</depend>
3232
<depend>opencensus_cpp_vendor</depend>
3333
<depend>ortools_vendor</depend>
34+
<depend>python3-protobuf</depend>
35+
<depend>python3-grpcio</depend>
3436

3537
<export>
3638
<build_type>cmake</build_type>

0 commit comments

Comments
 (0)