Skip to content

Commit f6ded0b

Browse files
maksleventalmakslevental
authored
[MLIR][Python] don't generate type stubs when cross-compiling (#160793)
Stubgen doesn't work when cross-compiling (stubgen will run in the host interpreter and then fail to find the extension module for the host arch). So disable it when `CMAKE_CROSSCOMPILING=ON`.
1 parent be23cdc commit f6ded0b

File tree

2 files changed

+145
-129
lines changed

2 files changed

+145
-129
lines changed

‎mlir/examples/standalone/python/CMakeLists.txt‎

Lines changed: 58 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -74,81 +74,88 @@ add_mlir_python_common_capi_library(StandalonePythonCAPI
7474

7575
set(StandalonePythonModules_ROOT_PREFIX "${MLIR_BINARY_DIR}/${MLIR_BINDINGS_PYTHON_INSTALL_PREFIX}")
7676

77-
# Everything here is very tightly coupled. See the ample descriptions at the bottom of
78-
# mlir/python/CMakeLists.txt.
79-
80-
# For a non-external projects build (e.g., installed distro) the type gen targets for the core _mlir module
81-
# need to be re-declared. On the contrary, for an external projects build, the MLIRPythonExtension.Core.type_stub_gen
82-
# target already exists and can just be added to DECLARED_SOURCES (see below).
83-
if(NOT EXTERNAL_PROJECT_BUILD)
84-
set(_core_type_stub_sources
85-
_mlir/__init__.pyi
86-
_mlir/ir.pyi
87-
_mlir/passmanager.pyi
88-
_mlir/rewrite.pyi
89-
)
90-
get_target_property(_core_extension_srcs MLIRPythonExtension.Core INTERFACE_SOURCES)
77+
if(NOT CMAKE_CROSSCOMPILING)
78+
# Everything here is very tightly coupled. See the ample descriptions at the bottom of
79+
# mlir/python/CMakeLists.txt.
80+
81+
# For a non-external projects build (e.g., installed distro) the type gen targets for the core _mlir module
82+
# need to be re-declared. On the contrary, for an external projects build, the MLIRPythonExtension.Core.type_stub_gen
83+
# target already exists and can just be added to DECLARED_SOURCES (see below).
84+
if(NOT EXTERNAL_PROJECT_BUILD)
85+
set(_core_type_stub_sources
86+
_mlir/__init__.pyi
87+
_mlir/ir.pyi
88+
_mlir/passmanager.pyi
89+
_mlir/rewrite.pyi
90+
)
91+
get_target_property(_core_extension_srcs MLIRPythonExtension.Core INTERFACE_SOURCES)
92+
mlir_generate_type_stubs(
93+
MODULE_NAME _mlir
94+
DEPENDS_TARGETS StandalonePythonModules.extension._mlir.dso
95+
OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs"
96+
OUTPUTS "${_core_type_stub_sources}"
97+
DEPENDS_TARGET_SRC_DEPS "${_core_extension_srcs}"
98+
IMPORT_PATHS "${StandalonePythonModules_ROOT_PREFIX}/_mlir_libs"
99+
VERBOSE
100+
)
101+
set(_mlir_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}")
102+
103+
list(TRANSFORM _core_type_stub_sources PREPEND "_mlir_libs/")
104+
declare_mlir_python_sources(
105+
StandalonePythonExtension.Core.type_stub_gen
106+
ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs"
107+
ADD_TO_PARENT StandalonePythonSources
108+
SOURCES "${_core_type_stub_sources}"
109+
)
110+
endif()
111+
112+
get_target_property(_standalone_extension_srcs StandalonePythonSources.NanobindExtension INTERFACE_SOURCES)
91113
mlir_generate_type_stubs(
92-
MODULE_NAME _mlir
93-
DEPENDS_TARGETS StandalonePythonModules.extension._mlir.dso
114+
MODULE_NAME mlir_standalone._mlir_libs._standaloneDialectsNanobind
115+
DEPENDS_TARGETS
116+
StandalonePythonModules.extension._mlir.dso
117+
StandalonePythonModules.extension._standaloneDialectsNanobind.dso
94118
OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs"
95-
OUTPUTS "${_core_type_stub_sources}"
96-
DEPENDS_TARGET_SRC_DEPS "${_core_extension_srcs}"
97-
IMPORT_PATHS "${StandalonePythonModules_ROOT_PREFIX}/_mlir_libs"
98-
VERBOSE
119+
OUTPUTS
120+
_standaloneDialectsNanobind/__init__.pyi
121+
_standaloneDialectsNanobind/standalone.pyi
122+
DEPENDS_TARGET_SRC_DEPS "${_standalone_extension_srcs}"
123+
IMPORT_PATHS "${StandalonePythonModules_ROOT_PREFIX}/.."
99124
)
100-
set(_mlir_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}")
125+
set(_standaloneDialectsNanobind_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}")
101126

102-
list(TRANSFORM _core_type_stub_sources PREPEND "_mlir_libs/")
103127
declare_mlir_python_sources(
104-
StandalonePythonExtension.Core.type_stub_gen
128+
StandalonePythonSources.type_stub_gen
105129
ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs"
106130
ADD_TO_PARENT StandalonePythonSources
107-
SOURCES "${_core_type_stub_sources}"
131+
SOURCES
132+
_mlir_libs/_standaloneDialectsNanobind/__init__.pyi
133+
_mlir_libs/_standaloneDialectsNanobind/standalone.pyi
108134
)
109135
endif()
110136

111-
get_target_property(_standalone_extension_srcs StandalonePythonSources.NanobindExtension INTERFACE_SOURCES)
112-
mlir_generate_type_stubs(
113-
MODULE_NAME mlir_standalone._mlir_libs._standaloneDialectsNanobind
114-
DEPENDS_TARGETS
115-
StandalonePythonModules.extension._mlir.dso
116-
StandalonePythonModules.extension._standaloneDialectsNanobind.dso
117-
OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs"
118-
OUTPUTS
119-
_standaloneDialectsNanobind/__init__.pyi
120-
_standaloneDialectsNanobind/standalone.pyi
121-
DEPENDS_TARGET_SRC_DEPS "${_standalone_extension_srcs}"
122-
IMPORT_PATHS "${StandalonePythonModules_ROOT_PREFIX}/.."
123-
)
124-
set(_standaloneDialectsNanobind_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}")
125-
126-
declare_mlir_python_sources(
127-
StandalonePythonSources.type_stub_gen
128-
ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs"
129-
ADD_TO_PARENT StandalonePythonSources
130-
SOURCES
131-
_mlir_libs/_standaloneDialectsNanobind/__init__.pyi
132-
_mlir_libs/_standaloneDialectsNanobind/standalone.pyi
133-
)
134137
set(_declared_sources
135138
StandalonePythonSources
136139
MLIRPythonSources.Core
137140
MLIRPythonSources.Dialects.builtin
138141
)
139142
# For an external projects build, the MLIRPythonExtension.Core.type_stub_gen
140143
# target already exists and can just be added to DECLARED_SOURCES.
141-
if(EXTERNAL_PROJECT_BUILD)
144+
if(EXTERNAL_PROJECT_BUILD AND (NOT CMAKE_CROSSCOMPILING))
142145
list(APPEND _declared_sources MLIRPythonExtension.Core.type_stub_gen)
143146
endif()
147+
144148
add_mlir_python_modules(StandalonePythonModules
145149
ROOT_PREFIX "${StandalonePythonModules_ROOT_PREFIX}"
146150
INSTALL_PREFIX "${MLIR_BINDINGS_PYTHON_INSTALL_PREFIX}"
147151
DECLARED_SOURCES "${_declared_sources}"
148152
COMMON_CAPI_LINK_LIBS
149153
StandalonePythonCAPI
150154
)
151-
if(NOT EXTERNAL_PROJECT_BUILD)
152-
add_dependencies(StandalonePythonModules "${_mlir_typestub_gen_target}")
155+
156+
if(NOT CMAKE_CROSSCOMPILING)
157+
if(NOT EXTERNAL_PROJECT_BUILD)
158+
add_dependencies(StandalonePythonModules "${_mlir_typestub_gen_target}")
159+
endif()
160+
add_dependencies(StandalonePythonModules "${_standaloneDialectsNanobind_typestub_gen_target}")
153161
endif()
154-
add_dependencies(StandalonePythonModules "${_standaloneDialectsNanobind_typestub_gen_target}")

‎mlir/python/CMakeLists.txt‎

Lines changed: 87 additions & 78 deletions
Original file line numberDiff line numberDiff line change
@@ -873,104 +873,113 @@ if(NOT LLVM_ENABLE_IDE)
873873
)
874874
endif()
875875

876-
# _mlir stubgen
877-
# Note: All this needs to come before add_mlir_python_modules(MLIRPythonModules so that the install targets for the
878-
# generated type stubs get created.
879-
880-
set(_core_type_stub_sources
881-
_mlir/__init__.pyi
882-
_mlir/ir.pyi
883-
_mlir/passmanager.pyi
884-
_mlir/rewrite.pyi
885-
)
886-
887-
# Note 1: INTERFACE_SOURCES is a genex ($<BUILD_INTERFACE> $<INSTALL_INTERFACE>)
888-
# which will be evaluated by file(GENERATE ...) inside mlir_generate_type_stubs. This will evaluate to the correct
889-
# thing in the build dir (i.e., actual source dir paths) and in the install dir
890-
# (where it's a conventional path; see install/lib/cmake/mlir/MLIRTargets.cmake).
891-
#
892-
# Note 2: MLIRPythonExtension.Core is the target that is defined using target_sources(INTERFACE)
893-
# **NOT** MLIRPythonModules.extension._mlir.dso. So be sure to use the correct target!
894-
get_target_property(_core_extension_srcs MLIRPythonExtension.Core INTERFACE_SOURCES)
895-
896-
# Why is MODULE_NAME _mlir here but mlir._mlir_libs._mlirPythonTestNanobind below???
897-
# The _mlir extension can be imported independently of any other python code and/or extension modules.
898-
# I.e., you could do `cd $MLIRPythonModules_ROOT_PREFIX/_mlir_libs && python -c "import _mlir"` (try it!).
899-
# _mlir is also (currently) the only extension for which this is possible because dialect extensions modules,
900-
# which generally make use of `mlir_value_subclass/mlir_type_subclass/mlir_attribute_subclass`, perform an
901-
# `import mlir` right when they're loaded (see the mlir_*_subclass ctors in NanobindAdaptors.h).
902-
# Note, this also why IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/_mlir_libs" here while below
903-
# "${MLIRPythonModules_ROOT_PREFIX}/.." (because MLIR_BINDINGS_PYTHON_INSTALL_PREFIX, by default, ends at mlir).
904-
#
905-
# Further note: this function creates file targets like
906-
# "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs/_mlir/__init__.pyi". These must match the file targets
907-
# that declare_mlir_python_sources expects, which are like "${ROOT_DIR}/${WHATEVER_SOURCE}".
908-
# This is why _mlir_libs is prepended below.
909-
mlir_generate_type_stubs(
910-
MODULE_NAME _mlir
911-
DEPENDS_TARGETS MLIRPythonModules.extension._mlir.dso
912-
OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs"
913-
OUTPUTS "${_core_type_stub_sources}"
914-
DEPENDS_TARGET_SRC_DEPS "${_core_extension_srcs}"
915-
IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/_mlir_libs"
916-
)
917-
set(_mlir_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}")
918-
919-
list(TRANSFORM _core_type_stub_sources PREPEND "_mlir_libs/")
920-
# Note, we do not do ADD_TO_PARENT here so that the type stubs are not associated (as mlir_DEPENDS) with
921-
# MLIRPythonSources.Core (or something) when a distro is installed/created. Otherwise they would not be regenerated
922-
# by users of the distro (the stubs are still installed in the distro - they are just not added to mlir_DEPENDS).
923-
declare_mlir_python_sources(
924-
MLIRPythonExtension.Core.type_stub_gen
925-
ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs"
926-
SOURCES "${_core_type_stub_sources}"
927-
)
928-
929-
# _mlirPythonTestNanobind stubgen
876+
# Stubgen doesn't work when cross-compiling (stubgen will run in the host interpreter and then fail
877+
# to find the extension module for the host arch).
878+
if(NOT CMAKE_CROSSCOMPILING)
879+
# _mlir stubgen
880+
# Note: All this needs to come before add_mlir_python_modules(MLIRPythonModules so that the install targets for the
881+
# generated type stubs get created.
882+
883+
set(_core_type_stub_sources
884+
_mlir/__init__.pyi
885+
_mlir/ir.pyi
886+
_mlir/passmanager.pyi
887+
_mlir/rewrite.pyi
888+
)
930889

931-
if(MLIR_INCLUDE_TESTS)
932-
get_target_property(_test_extension_srcs MLIRPythonTestSources.PythonTestExtensionNanobind INTERFACE_SOURCES)
890+
# Note 1: INTERFACE_SOURCES is a genex ($<BUILD_INTERFACE> $<INSTALL_INTERFACE>)
891+
# which will be evaluated by file(GENERATE ...) inside mlir_generate_type_stubs. This will evaluate to the correct
892+
# thing in the build dir (i.e., actual source dir paths) and in the install dir
893+
# (where it's a conventional path; see install/lib/cmake/mlir/MLIRTargets.cmake).
894+
#
895+
# Note 2: MLIRPythonExtension.Core is the target that is defined using target_sources(INTERFACE)
896+
# **NOT** MLIRPythonModules.extension._mlir.dso. So be sure to use the correct target!
897+
get_target_property(_core_extension_srcs MLIRPythonExtension.Core INTERFACE_SOURCES)
898+
899+
# Why is MODULE_NAME _mlir here but mlir._mlir_libs._mlirPythonTestNanobind below???
900+
# The _mlir extension can be imported independently of any other python code and/or extension modules.
901+
# I.e., you could do `cd $MLIRPythonModules_ROOT_PREFIX/_mlir_libs && python -c "import _mlir"` (try it!).
902+
# _mlir is also (currently) the only extension for which this is possible because dialect extensions modules,
903+
# which generally make use of `mlir_value_subclass/mlir_type_subclass/mlir_attribute_subclass`, perform an
904+
# `import mlir` right when they're loaded (see the mlir_*_subclass ctors in NanobindAdaptors.h).
905+
# Note, this also why IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/_mlir_libs" here while below
906+
# "${MLIRPythonModules_ROOT_PREFIX}/.." (because MLIR_BINDINGS_PYTHON_INSTALL_PREFIX, by default, ends at mlir).
907+
#
908+
# Further note: this function creates file targets like
909+
# "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs/_mlir/__init__.pyi". These must match the file targets
910+
# that declare_mlir_python_sources expects, which are like "${ROOT_DIR}/${WHATEVER_SOURCE}".
911+
# This is why _mlir_libs is prepended below.
933912
mlir_generate_type_stubs(
934-
# This is the FQN path because dialect modules import _mlir when loaded. See above.
935-
MODULE_NAME mlir._mlir_libs._mlirPythonTestNanobind
936-
DEPENDS_TARGETS
937-
# You need both _mlir and _mlirPythonTestNanobind because dialect modules import _mlir when loaded
938-
# (so _mlir needs to be built before calling stubgen).
939-
MLIRPythonModules.extension._mlir.dso
940-
MLIRPythonModules.extension._mlirPythonTestNanobind.dso
941-
# You need this one so that ir.py "built" because mlir._mlir_libs.__init__.py import mlir.ir in _site_initialize.
942-
MLIRPythonModules.sources.MLIRPythonSources.Core.Python
913+
MODULE_NAME _mlir
914+
DEPENDS_TARGETS MLIRPythonModules.extension._mlir.dso
943915
OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs"
944-
OUTPUTS _mlirPythonTestNanobind.pyi
945-
DEPENDS_TARGET_SRC_DEPS "${_test_extension_srcs}"
946-
IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/.."
916+
OUTPUTS "${_core_type_stub_sources}"
917+
DEPENDS_TARGET_SRC_DEPS "${_core_extension_srcs}"
918+
IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/_mlir_libs"
947919
)
948-
set(_mlirPythonTestNanobind_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}")
920+
set(_mlir_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}")
921+
922+
list(TRANSFORM _core_type_stub_sources PREPEND "_mlir_libs/")
923+
# Note, we do not do ADD_TO_PARENT here so that the type stubs are not associated (as mlir_DEPENDS) with
924+
# MLIRPythonSources.Core (or something) when a distro is installed/created. Otherwise they would not be regenerated
925+
# by users of the distro (the stubs are still installed in the distro - they are just not added to mlir_DEPENDS).
949926
declare_mlir_python_sources(
950-
MLIRPythonTestSources.PythonTestExtensionNanobind.type_stub_gen
927+
MLIRPythonExtension.Core.type_stub_gen
951928
ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs"
952-
ADD_TO_PARENT MLIRPythonTestSources.Dialects
953-
SOURCES _mlir_libs/_mlirPythonTestNanobind.pyi
929+
SOURCES "${_core_type_stub_sources}"
954930
)
931+
932+
# _mlirPythonTestNanobind stubgen
933+
934+
if(MLIR_INCLUDE_TESTS)
935+
get_target_property(_test_extension_srcs MLIRPythonTestSources.PythonTestExtensionNanobind INTERFACE_SOURCES)
936+
mlir_generate_type_stubs(
937+
# This is the FQN path because dialect modules import _mlir when loaded. See above.
938+
MODULE_NAME mlir._mlir_libs._mlirPythonTestNanobind
939+
DEPENDS_TARGETS
940+
# You need both _mlir and _mlirPythonTestNanobind because dialect modules import _mlir when loaded
941+
# (so _mlir needs to be built before calling stubgen).
942+
MLIRPythonModules.extension._mlir.dso
943+
MLIRPythonModules.extension._mlirPythonTestNanobind.dso
944+
# You need this one so that ir.py "built" because mlir._mlir_libs.__init__.py import mlir.ir in _site_initialize.
945+
MLIRPythonModules.sources.MLIRPythonSources.Core.Python
946+
OUTPUT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs/_mlir_libs"
947+
OUTPUTS _mlirPythonTestNanobind.pyi
948+
DEPENDS_TARGET_SRC_DEPS "${_test_extension_srcs}"
949+
IMPORT_PATHS "${MLIRPythonModules_ROOT_PREFIX}/.."
950+
)
951+
set(_mlirPythonTestNanobind_typestub_gen_target "${NB_STUBGEN_CUSTOM_TARGET}")
952+
declare_mlir_python_sources(
953+
MLIRPythonTestSources.PythonTestExtensionNanobind.type_stub_gen
954+
ROOT_DIR "${CMAKE_CURRENT_BINARY_DIR}/type_stubs"
955+
ADD_TO_PARENT MLIRPythonTestSources.Dialects
956+
SOURCES _mlir_libs/_mlirPythonTestNanobind.pyi
957+
)
958+
endif()
955959
endif()
956960

957961
################################################################################
958962
# The fully assembled package of modules.
959963
# This must come last.
960964
################################################################################
961965

966+
set(_declared_sources MLIRPythonSources MLIRPythonExtension.RegisterEverything)
967+
if(NOT CMAKE_CROSSCOMPILING)
968+
list(APPEND _declared_sources MLIRPythonExtension.Core.type_stub_gen)
969+
endif()
970+
962971
add_mlir_python_modules(MLIRPythonModules
963972
ROOT_PREFIX ${MLIRPythonModules_ROOT_PREFIX}
964973
INSTALL_PREFIX "${MLIR_BINDINGS_PYTHON_INSTALL_PREFIX}"
965974
DECLARED_SOURCES
966-
MLIRPythonSources
967-
MLIRPythonExtension.RegisterEverything
968-
MLIRPythonExtension.Core.type_stub_gen
975+
${_declared_sources}
969976
${_ADDL_TEST_SOURCES}
970977
COMMON_CAPI_LINK_LIBS
971978
MLIRPythonCAPI
972979
)
973-
add_dependencies(MLIRPythonModules "${_mlir_typestub_gen_target}")
974-
if(MLIR_INCLUDE_TESTS)
975-
add_dependencies(MLIRPythonModules "${_mlirPythonTestNanobind_typestub_gen_target}")
980+
if(NOT CMAKE_CROSSCOMPILING)
981+
add_dependencies(MLIRPythonModules "${_mlir_typestub_gen_target}")
982+
if(MLIR_INCLUDE_TESTS)
983+
add_dependencies(MLIRPythonModules "${_mlirPythonTestNanobind_typestub_gen_target}")
984+
endif()
976985
endif()

0 commit comments

Comments
 (0)