refactor: cleanup build scripts, removing old commands and make them faster (#396)

Author: ruben-arts

backends/pixi-build-ros/src/pixi_build_ros/build_script.py

@@ -55,19 +55,25 @@ def load_from_template(
         else:
             raise ValueError(f"Unsupported build type: {pkg.get_build_type()}")
 
+        script_content = ""
         try:
             # Try to load from installed package data first
             templates_pkg = files("pixi_build_ros") / "templates"
             template_file = templates_pkg / template_name
             script_content = template_file.read_text()
         except (ImportError, FileNotFoundError):
-            # Fallback to development path
+            # Fallback to the development path
             templates_pkg = Path(__file__).parent.parent.parent / "templates"
             script_path = templates_pkg / template_name
             with open(script_path) as f:
                 script_content = f.read()
 
-        script_content = script_content.replace("@SRC_DIR@", str(source_dir)).replace("@DISTRO@", distro.name)
+        script_content = (
+            script_content.replace("@SRC_DIR@", str(source_dir))
+            .replace("@DISTRO@", distro.name)
+            .replace("@BUILD_DIR@", "build")
+            .replace("@BUILD_TYPE@", "Release")
+        )
 
         return cls(
             script_content=script_content,

backends/pixi-build-ros/templates/bld_ament_cmake.bat

@@ -5,15 +5,18 @@ setlocal EnableDelayedExpansion
 :: Rattler-build will not set the SRC_DIR anymore so we set it through templating
 set "SRC_DIR=@SRC_DIR@"
 
+set "BUILD_DIR=@BUILD_DIR@"
+set "BUILD_TYPE=@BUILD_TYPE@"
+
 set "PYTHONPATH=%LIBRARY_PREFIX%\lib\site-packages;%SP_DIR%"
 
 :: MSVC is preferred.
 set CC=cl.exe
 set CXX=cl.exe
 
-rd /s /q build
-mkdir build
-pushd build
+:: Create build directory when not available
+if not exist "%BUILD_DIR%" mkdir "%BUILD_DIR%"
+pushd %BUILD_DIR%
 
 :: set "CMAKE_GENERATOR=Ninja"
 :: We use the Visual Studio generator as a workaround for
@@ -35,7 +38,7 @@ FOR /F "tokens=* USEBACKQ" %%i IN (`python -c "import os;print(os.path.relpath(o
 cmake ^
     -G "%CMAKE_GENERATOR%" ^
     -DCMAKE_INSTALL_PREFIX=%LIBRARY_PREFIX% ^
-    -DCMAKE_BUILD_TYPE=Release ^
+    -DCMAKE_BUILD_TYPE=%BUILD_TYPE% ^
     -DCMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_SKIP=True ^
     -DPYTHON_EXECUTABLE=%PYTHON% ^
     -DPython_EXECUTABLE=%PYTHON% ^

backends/pixi-build-ros/templates/build_ament_cmake.sh

@@ -2,125 +2,56 @@
 # DO NOT EDIT!
 set -eo pipefail
 
-rm -rf build
-mkdir build
-cd build
+BUILD_DIR=@BUILD_DIR@
+
+if [ ! -d "$BUILD_DIR" ]; then
+  mkdir -p "$BUILD_DIR"
+fi
+pushd $BUILD_DIR
+
+BUILD_TYPE=@BUILD_TYPE@
 
 # necessary for correctly linking SIP files (from python_qt_bindings)
 export LINK=$CXX
 
 if [[ "$CONDA_BUILD_CROSS_COMPILATION" != "1" ]]; then
   PYTHON_EXECUTABLE=$PREFIX/bin/python
   PKG_CONFIG_EXECUTABLE=$PREFIX/bin/pkg-config
-  OSX_DEPLOYMENT_TARGET="10.15"
+  export QT_HOST_PATH="$PREFIX"
 else
   PYTHON_EXECUTABLE=$BUILD_PREFIX/bin/python
   PKG_CONFIG_EXECUTABLE=$BUILD_PREFIX/bin/pkg-config
-  OSX_DEPLOYMENT_TARGET="11.0"
-fi
-
-if [[ "${CONDA_BUILD_CROSS_COMPILATION:-}" == "1" ]]; then
   export QT_HOST_PATH="$BUILD_PREFIX"
-else
-  export QT_HOST_PATH="$PREFIX"
 fi
 
-echo "USING PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}"
-echo "USING PKG_CONFIG_EXECUTABLE=${PKG_CONFIG_EXECUTABLE}"
-
-export ROS_PYTHON_VERSION=`$PYTHON_EXECUTABLE -c "import sys; print('%i.%i' % (sys.version_info[0:2]))"`
-echo "Using Python ${ROS_PYTHON_VERSION}"
-
-# see https://github.com/conda-forge/cross-python-feedstock/issues/24
-if [[ "$CONDA_BUILD_CROSS_COMPILATION" == "1" ]]; then
-  find $PREFIX/lib/cmake -type f -exec sed -i "s~\${_IMPORT_PREFIX}/lib/python${ROS_PYTHON_VERSION}/site-packages~${BUILD_PREFIX}/lib/python${ROS_PYTHON_VERSION}/site-packages~g" {} + || true
-  find $PREFIX/share/rosidl* -type f -exec sed -i "s~${PREFIX}/lib/python${ROS_PYTHON_VERSION}/site-packages~${BUILD_PREFIX}/lib/python${ROS_PYTHON_VERSION}/site-packages~g" {} + || true
-  find $PREFIX/share/rosidl* -type f -exec sed -i "s~\${_IMPORT_PREFIX}/lib/python${ROS_PYTHON_VERSION}/site-packages~${BUILD_PREFIX}/lib/python${ROS_PYTHON_VERSION}/site-packages~g" {} + || true
-  find $PREFIX/lib/cmake -type f -exec sed -i "s~message(FATAL_ERROR \"The imported target~message(WARNING \"The imported target~g" {} + || true
-fi
+# Only support 11.0+ for now
+OSX_DEPLOYMENT_TARGET="11.0"
 
+# Added to help make portable builds when using <inttypes.h> in C++ code.
 if [[ $target_platform =~ linux.* ]]; then
     export CFLAGS="${CFLAGS} -D__STDC_FORMAT_MACROS=1"
     export CXXFLAGS="${CXXFLAGS} -D__STDC_FORMAT_MACROS=1"
 fi;
 
-# Needed for qt-gui-cpp ..
-if [[ $target_platform =~ linux.* ]]; then
-  ln --symbolic --force $GCC ${BUILD_PREFIX}/bin/gcc
-  ln --symbolic --force $GXX ${BUILD_PREFIX}/bin/g++
-fi;
-
-
-# PYTHON_INSTALL_DIR should be a relative path, see
-# https://github.com/ament/ament_cmake/blob/2.3.2/ament_cmake_python/README.md
-# So we compute the relative path of $SP_DIR w.r.t. to $PREFIX,
-# but it is not trivial to do this in bash scripting, so let's do it via python
+echo "USING PYTHON_EXECUTABLE=${PYTHON_EXECUTABLE}"
+echo "USING PKG_CONFIG_EXECUTABLE=${PKG_CONFIG_EXECUTABLE}"
+export ROS_PYTHON_VERSION=3
+echo "Using Python ${ROS_PYTHON_VERSION}"
 echo "Using PREFIX: $PREFIX"
-# TODO: Remove when SPD_DIR is available in the environment
-export SP_DIR=${SP_DIR:-$PREFIX/lib/python3.12/site-packages}
-export PY_VER=3.12
 echo "Using SP_DIR: $SP_DIR"
 echo "Using CONDA_BUILD: $CONDA_BUILD"
 echo "Using CPU_COUNT: $CPU_COUNT"
+export PY_VER=${ROS_PYTHON_VERSION}
 echo "Using PY_VER: $PY_VER"
 echo "Using PYTHON: $PYTHON"
 echo "Using STDLIB_DIR: $STDLIB_DIR"
 echo "Using ROS_PYTHON_VERSION: $ROS_PYTHON_VERSION"
 echo "Using SRC_DIR: $SRC_DIR"
-
-export PYTHON_INSTALL_DIR=`python -c "import os;print(os.path.relpath(os.environ['SP_DIR'],os.environ['PREFIX']))"`
+export PYTHON_INSTALL_DIR=$(realpath --relative-to="$PREFIX" "$SP_DIR")
 echo "Using PYTHON_INSTALL_DIR: $PYTHON_INSTALL_DIR"
 
-if [[ $target_platform =~ emscripten.* ]]; then
-  export CONDA_BUILD_CROSS_COMPILATION="1"
-  PYTHON_EXECUTABLE=$BUILD_PREFIX/bin/python$PY_VER
-  echo "set_property(GLOBAL PROPERTY TARGET_SUPPORTS_SHARED_LIBS TRUE)"> $SRC_DIR/__vinca_shared_lib_patch.cmake
-  echo "set(CMAKE_STRIP FALSE)  # used by default in pybind11 on .so modules">> $SRC_DIR/__vinca_shared_lib_patch.cmake
-  echo "set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE BOTH)  # fixes an error where numpy header files are not found correctly">> $SRC_DIR/__vinca_shared_lib_patch.cmake
-
-  # if [ "${PKG_NAME}" == "ros-humble-examples-rclcpp-minimal-publisher" ] || [ "${PKG_NAME}" == "ros-humble-examples-rclcpp-minimal-subscriber" ] || [ "${PKG_NAME}" == "ros-humble-rclcpp-components" ]; then
-  #   echo "set(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS \"-s ASSERTIONS=1 -s SIDE_MODULE=1 -sWASM_BIGINT -s USE_PTHREADS=0 -s DEMANGLE_SUPPORT=1 -s ALLOW_MEMORY_GROWTH=1 \")">> $SRC_DIR/__vinca_shared_lib_patch.cmake
-  #   echo "set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS \"-s ASSERTIONS=1 -s SIDE_MODULE=1 -sWASM_BIGINT -s USE_PTHREADS=0 -s DEMANGLE_SUPPORT=1 -s ALLOW_MEMORY_GROWTH=1 -sASYNCIFY -O3 -s ASYNCIFY_STACK_SIZE=24576 \")">> $SRC_DIR/__vinca_shared_lib_patch.cmake
-  #   echo "set(CMAKE_EXE_LINKER_FLAGS \"-sMAIN_MODULE=1 -sASSERTIONS=1 -fexceptions -lembind -sWASM_BIGINT -s USE_PTHREADS=0 -s DEMANGLE_SUPPORT=1 -sALLOW_MEMORY_GROWTH=1 -sASYNCIFY -O3 -s ASYNCIFY_STACK_SIZE=24576 -L$SRC_DIR/build -L$PREFIX/lib\")  # remove SIDE_MODULE from exe linker flags">> $SRC_DIR/__vinca_shared_lib_patch.cmake
-  # else
-    echo "set(CMAKE_SHARED_LIBRARY_CREATE_C_FLAGS \"-s ASSERTIONS=1 -s SIDE_MODULE=1 -sWASM_BIGINT -s USE_PTHREADS=0 -s ALLOW_MEMORY_GROWTH=1 -s DEMANGLE_SUPPORT=1 \")">> $SRC_DIR/__vinca_shared_lib_patch.cmake
-    echo "set(CMAKE_SHARED_LIBRARY_CREATE_CXX_FLAGS \"-s ASSERTIONS=1 -s SIDE_MODULE=1 -sWASM_BIGINT -s USE_PTHREADS=0 -s ALLOW_MEMORY_GROWTH=1 -s DEMANGLE_SUPPORT=1 \")">> $SRC_DIR/__vinca_shared_lib_patch.cmake
-    echo "set(CMAKE_EXE_LINKER_FLAGS \"-sMAIN_MODULE=1 -sASSERTIONS=1 -fexceptions -lembind -sWASM_BIGINT -s USE_PTHREADS=0 -sALLOW_MEMORY_GROWTH=1 -s DEMANGLE_SUPPORT=1 -L$SRC_DIR/build -L$PREFIX/lib\")  # remove SIDE_MODULE from exe linker flags">> $SRC_DIR/__vinca_shared_lib_patch.cmake
-  # fi
-
-  export BUILD_TYPE="Debug"
-  export EXTRA_CMAKE_ARGS=" \
-      -DPYTHON_SOABI="cpython-${ROS_PYTHON_VERSION//./}-wasm32-emscripten" \
-      -DRMW_IMPLEMENTATION=rmw_wasm_cpp \
-      -DCMAKE_FIND_ROOT_PATH=$PREFIX \
-      -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE \
-      -DCMAKE_PROJECT_INCLUDE=$SRC_DIR/__vinca_shared_lib_patch.cmake \
-  "
-
-  unset -f cmake
-  export CMAKE_GEN="emcmake cmake"
-  export CMAKE_BLD="cmake"
-
-  export STATIC_ROSIDL_TYPESUPPORT_C=rosidl_typesupport_introspection_c
-  export STATIC_ROSIDL_TYPESUPPORT_CPP=rosidl_typesupport_introspection_cpp
-else
-  export BUILD_TYPE="Release"
-  export CMAKE_GEN="cmake"
-  export CMAKE_BLD="cmake"
-fi;
-
-if [ "${PKG_NAME}" == "ros-humble-rmw-wasm-cpp" ]; then
-  WORK_DIR=$SRC_DIR/$PKG_NAME/src/work/rmw_wasm_cpp
-elif [ "${PKG_NAME}" == "ros-humble-wasm-cpp" ]; then
-  WORK_DIR=$SRC_DIR/$PKG_NAME/src/work/wasm_cpp
-elif [ "${PKG_NAME}" == "dynmsg" ]; then
-  WORK_DIR=$SRC_DIR/$PKG_NAME/src/work/dynmsg
-else
-  WORK_DIR=$SRC_DIR
-fi;
-
 if [ ! -f "build.ninja" ]; then
-    $CMAKE_GEN \
+    cmake \
         -G "Ninja" \
         -S @SRC_DIR@ \
         -DCMAKE_BUILD_TYPE=$BUILD_TYPE \
@@ -137,7 +68,7 @@ if [ ! -f "build.ninja" ]; then
         -DSETUPTOOLS_DEB_LAYOUT=OFF \
         -DCATKIN_SKIP_TESTING=$SKIP_TESTING \
         -DCMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_SKIP=True \
-        -DBUILD_SHARED_LIBS=ON  \
+        -DBUILD_SHARED_LIBS=ON \
         -DBUILD_TESTING=OFF \
         -DCMAKE_IGNORE_PREFIX_PATH="/opt/homebrew;/usr/local/homebrew" \
         -DCMAKE_OSX_DEPLOYMENT_TARGET=$OSX_DEPLOYMENT_TARGET \
@@ -146,4 +77,4 @@ if [ ! -f "build.ninja" ]; then
         .
 fi
 
-$CMAKE_BLD --build . --config $BUILD_TYPE --target install
+cmake --build . --config $BUILD_TYPE --target install

backends/pixi-build-ros/templates/build_ament_python.sh

@@ -16,11 +16,10 @@ if [ -f setup.cfg ] && grep -q "install[-_]scripts" setup.cfg; then
   PKG_NAME_SHORT=${PKG_NAME_SHORT//-/_}
   INSTALL_SCRIPTS_ARG="--install-scripts=$PREFIX/lib/$PKG_NAME_SHORT"
   echo "WARNING: setup.cfg not set, will set INSTALL_SCRIPTS_ARG to: $INSTALL_SCRIPTS_ARG"
-  $PYTHON setup.py install --prefix="$PREFIX" --install-lib="$SP_DIR" $INSTALL_SCRIPTS_ARG --single-version-externally-managed --record=files.txt
+  $PYTHON setup.py install --prefix="$PREFIX" --install-lib="$SP_DIR" $INSTALL_SCRIPTS_ARG --single-version-externally-managed
   # Remove build artifacts from setup.py install
   rm -rf *.egg-info 2>/dev/null || true
   rm -rf build/ 2>/dev/null || true
-  rm -rf files.txt 2>/dev/null || true
 else
   $PYTHON -m pip install . --no-deps -vvv
 fi