Feature/cibuildwheel (#50)

* initial attempt at cibuildwheel for develop

* fix event syntax

* fix name and turn on dev branch

* (wip) add source dep options for cibuildwheel

* fix random build issues

* (wip) start splitting bindings for faster compile

* fix compilation and formatting

* fix get_layer_id usage

* replicate zmq interface for send/recv in python

* also port ZmqGraph implementation to python

* turn on other platforms

* get branch name right

* drop old osx

* use pack pramga for msvc

* fix missing headers and other msvc compile issues

* fix ub with uniform generator

* use filepath to avoid wstring issue

* switch to building on release only

* rework external deps to use system packages when available

* add dependencies to external projects

* switch back to fetch content

Author: nathanhhughes

.cmake-format.yaml

@@ -13,7 +13,7 @@ parse:
     pybind11_add_module:
       pargs: 1
 format:
-  line_width: 100
+  line_width: 120
   tab_size: 2
   use_tabchars: false
   max_subgroups_hwrap: 3

.github/workflows/ci.yaml

@@ -36,7 +36,7 @@ jobs:
       - name: Dependencies
         run: |
           python -m pip install --upgrade pip
-          sudo apt-get update && sudo apt install libgtest-dev libeigen3-dev nlohmann-json3-dev libzmq3-dev python3-dev
+          sudo apt-get update && sudo apt install python3-dev
       - name: Install
         run: python -m pip install ${{github.workspace}}
       - name: Test

.github/workflows/wheels.yaml

@@ -0,0 +1,23 @@
+---
+# Adapted from:
+# https://github.com/pypa/cibuildwheel?tab=readme-ov-file#example-setup
+name: 'Spark-DSG Wheels'
+on: {release: {types: [published]}}
+jobs:
+  build_wheels:
+    name: Build wheels on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-latest]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+      - name: Install cibuildwheel
+        run: python -m pip install cibuildwheel
+      - name: Build wheels
+        run: python -m cibuildwheel --output-dir wheelhouse
+      - uses: actions/upload-artifact@v4
+        with:
+          name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
+          path: ./wheelhouse/*.whl

.gitignore

@@ -193,3 +193,4 @@ Testing/
 # Versioning file
 include/spark_dsg_version.h
 .clangd
+wheelhouse

CMakeLists.txt

@@ -24,18 +24,64 @@ option(SPARK_DSG_BUILD_EXAMPLES "Build examples" ON)
 option(SPARK_DSG_BUILD_PYTHON "Build python bindings" OFF)
 option(SPARK_DSG_BUILD_ZMQ "Build zmq message interface" ON)
 
-find_package(nlohmann_json REQUIRED)
-find_package(Eigen3 REQUIRED)
-find_package(Threads REQUIRED)
+set(SPARK_DSG_INSTALL_PREFIX "")
+set(FIND_MODE REQUIRED)
+if(DEFINED SKBUILD AND "${SKBUILD}" GREATER 0)
+  # nest headers, cmake and built lib under spark_dsg to make installation into sitepackages / wheel work correctly
+  set(SPARK_DSG_INSTALL_PREFIX "${PROJECT_NAME}/")
+  set(SPARK_DSG_BUILD_ZMQ OFF)
+  set(FIND_MODE QUIET)
+endif()
 
-find_package(PkgConfig REQUIRED)
-pkg_check_modules(zmq libzmq)
-if(SPARK_DSG_BUILD_ZMQ AND zmq_FOUND)
-  set(SPARK_DSG_USE_ZMQ 1)
-else()
-  message(WARNING "ZMQ not found! Disabling ZMQ interface")
-  set(SPARK_DSG_USE_ZMQ 0)
+include(FetchContent)
+
+find_package(nlohmann_json ${FIND_MODE})
+if(NOT ${nlohmann_json_FOUND})
+  FetchContent_Declare(
+    json
+    GIT_REPOSITORY https://github.com/nlohmann/json.git
+    GIT_TAG v3.11.3
+    GIT_SHALLOW TRUE
+  )
+
+  FetchContent_GetProperties(json)
+  if(NOT json_POPULATED)
+    FetchContent_Populate(json)
+  endif()
+
+  add_library(nlohmann_json::nlohmann_json INTERFACE IMPORTED)
+  target_include_directories(nlohmann_json::nlohmann_json INTERFACE ${json_SOURCE_DIR}/single_include)
+endif()
+
+find_package(Eigen3 ${FIND_MODE})
+if(NOT ${Eigen3_FOUND})
+  FetchContent_Declare(
+    eigen
+    URL https://gitlab.com/libeigen/eigen/-/archive/3.4.0/eigen-3.4.0.zip
+    URL_HASH MD5=a83cb9a2cbba2dd52c137ac62d33d847
+    DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+  )
+
+  FetchContent_GetProperties(eigen)
+  if(NOT eigen_POPULATED)
+    FetchContent_Populate(eigen)
+  endif()
+
+  add_library(Eigen3::Eigen INTERFACE IMPORTED)
+  target_include_directories(Eigen3::Eigen INTERFACE "${eigen_SOURCE_DIR}")
+endif()
+
+set(SPARK_DSG_USE_ZMQ 0)
+if(SPARK_DSG_BUILD_ZMQ)
+  find_package(PkgConfig REQUIRED)
+  pkg_check_modules(zmq libzmq)
+  if(zmq_FOUND)
+    set(SPARK_DSG_USE_ZMQ 1)
+  else()
+    message(WARNING "ZMQ not found! Disabling ZMQ interface")
+  endif()
 endif()
+
 configure_file(cmake/spark_dsg_version.h.in include/spark_dsg_version.h)
 
 add_library(
@@ -73,19 +119,18 @@ add_library(
 )
 
 target_include_directories(
-  ${PROJECT_NAME}
-  PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
+  ${PROJECT_NAME} PUBLIC $<INSTALL_INTERFACE:include> $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   PRIVATE $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/include>
 )
-target_link_libraries(
-  ${PROJECT_NAME} PUBLIC Eigen3::Eigen nlohmann_json::nlohmann_json
-  PRIVATE Threads::Threads
-)
+target_link_libraries(${PROJECT_NAME} PUBLIC Eigen3::Eigen nlohmann_json::nlohmann_json)
+
 if(NOT BUILD_SHARED_LIBS)
   set_property(TARGET ${PROJECT_NAME} PROPERTY POSITION_INDEPENDENT_CODE 1)
 endif()
+
 if(SPARK_DSG_BUILD_ZMQ AND zmq_FOUND)
-  target_link_libraries(${PROJECT_NAME} PRIVATE ${zmq_LIBRARIES})
+  find_package(Threads REQUIRED)
+  target_link_libraries(${PROJECT_NAME} PRIVATE ${zmq_LIBRARIES} Threads::Threads)
   target_include_directories(${PROJECT_NAME} PRIVATE ${zmq_INCLUDE_DIRS})
 endif()
 
@@ -109,20 +154,11 @@ include(GNUInstallDirs)
 include(CMakePackageConfigHelpers)
 
 write_basic_package_version_file(
-  ${CMAKE_CURRENT_BINARY_DIR}/spark_dsgConfigVersion.cmake VERSION ${PROJECT_VERSION}
-  COMPATIBILITY AnyNewerVersion
+  ${CMAKE_CURRENT_BINARY_DIR}/spark_dsgConfigVersion.cmake VERSION ${PROJECT_VERSION} COMPATIBILITY AnyNewerVersion
 )
 
-set(SPARK_DSG_INSTALL_PREFIX "")
-if(SPARK_DSG_BUILDING_PYTHON_PACKAGE)
-  # nest headers, cmake and built lib under spark_dsg to make installation
-  # into sitepackages / wheel work correctly
-  set(SPARK_DSG_INSTALL_PREFIX "${PROJECT_NAME}/")
-endif()
-
 configure_package_config_file(
-  ${CMAKE_CURRENT_LIST_DIR}/cmake/spark_dsgConfig.cmake.in
-  ${CMAKE_CURRENT_BINARY_DIR}/spark_dsgConfig.cmake
+  ${CMAKE_CURRENT_LIST_DIR}/cmake/spark_dsgConfig.cmake.in ${CMAKE_CURRENT_BINARY_DIR}/spark_dsgConfig.cmake
   INSTALL_DESTINATION "${SPARK_DSG_INSTALL_PREFIX}${CMAKE_INSTALL_LIBDIR}/cmake/spark_dsg"
 )
 
@@ -142,8 +178,6 @@ install(
   DESTINATION "${SPARK_DSG_INSTALL_PREFIX}${CMAKE_INSTALL_LIBDIR}/cmake/spark_dsg"
 )
 
-install(
-  FILES ${CMAKE_CURRENT_BINARY_DIR}/spark_dsgConfig.cmake
-        ${CMAKE_CURRENT_BINARY_DIR}/spark_dsgConfigVersion.cmake
-  DESTINATION "${SPARK_DSG_INSTALL_PREFIX}${CMAKE_INSTALL_LIBDIR}/cmake/spark_dsg"
+install(FILES ${CMAKE_CURRENT_BINARY_DIR}/spark_dsgConfig.cmake ${CMAKE_CURRENT_BINARY_DIR}/spark_dsgConfigVersion.cmake
+        DESTINATION "${SPARK_DSG_INSTALL_PREFIX}${CMAKE_INSTALL_LIBDIR}/cmake/spark_dsg"
 )

include/spark_dsg/dynamic_scene_graph.h

@@ -34,6 +34,7 @@
  * -------------------------------------------------------------------------- */
 #pragma once
 #include <Eigen/Core>
+#include <filesystem>
 #include <nlohmann/json.hpp>
 #include <type_traits>
 
@@ -454,14 +455,14 @@ class DynamicSceneGraph {
    * @param filepath Filepath to save graph to.
    * @param include_mesh Optionally encode mesh (defaults to true)
    */
-  void save(std::string filepath, bool include_mesh = true) const;
+  void save(std::filesystem::path filepath, bool include_mesh = true) const;
 
   /**
    * @brief Parse graph from binary or JSON file
    * @param filepath Complete path to file to read, including extension.
    * @returns Resulting parsed scene graph
    */
-  static Ptr load(std::string filepath);
+  static Ptr load(std::filesystem::path filepath);
 
   //! @brief Set the scene graph mesh
   void setMesh(const std::shared_ptr<Mesh>& mesh);

include/spark_dsg/mesh.h

@@ -37,6 +37,7 @@
 #include <Eigen/Dense>
 #include <array>
 #include <cstdint>
+#include <filesystem>
 #include <memory>
 #include <unordered_set>
 #include <vector>
@@ -212,14 +213,14 @@ class Mesh {
    *
    * @param filepath Filepath to save graph to.
    */
-  void save(std::string filepath) const;
+  void save(std::filesystem::path filepath) const;
 
   /**
    * @brief parse mesh from binary or JSON file
    * @param filepath Complete path to file to read, including extension.
    * @returns Resulting parsed mesh
    */
-  static Ptr load(std::string filepath);
+  static Ptr load(std::filesystem::path filepath);
 
   // ------ Modification ------
 

include/spark_dsg/node_attributes.h

@@ -169,7 +169,7 @@ struct SemanticNodeAttributes : public NodeAttributes {
   SemanticNodeAttributes();
   virtual ~SemanticNodeAttributes() = default;
   NodeAttributes::Ptr clone() const override;
-  virtual void transform(const Eigen::Isometry3d& transform) override;
+  void transform(const Eigen::Isometry3d& transform) override;
 
   bool hasLabel() const;
   bool hasFeature() const;
@@ -210,7 +210,7 @@ struct ObjectNodeAttributes : public SemanticNodeAttributes {
   ObjectNodeAttributes();
   virtual ~ObjectNodeAttributes() = default;
   NodeAttributes::Ptr clone() const override;
-  virtual void transform(const Eigen::Isometry3d& transform);
+  void transform(const Eigen::Isometry3d& transform) override;
 
   //! Mesh vertice connections
   std::list<size_t> mesh_connections;
@@ -375,7 +375,7 @@ struct AgentNodeAttributes : public NodeAttributes {
                       NodeId external_key);
   virtual ~AgentNodeAttributes() = default;
   NodeAttributes::Ptr clone() const override;
-  virtual void transform(const Eigen::Isometry3d& transform) override;
+  void transform(const Eigen::Isometry3d& transform) override;
 
   std::chrono::nanoseconds timestamp;
   Eigen::Quaterniond world_R_body;

include/spark_dsg/node_symbol.h

@@ -93,10 +93,12 @@ class NodeSymbol {
  private:
   union {
     NodeId value;
-    struct __attribute__((packed)) {
+#pragma pack(push, 1)
+    struct {
       NodeId index : 56;
       char key : 8;
     } symbol;
+#pragma pack(pop)
   } value_;
 };
 

include/spark_dsg/serialization/file_io.h

@@ -34,6 +34,7 @@
  * -------------------------------------------------------------------------- */
 #pragma once
 
+#include <filesystem>
 #include <memory>
 #include <optional>
 #include <string>
@@ -53,7 +54,7 @@ enum class FileType { JSON, BINARY, NONE, UNKNOWN };
  * @brief Identify the file type of a given filepath.
  * @param filepath The complete filepath to identify.
  */
-FileType identifyFileType(const std::string& filepath);
+FileType identifyFileType(const std::filesystem::path& filepath);
 
 /**
  * @brief Verify that the file extension of a given filepath is valid. If no extension
@@ -62,7 +63,7 @@ FileType identifyFileType(const std::string& filepath);
  * @throws std::runtime_error if the extension is invalid.
  * @return The file type of the resulting filepath.
  */
-FileType verifyFileExtension(std::string& filepath);
+FileType verifyFileExtension(std::filesystem::path& filepath);
 
 /**
  * @brief Save a DynamicSceneGraph to a JSON file.
@@ -71,15 +72,15 @@ FileType verifyFileExtension(std::string& filepath);
  * @param include_mesh If true, save the mesh data for each node.
  */
 void saveDsgJson(const DynamicSceneGraph& graph,
-                 const std::string& filepath,
+                 const std::filesystem::path& filepath,
                  bool include_mesh = false);
 
 /**
  * @brief Load a DynamicSceneGraph from a JSON file.
  * @param filepath The filepath including extension to load from.
  * @return A pointer to the loaded graph or nullptr if loading failed.
  */
-std::shared_ptr<DynamicSceneGraph> loadDsgJson(const std::string& filepath);
+std::shared_ptr<DynamicSceneGraph> loadDsgJson(const std::filesystem::path& filepath);
 
 /**
  * @brief Save a DynamicSceneGraph to a file in binary serialization.
@@ -88,14 +89,14 @@ std::shared_ptr<DynamicSceneGraph> loadDsgJson(const std::string& filepath);
  * @param include_mesh If true, save the mesh data for each node.
  */
 void saveDsgBinary(const DynamicSceneGraph& graph,
-                   const std::string& filepath,
+                   const std::filesystem::path& filepath,
                    bool include_mesh = false);
 
 /**
  * @brief Load a DynamicSceneGraph from a file in binary serialization.
  * @param filepath The filepath including extension to load from.
  * @return A pointer to the loaded graph or nullptr if loading failed.
  */
-std::shared_ptr<DynamicSceneGraph> loadDsgBinary(const std::string& filepath);
+std::shared_ptr<DynamicSceneGraph> loadDsgBinary(const std::filesystem::path& filepath);
 
 }  // namespace spark_dsg::io