Bring depthai_nodes into depthai-core

CMakeLists.txt

@@ -1014,6 +1014,54 @@ if(DEPTHAI_BASALT_SUPPORT)
     endif()
 endif()
 
+########################
+# Extra host nodes
+########################
+set(TARGET_EXTRA_ALIAS hostNodesExt)
+set(TARGET_EXTRA_NAME ${PROJECT_NAME}-${TARGET_EXTRA_ALIAS})
+if (DEPTHAI_BUILD_EXT_HOST_NODES)
+    set(TARGET_EXT_NODES_SOURCES
+            ParserGenerator.cpp
+            host_nodes_ext/ParsingNeuralNetwork.cpp
+            host_nodes_ext/parsers/BaseParser.cpp
+            host_nodes_ext/parsers/SimCCKeypointParser.cpp
+            host_nodes_ext/parsers/KeypointParser.cpp
+            host_nodes_ext/messages/Keypoints.cpp
+    )
+    list(TRANSFORM TARGET_EXT_NODES_SOURCES PREPEND "src/pipeline/node/host/contrib/")
+#
+
+    # Add depthai-extraHostNodes library and importable target/alias depthai::extraHostNodes
+    add_library(${TARGET_EXTRA_NAME} ${TARGET_EXT_NODES_SOURCES})
+    add_library("${PROJECT_NAME}::${TARGET_EXTRA_ALIAS}" ALIAS ${TARGET_EXTRA_NAME})
+    set_target_properties(${TARGET_EXTRA_NAME} PROPERTIES EXPORT_NAME ${TARGET_EXTRA_ALIAS})
+
+
+    target_include_directories(${TARGET_EXTRA_NAME}
+            PUBLIC
+            "$<INSTALL_INTERFACE:include/depthai/pipeline/node/host/contrib/host_nodes_ext>"
+
+#            "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/${INC_DIR}>"
+#            "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/${INC_DIR}host/>"
+            "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include/depthai/pipeline/node/host/contrib/host_nodes_ext>"
+            PRIVATE
+            "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/include/depthai>"
+            "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/src/pipeline/node/host/contrib/host_nodes_ext>"
+            "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/src/pipeline/node/host/contrib>"
+            "$<BUILD_INTERFACE:${CMAKE_CURRENT_LIST_DIR}/src>"
+    )
+
+    target_link_libraries(${TARGET_EXTRA_NAME}
+            PRIVATE
+            spdlog::spdlog
+            xtensor
+            PUBLIC
+            depthai::core
+    )
+
+    list(APPEND targets_to_export ${TARGET_EXTRA_NAME})
+endif ()
+
 ########################
 # Combined target
 ########################

cmake/depthaiOptions.cmake

@@ -34,6 +34,7 @@ option(DEPTHAI_BUILD_TESTS "Build tests" OFF)
 option(DEPTHAI_BUILD_EXAMPLES "Build examples - Requires OpenCV library to be installed" OFF)
 option(DEPTHAI_BUILD_DOCS "Build documentation - requires doxygen to be installed" OFF)
 option(DEPTHAI_BUILD_ZOO_HELPER "Build the Zoo helper" OFF)
+option(DEPTHAI_BUILD_EXT_HOST_NODES "Build the contrib host nodes lib" OFF)
 option(DEPTHAI_NEW_FIND_PYTHON "Use new FindPython module" ON)
 option(DEPTHAI_INSTALL "Enable install target for depthai-core targets" ON)
 

examples/cpp/HostNodes/CMakeLists.txt

@@ -24,4 +24,8 @@ dai_set_example_test_labels(threaded_host_node ondevice rvc2_all rvc4 ci)
 dai_add_example(host_pipeline_synced_node host_pipeline_synced_node.cpp ON OFF)
 dai_set_example_test_labels(host_pipeline_synced_node ondevice rvc2_all rvc4 ci)
 
-dai_add_example(host_only_camera host_only_camera.cpp OFF OFF)
+dai_add_example(host_only_camera host_only_camera.cpp OFF OFF)
+
+if(DEPTHAI_BUILD_EXT_HOST_NODES)
+    add_subdirectory(HostNodesExtended)
+endif ()

examples/cpp/HostNodes/HostNodesExtended/CMakeLists.txt

@@ -0,0 +1,3 @@
+dai_add_example(simcc_parser simcc_parser.cpp ON OFF)
+dai_set_example_test_labels(simcc_parser ondevice rvc4 ci)
+target_link_libraries(simcc_parser PRIVATE utility depthai::hostNodesExt)

examples/cpp/HostNodes/HostNodesExtended/simcc_parser.cpp

@@ -0,0 +1,30 @@
+#include "depthai/pipeline/node/host/contrib/host_nodes_ext/ParsingNeuralNetwork.hpp"
+#include "messages/Keypoints.hpp"
+
+int main() {
+
+    // Create device
+    std::shared_ptr<dai::Device> device = std::make_shared<dai::Device>();
+
+    // Create pipeline
+    dai::Pipeline pipeline(device);
+
+    // Create nodes
+    auto camera = pipeline.create<dai::node::Camera>()->build();
+    auto output = camera->requestOutput(std::make_pair(288, 384), dai::ImgFrame::Type::BGR888i, dai::ImgResizeMode::STRETCH);
+
+    dai::NNModelDescription modelDescription{.model ="pedestl/rtmpose3d-open-mmlab-mmpose-large:v1-0-0:latest", .platform = pipeline.getDefaultDevice()->getPlatformAsString()};
+    auto archive = dai::NNArchive(dai::getModelFromZoo(modelDescription));
+
+    auto parsed_nn = pipeline.create<dai::node::ParsingNeuralNetwork>()->build(*output, archive);
+    // Create output queue, note that parsed_nn->out only exists if the NN has only a single parser head
+    auto out = parsed_nn->out.value().get().createOutputQueue();
+
+    // Start pipeline
+    pipeline.start();
+    pipeline.processTasks();
+    auto msg = out->get<dai::Buffer>();
+    std::shared_ptr<dai::Keypoints3D3C> keypoints = std::dynamic_pointer_cast<dai::Keypoints3D3C>(msg);
+    pipeline.stop();
+    assert(keypoints != nullptr);
+}

include/depthai/pipeline/node/host/contrib/ParserGenerator.hpp

@@ -0,0 +1,35 @@
+//
+// Created by thwdpc on 7/24/25.
+//
+
+#pragma once
+#include <variant>
+
+#include "depthai/depthai.hpp"
+#include "host_nodes_ext/parsers/BaseParser.hpp"
+
+namespace dai::node {
+
+typedef std::variant<std::shared_ptr<BaseParser>, std::shared_ptr<DetectionParser>> HostOrDeviceParser;
+template<typename V>
+constexpr bool all_alternatives_shared_ptr = false;
+template<typename... Ts>
+constexpr bool all_alternatives_shared_ptr<std::variant<Ts...>> =
+    (std::conjunction_v<std::is_same<std::shared_ptr<typename Ts::element_type>, Ts>...>);
+static_assert(all_alternatives_shared_ptr<HostOrDeviceParser>, "All alternatives must be std::shared_ptr<T>");
+
+struct ConfigModelWithHeads {
+    nn_archive::v1::Model model;
+    std::vector<nn_archive::v1::Head> heads;
+};
+
+class ParserGenerator {
+   public:
+    static std::vector<HostOrDeviceParser> generateAllParsers(Pipeline pipeline, const NNArchive& nnArchive, bool hostOnly = false);
+
+   private:
+    static ConfigModelWithHeads archiveGetModelEnsureOneHeadV1(const NNArchive& nnArchive, Platform targetPlatform);
+    static HostOrDeviceParser generateOneV1Parser(
+        Pipeline& pipeline, const NNArchive& owningArchive, const nn_archive::v1::Head& head, const nn_archive::v1::Model& model, bool hostOnly = false);
+};
+}  // namespace dai::node

include/depthai/pipeline/node/host/contrib/host_nodes_ext/ParsingNeuralNetwork.hpp

@@ -0,0 +1,50 @@
+//
+// Created by thwdpc on 7/24/25.
+//
+#pragma once
+#include <variant>
+#include <vector>
+
+#include "../ParserGenerator.hpp"
+#include "depthai/depthai.hpp"
+#include "parsers/BaseParser.hpp"
+#include "parsers/KeypointParser.hpp"
+
+namespace dai::node {
+
+class ParsingNeuralNetwork : public CustomThreadedNode<ParsingNeuralNetwork> {
+   public:
+    std::shared_ptr<ParsingNeuralNetwork> build(Output& input, const NNArchive& nnArchive);
+    std::shared_ptr<ParsingNeuralNetwork> build(const std::shared_ptr<Camera>& input, NNModelDescription modelDesc, std::optional<float> fps = std::nullopt);
+    std::shared_ptr<ParsingNeuralNetwork> build(const std::shared_ptr<Camera>& input, const NNArchive& nnArchive, std::optional<float> fps = std::nullopt);
+
+    InputMap& inputs = nn->inputs;
+    Input& input = nn->input;
+    std::optional<std::reference_wrapper<Output>> out = std::nullopt;
+    Output& passthrough = nn->passthrough;
+    OutputMap& passthroughs = nn->passthroughs;
+
+    template <typename T>
+    std::optional<size_t> getIndexOfFirstParserOfType() const {
+        const auto which = std::find_if(parsers.begin(), parsers.end(), [](const auto& p) {
+            return std::visit([](auto& anyP) { return std::dynamic_pointer_cast<T>(anyP) != nullptr; }, p);;
+        });
+        return which == parsers.end() ? std::nullopt : static_cast<std::optional<size_t>>(std::distance(parsers.begin(), which));
+    }
+
+    void run() override;
+
+   private:
+    std::vector<HostOrDeviceParser> getParserNodes(const NNArchive& nnArchive);
+
+    void updateParsers(const NNArchive& nnArchive);
+
+    void removeOldParserNodes();
+    std::shared_ptr<NeuralNetwork> nn;
+    std::optional<Subnode<Sync>> parserSync = std::nullopt;
+
+   protected:
+    std::vector<HostOrDeviceParser> parsers;
+};
+
+}  // namespace dai::node

include/depthai/pipeline/node/host/contrib/host_nodes_ext/messages/Keypoints.hpp

@@ -0,0 +1,56 @@
+//
+// Created by thwdpc on 7/28/25.
+//
+
+#pragma once
+#include <depthai/common/ImgTransformations.hpp>
+#include <depthai/pipeline/datatype/Buffer.hpp>
+#include <depthai/pipeline/datatype/NNData.hpp>
+
+namespace dai {
+
+struct ValueWithConfidence {
+    float_t value;
+    float_t confidence;
+};
+
+// Per-dimension confidence, strictly [v0, c0, v1, c1, ...]
+template <std::size_t D>
+struct KeypointPerDimConfidence {
+    ValueWithConfidence data[D];                 // value, confidence, value, confidence, ...
+    static constexpr std::size_t value = 2 * D;  // value,conf xD
+};
+
+// Per-keypoint confidence, strictly [v0, v1, v2 ... confidence]
+template <std::size_t D>
+struct KeypointPerKeypointConfidence {
+    float_t values[D];
+    float_t confidence;
+    static constexpr std::size_t value = D + 1;  // D values + 1 confidence
+};
+
+using Keypoint2D2C = KeypointPerDimConfidence<2>;
+using Keypoint2D1C = KeypointPerKeypointConfidence<2>;
+using Keypoint3D3C = KeypointPerDimConfidence<3>;
+using Keypoint3D1C = KeypointPerKeypointConfidence<3>;
+
+template <typename KP>
+class Keypoints : public Buffer {
+   public:
+    std::optional<ImgTransformation> transformation;
+
+    std::vector<KP> kpVec;
+
+    Keypoints(std::shared_ptr<NNData>&& other, xt::xarray<float>&& planarStackedKeypoints);
+};
+
+template class Keypoints<Keypoint2D1C>;
+typedef Keypoints<Keypoint2D1C> Keypoints2D;
+template class Keypoints<Keypoint2D2C>;
+typedef Keypoints<Keypoint2D2C> Keypoints2D2C;
+
+template class Keypoints<Keypoint3D1C>;
+typedef Keypoints<Keypoint3D1C> Keypoints3D;
+template class Keypoints<Keypoint3D3C>;
+typedef Keypoints<Keypoint3D3C> Keypoints3D3C;
+}  // namespace dai

include/depthai/pipeline/node/host/contrib/host_nodes_ext/parsers/BaseParser.hpp

@@ -0,0 +1,37 @@
+//
+// Created by thwdpc on 7/24/25.
+//
+
+#pragma once
+#include "depthai/depthai.hpp"
+
+namespace dai::node {
+class BaseParser : public ThreadedHostNode {
+   public:
+    Input input{*this, {"in", DEFAULT_GROUP, true, 5, {{{DatatypeEnum::NNData, true}}}, true}};
+    Output out{*this, {"out", DEFAULT_GROUP, {{{DatatypeEnum::Buffer, true}}}}};
+    virtual std::shared_ptr<BaseParser> build(const nn_archive::v1::Head& head, const nn_archive::v1::Model& model);
+
+   protected:
+    virtual void buildImpl(const nn_archive::v1::Head& head, const nn_archive::v1::Model& model) = 0;
+};
+
+/**
+ * @brief Custom node for parser. When creating a custom parser, inherit from this class!
+ * @tparam T Node type (same as the class you are creating)
+ *
+ * Example:
+ * @code{.cpp}
+ * class MyParser : public CustomParser<MyNode> {
+ *     std::shared_ptr<Buffer> processGroup(std::shared_ptr<dai::MessageGroup> in) override {
+ *         auto frame = in->get<dai::ImgFrame>("data");
+ *         // process frame
+ *         // ...
+ *         return nullptr; // Don't return anything, just process
+ *     }
+ * };
+ * @endcode
+ */
+template <typename T>
+using CustomParser = NodeCRTP<BaseParser, T>;
+}  // namespace dai::node

include/depthai/pipeline/node/host/contrib/host_nodes_ext/parsers/KeypointParser.hpp

@@ -0,0 +1,30 @@
+//
+// Created by thwdpc on 7/25/25.
+//
+
+#pragma once
+#include "BaseParser.hpp"
+
+namespace dai::node {
+
+enum class ValuesPerKeypoint: uint8_t {
+    Two = 2,
+    Three = 3
+};
+
+class KeypointParser : virtual public CustomParser<KeypointParser> {
+public:
+    constexpr static const char* NAME = "KeypointParser";
+
+protected:
+    void buildImpl(const nn_archive::v1::Head& head, const nn_archive::v1::Model& model) override;
+    void run() override;
+
+    std::vector<nn_archive::v1::Output> keypointsOutputs{};
+    uint16_t nKeypoints = 17;
+    // dimensionality: 2D or 3D
+    ValuesPerKeypoint valuesPerKeypoint = ValuesPerKeypoint::Two;
+    std::vector<std::string> keypointNames{};
+    std::vector<std::pair<uint8_t, uint8_t>> skeletonEdges{};
+};
+}

include/depthai/pipeline/node/host/contrib/host_nodes_ext/parsers/SimCCKeypointParser.hpp

@@ -0,0 +1,32 @@
+
+#pragma once
+#include "KeypointParser.hpp"
+#include "parsers/BaseParser.hpp"
+
+namespace dai::node {
+
+
+class SimCCKeypointParser final : public NodeCRTP<KeypointParser, SimCCKeypointParser> {
+   public:
+    constexpr static const char* NAME = "SimCCKeypointParser";
+
+   protected:
+    void buildImpl(const nn_archive::v1::Head& head, const nn_archive::v1::Model& model) override;
+    void run() override;
+    void foggyGuessesForOneDim(const nn_archive::v1::Head& head,
+                               const nn_archive::v1::Model& model,
+                               const nn_archive::v1::Input& imgInput,
+                               const std::pair<std::optional<int64_t>, std::optional<int64_t>>& imgWHMaybe);
+    void inferConfigFromMultipleOutputs(const nn_archive::v1::Head& head,
+                                        const nn_archive::v1::Model& model,
+                                        const nn_archive::v1::Input& imgInput,
+                                        std::pair<std::optional<int64_t>, std::optional<int64_t>>& imgWidthHeight);
+
+    uint8_t pixelSubdivisions = 2;
+    // Populated if the keypoint # dim and the XY(Z) dimensionality are collapsed(like yolo). Stores whether collapsed dim is interleaved(x1, x2 .. y1, y2 .. z1, z2)
+    //  or planar(x1, y1, z1, x2, y2, z2)
+    std::optional<bool> collapsedDimsAreInterleaved = std::nullopt;
+    bool replicateXDimToZDim = true;
+    std::vector<uint16_t> simCCDimLengths;
+};
+}  // namespace dai::node