Skip to content

Commit 6f55df1

Browse files
asahtikMatevz Morato
asahtik
and
Matevz Morato
authored
ImageManipV2 optimization and refactor (#1290)
* Refactor imagemanipv2 transform, optimize crop * WIP: ImageManipV2 opt * Refactor ImageManipV2Impl + use shared data instead of span * Refactor ImageManipV2 again * Fix output size calculation for NV12 * RVC4 FW: optimize common use cases in ImageManipV2 (crop, scale), fix incorrect striding in Camera gray output, refactor ImageManipV2 * Bugfixes, add test, WIP: test fails becouse of DsImgDispatcher issues * Fixed test to check Eva downscaler * Add backend property to imagemanip [no ci] * RVC4 FW: Fix ImageManipV2 optimization, add backend property, TODO: bump on RVC2 * Split test to RVC4 only part * Fix RVC2 ImageManipV2 rebuilding * RVC2 FW: Fix ImageManipV2 rebuilding * Fix image manip v2 rebuild test * Fix failing tests * RVC4 FW: Fix failing tests * Bump rvc4 fw * RVC4 FW: Fix black frame when downscaling 8K frames * Fix test * Make test more reliable [no ci] * Bump rvc4 fw * RVC4 FW: Fix edge case manipv2 bug * Fix april tags example [no ci] * RVC4 FW: Fix ImageManipV2 crash * Add performance mode configuration option [no ci] * Bump rvc4 fw [no ci] * Bump rvc2 fw * Add balanced option to manipv2 perf mode * Bump rvc4 fw * RVC4 FW: Added scale fallbacks * Use performance mode manip by default [no ci] * Bump rvc4 fw [no ci] * Bump rvc2 fw * Fix build [no ci] * RVC4 FW: Fix build * Bump rvc4 fw * Bump rvc2 fw [no ci] * PR fixes * Fix build * Expanded comments on new properties --------- Co-authored-by: Matevz Morato <[email protected]>
1 parent 8493867 commit 6f55df1

File tree

12 files changed

+1260
-1374
lines changed

12 files changed

+1260
-1374
lines changed

‎bindings/python/src/pipeline/node/ImageManipV2Bindings.cpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,23 @@ void bind_imagemanipv2(pybind11::module& m, void* pCallstack) {
2323
///////////////////////////////////////////////////////////////////////
2424
///////////////////////////////////////////////////////////////////////
2525
///////////////////////////////////////////////////////////////////////
26+
py::enum_<ImageManipV2::PerformanceMode> perfMode(imageManip, "PerformanceMode");
27+
perfMode.value("BALANCED", ImageManipV2::PerformanceMode::BALANCED)
28+
.value("PERFORMANCE", ImageManipV2::PerformanceMode::PERFORMANCE)
29+
.value("LOW_POWER", ImageManipV2::PerformanceMode::LOW_POWER);
30+
31+
py::enum_<ImageManipV2::Backend> backend(imageManip, "Backend");
32+
backend.value("HW", ImageManipV2::Backend::HW)
33+
.value("CPU", ImageManipV2::Backend::CPU);
2634

2735
// ImageManip Node
2836
imageManip.def_readonly("inputConfig", &ImageManipV2::inputConfig, DOC(dai, node, ImageManipV2, inputConfig))
2937
.def_readonly("inputImage", &ImageManipV2::inputImage, DOC(dai, node, ImageManipV2, inputImage))
3038
.def_readonly("out", &ImageManipV2::out, DOC(dai, node, ImageManipV2, out))
3139
.def_readonly("initialConfig", &ImageManipV2::initialConfig, DOC(dai, node, ImageManipV2, initialConfig))
3240
.def("setRunOnHost", &ImageManipV2::setRunOnHost, DOC(dai, node, ImageManipV2, setRunOnHost))
41+
.def("setBackend", &ImageManipV2::setBackend, DOC(dai, node, ImageManipV2, setBackend))
42+
.def("setPerformanceMode", &ImageManipV2::setPerformanceMode, DOC(dai, node, ImageManipV2, setPerformanceMode))
3343
.def("setNumFramesPool", &ImageManipV2::setNumFramesPool, DOC(dai, node, ImageManipV2, setNumFramesPool))
3444
.def("setMaxOutputFrameSize", &ImageManipV2::setMaxOutputFrameSize, DOC(dai, node, ImageManipV2, setMaxOutputFrameSize));
3545
}

‎cmake/Depthai/DepthaiDeviceRVC4Config.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@ set(DEPTHAI_DEVICE_RVC4_MATURITY "snapshot")
44

55
# "version if applicable"
66
# set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+93f7b75a885aa32f44c5e9f53b74470c49d2b1af")
7-
set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+c3853e7a42792202633735a204a1b45758371781")
7+
set(DEPTHAI_DEVICE_RVC4_VERSION "0.0.1+eaf4a54b6094d68775b41d8ced3b96a3c9aa0439")

‎cmake/Depthai/DepthaiDeviceSideConfig.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
set(DEPTHAI_DEVICE_SIDE_MATURITY "snapshot")
33

44
# "full commit hash of device side binary"
5-
set(DEPTHAI_DEVICE_SIDE_COMMIT "93eeaff81d82368bbe2c326465b1528f8ae93444")
5+
set(DEPTHAI_DEVICE_SIDE_COMMIT "211744dd005b80f51ceedec4d5a85d4bf59cee43")
66

77
# "version if applicable"
88
set(DEPTHAI_DEVICE_SIDE_VERSION "")

‎examples/python/AprilTags/april_tags_12mp.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
# ATM, AprilTags don't contain timestamps, so we can't sync them with frames
1919
manip = pipeline.create(dai.node.ImageManipV2)
2020
manip.initialConfig.setOutputSize(PREVIEW_SIZE[0], PREVIEW_SIZE[1], dai.ImageManipConfigV2.ResizeMode.STRETCH)
21-
manip.setMaxOutputFrameSize(2004096)
21+
manip.setMaxOutputFrameSize(2162688)
2222
outputCam.link(manip.inputImage)
2323
frameQ = manip.out.createOutputQueue()
2424

‎include/depthai/pipeline/node/ImageManipV2.hpp‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,9 @@ class ImageManipV2 : public DeviceNodeCRTP<DeviceNode, ImageManipV2, ImageManipP
2525
public:
2626
constexpr static const char* NAME = "ImageManipV2";
2727
using DeviceNodeCRTP::DeviceNodeCRTP;
28+
using Backend = ImageManipPropertiesV2::Backend;
29+
using PerformanceMode = ImageManipPropertiesV2::PerformanceMode;
30+
2831
ImageManipV2() = default;
2932
ImageManipV2(std::unique_ptr<Properties> props);
3033

@@ -70,6 +73,18 @@ class ImageManipV2 : public DeviceNodeCRTP<DeviceNode, ImageManipV2, ImageManipP
7073
*/
7174
ImageManipV2& setRunOnHost(bool runOnHost = true);
7275

76+
/**
77+
* Set CPU as backend preference
78+
* @param backend Backend preference
79+
*/
80+
ImageManipV2& setBackend(Backend backend);
81+
82+
/**
83+
* Set performance mode
84+
* @param performanceMode Performance mode
85+
*/
86+
ImageManipV2& setPerformanceMode(PerformanceMode performanceMode);
87+
7388
/**
7489
* Check if the node is set to run on host
7590
*/
@@ -82,8 +97,8 @@ class ImageManipV2 : public DeviceNodeCRTP<DeviceNode, ImageManipV2, ImageManipP
8297
const ImageManipConfigV2& initialConfig,
8398
std::shared_ptr<spdlog::async_logger> logger,
8499
std::function<size_t(const ImageManipConfigV2&, const ImgFrame&)> build,
85-
std::function<bool(std::shared_ptr<Memory>&, span<uint8_t>)> apply,
86-
std::function<void(const ImageManipConfigV2&, const ImgFrame&, ImgFrame&)> getFrame);
100+
std::function<bool(std::shared_ptr<Memory>&, std::shared_ptr<ImageManipData>)> apply,
101+
std::function<void(const ImgFrame&, ImgFrame&)> getFrame);
87102
};
88103

89104
} // namespace node
@@ -97,8 +112,8 @@ void ImageManipV2::loop(N& node,
97112
const ImageManipConfigV2& initialConfig,
98113
std::shared_ptr<spdlog::async_logger> logger,
99114
std::function<size_t(const ImageManipConfigV2&, const ImgFrame&)> build,
100-
std::function<bool(std::shared_ptr<Memory>&, span<uint8_t>)> apply,
101-
std::function<void(const ImageManipConfigV2&, const ImgFrame&, ImgFrame&)> getFrame) {
115+
std::function<bool(std::shared_ptr<Memory>&, std::shared_ptr<ImageManipData>)> apply,
116+
std::function<void(const ImgFrame&, ImgFrame&)> getFrame) {
102117
using namespace std::chrono;
103118
auto config = initialConfig;
104119

@@ -166,10 +181,10 @@ void ImageManipV2::loop(N& node,
166181
bool success = true;
167182
{
168183
auto t3 = steady_clock::now();
169-
success = apply(inImage->data, outImageData->getData());
184+
success = apply(inImage->data, outImageData);
170185
auto t4 = steady_clock::now();
171186

172-
getFrame(config, *inImage, *outImage);
187+
getFrame(*inImage, *outImage);
173188

174189
logger->trace("Build time: {}us, Process time: {}us, Total time: {}us, image manip id: {}",
175190
duration_cast<microseconds>(t2 - t1).count(),

‎include/depthai/properties/ImageManipPropertiesV2.hpp‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,22 @@ namespace dai {
1010
* Specify properties for ImageManip
1111
*/
1212
struct ImageManipPropertiesV2 : PropertiesSerializable<Properties, ImageManipPropertiesV2> {
13+
/**
14+
* Enable hardware accelerated image manipulation if set to HW. Only applied on RVC4.
15+
* This can cause some unexpected behavior when using multiple ImageManipV2 nodes in series.
16+
* Currently, the only operation affected is downscaling.
17+
*/
18+
enum class Backend : uint8_t { CPU, HW };
19+
/**
20+
* Set performance mode for ImageManip with a tradeoff between performance and power consumption. Only applied on RVC4.
21+
* This only affects scaling NV12 and GRAY images.
22+
* - PERFORMANCE: High performance, high power consumption. Uses the OpenCV backend.
23+
* - BALANCED: Balanced performance and power consumption. Uses the FastCV backend configured for high performance where possible with a fallback to
24+
* OpenCV.
25+
* - LOW_POWER: Low performance, low power consumption. Uses the FastCV backend configured for low power where possible with a fallback to OpenCV.
26+
*/
27+
enum class PerformanceMode : uint8_t { PERFORMANCE, BALANCED, LOW_POWER };
28+
1329
/// Initial configuration for ImageManip node
1430
ImageManipConfigV2 initialConfig;
1531

@@ -18,8 +34,12 @@ struct ImageManipPropertiesV2 : PropertiesSerializable<Properties, ImageManipPro
1834

1935
/// Num frames in output pool
2036
int numFramesPool = 4;
37+
38+
/// Using HW backend can cause some unexpected behavior when using multiple ImageManipV2 nodes in series
39+
Backend backend = Backend::CPU;
40+
PerformanceMode performanceMode = PerformanceMode::PERFORMANCE;
2141
};
2242

23-
DEPTHAI_SERIALIZE_EXT(ImageManipPropertiesV2, initialConfig, outputFrameSize, numFramesPool);
43+
DEPTHAI_SERIALIZE_EXT(ImageManipPropertiesV2, initialConfig, outputFrameSize, numFramesPool, backend, performanceMode);
2444

2545
} // namespace dai

0 commit comments

Comments
 (0)