From 7c914c017ebb185eef1f5918b101c926b606d1b7 Mon Sep 17 00:00:00 2001 From: jksemple Date: Mon, 6 May 2024 17:31:49 +0100 Subject: [PATCH 1/5] return-moving-points-ratio Update an external float variable with the movingRatio calculated internally to permit calibration of the detection ration threshold within an app --- src/eloquent_esp32cam/motion/detection.h | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/eloquent_esp32cam/motion/detection.h b/src/eloquent_esp32cam/motion/detection.h index a62f9398..ec19460f 100644 --- a/src/eloquent_esp32cam/motion/detection.h +++ b/src/eloquent_esp32cam/motion/detection.h @@ -100,7 +100,7 @@ namespace Eloquent { /** * */ - Exception& run() { + Exception& run(float& ratio ) { // skip fre first frames if (_skip > 0 && _skip-- > 0) return exception.set(String("Still ") + _skip + " frames to skip..."); @@ -131,7 +131,7 @@ namespace Eloquent { movingRatio = ((float) movingPoints) / camera.rgb565.length * _stride * _stride; copy(camera.rgb565); }); - + ratio = movingRatio; ESP_LOGD("MotionDetection", "moving points ratio: %.2f", movingRatio); // rate limit @@ -143,7 +143,10 @@ namespace Eloquent { return exception.clear(); } - + Exception& run() { + float dummy; + return run(dummy); + } /** * @brief Convert to JSON */ From 9aeb439c453491b585692ac5a24633deb12c018e Mon Sep 17 00:00:00 2001 From: jksemple Date: Tue, 7 May 2024 13:01:24 +0100 Subject: [PATCH 2/5] Anomaly detection Detect anomalies = like motion detection but producing a signal throughout the period that an anomaly is present in the camera frame --- .../Anomaly_Detection/Anomaly_Detection.ino | 78 +++++++ .../Anomaly_Detection_Higher_Resolution.ino | 101 ++++++++ src/eloquent_esp32cam/anomaly/daemon.h | 92 ++++++++ src/eloquent_esp32cam/anomaly/detection.h | 219 ++++++++++++++++++ src/eloquent_esp32cam/anomaly/roi_detection.h | 158 +++++++++++++ 5 files changed, 648 insertions(+) create mode 100644 examples/Anomaly_Detection/Anomaly_Detection.ino create mode 100644 examples/Anomaly_Detection_Higher_Resolution/Anomaly_Detection_Higher_Resolution.ino create mode 100644 src/eloquent_esp32cam/anomaly/daemon.h create mode 100644 src/eloquent_esp32cam/anomaly/detection.h create mode 100644 src/eloquent_esp32cam/anomaly/roi_detection.h diff --git a/examples/Anomaly_Detection/Anomaly_Detection.ino b/examples/Anomaly_Detection/Anomaly_Detection.ino new file mode 100644 index 00000000..0ccc7387 --- /dev/null +++ b/examples/Anomaly_Detection/Anomaly_Detection.ino @@ -0,0 +1,78 @@ +/** + * Anomaly detection + * Detect when the frame changes by a reasonable amount + * + * BE SURE TO SET "TOOLS > CORE DEBUG LEVEL = DEBUG" + * to turn on debug messages + */ +#include +#include + +using eloq::camera; +using eloq::anomaly::detection; + +/** + * + */ +void setup() { + delay(3000); + Serial.begin(115200); + Serial.println("___ANOMALY DETECTION___"); + + // camera settings + // replace with your own model! + camera.pinout.xiao(); + camera.brownout.disable(); + camera.resolution.vga(); + camera.quality.high(); + + // configure anomaly detection + detection.skip(4); + // the higher the stride, the faster the detection + // the higher the stride, the less the granularity + detection.stride(1); + // the higher the threshold, the less the sensitivity + // (at pixel level) + detection.threshold(5); + // the higher the detectionRatio, the less the sensitivity + // (at image level, from 0 to 1) + detection.detectionRatio(0.5); + // the higher the referenceRatio, the more the reference image can change over time + // (at image level, from 0 to 1) + detection.referenceRatio(0.2); + // optionally, you can enable rate limiting (aka debounce) + // anomaly won't trigger more often than the specified frequency + //detection.rate.atMostOnceEvery(5).seconds(); + + // init camera + while (!camera.begin().isOk()) + Serial.println(camera.exception.toString()); + + Serial.println("Camera OK"); + Serial.println("Awaiting anomaly..."); +} + +/** + * + */ +void loop() { + + // Don't run more often than the time for an anomaly to come into view as the reference image can 'drift' away from 'normal' + delay(1000); + // capture picture + if (!camera.capture().isOk()) { + Serial.println(camera.exception.toString()); + return; + } + float anomalyProbability; + // run anomaly detection + if (!detection.run(anomalyProbability).isOk()) { + Serial.println(detection.exception.toString()); + return; + } + + // on anomaly, perform action + if (detection.triggered()) { + Serial.print("Anomaly detected: "); Serial.println(anomalyProbability); + } +} \ No newline at end of file diff --git a/examples/Anomaly_Detection_Higher_Resolution/Anomaly_Detection_Higher_Resolution.ino b/examples/Anomaly_Detection_Higher_Resolution/Anomaly_Detection_Higher_Resolution.ino new file mode 100644 index 00000000..ac01c855 --- /dev/null +++ b/examples/Anomaly_Detection_Higher_Resolution/Anomaly_Detection_Higher_Resolution.ino @@ -0,0 +1,101 @@ +/** + * Run anomaly detection at low resolution. + * On anomaly, capture frame at higher resolution + * for SD storage. + * + * BE SURE TO SET "TOOLS > CORE DEBUG LEVEL = INFO" + * to turn on debug messages + */ +#include +#include + +using eloq::camera; +using eloq::anomaly::detection; + + +/** + * + */ +void setup() { + delay(3000); + Serial.begin(115200); + Serial.println("___ANOMALY DETECTION + SWITCH RESOLUTION___"); + + // camera settings + // replace with your own model! + camera.pinout.freenove_s3(); + camera.brownout.disable(); + camera.resolution.vga(); + camera.quality.high(); + + // see example of anomaly detection for config values + detection.skip(5); + detection.stride(1); + detection.threshold(5); + // the higher the detectionRatio, the less the sensitivity + // (at image level, from 0 to 1) + detection.detectionRatio(0.5); + // the higher the referenceRatio, the more the reference image can change over time + // (at image level, from 0 to 1) + detection.referenceRatio(0.2); + + // init camera + while (!camera.begin().isOk()) + Serial.println(camera.exception.toString()); + + Serial.println("Camera OK"); + Serial.println("Awaiting for anomaly..."); +} + +/** + * + */ +void loop() { + // Don't run more often than the time for an anomaly to come into view as the reference image can 'drift' away from 'normal' + delay(1000); + // capture picture + if (!camera.capture().isOk()) { + Serial.println(camera.exception.toString()); + return; + } + + float anomalyProbability; + // run anomaly detection + if (!detection.run(anomalyProbability).isOk()) { + Serial.println(detection.exception.toString()); + return; + } + + // on anomaly, perform action + if (detection.triggered()) { + Serial.printf( + "Anomaly of %.2f detected on frame of size %dx%d (%d bytes)\n", + anomalyProbability, + camera.resolution.getWidth(), + camera.resolution.getHeight(), + camera.getSizeInBytes() + ); + + Serial.println("Taking photo of anomaly at higher resolution"); + + camera.resolution.at(FRAMESIZE_UXGA, []() { + Serial.printf( + "Switched to higher resolution: %dx%d. It took %d ms to switch\n", + camera.resolution.getWidth(), + camera.resolution.getHeight(), + camera.resolution.benchmark.millis() + ); + + camera.capture(); + + Serial.printf( + "Frame size is now %d bytes\n", + camera.getSizeInBytes() + ); + + // save to SD... + }); + + Serial.println("Resolution switched back to VGA"); + } +} \ No newline at end of file diff --git a/src/eloquent_esp32cam/anomaly/daemon.h b/src/eloquent_esp32cam/anomaly/daemon.h new file mode 100644 index 00000000..4f494a3f --- /dev/null +++ b/src/eloquent_esp32cam/anomaly/daemon.h @@ -0,0 +1,92 @@ +#ifndef ELOQUENT_ESP32CAM_ANOMALY_DAEMON_H +#define ELOQUENT_ESP32CAM_ANOMALY_DAEMON_H + +#include +#include "../camera/camera.h" +#include "../extra/esp32/multiprocessing/thread.h" + +using eloq::camera; +using Eloquent::Extra::Esp32::Multiprocessing::Thread; +using OnAnomalyCallback = std::function; + + +namespace Eloquent { + namespace Esp32cam { + namespace Anomaly { + /** + * Run anomaly detection in a task + * + * @class Daemon + * @author Simone + * @date 13/12/2023 + * @file daemon.h + * @brief + */ + template + class Daemon { + public: + Thread thread; + + /** + * Constructor + * + * @brief + */ + Daemon(T* detection) : + thread("AnomalyDetection"), + _detection(detection) { + + } + + /** + * Run function when a face is detected + * + * @brief + * @param callback + */ + void onAnomaly(OnAnomalyCallback callback) { + _onAnomaly = callback; + } + + /** + * Start anomaly detection in background + * + * @brief + */ + void start() { + thread + .withArgs((void*) this) + .withStackSize(5000) + .run([](void *args) { + Daemon *self = (Daemon*) args; + + delay(3000); + + while (true) { + yield(); + delay(1); + + if (!camera.capture().isOk()) + continue; + + if (!self->_detection->run().isOk()) + continue; + + if (!self->_detection->triggered()) + continue; + + self->_onAnomaly(); + } + }); + } + + protected: + T *_detection; + OnAnomalyCallback _onAnomaly; + }; + } + } +} + + +#endif \ No newline at end of file diff --git a/src/eloquent_esp32cam/anomaly/detection.h b/src/eloquent_esp32cam/anomaly/detection.h new file mode 100644 index 00000000..3ea9630b --- /dev/null +++ b/src/eloquent_esp32cam/anomaly/detection.h @@ -0,0 +1,219 @@ +#ifndef ELOQUENT_ESP32CAM_ANOMALY_DETECTION +#define ELOQUENT_ESP32CAM_ANOMALY_DETECTION + +#include +#include "../extra/exception.h" +#include "../extra/time/benchmark.h" +#include "../extra/time/rate_limit.h" +#include "../extra/pubsub.h" +#include "./daemon.h" + +using eloq::camera; +using Eloquent::Error::Exception; +using Eloquent::Extra::Time::Benchmark; +using Eloquent::Extra::Time::RateLimit; +#if defined(ELOQUENT_EXTRA_PUBSUB_H) +using Eloquent::Extra::PubSub; +#endif + + +namespace Eloquent { + namespace Esp32cam { + namespace Anomaly { + /** + * Detect anomaly using "fast" algorithm + */ + class Detection { + public: + float movingRatio; + Exception exception; + Benchmark benchmark; + RateLimit rate; + Daemon daemon; + #if defined(ELOQUENT_EXTRA_PUBSUB_H) + PubSub mqtt; + #endif + + /** + * + */ + Detection() : + _stride(4), + _threshold(5), + _detectionRatio(0.2), + _referenceRatio(0.05), + _reference(NULL), + _skip(5), + movingRatio(0), + daemon(this), + #if defined(ELOQUENT_EXTRA_PUBSUB_H) + mqtt(this), + #endif + exception("AnomalyDetection") { + + } + + /** + * Set detection stride. + * The greater the value, the less accurate. + * The greater the value, the faster. + */ + void stride(uint8_t stride) { + _stride = stride; + } + + /** + * Set detection sensitivity (pixel level). + * The greater the value, the less sensitive the detection. + */ + void threshold(uint8_t threshold) { + _threshold = threshold; + } + + /** + * @brief Skip first frames (to avoid false detection) + * @param skip + */ + void skip(uint8_t skip) { + _skip = skip; + } + + /** + * Set reference image sensitivity (image level). + * The greater the value, the more the reference image can vary over time. + */ + void referenceRatio(float ratio) { + if (ratio < 0 || ratio >= _detectionRatio) { + ESP_LOGE("AnomalyDetection", "referenceRatio MUST be between 0 (inclusive) and _detectionRatio (exclusive)"); + return; + } + + _referenceRatio = ratio; + } + + /** + * Set detection sensitivity (image level). + * The greater the value, the less sensitive the detection. + */ + void detectionRatio(float ratio) { + if (ratio <= _referenceRatio || ratio > 1) { + ESP_LOGE("AnomalyDetection", "detectionRatio MUST be between _referenceRatio (exclusive) and 1 (inclusive)"); + return; + } + _detectionRatio = ratio; + } + + /** + * Test if anomaly triggered + */ + inline bool triggered() { + return movingRatio >= _detectionRatio; + } + + Exception& setReference() { + // convert JPEG to RGB565 + // this reduces the frame to 1/8th + if (!camera.rgb565.convert().isOk()) + return camera.rgb565.exception; + + if (_reference == NULL) { + _reference = (uint16_t*) malloc(camera.rgb565.length * sizeof(uint16_t)); + } + copy(camera.rgb565); + + return exception.clear(); + } + /** + * + */ + Exception& run(float& ratio ) { + // skip fre first frames + if (_skip > 0 && _skip-- > 0) + return exception.set(String("Still ") + _skip + " frames to skip..."); + + // convert JPEG to RGB565 + // this reduces the frame to 1/8th + if (!camera.rgb565.convert().isOk()) + return camera.rgb565.exception; + + // first frame, only copy frame to prev + if (_reference == NULL) { + _reference = (uint16_t*) malloc(camera.rgb565.length * sizeof(uint16_t)); + copy(camera.rgb565); + + return exception.set("First frame, can't detect anomaly").soft(); + } + + benchmark.timeit([this]() { + int movingPoints = dl::image::get_moving_point_number( + camera.rgb565.data, + _reference, + camera.rgb565.height, + camera.rgb565.width, + _stride, + _threshold + ); + + movingRatio = ((float) movingPoints) / camera.rgb565.length * _stride * _stride; + if (movingRatio < _referenceRatio) { + ESP_LOGD("AnomalyDetection", "Replacing reference frame - referenceRatio = %.2f", movingRatio); + copy(camera.rgb565); + } + }); + ratio = movingRatio; // Update the caller's reference to the movingRatio + ESP_LOGD("AnomalyDetection", "moving points ratio: %.2f", movingRatio); + + // rate limit + if (triggered() && !rate) + return exception.set(rate.getRetryInMessage()).soft(); + + if (triggered()) + rate.touch(); + + return exception.clear(); + } + Exception& run() { + float dummy; + return run(dummy); + } + /** + * @brief Convert to JSON + */ + String toJSON() { + return String("{\"anomaly\":") + (triggered() ? "true" : "false") + "}"; + } + + /** + * @brief Test if an MQTT message should be published + */ + bool shouldPub() { + return triggered(); + } + + protected: + uint8_t _skip; + uint16_t *_reference; + uint8_t _stride; + uint8_t _threshold; + float _detectionRatio; + float _referenceRatio; + + /** + * + */ + template + void copy(Frame frame) { + memcpy((uint8_t*) _reference, (uint8_t*) frame.data, frame.length * sizeof(uint16_t)); + } + }; + } + } +} + +namespace eloq { + namespace anomaly { + static Eloquent::Esp32cam::Anomaly::Detection detection; + } +} + +#endif \ No newline at end of file diff --git a/src/eloquent_esp32cam/anomaly/roi_detection.h b/src/eloquent_esp32cam/anomaly/roi_detection.h new file mode 100644 index 00000000..4b4a2b20 --- /dev/null +++ b/src/eloquent_esp32cam/anomaly/roi_detection.h @@ -0,0 +1,158 @@ +#ifndef ELOQUENT_ESP32CAM_ANOMALY_ROI_DETECTION +#define ELOQUENT_ESP32CAM_ANOMALY_ROI_DETECTION + +#include "./detection.h" + +namespace Eloquent { + namespace Esp32cam { + namespace Anomaly { + /** + * Perform anomaly detection on Region of Interest + */ + class RoI : public Detection { + public: + struct { + uint16_t x; + uint16_t y; + uint16_t width; + uint16_t height; + uint16_t x1; + uint16_t x2; + uint16_t y1; + uint16_t y2; + } coords; + + /** + * + */ + RoI() : + _x(0), + _y(0), + _w(0), + _h(0) { + + } + + /** + * Set x coordinate (top-left corner) + */ + void x(float x) { + _x = x; + } + + /** + * Set y coordinate (top-left corner) + */ + void y(float y) { + _y = y; + } + + /** + * Set width of RoI + */ + void width(float width) { + _w = width; + } + + /** + * Set height of RoI + */ + void height(float height) { + _h = height; + } + + /** + * + */ + void updateCoords(uint16_t width, uint16_t height) { + coords.x = max(0, _x < 1 ? _x * width : _x); + coords.y = max(0, _y < 1 ? _y * height : _y); + coords.width = min(width - coords.x, _w < 1 ? _w * width : _w); + coords.height = min(height - coords.y, _h < 1 ? _h * height : _h); + coords.x1 = coords.x; + coords.y1 = coords.y; + coords.x2 = coords.x + coords.width; + coords.y2 = coords.y + coords.height; + } + + /** + * Detect anomaly + */ + template + Exception& update(Frame& frame) { + if (!_w || !_h) + return exception.set("You MUST set a width and height for the RoI"); + + if (_reference == NULL) { + _reference = (uint8_t*) malloc(_w * _h * sizeof(uint16_t)); + _roi = (uint8_t*) malloc(_w * _h * sizeof(uint16_t)); + copy(frame, _reference); + + return exception.set("First frame, can't detect anomaly").soft(); + } + + updateCoords(frame.width, frame.height); + + benchmark.timeit([this, &frame]() { + copy(frame, _roi); + + int movingPoints = dl::image::get_moving_point_number((uint16_t *) _roi, (uint16_t*) _reference, coords.height, coords.width, _stride, _threshold); + moving_ratio = ((float) movingPoints) / sizeof(_roi) * _stride * _stride; + memcpy(_reference, _roi, sizeof(_reference)); + }); + if (moving_ratio < _referenceRatio) { + // update reference + copy(frame, _reference); + } + + ESP_LOGD( + "RoI AnomalyDetection", + "roi: (x=%d, y=%d, width=%d, height=%d). moving points: %.2f%%", + coords.x, + coords.y, + coords.width, + coords.height, + moving_ratio + ); + if (triggered() && !rate_limiter) + return exception.set(rate.getRetryInMessage()).soft(); + + if (triggered()) + rate_limiter.touch(); + + return exception.clear(); + } + + + protected: + float _x; + float _y; + float _w; + float _h; + uint8_t *_roi; + + /** + * Copy RoI of frame into buffer + */ + template + void copy(Frame& frame, uint8_t *dest) { + for (int i = coords.y1; i < coords.y2; i++) + memcpy( + dest + coords.width * (i - coords.y1) * sizeof(uint16_t), + frame.data + (frame.width * i + coords.x) * sizeof(uint16_t), + coords.width * sizeof(uint16_t) + ); + } + }; + } + } +} + +namespace eloq { + namespace anomaly { + // create class alias + class RoI : public Eloquent::Esp32cam::Anomaly::RoI {}; + } +} + +#endif \ No newline at end of file From 6904276790294512e0b84700c228fdbae95d5b1b Mon Sep 17 00:00:00 2001 From: jksemple Date: Mon, 8 Jul 2024 11:13:19 +0100 Subject: [PATCH 3/5] Update detection.h Remove unnecessary exposure of movingRatio member --- src/eloquent_esp32cam/motion/detection.h | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/eloquent_esp32cam/motion/detection.h b/src/eloquent_esp32cam/motion/detection.h index ec19460f..f345c894 100644 --- a/src/eloquent_esp32cam/motion/detection.h +++ b/src/eloquent_esp32cam/motion/detection.h @@ -100,7 +100,7 @@ namespace Eloquent { /** * */ - Exception& run(float& ratio ) { + Exception& run() { // skip fre first frames if (_skip > 0 && _skip-- > 0) return exception.set(String("Still ") + _skip + " frames to skip..."); @@ -131,7 +131,6 @@ namespace Eloquent { movingRatio = ((float) movingPoints) / camera.rgb565.length * _stride * _stride; copy(camera.rgb565); }); - ratio = movingRatio; ESP_LOGD("MotionDetection", "moving points ratio: %.2f", movingRatio); // rate limit @@ -143,10 +142,6 @@ namespace Eloquent { return exception.clear(); } - Exception& run() { - float dummy; - return run(dummy); - } /** * @brief Convert to JSON */ From 8a9b6e534c44fcd3a279b9d9a55e7e7d61219d74 Mon Sep 17 00:00:00 2001 From: jksemple Date: Sun, 14 Jul 2024 12:56:10 +0100 Subject: [PATCH 4/5] Anomaly detection Anomaly detection --- .../Anomaly_Detection/Anomaly_Detection.ino | 6 +-- .../Anomaly_Detection_Higher_Resolution.ino | 7 ++- src/eloquent_esp32cam/anomaly/daemon.h | 9 ++-- src/eloquent_esp32cam/anomaly/detection.h | 53 +++++++++---------- src/eloquent_esp32cam/anomaly/roi_detection.h | 10 ++-- src/eloquent_esp32cam/camera/rgb_565.h | 10 ++-- src/eloquent_esp32cam/transform/crop.h | 52 ++++++++++++------ 7 files changed, 81 insertions(+), 66 deletions(-) diff --git a/examples/Anomaly_Detection/Anomaly_Detection.ino b/examples/Anomaly_Detection/Anomaly_Detection.ino index 0ccc7387..ef7aae8c 100644 --- a/examples/Anomaly_Detection/Anomaly_Detection.ino +++ b/examples/Anomaly_Detection/Anomaly_Detection.ino @@ -65,14 +65,14 @@ void loop() { return; } float anomalyProbability; + // run anomaly detection - if (!detection.run(anomalyProbability).isOk()) { Serial.println(detection.exception.toString()); return; } // on anomaly, perform action if (detection.triggered()) { - Serial.print("Anomaly detected: "); Serial.println(anomalyProbability); + Serial.print("Anomaly detected: "); Serial.println(detection.movingRatio); } -} \ No newline at end of file + diff --git a/examples/Anomaly_Detection_Higher_Resolution/Anomaly_Detection_Higher_Resolution.ino b/examples/Anomaly_Detection_Higher_Resolution/Anomaly_Detection_Higher_Resolution.ino index ac01c855..9afa0853 100644 --- a/examples/Anomaly_Detection_Higher_Resolution/Anomaly_Detection_Higher_Resolution.ino +++ b/examples/Anomaly_Detection_Higher_Resolution/Anomaly_Detection_Higher_Resolution.ino @@ -59,9 +59,8 @@ void loop() { return; } - float anomalyProbability; // run anomaly detection - if (!detection.run(anomalyProbability).isOk()) { + if (!detection.run().isOk()) { Serial.println(detection.exception.toString()); return; } @@ -70,7 +69,7 @@ void loop() { if (detection.triggered()) { Serial.printf( "Anomaly of %.2f detected on frame of size %dx%d (%d bytes)\n", - anomalyProbability, + detection.movingRatio, camera.resolution.getWidth(), camera.resolution.getHeight(), camera.getSizeInBytes() @@ -87,7 +86,7 @@ void loop() { ); camera.capture(); - + Serial.printf( "Frame size is now %d bytes\n", camera.getSizeInBytes() diff --git a/src/eloquent_esp32cam/anomaly/daemon.h b/src/eloquent_esp32cam/anomaly/daemon.h index 4f494a3f..afbb8cb8 100644 --- a/src/eloquent_esp32cam/anomaly/daemon.h +++ b/src/eloquent_esp32cam/anomaly/daemon.h @@ -17,8 +17,8 @@ namespace Eloquent { * Run anomaly detection in a task * * @class Daemon - * @author Simone - * @date 13/12/2023 + * @author jksemple + * @date 11/07/2024 * @file daemon.h * @brief */ @@ -39,7 +39,7 @@ namespace Eloquent { } /** - * Run function when a face is detected + * Run function when a difference from 'normal' is detected * * @brief * @param callback @@ -88,5 +88,4 @@ namespace Eloquent { } } - -#endif \ No newline at end of file +#endif diff --git a/src/eloquent_esp32cam/anomaly/detection.h b/src/eloquent_esp32cam/anomaly/detection.h index 3ea9630b..c6971297 100644 --- a/src/eloquent_esp32cam/anomaly/detection.h +++ b/src/eloquent_esp32cam/anomaly/detection.h @@ -33,14 +33,15 @@ namespace Eloquent { #if defined(ELOQUENT_EXTRA_PUBSUB_H) PubSub mqtt; #endif - + /** * */ Detection() : _stride(4), _threshold(5), - _detectionRatio(0.2), + _lowerDetectionRatio(0.2), + _upperDetectionRatio(0.5), _referenceRatio(0.05), _reference(NULL), _skip(5), @@ -69,7 +70,7 @@ namespace Eloquent { void threshold(uint8_t threshold) { _threshold = threshold; } - + /** * @brief Skip first frames (to avoid false detection) * @param skip @@ -83,11 +84,6 @@ namespace Eloquent { * The greater the value, the more the reference image can vary over time. */ void referenceRatio(float ratio) { - if (ratio < 0 || ratio >= _detectionRatio) { - ESP_LOGE("AnomalyDetection", "referenceRatio MUST be between 0 (inclusive) and _detectionRatio (exclusive)"); - return; - } - _referenceRatio = ratio; } @@ -95,42 +91,45 @@ namespace Eloquent { * Set detection sensitivity (image level). * The greater the value, the less sensitive the detection. */ - void detectionRatio(float ratio) { - if (ratio <= _referenceRatio || ratio > 1) { - ESP_LOGE("AnomalyDetection", "detectionRatio MUST be between _referenceRatio (exclusive) and 1 (inclusive)"); - return; - } - _detectionRatio = ratio; + void lowerDetectionRatio(float ratio) { + _lowerDetectionRatio = ratio; } + /** + * Set maximum detection sensitivity (image level). + * This protects against false detections when the light level across the whole image changes + * Useful when you know objects being detected will never fill more than a fraction of the image + */ + void upperDetectionRatio(float ratio) { + _upperDetectionRatio = ratio; + } /** * Test if anomaly triggered */ inline bool triggered() { - return movingRatio >= _detectionRatio; + return movingRatio >= _lowerDetectionRatio && movingRatio <= _upperDetectionRatio; } - + Exception& setReference() { // convert JPEG to RGB565 - // this reduces the frame to 1/8th if (!camera.rgb565.convert().isOk()) return camera.rgb565.exception; if (_reference == NULL) { - _reference = (uint16_t*) malloc(camera.rgb565.length * sizeof(uint16_t)); + _reference = (uint16_t*) ps_malloc(camera.rgb565.length * sizeof(uint16_t)); } copy(camera.rgb565); - + return exception.clear(); } /** * */ - Exception& run(float& ratio ) { + Exception& run() { // skip fre first frames if (_skip > 0 && _skip-- > 0) return exception.set(String("Still ") + _skip + " frames to skip..."); - + // convert JPEG to RGB565 // this reduces the frame to 1/8th if (!camera.rgb565.convert().isOk()) @@ -160,7 +159,6 @@ namespace Eloquent { copy(camera.rgb565); } }); - ratio = movingRatio; // Update the caller's reference to the movingRatio ESP_LOGD("AnomalyDetection", "moving points ratio: %.2f", movingRatio); // rate limit @@ -172,17 +170,13 @@ namespace Eloquent { return exception.clear(); } - Exception& run() { - float dummy; - return run(dummy); - } /** * @brief Convert to JSON */ String toJSON() { return String("{\"anomaly\":") + (triggered() ? "true" : "false") + "}"; } - + /** * @brief Test if an MQTT message should be published */ @@ -195,7 +189,8 @@ namespace Eloquent { uint16_t *_reference; uint8_t _stride; uint8_t _threshold; - float _detectionRatio; + float _lowerDetectionRatio; + float _upperDetectionRatio; float _referenceRatio; /** @@ -216,4 +211,4 @@ namespace eloq { } } -#endif \ No newline at end of file +#endif diff --git a/src/eloquent_esp32cam/anomaly/roi_detection.h b/src/eloquent_esp32cam/anomaly/roi_detection.h index 4b4a2b20..412d9b45 100644 --- a/src/eloquent_esp32cam/anomaly/roi_detection.h +++ b/src/eloquent_esp32cam/anomaly/roi_detection.h @@ -84,8 +84,8 @@ namespace Eloquent { return exception.set("You MUST set a width and height for the RoI"); if (_reference == NULL) { - _reference = (uint8_t*) malloc(_w * _h * sizeof(uint16_t)); - _roi = (uint8_t*) malloc(_w * _h * sizeof(uint16_t)); + _reference = (uint8_t*) ps_malloc(_w * _h * sizeof(uint16_t)); + _roi = (uint8_t*) ps_malloc(_w * _h * sizeof(uint16_t)); copy(frame, _reference); return exception.set("First frame, can't detect anomaly").soft(); @@ -97,14 +97,14 @@ namespace Eloquent { copy(frame, _roi); int movingPoints = dl::image::get_moving_point_number((uint16_t *) _roi, (uint16_t*) _reference, coords.height, coords.width, _stride, _threshold); - moving_ratio = ((float) movingPoints) / sizeof(_roi) * _stride * _stride; + movingRatio = ((float) movingPoints) / sizeof(_roi) * _stride * _stride; memcpy(_reference, _roi, sizeof(_reference)); }); - if (moving_ratio < _referenceRatio) { + if (movingRatio < _referenceRatio) { // update reference copy(frame, _reference); } - + ESP_LOGD( "RoI AnomalyDetection", "roi: (x=%d, y=%d, width=%d, height=%d). moving points: %.2f%%", diff --git a/src/eloquent_esp32cam/camera/rgb_565.h b/src/eloquent_esp32cam/camera/rgb_565.h index 68dd0e11..c11cee44 100644 --- a/src/eloquent_esp32cam/camera/rgb_565.h +++ b/src/eloquent_esp32cam/camera/rgb_565.h @@ -21,6 +21,7 @@ namespace Eloquent { size_t length; size_t width; size_t height; + jpg_scale_t scaling; /** * @@ -30,7 +31,8 @@ namespace Eloquent { camera(cam), length(0), width(0), - height(0) { + height(0), + scaling(JPG_SCALE_8X) { } /** @@ -94,8 +96,8 @@ namespace Eloquent { return exception.set("Can't convert empty frame to RGB565"); if (!width) { - width = camera->resolution.getWidth() / 8; - height = camera->resolution.getHeight() / 8; + width = camera->resolution.getWidth() >> scaling; + height = camera->resolution.getHeight() >> scaling; length = width * height; ESP_LOGI("Camera", "Allocating %d bytes for %dx%d RGB565 image", length * 2, width, height); @@ -106,7 +108,7 @@ namespace Eloquent { return exception.set("Cannot allocate memory"); camera->mutex.threadsafe([this]() { - if (!jpg2rgb565(camera->frame->buf, camera->frame->len, (uint8_t*) data, JPG_SCALE_8X)) + if (!jpg2rgb565(camera->frame->buf, camera->frame->len, (uint8_t*) data, scaling)) exception.set("Error converting frame from JPEG to RGB565"); }); diff --git a/src/eloquent_esp32cam/transform/crop.h b/src/eloquent_esp32cam/transform/crop.h index b45fd9b5..d018a683 100644 --- a/src/eloquent_esp32cam/transform/crop.h +++ b/src/eloquent_esp32cam/transform/crop.h @@ -42,8 +42,8 @@ namespace Eloquent { _src.height = height; _src.x1 = 0; _src.y1 = 0; - _src.x2 = width; - _src.y2 = height; + _src.x2 = width - 1; + _src.y2 = height - 1; return *this; } @@ -64,8 +64,8 @@ namespace Eloquent { _out.height = height; _out.x1 = 0; _out.y1 = 0; - _out.x2 = width; - _out.y2 = height; + _out.x2 = width - 1; + _out.y2 = height - 1; return *this; } @@ -93,13 +93,13 @@ namespace Eloquent { */ Crop& squash() { _src.x1 = 0; - _src.x2 = _src.width; + _src.x2 = _src.width - 1; _src.y1 = 0; - _src.y2 = _src.height; + _src.y2 = _src.height - 1; _out.x1 = 0; - _out.x2 = _out.width; + _out.x2 = _out.width - 1; _out.y1 = 0; - _out.y2 = _out.height; + _out.y2 = _out.height - 1; return *this; } @@ -114,12 +114,14 @@ namespace Eloquent { _src.x1 = dx; _src.y1 = dy; - _src.x2 = _src.width - dx; - _src.y2 = _src.height - dy; + _src.x2 = _src.width - dx - 1; + _src.y2 = _src.height - dy - 1; _out.x1 = 0; _out.y1 = 0; - _out.x2 = _out.width; - _out.y2 = _out.height; + _out.x2 = _out.width - 1; + _out.y2 = _out.height - 1; + + return *this; } else if (_out.width > _src.width) { uint16_t dx = (_out.width - _src.width) / 2; @@ -127,17 +129,35 @@ namespace Eloquent { _out.x1 = dx; _out.y1 = dy; - _out.x2 = _out.width - dx; - _out.y2 = _out.height - dy; + _out.x2 = _out.width - dx - 1; + _out.y2 = _out.height - dy - 1; _src.x1 = 0; _src.y1 = 0; - _src.x2 = _src.width; - _src.y2 = _src.height; + _src.x2 = _src.width - 1; + _src.y2 = _src.height - 1; } return *this; } + /** + * Manually set crop area origin + * @param x + * @param y + * @return + */ + Crop& offset(int16_t x, int16_t y) { + if (x < 0) x += _src.width; + if (y < 0) y += _src.height; + + _src.x1 = x; + _src.x2 = x + _out.width - 1; + _src.y1 = y; + _src.y2 = y + _out.height - 1; + + return *this; + } + /** * No interpolation */ From 67c4353bcc531160bad156e7911ce643c5e187bf Mon Sep 17 00:00:00 2001 From: jksemple Date: Sun, 14 Jul 2024 12:56:32 +0100 Subject: [PATCH 5/5] Update Anomaly_Detection.ino --- examples/Anomaly_Detection/Anomaly_Detection.ino | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/Anomaly_Detection/Anomaly_Detection.ino b/examples/Anomaly_Detection/Anomaly_Detection.ino index ef7aae8c..c5aafe1f 100644 --- a/examples/Anomaly_Detection/Anomaly_Detection.ino +++ b/examples/Anomaly_Detection/Anomaly_Detection.ino @@ -64,9 +64,9 @@ void loop() { Serial.println(camera.exception.toString()); return; } - float anomalyProbability; // run anomaly detection + if (!detection.run().isOk()) { Serial.println(detection.exception.toString()); return; } @@ -75,4 +75,5 @@ void loop() { if (detection.triggered()) { Serial.print("Anomaly detected: "); Serial.println(detection.movingRatio); } +}