Merge branch 'ai_server' of github.com:connortechnology/zoneminder into ai_server

Author: connortechnology

misc/zoneminder.gortc.service

@@ -0,0 +1,21 @@
+[Unit]
+Description=go2rtc for use with ZoneMinder
+After=network.target
+
+[Service]
+# Does not have to be www-data.  Could also be an unprivileged user
+User=root
+Type=simple
+#Where ever you put go2rtc
+WorkingDirectory=/opt/go2rtc
+# This first entry will get overwritten when you add streams and can get wiped out.  Place the permanent config in the second file
+ExecStart=/opt/go2rtc/go2rtc_linux_arm64 -c go2rtc.yaml -c /etc/zm/go2rtc.yaml
+Restart=always
+RuntimeMaxSec=1d
+RestartSec=10
+Environment=TZ=:/etc/localtime
+TimeoutSec=600
+
+[Install]
+WantedBy=multi-user.target
+

scripts/ZoneMinder/lib/ZoneMinder/Event.pm

@@ -419,7 +419,7 @@ sub delete {
     $ZoneMinder::Database::dbh->commit() if ! $in_transaction;
 
     my $storage = $event->Storage();
-    $storage->save({DiskSpace=>$storage->DiskSpace()-$event->DiskSpace()}) if $event->DiskSpace();
+    $storage->save({DiskSpace=>$storage->DiskSpace()-$event->DiskSpace()}) if $event->DiskSpace() and $storage->Id();
   }
 
   if ( ( $in_zmaudit or (!$Config{ZM_OPT_FAST_DELETE})) and $event->Storage()->DoDelete() ) {
@@ -782,8 +782,8 @@ sub MoveTo {
   $ZoneMinder::Database::dbh->commit() if !$was_in_transaction;
 
   # Update storage diskspace.  The triggers no longer do this. This is ... less important so do it outside the transaction
-  $OldStorage->save({DiskSpace => $OldStorage->DiskSpace()-$old_diskspace}) if $old_diskspace;
-  $NewStorage->save({DiskSpace => $NewStorage->DiskSpace()+$new_diskspace}) if $new_diskspace;
+  $OldStorage->save({DiskSpace => $OldStorage->DiskSpace()-$old_diskspace}) if $old_diskspace and $OldStorage->Id();
+  $NewStorage->save({DiskSpace => $NewStorage->DiskSpace()+$new_diskspace}) if $new_diskspace and $NewStorage->Id();
 
   $self->delete_files($OldStorage);
   return $error;

src/zm_ai_server.cpp

@@ -473,7 +473,7 @@ void AIThread::Run() {
       packet->image_index = image_index;
 
       {
-        std::unique_lock<std::mutex> lck(mutex_);
+        std::lock_guard<std::mutex> lck(mutex_);
         send_queue.push_back(packet);
       }
       Debug(4, "send queue size %zu", send_queue.size());
@@ -508,8 +508,7 @@ void AIThread::Run() {
     }  // end if have a new image
   }  // end while !zm_terminate
   if (monitor_->ShmValid()) shared_data->analysis_image_count = 0;
-} // end SpeedAIDetect   
-
+} // end void AIThread::Run()
 
 int draw_boxes(Image *in_image, Image *out_image, const nlohmann::json &coco_object, int font_size, int line_width=2) {
   out_image->Assign(*in_image);

src/zm_db.cpp

@@ -138,6 +138,7 @@ MYSQL_RES *zmDbFetch(const std::string &query) {
   }
 
 
+  Debug(3, "Doing Fetch %s", query.c_str());
   int rc = mysql_query(&dbconn, query.c_str());
   if (rc) {
     std::string reason = mysql_error(&dbconn); // store for later because it will get clobbered by second attempt

src/zm_image.cpp

@@ -5913,13 +5913,13 @@ int Image::draw_boxes(
     if (coco_object.size()) {
       for (auto it = coco_object.begin(); it != coco_object.end(); ++it) {
         nlohmann::json detection = *it;
-        Debug(1, "CURL detections detection %s", detection.dump().c_str());
-        nlohmann::json bbox = detection["box"];
+        Debug(1, "detections detection %s", detection.dump().c_str());
+        nlohmann::json bbox = detection["box"] == nullptr ? detection["bbox"] : detection["box"];
         if ( bbox == nullptr) {
           Debug(1, "Null box");
           continue;
         }
-        Debug(1, "CURL detections box %s", bbox.dump().c_str());
+        Debug(1, "detections box %s", bbox.dump().c_str());
         if ( bbox == nullptr) {
           Debug(1, "Null box");
           continue;
@@ -5931,7 +5931,7 @@ int Image::draw_boxes(
         int x2 = bbox[2];
         int y2 = bbox[3];
         std::string coco_class = detection["class"];
-        float score = detection["confidence"];
+        float score = detection["confidence"] != nullptr ? detection["confidence"] : detection["score"];
         std::string annotation = stringtf("%s %d%%", coco_class.c_str(), static_cast<int>(100*score));
 
 #if 0

src/zm_monitor.cpp

@@ -36,6 +36,9 @@
 #include "zm_utils.h"
 #include "zm_zone.h"
 
+#define DEBUG_TIMING 1
+#define AI_IN_DECODE 0
+
 #if ZM_HAS_V4L2
 #include "zm_local_camera.h"
 #endif  // ZM_HAS_V4L2
@@ -302,6 +305,10 @@ Monitor::Monitor() :
 #if HAVE_QUADRA
   //quadra(nullptr),
   quadra_yolo(nullptr),
+#endif
+#if HAVE_MX_ACCL
+  mx_accl(nullptr),
+  mx_accl_job(nullptr),
 #endif
   //nlohmann::json last_detections;
   last_detection_count(0),
@@ -404,6 +411,8 @@ void Monitor::Load(MYSQL_ROW dbrow, bool load_zones = true, Purpose p = QUERY) {
     objectdetection = OBJECT_DETECTION_NONE;
   } else if (od == "memx") {
     objectdetection = OBJECT_DETECTION_MEMX;
+  } else if (od == "mx_accl") {
+    objectdetection = OBJECT_DETECTION_MX_ACCL;
   } else if (od == "quadra") {
     objectdetection = OBJECT_DETECTION_QUADRA;
   } else if (od == "speedai") {
@@ -2140,10 +2149,17 @@ int Monitor::Analyse() {
                     packet->ai_image = new Image(*analysis_image_buffer[packet->image_index % image_buffer_count]);
                   }
                 }
+#if !AI_IN_DECODE
 #if HAVE_QUADRA
-                else {
-                  //std::pair<int, std::string> results = Analyse_Quadra(packet);
+                else if (objectdetection == OBJECT_DETECTION_QUADRA) {
+                  std::pair<int, std::string> results = Analyse_Quadra(packet);
+                }
+#endif
+#if HAVE_MX_ACCL_H
+                else if (objectdetection == OBJECT_DETECTION_MX_ACCL) {
+                  std::pair<int, std::string> results = Analyse_MxAccl(packet);
                 }
+#endif
 #endif
                 if (objectdetection == OBJECT_DETECTION_UVICORN) {
                   std::pair<int, std::string> results = Analyse_UVICORN(packet);
@@ -2423,7 +2439,11 @@ int Monitor::Analyse() {
       packetqueue.clearPackets(packet);
 
       unsigned int index = (shared_data->last_analysis_index+1) % image_buffer_count;
-      if (objectdetection == OBJECT_DETECTION_QUADRA || objectdetection == OBJECT_DETECTION_UVICORN) {
+      if (
+          objectdetection == OBJECT_DETECTION_QUADRA 
+          || objectdetection == OBJECT_DETECTION_MX_ACCL 
+          || objectdetection == OBJECT_DETECTION_UVICORN
+          ) {
         // Only do these if it's a video packet.
         if (packet->ai_frame) {
           analysis_image_buffer[index]->AVPixFormat(static_cast<AVPixelFormat>(packet->ai_frame->format));
@@ -2477,6 +2497,8 @@ int Monitor::Analyse() {
         shared_data->last_analysis_index = index;
         shared_data->last_read_index = index;
         shared_data->analysis_image_count = analysis_image_count;
+      } else {
+        Warning("Unknown value for Object_Detection");
       }
     } else {
       Debug(3, "Not video, not clearing packets");
@@ -2512,6 +2534,67 @@ int Monitor::Analyse() {
   return 1;
 } // end Monitor::Analyse
 
+#if HAVE_MX_ACCL_H
+std::pair<int, std::string> Monitor::Analyse_MxAccl(std::shared_ptr<ZMPacket> packet) {
+  int score = 0;
+  std::string cause;
+
+  if (packet->needs_hw_transfer(mVideoCodecContext))
+    packet->get_hwframe(mVideoCodecContext);
+  AVFrame *frame = packet->in_frame.get();
+
+  if (!mx_accl and mVideoCodecContext) {
+    mx_accl = new MxAccl();
+    if (!mx_accl->setup( "", (staticConfig.DIR_MODELS+"/"+objectdetection_model))) {
+      Warning("Failed setting up Mx_Accl");
+      delete mx_accl;
+      mx_accl = nullptr;
+      return std::make_pair(0, "");
+    }
+    mx_accl_job = mx_accl->get_job();
+  }
+
+  if (frame) {
+    if (!(shared_data->analysis_image_count % (motion_frame_skip+1))) {
+#if DEBUG_TIMING
+      SystemTimePoint starttime = std::chrono::system_clock::now();
+#endif
+      int ret = packet->image ? mx_accl->send_image(mx_accl_job, packet->image) : mx_accl->send_frame(mx_accl_job, frame);
+      if (ret <= 0) {
+        Debug(1, "Can't send_packet %d", packet->image_index);
+        return std::make_pair(ret, cause);
+      }
+#if DEBUG_TIMING 
+      SystemTimePoint endtime = std::chrono::system_clock::now();
+      if (endtime - starttime > Seconds(1)) {
+        Warning("AI is to slow: %.2f seconds", FPSeconds(endtime - starttime).count());
+      } else {
+        Debug(1, "AI took: %.2f seconds", FPSeconds(endtime - starttime).count());
+      }
+#endif
+      last_detection_count = 2;
+      const nlohmann::json detections = mx_accl->receive_detections(mx_accl_job, objectdetection_object_threshold);
+      if (detections.size()) {
+        Image *ai_image = new Image(packet->in_frame.get(), packet->in_frame->width, packet->in_frame->height); //copies
+        ai_image->draw_boxes(detections, LabelSize(), LabelSize());
+        packet->ai_image = ai_image;
+        // Populate ai_frame as well
+        packet->ai_frame = av_frame_ptr(av_frame_alloc());
+        ai_image->PopulateFrame(packet->ai_frame.get());
+        packet->detections = detections;
+      }
+    } else { // skipping
+      if (last_detection_count>0) {
+        last_detection_count --;
+        //TODO last_detection shouldn't be AI specific
+        //quadra_yolo->draw_last_roi(packet);
+      }
+    } // end if skip_frame
+  } // end if has input_frame/hw_frame
+  return std::make_pair(score, cause);
+} // end Monitor::Analyse_MxAccl(Packet)
+#endif
+
 #if HAVE_QUADRA
 std::pair<int, std::string> Monitor::Analyse_Quadra(std::shared_ptr<ZMPacket> packet) {
   int score = 0;
@@ -3581,14 +3664,18 @@ int Monitor::Decode() {
     Debug(1, "Assigning %s for index %d to %s", packet->image->toString().c_str(), index, image_buffer[index]->toString().c_str());
     image_buffer[index]->Assign(*(packet->image));
   } else if (packet->in_frame) {
-#if HAVE_QUADRA
-    //if (!quadra_yolo)
-#endif
     {
+#if AI_IN_DECODE
+
 #if HAVE_QUADRA
       if (objectdetection == OBJECT_DETECTION_QUADRA) {
         std::pair<int, std::string> results = Analyse_Quadra(packet);
       }
+#endif
+#if HAVE_MX_ACCL_H
+      if (objectdetection == OBJECT_DETECTION_MX_ACCL) {
+        std::pair<int, std::string> results = Analyse_MxAccl(packet);
+      }
 #endif
       if (packet->ai_frame) {
         Debug(1, "Assigning ai_frame for index %d", index);
@@ -3604,6 +3691,8 @@ int Monitor::Decode() {
         image_buffer[index]->AVPixFormat(image_pixelformats[index] = static_cast<AVPixelFormat>(packet->in_frame->format));
         image_buffer[index]->Assign(packet->in_frame.get());
       }
+#endif // AI_IN_DECODE
+       
       //if (packet->detections.size())
         //zm_terminate = true;
       if (objectdetection == OBJECT_DETECTION_SPEEDAI) {

src/zm_monitor.h

@@ -57,6 +57,9 @@ extern "C" {
 }
 #include "zm_netint_yolo.h"
 #endif
+#if HAVE_MX_ACCL_H
+#include "zm_mx_accl.h"
+#endif
 
 class Group;
 class MonitorLinkExpression;
@@ -100,6 +103,7 @@ class Monitor : public std::enable_shared_from_this<Monitor> {
     OBJECT_DETECTION_SPEEDAI,
     OBJECT_DETECTION_UVICORN,
     OBJECT_DETECTION_MEMX,
+    OBJECT_DETECTION_MX_ACCL,
   } ObjectDetectionOption;
 
   typedef enum {
@@ -789,6 +793,10 @@ class Monitor : public std::enable_shared_from_this<Monitor> {
   //Quadra_Yolo *quadra;
   Quadra_Yolo *quadra_yolo;
   std::mutex   quadra_mutex;
+#endif
+#if HAVE_MX_ACCL_H
+  MxAccl *mx_accl;
+  MxAccl::Job *mx_accl_job;
 #endif
   nlohmann::json last_detections;
   int last_detection_count;
@@ -1077,6 +1085,9 @@ class Monitor : public std::enable_shared_from_this<Monitor> {
   bool CheckSignal( const Image *image );
   int Analyse();
   std::pair<int, std::string> Analyse_Quadra(std::shared_ptr<ZMPacket> packet);
+#if HAVE_MX_ACCL_H
+  std::pair<int, std::string> Analyse_MxAccl(std::shared_ptr<ZMPacket> packet);
+#endif
   struct transfer
 {
     uint8_t *buf;