Merge branch 'develop'

Author: bafu

CONTRIBUTING.md

@@ -0,0 +1,7 @@
+We use [Developer Certificate of Origin](https://developercertificate.org/).
+
+To use DCO, you only need to add your signature into a Git commit:
+
+```
+$ git commit -s -m "your commit message."
+```

README.md

@@ -78,16 +78,16 @@ For more details about dashboard configuration (e.g. how to add widgets), please
 
 # Provide Image Input
 
-To capture an image via Pi camera
+To capture an image via configured IP camera
 
 ```
-$ mosquitto_pub -h localhost -t berrynet/event/camera -m snapshot_picam
+$ mosquitto_pub -h localhost -t berrynet/event/camera -m snapshot_ipcam
 ```
 
-To capture an image via configured IP camera
+To capture an image via board-connected camera (RPi camera or USB webcam)
 
 ```
-$ mosquitto_pub -h localhost -t berrynet/event/camera -m snapshot_ipcam
+$ mosquitto_pub -h localhost -t berrynet/event/camera -m snapshot_boardcam
 ```
 
 To provide a local image
@@ -96,6 +96,49 @@ To provide a local image
 $ mosquitto_pub -h localhost -t berrynet/event/localImage -m <image_path>
 ```
 
+To start and stop streaming from board-connected camera
+
+```
+$ mosquitto_pub -h localhost -t berrynet/event/camera -m stream_boardcam_start
+$ mosquitto_pub -h localhost -t berrynet/event/camera -m stream_boardcam_stop
+```
+
+To start and stop streaming from Nest IP camera
+
+```
+$ mosquitto_pub -h localhost -t berrynet/event/camera -m stream_nest_ipcam_start
+$ mosquitto_pub -h localhost -t berrynet/event/camera -m stream_nest_ipcam_stop
+```
+
+
+# Enable Data Collector
+
+You might want to store the snapshot and inference results for data analysis.
+
+To enable data collector, you can set the storage directory path in config.js:
+
+```
+config.storageDirPath = '<data-storage-dirpath>';
+```
+
+and restart BerryNet.
+
+
+# Use Your Data To Train
+
+The original instruction of retraining YOLOv2 model see [github repository of darknet](https://github.com/AlexeyAB/darknet#how-to-train-to-detect-your-custom-objects)
+
+In the current of BerryNet, TinyYolo is used instead of YOLOv2. 
+The major differences are:
+
+1. Create file yolo-obj.cfg with the same content as in `tiny-yolo.cfg`
+2. Download pre-trained weights of darknet reference model, `darknet.weights.12`, for the convolutional layers (6.1MB)
+https://drive.google.com/drive/folders/0B-oZJEwmkAObMzAtc2QzZDhyVGM?usp=sharing
+
+The rest parts are the same as retraining YOLO.
+
+If you use [LabelMe](http://labelme.csail.mit.edu/Release3.0/) to annotate data, `utils/xmlTotxt.py` can help convert the xml format to the text format that darknet uses.
+
 
 # Discussion
 

berrynet-manager

@@ -1,19 +1,19 @@
 #! /bin/sh
 #
 # Copyright 2017 DT42
-# 
+#
 # This file is part of BerryNet.
-# 
+#
 # BerryNet is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
-# 
+#
 # BerryNet is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with BerryNet.  If not, see <http://www.gnu.org/licenses/>.
 
@@ -26,25 +26,34 @@ help() {
     exit 1
 }
 
-if [ $# -lt 1 ]
-    then
-        help
+if [ $# -lt 1 ]; then
+    help
 fi
 
 case $1 in
-  start | stop | status)
-	sudo systemctl $1 detection_server.service agent.service broker.service dashboard.service localimg.service camera.service journal.service cleaner.timer
-    ;;
-  log)
-	sudo journalctl -x --no-pager -u detection_server.service
+    start | stop | status)
+	sudo systemctl $1 \
+            detection_fast_server.service \
+            agent.service \
+            broker.service \
+            dashboard.service \
+            localimg.service \
+            camera.service \
+            journal.service \
+            data_collector.service \
+            line.service
+        ;;
+    log)
+	sudo journalctl -x --no-pager -u detection_fast_server.service
 	sudo journalctl -x --no-pager -u agent.service
 	sudo journalctl -x --no-pager -u broker.service
 	sudo journalctl -x --no-pager -u dashboard.service
 	sudo journalctl -x --no-pager -u localimg.service
 	sudo journalctl -x --no-pager -u camera.service
 	sudo journalctl -x --no-pager -u journal.service
-	sudo journalctl -x --no-pager -u cleaner.timer
-    ;;
-  *)
-    help
-esac 
+	sudo journalctl -x --no-pager -u data_collector.service
+	sudo journalctl -x --no-pager -u line.service
+        ;;
+    *)
+        help
+esac

camera.js

@@ -22,6 +22,7 @@ const mqtt = require('mqtt');
 const request = require('request');
 const spawnsync = require('child_process').spawnSync;
 const config = require('./config');
+const cv = require('opencv');
 
 const broker = config.brokerHost;
 const client = mqtt.connect(broker);
@@ -30,10 +31,21 @@ const topicActionInference = config.topicActionInference;
 const topicEventCamera = config.topicEventCamera;
 const cameraURI = config.ipcameraSnapshot;
 const snapshotFile = '/tmp/snapshot.jpg';
+const snapshotWidth = config.boardcameraImageWidth;
+const snapshotHeight = config.boardcameraImageHeight;
 const cameraCmd = '/usr/bin/raspistill';
 const cameraArgs = ['-vf', '-hf',
-  '-w', '1024', '-h', '768',
+  '-w', snapshotWidth,
+  '-h', snapshotHeight,
   '-o', snapshotFile];
+const usbCameraCmd = '/usr/bin/fswebcam';
+const usbCameraArgs = ['-r', snapshotWidth + 'x' + snapshotHeight,
+  '--no-banner', '-D', '0.5', snapshotFile];
+const fps = 30;
+var cameraIntervalID = null;
+var cameraInterval = 1000.0 / fps;
+var cameraCV = null;
+var frameCounter = 0;
 
 function log(m) {
   client.publish(topicActionLog, m);
@@ -50,6 +62,13 @@ client.on('message', (t, m) => {
 
   const action = m.toString();
   if (action == 'snapshot_picam') {
+    /* NOTE: We use V4L2 to support RPi camera, so RPi camera's usage is
+     *       the same as USB camera. Both RPi and USB cameras are called
+     *       "board camera".
+     *
+     *       This action is obsoleted and will be removed in the future.
+     */
+
     // Take a snapshot from RPi3 camera. The snapshot will be displayed
     // on dashboard.
     spawnsync(cameraCmd, cameraArgs);
@@ -75,6 +94,75 @@ client.on('message', (t, m) => {
         }
       }
     );
+  } else if (action == 'snapshot_boardcam') {
+    // Take a snapshot from USB camera.
+    spawnsync(usbCameraCmd, usbCameraArgs);
+    fs.readFile(snapshotFile, function(err, data) {
+      if (err) {
+        log('camera client: cannot get image.');
+      } else {
+        log('camera client: publishing image.');
+        client.publish(topicActionInference, data);
+      }
+    });
+  } else if (action == 'stream_boardcam_start') {
+    if ((!cameraCV) && (!cameraIntervalID)) {
+      cameraCV = new cv.VideoCapture(0);
+      cameraCV.setWidth(snapshotWidth);
+      cameraCV.setHeight(snapshotHeight);
+      cameraIntervalID = setInterval(function() {
+        cameraCV.read(function(err, im) {
+          if (err) {
+            throw err;
+          }
+          if (frameCounter < fps * 2) {
+            frameCounter++;
+          } else {
+            frameCounter = 0;
+            im.save(snapshotFile);
+            fs.readFile(snapshotFile, function(err, data) {
+              if (err) {
+                log('camera client: cannot get image.');
+              } else {
+                log('camera client: publishing image.');
+                client.publish(topicActionInference, data);
+              }
+            });
+          }
+          im.release();
+        });
+      }, cameraInterval);
+    }
+  } else if (action == 'stream_boardcam_stop') {
+    if (cameraCV) {
+      cameraCV.release();
+      cameraCV = null;
+    }
+    if (cameraIntervalID) {
+      clearInterval(cameraIntervalID);
+      cameraIntervalID = null;
+    }
+  } else if (action == 'stream_nest_ipcam_start') {
+    if (!cameraIntervalID) {
+      cameraIntervalID = setInterval(function() {
+        request.get(
+          {uri: cameraURI, encoding: null},
+          (e, res, body) => {
+            if (!e && res.statusCode == 200) {
+              log('camera client: publishing image.');
+              client.publish(topicActionInference, body);
+            } else {
+              log('camera client: cannot get image.');
+            }
+          }
+        );
+      }, cameraInterval);
+    }
+  } else if (action == 'stream_nest_ipcam_stop') {
+    if (cameraIntervalID) {
+      clearInterval(cameraIntervalID);
+      cameraIntervalID = null;
+    }
   } else {
     log('camera client: unkown action.');
   }

cleaner.sh

@@ -1,14 +1,14 @@
-var mqtt = require('mqtt')
-//var client  = mqtt.connect('mqtt://test.mosquitto.org')
-var client  = mqtt.connect('mqtt://localhost:1883')
+var mqtt = require('mqtt');
+//var client  = mqtt.connect('mqtt://test.mosquitto.org');
+var client = mqtt.connect('mqtt://localhost:1883');
 
 client.on('connect', function () {
-  client.subscribe('presence')
-  client.publish('presence', 'Hello mqtt')
-})
+  client.subscribe('presence');
+  client.publish('presence', 'Hello mqtt');
+});
 
 client.on('message', function (topic, message) {
   // message is Buffer
-  console.log(message.toString())
-  client.end()
-})
+  console.log(message.toString());
+  client.end();
+});

client.js

@@ -41,22 +41,36 @@ config.topicEventCamera = padTopicBase('event/camera');
 config.topicEventLocalImage = padTopicBase('event/localImage');
 config.topicNotifyEmail = padTopicBase('notify/email');
 config.topicNotifySMS = padTopicBase('notify/sms');
+config.topicNotifyLINE = padTopicBase('notify/line');
 config.topicDashboardLog = padTopicBase('dashboard/log');
 config.topicDashboardSnapshot = padTopicBase('dashboard/snapshot');
 config.topicDashboardInferenceResult = padTopicBase('dashboard/inferenceResult');
+config.topicJSONInferenceResult = padTopicBase('data/jsonInferenceResult');
 
 // IP camera
 config.ipcameraSnapshot = '';
 
+// Board camera, e.g. USB and RPi cameras
+config.boardcameraImageWidth = 640;
+config.boardcameraImageHeight = 480;
+
+// data collector configs
+config.storageDirPath = '';
+
 // email notification
-config.senderEmail = 'SENDER_EMAIL';
-config.senderPassword = 'SENDER_PASSWORD';
-config.receiverEmail = 'RECEIVER_EMAIL';
+config.senderEmail = '';
+config.senderPassword = '';
+config.receiverEmail = '';
 
 // for compatibility
 config.sender_email = config.senderEmail;
 config.sender_password = config.senderPassword;
 config.receiver_email = config.receiverEmail;
 
+// Authentication and channel information for LINE
+config.LINETargetUserID = '';
+config.LINEChannelSecret = '';
+config.LINEChannelAccessToken = '';
+
 // make config importable
 module.exports = config;

config.js

@@ -0,0 +1,3 @@
+# BerryNet supports accessing RPi camera access via OpenCV.
+
+bcm2835_v4l2