wip

Author: previ

camera.py

@@ -22,17 +22,19 @@
 import os
 import sys
 import math
+import json
 from PIL import Image as PILImage
 from StringIO import StringIO
 from threading import Thread, Lock
 import logging
 
-from viz import camera, streamer, image, blob
+from cv import camera, streamer, image, blob
 from cnn_classifier import CNNClassifier
 import config
 
 MAX_IMAGE_AGE = 0.0
 PHOTO_PATH = "./photos"
+PHOTO_METADATA_FILE = "./photos/metadata.json"
 PHOTO_PREFIX = "DSC"
 VIDEO_PREFIX = "VID"
 PHOTO_THUMB_SUFFIX = "_thumb"
@@ -68,13 +70,16 @@ def __init__(self):
     self._path_object_size_min = int(config.Config.get().get("camera_path_object_size_min", 80)) / (self._cv_image_factor * self._cv_image_factor)
     self._path_object_size_max = int(config.Config.get().get("camera_path_object_size_max", 32000)) / (self._cv_image_factor * self._cv_image_factor)
     self._photos = []
-   
-    for dirname, dirnames, filenames,  in os.walk(PHOTO_PATH):
-      for filename in filenames:
-        if (PHOTO_PREFIX in filename or VIDEO_PREFIX in filename) and PHOTO_THUMB_SUFFIX not in filename:
-          self._photos.append(filename)
-
-    self._cnn_classifier = CNNClassifier("cnn_models/applekiwi_1_128.pb", "cnn_models/applekiwi.txt", "input", "final_result", 128, 128, 0.0, 255.0)
+    self.load_photo_metadata()
+    if len(self._photos) == 0:
+      self._photos = []
+      for dirname, dirnames, filenames,  in os.walk(PHOTO_PATH):
+        for filename in filenames:
+          if (PHOTO_PREFIX in filename or VIDEO_PREFIX in filename) and PHOTO_THUMB_SUFFIX not in filename:
+            self._photos.append({'name': filename})
+      self.save_photo_metadata()
+
+    self._cnn_classifier = CNNClassifier("cnn_models/applekiwi_0_5_128.pb", "cnn_models/applekiwi_0_5_128.txt", "input", "final_result", 128, 128, 0.0, 255.0)
 
     super(Camera, self).__init__()
 
@@ -118,6 +123,25 @@ def get_image_jpeg(self):
   def set_text(self, text):
     self._camera.set_overlay_text(str(text))
 
+  def load_photo_metadata(self):
+    try:
+      f = open(PHOTO_METADATA_FILE)
+      self._photos = json.load(f)
+      f.close()
+    except IOError:
+      logging.warning("no metadata file")
+
+  def save_photo_metadata(self):
+    f = open(PHOTO_METADATA_FILE, "w")
+    json.dump(self._photos, f)
+    f.close()
+
+  def update_photo(self, photo):
+    for p in self._photos:
+      if p["name"] == photo["name"]:
+        p["tag"] = photo["tag"]
+    self.save_photo_metadata()
+
   def get_next_photo_index(self):
     last_photo_index = 0
     for p in self._photos:
@@ -140,7 +164,8 @@ def photo_take(self):
     # thumb
     im_pil = PILImage.open(StringIO(im_str)) 
     im_pil.resize(PHOTO_THUMB_SIZE).save(oft)
-    self._photos.append(filename)
+    self._photos.append({"name":filename})
+    self.save_photo_metadata()
 
   def is_recording(self):
     return self.recording
@@ -160,15 +185,16 @@ def video_rec(self, video_name=None):
       try:
         #remove previous file and reference in album
         os.remove(PHOTO_PATH + "/" + filename)
-        self._photos.remove(filename)
+        self._photos.remove({"name":filename})
       except:
         pass
 
     oft = open(PHOTO_PATH +  "/" + filename_thumb, "w")
     im_str = self._camera.get_image_jpeg()
     im_pil = PILImage.open(StringIO(im_str)) 
     im_pil.resize(PHOTO_THUMB_SIZE).save(oft)
-    self._photos.append(filename)
+    self._photos.append({"name":filename})
+    self.save_photo_metadata()
     self._camera.video_rec(PHOTO_PATH + "/" + filename)
     self.video_start_time = time.time()
 
@@ -190,7 +216,10 @@ def delete_photo(self, filename):
     logging.info("delete photo: " + filename)
     os.remove(PHOTO_PATH + "/" + filename)
     os.remove(PHOTO_PATH + "/" + filename[:filename.rfind(".")] + PHOTO_THUMB_SUFFIX + self._camera.PHOTO_FILE_EXT)
-    self._photos.remove(filename)
+    for photo in self._photos:
+      if photo["name"] == filename:
+      	self._photos.remove(photo)
+    self.save_photo_metadata()
 
   def exit(self):
     #self._streamer.server.shutdown()

cnn_manager.py

@@ -0,0 +1,109 @@
+import os
+import shutil
+import logging
+import json
+import threading
+
+from cnn_train import CNNTrainer
+
+MODEL_PATH = "./cnn_models"
+MODEL_TMP_PATH = "/tmp/images"
+MODEL_METADATA = "./cnn_models/models.json"
+PHOTO_PATH = "./photos"
+
+class CNNManager:
+  instance = None
+
+  @classmethod
+  def get_instance(cls):
+    if cls.instance is None:
+      cls.instance = CNNManager()
+    return cls.instance
+
+
+  def __init__(self):
+    try:
+      f = open(MODEL_METADATA, "r")
+      self._models = json.load(f)
+      f.close
+    except IOError:
+      self._models = {}
+      f = open(MODEL_METADATA, "w")
+      json.dump(self._models, f)
+      f.close
+    self._trainers = {} 
+
+  def get_models(self):
+    return self._models
+
+  def get_model_status(self, model_name):
+    return self._models[model_name]
+
+  def delete_model(self, model_name):
+    if self._models.get(model_name):
+      os.remove(MODEL_PATH + "/" + model_name + ".*")
+      del self._models[model_name]    
+
+  def train_new_model(self,
+                      model_name, 
+                      architecture,
+                      image_tags,
+                      photos_meta,
+                      training_steps,
+                      learning_rate):
+
+    logging.info("starting")
+    trainer = self.TrainThread(self, model_name, architecture, image_tags, photos_meta, training_steps, learning_rate)
+    trainer.start()
+    self._trainers[model_name] = trainer
+    #trainer.join()
+    
+  def save_model(self, model_name, architecture):
+    model_info = architecture.split("_")
+    self._models[model_name] = {"status": 1, "image_height": model_info[2], "image_width": model_info[2]}
+    f = open(MODEL_METADATA, "w")
+    json.dump(self._models, f)
+    f.close()
+
+  class TrainThread(threading.Thread):  
+    
+    def __init__(self, manager, model_name, architecture, image_tags, photos_metadata, training_steps, learning_rate):
+      super(CNNManager.TrainThread, self).__init__() 
+      self.manager = manager
+      self.model_name = model_name
+      self.architecture = architecture
+      self.image_tags = image_tags
+      self.photos_metadata = photos_metadata
+      self.learning_rate = learning_rate
+      self.training_steps = training_steps
+      self.trainer = CNNTrainer(architecture)
+
+    def update_train_status(self, model_name, status):
+      model = self.manager._models.get(model_name)
+      model["status"] = status
+
+    def run(self):
+      image_dir = self.prepare_images()
+      logging.info("retrain")
+      self.trainer.retrain(image_dir, MODEL_PATH + "/" + self.model_name, self.training_steps, self.learning_rate)
+      self.manager.save_model(self.model_name, self.architecture)
+      self.clear_filesystem()
+      logging.info("finish")
+              
+    def prepare_images(self):
+      logging.info("prepare_images")
+      photo_abs_path = os.path.abspath(PHOTO_PATH)
+      model_image_path = MODEL_TMP_PATH + "/" + self.model_name
+      os.makedirs(model_image_path)
+      for t in self.image_tags:
+        tag_path = model_image_path + "/" + t
+        os.makedirs(tag_path)
+        for p in self.photos_metadata:
+          if p.get("tag", "---") == t:
+            os.symlink(photo_abs_path + "/" + p["name"], tag_path + "/" + p["name"])
+
+      return model_image_path
+
+    def clear_filesystem(self):
+      shutil.rmtree(MODEL_TMP_PATH + "/" + self.model_name)
+

cnn_train.py

@@ -149,7 +149,7 @@ def __init__(self, architecture):
     self.model_info = self.create_model_info(self.architecture)
     if not self.model_info:
       tf.logging.error('Did not recognize architecture flag')
-      raise
+      raise Exception("Did not recognize architecture flag'")
 
     # Set up the pre-trained graph.
     self.maybe_download_and_extract(self.model_info['data_url'])
@@ -1167,9 +1167,9 @@ def add_jpeg_decoding(self, input_width, input_height, input_depth, input_mean,
 
 
 if __name__ == '__main__':
-  cnn_trainer = CNNTrainer("mobilenet_1.0_128")
+  cnn_trainer = CNNTrainer("mobilenet_0.50_128")
   cnn_trainer.retrain(
               image_dir="/home/pi/tensorflow/data/applekiwi",
-              output_graph="./cnn_test_1",
+              output_graph="./cnn_models/applewiki_0_5_128.pb",
               training_steps=10,
               learning_rate=0.1)

viz/__init__.py renamed to cv/__init__.py

@@ -31,8 +31,9 @@
 from audio import Audio
 from program import ProgramEngine, Program
 from config import Config
+from cnn_manager import CNNManager
 
-from flask import Flask, render_template, request, send_file, redirect, Response
+from flask import Flask, render_template, request, send_file, redirect, Response, jsonify
 from flask_babel import Babel
 #from flask_sockets import Sockets
 
@@ -47,6 +48,7 @@
 cam = None
 motion = None
 audio = None
+cnn = None
 
 app = Flask(__name__,static_url_path="")
 #app.config.from_pyfile('coderbot.cfg')
@@ -201,7 +203,7 @@ def handle_photos():
     return json.dumps(cam.get_photo_list())
 
 @app.route("/photos/<filename>", methods=["GET"])
-def handle_photo(filename):
+def handle_photo_get(filename):
     logging.info("photo")
     mimetype = {'jpeg': 'image/jpeg', 'h264': 'video/mp4'}
     video = None
@@ -212,7 +214,16 @@ def handle_photo(filename):
 
     return send_file(video, mimetype.get(filename[:-3],'image'), cache_timeout=0)
 
-@app.route("/photos/<filename>", methods=["POST"])
+@app.route("/photos/<filename>", methods=["PUT"])
+def handle_photo_put(filename):
+    logging.info("photo update")
+    data = request.get_data()
+    logging.info(data);
+    data = json.loads(data)
+    cam.update_photo({"name": filename, "tag":data["tag"]});
+    return jsonify({"res":"ok"}) 
+
+@app.route("/photos/<filename>", methods=["DELETE"])
 def handle_photo_cmd(filename):
     logging.debug("photo delete")
     cam.delete_photo(filename)
@@ -276,9 +287,38 @@ def handle_program_status():
       prog = app.prog
     return json.dumps({'name': prog.name, "running": prog.is_running()}) 
 
-@app.route("/tutorial")
-def handle_tutorial():
-    return redirect("/blockly-tutorial/apps/index.html", code=302)
+@app.route("/cnnmodels", methods=["GET"])
+def handle_cnn_models_list():
+    logging.info("cnn_models_list")
+    return json.dumps(app.cnn.get_models())
+
+@app.route("/cnnmodels", methods=["POST"])
+def handle_cnn_models_new():
+    logging.info("cnn_models_new")
+    data = json.loads(request.get_data())
+    app.cnn.train_new_model(model_name=data["model_name"],
+                        architecture=data["architecture"],
+                        image_tags=data["image_tags"],
+                        photos_meta=cam.get_photo_list(),
+                        training_steps=data["training_steps"],
+                        learning_rate=data["training_rate"])
+
+    return json.dumps({"name": data["model_name"], "status": 0})
+
+@app.route("/cnnmodels/<model_name>", methods=["GET"])
+def handle_cnn_models_status(model_name):
+    logging.info("cnn_models_status")
+    model_status = app.cnn.get_model(model_name=model_name)
+
+    return json.dumps(model_status)
+
+@app.route("/cnnmodels/<model_name>", methods=["DELETE"])
+def handle_cnn_models_delete(model_name):
+    logging.info("cnn_models_delete")
+    model_status = app.cnn.delete_model(model_name=model_name)
+
+    return json.dumps(model_status)
+
 
 def execute(command):
   process = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
@@ -303,6 +343,7 @@ def run_server():
   global cam
   global motion
   global audio
+  global cnn
   try:
     try:
       app.bot_config = Config.read()
@@ -314,7 +355,10 @@ def run_server():
         #motion = Motion.get_instance()
       except picamera.exc.PiCameraError:
         logging.error("Camera not present")
-      
+
+      cnn = CNNManager.get_instance()
+      app.cnn = cnn
+
       if app.bot_config.get('load_at_start') and len(app.bot_config.get('load_at_start')):
         app.prog = app.prog_engine.load(app.bot_config.get('load_at_start'))
         app.prog.execute()

viz/blob.py renamed to cv/blob.py

@@ -22,7 +22,7 @@
 import numpy as np
 import logging
 from time import clock, time, sleep
-from viz import image, streamer
+from cv import image, streamer
 
 from coderbot import CoderBot
 from camera import Camera