Merge branch 'main' into camera_tuning

Erol444 · web-flow · commit 080d0a0c1bde · 2022-01-17T10:55:41.000-08:00
diff --git a/docs/source/_static/images/components/depth_diagram.jpeg b/docs/source/_static/images/components/depth_diagram.jpeg
diff --git a/docs/source/_static/images/components/depth_diagram.png b/docs/source/_static/images/components/depth_diagram.png
diff --git a/docs/source/_static/images/examples/depth_comparison.png b/docs/source/_static/images/examples/depth_comparison.png
diff --git a/docs/source/components/nodes/script.rst b/docs/source/components/nodes/script.rst
@@ -106,7 +106,7 @@ In the script node you can interface with GPIOs of the VPU using module GPIO. Cu
 
   # Interrupts
   GPIO.waitInterruptEvent(gpio = -1) # blocks until any interrupt or interrupt by specified gpio is fired. Interrupts with callbacks are ignored here
-  GPIO.hasInterruptEvent(gpio = -1) # returns whether interrupt happened on any or specfied gpio. Interrupts with callbacks are ignored here
+  GPIO.hasInterruptEvent(gpio = -1) # returns whether interrupt happened on any or specified gpio. Interrupts with callbacks are ignored here
   GPIO.setInterrupt(gpio, edge, priority, callback = None) # adds interrupt to specified pin
   GPIO.clearInterrupt(gpio) # clears interrupt of specified pin
 
diff --git a/docs/source/components/nodes/stereo_depth.rst b/docs/source/components/nodes/stereo_depth.rst
@@ -73,7 +73,7 @@ Inputs and Outputs
 Internal block diagram of StereoDepth node
 ##########################################
 
-.. image:: /_static/images/components/depth_diagram.jpeg
+.. image:: /_static/images/components/depth_diagram.png
    :target: https://whimsical.com/stereo-node-EKcfcXGjGpNL6cwRPV6NPv
 
 On the diagram, red rectangle are firmware settings that are configurable via the API. Gray rectangles are settings that that are not yet
@@ -126,9 +126,15 @@ Currently configurable blocks
 
         For comparison of normal disparity vs. subpixel disparity images, click `here <https://github.com/luxonis/depthai/issues/184>`__.
 
-  .. tab:: Mesh file / Homography matrix
+  .. tab:: Depth Filters
 
-    Mesh files are generated using the camera intrinsics, distortion coeffs, and rectification rotations.
+    **Depth Filtering** / **Depth Post-Processing** is performed at the end of the depth pipeline. It helps with noise reduction and overall depth quality.
+
+    .. include::  ../../includes/depth-filters.rst
+
+  .. tab:: Mesh files
+
+    Mesh files (homography matrix) are generated using the camera intrinsics, distortion coeffs, and rectification rotations.
     These files helps in overcoming the distortions in the camera increasing the accuracy and also help in when `wide FOV <https://docs.luxonis.com/projects/hardware/en/latest/pages/arducam.html#arducam-compatible-cameras>`__ lens are used.
 
     .. note::
@@ -148,7 +154,7 @@ Currently configurable blocks
 
   .. tab:: Confidence Threshold
 
-    - **Confidence threshold**: Stereo depth algorithm searches for the matching feature from right camera point to the left image (along the 96 disparity levels). During this process it computes the cost for each disparity level and choses the minimal cost between two disparities and uses it to compute the confidence at each pixel. Stereo node will output disparity/depth pixels only where depth confidence is below the **confidence threshold** (lower the confidence value means better depth accuracy). Note: This threshold only applies to Normal stereo mode as of now.
+    - **Confidence threshold**: Stereo depth algorithm searches for the matching feature from right camera point to the left image (along the 96 disparity levels). During this process it computes the cost for each disparity level and chooses the minimal cost between two disparities and uses it to compute the confidence at each pixel. Stereo node will output disparity/depth pixels only where depth confidence is below the **confidence threshold** (lower the confidence value means better depth accuracy).
     - **LR check threshold**: Disparity is considered for the output when the difference between LR and RL disparities is smaller than the LR check threshold.
 
     .. doxygenfunction:: dai::StereoDepthConfig::setConfidenceThreshold
@@ -159,8 +165,8 @@ Currently configurable blocks
         :project: depthai-core
         :no-link:
 
-Current limitations
-###################
+Limitations
+###########
 
 - Median filtering is disabled when subpixel mode is set to 4 or 5 bits.
 
diff --git a/docs/source/includes/depth-filters.rst b/docs/source/includes/depth-filters.rst
@@ -0,0 +1,65 @@
+.. tabs::
+
+    .. tab:: Median
+
+        This is a non-edge preserving `Median filter <https://en.wikipedia.org/wiki/Median_filter>`__, which can be used
+        to reduce noise and smoothen the depth map. Median filter is implemented in hardware, so it's the fastest filter.
+
+        .. doxygenenum:: dai::MedianFilter
+            :project: depthai-core
+            :no-link:
+
+    .. tab:: Speckle
+
+        **Speckle Filter** is used to reduce the speckle noise. Speckle noise is a region with huge
+        variance between neighboring disparity/depth pixels, and speckle filter tries to filter this region.
+
+        .. doxygenstruct:: dai::RawStereoDepthConfig::PostProcessing::SpeckleFilter
+            :project: depthai-core
+            :no-link:
+            :members:
+
+    .. tab:: Temporal
+
+        **Temporal Filter** is intended to improve the depth data persistency by manipulating per-pixel
+        values based on previous frames. The filter performs a single pass on the data, adjusting the depth
+        values while also updating the tracking history. In cases where the pixel data is missing or invalid,
+        the filter uses a user-defined persistency mode to decide whether the missing value should be
+        rectified with stored data. Note that due to its reliance on historic data the filter may introduce
+        visible blurring/smearing artifacts, and therefore is best-suited for static scenes.
+
+        .. doxygenstruct:: dai::RawStereoDepthConfig::PostProcessing::TemporalFilter
+            :project: depthai-core
+            :no-link:
+            :members:
+
+    .. tab:: Spatial
+
+        **Spatial Edge-Preserving Filter** will fill invalid depth pixels with valid neighboring depth pixels.
+        It performs a series of 1D horizontal and vertical passes or iterations, to enhance the smoothness of
+        the reconstructed data. It is based on `this research paper <https://www.inf.ufrgs.br/~eslgastal/DomainTransform/>`__.
+
+        .. doxygenstruct:: dai::RawStereoDepthConfig::PostProcessing::SpatialFilter
+            :project: depthai-core
+            :no-link:
+            :members:
+
+    .. tab:: Threshold
+
+        **Threshold Filter** filters out all disparity/depth pixels outside the configured min/max
+        threshold values.
+
+        .. autoclass:: depthai.RawStereoDepthConfig.PostProcessing.ThresholdFilter
+            :members:
+            :inherited-members:
+            :noindex:
+
+    .. tab:: Decimation
+
+        **Decimation Filter** will sub-samples the depth map, which means it reduces the depth scene complexity and allows
+        other filters to run faster. Setting :code:`decimationFactor` to 2 will downscale 1280x800 depth map to 640x400.
+
+        .. doxygenstruct:: dai::RawStereoDepthConfig::PostProcessing::DecimationFilter
+            :members:
+            :project: depthai-core
+            :no-link:
diff --git a/docs/source/install.rst b/docs/source/install.rst
@@ -241,7 +241,7 @@ Kernel Virtual Machine
 To access the OAK-D camera in the `Kernel Virtual Machine <https://www.linux-kvm.org/page/Main_Page>`__, there is a need to attach and detach USB 
 devices on the fly when the host machine detects changes in the USB bus.
 
-OAK-D camera changes the USB device type when it is used by DepthAI API. This happens in backgound when the camera is used natively.
+OAK-D camera changes the USB device type when it is used by DepthAI API. This happens in background when the camera is used natively.
 But when the camera is used in a virtual environment the situation is different.
 
 On your host machine, use the following code:
@@ -292,7 +292,7 @@ Note that when the device is disconnected from the USB bus, some udev environmen
 that is why you need to use :code:`PRODUCT` environmental variable to identify which device has been disconnected.
 
 The virtual machine where DepthAI API application is running should have defined a udev rules that identify the OAK-D camera.
-The udev rule is decribed `here <https://docs.luxonis.com/en/latest/pages/faq/#does-depthai-work-on-the-nvidia-jetson-series>`__
+The udev rule is described `here <https://docs.luxonis.com/en/latest/pages/faq/#does-depthai-work-on-the-nvidia-jetson-series>`__
 
 Solution provided by `Manuel Segarra-Abad <https://github.com/maseabunikie>`__
 
diff --git a/docs/source/samples/Script/script_json_communication.rst b/docs/source/samples/Script/script_json_communication.rst
@@ -15,7 +15,7 @@ and when it's noon, it would send a simple message to the Script node which woul
 
 **What it does:**
 
-Host creates a dictionary, serializes it, sends it to the Script node. Script node recieves the Buffer message, deserializes the dictionary,
+Host creates a dictionary, serializes it, sends it to the Script node. Script node receives the Buffer message, deserializes the dictionary,
 changes values a bit, serializes the dictionary again and sends it to the host, which deserializes the changed dictionary and prints the new values.
 
 Demo
diff --git a/docs/source/samples/StereoDepth/depth_post_processing.rst b/docs/source/samples/StereoDepth/depth_post_processing.rst
@@ -0,0 +1,49 @@
+Depth Post-Processing
+=====================
+
+This example shows how you can run depth post-processing filters on the device itself to reduce noise,
+smooth the depth map and overall improve the depth map quality. Post-processing can be added to :ref:`StereoDepth` node.
+
+
+Demo
+####
+
+.. image:: /_static/images/examples/depth_comparison.png
+
+Depth filters
+#############
+
+.. include::  /includes/depth-filters.rst
+
+.. rubric:: Similar samples:
+
+- :ref:`Depth Preview`
+- :ref:`Stereo Depth from host`
+
+Setup
+#####
+
+.. include::  /includes/install_from_pypi.rst
+
+Source code
+###########
+
+.. tabs::
+
+    .. tab:: Python
+
+        Also `available on GitHub <https://github.com/luxonis/depthai-python/blob/main/examples/StereoDepth/depth_post_processing.py>`__
+
+        .. literalinclude:: ../../../../examples/StereoDepth/depth_post_processing.py
+           :language: python
+           :linenos:
+
+    .. tab:: C++
+
+        Also `available on GitHub <https://github.com/luxonis/depthai-core/blob/main/examples/StereoDepth/depth_post_processing.cpp>`__
+
+        .. literalinclude:: ../../../../depthai-core/examples/StereoDepth/depth_post_processing.cpp
+           :language: cpp
+           :linenos:
+
+.. include::  /includes/footer-short.rst
diff --git a/docs/source/tutorials/code_samples.rst b/docs/source/tutorials/code_samples.rst
@@ -122,6 +122,7 @@ are presented with code.
 .. rubric:: StereoDepth
 
 - :ref:`Depth Crop Control` - Demonstrates how to control cropping of depth frames from the host
+- :ref:`Depth Post-Processing` - Depth post-processing filters
 - :ref:`Depth Preview` - Displays colorized stereo disparity
 - :ref:`Stereo Depth from host` - Generates stereo depth frame from a set of mono images from the host
 - :ref:`Stereo Depth Video` - An extended version of **Depth Preview**
diff --git a/examples/ColorCamera/autoexposure_roi.py b/examples/ColorCamera/autoexposure_roi.py
@@ -130,7 +130,7 @@ def displayFrame(name, frame):
         cv2.imshow(name, frame)
 
     while True:
-        # Instead of get (blocking), we use tryGet (nonblocking) which will return the available data or None otherwise
+        # Instead of get (blocking), we use tryGet (non-blocking) which will return the available data or None otherwise
         inRgb = qRgb.tryGet()
         inDet = qDet.tryGet()
 
diff --git a/examples/MobileNet/mono_mobilenet.py b/examples/MobileNet/mono_mobilenet.py
@@ -79,7 +79,7 @@ def displayFrame(name, frame):
         cv2.imshow(name, frame)
 
     while True:
-        # Instead of get (blocking), we use tryGet (nonblocking) which will return the available data or None otherwise
+        # Instead of get (blocking), we use tryGet (non-blocking) which will return the available data or None otherwise
         inRight = qRight.tryGet()
         inDet = qDet.tryGet()
 
diff --git a/examples/MobileNet/rgb_mobilenet.py b/examples/MobileNet/rgb_mobilenet.py
@@ -87,7 +87,7 @@ def displayFrame(name, frame):
             inRgb = qRgb.get()
             inDet = qDet.get()
         else:
-            # Instead of get (blocking), we use tryGet (nonblocking) which will return the available data or None otherwise
+            # Instead of get (blocking), we use tryGet (non-blocking) which will return the available data or None otherwise
             inRgb = qRgb.tryGet()
             inDet = qDet.tryGet()
 
diff --git a/examples/MobileNet/rgb_mobilenet_4k.py b/examples/MobileNet/rgb_mobilenet_4k.py
@@ -84,7 +84,7 @@ def displayFrame(name, frame):
     print("Resize video window with mouse drag!")
 
     while True:
-        # Instead of get (blocking), we use tryGet (nonblocking) which will return the available data or None otherwise
+        # Instead of get (blocking), we use tryGet (non-blocking) which will return the available data or None otherwise
         inVideo = qVideo.tryGet()
         inPreview = qPreview.tryGet()
         inDet = qDet.tryGet()
diff --git a/examples/MonoCamera/mono_preview.py b/examples/MonoCamera/mono_preview.py
@@ -33,7 +33,7 @@
     qRight = device.getOutputQueue(name="right", maxSize=4, blocking=False)
 
     while True:
-        # Instead of get (blocking), we use tryGet (nonblocking) which will return the available data or None otherwise
+        # Instead of get (blocking), we use tryGet (non-blocking) which will return the available data or None otherwise
         inLeft = qLeft.tryGet()
         inRight = qRight.tryGet()
 
diff --git a/examples/Script/script_json_communication.py b/examples/Script/script_json_communication.py
@@ -12,7 +12,7 @@
 script.setScript("""
     import json
 
-    # Recieve bytes from the host
+    # Receive bytes from the host
     data = node.io['in'].get().getData()
     jsonStr = str(data, 'utf-8')
     dict = json.loads(jsonStr)
diff --git a/examples/StereoDepth/stereo_depth_from_host.py b/examples/StereoDepth/stereo_depth_from_host.py
@@ -230,14 +230,14 @@ def handleKeypress(key, stereoDepthConfigInQueue):
             StereoConfigHandler.config.postProcessing.decimationFilter.decimationMode = nextDecimation
         if key == ord('a'):
             StereoConfigHandler.newConfig = True
-            aligmentSettings = [dai.StereoDepthConfig.AlgorithmControl.DepthAlign.RECTIFIED_RIGHT,
+            alignmentSettings = [dai.StereoDepthConfig.AlgorithmControl.DepthAlign.RECTIFIED_RIGHT,
             dai.StereoDepthConfig.AlgorithmControl.DepthAlign.RECTIFIED_LEFT,
             dai.StereoDepthConfig.AlgorithmControl.DepthAlign.CENTER,
             ]
-            currentAligment = StereoConfigHandler.config.algorithmControl.depthAlign
-            nextAligment = aligmentSettings[(aligmentSettings.index(currentAligment)+1) % len(aligmentSettings)]
-            print(f"Changing aligment mode to {nextAligment.name} from {currentAligment.name}")
-            StereoConfigHandler.config.algorithmControl.depthAlign = nextAligment
+            currentAlignment = StereoConfigHandler.config.algorithmControl.depthAlign
+            nextAlignment = alignmentSettings[(alignmentSettings.index(currentAlignment)+1) % len(alignmentSettings)]
+            print(f"Changing alignment mode to {nextAlignment.name} from {currentAlignment.name}")
+            StereoConfigHandler.config.algorithmControl.depthAlign = nextAlignment
         elif key == ord('c'):
             StereoConfigHandler.newConfig = True
             censusSettings = [dai.StereoDepthConfig.CensusTransform.KernelSize.AUTO, dai.StereoDepthConfig.CensusTransform.KernelSize.KERNEL_5x5, dai.StereoDepthConfig.CensusTransform.KernelSize.KERNEL_7x7, dai.StereoDepthConfig.CensusTransform.KernelSize.KERNEL_7x9]
@@ -318,7 +318,7 @@ def __init__(self, config):
         print("Control disparity search range using the 'd' key.")
         print("Control disparity companding using the 'f' key.")
         print("Control census transform mean mode using the 'v' key.")
-        print("Control depth aligment using the 'a' key.")
+        print("Control depth alignment using the 'a' key.")
         print("Control decimation algorithm using the 'a' key.")
         print("Control temporal persistency mode using the 'r' key.")
         print("Control spatial filter using the 'w' key.")
diff --git a/examples/mixed/mono_depth_mobilenetssd.py b/examples/mixed/mono_depth_mobilenetssd.py
@@ -98,7 +98,7 @@ def show(name, frame):
     disparityMultiplier = 255 / stereo.initialConfig.getMaxDisparity()
 
     while True:
-        # Instead of get (blocking), we use tryGet (nonblocking) which will return the available data or None otherwise
+        # Instead of get (blocking), we use tryGet (non-blocking) which will return the available data or None otherwise
         inRight = qRight.tryGet()
         inDet = qDet.tryGet()
         inDisparity = qDisparity.tryGet()
