Improve capture interface.

configs/capture.yaml

@@ -1,19 +1,20 @@
 sensor: rpi_hq
-bayer: True
+legacy: True   # Legacy image capture software (raspistill) or new one (libcamera)
+bayer: True    # whether to capture Bayer data, otherwise directly capture RGB (demosaiced) data
+nbits_capture: 12   # (if legacy=True) output bit depth for Bayer data. should correspond to supported bit depths: lensless/hardware/sensor.py
 fn: test
 exp: 0.5
 config_pause: 2
-# {'off': 0, 'auto': 1, 'sunlight': 2, 'cloudy': 3,
-# 'shade': 4, 'tungsten': 5, 'fluorescent': 6, 
-# 'incandescent': 7, 'flash': 8, 'horizon': 9}
-sensor_mode: "0"
+sensor_mode: "0"   # {'off': 0, 'auto': 1, 'sunlight': 2, 'cloudy': 3, 'shade': 4, 'tungsten': 5, 'fluorescent': 6, 'incandescent': 7, 'flash': 8, 'horizon': 9}
+iso: 100
+# awb_gains: null   # to use gains estimated by system
+awb_gains: [1.9, 1.2]
+
+# -- only one of below should be set
+down: null  # downsample from maximum resolution
+res: null   # desired resolution
+
+# -- below only used if bayer=True, i.e. capture Bayer data but return as RGB/grayscale
 rgb: False
 gray: False
-iso: 100
-sixteen: True   # whether 16 bits or 8 (from Bayer data)
-legacy: True    
-down: null
-res: null
-nbits_out: 8
-awb_gains: null
-# awb_gains: [1.9, 1.2]   # red, blue
+nbits_out: 8

configs/collect_dataset.yaml

@@ -35,6 +35,9 @@ display:
   rot90: 3
 
 capture:
+  sensor: rpi_hq
+  nbits_capture: 12
+  nbits_out: 8
   delay: 2    # to allow picture to display
   config_pause: 2
   iso: 100

configs/demo.yaml

@@ -30,23 +30,26 @@ display:
   white: False
 
 capture:
-  sensor: rpi_hq
   gamma: null # for visualization
-  exp: 0.02
-  delay: 2
   script: ~/LenslessPiCam/scripts/measure/on_device_capture.py
-  iso: 100
+  delay: 2
+
+  # -- capture settings
+  sensor: rpi_hq
+  legacy: True   # Legacy image capture software (raspistill) or new one (libcamera)
+  bayer: True    # whether to capture Bayer data, otherwise directly capture RGB (demosaiced) data
+  nbits_capture: 12   # bit-depth to capture, check support bit depth of sensor: lensless/hardware/sensor.py
+  fn: raw_data    # output file name
+  exp: 0.02
   config_pause: 2
-  sensor_mode: "0"
-  # nbits_out: 12
-  nbits_out: 8    # light data transer, doesn't seem to worsen performance
-  nbits: 12
-  legacy: True
+  sensor_mode: "0"   # {'off': 0, 'auto': 1, 'sunlight': 2, 'cloudy': 3, 'shade': 4, 'tungsten': 5, 'fluorescent': 6, 'incandescent': 7, 'flash': 8, 'horizon': 9}
+  iso: 100
+
+  # -- if bayer=True, capture Bayer data but return as RGB or grayscale
   gray: False     # only for legacy=True, if bayer=True, remote script returns grayscale data
   # rgb: False   # only for legacy=True, if bayer=True, remote script return RGB data
-  raw_data_fn: raw_data
-  bayer: True
-  source: white
+  # nbits_out: 12
+  nbits_out: 8   # used if gray=True or rgb=True
 
   # remote script return Bayer data, full res
   # awb_gains: null

lensless/hardware/sensor.py

@@ -60,6 +60,8 @@ class SensorParam:
     BIT_DEPTH = "bit_depth"
     MAX_EXPOSURE = "max_exposure"  # in seconds
     MIN_EXPOSURE = "min_exposure"  # in seconds
+    BLACK_LEVEL = "black_level"  # normalized to [0,1]
+    CCM_MATRIX = "ccm_matrix"
 
 
 """
@@ -79,6 +81,15 @@ class SensorParam:
         SensorParam.BIT_DEPTH: [8, 12],
         SensorParam.MAX_EXPOSURE: 670.74,
         SensorParam.MIN_EXPOSURE: 0.02,
+        # https://www.strollswithmydog.com/open-raspberry-pi-high-quality-camera-raw
+        SensorParam.BLACK_LEVEL: 256.3 / (2**12 - 1),
+        SensorParam.CCM_MATRIX: np.array(
+            [
+                [2.0659, -0.93119, -0.13421],
+                [-0.11615, 1.5593, -0.44314],
+                [0.073694, -0.4368, 1.3636],
+            ]
+        ),
     },
     # Raspberry Pi Global Shutter Camera
     # https://www.raspberrypi.com/products/raspberry-pi-global-shutter-camera/

lensless/hardware/utils.py

@@ -6,7 +6,7 @@
 import paramiko
 from pprint import pprint
 from paramiko.ssh_exception import AuthenticationException, BadHostKeyException, SSHException
-from lensless.hardware.sensor import SensorOptions
+from lensless.hardware.sensor import SensorOptions, sensor_dict, SensorParam
 import cv2
 from lensless.utils.image import print_image_info
 from lensless.utils.io import load_image
@@ -17,6 +17,73 @@
 logging.getLogger("paramiko").setLevel(logging.WARNING)
 
 
+def check_capture_config(
+    sensor,
+    nbits_capture,
+    nbits_out,
+    exp,
+    rgb,
+    gray,
+    legacy,
+    down,
+    awb_gains,
+    res=None,
+    **kwargs,
+):
+
+    # sensor = config.sensor
+    # nbits_capture = config.nbits_capture
+    # nbits_out = config.nbits_out
+    # exp = config.exp
+    # rgb = config.rgb
+    # gray = config.gray
+    # legacy = config.legacy
+    # down = config.down
+    # res = config.res if "res" in config.keys() else None
+    # awb_gains = config.awb_gains
+
+    assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}"
+
+    assert res is None or down is None, "Cannot specify both res and down"
+    if res is not None:
+        assert len(res) == 2, "res must be a tuple of length 2"
+    # if down is not None or res is not None:
+    #     if rgb is False and gray is False:
+    #         warnings.warn("Bayer data will be returned in original dimension, i.e. 'down' and 'res' will be ignored.")
+
+    assert rgb is False or gray is False, "Cannot set both rgb and gray"
+
+    supported_bit_depth = sensor_dict[sensor][SensorParam.BIT_DEPTH]
+    assert (
+        nbits_capture in supported_bit_depth
+    ), f"nbits_capture must be one of {supported_bit_depth} for sensor {sensor}"
+
+    assert nbits_capture >= nbits_out, "nbits_capture must be greater than or equal to nbits_out"
+
+    if SensorParam.BLACK_LEVEL in sensor_dict[sensor]:
+        black_level = sensor_dict[sensor][SensorParam.BLACK_LEVEL] * (2**nbits_capture - 1)
+    else:
+        black_level = 0
+    if SensorParam.CCM_MATRIX in sensor_dict[sensor]:
+        ccm = sensor_dict[sensor][SensorParam.CCM_MATRIX]
+    else:
+        ccm = None
+
+    # https://www.raspberrypi.com/documentation/accessories/camera.html#hardware-specification
+    sensor_param = sensor_dict[sensor]
+    assert exp <= sensor_param[SensorParam.MAX_EXPOSURE]
+    assert exp >= sensor_param[SensorParam.MIN_EXPOSURE]
+
+    if awb_gains is not None:
+        assert len(awb_gains) == 2, "awb_gains must be a tuple of length 2"
+
+    if sensor == SensorOptions.RPI_GS.value:
+        assert not legacy, "Legacy capture software not supported for Global Shutter sensor"
+
+    # return sensor parameters
+    return black_level, ccm, supported_bit_depth
+
+
 def capture(
     rpi_username,
     rpi_hostname,
@@ -28,16 +95,16 @@ def capture(
     config_pause=2,
     sensor_mode="0",
     nbits_out=12,
+    nbits_capture=12,
     legacy=True,
     rgb=False,
     gray=False,
-    nbits=12,
     down=None,
     awb_gains=None,
     rpi_python="~/LenslessPiCam/lensless_env/bin/python",
     capture_script="~/LenslessPiCam/scripts/measure/on_device_capture.py",
     verbose=False,
-    output_path=None,
+    output_dir=None,
     **kwargs,
 ):
     """
@@ -71,10 +138,10 @@ def capture(
         Whether to capture RGB image.
     gray : bool
         Whether to capture grayscale image.
-    nbits : int
-        Number of bits of image.
+    nbits_capture : int
+        Number of bits of image captured by sensor.
     down : int
-        Downsample factor.
+        Downsample factor. Only used if non-Bayer data is returned.
     awb_gains : list
         AWB gains (red, blue).
     rpi_python : str
@@ -88,6 +155,19 @@ def capture(
 
     """
 
+    black_level, ccm, supported_bit_depth = check_capture_config(
+        sensor=sensor,
+        nbits_capture=nbits_capture,
+        nbits_out=nbits_out,
+        exp=exp,
+        rgb=rgb,
+        gray=gray,
+        legacy=legacy,
+        down=down,
+        awb_gains=awb_gains,
+        **kwargs,
+    )
+
     # check_username_hostname(rpi_username, rpi_hostname)
     assert sensor in SensorOptions.values(), f"Sensor must be one of {SensorOptions.values()}"
 
@@ -96,10 +176,8 @@ def capture(
     pic_command = (
         f"{rpi_python} {capture_script} sensor={sensor} bayer={bayer} fn={remote_fn} exp={exp} iso={iso} "
         f"config_pause={config_pause} sensor_mode={sensor_mode} nbits_out={nbits_out} "
-        f"legacy={legacy} rgb={rgb} gray={gray} "
+        f"legacy={legacy} rgb={rgb} gray={gray} nbits_capture={nbits_capture} "
     )
-    if nbits > 8:
-        pic_command += " sixteen=True"
     if down:
         pic_command += f" down={down}"
     if awb_gains:
@@ -151,16 +229,24 @@ def capture(
             # copy over DNG file
             remotefile = f"~/{remote_fn}.dng"
             localfile = f"{fn}.dng"
-            if output_path is not None:
-                localfile = os.path.join(output_path, localfile)
+            if output_dir is not None:
+                localfile = os.path.join(output_dir, localfile)
             if verbose:
                 print(f"\nCopying over picture as {localfile}...")
             os.system(
                 'scp "%s@%s:%s" %s >/dev/null 2>&1'
                 % (rpi_username, rpi_hostname, remotefile, localfile)
             )
 
-            img = load_image(localfile, verbose=True, bayer=bayer, nbits_out=nbits_out)
+            img = load_image(
+                localfile,
+                verbose=True,
+                bayer=bayer,
+                nbits_out=nbits_out,
+                black_level=black_level,
+                ccm=ccm,
+                downsample=down,
+            )
 
             # print image properties
             print_image_info(img)
@@ -174,8 +260,8 @@ def capture(
 
             remotefile = f"~/{remote_fn}.png"
             localfile = f"{fn}.png"
-            if output_path is not None:
-                localfile = os.path.join(output_path, localfile)
+            if output_dir is not None:
+                localfile = os.path.join(output_dir, localfile)
             if verbose:
                 print(f"\nCopying over picture as {localfile}...")
             os.system(
@@ -191,8 +277,8 @@ def capture(
         # more pythonic? https://stackoverflow.com/questions/250283/how-to-scp-in-python
         remotefile = f"~/{remote_fn}.png"
         localfile = f"{fn}.png"
-        if output_path is not None:
-            localfile = os.path.join(output_path, localfile)
+        if output_dir is not None:
+            localfile = os.path.join(output_dir, localfile)
         if verbose:
             print(f"\nCopying over picture as {localfile}...")
         os.system(
@@ -205,14 +291,16 @@ def capture(
 
         else:
 
-            if not bayer:
-                # red_gain = config.camera.red_gain
-                # blue_gain = config.camera.blue_gain
-                red_gain = awb_gains[0]
-                blue_gain = awb_gains[1]
-            else:
-                red_gain = None
-                blue_gain = None
+            red_gain = awb_gains[0]
+            blue_gain = awb_gains[1]
+            # if not bayer:
+            #     # red_gain = config.camera.red_gain
+            #     # blue_gain = config.camera.blue_gain
+            #     red_gain = awb_gains[0]
+            #     blue_gain = awb_gains[1]
+            # else:
+            #     red_gain = None
+            #     blue_gain = None
 
             # load image
             if verbose:
@@ -225,6 +313,9 @@ def capture(
                 blue_gain=blue_gain,
                 red_gain=red_gain,
                 nbits_out=nbits_out,
+                black_level=black_level,
+                ccm=ccm,
+                downsample=down,
             )
 
             # write RGB data