libjade: enable building the Jade firmware as a native library

Implements an in-process instance of the Jade firmware. This can be
thought of as an emulated or virtual Jade device that runs in the address
space of the application that links to it.

This initial implementation is highly experimental and incomplete, and
should under no circumstances be used beyond development and testing.

Co-authored-by: Lawrence Nahum <[email protected]>

Author: jgriffiths

.gitignore

@@ -1,6 +1,7 @@
 pins
 pinsdir
 build
+build_linux
 sdkconfig
 sdkconfig.defaults
 sdkconfig.old

.gitlab-ci.yml

@@ -20,6 +20,7 @@ include:
   - gitlab/apidocs.yml
   - gitlab/test_fw.yml
   - gitlab/dev_fw.yml
+  - gitlab/test_libjade.yml
   - gitlab/test.yml
   - gitlab/flash.yml
   - gitlab/release.yml

CMakeLists.txt

@@ -1,6 +1,9 @@
 cmake_minimum_required(VERSION 3.16)
-set(EXTRA_COMPONENT_DIRS bootloader_components/bootloader_support)
-
-include($ENV{IDF_PATH}/tools/cmake/project.cmake)
-idf_build_set_property(DEPENDENCIES_LOCK dependencies.lock.${IDF_TARGET})
-project(jade)
+if(DEFINED ESP_PLATFORM AND ESP_PLATFORM EQUAL 1)
+    set(EXTRA_COMPONENT_DIRS bootloader_components/bootloader_support)
+    include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+    idf_build_set_property(DEPENDENCIES_LOCK dependencies.lock.${IDF_TARGET})
+    project(jade)
+else()
+    add_subdirectory(libjade)
+endif()

format.sh

@@ -2,6 +2,7 @@
 set -eo pipefail
 
 (cd main && clang-format -i *.c *.h */*.{c,h,inc})
+(cd libjade && clang-format -i *.c *.h */*.h */*/*.h)
 
 clang-format -i tools/bip85_rsa_key_gen/main.c
 

gitlab/test_libjade.yml

@@ -0,0 +1,34 @@
+test_libjade:
+  tags: [ ga ]
+  stage: test
+  script:
+    - . $HOME/esp/esp-idf/export.sh
+    - cp configs/sdkconfig_jade.defaults sdkconfig.defaults
+    - idf.py reconfigure
+    - source /venv/bin/activate
+    - pip install -r requirements.txt
+    - pip install -r pinserver/requirements.txt
+    - pip install .
+    - echo "----------------------- BUILD ---------------------------------"
+    - python ./components/assets/gen_assets.py ./components/assets/asset_data.json ./build/esp-idf/assets/asset_data.inc
+    - python ./components/assets/gen_assets.py ./components/assets/asset_data_testnet.json ./build/esp-idf/assets/asset_data_testnet.inc
+    - ./make_libjade.sh Sanitize
+    - export ASAN_OPTIONS=symbolize=1,detect_leaks=0
+    - export UBSAN_OPTIONS=print_stacktrace=1
+    - export ASAN_SO=/usr/lib/gcc/x86_64-linux-gnu/12/libasan.so
+    - export LD_LIBRARY_PATH=$PWD/build_linux/libjade
+    - export SOCKET_LINK=$PWD/tmp_socket
+    - echo "----------------------- STANDARD TESTS ------------------------"
+    - LD_PRELOAD=$ASAN_SO python ./test_jade.py --log CRITICAL --libjade
+    - echo "----------------------- NON-LEGACY TESTS ----------------------"
+    - LD_PRELOAD=$ASAN_SO python ./test_jade.py --log CRITICAL --libjade --nolegacyflow
+    - echo "-----------------------   SERIAL TESTS   ----------------------"
+    - LD_PRELOAD=$ASAN_SO setsid $PWD/build_linux/libjade/libjade_daemon --serialport $SOCKET_LINK >daemon.log 2>&1 &
+    - DAEMON_PID=$!
+    - LD_PRELOAD=$ASAN_SO python ./test_jade.py --log CRITICAL --nolegacyflow --serialport $SOCKET_LINK --serialtimeout 30
+    - kill -- -$DAEMON_PID
+  artifacts:
+    expire_in: 2 days
+    name: libjade_serial_log
+    paths:
+    - daemon.log

jadepy/jade.py

@@ -29,6 +29,14 @@
     logger.warning(e)
     logger.warning('BLE scanning/connectivity will not be available')
 
+# libjade in-process software emulation is optional.
+# It relies on the libjade.so shared library being in LD_LIBRARY_PATH.
+try:
+    from .jade_sw import JadeSoftwareImpl
+except (ImportError, FileNotFoundError) as e:
+    logger.debug(e)
+    logger.debug('libjade Software Jade emulation will not be available')
+
 
 # Default serial connection
 DEFAULT_BAUD_RATE = 115200
@@ -39,6 +47,9 @@
 DEFAULT_BLE_SERIAL_NUMBER = None
 DEFAULT_BLE_SCAN_TIMEOUT = 60
 
+# Default libjade connection
+DEFAULT_LIBJADE_TIMEOUT = 5
+
 
 def _hexlify(data):
     """
@@ -255,6 +266,31 @@ def create_ble(device_name=None, serial_number=None,
                                         scan_timeout, loop)
         return JadeAPI(impl)
 
+    @staticmethod
+    def create_libjade(timeout=None):
+        """
+        Create a JadeAPI object using libjade (in-process software emulation).
+        WARNING: libjade is BETA and should not be used with real funds.
+        NOTE: raises JadeError if libjade dependencies not installed.
+
+        Parameters
+        ----------
+        timeout : int, optional
+            The read timeout when awaiting messages (Uses 5s if not given).
+
+        Returns
+        -------
+        JadeAPI
+            API object configured to use libjade.
+            NOTE: The caller must call 'connect()' before using the instance.
+
+        Raises
+        ------
+        JadeError if libjade is not available (libjade.so not installed)
+        """
+        impl = JadeInterface.create_libjade(timeout)
+        return JadeAPI(impl)
+
     def connect(self):
         """
         Try to connect the underlying transport interface (eg. serial, ble, etc.)
@@ -1993,18 +2029,18 @@ def sign_psbt(self, network, psbt, additional_info=None):
 
 class JadeInterface:
     """
-    Mid-level interface to Jade
-    Wraps either a serial or a ble connection
+    Mid-level interface to Jade.
+    Wraps either a serial or ble connection, or an in-process libjade instance.
     Calls to send and receive bytes and cbor messages over the interface.
 
     Either:
      a) use wrapped with JadeAPI
     (recommended)
     or:
-     b) use with JadeInterface.create_[serial|ble]() as jade:
+     b) use with JadeInterface.create_[serial|ble|libjade]() as jade:
           ...
     or:
-     c) use JadeInterface.create_[serial|ble], then call connect() before
+     c) use JadeInterface.create_[serial|ble|libjade], then call connect() before
         using, and disconnect() when finished
     (caveat cranium)
     or:
@@ -2110,6 +2146,31 @@ def create_ble(device_name=None, serial_number=None,
                            loop=loop)
         return JadeInterface(impl)
 
+    @staticmethod
+    def create_libjade(timeout=None):
+        """
+        Create a JadeInterface object object using libjade (in-process software emulation).
+        WARNING: libjade is BETA and should not be used with real funds.
+        NOTE: raises JadeError if libjade dependencies not installed.
+
+        Parameters
+        ----------
+        timeout : int, optional
+            The read timeout when awaiting messages (Uses 5s if not given).
+
+        Returns
+        -------
+        JadeInterface
+            Interface object configured to use the libjade.
+            NOTE: The caller must call 'connect()' before using the instance.
+        """
+        this_module = sys.modules[__name__]
+        if not hasattr(this_module, "JadeSoftwareImpl"):
+            raise JadeError(1, "libjade support not installed", None)
+
+        impl = JadeSoftwareImpl(timeout or DEFAULT_LIBJADE_TIMEOUT)
+        return JadeInterface(impl)
+
     def connect(self):
         """
         Try to connect the underlying transport interface (eg. serial, ble, etc.)

jadepy/jade_serial.py

@@ -60,8 +60,9 @@ def connect(self):
                 raise JadeError(1, 'Unable to open port', self.device)
 
         # Ensure RTS and DTR are not set (as this can cause the hw to reboot)
-        self.ser.setRTS(False)
-        self.ser.setDTR(False)
+        if self.device.startswith('/dev/tty'):
+            self.ser.setRTS(False)
+            self.ser.setDTR(False)
 
         logger.info('Connected')
 
@@ -70,8 +71,9 @@ def disconnect(self):
 
         # Ensure RTS and DTR are not set (as this can cause the hw to reboot)
         # and then close the connection
-        self.ser.setRTS(False)
-        self.ser.setDTR(False)
+        if self.device.startswith('/dev/tty'):
+            self.ser.setRTS(False)
+            self.ser.setDTR(False)
         self.ser.close()
 
         # Reset state

jadepy/jade_sw.py

@@ -0,0 +1,92 @@
+from ctypes import CDLL, POINTER, c_ubyte, c_size_t, byref
+import logging
+from .jade_error import JadeError
+
+
+logger = logging.getLogger(__name__)
+
+try:
+    _libjade = CDLL('libjade.so')
+    _libjade.libjade_receive.restype = POINTER(c_ubyte)
+except Exception as _:
+    raise ImportError  # libjade.so not available
+
+
+#
+# Experimental, internal, in-process interface to Jade
+# Intended for use via JadeInterface wrapper.
+#
+# Use via JadeInterface.create_libjade() (see JadeInterface)
+#
+class JadeSoftwareImpl:
+
+    _log_levels = {
+        logging.DEBUG: 1,
+        logging.INFO: 2,
+        logging.WARNING: 3,
+        logging.ERROR: 4,
+        logging.CRITICAL: 5,  # Note we have no critical logs
+        logging.NOTSET: 5  # Default to critical (i.e. no logging)
+    }
+
+    def __init__(self, timeout):
+        self.timeout = timeout
+        self.libjade = None
+        self.msg = None  # Bytes of the current message being read, if any
+
+    def connect(self):
+        assert self.libjade is None
+        self.libjade = _libjade
+        # Respect the python log level for Jade logging
+        log_level = self._log_levels[logger.getEffectiveLevel()]
+        self.libjade.libjade_set_log_level(log_level)
+        # Starts the firmware in a separate thread
+        self.libjade.libjade_start()
+        logger.info('Connected to in-process software Jade')
+
+    def disconnect(self):
+        assert self.libjade is not None
+        self.libjade.libjade_stop()
+        self.libjade = None
+
+    def write(self, bytes_):
+        assert self.libjade is not None
+        if logger.isEnabledFor(logging.DEBUG):
+            logger.debug(f'Pushing {bytes_.hex()}\n')
+        # The software interface takes the whole message in one go
+        num_bytes = len(bytes_)
+        if not self.libjade.libjade_send(bytes_, num_bytes):
+            raise JadeError(1, f'Failed to send {num_bytes} bytes', None)
+        return num_bytes  # Let the caller know we wrote all bytes
+
+    def read(self, n):
+        assert self.libjade is not None
+        if n == 0:
+            # Sometimes read is called with 0 bytes.
+            # Treat this as a no-op.
+            logger.debug('Read of 0 bytes requested')
+            return bytes()
+
+        if not self.msg:
+            # The software interface reads the whole message in one go.
+            # Fetch it here, then return it in chunks below
+            logger.debug(f'Calling libjade_receive() with timeout {self.timeout}\n')
+            bytes_len = c_size_t()
+            buff = self.libjade.libjade_receive(self.timeout, byref(bytes_len))
+            if not buff:
+                logger.debug('Timeout calling libjade_receive()\n')
+                return bytes()
+            self.msg = bytes([buff[i] for i in range(bytes_len.value)])
+            self.libjade.libjade_release(buff)
+            if logger.isEnabledFor(logging.DEBUG):
+                logger.debug(f'Received message {self.msg.hex()}\n')
+
+        # Return as much of the message as the caller asked for
+        ret = self.msg[:n]
+        self.msg = self.msg[n:]
+        if False and logger.isEnabledFor(logging.DEBUG):
+            # Not generally useful unless debugging serialization failures
+            logger.debug(f'Returning {ret.hex()} leaving {self.msg.hex()}')
+        elif not self.msg:
+            logger.debug('Message consumed')
+        return ret

jadepy/jade_tcp.py

@@ -1,5 +1,6 @@
 import socket
 import logging
+import os
 
 
 logger = logging.getLogger(__name__)
@@ -34,21 +35,35 @@ def __init__(self, device, timeout):
     def connect(self):
         assert self.isSupportedDevice(self.device)
         assert self.tcp_sock is None
-
         logger.info(f'Connecting to {self.device}')
-        self.tcp_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        self.tcp_sock.settimeout(self.timeout)
+        conn_path = self.device[len(self.PROTOCOL_PREFIX):]
 
-        url = self.device[len(self.PROTOCOL_PREFIX):].split(':')
-        self.tcp_sock.connect((url[0], int(url[1])))
-        assert self.tcp_sock is not None
+        is_unix_socket = False
+        if conn_path.startswith('/') or os.path.exists(conn_path):
+            is_unix_socket = True
+        elif '/' in conn_path and ':' not in conn_path:
+            is_unix_socket = True
 
+        if is_unix_socket:
+            self.tcp_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+            self.tcp_sock.settimeout(self.timeout)
+            self.tcp_sock.connect(conn_path)
+        else:
+            self.tcp_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            self.tcp_sock.settimeout(self.timeout)
+            if ':' in conn_path:
+                url = conn_path.split(':')
+                self.tcp_sock.connect((url[0], int(url[1])))
+            else:
+                self.tcp_sock.connect((conn_path, 80))
+
+        assert self.tcp_sock is not None
         self.tcp_sock.__enter__()
         logger.info('Connected')
 
     def disconnect(self):
         assert self.tcp_sock is not None
-        self.tcp_sock.__exit__()
+        self.tcp_sock.__exit__(None, None, None)
 
         # Reset state
         self.tcp_sock = None