Add PacketBuffer and MTU negotiation support.

PacketBuffer facilitates packet oriented BLE protocols such as BLE
MIDI and the Apple Media Service.

This also adds PHY, MTU and connection event extension negotiation
to speed up data transfer when possible.

Author: tannewt

ports/nrf/bluetooth/ble_drv.c

@@ -46,7 +46,7 @@ nrf_nvic_state_t nrf_nvic_state = { 0 };
 volatile sd_flash_operation_status_t sd_flash_operation_status;
 
 __attribute__((aligned(4)))
-static uint8_t m_ble_evt_buf[sizeof(ble_evt_t) + (BLE_GATT_ATT_MTU_DEFAULT)];
+static uint8_t m_ble_evt_buf[sizeof(ble_evt_t) + (BLE_GATTS_VAR_ATTR_LEN_MAX)];
 
 void ble_drv_reset() {
     // Linked list items will be gc'd.

ports/nrf/boards/common.template.ld

@@ -16,7 +16,13 @@ MEMORY
     FLASH_FATFS (r) : ORIGIN = ${CIRCUITPY_INTERNAL_FLASH_FILESYSTEM_START_ADDR}, LENGTH = ${CIRCUITPY_INTERNAL_FLASH_FILESYSTEM_SIZE}
     FLASH_BOOTLOADER (rx) : ORIGIN = ${BOOTLOADER_START_ADDR}, LENGTH = ${BOOTLOADER_SIZE}
     FLASH_BOOTLOADER_SETTINGS (r) : ORIGIN = ${BOOTLOADER_SETTINGS_START_ADDR}, LENGTH = ${BOOTLOADER_SETTINGS_SIZE}
-    RAM (xrw)       : ORIGIN = 0x20004000, LENGTH = 0x03C000 /* 240 KiB */
+
+
+    /* 0x2000000 - RAM:ORIGIN is reserved for Softdevice */
+    /* SoftDevice 6.1.0 takes 0x7b78 bytes (30.86 kb) minimum with high ATT MTU. */
+    /* To measure the minimum required amount of memory for given configuration, set this number
+       high enough to work and then check the mutation of the value done by sd_ble_enable. */
+    RAM (xrw)       : ORIGIN = 0x20000000 + 32K, LENGTH = 256K - 32K
 }
 
 /* produce a link error if there is not this amount of RAM for these sections */

ports/nrf/common-hal/_bleio/Adapter.c

@@ -67,7 +67,7 @@ STATIC void softdevice_assert_handler(uint32_t id, uint32_t pc, uint32_t info) {
     reset_into_safe_mode(NORDIC_SOFT_DEVICE_ASSERT);
 }
 
-bleio_connection_internal_t connections[BLEIO_TOTAL_CONNECTION_COUNT];
+bleio_connection_internal_t bleio_connections[BLEIO_TOTAL_CONNECTION_COUNT];
 
 // Linker script provided ram start.
 extern uint32_t _ram_start;
@@ -133,6 +133,15 @@ STATIC uint32_t ble_stack_enable(void) {
         return err_code;
     }
 
+    // Set ATT_MTU so that the maximum MTU we can negotiate is up to the full characteristic size.
+    memset(&ble_conf, 0, sizeof(ble_conf));
+    ble_conf.conn_cfg.conn_cfg_tag = BLE_CONN_CFG_TAG_CUSTOM;
+    ble_conf.conn_cfg.params.gatt_conn_cfg.att_mtu = BLE_GATTS_VAR_ATTR_LEN_MAX;
+    err_code = sd_ble_cfg_set(BLE_CONN_CFG_GATT, &ble_conf, app_ram_start);
+    if (err_code != NRF_SUCCESS) {
+        return err_code;
+    }
+
     // Triple the GATT Server attribute size to accomodate both the CircuitPython built-in service
     // and anything the user does.
     memset(&ble_conf, 0, sizeof(ble_conf));
@@ -142,14 +151,29 @@ STATIC uint32_t ble_stack_enable(void) {
         return err_code;
     }
 
-    // TODO set ATT_MTU so that the maximum MTU we can negotiate is higher than the default.
+    // Increase the number of vendor UUIDs supported. Apple uses a complete random number per
+    // service and characteristic.
+    memset(&ble_conf, 0, sizeof(ble_conf));
+    ble_conf.common_cfg.vs_uuid_cfg.vs_uuid_count = 32; // Defaults to 10.
+    err_code = sd_ble_cfg_set(BLE_COMMON_CFG_VS_UUID, &ble_conf, app_ram_start);
+    if (err_code != NRF_SUCCESS) {
+        return err_code;
+    }
 
     // This sets app_ram_start to the minimum value needed for the settings set above.
     err_code = sd_ble_enable(&app_ram_start);
     if (err_code != NRF_SUCCESS) {
         return err_code;
     }
 
+    // Turn on connection event extension so we can transmit for a longer period of time as needed.
+    ble_opt_t opt;
+    opt.common_opt.conn_evt_ext.enable = true;
+    err_code = sd_ble_opt_set(BLE_COMMON_OPT_CONN_EVT_EXT, &opt);
+    if (err_code != NRF_SUCCESS) {
+        return err_code;
+    }
+
     ble_gap_conn_params_t gap_conn_params = {
         .min_conn_interval = BLE_MIN_CONN_INTERVAL,
         .max_conn_interval = BLE_MAX_CONN_INTERVAL,
@@ -177,7 +201,7 @@ STATIC bool adapter_on_ble_evt(ble_evt_t *ble_evt, void *self_in) {
             // total connection limit.
             bleio_connection_internal_t *connection;
             for (size_t i = 0; i < BLEIO_TOTAL_CONNECTION_COUNT; i++) {
-                connection = &connections[i];
+                connection = &bleio_connections[i];
                 if (connection->conn_handle == BLE_CONN_HANDLE_INVALID) {
                     break;
                 }
@@ -189,6 +213,7 @@ STATIC bool adapter_on_ble_evt(ble_evt_t *ble_evt, void *self_in) {
             connection->conn_handle = ble_evt->evt.gap_evt.conn_handle;
             connection->connection_obj = mp_const_none;
             connection->pair_status = PAIR_NOT_PAIRED;
+            connection->mtu = 0;
 
             ble_drv_add_event_handler_entry(&connection->handler_entry, connection_on_ble_evt, connection);
             self->connection_objs = NULL;
@@ -216,7 +241,7 @@ STATIC bool adapter_on_ble_evt(ble_evt_t *ble_evt, void *self_in) {
             // Find the connection that was disconnected.
             bleio_connection_internal_t *connection;
             for (size_t i = 0; i < BLEIO_TOTAL_CONNECTION_COUNT; i++) {
-                connection = &connections[i];
+                connection = &bleio_connections[i];
                 if (connection->conn_handle == ble_evt->evt.gap_evt.conn_handle) {
                     break;
                 }
@@ -293,7 +318,7 @@ void common_hal_bleio_adapter_set_enabled(bleio_adapter_obj_t *self, bool enable
     // Add a handler for incoming peripheral connections.
     if (enabled) {
         for (size_t i = 0; i < BLEIO_TOTAL_CONNECTION_COUNT; i++) {
-            bleio_connection_internal_t *connection = &connections[i];
+            bleio_connection_internal_t *connection = &bleio_connections[i];
             connection->conn_handle = BLE_CONN_HANDLE_INVALID;
         }
         bleio_adapter_reset_name(self);
@@ -497,14 +522,25 @@ mp_obj_t common_hal_bleio_adapter_connect(bleio_adapter_obj_t *self, bleio_addre
 
     ble_drv_remove_event_handler(connect_on_ble_evt, &event_info);
 
-    if (event_info.conn_handle == BLE_CONN_HANDLE_INVALID) {
+    uint16_t conn_handle = event_info.conn_handle;
+    if (conn_handle == BLE_CONN_HANDLE_INVALID) {
         mp_raise_bleio_BluetoothError(translate("Failed to connect: timeout"));
     }
 
+    // Negotiate for better PHY, larger MTU and data lengths since we are the central. These are
+    // nice-to-haves so ignore any errors.
+    ble_gap_phys_t const phys = {
+        .rx_phys = BLE_GAP_PHY_AUTO,
+        .tx_phys = BLE_GAP_PHY_AUTO,
+    };
+    sd_ble_gap_phy_update(conn_handle, &phys);
+    sd_ble_gattc_exchange_mtu_request(conn_handle, BLE_GATTS_VAR_ATTR_LEN_MAX);
+    sd_ble_gap_data_length_update(conn_handle, NULL, NULL);
+
     // Make the connection object and return it.
     for (size_t i = 0; i < BLEIO_TOTAL_CONNECTION_COUNT; i++) {
-        bleio_connection_internal_t *connection = &connections[i];
-        if (connection->conn_handle == event_info.conn_handle) {
+        bleio_connection_internal_t *connection = &bleio_connections[i];
+        if (connection->conn_handle == conn_handle) {
             return bleio_connection_new_from_internal(connection);
         }
     }
@@ -628,7 +664,7 @@ void common_hal_bleio_adapter_stop_advertising(bleio_adapter_obj_t *self) {
 
 bool common_hal_bleio_adapter_get_connected(bleio_adapter_obj_t *self) {
     for (size_t i = 0; i < BLEIO_TOTAL_CONNECTION_COUNT; i++) {
-        bleio_connection_internal_t *connection = &connections[i];
+        bleio_connection_internal_t *connection = &bleio_connections[i];
         if (connection->conn_handle != BLE_CONN_HANDLE_INVALID) {
             return true;
         }
@@ -643,7 +679,7 @@ mp_obj_t common_hal_bleio_adapter_get_connections(bleio_adapter_obj_t *self) {
     size_t total_connected = 0;
     mp_obj_t items[BLEIO_TOTAL_CONNECTION_COUNT];
     for (size_t i = 0; i < BLEIO_TOTAL_CONNECTION_COUNT; i++) {
-        bleio_connection_internal_t *connection = &connections[i];
+        bleio_connection_internal_t *connection = &bleio_connections[i];
         if (connection->conn_handle != BLE_CONN_HANDLE_INVALID) {
             if (connection->connection_obj == mp_const_none) {
                 connection->connection_obj = bleio_connection_new_from_internal(connection);
@@ -658,15 +694,15 @@ mp_obj_t common_hal_bleio_adapter_get_connections(bleio_adapter_obj_t *self) {
 
 void bleio_adapter_gc_collect(bleio_adapter_obj_t* adapter) {
     gc_collect_root((void**)adapter, sizeof(bleio_adapter_obj_t) / sizeof(size_t));
-    gc_collect_root((void**)connections, sizeof(connections) / sizeof(size_t));
+    gc_collect_root((void**)bleio_connections, sizeof(bleio_connections) / sizeof(size_t));
 }
 
 void bleio_adapter_reset(bleio_adapter_obj_t* adapter) {
     common_hal_bleio_adapter_stop_scan(adapter);
     common_hal_bleio_adapter_stop_advertising(adapter);
     adapter->connection_objs = NULL;
     for (size_t i = 0; i < BLEIO_TOTAL_CONNECTION_COUNT; i++) {
-        bleio_connection_internal_t *connection = &connections[i];
+        bleio_connection_internal_t *connection = &bleio_connections[i];
         connection->connection_obj = mp_const_none;
     }
 }

ports/nrf/common-hal/_bleio/Adapter.h

@@ -37,7 +37,7 @@
 
 #define BLEIO_TOTAL_CONNECTION_COUNT 2
 
-extern bleio_connection_internal_t connections[BLEIO_TOTAL_CONNECTION_COUNT];
+extern bleio_connection_internal_t bleio_connections[BLEIO_TOTAL_CONNECTION_COUNT];
 
 typedef struct {
     mp_obj_base_t base;

ports/nrf/common-hal/_bleio/Characteristic.c

@@ -154,7 +154,7 @@ void common_hal_bleio_characteristic_set_value(bleio_characteristic_obj_t *self,
             common_hal_bleio_gatts_write(self->handle, BLE_CONN_HANDLE_INVALID, bufinfo);
             // Check to see if we need to notify or indicate any active connections.
             for (size_t i = 0; i < BLEIO_TOTAL_CONNECTION_COUNT; i++) {
-                bleio_connection_internal_t *connection = &connections[i];
+                bleio_connection_internal_t *connection = &bleio_connections[i];
                 uint16_t conn_handle = connection->conn_handle;
                 if (connection->conn_handle == BLE_CONN_HANDLE_INVALID) {
                     continue;

ports/nrf/common-hal/_bleio/Connection.c

@@ -85,6 +85,7 @@ bool connection_on_ble_evt(ble_evt_t *ble_evt, void *self_in) {
     switch (ble_evt->header.evt_id) {
         case BLE_GAP_EVT_DISCONNECTED:
             break;
+
         case BLE_GAP_EVT_PHY_UPDATE_REQUEST: {
             ble_gap_phys_t const phys = {
                 .rx_phys = BLE_GAP_PHY_AUTO,
@@ -94,20 +95,45 @@ bool connection_on_ble_evt(ble_evt_t *ble_evt, void *self_in) {
             break;
         }
 
-        case BLE_GAP_EVT_PHY_UPDATE: // 0x22
+        case BLE_GAP_EVT_PHY_UPDATE: { // 0x22
             break;
+        }
 
         case BLE_GAP_EVT_DATA_LENGTH_UPDATE_REQUEST:
             // SoftDevice will respond to a length update request.
             sd_ble_gap_data_length_update(self->conn_handle, NULL, NULL);
             break;
 
-        case BLE_GAP_EVT_DATA_LENGTH_UPDATE: // 0x24
+        case BLE_GAP_EVT_DATA_LENGTH_UPDATE: { // 0x24
             break;
+        }
 
         case BLE_GATTS_EVT_EXCHANGE_MTU_REQUEST: {
-            // We only handle MTU of size BLE_GATT_ATT_MTU_DEFAULT.
-            sd_ble_gatts_exchange_mtu_reply(self->conn_handle, BLE_GATT_ATT_MTU_DEFAULT);
+            ble_gatts_evt_exchange_mtu_request_t *request =
+                    &ble_evt->evt.gatts_evt.params.exchange_mtu_request;
+
+            uint16_t new_mtu = BLE_GATTS_VAR_ATTR_LEN_MAX;
+            if (request->client_rx_mtu < new_mtu) {
+                new_mtu = request->client_rx_mtu;
+            }
+            if (new_mtu < BLE_GATT_ATT_MTU_DEFAULT) {
+                new_mtu = BLE_GATT_ATT_MTU_DEFAULT;
+            }
+            if (self->mtu > 0) {
+                new_mtu = self->mtu;
+            }
+
+            self->mtu = new_mtu;
+            sd_ble_gatts_exchange_mtu_reply(self->conn_handle, new_mtu);
+            break;
+        }
+
+
+        case BLE_GATTC_EVT_EXCHANGE_MTU_RSP: {
+            ble_gattc_evt_exchange_mtu_rsp_t *response =
+                    &ble_evt->evt.gattc_evt.params.exchange_mtu_rsp;
+
+            self->mtu = response->server_rx_mtu;
             break;
         }
 
@@ -319,8 +345,11 @@ STATIC bool discover_next_services(bleio_connection_internal_t* connection, uint
     m_discovery_successful = false;
     m_discovery_in_process = true;
 
-    check_nrf_error(sd_ble_gattc_primary_services_discover(connection->conn_handle,
-                                                           start_handle, service_uuid));
+    uint32_t nrf_err = NRF_ERROR_BUSY;
+    while (nrf_err == NRF_ERROR_BUSY) {
+        nrf_err = sd_ble_gattc_primary_services_discover(connection->conn_handle, start_handle, service_uuid);
+    }
+    check_nrf_error(nrf_err);
 
     // Wait for a discovery event.
     while (m_discovery_in_process) {

ports/nrf/common-hal/_bleio/Connection.h

@@ -65,6 +65,7 @@ typedef struct {
     ble_drv_evt_handler_entry_t handler_entry;
     ble_gap_conn_params_t conn_params;
     volatile bool conn_params_updating;
+    uint16_t mtu;
 } bleio_connection_internal_t;
 
 typedef struct {