!wip: pbio/drv/usb_ev3: Fix lockup for stdout.

Receiving a command such as stop while stdout is active locks up Pybricks Code.

This is a work-in-progress fix that makes the logic a bit more similar to the STM32 version.

Fixes pybricks/support#2374

Author: laurensvalk

lib/pbio/drv/usb/usb_ev3.c

@@ -225,12 +225,11 @@ static uint8_t ep1_rx_buf[PYBRICKS_EP_PKT_SZ_HS] PBDRV_DMA_BUF;
 static uint8_t ep1_tx_response_buf[PYBRICKS_EP_PKT_SZ_HS];
 static uint8_t ep1_tx_status_buf[PYBRICKS_EP_PKT_SZ_HS];
 static uint8_t ep1_tx_stdout_buf[PYBRICKS_EP_PKT_SZ_HS];
+static uint32_t ep1_tx_stdout_sz;
 
 // Buffer status flags
 static volatile bool usb_rx_is_ready;
-static volatile bool usb_tx_response_is_not_ready;
-static volatile bool usb_tx_status_is_not_ready;
-static volatile bool usb_tx_stdout_is_not_ready;
+static volatile bool transmitting;
 
 // CPPI DMA support code
 
@@ -261,8 +260,7 @@ _Static_assert(sizeof(usb_cppi_hpd_t) <= CPPI_DESCRIPTOR_ALIGN);
 enum {
     CPPI_DESC_RX,
     CPPI_DESC_TX_RESPONSE,
-    CPPI_DESC_TX_STATUS,
-    CPPI_DESC_TX_STDOUT,
+    CPPI_DESC_TX_PYBRICKS_EVENT,
     // the minimum number of descriptors we can allocate is 32,
     // even though we do not use nearly all of them
     CPPI_DESC_COUNT = 32,
@@ -860,13 +858,10 @@ static void usb_device_intr(void) {
         uint32_t qctrld = HWREG(USB_0_OTGBASE + CPDMA_QUEUE_REGISTER_D + TX_COMPQ1 * 16) & ~CPDMA_QUEUE_REGISTER_DESC_SIZE_MASK;
 
         if (qctrld == (uint32_t)(&cppi_descriptors[CPPI_DESC_TX_RESPONSE])) {
-            usb_tx_response_is_not_ready = false;
+            transmitting = false;
             pbio_os_request_poll();
-        } else if (qctrld == (uint32_t)(&cppi_descriptors[CPPI_DESC_TX_STATUS])) {
-            usb_tx_status_is_not_ready = false;
-            pbio_os_request_poll();
-        } else if (qctrld == (uint32_t)(&cppi_descriptors[CPPI_DESC_TX_STDOUT])) {
-            usb_tx_stdout_is_not_ready = false;
+        } else if (qctrld == (uint32_t)(&cppi_descriptors[CPPI_DESC_TX_PYBRICKS_EVENT])) {
+            transmitting = false;
             pbio_os_request_poll();
         }
 
@@ -906,124 +901,142 @@ void pbdrv_usb_schedule_status_update(const uint8_t *status_msg) {
     pbio_os_request_poll();
 }
 
-static pbio_os_process_t pbdrv_usb_ev3_process;
+static bool usb_send_response;
 
-static pbio_error_t pbdrv_usb_ev3_process_thread(pbio_os_state_t *state, void *context) {
-    static pbio_os_timer_t delay_timer;
-    static pbio_os_timer_t tx_timeout_timer;
-    static pbio_os_timer_t keepalive_timer;
-    static bool was_transmitting = false;
-    static bool is_transmitting = false;
+/**
+ * Non-blocking poll handler to process pending incoming messages.
+ */
+static void pbdrv_usb_handle_data_in(void) {
+    if (!usb_rx_is_ready) {
+        return;
+    }
+    // This barrier prevents *subsequent* memory reads from being
+    // speculatively moved *earlier*, outside the if statement
+    // (which is technically allowed by the as-if rule).
+    pbdrv_compiler_memory_barrier();
 
-    PBIO_OS_ASYNC_BEGIN(state);
+    uint32_t usb_rx_sz = PBDRV_UNCACHED(cppi_descriptors[CPPI_DESC_RX].word0).pktLength;
+    pbio_pybricks_error_t result;
 
-    for (;;) {
-        if (pbdrv_usb_config == 0) {
-            pbdrv_usb_is_events_subscribed = false;
-            // Resend status flags when host subscribes
-            pbdrv_usb_status_data_pending = true;
-        }
+    // Skip empty commands
+    if (usb_rx_sz) {
+        pbdrv_cache_prepare_after_dma(ep1_rx_buf, sizeof(ep1_rx_buf));
 
-        if (usb_rx_is_ready) {
-            // This barrier prevents *subsequent* memory reads from being
-            // speculatively moved *earlier*, outside the if statement
-            // (which is technically allowed by the as-if rule).
-            pbdrv_compiler_memory_barrier();
-
-            uint32_t usb_rx_sz = PBDRV_UNCACHED(cppi_descriptors[CPPI_DESC_RX].word0).pktLength;
-            pbio_pybricks_error_t result;
-            bool usb_send_response = false;
-            // Skip empty commands, or commands sent when previous response is still busy
-            if (usb_rx_sz && !usb_tx_response_is_not_ready) {
-                pbdrv_cache_prepare_after_dma(ep1_rx_buf, sizeof(ep1_rx_buf));
-
-                switch (ep1_rx_buf[0]) {
-                    case PBIO_PYBRICKS_OUT_EP_MSG_SUBSCRIBE:
-                        pbio_os_timer_set(&keepalive_timer, 1000);
-                        pbdrv_usb_is_events_subscribed = ep1_rx_buf[1];
-                        ep1_tx_response_buf[0] = PBIO_PYBRICKS_IN_EP_MSG_RESPONSE;
-                        pbio_set_uint32_le(&ep1_tx_response_buf[1], PBIO_PYBRICKS_ERROR_OK);
-                        usb_send_response = true;
-                        break;
-                    case PBIO_PYBRICKS_OUT_EP_MSG_COMMAND:
-                        if (!pbdrv_usb_receive_handler) {
-                            break;
-                        }
-                        result = pbdrv_usb_receive_handler(ep1_rx_buf + 1, usb_rx_sz - 1);
-                        ep1_tx_response_buf[0] = PBIO_PYBRICKS_IN_EP_MSG_RESPONSE;
-                        pbio_set_uint32_le(&ep1_tx_response_buf[1], result);
-                        usb_send_response = true;
-                        break;
+        switch (ep1_rx_buf[0]) {
+            case PBIO_PYBRICKS_OUT_EP_MSG_SUBSCRIBE:
+                pbdrv_usb_is_events_subscribed = ep1_rx_buf[1];
+                ep1_tx_response_buf[0] = PBIO_PYBRICKS_IN_EP_MSG_RESPONSE;
+                pbio_set_uint32_le(&ep1_tx_response_buf[1], PBIO_PYBRICKS_ERROR_OK);
+                usb_send_response = true;
+                // Resend status flags when host subscribes
+                pbdrv_usb_status_data_pending = true;
+                break;
+            case PBIO_PYBRICKS_OUT_EP_MSG_COMMAND:
+                if (!pbdrv_usb_receive_handler) {
+                    break;
                 }
-            }
+                result = pbdrv_usb_receive_handler(ep1_rx_buf + 1, usb_rx_sz - 1);
+                ep1_tx_response_buf[0] = PBIO_PYBRICKS_IN_EP_MSG_RESPONSE;
+                pbio_set_uint32_le(&ep1_tx_response_buf[1], result);
+                usb_send_response = true;
+                break;
+        }
+    }
 
-            if (usb_send_response) {
-                usb_tx_response_is_not_ready = true;
-                pbdrv_cache_prepare_before_dma(ep1_tx_response_buf, sizeof(ep1_tx_response_buf));
-                usb_setup_tx_dma_desc(CPPI_DESC_TX_RESPONSE, ep1_tx_response_buf, PBIO_PYBRICKS_USB_MESSAGE_SIZE(sizeof(uint32_t)));
-            }
+    // Re-queue RX buffer after processing is complete
+    usb_rx_is_ready = false;
+    usb_setup_rx_dma_desc();
+}
+
+static pbio_error_t pbdrv_usb_handle_data_out(void) {
+
+    static pbio_os_timer_t transmit_timer;
 
-            // Re-queue RX buffer after processing is complete
-            usb_rx_is_ready = false;
-            usb_setup_rx_dma_desc();
+    if (transmitting) {
+        if (!pbio_os_timer_is_expired(&transmit_timer)) {
+            // Still transmitting, can't do anything for now.
+            return PBIO_SUCCESS;
         }
 
-        // Send status flags if they've changed (and we can)
-        if (pbdrv_usb_is_events_subscribed && !usb_tx_status_is_not_ready &&
-            (pbdrv_usb_status_data_pending || pbio_os_timer_is_expired(&keepalive_timer))) {
-            ep1_tx_status_buf[0] = PBIO_PYBRICKS_IN_EP_MSG_EVENT;
-            uint32_t usb_status_sz = PBIO_PYBRICKS_USB_MESSAGE_SIZE(pbsys_status_get_status_report(&ep1_tx_status_buf[1]));
+        // Transmission has taken too long, so reset the state to allow
+        // new transmissions. This can happen if the host stops reading
+        // data for some reason.
+        return PBIO_ERROR_TIMEDOUT;
+    }
 
-            usb_tx_status_is_not_ready = true;
-            pbdrv_cache_prepare_before_dma(ep1_tx_status_buf, sizeof(ep1_tx_status_buf));
-            usb_setup_tx_dma_desc(CPPI_DESC_TX_STATUS, ep1_tx_status_buf, usb_status_sz);
+    // Transmit. Give priority to response, then status updates, then stdout.
+    if (usb_send_response) {
+        usb_send_response = false;
+        transmitting = true;
+        pbdrv_cache_prepare_before_dma(ep1_tx_response_buf, sizeof(ep1_tx_response_buf));
+        usb_setup_tx_dma_desc(CPPI_DESC_TX_RESPONSE, ep1_tx_response_buf, PBIO_PYBRICKS_USB_MESSAGE_SIZE(sizeof(uint32_t)));
+    } else if (pbdrv_usb_is_events_subscribed && pbdrv_usb_status_data_pending) {
+        pbdrv_usb_status_data_pending = false;
+        ep1_tx_status_buf[0] = PBIO_PYBRICKS_IN_EP_MSG_EVENT;
+        uint32_t usb_status_sz = PBIO_PYBRICKS_USB_MESSAGE_SIZE(pbsys_status_get_status_report(&ep1_tx_status_buf[1]));
+
+        transmitting = true;
+        pbdrv_cache_prepare_before_dma(ep1_tx_status_buf, sizeof(ep1_tx_status_buf));
+        usb_setup_tx_dma_desc(CPPI_DESC_TX_PYBRICKS_EVENT, ep1_tx_status_buf, usb_status_sz);
+    } else if (ep1_tx_stdout_sz) {
+        transmitting = true;
+        pbdrv_cache_prepare_before_dma(ep1_tx_stdout_buf, sizeof(ep1_tx_stdout_buf));
+        usb_setup_tx_dma_desc(CPPI_DESC_TX_PYBRICKS_EVENT, ep1_tx_stdout_buf, ep1_tx_stdout_sz);
+        ep1_tx_stdout_sz = 0;
+    }
 
-            if (pbdrv_usb_status_data_pending) {
-                // If we are sending a status because the flags have changed,
-                // we can bump out the keepalive timer.
-                pbio_os_timer_set(&keepalive_timer, 1000);
-            } else {
-                // Otherwise, we want to send keepalives at a particular rate.
-                pbio_os_timer_extend(&keepalive_timer);
-            }
-            pbdrv_usb_status_data_pending = false;
+    if (transmitting) {
+        // If the FIFO isn't emptied quickly, then there probably isn't an
+        // app anymore. This timer is used to detect such a condition.
+        pbio_os_timer_set(&transmit_timer, 50);
+    }
+    return PBIO_SUCCESS;
+}
+
+static pbio_os_process_t pbdrv_usb_ev3_process;
+
+static pbio_error_t pbdrv_usb_ev3_process_thread(pbio_os_state_t *state, void *context) {
+
+    static pbio_os_timer_t timer;
+
+    pbdrv_usb_handle_data_in();
+
+    if (pbdrv_usb_config == 0) {
+        pbdrv_usb_is_events_subscribed = false;
+    }
+
+    PBIO_OS_ASYNC_BEGIN(state);
+
+    // REVISIT: Exit on pbdrv_usb_deinit
+    for (;;) {
+
+        PBIO_OS_ASYNC_SET_CHECKPOINT(state);
+        pbio_error_t err = pbdrv_usb_handle_data_out();
+        if (err == PBIO_SUCCESS) {
+            return PBIO_ERROR_AGAIN;
         }
 
-        // Handle timeouts
-        is_transmitting = usb_tx_response_is_not_ready || usb_tx_status_is_not_ready || usb_tx_stdout_is_not_ready;
-        if (was_transmitting && is_transmitting) {
-            if (pbio_os_timer_is_expired(&tx_timeout_timer)) {
-                // Flush _all_ TX packets
-                while (HWREGB(USB0_BASE + USB_O_TXCSRL1) & USB_TXCSRL1_TXRDY) {
-                    HWREGB(USB0_BASE + USB_O_TXCSRL1) = USB_TXCSRL1_FLUSH;
-                    // We need to wait a bit until the DMA refills the FIFO.
-                    // There doesn't seem to be a good way to figure out if
-                    // there are packets in flight *within* the DMA engine itself
-                    // (i.e. no longer in the queue but in the transfer engine).
-                    PBIO_OS_AWAIT_MS(state, &delay_timer, 1);
-                }
-                usb_tx_response_is_not_ready = false;
-                usb_tx_status_is_not_ready = false;
-                usb_tx_stdout_is_not_ready = false;
-                pbdrv_usb_is_events_subscribed = false;
-                // Resend status flags when host subscribes
-                pbdrv_usb_status_data_pending = true;
-            }
-        } else if (!was_transmitting && is_transmitting) {
-            pbio_os_timer_set(&tx_timeout_timer, 50);
+        // Flush _all_ TX packets
+        while (HWREGB(USB0_BASE + USB_O_TXCSRL1) & USB_TXCSRL1_TXRDY) {
+            HWREGB(USB0_BASE + USB_O_TXCSRL1) = USB_TXCSRL1_FLUSH;
+            // We need to wait a bit until the DMA refills the FIFO.
+            // There doesn't seem to be a good way to figure out if
+            // there are packets in flight *within* the DMA engine itself
+            // (i.e. no longer in the queue but in the transfer engine).
+            PBIO_OS_AWAIT_MS(state, &timer, 1);
         }
-        was_transmitting = is_transmitting;
+        transmitting = false;
+        pbdrv_usb_is_events_subscribed = false;
 
-        // Make this process loop run from the beginning.
-        // Because we check so many different conditions,
-        // we don't use the normal await macros.
-        PBIO_OS_ASYNC_RESET(state);
-        return PBIO_ERROR_AGAIN;
     }
 
     PBIO_OS_ASYNC_END(PBIO_SUCCESS);
 }
 
+void pbdrv_usb_deinit(void) {
+    // todo
+}
+
 void pbdrv_usb_init(void) {
     // If we came straight from a firmware update, we need to send a disconnect
     // to the host, then reset the USB controller.
@@ -1138,7 +1151,7 @@ pbio_error_t pbdrv_usb_stdout_tx(const uint8_t *data, uint32_t *size) {
     uint8_t *ptr = ep1_tx_stdout_buf;
     uint32_t ptr_len = pbdrv_usb_is_usb_hs ? PYBRICKS_EP_PKT_SZ_HS : PYBRICKS_EP_PKT_SZ_FS;
 
-    if (usb_tx_stdout_is_not_ready) {
+    if (transmitting) {
         *size = 0;
         return PBIO_ERROR_AGAIN;
     }
@@ -1153,10 +1166,8 @@ pbio_error_t pbdrv_usb_stdout_tx(const uint8_t *data, uint32_t *size) {
         *size = ptr_len;
     }
     memcpy(ptr, data, *size);
-
-    usb_tx_stdout_is_not_ready = true;
-    pbdrv_cache_prepare_before_dma(ep1_tx_stdout_buf, sizeof(ep1_tx_stdout_buf));
-    usb_setup_tx_dma_desc(CPPI_DESC_TX_STDOUT, ep1_tx_stdout_buf, 2 + *size);
+    ep1_tx_stdout_sz = 2 + *size;
+    pbio_os_request_poll();
 
     return PBIO_SUCCESS;
 }
@@ -1171,7 +1182,7 @@ uint32_t pbdrv_usb_stdout_tx_available(void) {
         return UINT32_MAX;
     }
 
-    if (usb_tx_stdout_is_not_ready) {
+    if (transmitting) {
         return 0;
     }
 
@@ -1183,7 +1194,7 @@ uint32_t pbdrv_usb_stdout_tx_available(void) {
 }
 
 bool pbdrv_usb_stdout_tx_is_idle(void) {
-    return !usb_tx_stdout_is_not_ready;
+    return !transmitting;
 }
 
 #endif // PBDRV_CONFIG_USB_EV3