[nrf fromtree] drivers: usb: dwc2: Do cache operations on ownership change

Perform cache operations in thread context when the buffer ownership
changes between USB stack and UDC driver. This offers clearly measurable
throughput increase on Mass Storage when double buffering is enabled.
When endpoint operations are not double buffered the difference is
negligible. The positive side effect is reducing number of operations
performed in interrupt context.

Signed-off-by: Tomasz Moń <[email protected]>
(cherry picked from commit 614dd57)

Author: tmon-nordic

drivers/usb/udc/udc_dwc2.c

@@ -419,6 +419,11 @@ static int dwc2_ctrl_feed_dout(const struct device *dev, const size_t length)
 		return -ENOMEM;
 	}
 
+	if (dwc2_in_buffer_dma_mode(dev)) {
+		/* Get rid of all dirty cache lines */
+		sys_cache_data_invd_range(buf->data, net_buf_tailroom(buf));
+	}
+
 	udc_buf_put(ep_cfg, buf);
 	atomic_set_bit(&priv->xfer_new, 16);
 	k_event_post(&priv->drv_evt, BIT(DWC2_DRV_EVT_XFER));
@@ -589,8 +594,6 @@ static int dwc2_tx_fifo_write(const struct device *dev,
 
 		sys_write32((uint32_t)buf->data,
 			    (mem_addr_t)&base->in_ep[ep_idx].diepdma);
-
-		sys_cache_data_flush_range(buf->data, len);
 	}
 
 	diepctl = sys_read32(diepctl_reg);
@@ -790,8 +793,6 @@ static void dwc2_prep_rx(const struct device *dev, struct net_buf *buf,
 
 		sys_write32((uint32_t)data,
 			    (mem_addr_t)&base->out_ep[ep_idx].doepdma);
-
-		sys_cache_data_invd_range(data, xfersize);
 	}
 
 	sys_write32(doepctl, doepctl_reg);
@@ -968,6 +969,10 @@ static inline int dwc2_handle_evt_dout(const struct device *dev,
 		return -ENODATA;
 	}
 
+	if (dwc2_in_buffer_dma_mode(dev)) {
+		sys_cache_data_invd_range(buf->data, buf->len);
+	}
+
 	udc_ep_set_busy(cfg, false);
 
 	if (cfg->addr == USB_CONTROL_EP_OUT) {
@@ -1835,6 +1840,17 @@ static int udc_dwc2_ep_enqueue(const struct device *dev,
 	struct udc_dwc2_data *const priv = udc_get_private(dev);
 
 	LOG_DBG("%p enqueue %x %p", dev, cfg->addr, buf);
+
+	if (dwc2_in_buffer_dma_mode(dev)) {
+		if (USB_EP_DIR_IS_IN(cfg->addr)) {
+			/* Write all dirty cache lines to memory */
+			sys_cache_data_flush_range(buf->data, buf->len);
+		} else {
+			/* Get rid of all dirty cache lines */
+			sys_cache_data_invd_range(buf->data, net_buf_tailroom(buf));
+		}
+	}
+
 	udc_buf_put(cfg, buf);
 
 	if (!cfg->stat.halted) {
@@ -1861,6 +1877,13 @@ static int udc_dwc2_ep_dequeue(const struct device *dev,
 	udc_dwc2_ep_disable(dev, cfg, false, true);
 
 	buf = udc_buf_get_all(cfg);
+
+	if (dwc2_in_buffer_dma_mode(dev) && USB_EP_DIR_IS_OUT(cfg->addr)) {
+		for (struct net_buf *iter = buf; iter; iter = iter->frags) {
+			sys_cache_data_invd_range(iter->data, iter->len);
+		}
+	}
+
 	if (buf) {
 		udc_submit_ep_event(dev, buf, -ECONNABORTED);
 	}
@@ -2821,7 +2844,9 @@ static inline void dwc2_handle_out_xfercompl(const struct device *dev,
 	}
 
 	if (dwc2_in_buffer_dma_mode(dev) && bcnt) {
-		sys_cache_data_invd_range(net_buf_tail(buf), bcnt);
+		/* Update just the length, cache will be invalidated in thread
+		 * context after transfer if finished or cancelled.
+		 */
 		net_buf_add(buf, bcnt);
 	}