drivers: dma: edma: eDMA loop SG mode support

Existing dynamic SG mode have some issues which cause uart sync api and
I2S speed test failed.

These issues are: a wrong "Done" bit issue found on UART async api
test. An invalid destination address issue found on I2S speed
test. By introducing loop SG mode, these issues are all fixed.

Some data structures are different between eDMA and eDMA V4, use
different macros defined in SDK driver for each of them.

Signed-off-by: Raymond Lei <[email protected]>

Author: Raymond0225

drivers/dma/dma_mcux_edma.c

@@ -92,6 +92,14 @@ struct dma_mcux_channel_transfer_edma_settings {
 	enum dma_channel_direction direction;
 	edma_transfer_type_t transfer_type;
 	bool valid;
+	/* This var indicate it is dynamic SG mode or loop SG mode. */
+	bool cyclic;
+	/* These parameters are for cyclic mode only.
+	 * Next empty TCD idx which can be used for transfer
+	 */
+	volatile int8_t write_idx;
+	/* How many TCDs in TCD pool is emtpy(can be used to write transfer parameters) */
+	volatile uint8_t empty_tcds;
 };
 
 
@@ -130,6 +138,35 @@ struct dma_mcux_edma_data {
 	(ch % DEV_CFG(dev)->channels_per_mux) ^ (DEV_CFG(dev)->dmamux_reg_offset)
 #endif
 
+/* Definations for SW TCD fields */
+#if defined(CONFIG_DMA_MCUX_EDMA) || defined(CONFIG_DMA_MCUX_EDMA_V3)
+#define EDMA_TCD_SADDR(tcd, flag)     ((tcd)->SADDR)
+#define EDMA_TCD_DADDR(tcd, flag)     ((tcd)->DADDR)
+#define EDMA_TCD_BITER(tcd, flag)     ((tcd)->BITER)
+#define EDMA_TCD_CITER(tcd, flag)     ((tcd)->CITER)
+#define EDMA_TCD_CSR(tcd, flag)       ((tcd)->CSR)
+#define EDMA_TCD_DLAST_SGA(tcd, flag) ((tcd)->DLAST_SGA)
+#define EDMA_HW_TCD_CH_ACTIVE_MASK    (DMA_CSR_ACTIVE_MASK)
+#elif defined(CONFIG_DMA_MCUX_EDMA_V4)
+/* Above macros have been defined in fsl_edma_core.h */
+#define EDMA_HW_TCD_CH_ACTIVE_MASK (DMA_CH_CSR_ACTIVE_MASK)
+#endif
+
+/* Definations for HW TCD fields */
+#ifdef CONFIG_DMA_MCUX_EDMA
+#define EDMA_HW_TCD_SADDR(dev, ch) (DEV_BASE(dev)->TCD[ch].SADDR)
+#define EDMA_HW_TCD_DADDR(dev, ch) (DEV_BASE(dev)->TCD[ch].DADDR)
+#define EDMA_HW_TCD_BITER(dev, ch) (DEV_BASE(dev)->TCD[ch].BITER_ELINKNO)
+#define EDMA_HW_TCD_CITER(dev, ch) (DEV_BASE(dev)->TCD[ch].CITER_ELINKNO)
+#define EDMA_HW_TCD_CSR(dev, ch)   (DEV_BASE(dev)->TCD[ch].CSR)
+#elif defined(CONFIG_DMA_MCUX_EDMA_V3) || defined(CONFIG_DMA_MCUX_EDMA_V4)
+#define EDMA_HW_TCD_SADDR(dev, ch) (DEV_BASE(dev)->CH[ch].TCD_SADDR)
+#define EDMA_HW_TCD_DADDR(dev, ch) (DEV_BASE(dev)->CH[ch].TCD_DADDR)
+#define EDMA_HW_TCD_BITER(dev, ch) (DEV_BASE(dev)->CH[ch].TCD_BITER_ELINKNO)
+#define EDMA_HW_TCD_CITER(dev, ch) (DEV_BASE(dev)->CH[ch].TCD_CITER_ELINKNO)
+#define EDMA_HW_TCD_CSR(dev, ch)   (DEV_BASE(dev)->CH[ch].TCD_CSR)
+#endif
+
 /*
  * The hardware channel (takes the gap into account) is used when access DMA registers.
  * For data structures in the shim driver still use the primitive channel.
@@ -178,10 +215,14 @@ static void nxp_edma_callback(edma_handle_t *handle, void *param, bool transferD
 {
 	int ret = -EIO;
 	struct call_back *data = (struct call_back *)param;
-
 	uint32_t channel = dma_mcux_edma_remove_channel_gap(data->dev, handle->channel);
 
-	if (transferDone) {
+	if (data->transfer_settings.cyclic) {
+		data->transfer_settings.empty_tcds++;
+		/*In loop mode, DMA is always busy*/
+		data->busy = 1;
+		ret = DMA_STATUS_COMPLETE;
+	} else if (transferDone) {
 		/* DMA is no longer busy when there are no remaining TCDs to transfer */
 		data->busy = (handle->tcdPool != NULL) && (handle->tcdUsed > 0);
 		ret = DMA_STATUS_COMPLETE;
@@ -254,6 +295,7 @@ static int dma_mcux_edma_configure(const struct device *dev, uint32_t channel,
 	edma_transfer_type_t transfer_type;
 	unsigned int key;
 	int ret = 0;
+	edma_tcd_t *tcd = NULL;
 
 	if (slot >= DEV_CFG(dev)->dma_requests) {
 		LOG_ERR("source number is out of scope %d", slot);
@@ -316,7 +358,7 @@ static int dma_mcux_edma_configure(const struct device *dev, uint32_t channel,
 	data->transfer_settings.direction = config->channel_direction;
 	data->transfer_settings.transfer_type = transfer_type;
 	data->transfer_settings.valid = true;
-
+	data->transfer_settings.cyclic = config->cyclic;
 
 	/* Lock and page in the channel configuration */
 	key = irq_lock();
@@ -357,26 +399,93 @@ static int dma_mcux_edma_configure(const struct device *dev, uint32_t channel,
 	LOG_DBG("channel is %d", channel);
 	EDMA_EnableChannelInterrupts(DEV_BASE(dev), hw_channel, kEDMA_ErrorInterruptEnable);
 
+	/* Initialize all TCD pool as 0*/
+	for (int i = 0; i < CONFIG_DMA_TCD_QUEUE_SIZE; i++) {
+		memset(&DEV_CFG(dev)->tcdpool[channel][i], 0,
+		       sizeof(DEV_CFG(dev)->tcdpool[channel][i]));
+	}
+
 	if (block_config->source_gather_en || block_config->dest_scatter_en) {
-		EDMA_InstallTCDMemory(p_handle, DEV_CFG(dev)->tcdpool[channel],
-				      CONFIG_DMA_TCD_QUEUE_SIZE);
-		while (block_config != NULL) {
+		if (config->cyclic) {
+			/* Loop SG mode */
+			data->transfer_settings.write_idx = 0;
+			data->transfer_settings.empty_tcds = CONFIG_DMA_TCD_QUEUE_SIZE;
+
 			EDMA_PrepareTransfer(
-				&(data->transferConfig),
-				(void *)block_config->source_address,
-				config->source_data_size,
-				(void *)block_config->dest_address,
-				config->dest_data_size,
-				config->source_burst_length,
+				&data->transferConfig, (void *)block_config->source_address,
+				config->source_data_size, (void *)block_config->dest_address,
+				config->dest_data_size, config->source_burst_length,
 				block_config->block_size, transfer_type);
 
-			const status_t submit_status =
-				EDMA_SubmitTransfer(p_handle, &(data->transferConfig));
-			if (submit_status != kStatus_Success) {
-				LOG_ERR("Error submitting EDMA Transfer: 0x%x", submit_status);
-				ret = -EFAULT;
+			/* Init all TCDs with the para in transfer config and link them. */
+			for (int i = 0; i < CONFIG_DMA_TCD_QUEUE_SIZE; i++) {
+				EDMA_TcdSetTransferConfig(
+					&DEV_CFG(dev)->tcdpool[channel][i], &data->transferConfig,
+					&DEV_CFG(dev)->tcdpool[channel][(i + 1) %
+									CONFIG_DMA_TCD_QUEUE_SIZE]);
+
+				/* Enable Major loop interrupt.*/
+				EDMA_TcdEnableInterrupts(&DEV_CFG(dev)->tcdpool[channel][i],
+							 kEDMA_MajorInterruptEnable);
+			}
+
+			/* Load valid transfer parameters */
+			while (block_config != NULL && data->transfer_settings.empty_tcds > 0) {
+				tcd = &(DEV_CFG(dev)->tcdpool[channel]
+							     [data->transfer_settings.write_idx]);
+
+				EDMA_TCD_SADDR(tcd, kEDMA_EDMA4Flag) = block_config->source_address;
+				EDMA_TCD_DADDR(tcd, kEDMA_EDMA4Flag) = block_config->dest_address;
+				EDMA_TCD_BITER(tcd, kEDMA_EDMA4Flag) =
+					block_config->block_size / config->source_data_size;
+				EDMA_TCD_CITER(tcd, kEDMA_EDMA4Flag) =
+					block_config->block_size / config->source_data_size;
+				/*Enable auto stop for last transfer.*/
+				if (block_config->next_block == NULL) {
+					EDMA_TCD_CSR(tcd, kEDMA_EDMA4Flag) |= DMA_CSR_DREQ(1U);
+				} else {
+					EDMA_TCD_CSR(tcd, kEDMA_EDMA4Flag) &= ~DMA_CSR_DREQ(1U);
+				}
+
+				data->transfer_settings.write_idx =
+					(data->transfer_settings.write_idx + 1) %
+					CONFIG_DMA_TCD_QUEUE_SIZE;
+				data->transfer_settings.empty_tcds--;
+				block_config = block_config->next_block;
+			}
+
+			if (block_config != NULL && data->transfer_settings.empty_tcds == 0) {
+				/* User input more blocks than TCD number, return error */
+				LOG_ERR("Too much request blocks,increase TCD buffer size!");
+				ret = -ENOBUFS;
+			}
+			/* Push the 1st TCD into HW */
+			EDMA_InstallTCD(p_handle->base, hw_channel,
+					&DEV_CFG(dev)->tcdpool[channel][0]);
+
+		} else {
+			/* Dynamic Scatter Gather mode */
+			EDMA_InstallTCDMemory(p_handle, DEV_CFG(dev)->tcdpool[channel],
+					      CONFIG_DMA_TCD_QUEUE_SIZE);
+
+			while (block_config != NULL) {
+				EDMA_PrepareTransfer(&(data->transferConfig),
+						     (void *)block_config->source_address,
+						     config->source_data_size,
+						     (void *)block_config->dest_address,
+						     config->dest_data_size,
+						     config->source_burst_length,
+						     block_config->block_size, transfer_type);
+
+				const status_t submit_status =
+					EDMA_SubmitTransfer(p_handle, &(data->transferConfig));
+				if (submit_status != kStatus_Success) {
+					LOG_ERR("Error submitting EDMA Transfer: 0x%x",
+						submit_status);
+					ret = -EFAULT;
+				}
+				block_config = block_config->next_block;
 			}
-			block_config = block_config->next_block;
 		}
 	} else {
 		/* block_count shall be 1 */
@@ -395,11 +504,8 @@ static int dma_mcux_edma_configure(const struct device *dev, uint32_t channel,
 			LOG_ERR("Error submitting EDMA Transfer: 0x%x", submit_status);
 			ret = -EFAULT;
 		}
-#if defined(CONFIG_DMA_MCUX_EDMA_V3) || defined(CONFIG_DMA_MCUX_EDMA_V4)
-		LOG_DBG("DMA TCD_CSR 0x%x", DEV_BASE(dev)->CH[hw_channel].TCD_CSR);
-#else
-		LOG_DBG("data csr is 0x%x", DEV_BASE(dev)->TCD[hw_channel].CSR);
-#endif
+
+		LOG_DBG("DMA TCD CSR 0x%x", EDMA_HW_TCD_CSR(dev, hw_channel));
 	}
 
 	if (config->dest_chaining_en) {
@@ -491,11 +597,23 @@ static int dma_mcux_edma_resume(const struct device *dev, uint32_t channel)
 	return 0;
 }
 
+static void dma_mcux_edma_update_hw_tcd(const struct device *dev, uint32_t channel, uint32_t src,
+					uint32_t dst, size_t size)
+{
+	EDMA_HW_TCD_SADDR(dev, channel) = src;
+	EDMA_HW_TCD_DADDR(dev, channel) = dst;
+	EDMA_HW_TCD_BITER(dev, channel) = size;
+	EDMA_HW_TCD_CITER(dev, channel) = size;
+	EDMA_HW_TCD_CSR(dev, channel) |= DMA_CSR_DREQ(1U);
+}
 
 static int dma_mcux_edma_reload(const struct device *dev, uint32_t channel,
 				uint32_t src, uint32_t dst, size_t size)
 {
 	struct call_back *data = DEV_CHANNEL_DATA(dev, channel);
+	edma_tcd_t *tcd = NULL;
+	edma_tcd_t *pre_tcd = NULL;
+	uint32_t hw_id, sw_id;
 
 	/* Lock the channel configuration */
 	const unsigned int key = irq_lock();
@@ -507,29 +625,115 @@ static int dma_mcux_edma_reload(const struct device *dev, uint32_t channel,
 		goto cleanup;
 	}
 
-	/* If the tcdPool is not in use (no s/g) then only a single TCD can be active at once. */
-	if (data->busy && data->edma_handle.tcdPool == NULL) {
-		LOG_ERR("EDMA busy. Wait until the transfer completes before reloading.");
-		ret = -EBUSY;
-		goto cleanup;
-	}
+	if (data->transfer_settings.cyclic) {
+		if (data->transfer_settings.empty_tcds == 0) {
+			LOG_ERR("TCD list is full in loop mode.");
+			ret = -ENOBUFS;
+			goto cleanup;
+		}
 
-	EDMA_PrepareTransfer(
-		&(data->transferConfig),
-		(void *)src,
-		data->transfer_settings.source_data_size,
-		(void *)dst,
-		data->transfer_settings.dest_data_size,
-		data->transfer_settings.source_burst_length,
-		size,
-		data->transfer_settings.transfer_type);
-
-	const status_t submit_status =
-		EDMA_SubmitTransfer(DEV_EDMA_HANDLE(dev, channel), &(data->transferConfig));
-
-	if (submit_status != kStatus_Success) {
-		LOG_ERR("Error submitting EDMA Transfer: 0x%x", submit_status);
-		ret = -EFAULT;
+		/* Convert size into major loop count */
+		size = size / data->transfer_settings.dest_data_size;
+
+		/* Configure a TCD for the transfer */
+		tcd = &(DEV_CFG(dev)->tcdpool[channel][data->transfer_settings.write_idx]);
+		pre_tcd = &(DEV_CFG(dev)->tcdpool[channel][(data->transfer_settings.write_idx - 1) %
+							   CONFIG_DMA_TCD_QUEUE_SIZE]);
+
+		EDMA_TCD_SADDR(tcd, kEDMA_EDMA4Flag) = src;
+		EDMA_TCD_DADDR(tcd, kEDMA_EDMA4Flag) = dst;
+		EDMA_TCD_BITER(tcd, kEDMA_EDMA4Flag) = size;
+		EDMA_TCD_CITER(tcd, kEDMA_EDMA4Flag) = size;
+		/* Enable automatically stop */
+		EDMA_TCD_CSR(tcd, kEDMA_EDMA4Flag) |= DMA_CSR_DREQ(1U);
+		sw_id = EDMA_TCD_DLAST_SGA(tcd, kEDMA_EDMA4Flag);
+
+		/* Block the peripheral's hardware request trigger to prevent
+		 * starting the DMA before updating the TCDs.  Make sure the
+		 * code between EDMA_DisableChannelRequest() and
+		 * EDMA_EnableChannelRequest() is minimum.
+		 */
+		EDMA_DisableChannelRequest(DEV_BASE(dev), channel);
+
+		/* Wait for the DMA to be inactive before updating the TCDs.
+		 * The CSR[ACTIVE] bit will deassert quickly after the EDMA's
+		 * minor loop burst completes.
+		 */
+		while (EDMA_HW_TCD_CSR(dev, channel) & EDMA_HW_TCD_CH_ACTIVE_MASK) {
+			;
+		}
+
+		/* Identify the current active TCD.  Use DLAST_SGA as the HW ID */
+		hw_id = EDMA_GetNextTCDAddress(DEV_EDMA_HANDLE(dev, channel));
+		if (data->transfer_settings.empty_tcds >= CONFIG_DMA_TCD_QUEUE_SIZE ||
+		    hw_id == sw_id) {
+			/* All transfers have been done.DMA is stopped automatically,
+			 * invalid TCD has been loaded into the HW, update HW.
+			 */
+			dma_mcux_edma_update_hw_tcd(dev, channel, src, dst, size);
+			LOG_DBG("Transfer done,auto stop");
+
+		} else {
+			/* Previous TCD can automatically start this TCD.
+			 * Enable the peripheral DMA request in the previous TCD
+			 */
+			EDMA_TCD_CSR(pre_tcd, kEDMA_EDMA4Flag) &= ~DMA_CSR_DREQ(1U);
+
+			if (data->transfer_settings.empty_tcds == CONFIG_DMA_TCD_QUEUE_SIZE - 1 ||
+			    hw_id == (uint32_t)tcd) {
+				/* DMA is running on last transfer. HW has loaded the last one,
+				 * we need ensure it's DREQ is cleared.
+				 */
+				EDMA_EnableAutoStopRequest(DEV_BASE(dev), channel, false);
+				LOG_DBG("Last transfer.");
+			}
+			LOG_DBG("Manu stop");
+		}
+
+#ifdef CONFIG_DMA_MCUX_EDMA
+		/* It seems that there is HW issue which may cause ESG bit is cleared.
+		 * This is a workaround. Clear the DONE bit before setting ESG bit.
+		 */
+		EDMA_ClearChannelStatusFlags(DEV_BASE(dev), channel, kEDMA_DoneFlag);
+		EDMA_HW_TCD_CSR(dev, channel) |= DMA_CSR_ESG_MASK;
+#elif (CONFIG_DMA_MCUX_EDMA_V3 || CONFIG_DMA_MCUX_EDMA_V4)
+		/*We have not verified if this issue exist on V3/V4 HW, jut place a holder here. */
+#endif
+		/* TCDs are configured.  Resume DMA */
+		EDMA_EnableChannelRequest(DEV_BASE(dev), channel);
+
+		/* Update the write index and available TCD numbers. */
+		data->transfer_settings.write_idx =
+			(data->transfer_settings.write_idx + 1) % CONFIG_DMA_TCD_QUEUE_SIZE;
+		data->transfer_settings.empty_tcds--;
+
+		LOG_DBG("w_idx:%d no:%d(ch:%d)", data->transfer_settings.write_idx,
+			data->transfer_settings.empty_tcds, channel);
+
+	} else {
+		/* Dynamice Scatter/Gather mode:
+		 * If the tcdPool is not in use (no s/g) then only a single TCD
+		 * can be active at once.
+		 */
+		if (data->busy && data->edma_handle.tcdPool == NULL) {
+			LOG_ERR("EDMA busy. Wait until the transfer completes before reloading.");
+			ret = -EBUSY;
+			goto cleanup;
+		}
+
+		EDMA_PrepareTransfer(&(data->transferConfig), (void *)src,
+				     data->transfer_settings.source_data_size, (void *)dst,
+				     data->transfer_settings.dest_data_size,
+				     data->transfer_settings.source_burst_length, size,
+				     data->transfer_settings.transfer_type);
+
+		const status_t submit_status =
+			EDMA_SubmitTransfer(DEV_EDMA_HANDLE(dev, channel), &(data->transferConfig));
+
+		if (submit_status != kStatus_Success) {
+			LOG_ERR("Error submitting EDMA Transfer: 0x%x", submit_status);
+			ret = -EFAULT;
+		}
 	}
 
 cleanup:
@@ -544,7 +748,9 @@ static int dma_mcux_edma_get_status(const struct device *dev, uint32_t channel,
 
 	if (DEV_CHANNEL_DATA(dev, channel)->busy) {
 		status->busy = true;
-		/* Be aware that here we need multiply NBYTES for each minor loops */
+		/* pending_length is in bytes.  Multiply remaining major loop
+		 * count by NBYTES for each minor loop
+		 */
 		status->pending_length =
 			EDMA_GetRemainingMajorLoopCount(DEV_BASE(dev), hw_channel) *
 			DEV_CHANNEL_DATA(dev, channel)->transfer_settings.source_data_size;