Improve PIO DMA performance

arch/arm64/boot/dts/broadcom/bcm2712-rpi.dtsi

@@ -2,6 +2,9 @@
 
 #include <dt-bindings/power/raspberrypi-power.h>
 
+#define DMA_SEL_WANTHEAVY       (1 << 8)
+#define DMA_SEL_ONLYHEAVY       (1 << 9)
+
 &soc {
 	firmware: firmware {
 		compatible = "raspberrypi,bcm2835-firmware", "simple-mfd";
@@ -91,7 +94,21 @@
 	};
 };
 
+&rp1_dma {
+	snps,sel-require =  <DMA_SEL_WANTHEAVY DMA_SEL_WANTHEAVY 0 0 0 0 0 0>;
+	snps,sel-preclude = <0 0 DMA_SEL_ONLYHEAVY DMA_SEL_ONLYHEAVY DMA_SEL_ONLYHEAVY
+			     DMA_SEL_ONLYHEAVY DMA_SEL_ONLYHEAVY DMA_SEL_ONLYHEAVY>;
+};
+
 pio: &rp1_pio {
+	dmas = <&rp1_dma (RP1_DMA_PIO_CH0_TX | DMA_SEL_WANTHEAVY)>,
+			<&rp1_dma (RP1_DMA_PIO_CH0_RX | DMA_SEL_WANTHEAVY)>,
+			<&rp1_dma (RP1_DMA_PIO_CH1_TX | DMA_SEL_WANTHEAVY)>,
+			<&rp1_dma (RP1_DMA_PIO_CH1_RX | DMA_SEL_WANTHEAVY)>,
+			<&rp1_dma (RP1_DMA_PIO_CH2_TX | DMA_SEL_WANTHEAVY)>,
+			<&rp1_dma (RP1_DMA_PIO_CH2_RX | DMA_SEL_WANTHEAVY)>,
+			<&rp1_dma (RP1_DMA_PIO_CH3_TX | DMA_SEL_WANTHEAVY)>,
+			<&rp1_dma (RP1_DMA_PIO_CH3_RX | DMA_SEL_WANTHEAVY)>;
 	status = "okay";
 };
 

arch/arm64/boot/dts/broadcom/rp1.dtsi

@@ -1111,7 +1111,7 @@
 			snps,data-width = <4>; // (8 << 4) == 128 bits
 			snps,block-size = <0x40000 0x40000 0x40000 0x40000 0x40000 0x40000 0x40000 0x40000>;
 			snps,priority = <0 1 2 3 4 5 6 7>;
-			snps,axi-max-burst-len = <4>;
+			snps,axi-max-burst-len = <8 8 4 4 4 4 4 4>;
 			status = "disabled";
 		};
 

drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c

@@ -809,7 +809,7 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan,
 	ctlhi = CH_CTL_H_LLI_VALID;
 
 	if (chan->chip->dw->hdata->restrict_axi_burst_len) {
-		burst_len = chan->chip->dw->hdata->axi_rw_burst_len;
+		burst_len = chan->chip->dw->hdata->axi_rw_burst_len[chan->id];
 		ctlhi |= CH_CTL_H_ARLEN_EN | CH_CTL_H_AWLEN_EN |
 			 burst_len << CH_CTL_H_ARLEN_POS |
 			 burst_len << CH_CTL_H_AWLEN_POS;
@@ -1087,7 +1087,7 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr,
 
 		reg = CH_CTL_H_LLI_VALID;
 		if (chan->chip->dw->hdata->restrict_axi_burst_len) {
-			u32 burst_len = chan->chip->dw->hdata->axi_rw_burst_len;
+			u32 burst_len = chan->chip->dw->hdata->axi_rw_burst_len[chan->id];
 
 			reg |= (CH_CTL_H_ARLEN_EN |
 				burst_len << CH_CTL_H_ARLEN_POS |
@@ -1476,26 +1476,54 @@ static int __maybe_unused axi_dma_runtime_resume(struct device *dev)
 	return axi_dma_resume(chip);
 }
 
+static bool dw_axi_dma_filter_fn(struct dma_chan *dchan, void *filter_param)
+{
+	struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan);
+	uint32_t selector = *(const uint32_t *)filter_param;
+
+	return !!(selector & (1 << chan->id));
+}
+
 static struct dma_chan *dw_axi_dma_of_xlate(struct of_phandle_args *dma_spec,
 					    struct of_dma *ofdma)
 {
 	struct dw_axi_dma *dw = ofdma->of_dma_data;
 	struct axi_dma_chan *chan;
 	struct dma_chan *dchan;
+	uint32_t chan_mask = 0;
+	uint32_t chan_sel;
+	dma_cap_mask_t mask;
+	int i;
 
-	dchan = dma_get_any_slave_channel(&dw->dma);
+	/*
+	 * Walk through all channels looking for the best match.
+	 * Starting from 0, choose the first available slave channel which isn't precluded.
+	 */
+	chan_sel = dma_spec->args[0];
+
+	for (i = 0; i < dw->hdata->nr_channels; i++) {
+		if (((dw->sel_precluded[i] & chan_sel) == 0) &&
+		    ((dw->sel_required[i] & chan_sel) == dw->sel_required[i]))
+			chan_mask |= (1 << i);
+	}
+
+	dma_cap_zero(mask);
+	dma_cap_set(DMA_SLAVE, mask);
+
+	dchan = __dma_request_channel(&mask, dw_axi_dma_filter_fn, &chan_mask, ofdma->of_node);
 	if (!dchan)
 		return NULL;
 
 	chan = dchan_to_axi_dma_chan(dchan);
-	chan->hw_handshake_num = dma_spec->args[0];
+	chan->hw_handshake_num = (u8)chan_sel;
 	return dchan;
 }
 
 static int parse_device_properties(struct axi_dma_chip *chip)
 {
 	struct device *dev = chip->dev;
 	u32 tmp, carr[DMAC_MAX_CHANNELS];
+	u32 val;
 	int ret;
 
 	ret = device_property_read_u32(dev, "dma-channels", &tmp);
@@ -1552,17 +1580,35 @@ static int parse_device_properties(struct axi_dma_chip *chip)
 	}
 
 	/* axi-max-burst-len is optional property */
-	ret = device_property_read_u32(dev, "snps,axi-max-burst-len", &tmp);
-	if (!ret) {
-		if (tmp > DWAXIDMAC_ARWLEN_MAX + 1)
-			return -EINVAL;
-		if (tmp < DWAXIDMAC_ARWLEN_MIN + 1)
-			return -EINVAL;
-
+	ret = device_property_read_u32_array(dev, "snps,axi-max-burst-len", NULL,
+					     chip->dw->hdata->nr_channels);
+	if ((ret > 0) &&
+	    !device_property_read_u32_array(dev, "snps,axi-max-burst-len",
+					    carr, ret)) {
 		chip->dw->hdata->restrict_axi_burst_len = true;
-		chip->dw->hdata->axi_rw_burst_len = tmp;
+		for (tmp = 0; tmp < chip->dw->hdata->nr_channels; tmp++) {
+			// Replicate the last value to any remaining channels
+			val = carr[min(tmp, (u32)ret - 1)];
+			if (val > DWAXIDMAC_ARWLEN_MAX + 1)
+				return -EINVAL;
+			if (val < DWAXIDMAC_ARWLEN_MIN + 1)
+				return -EINVAL;
+			chip->dw->hdata->axi_rw_burst_len[tmp] = val;
+		}
 	}
 
+	/* sel-require is optional */
+	memset(chip->dw->sel_required, 0, sizeof(chip->dw->sel_required));
+	device_property_read_u32_array(dev, "snps,sel-require",
+				       chip->dw->sel_required,
+				       chip->dw->hdata->nr_channels);
+
+	/* sel-preclude is optional */
+	memset(chip->dw->sel_precluded, 0, sizeof(chip->dw->sel_precluded));
+	device_property_read_u32_array(dev, "snps,sel-preclude",
+				       chip->dw->sel_precluded,
+				       chip->dw->hdata->nr_channels);
+
 	return 0;
 }
 
@@ -1674,7 +1720,7 @@ static int dw_probe(struct platform_device *pdev)
 	dma_cap_set(DMA_CYCLIC, dw->dma.cap_mask);
 
 	/* DMA capabilities */
-	dw->dma.max_burst = hdata->axi_rw_burst_len;
+	dw->dma.max_burst = hdata->axi_rw_burst_len[0];
 	dw->dma.src_addr_widths = AXI_DMA_BUSWIDTHS;
 	dw->dma.dst_addr_widths = AXI_DMA_BUSWIDTHS;
 	dw->dma.directions = BIT(DMA_MEM_TO_MEM);

drivers/dma/dw-axi-dmac/dw-axi-dmac.h

@@ -29,7 +29,7 @@ struct dw_axi_dma_hcfg {
 	u32	block_size[DMAC_MAX_CHANNELS];
 	u32	priority[DMAC_MAX_CHANNELS];
 	/* maximum supported axi burst length */
-	u32	axi_rw_burst_len;
+	u32	axi_rw_burst_len[DMAC_MAX_CHANNELS];
 	/* Register map for DMAX_NUM_CHANNELS <= 8 */
 	bool	reg_map_8_channels;
 	bool	restrict_axi_burst_len;
@@ -58,6 +58,8 @@ struct dw_axi_dma {
 	struct dma_device	dma;
 	struct dw_axi_dma_hcfg	*hdata;
 	struct device_dma_parameters	dma_parms;
+	u32	sel_required[DMAC_MAX_CHANNELS];
+	u32	sel_precluded[DMAC_MAX_CHANNELS];
 
 	/* channels */
 	struct axi_dma_chan	*chan;

drivers/misc/rp1-pio.c

@@ -50,7 +50,7 @@
 #define RP1_PIO_FIFO_RX2	0x18
 #define RP1_PIO_FIFO_RX3	0x1c
 
-#define RP1_PIO_DMACTRL_DEFAULT	0x80000104
+#define RP1_PIO_DMACTRL_DEFAULT	0x80000108
 
 #define HANDLER(_n, _f) \
 	[_IOC_NR(PIO_IOC_ ## _n)] = { #_n, rp1_pio_ ## _f, _IOC_SIZE(PIO_IOC_ ## _n) }
@@ -648,8 +648,10 @@ static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint s
 	chan_name[3] = '\0';
 
 	dma->chan = dma_request_chan(dev, chan_name);
-	if (IS_ERR(dma->chan))
-		return PTR_ERR(dma->chan);
+	if (IS_ERR(dma->chan)) {
+		ret = PTR_ERR(dma->chan);
+		goto err_unclaim;
+	}
 
 	/* Alloc and map bounce buffers */
 	for (dma->buf_count = 0; dma->buf_count < buf_count; dma->buf_count++) {
@@ -674,6 +676,10 @@ static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint s
 	config.src_addr = fifo_addr;
 	config.dst_addr = fifo_addr;
 	config.direction = (dir == RP1_PIO_DIR_TO_SM) ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM;
+	if (dir == RP1_PIO_DIR_TO_SM)
+		config.dst_maxburst = 8;
+	else
+		config.src_maxburst = 8;
 
 	ret = dmaengine_slave_config(dma->chan, &config);
 	if (ret)
@@ -694,6 +700,7 @@ static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint s
 err_dma_free:
 	rp1_pio_sm_dma_free(dev, dma);
 
+err_unclaim:
 	spin_lock(&pio->lock);
 	client->claimed_dmas &= ~dma_mask;
 	pio->claimed_dmas &= ~dma_mask;