From 4c325ef7d17b1d950b7f0fddb29b0ba817837095 Mon Sep 17 00:00:00 2001 From: Phil Elwell Date: Mon, 11 Aug 2025 16:41:10 +0100 Subject: [PATCH 1/6] misc: rp1-pio: Fix a config_xfer error path If the DMA channel allocation fails, the relevant dma_configs entry should be marked as no longer claimed, otherwise rp1_pio_sm_dma_free will be called with an error number as a DMA channel pointer. Signed-off-by: Phil Elwell --- drivers/misc/rp1-pio.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/drivers/misc/rp1-pio.c b/drivers/misc/rp1-pio.c index 91c20ad5d696f9..3a8e5aa8950b90 100644 --- a/drivers/misc/rp1-pio.c +++ b/drivers/misc/rp1-pio.c @@ -648,8 +648,10 @@ static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint s chan_name[3] = '\0'; dma->chan = dma_request_chan(dev, chan_name); - if (IS_ERR(dma->chan)) - return PTR_ERR(dma->chan); + if (IS_ERR(dma->chan)) { + ret = PTR_ERR(dma->chan); + goto err_unclaim; + } /* Alloc and map bounce buffers */ for (dma->buf_count = 0; dma->buf_count < buf_count; dma->buf_count++) { @@ -694,6 +696,7 @@ static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint s err_dma_free: rp1_pio_sm_dma_free(dev, dma); +err_unclaim: spin_lock(&pio->lock); client->claimed_dmas &= ~dma_mask; pio->claimed_dmas &= ~dma_mask; From 29b478e036d85d0facb83093b93a725eeca02e64 Mon Sep 17 00:00:00 2001 From: Phil Elwell Date: Thu, 30 Jan 2025 15:29:31 +0000 Subject: [PATCH 2/6] dmaengine: dw-axi-dmac: Per-channel burst limits The DesignWare AXI DMAC IP can be configured with heterogeneous channel parameters. Allow maximum burst length to be set per-channel by making snps,axi-max-burst-len an array. Signed-off-by: Phil Elwell --- .../dma/dw-axi-dmac/dw-axi-dmac-platform.c | 29 ++++++++++++------- drivers/dma/dw-axi-dmac/dw-axi-dmac.h | 2 +- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c index 179d7dd0b7cbef..3a8a6c57ff8539 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -809,7 +809,7 @@ static int dw_axi_dma_set_hw_desc(struct axi_dma_chan *chan, ctlhi = CH_CTL_H_LLI_VALID; if (chan->chip->dw->hdata->restrict_axi_burst_len) { - burst_len = chan->chip->dw->hdata->axi_rw_burst_len; + burst_len = chan->chip->dw->hdata->axi_rw_burst_len[chan->id]; ctlhi |= CH_CTL_H_ARLEN_EN | CH_CTL_H_AWLEN_EN | burst_len << CH_CTL_H_ARLEN_POS | burst_len << CH_CTL_H_AWLEN_POS; @@ -1087,7 +1087,7 @@ dma_chan_prep_dma_memcpy(struct dma_chan *dchan, dma_addr_t dst_adr, reg = CH_CTL_H_LLI_VALID; if (chan->chip->dw->hdata->restrict_axi_burst_len) { - u32 burst_len = chan->chip->dw->hdata->axi_rw_burst_len; + u32 burst_len = chan->chip->dw->hdata->axi_rw_burst_len[chan->id]; reg |= (CH_CTL_H_ARLEN_EN | burst_len << CH_CTL_H_ARLEN_POS | @@ -1496,6 +1496,7 @@ static int parse_device_properties(struct axi_dma_chip *chip) { struct device *dev = chip->dev; u32 tmp, carr[DMAC_MAX_CHANNELS]; + u32 val; int ret; ret = device_property_read_u32(dev, "dma-channels", &tmp); @@ -1552,15 +1553,21 @@ static int parse_device_properties(struct axi_dma_chip *chip) } /* axi-max-burst-len is optional property */ - ret = device_property_read_u32(dev, "snps,axi-max-burst-len", &tmp); - if (!ret) { - if (tmp > DWAXIDMAC_ARWLEN_MAX + 1) - return -EINVAL; - if (tmp < DWAXIDMAC_ARWLEN_MIN + 1) - return -EINVAL; - + ret = device_property_read_u32_array(dev, "snps,axi-max-burst-len", NULL, + chip->dw->hdata->nr_channels); + if ((ret > 0) && + !device_property_read_u32_array(dev, "snps,axi-max-burst-len", + carr, ret)) { chip->dw->hdata->restrict_axi_burst_len = true; - chip->dw->hdata->axi_rw_burst_len = tmp; + for (tmp = 0; tmp < chip->dw->hdata->nr_channels; tmp++) { + // Replicate the last value to any remaining channels + val = carr[min(tmp, (u32)ret - 1)]; + if (val > DWAXIDMAC_ARWLEN_MAX + 1) + return -EINVAL; + if (val < DWAXIDMAC_ARWLEN_MIN + 1) + return -EINVAL; + chip->dw->hdata->axi_rw_burst_len[tmp] = val; + } } return 0; @@ -1674,7 +1681,7 @@ static int dw_probe(struct platform_device *pdev) dma_cap_set(DMA_CYCLIC, dw->dma.cap_mask); /* DMA capabilities */ - dw->dma.max_burst = hdata->axi_rw_burst_len; + dw->dma.max_burst = hdata->axi_rw_burst_len[0]; dw->dma.src_addr_widths = AXI_DMA_BUSWIDTHS; dw->dma.dst_addr_widths = AXI_DMA_BUSWIDTHS; dw->dma.directions = BIT(DMA_MEM_TO_MEM); diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h index b842e6a8d90d3d..7879f246253957 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h @@ -29,7 +29,7 @@ struct dw_axi_dma_hcfg { u32 block_size[DMAC_MAX_CHANNELS]; u32 priority[DMAC_MAX_CHANNELS]; /* maximum supported axi burst length */ - u32 axi_rw_burst_len; + u32 axi_rw_burst_len[DMAC_MAX_CHANNELS]; /* Register map for DMAX_NUM_CHANNELS <= 8 */ bool reg_map_8_channels; bool restrict_axi_burst_len; From d87d457b141e0d69fafa4cc767cd8f8640ddd206 Mon Sep 17 00:00:00 2001 From: Phil Elwell Date: Mon, 11 Aug 2025 17:15:43 +0100 Subject: [PATCH 3/6] dmaengine: dw-axi-dmac: Add DMA channel selection Add a mechanism to allow clients to prefer some DMA channels over others. This is required to allow high-bandwidth clients to request one of the two "heavy" channels, but could also be used to prevent some clients from hogging all channels. Signed-off-by: Phil Elwell --- .../dma/dw-axi-dmac/dw-axi-dmac-platform.c | 43 ++++++++++++++++++- drivers/dma/dw-axi-dmac/dw-axi-dmac.h | 2 + 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c index 3a8a6c57ff8539..56b2bbcbc51753 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac-platform.c @@ -1476,19 +1476,46 @@ static int __maybe_unused axi_dma_runtime_resume(struct device *dev) return axi_dma_resume(chip); } +static bool dw_axi_dma_filter_fn(struct dma_chan *dchan, void *filter_param) +{ + struct axi_dma_chan *chan = dchan_to_axi_dma_chan(dchan); + uint32_t selector = *(const uint32_t *)filter_param; + + return !!(selector & (1 << chan->id)); +} + static struct dma_chan *dw_axi_dma_of_xlate(struct of_phandle_args *dma_spec, struct of_dma *ofdma) { struct dw_axi_dma *dw = ofdma->of_dma_data; struct axi_dma_chan *chan; struct dma_chan *dchan; + uint32_t chan_mask = 0; + uint32_t chan_sel; + dma_cap_mask_t mask; + int i; + + /* + * Walk through all channels looking for the best match. + * Starting from 0, choose the first available slave channel which isn't precluded. + */ + chan_sel = dma_spec->args[0]; - dchan = dma_get_any_slave_channel(&dw->dma); + for (i = 0; i < dw->hdata->nr_channels; i++) { + if (((dw->sel_precluded[i] & chan_sel) == 0) && + ((dw->sel_required[i] & chan_sel) == dw->sel_required[i])) + chan_mask |= (1 << i); + } + + dma_cap_zero(mask); + dma_cap_set(DMA_SLAVE, mask); + + dchan = __dma_request_channel(&mask, dw_axi_dma_filter_fn, &chan_mask, ofdma->of_node); if (!dchan) return NULL; chan = dchan_to_axi_dma_chan(dchan); - chan->hw_handshake_num = dma_spec->args[0]; + chan->hw_handshake_num = (u8)chan_sel; return dchan; } @@ -1570,6 +1597,18 @@ static int parse_device_properties(struct axi_dma_chip *chip) } } + /* sel-require is optional */ + memset(chip->dw->sel_required, 0, sizeof(chip->dw->sel_required)); + device_property_read_u32_array(dev, "snps,sel-require", + chip->dw->sel_required, + chip->dw->hdata->nr_channels); + + /* sel-preclude is optional */ + memset(chip->dw->sel_precluded, 0, sizeof(chip->dw->sel_precluded)); + device_property_read_u32_array(dev, "snps,sel-preclude", + chip->dw->sel_precluded, + chip->dw->hdata->nr_channels); + return 0; } diff --git a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h index 7879f246253957..5bf8bc890e6cb8 100644 --- a/drivers/dma/dw-axi-dmac/dw-axi-dmac.h +++ b/drivers/dma/dw-axi-dmac/dw-axi-dmac.h @@ -58,6 +58,8 @@ struct dw_axi_dma { struct dma_device dma; struct dw_axi_dma_hcfg *hdata; struct device_dma_parameters dma_parms; + u32 sel_required[DMAC_MAX_CHANNELS]; + u32 sel_precluded[DMAC_MAX_CHANNELS]; /* channels */ struct axi_dma_chan *chan; From 865d564424fd758ae2093b059246b111861e114f Mon Sep 17 00:00:00 2001 From: Phil Elwell Date: Tue, 12 Aug 2025 09:18:24 +0100 Subject: [PATCH 4/6] dts: rp1: Set per-channel DMA burst limit Give the DMAC property "snps,axi-max-burst-len" a value for each DMA channel, encoding the fact that channels 1 and 2 are more capable ("heavy"). Signed-off-by: Phil Elwell --- arch/arm64/boot/dts/broadcom/rp1.dtsi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm64/boot/dts/broadcom/rp1.dtsi b/arch/arm64/boot/dts/broadcom/rp1.dtsi index b746de80010863..6a29960db57665 100644 --- a/arch/arm64/boot/dts/broadcom/rp1.dtsi +++ b/arch/arm64/boot/dts/broadcom/rp1.dtsi @@ -1111,7 +1111,7 @@ snps,data-width = <4>; // (8 << 4) == 128 bits snps,block-size = <0x40000 0x40000 0x40000 0x40000 0x40000 0x40000 0x40000 0x40000>; snps,priority = <0 1 2 3 4 5 6 7>; - snps,axi-max-burst-len = <4>; + snps,axi-max-burst-len = <8 8 4 4 4 4 4 4>; status = "disabled"; }; From c9d3b29757a5c33567787262eb6ac0be5fdbe3cf Mon Sep 17 00:00:00 2001 From: Phil Elwell Date: Tue, 12 Aug 2025 09:38:08 +0100 Subject: [PATCH 5/6] dts: bcm2712-rpi: Give PIO the "heavy" DMA channels PIO benefits from increased DMA bandwidth when used with DMA channels 0 or 1, because they support longer bursts. Add DMA channel selection attributes to prevent other users from claiming them. Signed-off-by: Phil Elwell --- arch/arm64/boot/dts/broadcom/bcm2712-rpi.dtsi | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/arch/arm64/boot/dts/broadcom/bcm2712-rpi.dtsi b/arch/arm64/boot/dts/broadcom/bcm2712-rpi.dtsi index 2f4d359a69a640..3ea93f05248c32 100644 --- a/arch/arm64/boot/dts/broadcom/bcm2712-rpi.dtsi +++ b/arch/arm64/boot/dts/broadcom/bcm2712-rpi.dtsi @@ -2,6 +2,9 @@ #include +#define DMA_SEL_WANTHEAVY (1 << 8) +#define DMA_SEL_ONLYHEAVY (1 << 9) + &soc { firmware: firmware { compatible = "raspberrypi,bcm2835-firmware", "simple-mfd"; @@ -91,7 +94,21 @@ }; }; +&rp1_dma { + snps,sel-require = ; + snps,sel-preclude = <0 0 DMA_SEL_ONLYHEAVY DMA_SEL_ONLYHEAVY DMA_SEL_ONLYHEAVY + DMA_SEL_ONLYHEAVY DMA_SEL_ONLYHEAVY DMA_SEL_ONLYHEAVY>; +}; + pio: &rp1_pio { + dmas = <&rp1_dma (RP1_DMA_PIO_CH0_TX | DMA_SEL_WANTHEAVY)>, + <&rp1_dma (RP1_DMA_PIO_CH0_RX | DMA_SEL_WANTHEAVY)>, + <&rp1_dma (RP1_DMA_PIO_CH1_TX | DMA_SEL_WANTHEAVY)>, + <&rp1_dma (RP1_DMA_PIO_CH1_RX | DMA_SEL_WANTHEAVY)>, + <&rp1_dma (RP1_DMA_PIO_CH2_TX | DMA_SEL_WANTHEAVY)>, + <&rp1_dma (RP1_DMA_PIO_CH2_RX | DMA_SEL_WANTHEAVY)>, + <&rp1_dma (RP1_DMA_PIO_CH3_TX | DMA_SEL_WANTHEAVY)>, + <&rp1_dma (RP1_DMA_PIO_CH3_RX | DMA_SEL_WANTHEAVY)>; status = "okay"; }; From a3b5a5469d07e597d743ca2fa089c79f51bd8222 Mon Sep 17 00:00:00 2001 From: Phil Elwell Date: Tue, 12 Aug 2025 11:56:55 +0100 Subject: [PATCH 6/6] misc: rp1-pio: Request a DMA burst size of 8 Improve DMA performance by increasing the burst size to 8. Signed-off-by: Phil Elwell --- drivers/misc/rp1-pio.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/drivers/misc/rp1-pio.c b/drivers/misc/rp1-pio.c index 3a8e5aa8950b90..e3ca8e451a57e5 100644 --- a/drivers/misc/rp1-pio.c +++ b/drivers/misc/rp1-pio.c @@ -50,7 +50,7 @@ #define RP1_PIO_FIFO_RX2 0x18 #define RP1_PIO_FIFO_RX3 0x1c -#define RP1_PIO_DMACTRL_DEFAULT 0x80000104 +#define RP1_PIO_DMACTRL_DEFAULT 0x80000108 #define HANDLER(_n, _f) \ [_IOC_NR(PIO_IOC_ ## _n)] = { #_n, rp1_pio_ ## _f, _IOC_SIZE(PIO_IOC_ ## _n) } @@ -676,6 +676,10 @@ static int rp1_pio_sm_config_xfer_internal(struct rp1_pio_client *client, uint s config.src_addr = fifo_addr; config.dst_addr = fifo_addr; config.direction = (dir == RP1_PIO_DIR_TO_SM) ? DMA_MEM_TO_DEV : DMA_DEV_TO_MEM; + if (dir == RP1_PIO_DIR_TO_SM) + config.dst_maxburst = 8; + else + config.src_maxburst = 8; ret = dmaengine_slave_config(dma->chan, &config); if (ret)