spi: Add RTIO support to SPI

Provides a macro and submit API for SPI drivers to support RTIO.

A copy function enables compatibility with the existing blocking API
and very easily the existing async API as well.

Signed-off-by: Tom Burdick <[email protected]>

Author: teburd

drivers/spi/CMakeLists.txt

@@ -37,5 +37,6 @@ zephyr_library_sources_ifdef(CONFIG_NXP_S32_SPI spi_nxp_s32.c)
 zephyr_library_sources_ifdef(CONFIG_SPI_XMC4XXX     spi_xmc4xxx.c)
 zephyr_library_sources_ifdef(CONFIG_SPI_PW		spi_pw.c)
 
+zephyr_library_sources_ifdef(CONFIG_SPI_RTIO spi_rtio.c)
 zephyr_library_sources_ifdef(CONFIG_SPI_ASYNC spi_signal.c)
 zephyr_library_sources_ifdef(CONFIG_USERSPACE		spi_handlers.c)

drivers/spi/Kconfig

@@ -19,6 +19,14 @@ config SPI_ASYNC
 	help
 	  This option enables the asynchronous API calls.
 
+config SPI_RTIO
+	bool "RTIO support [EXPERIMENTAL]"
+	select EXPERIMENTAL
+	select RTIO
+	help
+	  This option enables the RTIO API calls. RTIO support is
+	  experimental as the API itself is unstable.
+
 config SPI_SLAVE
 	bool "Slave support [EXPERIMENTAL]"
 	select EXPERIMENTAL

drivers/spi/spi_rtio.c

@@ -0,0 +1,11 @@
+/*
+ * Copyright (c) 2023 Intel Corporation
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+#include <zephyr/drivers/spi.h>
+
+const struct rtio_iodev_api spi_iodev_api = {
+	.submit = spi_iodev_submit,
+};

include/zephyr/drivers/spi.h

@@ -25,6 +25,8 @@
 #include <zephyr/dt-bindings/spi/spi.h>
 #include <zephyr/drivers/gpio.h>
 #include <zephyr/kernel.h>
+#include <zephyr/sys/__assert.h>
+#include <zephyr/rtio/rtio.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -480,6 +482,16 @@ typedef int (*spi_api_io_async)(const struct device *dev,
 				spi_callback_t cb,
 				void *userdata);
 
+#if defined(CONFIG_SPI_RTIO) || defined(DOXYGEN)
+
+/**
+ * @typedef spi_api_iodev_submit
+ * @brief Callback API for submitting work to a SPI device with RTIO
+ */
+typedef void (*spi_api_iodev_submit)(const struct device *dev,
+				     struct rtio_iodev_sqe *iodev_sqe);
+#endif /* CONFIG_SPI_RTIO */
+
 /**
  * @typedef spi_api_release
  * @brief Callback API for unlocking SPI device.
@@ -498,6 +510,9 @@ __subsystem struct spi_driver_api {
 #ifdef CONFIG_SPI_ASYNC
 	spi_api_io_async transceive_async;
 #endif /* CONFIG_SPI_ASYNC */
+#ifdef CONFIG_SPI_RTIO
+	spi_api_iodev_submit iodev_submit;
+#endif /* CONFIG_SPI_RTIO */
 	spi_api_release release;
 };
 
@@ -545,6 +560,7 @@ static inline bool spi_is_ready_dt(const struct spi_dt_spec *spec)
 	}
 	return true;
 }
+
 /**
  * @brief Read/write the specified amount of data from the SPI driver.
  *
@@ -878,6 +894,209 @@ __deprecated static inline int spi_write_async(const struct device *dev,
 
 #endif /* CONFIG_SPI_ASYNC */
 
+
+#if defined(CONFIG_SPI_RTIO) || defined(DOXYGEN)
+
+/**
+ * @brief Submit a SPI device with a request
+ *
+ * @param dev SPI device
+ * @param iodev_sqe Prepared submissions queue entry connected to an iodev
+ *                  defined by SPI_IODEV_DEFINE.
+ *                  Must live as long as the request is in flight.
+ *
+ * @retval 0 If successful.
+ * @retval -errno Negative errno code on failure.
+ */
+static inline void spi_iodev_submit(struct rtio_iodev_sqe *iodev_sqe)
+{
+	const struct spi_dt_spec *dt_spec = iodev_sqe->sqe->iodev->data;
+	const struct device *dev = dt_spec->bus;
+	const struct spi_driver_api *api = (const struct spi_driver_api *)dev->api;
+
+	api->iodev_submit(dt_spec->bus, iodev_sqe);
+}
+
+extern const struct rtio_iodev_api spi_iodev_api;
+
+/**
+ * @brief Define an iodev for a given dt node on the bus
+ *
+ * These do not need to be shared globally but doing so
+ * will save a small amount of memory.
+ *
+ * @param node DT_NODE
+ */
+#define SPI_DT_IODEV_DEFINE(name, node_id, operation_, delay_)			\
+	const struct spi_dt_spec _spi_dt_spec_##name =				\
+		SPI_DT_SPEC_GET(node_id, operation_, delay_);			\
+	RTIO_IODEV_DEFINE(name, &spi_iodev_api, (void *)&_spi_dt_spec_##name)
+
+/**
+ * @brief Validate that SPI bus (and CS gpio if defined) is ready.
+ *
+ * @param spi_iodev SPI iodev defined with SPI_DT_IODEV_DEFINE
+ *
+ * @retval true if the SPI bus is ready for use.
+ * @retval false if the SPI bus (or the CS gpio defined) is not ready for use.
+ */
+static inline bool spi_is_ready_iodev(const struct rtio_iodev *spi_iodev)
+{
+	struct spi_dt_spec *spec = spi_iodev->data;
+
+	return spi_is_ready_dt(spec);
+}
+
+/**
+ * @brief Copy the tx_bufs and rx_bufs into a set of RTIO requests
+ *
+ * @param r RTIO context
+ * @param tx_bufs Transmit buffer set
+ * @param rx_bufs Receive buffer set
+ *
+ * @retval sqe Last submission in the queue added
+ * @retval NULL Not enough memory in the context to copy the requests
+ */
+static inline struct rtio_sqe *spi_rtio_copy(struct rtio *r,
+					     struct rtio_iodev *iodev,
+					     const struct spi_buf_set *tx_bufs,
+					     const struct spi_buf_set *rx_bufs)
+{
+	struct rtio_sqe *sqe = NULL;
+	size_t tx_count = tx_bufs ? tx_bufs->count : 0;
+	size_t rx_count = rx_bufs ? rx_bufs->count : 0;
+
+	uint32_t tx = 0, tx_len = 0;
+	uint32_t rx = 0, rx_len = 0;
+	uint8_t *tx_buf, *rx_buf;
+
+	if (tx < tx_count) {
+		tx_buf = tx_bufs->buffers[tx].buf;
+		tx_len = tx_bufs->buffers[tx].len;
+	} else {
+		tx_buf = NULL;
+		tx_len = rx_bufs->buffers[rx].len;
+	}
+
+	if (rx < rx_count) {
+		rx_buf = rx_bufs->buffers[rx].buf;
+		rx_len = rx_bufs->buffers[rx].len;
+	} else {
+		rx_buf = NULL;
+		rx_len = tx_bufs->buffers[tx].len;
+	}
+
+
+	while ((tx < tx_count || rx < rx_count) && (tx_len > 0 || rx_len > 0)) {
+		sqe = rtio_sqe_acquire(r);
+
+		if (sqe == NULL) {
+			rtio_spsc_drop_all(r->sq);
+			return NULL;
+		}
+
+		/* If tx/rx len are same, we can do a simple transceive */
+		if (tx_len == rx_len) {
+			if (tx_buf == NULL) {
+				rtio_sqe_prep_read(sqe, iodev, RTIO_PRIO_NORM,
+						   rx_buf, rx_len, NULL);
+			} else if (rx_buf == NULL) {
+				rtio_sqe_prep_write(sqe, iodev, RTIO_PRIO_NORM,
+						    tx_buf, tx_len, NULL);
+			} else {
+				rtio_sqe_prep_transceive(sqe, iodev, RTIO_PRIO_NORM,
+							 tx_buf, rx_buf, rx_len, NULL);
+			}
+			tx++;
+			rx++;
+			if (rx < rx_count) {
+				rx_buf = rx_bufs->buffers[rx].buf;
+				rx_len = rx_bufs->buffers[rx].len;
+			} else {
+				rx_buf = NULL;
+				rx_len = 0;
+			}
+			if (tx < tx_count) {
+				tx_buf = tx_bufs->buffers[tx].buf;
+				tx_len = tx_bufs->buffers[tx].len;
+			} else {
+				tx_buf = NULL;
+				tx_len = 0;
+			}
+		} else if (tx_len == 0) {
+			rtio_sqe_prep_read(sqe, iodev, RTIO_PRIO_NORM,
+					   (uint8_t *)rx_buf,
+					   (uint32_t)rx_len,
+					   NULL);
+			rx++;
+			if (rx < rx_count) {
+				rx_buf = rx_bufs->buffers[rx].buf;
+				rx_len = rx_bufs->buffers[rx].len;
+			} else {
+				rx_buf = NULL;
+				rx_len = 0;
+			}
+		} else if (rx_len == 0) {
+			rtio_sqe_prep_write(sqe, iodev, RTIO_PRIO_NORM,
+					    (uint8_t *)tx_buf,
+					    (uint32_t)tx_len,
+					    NULL);
+			tx++;
+			if (tx < tx_count) {
+				tx_buf = rx_bufs->buffers[rx].buf;
+				tx_len = rx_bufs->buffers[rx].len;
+			} else {
+				tx_buf = NULL;
+				tx_len = 0;
+			}
+		} else if (tx_len > rx_len) {
+			rtio_sqe_prep_transceive(sqe, iodev, RTIO_PRIO_NORM,
+						 (uint8_t *)tx_buf,
+						 (uint8_t *)rx_buf,
+						 (uint32_t)rx_len,
+						 NULL);
+			tx_len -= rx_len;
+			tx_buf += rx_len;
+			rx++;
+			if (rx < rx_count) {
+				rx_buf = rx_bufs->buffers[rx].buf;
+				rx_len = rx_bufs->buffers[rx].len;
+			} else {
+				rx_buf = NULL;
+				rx_len = tx_len;
+			}
+		} else if (rx_len > tx_len) {
+			rtio_sqe_prep_transceive(sqe, iodev, RTIO_PRIO_NORM,
+						 (uint8_t *)tx_buf,
+						 (uint8_t *)rx_buf,
+						 (uint32_t)tx_len,
+						 NULL);
+			rx_len -= tx_len;
+			rx_buf += tx_len;
+			tx++;
+			if (tx < tx_count) {
+				tx_buf = tx_bufs->buffers[tx].buf;
+				tx_len = tx_bufs->buffers[tx].len;
+			} else {
+				tx_buf = NULL;
+				tx_len = rx_len;
+			}
+		} else {
+			__ASSERT_NO_MSG("Invalid spi_rtio_copy state");
+		}
+
+		sqe->flags = RTIO_SQE_TRANSACTION;
+	}
+
+	if (sqe != NULL) {
+		sqe->flags = 0;
+	}
+
+	return sqe;
+}
+
+#endif /* CONFIG_SPI_RTIO */
+
 /**
  * @brief Release the SPI device locked on and/or the CS by the current config
  *