RK3588: NPU: Update to 0.9.6 version

Author: yml

arch/arm64/boot/dts/rockchip/overlay/rk3588-fixup.scr

@@ -10,6 +10,7 @@
 #include <linux/completion.h>
 #include <linux/device.h>
 #include <linux/kref.h>
+#include <linux/irq.h>
 #include <linux/platform_device.h>
 #include <linux/spinlock.h>
 #include <linux/regulator/consumer.h>
@@ -28,10 +29,10 @@
 
 #define DRIVER_NAME "rknpu"
 #define DRIVER_DESC "RKNPU driver"
-#define DRIVER_DATE "20231121"
+#define DRIVER_DATE "20240322"
 #define DRIVER_MAJOR 0
 #define DRIVER_MINOR 9
-#define DRIVER_PATCHLEVEL 3
+#define DRIVER_PATCHLEVEL 6
 
 #define LOG_TAG "RKNPU"
 
@@ -52,9 +53,18 @@
 #define LOG_DEV_DEBUG(dev, fmt, args...) dev_dbg(dev, LOG_TAG ": " fmt, ##args)
 #define LOG_DEV_ERROR(dev, fmt, args...) dev_err(dev, LOG_TAG ": " fmt, ##args)
 
-struct rknpu_reset_data {
-	const char *srst_a_name;
-	const char *srst_h_name;
+#define RKNPU_MAX_IOMMU_DOMAIN_NUM 16
+
+struct rknpu_irqs_data {
+	const char *name;
+	irqreturn_t (*irq_hdl)(int irq, void *ctx);
+};
+
+struct rknpu_amount_data {
+	uint16_t offset_clr_all;
+	uint16_t offset_dt_wr;
+	uint16_t offset_dt_rd;
+	uint16_t offset_wt_rd;
 };
 
 struct rknpu_config {
@@ -66,15 +76,14 @@ struct rknpu_config {
 	__u32 pc_task_number_mask;
 	__u32 pc_task_status_offset;
 	__u32 pc_dma_ctrl;
-	__u32 bw_enable;
 	const struct rknpu_irqs_data *irqs;
-	const struct rknpu_reset_data *resets;
 	int num_irqs;
-	int num_resets;
 	__u64 nbuf_phyaddr;
 	__u64 nbuf_size;
 	__u64 max_submit_number;
 	__u32 core_mask;
+	const struct rknpu_amount_data *amount_top;
+	const struct rknpu_amount_data *amount_core;
 };
 
 struct rknpu_timer {
@@ -113,13 +122,14 @@ struct rknpu_device {
 	spinlock_t irq_lock;
 	struct mutex power_lock;
 	struct mutex reset_lock;
+	struct mutex domain_lock;
 	struct rknpu_subcore_data subcore_datas[RKNPU_MAX_CORES];
 	const struct rknpu_config *config;
 	void __iomem *bw_priority_base;
 	struct rknpu_fence_context *fence_ctx;
 	bool iommu_en;
-	struct reset_control *srst_a[RKNPU_MAX_CORES];
-	struct reset_control *srst_h[RKNPU_MAX_CORES];
+	struct reset_control **srsts;
+	int num_srsts;
 	struct clk_bulk_data *clks;
 	int num_clks;
 	struct regulator *vdd;
@@ -156,6 +166,10 @@ struct rknpu_device {
 	void __iomem *nbuf_base_io;
 	struct rknpu_mm *sram_mm;
 	unsigned long power_put_delay;
+	struct iommu_group *iommu_group;
+	int iommu_domain_num;
+	int iommu_domain_id;
+	struct iommu_domain *iommu_domains[RKNPU_MAX_IOMMU_DOMAIN_NUM];
 };
 
 struct rknpu_session {

drivers/rknpu/include/rknpu_drv.h

@@ -60,6 +60,7 @@ struct rknpu_gem_object {
 	struct page **pages;
 	struct sg_table *sgt;
 	struct drm_mm_node mm_node;
+	int iommu_domain_id;
 };
 
 enum rknpu_cache_type {
@@ -71,7 +72,8 @@ enum rknpu_cache_type {
 struct rknpu_gem_object *rknpu_gem_object_create(struct drm_device *dev,
 						 unsigned int flags,
 						 unsigned long size,
-						 unsigned long sram_size);
+						 unsigned long sram_size,
+						 int iommu_domain_id);
 
 /* destroy a buffer with gem object */
 void rknpu_gem_object_destroy(struct rknpu_gem_object *rknpu_obj);

drivers/rknpu/include/rknpu_gem.h

@@ -31,11 +31,6 @@
 #define RKNPU_OFFSET_INT_STATUS 0x28
 #define RKNPU_OFFSET_INT_RAW_STATUS 0x2c
 
-#define RKNPU_OFFSET_CLR_ALL_RW_AMOUNT 0x8010
-#define RKNPU_OFFSET_DT_WR_AMOUNT 0x8034
-#define RKNPU_OFFSET_DT_RD_AMOUNT 0x8038
-#define RKNPU_OFFSET_WT_RD_AMOUNT 0x803c
-
 #define RKNPU_OFFSET_ENABLE_MASK 0xf008
 
 #define RKNPU_INT_CLEAR 0x1ffff
@@ -134,6 +129,8 @@ enum e_rknpu_action {
 	RKNPU_POWER_OFF = 21,
 	RKNPU_GET_TOTAL_SRAM_SIZE = 22,
 	RKNPU_GET_FREE_SRAM_SIZE = 23,
+	RKNPU_GET_IOMMU_DOMAIN_ID = 24,
+	RKNPU_SET_IOMMU_DOMAIN_ID = 25,
 };
 
 /**
@@ -147,6 +144,8 @@ enum e_rknpu_action {
  * @dma_addr: dma address that access by rknpu.
  * @sram_size: user-desired sram memory allocation size.
  *  - this size value would be page-aligned internally.
+ * @iommu_domain_id: iommu domain id
+ * @reserved: just padding to be 64-bit aligned.
  */
 struct rknpu_mem_create {
 	__u32 handle;
@@ -155,6 +154,8 @@ struct rknpu_mem_create {
 	__u64 obj_addr;
 	__u64 dma_addr;
 	__u64 sram_size;
+	__s32 iommu_domain_id;
+	__u32 reserved;
 };
 
 /**
@@ -249,7 +250,8 @@ struct rknpu_subcore_task {
  * @task_counter: task counter
  * @priority: submit priority
  * @task_obj_addr: address of task object
- * @regcfg_obj_addr: address of register config object
+ * @iommu_domain_id: iommu domain id
+ * @reserved: just padding to be 64-bit aligned.
  * @task_base_addr: task base address
  * @hw_elapse_time: hardware elapse time
  * @core_mask: core mask of rknpu
@@ -265,7 +267,8 @@ struct rknpu_submit {
 	__u32 task_counter;
 	__s32 priority;
 	__u64 task_obj_addr;
-	__u64 regcfg_obj_addr;
+	__u32 iommu_domain_id;
+	__u32 reserved;
 	__u64 task_base_addr;
 	__s64 hw_elapse_time;
 	__u32 core_mask;

drivers/rknpu/include/rknpu_ioctl.h

@@ -37,4 +37,12 @@ dma_addr_t rknpu_iommu_dma_alloc_iova(struct iommu_domain *domain, size_t size,
 void rknpu_iommu_dma_free_iova(struct rknpu_iommu_dma_cookie *cookie,
 			       dma_addr_t iova, size_t size);
 
+int rknpu_iommu_init_domain(struct rknpu_device *rknpu_dev);
+int rknpu_iommu_switch_domain(struct rknpu_device *rknpu_dev, int domain_id);
+void rknpu_iommu_free_domains(struct rknpu_device *rknpu_dev);
+
+#if KERNEL_VERSION(5, 10, 0) < LINUX_VERSION_CODE
+int iommu_get_dma_cookie(struct iommu_domain *domain);
+#endif
+
 #endif

drivers/rknpu/include/rknpu_iommu.h

@@ -48,6 +48,7 @@ struct rknpu_job {
 	ktime_t hw_recoder_time;
 	ktime_t hw_elapse_time;
 	atomic_t submit_count[RKNPU_MAX_CORES];
+	int iommu_domain_id;
 };
 
 irqreturn_t rknpu_core0_irq_handler(int irq, void *data);

drivers/rknpu/include/rknpu_job.h

@@ -37,10 +37,10 @@ struct rknpu_mem_object {
 	unsigned int owner;
 };
 
-int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, unsigned long data,
-			   struct file *file);
-int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, unsigned long data,
-			    struct file *file);
+int rknpu_mem_create_ioctl(struct rknpu_device *rknpu_dev, struct file *file,
+			   unsigned int cmd, unsigned long data);
+int rknpu_mem_destroy_ioctl(struct rknpu_device *rknpu_dev, struct file *file,
+			    unsigned long data);
 int rknpu_mem_sync_ioctl(struct rknpu_device *rknpu_dev, unsigned long data);
 
 #endif

drivers/rknpu/include/rknpu_mem.h

@@ -75,6 +75,25 @@ static struct devfreq_governor devfreq_rknpu_ondemand = {
 	.event_handler = devfreq_rknpu_ondemand_handler,
 };
 
+static int rk3576_npu_set_read_margin(struct device *dev,
+				      struct rockchip_opp_info *opp_info,
+				      u32 rm)
+{
+	if (!opp_info->grf || !opp_info->volt_rm_tbl)
+		return 0;
+
+	if (rm == opp_info->current_rm || rm == UINT_MAX)
+		return 0;
+
+	LOG_DEV_DEBUG(dev, "set rm to %d\n", rm);
+
+	regmap_write(opp_info->grf, 0x08, 0x001c0000 | (rm << 2));
+	regmap_write(opp_info->grf, 0x0c, 0x003c0000 | (rm << 2));
+	regmap_write(opp_info->grf, 0x10, 0x001c0000 | (rm << 2));
+
+	return 0;
+}
+
 static int rk3588_npu_get_soc_info(struct device *dev, struct device_node *np,
 				   int *bin, int *process)
 {
@@ -162,7 +181,6 @@ static int rk3588_npu_set_read_margin(struct device *dev,
 				      struct rockchip_opp_info *opp_info,
 				      u32 rm)
 {
-	struct rknpu_device *rknpu_dev = dev_get_drvdata(dev);
 	u32 offset = 0, val = 0;
 	int i, ret = 0;
 
@@ -174,7 +192,7 @@ static int rk3588_npu_set_read_margin(struct device *dev,
 
 	LOG_DEV_DEBUG(dev, "set rm to %d\n", rm);
 
-	for (i = 0; i < rknpu_dev->config->num_irqs; i++) {
+	for (i = 0; i < 3; i++) {
 		ret = regmap_read(opp_info->grf, offset, &val);
 		if (ret < 0) {
 			LOG_DEV_ERROR(dev, "failed to get rm from 0x%x\n",
@@ -214,6 +232,14 @@ static int npu_opp_config_clks(struct device *dev, struct opp_table *opp_table,
 }
 #endif
 
+static const struct rockchip_opp_data rk3576_npu_opp_data = {
+	.set_read_margin = rk3576_npu_set_read_margin,
+#if KERNEL_VERSION(6, 1, 0) <= LINUX_VERSION_CODE
+	.config_regulators = npu_opp_config_regulators,
+	.config_clks = npu_opp_config_clks,
+#endif
+};
+
 static const struct rockchip_opp_data rk3588_npu_opp_data = {
 	.get_soc_info = rk3588_npu_get_soc_info,
 	.set_soc_info = rk3588_npu_set_soc_info,
@@ -225,6 +251,10 @@ static const struct rockchip_opp_data rk3588_npu_opp_data = {
 };
 
 static const struct of_device_id rockchip_npu_of_match[] = {
+	{
+		.compatible = "rockchip,rk3576",
+		.data = (void *)&rk3576_npu_opp_data,
+	},
 	{
 		.compatible = "rockchip,rk3588",
 		.data = (void *)&rk3588_npu_opp_data,