Skip to content

Commit 07838f7

Browse files
LuBaolujgunthorpe
LuBaolu
authored and
jgunthorpe
committed
iommufd: Add iommufd fault object
An iommufd fault object provides an interface for delivering I/O page faults to user space. These objects are created and destroyed by user space, and they can be associated with or dissociated from hardware page table objects during page table allocation or destruction. User space interacts with the fault object through a file interface. This interface offers a straightforward and efficient way for user space to handle page faults. It allows user space to read fault messages sequentially and respond to them by writing to the same file. The file interface supports reading messages in poll mode, so it's recommended that user space applications use io_uring to enhance read and write efficiency. A fault object can be associated with any iopf-capable iommufd_hw_pgtable during the pgtable's allocation. All I/O page faults triggered by devices when accessing the I/O addresses of an iommufd_hw_pgtable are routed through the fault object to user space. Similarly, user space's responses to these page faults are routed back to the iommu device driver through the same fault object. Link: https://lore.kernel.org/r/[email protected] Signed-off-by: Lu Baolu <[email protected]> Reviewed-by: Jason Gunthorpe <[email protected]> Reviewed-by: Kevin Tian <[email protected]> Signed-off-by: Jason Gunthorpe <[email protected]>
1 parent c714f15 commit 07838f7

File tree

7 files changed

+287
-0
lines changed

7 files changed

+287
-0
lines changed

‎drivers/iommu/io-pgfault.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,6 +110,8 @@ static struct iopf_group *iopf_group_alloc(struct iommu_fault_param *iopf_param,
110110
list_add(&group->pending_node, &iopf_param->faults);
111111
mutex_unlock(&iopf_param->lock);
112112

113+
group->fault_count = list_count_nodes(&group->faults);
114+
113115
return group;
114116
}
115117

‎drivers/iommu/iommufd/Makefile

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
# SPDX-License-Identifier: GPL-2.0-only
22
iommufd-y := \
33
device.o \
4+
fault.o \
45
hw_pagetable.o \
56
io_pagetable.o \
67
ioas.o \

‎drivers/iommu/iommufd/fault.c

Lines changed: 226 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,226 @@
1+
// SPDX-License-Identifier: GPL-2.0-only
2+
/* Copyright (C) 2024 Intel Corporation
3+
*/
4+
#define pr_fmt(fmt) "iommufd: " fmt
5+
6+
#include <linux/file.h>
7+
#include <linux/fs.h>
8+
#include <linux/module.h>
9+
#include <linux/mutex.h>
10+
#include <linux/iommufd.h>
11+
#include <linux/poll.h>
12+
#include <linux/anon_inodes.h>
13+
#include <uapi/linux/iommufd.h>
14+
15+
#include "../iommu-priv.h"
16+
#include "iommufd_private.h"
17+
18+
void iommufd_fault_destroy(struct iommufd_object *obj)
19+
{
20+
struct iommufd_fault *fault = container_of(obj, struct iommufd_fault, obj);
21+
struct iopf_group *group, *next;
22+
23+
/*
24+
* The iommufd object's reference count is zero at this point.
25+
* We can be confident that no other threads are currently
26+
* accessing this pointer. Therefore, acquiring the mutex here
27+
* is unnecessary.
28+
*/
29+
list_for_each_entry_safe(group, next, &fault->deliver, node) {
30+
list_del(&group->node);
31+
iopf_group_response(group, IOMMU_PAGE_RESP_INVALID);
32+
iopf_free_group(group);
33+
}
34+
}
35+
36+
static void iommufd_compose_fault_message(struct iommu_fault *fault,
37+
struct iommu_hwpt_pgfault *hwpt_fault,
38+
struct iommufd_device *idev,
39+
u32 cookie)
40+
{
41+
hwpt_fault->flags = fault->prm.flags;
42+
hwpt_fault->dev_id = idev->obj.id;
43+
hwpt_fault->pasid = fault->prm.pasid;
44+
hwpt_fault->grpid = fault->prm.grpid;
45+
hwpt_fault->perm = fault->prm.perm;
46+
hwpt_fault->addr = fault->prm.addr;
47+
hwpt_fault->length = 0;
48+
hwpt_fault->cookie = cookie;
49+
}
50+
51+
static ssize_t iommufd_fault_fops_read(struct file *filep, char __user *buf,
52+
size_t count, loff_t *ppos)
53+
{
54+
size_t fault_size = sizeof(struct iommu_hwpt_pgfault);
55+
struct iommufd_fault *fault = filep->private_data;
56+
struct iommu_hwpt_pgfault data;
57+
struct iommufd_device *idev;
58+
struct iopf_group *group;
59+
struct iopf_fault *iopf;
60+
size_t done = 0;
61+
int rc = 0;
62+
63+
if (*ppos || count % fault_size)
64+
return -ESPIPE;
65+
66+
mutex_lock(&fault->mutex);
67+
while (!list_empty(&fault->deliver) && count > done) {
68+
group = list_first_entry(&fault->deliver,
69+
struct iopf_group, node);
70+
71+
if (group->fault_count * fault_size > count - done)
72+
break;
73+
74+
rc = xa_alloc(&fault->response, &group->cookie, group,
75+
xa_limit_32b, GFP_KERNEL);
76+
if (rc)
77+
break;
78+
79+
idev = to_iommufd_handle(group->attach_handle)->idev;
80+
list_for_each_entry(iopf, &group->faults, list) {
81+
iommufd_compose_fault_message(&iopf->fault,
82+
&data, idev,
83+
group->cookie);
84+
if (copy_to_user(buf + done, &data, fault_size)) {
85+
xa_erase(&fault->response, group->cookie);
86+
rc = -EFAULT;
87+
break;
88+
}
89+
done += fault_size;
90+
}
91+
92+
list_del(&group->node);
93+
}
94+
mutex_unlock(&fault->mutex);
95+
96+
return done == 0 ? rc : done;
97+
}
98+
99+
static ssize_t iommufd_fault_fops_write(struct file *filep, const char __user *buf,
100+
size_t count, loff_t *ppos)
101+
{
102+
size_t response_size = sizeof(struct iommu_hwpt_page_response);
103+
struct iommufd_fault *fault = filep->private_data;
104+
struct iommu_hwpt_page_response response;
105+
struct iopf_group *group;
106+
size_t done = 0;
107+
int rc = 0;
108+
109+
if (*ppos || count % response_size)
110+
return -ESPIPE;
111+
112+
mutex_lock(&fault->mutex);
113+
while (count > done) {
114+
rc = copy_from_user(&response, buf + done, response_size);
115+
if (rc)
116+
break;
117+
118+
group = xa_erase(&fault->response, response.cookie);
119+
if (!group) {
120+
rc = -EINVAL;
121+
break;
122+
}
123+
124+
iopf_group_response(group, response.code);
125+
iopf_free_group(group);
126+
done += response_size;
127+
}
128+
mutex_unlock(&fault->mutex);
129+
130+
return done == 0 ? rc : done;
131+
}
132+
133+
static __poll_t iommufd_fault_fops_poll(struct file *filep,
134+
struct poll_table_struct *wait)
135+
{
136+
struct iommufd_fault *fault = filep->private_data;
137+
__poll_t pollflags = EPOLLOUT;
138+
139+
poll_wait(filep, &fault->wait_queue, wait);
140+
mutex_lock(&fault->mutex);
141+
if (!list_empty(&fault->deliver))
142+
pollflags |= EPOLLIN | EPOLLRDNORM;
143+
mutex_unlock(&fault->mutex);
144+
145+
return pollflags;
146+
}
147+
148+
static int iommufd_fault_fops_release(struct inode *inode, struct file *filep)
149+
{
150+
struct iommufd_fault *fault = filep->private_data;
151+
152+
refcount_dec(&fault->obj.users);
153+
iommufd_ctx_put(fault->ictx);
154+
return 0;
155+
}
156+
157+
static const struct file_operations iommufd_fault_fops = {
158+
.owner = THIS_MODULE,
159+
.open = nonseekable_open,
160+
.read = iommufd_fault_fops_read,
161+
.write = iommufd_fault_fops_write,
162+
.poll = iommufd_fault_fops_poll,
163+
.release = iommufd_fault_fops_release,
164+
.llseek = no_llseek,
165+
};
166+
167+
int iommufd_fault_alloc(struct iommufd_ucmd *ucmd)
168+
{
169+
struct iommu_fault_alloc *cmd = ucmd->cmd;
170+
struct iommufd_fault *fault;
171+
struct file *filep;
172+
int fdno;
173+
int rc;
174+
175+
if (cmd->flags)
176+
return -EOPNOTSUPP;
177+
178+
fault = iommufd_object_alloc(ucmd->ictx, fault, IOMMUFD_OBJ_FAULT);
179+
if (IS_ERR(fault))
180+
return PTR_ERR(fault);
181+
182+
fault->ictx = ucmd->ictx;
183+
INIT_LIST_HEAD(&fault->deliver);
184+
xa_init_flags(&fault->response, XA_FLAGS_ALLOC1);
185+
mutex_init(&fault->mutex);
186+
init_waitqueue_head(&fault->wait_queue);
187+
188+
filep = anon_inode_getfile("[iommufd-pgfault]", &iommufd_fault_fops,
189+
fault, O_RDWR);
190+
if (IS_ERR(filep)) {
191+
rc = PTR_ERR(filep);
192+
goto out_abort;
193+
}
194+
195+
refcount_inc(&fault->obj.users);
196+
iommufd_ctx_get(fault->ictx);
197+
fault->filep = filep;
198+
199+
fdno = get_unused_fd_flags(O_CLOEXEC);
200+
if (fdno < 0) {
201+
rc = fdno;
202+
goto out_fput;
203+
}
204+
205+
cmd->out_fault_id = fault->obj.id;
206+
cmd->out_fault_fd = fdno;
207+
208+
rc = iommufd_ucmd_respond(ucmd, sizeof(*cmd));
209+
if (rc)
210+
goto out_put_fdno;
211+
iommufd_object_finalize(ucmd->ictx, &fault->obj);
212+
213+
fd_install(fdno, fault->filep);
214+
215+
return 0;
216+
out_put_fdno:
217+
put_unused_fd(fdno);
218+
out_fput:
219+
fput(filep);
220+
refcount_dec(&fault->obj.users);
221+
iommufd_ctx_put(fault->ictx);
222+
out_abort:
223+
iommufd_object_abort_and_destroy(ucmd->ictx, &fault->obj);
224+
225+
return rc;
226+
}

‎drivers/iommu/iommufd/iommufd_private.h

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,7 @@ enum iommufd_object_type {
128128
IOMMUFD_OBJ_HWPT_NESTED,
129129
IOMMUFD_OBJ_IOAS,
130130
IOMMUFD_OBJ_ACCESS,
131+
IOMMUFD_OBJ_FAULT,
131132
#ifdef CONFIG_IOMMUFD_TEST
132133
IOMMUFD_OBJ_SELFTEST,
133134
#endif
@@ -426,6 +427,35 @@ void iopt_remove_access(struct io_pagetable *iopt,
426427
u32 iopt_access_list_id);
427428
void iommufd_access_destroy_object(struct iommufd_object *obj);
428429

430+
/*
431+
* An iommufd_fault object represents an interface to deliver I/O page faults
432+
* to the user space. These objects are created/destroyed by the user space and
433+
* associated with hardware page table objects during page-table allocation.
434+
*/
435+
struct iommufd_fault {
436+
struct iommufd_object obj;
437+
struct iommufd_ctx *ictx;
438+
struct file *filep;
439+
440+
/* The lists of outstanding faults protected by below mutex. */
441+
struct mutex mutex;
442+
struct list_head deliver;
443+
struct xarray response;
444+
445+
struct wait_queue_head wait_queue;
446+
};
447+
448+
struct iommufd_attach_handle {
449+
struct iommu_attach_handle handle;
450+
struct iommufd_device *idev;
451+
};
452+
453+
/* Convert an iommu attach handle to iommufd handle. */
454+
#define to_iommufd_handle(hdl) container_of(hdl, struct iommufd_attach_handle, handle)
455+
456+
int iommufd_fault_alloc(struct iommufd_ucmd *ucmd);
457+
void iommufd_fault_destroy(struct iommufd_object *obj);
458+
429459
#ifdef CONFIG_IOMMUFD_TEST
430460
int iommufd_test(struct iommufd_ucmd *ucmd);
431461
void iommufd_selftest_destroy(struct iommufd_object *obj);

‎drivers/iommu/iommufd/main.c

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,7 @@ static int iommufd_option(struct iommufd_ucmd *ucmd)
319319

320320
union ucmd_buffer {
321321
struct iommu_destroy destroy;
322+
struct iommu_fault_alloc fault;
322323
struct iommu_hw_info info;
323324
struct iommu_hwpt_alloc hwpt;
324325
struct iommu_hwpt_get_dirty_bitmap get_dirty_bitmap;
@@ -355,6 +356,8 @@ struct iommufd_ioctl_op {
355356
}
356357
static const struct iommufd_ioctl_op iommufd_ioctl_ops[] = {
357358
IOCTL_OP(IOMMU_DESTROY, iommufd_destroy, struct iommu_destroy, id),
359+
IOCTL_OP(IOMMU_FAULT_QUEUE_ALLOC, iommufd_fault_alloc, struct iommu_fault_alloc,
360+
out_fault_fd),
358361
IOCTL_OP(IOMMU_GET_HW_INFO, iommufd_get_hw_info, struct iommu_hw_info,
359362
__reserved),
360363
IOCTL_OP(IOMMU_HWPT_ALLOC, iommufd_hwpt_alloc, struct iommu_hwpt_alloc,
@@ -513,6 +516,9 @@ static const struct iommufd_object_ops iommufd_object_ops[] = {
513516
.destroy = iommufd_hwpt_nested_destroy,
514517
.abort = iommufd_hwpt_nested_abort,
515518
},
519+
[IOMMUFD_OBJ_FAULT] = {
520+
.destroy = iommufd_fault_destroy,
521+
},
516522
#ifdef CONFIG_IOMMUFD_TEST
517523
[IOMMUFD_OBJ_SELFTEST] = {
518524
.destroy = iommufd_selftest_destroy,

‎include/linux/iommu.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,12 +124,16 @@ struct iopf_fault {
124124
struct iopf_group {
125125
struct iopf_fault last_fault;
126126
struct list_head faults;
127+
size_t fault_count;
127128
/* list node for iommu_fault_param::faults */
128129
struct list_head pending_node;
129130
struct work_struct work;
130131
struct iommu_attach_handle *attach_handle;
131132
/* The device's fault data parameter. */
132133
struct iommu_fault_param *fault_param;
134+
/* Used by handler provider to hook the group on its own lists. */
135+
struct list_head node;
136+
u32 cookie;
133137
};
134138

135139
/**

‎include/uapi/linux/iommufd.h

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ enum {
5050
IOMMUFD_CMD_HWPT_SET_DIRTY_TRACKING,
5151
IOMMUFD_CMD_HWPT_GET_DIRTY_BITMAP,
5252
IOMMUFD_CMD_HWPT_INVALIDATE,
53+
IOMMUFD_CMD_FAULT_QUEUE_ALLOC,
5354
};
5455

5556
/**
@@ -775,4 +776,21 @@ struct iommu_hwpt_page_response {
775776
__u32 cookie;
776777
__u32 code;
777778
};
779+
780+
/**
781+
* struct iommu_fault_alloc - ioctl(IOMMU_FAULT_QUEUE_ALLOC)
782+
* @size: sizeof(struct iommu_fault_alloc)
783+
* @flags: Must be 0
784+
* @out_fault_id: The ID of the new FAULT
785+
* @out_fault_fd: The fd of the new FAULT
786+
*
787+
* Explicitly allocate a fault handling object.
788+
*/
789+
struct iommu_fault_alloc {
790+
__u32 size;
791+
__u32 flags;
792+
__u32 out_fault_id;
793+
__u32 out_fault_fd;
794+
};
795+
#define IOMMU_FAULT_QUEUE_ALLOC _IO(IOMMUFD_TYPE, IOMMUFD_CMD_FAULT_QUEUE_ALLOC)
778796
#endif

0 commit comments

Comments
 (0)