Skip to content

Commit f7f0748

Browse files
rmurphy-armjoergroedel
rmurphy-arm
authored and
joergroedel
committed
iommu/iova: Move flush queue code to iommu-dma
Flush queues are specific to DMA ops, which are now handled exclusively by iommu-dma. As such, now that the historical artefacts from being shared directly with drivers have been cleaned up, move the flush queue code into iommu-dma itself to get it out of the way of other IOVA users. This is pure code movement with no functional change; refactoring to clean up the headers and definitions will follow. Reviewed-by: John Garry <[email protected]> Signed-off-by: Robin Murphy <[email protected]> Link: https://lore.kernel.org/r/1d9a1ee1392e96eaae5e6467181b3e83edfdfbad.1639753638.git.robin.murphy@arm.com Signed-off-by: Joerg Roedel <[email protected]>
1 parent ea4d71b commit f7f0748

File tree

2 files changed

+176
-176
lines changed

2 files changed

+176
-176
lines changed

‎drivers/iommu/dma-iommu.c

Lines changed: 176 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,179 @@ static int __init iommu_dma_forcedac_setup(char *str)
6464
}
6565
early_param("iommu.forcedac", iommu_dma_forcedac_setup);
6666

67+
#define fq_ring_for_each(i, fq) \
68+
for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE)
69+
70+
static inline bool fq_full(struct iova_fq *fq)
71+
{
72+
assert_spin_locked(&fq->lock);
73+
return (((fq->tail + 1) % IOVA_FQ_SIZE) == fq->head);
74+
}
75+
76+
static inline unsigned fq_ring_add(struct iova_fq *fq)
77+
{
78+
unsigned idx = fq->tail;
79+
80+
assert_spin_locked(&fq->lock);
81+
82+
fq->tail = (idx + 1) % IOVA_FQ_SIZE;
83+
84+
return idx;
85+
}
86+
87+
static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq)
88+
{
89+
u64 counter = atomic64_read(&iovad->fq_flush_finish_cnt);
90+
unsigned idx;
91+
92+
assert_spin_locked(&fq->lock);
93+
94+
fq_ring_for_each(idx, fq) {
95+
96+
if (fq->entries[idx].counter >= counter)
97+
break;
98+
99+
put_pages_list(&fq->entries[idx].freelist);
100+
free_iova_fast(iovad,
101+
fq->entries[idx].iova_pfn,
102+
fq->entries[idx].pages);
103+
104+
fq->head = (fq->head + 1) % IOVA_FQ_SIZE;
105+
}
106+
}
107+
108+
static void iova_domain_flush(struct iova_domain *iovad)
109+
{
110+
atomic64_inc(&iovad->fq_flush_start_cnt);
111+
iovad->fq_domain->ops->flush_iotlb_all(iovad->fq_domain);
112+
atomic64_inc(&iovad->fq_flush_finish_cnt);
113+
}
114+
115+
static void fq_flush_timeout(struct timer_list *t)
116+
{
117+
struct iova_domain *iovad = from_timer(iovad, t, fq_timer);
118+
int cpu;
119+
120+
atomic_set(&iovad->fq_timer_on, 0);
121+
iova_domain_flush(iovad);
122+
123+
for_each_possible_cpu(cpu) {
124+
unsigned long flags;
125+
struct iova_fq *fq;
126+
127+
fq = per_cpu_ptr(iovad->fq, cpu);
128+
spin_lock_irqsave(&fq->lock, flags);
129+
fq_ring_free(iovad, fq);
130+
spin_unlock_irqrestore(&fq->lock, flags);
131+
}
132+
}
133+
134+
void queue_iova(struct iova_domain *iovad,
135+
unsigned long pfn, unsigned long pages,
136+
struct list_head *freelist)
137+
{
138+
struct iova_fq *fq;
139+
unsigned long flags;
140+
unsigned idx;
141+
142+
/*
143+
* Order against the IOMMU driver's pagetable update from unmapping
144+
* @pte, to guarantee that iova_domain_flush() observes that if called
145+
* from a different CPU before we release the lock below. Full barrier
146+
* so it also pairs with iommu_dma_init_fq() to avoid seeing partially
147+
* written fq state here.
148+
*/
149+
smp_mb();
150+
151+
fq = raw_cpu_ptr(iovad->fq);
152+
spin_lock_irqsave(&fq->lock, flags);
153+
154+
/*
155+
* First remove all entries from the flush queue that have already been
156+
* flushed out on another CPU. This makes the fq_full() check below less
157+
* likely to be true.
158+
*/
159+
fq_ring_free(iovad, fq);
160+
161+
if (fq_full(fq)) {
162+
iova_domain_flush(iovad);
163+
fq_ring_free(iovad, fq);
164+
}
165+
166+
idx = fq_ring_add(fq);
167+
168+
fq->entries[idx].iova_pfn = pfn;
169+
fq->entries[idx].pages = pages;
170+
fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt);
171+
list_splice(freelist, &fq->entries[idx].freelist);
172+
173+
spin_unlock_irqrestore(&fq->lock, flags);
174+
175+
/* Avoid false sharing as much as possible. */
176+
if (!atomic_read(&iovad->fq_timer_on) &&
177+
!atomic_xchg(&iovad->fq_timer_on, 1))
178+
mod_timer(&iovad->fq_timer,
179+
jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT));
180+
}
181+
182+
static void free_iova_flush_queue(struct iova_domain *iovad)
183+
{
184+
int cpu, idx;
185+
186+
if (!iovad->fq)
187+
return;
188+
189+
del_timer_sync(&iovad->fq_timer);
190+
/*
191+
* This code runs when the iova_domain is being detroyed, so don't
192+
* bother to free iovas, just free any remaining pagetable pages.
193+
*/
194+
for_each_possible_cpu(cpu) {
195+
struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu);
196+
197+
fq_ring_for_each(idx, fq)
198+
put_pages_list(&fq->entries[idx].freelist);
199+
}
200+
201+
free_percpu(iovad->fq);
202+
203+
iovad->fq = NULL;
204+
iovad->fq_domain = NULL;
205+
}
206+
207+
int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain)
208+
{
209+
struct iova_fq __percpu *queue;
210+
int i, cpu;
211+
212+
atomic64_set(&iovad->fq_flush_start_cnt, 0);
213+
atomic64_set(&iovad->fq_flush_finish_cnt, 0);
214+
215+
queue = alloc_percpu(struct iova_fq);
216+
if (!queue)
217+
return -ENOMEM;
218+
219+
for_each_possible_cpu(cpu) {
220+
struct iova_fq *fq = per_cpu_ptr(queue, cpu);
221+
222+
fq->head = 0;
223+
fq->tail = 0;
224+
225+
spin_lock_init(&fq->lock);
226+
227+
for (i = 0; i < IOVA_FQ_SIZE; i++)
228+
INIT_LIST_HEAD(&fq->entries[i].freelist);
229+
}
230+
231+
iovad->fq_domain = fq_domain;
232+
iovad->fq = queue;
233+
234+
timer_setup(&iovad->fq_timer, fq_flush_timeout, 0);
235+
atomic_set(&iovad->fq_timer_on, 0);
236+
237+
return 0;
238+
}
239+
67240
static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)
68241
{
69242
if (cookie->type == IOMMU_DMA_IOVA_COOKIE)
@@ -144,8 +317,10 @@ void iommu_put_dma_cookie(struct iommu_domain *domain)
144317
if (!cookie)
145318
return;
146319

147-
if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule)
320+
if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) {
321+
free_iova_flush_queue(&cookie->iovad);
148322
put_iova_domain(&cookie->iovad);
323+
}
149324

150325
list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) {
151326
list_del(&msi->list);

‎drivers/iommu/iova.c

Lines changed: 0 additions & 175 deletions
Original file line numberDiff line numberDiff line change
@@ -490,179 +490,6 @@ free_iova_fast(struct iova_domain *iovad, unsigned long pfn, unsigned long size)
490490
}
491491
EXPORT_SYMBOL_GPL(free_iova_fast);
492492

493-
#define fq_ring_for_each(i, fq) \
494-
for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE)
495-
496-
static inline bool fq_full(struct iova_fq *fq)
497-
{
498-
assert_spin_locked(&fq->lock);
499-
return (((fq->tail + 1) % IOVA_FQ_SIZE) == fq->head);
500-
}
501-
502-
static inline unsigned fq_ring_add(struct iova_fq *fq)
503-
{
504-
unsigned idx = fq->tail;
505-
506-
assert_spin_locked(&fq->lock);
507-
508-
fq->tail = (idx + 1) % IOVA_FQ_SIZE;
509-
510-
return idx;
511-
}
512-
513-
static void fq_ring_free(struct iova_domain *iovad, struct iova_fq *fq)
514-
{
515-
u64 counter = atomic64_read(&iovad->fq_flush_finish_cnt);
516-
unsigned idx;
517-
518-
assert_spin_locked(&fq->lock);
519-
520-
fq_ring_for_each(idx, fq) {
521-
522-
if (fq->entries[idx].counter >= counter)
523-
break;
524-
525-
put_pages_list(&fq->entries[idx].freelist);
526-
free_iova_fast(iovad,
527-
fq->entries[idx].iova_pfn,
528-
fq->entries[idx].pages);
529-
530-
fq->head = (fq->head + 1) % IOVA_FQ_SIZE;
531-
}
532-
}
533-
534-
static void iova_domain_flush(struct iova_domain *iovad)
535-
{
536-
atomic64_inc(&iovad->fq_flush_start_cnt);
537-
iovad->fq_domain->ops->flush_iotlb_all(iovad->fq_domain);
538-
atomic64_inc(&iovad->fq_flush_finish_cnt);
539-
}
540-
541-
static void fq_flush_timeout(struct timer_list *t)
542-
{
543-
struct iova_domain *iovad = from_timer(iovad, t, fq_timer);
544-
int cpu;
545-
546-
atomic_set(&iovad->fq_timer_on, 0);
547-
iova_domain_flush(iovad);
548-
549-
for_each_possible_cpu(cpu) {
550-
unsigned long flags;
551-
struct iova_fq *fq;
552-
553-
fq = per_cpu_ptr(iovad->fq, cpu);
554-
spin_lock_irqsave(&fq->lock, flags);
555-
fq_ring_free(iovad, fq);
556-
spin_unlock_irqrestore(&fq->lock, flags);
557-
}
558-
}
559-
560-
void queue_iova(struct iova_domain *iovad,
561-
unsigned long pfn, unsigned long pages,
562-
struct list_head *freelist)
563-
{
564-
struct iova_fq *fq;
565-
unsigned long flags;
566-
unsigned idx;
567-
568-
/*
569-
* Order against the IOMMU driver's pagetable update from unmapping
570-
* @pte, to guarantee that iova_domain_flush() observes that if called
571-
* from a different CPU before we release the lock below. Full barrier
572-
* so it also pairs with iommu_dma_init_fq() to avoid seeing partially
573-
* written fq state here.
574-
*/
575-
smp_mb();
576-
577-
fq = raw_cpu_ptr(iovad->fq);
578-
spin_lock_irqsave(&fq->lock, flags);
579-
580-
/*
581-
* First remove all entries from the flush queue that have already been
582-
* flushed out on another CPU. This makes the fq_full() check below less
583-
* likely to be true.
584-
*/
585-
fq_ring_free(iovad, fq);
586-
587-
if (fq_full(fq)) {
588-
iova_domain_flush(iovad);
589-
fq_ring_free(iovad, fq);
590-
}
591-
592-
idx = fq_ring_add(fq);
593-
594-
fq->entries[idx].iova_pfn = pfn;
595-
fq->entries[idx].pages = pages;
596-
fq->entries[idx].counter = atomic64_read(&iovad->fq_flush_start_cnt);
597-
list_splice(freelist, &fq->entries[idx].freelist);
598-
599-
spin_unlock_irqrestore(&fq->lock, flags);
600-
601-
/* Avoid false sharing as much as possible. */
602-
if (!atomic_read(&iovad->fq_timer_on) &&
603-
!atomic_xchg(&iovad->fq_timer_on, 1))
604-
mod_timer(&iovad->fq_timer,
605-
jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT));
606-
}
607-
608-
static void free_iova_flush_queue(struct iova_domain *iovad)
609-
{
610-
int cpu, idx;
611-
612-
if (!iovad->fq)
613-
return;
614-
615-
del_timer_sync(&iovad->fq_timer);
616-
/*
617-
* This code runs when the iova_domain is being detroyed, so don't
618-
* bother to free iovas, just free any remaining pagetable pages.
619-
*/
620-
for_each_possible_cpu(cpu) {
621-
struct iova_fq *fq = per_cpu_ptr(iovad->fq, cpu);
622-
623-
fq_ring_for_each(idx, fq)
624-
put_pages_list(&fq->entries[idx].freelist);
625-
}
626-
627-
free_percpu(iovad->fq);
628-
629-
iovad->fq = NULL;
630-
iovad->fq_domain = NULL;
631-
}
632-
633-
int init_iova_flush_queue(struct iova_domain *iovad, struct iommu_domain *fq_domain)
634-
{
635-
struct iova_fq __percpu *queue;
636-
int i, cpu;
637-
638-
atomic64_set(&iovad->fq_flush_start_cnt, 0);
639-
atomic64_set(&iovad->fq_flush_finish_cnt, 0);
640-
641-
queue = alloc_percpu(struct iova_fq);
642-
if (!queue)
643-
return -ENOMEM;
644-
645-
for_each_possible_cpu(cpu) {
646-
struct iova_fq *fq = per_cpu_ptr(queue, cpu);
647-
648-
fq->head = 0;
649-
fq->tail = 0;
650-
651-
spin_lock_init(&fq->lock);
652-
653-
for (i = 0; i < IOVA_FQ_SIZE; i++)
654-
INIT_LIST_HEAD(&fq->entries[i].freelist);
655-
}
656-
657-
iovad->fq_domain = fq_domain;
658-
iovad->fq = queue;
659-
660-
timer_setup(&iovad->fq_timer, fq_flush_timeout, 0);
661-
atomic_set(&iovad->fq_timer_on, 0);
662-
663-
return 0;
664-
}
665-
666493
/**
667494
* put_iova_domain - destroys the iova domain
668495
* @iovad: - iova domain in question.
@@ -674,8 +501,6 @@ void put_iova_domain(struct iova_domain *iovad)
674501

675502
cpuhp_state_remove_instance_nocalls(CPUHP_IOMMU_IOVA_DEAD,
676503
&iovad->cpuhp_dead);
677-
678-
free_iova_flush_queue(iovad);
679504
free_iova_rcaches(iovad);
680505
rbtree_postorder_for_each_entry_safe(iova, tmp, &iovad->rbroot, node)
681506
free_iova_mem(iova);

0 commit comments

Comments
 (0)