33
33
#include <linux/slab.h>
34
34
#include <linux/irqnr.h>
35
35
#include <linux/pci.h>
36
+ #include <linux/rcupdate.h>
36
37
#include <linux/spinlock.h>
37
38
#include <linux/cpuhotplug.h>
38
39
#include <linux/atomic.h>
@@ -96,6 +97,7 @@ enum xen_irq_type {
96
97
struct irq_info {
97
98
struct list_head list ;
98
99
struct list_head eoi_list ;
100
+ struct rcu_work rwork ;
99
101
short refcnt ;
100
102
u8 spurious_cnt ;
101
103
u8 is_accounted ;
@@ -146,23 +148,13 @@ const struct evtchn_ops *evtchn_ops;
146
148
*/
147
149
static DEFINE_MUTEX (irq_mapping_update_lock );
148
150
149
- /*
150
- * Lock protecting event handling loop against removing event channels.
151
- * Adding of event channels is no issue as the associated IRQ becomes active
152
- * only after everything is setup (before request_[threaded_]irq() the handler
153
- * can't be entered for an event, as the event channel will be unmasked only
154
- * then).
155
- */
156
- static DEFINE_RWLOCK (evtchn_rwlock );
157
-
158
151
/*
159
152
* Lock hierarchy:
160
153
*
161
154
* irq_mapping_update_lock
162
- * evtchn_rwlock
163
- * IRQ-desc lock
164
- * percpu eoi_list_lock
165
- * irq_info->lock
155
+ * IRQ-desc lock
156
+ * percpu eoi_list_lock
157
+ * irq_info->lock
166
158
*/
167
159
168
160
static LIST_HEAD (xen_irq_list_head );
@@ -306,6 +298,22 @@ static void channels_on_cpu_inc(struct irq_info *info)
306
298
info -> is_accounted = 1 ;
307
299
}
308
300
301
+ static void delayed_free_irq (struct work_struct * work )
302
+ {
303
+ struct irq_info * info = container_of (to_rcu_work (work ), struct irq_info ,
304
+ rwork );
305
+ unsigned int irq = info -> irq ;
306
+
307
+ /* Remove the info pointer only now, with no potential users left. */
308
+ set_info_for_irq (irq , NULL );
309
+
310
+ kfree (info );
311
+
312
+ /* Legacy IRQ descriptors are managed by the arch. */
313
+ if (irq >= nr_legacy_irqs ())
314
+ irq_free_desc (irq );
315
+ }
316
+
309
317
/* Constructors for packed IRQ information. */
310
318
static int xen_irq_info_common_setup (struct irq_info * info ,
311
319
unsigned irq ,
@@ -668,33 +676,36 @@ static void xen_irq_lateeoi_worker(struct work_struct *work)
668
676
669
677
eoi = container_of (to_delayed_work (work ), struct lateeoi_work , delayed );
670
678
671
- read_lock_irqsave ( & evtchn_rwlock , flags );
679
+ rcu_read_lock ( );
672
680
673
681
while (true) {
674
- spin_lock (& eoi -> eoi_list_lock );
682
+ spin_lock_irqsave (& eoi -> eoi_list_lock , flags );
675
683
676
684
info = list_first_entry_or_null (& eoi -> eoi_list , struct irq_info ,
677
685
eoi_list );
678
686
679
- if (info == NULL || now < info -> eoi_time ) {
680
- spin_unlock (& eoi -> eoi_list_lock );
687
+ if (info == NULL )
688
+ break ;
689
+
690
+ if (now < info -> eoi_time ) {
691
+ mod_delayed_work_on (info -> eoi_cpu , system_wq ,
692
+ & eoi -> delayed ,
693
+ info -> eoi_time - now );
681
694
break ;
682
695
}
683
696
684
697
list_del_init (& info -> eoi_list );
685
698
686
- spin_unlock (& eoi -> eoi_list_lock );
699
+ spin_unlock_irqrestore (& eoi -> eoi_list_lock , flags );
687
700
688
701
info -> eoi_time = 0 ;
689
702
690
703
xen_irq_lateeoi_locked (info , false);
691
704
}
692
705
693
- if (info )
694
- mod_delayed_work_on (info -> eoi_cpu , system_wq ,
695
- & eoi -> delayed , info -> eoi_time - now );
706
+ spin_unlock_irqrestore (& eoi -> eoi_list_lock , flags );
696
707
697
- read_unlock_irqrestore ( & evtchn_rwlock , flags );
708
+ rcu_read_unlock ( );
698
709
}
699
710
700
711
static void xen_cpu_init_eoi (unsigned int cpu )
@@ -709,16 +720,15 @@ static void xen_cpu_init_eoi(unsigned int cpu)
709
720
void xen_irq_lateeoi (unsigned int irq , unsigned int eoi_flags )
710
721
{
711
722
struct irq_info * info ;
712
- unsigned long flags ;
713
723
714
- read_lock_irqsave ( & evtchn_rwlock , flags );
724
+ rcu_read_lock ( );
715
725
716
726
info = info_for_irq (irq );
717
727
718
728
if (info )
719
729
xen_irq_lateeoi_locked (info , eoi_flags & XEN_EOI_FLAG_SPURIOUS );
720
730
721
- read_unlock_irqrestore ( & evtchn_rwlock , flags );
731
+ rcu_read_unlock ( );
722
732
}
723
733
EXPORT_SYMBOL_GPL (xen_irq_lateeoi );
724
734
@@ -732,6 +742,7 @@ static void xen_irq_init(unsigned irq)
732
742
733
743
info -> type = IRQT_UNBOUND ;
734
744
info -> refcnt = -1 ;
745
+ INIT_RCU_WORK (& info -> rwork , delayed_free_irq );
735
746
736
747
set_info_for_irq (irq , info );
737
748
/*
@@ -789,31 +800,18 @@ static int __must_check xen_allocate_irq_gsi(unsigned gsi)
789
800
static void xen_free_irq (unsigned irq )
790
801
{
791
802
struct irq_info * info = info_for_irq (irq );
792
- unsigned long flags ;
793
803
794
804
if (WARN_ON (!info ))
795
805
return ;
796
806
797
- write_lock_irqsave (& evtchn_rwlock , flags );
798
-
799
807
if (!list_empty (& info -> eoi_list ))
800
808
lateeoi_list_del (info );
801
809
802
810
list_del (& info -> list );
803
811
804
- set_info_for_irq (irq , NULL );
805
-
806
812
WARN_ON (info -> refcnt > 0 );
807
813
808
- write_unlock_irqrestore (& evtchn_rwlock , flags );
809
-
810
- kfree (info );
811
-
812
- /* Legacy IRQ descriptors are managed by the arch. */
813
- if (irq < nr_legacy_irqs ())
814
- return ;
815
-
816
- irq_free_desc (irq );
814
+ queue_rcu_work (system_wq , & info -> rwork );
817
815
}
818
816
819
817
/* Not called for lateeoi events. */
@@ -1711,7 +1709,14 @@ int xen_evtchn_do_upcall(void)
1711
1709
int cpu = smp_processor_id ();
1712
1710
struct evtchn_loop_ctrl ctrl = { 0 };
1713
1711
1714
- read_lock (& evtchn_rwlock );
1712
+ /*
1713
+ * When closing an event channel the associated IRQ must not be freed
1714
+ * until all cpus have left the event handling loop. This is ensured
1715
+ * by taking the rcu_read_lock() while handling events, as freeing of
1716
+ * the IRQ is handled via queue_rcu_work() _after_ closing the event
1717
+ * channel.
1718
+ */
1719
+ rcu_read_lock ();
1715
1720
1716
1721
do {
1717
1722
vcpu_info -> evtchn_upcall_pending = 0 ;
@@ -1724,7 +1729,7 @@ int xen_evtchn_do_upcall(void)
1724
1729
1725
1730
} while (vcpu_info -> evtchn_upcall_pending );
1726
1731
1727
- read_unlock ( & evtchn_rwlock );
1732
+ rcu_read_unlock ( );
1728
1733
1729
1734
/*
1730
1735
* Increment irq_epoch only now to defer EOIs only for
0 commit comments