@@ -86,6 +86,9 @@ struct Inner {
86
86
87
87
/// A linked list holding registered listeners.
88
88
list : Spinlock < List > ,
89
+
90
+ /// A single cached list entry to avoid allocations on the fast path of the insertion.
91
+ cache : UnsafeCell < Entry > ,
89
92
}
90
93
91
94
impl Inner {
@@ -96,6 +99,12 @@ impl Inner {
96
99
guard : self . list . lock ( ) ,
97
100
}
98
101
}
102
+
103
+ /// Returns the pointer to the single cached list entry.
104
+ #[ inline( always) ]
105
+ fn cache_ptr ( & self ) -> NonNull < Entry > {
106
+ unsafe { NonNull :: new_unchecked ( self . cache . get ( ) ) }
107
+ }
99
108
}
100
109
101
110
/// A synchronization primitive for notifying async tasks and threads.
@@ -166,7 +175,7 @@ impl Event {
166
175
let inner = self . inner ( ) ;
167
176
let listener = EventListener {
168
177
inner : unsafe { Arc :: clone ( & ManuallyDrop :: new ( Arc :: from_raw ( inner) ) ) } ,
169
- entry : Some ( inner. lock ( ) . insert ( ) ) ,
178
+ entry : Some ( inner. lock ( ) . insert ( inner . cache_ptr ( ) ) ) ,
170
179
} ;
171
180
172
181
// Make sure the listener is registered before whatever happens next.
@@ -372,11 +381,11 @@ impl Event {
372
381
len : 0 ,
373
382
notified : 0 ,
374
383
cache_used : false ,
375
- cache : UnsafeCell :: new ( Entry {
376
- state : Cell :: new ( State :: Created ) ,
377
- prev : Cell :: new ( None ) ,
378
- next : Cell :: new ( None ) ,
379
- } ) ,
384
+ } ) ,
385
+ cache : UnsafeCell :: new ( Entry {
386
+ state : Cell :: new ( State :: Created ) ,
387
+ prev : Cell :: new ( None ) ,
388
+ next : Cell :: new ( None ) ,
380
389
} ) ,
381
390
} ) ;
382
391
// Convert the heap-allocated state into a raw pointer.
@@ -563,7 +572,7 @@ impl EventListener {
563
572
match e. state . replace ( State :: Notified ( false ) ) {
564
573
State :: Notified ( _) => {
565
574
// If this listener has been notified, remove it from the list and return.
566
- list. remove ( entry) ;
575
+ list. remove ( entry, self . inner . cache_ptr ( ) ) ;
567
576
return true ;
568
577
}
569
578
// Otherwise, set the state to `Waiting`.
@@ -581,11 +590,11 @@ impl EventListener {
581
590
let now = Instant :: now ( ) ;
582
591
if now >= deadline {
583
592
// Remove the entry and check if notified.
584
- if self . inner . lock ( ) . remove ( entry ) . is_notified ( ) {
585
- return true ;
586
- } else {
587
- return false ;
588
- }
593
+ return self
594
+ . inner
595
+ . lock ( )
596
+ . remove ( entry , self . inner . cache_ptr ( ) )
597
+ . is_notified ( ) ;
589
598
}
590
599
591
600
// Park until the deadline.
@@ -600,7 +609,7 @@ impl EventListener {
600
609
match e. state . replace ( State :: Notified ( false ) ) {
601
610
State :: Notified ( _) => {
602
611
// If this listener has been notified, remove it from the list and return.
603
- list. remove ( entry) ;
612
+ list. remove ( entry, self . inner . cache_ptr ( ) ) ;
604
613
return true ;
605
614
}
606
615
// Otherwise, set the state back to `Waiting`.
@@ -632,7 +641,7 @@ impl Future for EventListener {
632
641
match state. replace ( State :: Notified ( false ) ) {
633
642
State :: Notified ( _) => {
634
643
// If this listener has been notified, remove it from the list and return.
635
- list. remove ( entry) ;
644
+ list. remove ( entry, self . inner . cache_ptr ( ) ) ;
636
645
drop ( list) ;
637
646
self . entry = None ;
638
647
return Poll :: Ready ( ( ) ) ;
@@ -665,7 +674,7 @@ impl Drop for EventListener {
665
674
let mut list = self . inner . lock ( ) ;
666
675
667
676
// But if a notification was delivered to it...
668
- if let State :: Notified ( additional) = list. remove ( entry) {
677
+ if let State :: Notified ( additional) = list. remove ( entry, self . inner . cache_ptr ( ) ) {
669
678
// Then pass it on to another active listener.
670
679
if additional {
671
680
list. notify_additional ( 1 ) ;
@@ -776,14 +785,11 @@ struct List {
776
785
777
786
/// Whether the cached entry is used.
778
787
cache_used : bool ,
779
-
780
- /// A single cached entry to avoid allocations on the fast path.
781
- cache : UnsafeCell < Entry > ,
782
788
}
783
789
784
790
impl List {
785
791
/// Inserts a new entry into the list.
786
- fn insert ( & mut self ) -> NonNull < Entry > {
792
+ fn insert ( & mut self , cache : NonNull < Entry > ) -> NonNull < Entry > {
787
793
unsafe {
788
794
let entry = Entry {
789
795
state : Cell :: new ( State :: Created ) ,
@@ -797,8 +803,8 @@ impl List {
797
803
} else {
798
804
// No need to allocate - we can use the cached entry.
799
805
self . cache_used = true ;
800
- * self . cache . get ( ) = entry;
801
- NonNull :: new_unchecked ( self . cache . get ( ) )
806
+ cache. as_ptr ( ) . write ( entry) ;
807
+ cache
802
808
} ;
803
809
804
810
// Replace the tail with the new entry.
@@ -820,7 +826,7 @@ impl List {
820
826
}
821
827
822
828
/// Removes an entry from the list and returns its state.
823
- fn remove ( & mut self , entry : NonNull < Entry > ) -> State {
829
+ fn remove ( & mut self , entry : NonNull < Entry > , cache : NonNull < Entry > ) -> State {
824
830
unsafe {
825
831
let prev = entry. as_ref ( ) . prev . get ( ) ;
826
832
let next = entry. as_ref ( ) . next . get ( ) ;
@@ -843,7 +849,7 @@ impl List {
843
849
}
844
850
845
851
// Extract the state.
846
- let state = if ptr:: eq ( entry. as_ptr ( ) , self . cache . get ( ) ) {
852
+ let state = if ptr:: eq ( entry. as_ptr ( ) , cache. as_ptr ( ) ) {
847
853
// Free the cached entry.
848
854
self . cache_used = false ;
849
855
entry. as_ref ( ) . state . replace ( State :: Created )
0 commit comments