@@ -10,13 +10,18 @@ use core::{
1010 ops:: { Deref , DerefMut , Index , IndexMut } ,
1111 ptr,
1212 slice:: SliceIndex ,
13- sync:: atomic:: { AtomicUsize , Ordering } ,
1413} ;
1514
15+ #[ cfg( feature = "atomic_append" ) ]
16+ use core:: sync:: atomic:: { AtomicUsize , Ordering } ;
17+
1618struct HeaderVecHeader < H > {
1719 head : H ,
1820 capacity : usize ,
21+ #[ cfg( feature = "atomic_append" ) ]
1922 len : AtomicUsize ,
23+ #[ cfg( not( feature = "atomic_append" ) ) ]
24+ len : usize ,
2025}
2126
2227/// A vector with a header of your choosing behind a thin pointer
@@ -75,82 +80,94 @@ impl<H, T> HeaderVec<H, T> {
7580 unsafe { core:: ptr:: write ( & mut header. head , head) } ;
7681 // These primitive types don't have drop implementations.
7782 header. capacity = capacity;
78- header. len = AtomicUsize :: new ( 0 ) ;
83+ header. len = 0usize . into ( ) ;
7984
8085 this
8186 }
8287
83- /// Get the length of the vector from a mutable reference.
88+ /// Get the length of the vector from a mutable reference. When one has a `&mut
89+ /// HeaderVec`, this is the method is always exact and can be slightly faster than the non
90+ /// mutable `len()`.
91+ #[ cfg( feature = "atomic_append" ) ]
8492 #[ inline( always) ]
85- pub fn len_from_mut ( & mut self ) -> usize {
93+ pub fn len_exact ( & mut self ) -> usize {
8694 * self . header_mut ( ) . len . get_mut ( )
8795 }
88-
89- /// Get the length of the vector with `Ordering::Acquire`. This ensures that the length is
90- /// properly synchronized after it got atomically updated.
96+ #[ cfg( not( feature = "atomic_append" ) ) ]
9197 #[ inline( always) ]
92- pub fn len_atomic_acquire ( & self ) -> usize {
93- self . header ( ) . len . load ( Ordering :: Acquire )
98+ pub fn len_exact ( & mut self ) -> usize {
99+ self . header_mut ( ) . len
94100 }
95101
96- /// Get the length of the vector with `Ordering::Relaxed`. This is useful for when you don't
97- /// need exact synchronization semantic.
102+ /// This gives the length of the `HeaderVec`. This is the non synchronized variant may
103+ /// produce racy results in case another thread atomically appended to
104+ /// `&self`. Nevertheless it is always safe to use.
105+ #[ cfg( feature = "atomic_append" ) ]
98106 #[ inline( always) ]
99- pub fn len_atomic_relaxed ( & self ) -> usize {
100- self . header ( ) . len . load ( Ordering :: Relaxed )
107+ pub fn len ( & self ) -> usize {
108+ self . len_atomic_relaxed ( )
101109 }
102-
103- /// Alias for [`HeaderVec::len_atomic_relaxed`]. This gives always a valid view of the
104- /// length when used in isolation.
110+ #[ cfg( not( feature = "atomic_append" ) ) ]
105111 #[ inline( always) ]
106112 pub fn len ( & self ) -> usize {
107- self . len_atomic_relaxed ( )
113+ self . header ( ) . len
108114 }
109115
110- /// Add `n` to the length of the vector atomically with `Ordering::Release`.
111- ///
112- /// # Safety
113- ///
114- /// Before incrementing the length of the vector, you must ensure that new elements are
115- /// properly initialized.
116+ /// This gives the length of the `HeaderVec`. With `atomic_append` enabled this gives a
117+ /// exact result *after* another thread atomically appended to this `HeaderVec`. It still
118+ /// requires synchronization because the length may become invalidated when another thread
119+ /// atomically appends data to this `HeaderVec` while we still work with the result of
120+ /// this method.
121+ # [ cfg ( not ( feature = "atomic_append" ) ) ]
116122 #[ inline( always) ]
117- pub unsafe fn len_atomic_add_release ( & self , n : usize ) -> usize {
118- self . header ( ) . len . fetch_add ( n, Ordering :: Release )
123+ pub fn len_strict ( & self ) -> usize {
124+ self . header ( ) . len
125+ }
126+ #[ cfg( feature = "atomic_append" ) ]
127+ #[ inline( always) ]
128+ pub fn len_strict ( & self ) -> usize {
129+ self . len_atomic_acquire ( )
119130 }
120131
132+ /// Check whenever a `HeaderVec` is empty. This uses a `&mut self` reference and is
133+ /// always exact and may be slightly faster than the non mutable variant.
121134 #[ inline( always) ]
122- pub fn is_empty_from_mut ( & mut self ) -> bool {
123- self . len_from_mut ( ) == 0
135+ pub fn is_empty_exact ( & mut self ) -> bool {
136+ self . len_exact ( ) == 0
124137 }
125138
139+ /// Check whenever a `HeaderVec` is empty. This uses a `&self` reference and may be racy
140+ /// when another thread atomically appended to this `HeaderVec`.
126141 #[ inline( always) ]
127142 pub fn is_empty ( & self ) -> bool {
128- self . len_atomic_relaxed ( ) == 0
143+ self . len ( ) == 0
129144 }
130145
146+ /// Check whenever a `HeaderVec` is empty. see [`len_strict()`] about the exactness guarantees.
131147 #[ inline( always) ]
132- pub fn is_empty_atomic_acquire ( & self ) -> bool {
133- self . len_atomic_acquire ( ) == 0
148+ pub fn is_empty_strict ( & self ) -> bool {
149+ self . len_strict ( ) == 0
134150 }
135151
136152 #[ inline( always) ]
137153 pub fn capacity ( & self ) -> usize {
138154 self . header ( ) . capacity
139155 }
140156
157+ /// This is the amount of elements that can be added to the `HeaderVec` without reallocation.
141158 #[ inline( always) ]
142- pub fn as_slice ( & self ) -> & [ T ] {
143- unsafe { core :: slice :: from_raw_parts ( self . start_ptr ( ) , self . len_atomic_relaxed ( ) ) }
159+ pub fn spare_capacity ( & self ) -> usize {
160+ self . header ( ) . capacity - self . len_strict ( )
144161 }
145162
146163 #[ inline( always) ]
147- pub fn as_slice_atomic_acquire ( & self ) -> & [ T ] {
148- unsafe { core:: slice:: from_raw_parts ( self . start_ptr ( ) , self . len_atomic_acquire ( ) ) }
164+ pub fn as_slice ( & self ) -> & [ T ] {
165+ unsafe { core:: slice:: from_raw_parts ( self . start_ptr ( ) , self . len_strict ( ) ) }
149166 }
150167
151168 #[ inline( always) ]
152169 pub fn as_mut_slice ( & mut self ) -> & mut [ T ] {
153- unsafe { core:: slice:: from_raw_parts_mut ( self . start_ptr_mut ( ) , self . len_from_mut ( ) ) }
170+ unsafe { core:: slice:: from_raw_parts_mut ( self . start_ptr_mut ( ) , self . len_exact ( ) ) }
154171 }
155172
156173 /// This is useful to check if two nodes are the same. Use it with [`HeaderVec::is`].
@@ -241,7 +258,7 @@ impl<H, T> HeaderVec<H, T> {
241258 /// Returns `true` if the memory was moved to a new location.
242259 /// In this case, you are responsible for updating the weak nodes.
243260 pub fn push ( & mut self , item : T ) -> Option < * const ( ) > {
244- let old_len = self . len_from_mut ( ) ;
261+ let old_len = self . len_exact ( ) ;
245262 let new_len = old_len + 1 ;
246263 let old_capacity = self . capacity ( ) ;
247264 // If it isn't big enough.
@@ -266,7 +283,7 @@ impl<H, T> HeaderVec<H, T> {
266283 // This keeps track of the length (and next position) of the contiguous retained elements
267284 // at the beginning of the vector.
268285 let mut head = 0 ;
269- let original_len = self . len_from_mut ( ) ;
286+ let original_len = self . len_exact ( ) ;
270287 // Get the offset of the beginning of the slice.
271288 let start_ptr = self . start_ptr_mut ( ) ;
272289 // Go through each index.
@@ -346,11 +363,58 @@ impl<H, T> HeaderVec<H, T> {
346363 }
347364}
348365
366+ #[ cfg( feature = "atomic_append" ) ]
367+ /// The atomic append API is only enabled when the `atomic_append` feature flag is set (which
368+ /// is the default).
369+ impl < H , T > HeaderVec < H , T > {
370+ /// Get the length of the vector with `Ordering::Acquire`. This ensures that the length is
371+ /// properly synchronized after it got atomically updated.
372+ #[ inline( always) ]
373+ fn len_atomic_acquire ( & self ) -> usize {
374+ self . header ( ) . len . load ( Ordering :: Acquire )
375+ }
376+
377+ /// Get the length of the vector with `Ordering::Relaxed`. This is useful for when you don't
378+ /// need exact synchronization semantic.
379+ #[ inline( always) ]
380+ fn len_atomic_relaxed ( & self ) -> usize {
381+ self . header ( ) . len . load ( Ordering :: Relaxed )
382+ }
383+
384+ /// Add `n` to the length of the vector atomically with `Ordering::Release`.
385+ ///
386+ /// # Safety
387+ ///
388+ /// Before incrementing the length of the vector, you must ensure that new elements are
389+ /// properly initialized.
390+ #[ inline( always) ]
391+ unsafe fn len_atomic_add_release ( & self , n : usize ) -> usize {
392+ self . header ( ) . len . fetch_add ( n, Ordering :: Release )
393+ }
394+
395+ #[ inline( always) ]
396+ pub fn is_empty_atomic_acquire ( & self ) -> bool {
397+ self . len_atomic_acquire ( ) == 0
398+ }
399+
400+ #[ inline( always) ]
401+ pub fn as_slice_atomic_acquire ( & self ) -> & [ T ] {
402+ unsafe { core:: slice:: from_raw_parts ( self . start_ptr ( ) , self . len_atomic_acquire ( ) ) }
403+ }
404+
405+ /// Gets the pointer to the end of the slice. This returns a mutable pointer to
406+ /// uninitialized memory behind the last element.
407+ #[ inline( always) ]
408+ fn end_ptr_atomic_mut ( & self ) -> * mut T {
409+ unsafe { self . ptr . add ( Self :: offset ( ) ) . add ( self . len_atomic_acquire ( ) ) }
410+ }
411+ }
412+
349413impl < H , T > Drop for HeaderVec < H , T > {
350414 fn drop ( & mut self ) {
351415 unsafe {
352416 ptr:: drop_in_place ( & mut self . header_mut ( ) . head ) ;
353- for ix in 0 ..self . len_from_mut ( ) {
417+ for ix in 0 ..self . len_exact ( ) {
354418 ptr:: drop_in_place ( self . start_ptr_mut ( ) . add ( ix) ) ;
355419 }
356420 alloc:: alloc:: dealloc ( self . ptr as * mut u8 , Self :: layout ( self . capacity ( ) ) ) ;
@@ -412,8 +476,7 @@ where
412476 T : Clone ,
413477{
414478 fn clone ( & self ) -> Self {
415- let mut new_vec =
416- Self :: with_capacity ( self . len_atomic_acquire ( ) , self . header ( ) . head . clone ( ) ) ;
479+ let mut new_vec = Self :: with_capacity ( self . len_strict ( ) , self . header ( ) . head . clone ( ) ) ;
417480 for e in self . as_slice ( ) {
418481 new_vec. push ( e. clone ( ) ) ;
419482 }
0 commit comments