11//! Uring RingBuf
22
33use crate :: RingBufError ;
4+ use core:: cell:: Cell ;
45use core:: num:: NonZero ;
56use core:: sync:: atomic:: { AtomicU16 , Ordering } ;
67
@@ -106,6 +107,9 @@ impl RingBufChoice {
106107pub struct RingBufUnregistered {
107108 choice : RingBufChoice ,
108109 ring_start : AnonymousMmap ,
110+ shared_tail : * const AtomicU16 ,
111+ local_tail : Cell < u16 > ,
112+ base_ptr : * mut u8 ,
109113}
110114
111115impl RingBufUnregistered {
@@ -126,9 +130,9 @@ impl RingBufUnregistered {
126130 let entry_size = size_of :: < BufRingEntry > ( ) ; // 16B
127131 let ring_size = entry_size * choice. total_bufs_count ( ) as usize ;
128132 let ring_start = AnonymousMmap :: new ( ring_size) . map_err ( RingBufError :: Mmap ) ?;
133+ let entries = ring_start. as_ptr_mut ( ) as * mut BufRingEntry ;
129134
130135 for bid in 0u16 ..choice. total_bufs_count ( ) {
131- let entries = ring_start. as_ptr_mut ( ) as * mut BufRingEntry ;
132136 // SAFETY: Our owned AnonymousMmap holds the validity
133137 let new_entry = unsafe { & mut * entries. add ( bid as usize ) } ;
134138 // SAFETY: Our owned AnonymousMmap holds the validity
@@ -144,11 +148,55 @@ impl RingBufUnregistered {
144148
145149 // SAFETY: Nobody else is modifying it.
146150 unsafe {
147- ( * shared_tail) . store ( 256 , Ordering :: Release ) ;
151+ ( * shared_tail) . store ( choice . total_bufs_count ( ) , Ordering :: Release ) ;
148152 }
149153
150- Ok ( Self { choice, ring_start } )
154+ let local_tail = Cell :: new ( choice. total_bufs_count ( ) ) ;
155+
156+ Ok ( Self {
157+ choice,
158+ ring_start,
159+ shared_tail,
160+ local_tail,
161+ base_ptr,
162+ } )
163+ }
164+ #[ inline]
165+ fn buf_ring_sync ( & self ) {
166+ unsafe {
167+ ( * self . shared_tail ) . store ( self . local_tail . get ( ) , Ordering :: Release ) ;
168+ }
169+ }
170+ /// Add buffer with id to the ring. Updates the tail
171+ #[ inline]
172+ pub unsafe fn dropping_bid ( & mut self , bid : u16 ) {
173+ self . add_bid ( bid) ;
174+ self . buf_ring_sync ( ) ;
175+ }
176+ /// Add buffer with id to the ring. Does not update the tail
177+ #[ inline]
178+ pub unsafe fn add_bid ( & mut self , bid : u16 ) -> ( ) {
179+ let old_tail = self . local_tail . get ( ) ;
180+ self . local_tail . set ( old_tail. wrapping_add ( 1 ) ) ;
181+ let ring_idx = old_tail & 255 ;
182+
183+ let entries = self . ring_start . as_ptr_mut ( ) as * mut BufRingEntry ;
184+
185+ // SAFETY: Our owned AnonymousMmap holds the validity
186+ let new_entry = unsafe { & mut * entries. add ( ring_idx as usize ) } ;
187+ // SAFETY: Our owned AnonymousMmap holds the validity
188+ let aligned_bid_ptr = unsafe { self . base_ptr . add ( ring_idx as usize * 8192 ) } ;
189+ new_entry. set_addr ( aligned_bid_ptr as _ ) ;
190+ new_entry. set_len ( 8192 as _ ) ;
191+ new_entry. set_bid ( bid) ;
192+ }
193+ /// The current tail of the bufring
194+ #[ inline]
195+ pub fn cur_tail ( & mut self ) -> u16 {
196+ unsafe { ( * self . shared_tail ) . load ( Ordering :: Acquire ) }
151197 }
198+ #[ inline]
199+ pub fn update_tail ( & mut self , idx : u16 ) -> ( ) { }
152200 /// Provides the ring start mut ptr e.g. using it to register it with io_uring.
153201 ///
154202 /// ## Safety
@@ -223,6 +271,14 @@ impl RingBufRegistered {
223271 Ok ( ( ) ) => Ok ( self . inner_ring ) ,
224272 }
225273 }
274+ #[ inline]
275+ pub fn cur_tail ( & mut self ) -> u16 {
276+ self . inner_ring . cur_tail ( )
277+ }
278+ #[ inline]
279+ pub unsafe fn dropping_bid ( & mut self , bid : u16 ) -> ( ) {
280+ self . inner_ring . dropping_bid ( bid)
281+ }
226282}
227283
228284#[ cfg( test) ]
0 commit comments