@@ -32,118 +32,153 @@ int mca_btl_vader_xpmem_init (void)
3232 return OPAL_SUCCESS ;
3333}
3434
35+ struct vader_check_reg_ctx_t {
36+ mca_rcache_base_vma_module_t * vma_module ;
37+ mca_btl_base_endpoint_t * ep ;
38+ mca_rcache_base_registration_t * * reg ;
39+ uintptr_t base ;
40+ uintptr_t bound ;
41+ };
42+ typedef struct vader_check_reg_ctx_t vader_check_reg_ctx_t ;
43+
44+ static int vader_check_reg (mca_rcache_base_registration_t * reg , void * ctx )
45+ {
46+ vader_check_reg_ctx_t * vader_ctx = (vader_check_reg_ctx_t * ) ctx ;
47+
48+ if ((intptr_t ) reg -> alloc_base != vader_ctx -> ep -> peer_smp_rank ||
49+ (reg -> flags & MCA_RCACHE_FLAGS_PERSIST )) {
50+ /* ignore this registration */
51+ return OPAL_SUCCESS ;
52+ }
53+
54+ vader_ctx -> reg [0 ] = reg ;
55+
56+ if (vader_ctx -> bound <= (uintptr_t ) reg -> bound && vader_ctx -> base >= (uintptr_t ) reg -> base ) {
57+ (void )opal_atomic_add (& reg -> ref_count , 1 );
58+ return 1 ;
59+ }
60+
61+ /* remove this pointer from the rcache and decrement its reference count
62+ (so it is detached later) */
63+ mca_rcache_base_vma_delete (vader_ctx -> vma_module , reg );
64+
65+ return 2 ;
66+ }
67+
3568/* look up the remote pointer in the peer rcache and attach if
3669 * necessary */
3770mca_rcache_base_registration_t * vader_get_registation (struct mca_btl_base_endpoint_t * ep , void * rem_ptr ,
3871 size_t size , int flags , void * * local_ptr )
3972{
40- mca_rcache_base_vma_module_t * vma_module = ep -> segment_data .xpmem .vma_module ;
41- mca_rcache_base_registration_t * regs [10 ], * reg = NULL ;
73+ mca_rcache_base_vma_module_t * vma_module = mca_btl_vader_component .vma_module ;
74+ uint64_t attach_align = 1 << mca_btl_vader_component .log_attach_align ;
75+ mca_rcache_base_registration_t * reg = NULL ;
76+ vader_check_reg_ctx_t check_ctx = {.ep = ep , .reg = & reg , .vma_module = vma_module };
4277 xpmem_addr_t xpmem_addr ;
4378 uintptr_t base , bound ;
44- uint64_t attach_align = 1 << mca_btl_vader_component .log_attach_align ;
4579 int rc , i ;
4680
47- /* protect rcache access */
48- OPAL_THREAD_LOCK (& ep -> lock );
49-
50- /* use btl/self for self communication */
51- assert (ep -> peer_smp_rank != MCA_BTL_VADER_LOCAL_RANK );
52-
5381 base = OPAL_DOWN_ALIGN ((uintptr_t ) rem_ptr , attach_align , uintptr_t );
5482 bound = OPAL_ALIGN ((uintptr_t ) rem_ptr + size - 1 , attach_align , uintptr_t ) + 1 ;
5583 if (OPAL_UNLIKELY (bound > VADER_MAX_ADDRESS )) {
5684 bound = VADER_MAX_ADDRESS ;
5785 }
5886
59- /* several segments may match the base pointer */
60- rc = mca_rcache_base_vma_find_all (vma_module , (void * ) base , bound - base , regs , 10 );
61- for (i = 0 ; i < rc ; ++ i ) {
62- if (bound <= (uintptr_t )regs [i ]-> bound && base >= (uintptr_t )regs [i ]-> base ) {
63- (void )opal_atomic_add (& regs [i ]-> ref_count , 1 );
64- reg = regs [i ];
65- goto reg_found ;
66- }
67-
68- if (regs [i ]-> flags & MCA_RCACHE_FLAGS_PERSIST ) {
69- continue ;
70- }
71-
72- /* remove this pointer from the rcache and decrement its reference count
73- (so it is detached later) */
74- rc = mca_rcache_base_vma_delete (vma_module , regs [i ]);
75- if (OPAL_UNLIKELY (0 != rc )) {
76- /* someone beat us to it? */
77- break ;
78- }
87+ check_ctx .base = base ;
88+ check_ctx .bound = bound ;
7989
90+ /* several segments may match the base pointer */
91+ rc = mca_rcache_base_vma_iterate (vma_module , (void * ) base , bound - base , vader_check_reg , & check_ctx );
92+ if (2 == rc ) {
8093 /* start the new segment from the lower of the two bases */
81- base = (uintptr_t ) regs [i ]-> base < base ? (uintptr_t ) regs [i ]-> base : base ;
82-
83- (void )opal_atomic_add (& regs [i ]-> ref_count , -1 );
94+ base = (uintptr_t ) reg -> base < base ? (uintptr_t ) reg -> base : base ;
8495
85- if (OPAL_LIKELY (0 == regs [ i ] -> ref_count )) {
96+ if (OPAL_LIKELY (0 == opal_atomic_add_32 ( & reg -> ref_count , -1 ) )) {
8697 /* this pointer is not in use */
87- (void ) xpmem_detach (regs [ i ] -> rcache_context );
88- OBJ_RELEASE (regs [ i ] );
98+ (void ) xpmem_detach (reg -> rcache_context );
99+ OBJ_RELEASE (reg );
89100 }
90101
91- break ;
102+ reg = NULL ;
92103 }
93104
94- reg = OBJ_NEW (mca_rcache_base_registration_t );
95- if (OPAL_LIKELY (NULL != reg )) {
96- /* stick around for awhile */
97- reg -> ref_count = 2 ;
98- reg -> base = (unsigned char * ) base ;
99- reg -> bound = (unsigned char * ) bound ;
100- reg -> flags = flags ;
105+ if (NULL == reg ) {
106+ reg = OBJ_NEW (mca_rcache_base_registration_t );
107+ if (OPAL_LIKELY (NULL != reg )) {
108+ /* stick around for awhile */
109+ reg -> ref_count = 2 ;
110+ reg -> base = (unsigned char * ) base ;
111+ reg -> bound = (unsigned char * ) bound ;
112+ reg -> flags = flags ;
113+ reg -> alloc_base = (void * ) (intptr_t ) ep -> peer_smp_rank ;
101114
102115#if defined(HAVE_SN_XPMEM_H )
103- xpmem_addr .id = ep -> segment_data .xpmem .apid ;
116+ xpmem_addr .id = ep -> segment_data .xpmem .apid ;
104117#else
105- xpmem_addr .apid = ep -> segment_data .xpmem .apid ;
118+ xpmem_addr .apid = ep -> segment_data .xpmem .apid ;
106119#endif
107- xpmem_addr .offset = base ;
120+ xpmem_addr .offset = base ;
108121
109- reg -> rcache_context = xpmem_attach (xpmem_addr , bound - base , NULL );
110- if (OPAL_UNLIKELY ((void * )-1 == reg -> rcache_context )) {
111- OPAL_THREAD_UNLOCK (& ep -> lock );
112- OBJ_RELEASE (reg );
113- return NULL ;
114- }
122+ reg -> rcache_context = xpmem_attach (xpmem_addr , bound - base , NULL );
123+ if (OPAL_UNLIKELY ((void * )-1 == reg -> rcache_context )) {
124+ OBJ_RELEASE (reg );
125+ return NULL ;
126+ }
115127
116- opal_memchecker_base_mem_defined (reg -> rcache_context , bound - base );
128+ opal_memchecker_base_mem_defined (reg -> rcache_context , bound - base );
117129
118- mca_rcache_base_vma_insert (vma_module , reg , 0 );
130+ mca_rcache_base_vma_insert (vma_module , reg , 0 );
131+ }
119132 }
120133
121- reg_found :
122134 opal_atomic_wmb ();
123135 * local_ptr = (void * ) ((uintptr_t ) reg -> rcache_context +
124136 (ptrdiff_t )((uintptr_t ) rem_ptr - (uintptr_t ) reg -> base ));
125137
126- OPAL_THREAD_UNLOCK (& ep -> lock );
127-
128138 return reg ;
129139}
130140
131141void vader_return_registration (mca_rcache_base_registration_t * reg , struct mca_btl_base_endpoint_t * ep )
132142{
133- mca_rcache_base_vma_module_t * vma_module = ep -> segment_data . xpmem .vma_module ;
143+ mca_rcache_base_vma_module_t * vma_module = mca_btl_vader_component .vma_module ;
134144 int32_t ref_count ;
135145
136146 ref_count = opal_atomic_add_32 (& reg -> ref_count , -1 );
137147 if (OPAL_UNLIKELY (0 == ref_count && !(reg -> flags & MCA_RCACHE_FLAGS_PERSIST ))) {
138148 /* protect rcache access */
139- OPAL_THREAD_LOCK (& ep -> lock );
140149 mca_rcache_base_vma_delete (vma_module , reg );
141- OPAL_THREAD_UNLOCK (& ep -> lock );
142150
143151 opal_memchecker_base_mem_noaccess (reg -> rcache_context , (uintptr_t )(reg -> bound - reg -> base ));
144152 (void )xpmem_detach (reg -> rcache_context );
145153 OBJ_RELEASE (reg );
146154 }
147155}
148156
157+ static int mca_btl_vader_endpoint_xpmem_rcache_cleanup (mca_rcache_base_registration_t * reg , void * ctx )
158+ {
159+ mca_rcache_base_vma_module_t * vma_module = mca_btl_vader_component .vma_module ;
160+ mca_btl_vader_endpoint_t * ep = (mca_btl_vader_endpoint_t * ) ctx ;
161+ if ((intptr_t ) reg -> alloc_base == ep -> peer_smp_rank ) {
162+ /* otherwise dereg will fail on assert */
163+ reg -> ref_count = 0 ;
164+ (void ) mca_rcache_base_vma_delete (vma_module , reg );
165+ OBJ_RELEASE (reg );
166+ }
167+
168+ return OPAL_SUCCESS ;
169+ }
170+
171+ void mca_btl_vader_xpmem_cleanup_endpoint (struct mca_btl_base_endpoint_t * ep )
172+ {
173+ /* clean out the registration cache */
174+ (void ) mca_rcache_base_vma_iterate (mca_btl_vader_component .vma_module ,
175+ NULL , (size_t ) -1 ,
176+ mca_btl_vader_endpoint_xpmem_rcache_cleanup ,
177+ (void * ) ep );
178+ if (ep -> segment_base ) {
179+ xpmem_release (ep -> segment_data .xpmem .apid );
180+ ep -> segment_data .xpmem .apid = 0 ;
181+ }
182+ }
183+
149184#endif /* OPAL_BTL_VADER_HAVE_XPMEM */
0 commit comments