Skip to content

Commit 73d3d58

Browse files
jasonejasone
committed
Optimize witness fast path.
Short-circuit commonly called witness functions so that they only execute in debug builds, and remove equivalent guards from mutex functions. This avoids pointless code execution in witness_assert_lockless(), which is typically called twice per allocation/deallocation function invocation. Inline commonly called witness functions so that optimized builds can completely remove calls as dead code.
1 parent 7790a0b commit 73d3d58

File tree

4 files changed

+157
-132
lines changed

4 files changed

+157
-132
lines changed

‎include/jemalloc/internal/jemalloc_internal.h.in‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -531,9 +531,9 @@ void jemalloc_postfork_child(void);
531531
#include "jemalloc/internal/smoothstep.h"
532532
#include "jemalloc/internal/stats.h"
533533
#include "jemalloc/internal/ctl.h"
534+
#include "jemalloc/internal/tsd.h"
534535
#include "jemalloc/internal/witness.h"
535536
#include "jemalloc/internal/mutex.h"
536-
#include "jemalloc/internal/tsd.h"
537537
#include "jemalloc/internal/mb.h"
538538
#include "jemalloc/internal/extent.h"
539539
#include "jemalloc/internal/base.h"

‎include/jemalloc/internal/mutex.h‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -81,8 +81,7 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex)
8181
{
8282

8383
if (isthreaded) {
84-
if (config_debug)
85-
witness_assert_not_owner(tsdn, &mutex->witness);
84+
witness_assert_not_owner(tsdn, &mutex->witness);
8685
#ifdef _WIN32
8786
# if _WIN32_WINNT >= 0x0600
8887
AcquireSRWLockExclusive(&mutex->lock);
@@ -94,8 +93,7 @@ malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex)
9493
#else
9594
pthread_mutex_lock(&mutex->lock);
9695
#endif
97-
if (config_debug)
98-
witness_lock(tsdn, &mutex->witness);
96+
witness_lock(tsdn, &mutex->witness);
9997
}
10098
}
10199

@@ -104,8 +102,7 @@ malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex)
104102
{
105103

106104
if (isthreaded) {
107-
if (config_debug)
108-
witness_unlock(tsdn, &mutex->witness);
105+
witness_unlock(tsdn, &mutex->witness);
109106
#ifdef _WIN32
110107
# if _WIN32_WINNT >= 0x0600
111108
ReleaseSRWLockExclusive(&mutex->lock);
@@ -124,15 +121,15 @@ JEMALLOC_INLINE void
124121
malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex)
125122
{
126123

127-
if (isthreaded && config_debug)
124+
if (isthreaded)
128125
witness_assert_owner(tsdn, &mutex->witness);
129126
}
130127

131128
JEMALLOC_INLINE void
132129
malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex)
133130
{
134131

135-
if (isthreaded && config_debug)
132+
if (isthreaded)
136133
witness_assert_not_owner(tsdn, &mutex->witness);
137134
}
138135
#endif

‎include/jemalloc/internal/witness.h‎

Lines changed: 147 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -74,24 +74,28 @@ void witness_init(witness_t *witness, const char *name, witness_rank_t rank,
7474
#ifdef JEMALLOC_JET
7575
typedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *);
7676
extern witness_lock_error_t *witness_lock_error;
77+
#else
78+
void witness_lock_error(const witness_list_t *witnesses,
79+
const witness_t *witness);
7780
#endif
78-
void witness_lock(tsdn_t *tsdn, witness_t *witness);
79-
void witness_unlock(tsdn_t *tsdn, witness_t *witness);
8081
#ifdef JEMALLOC_JET
8182
typedef void (witness_owner_error_t)(const witness_t *);
8283
extern witness_owner_error_t *witness_owner_error;
84+
#else
85+
void witness_owner_error(const witness_t *witness);
8386
#endif
84-
void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness);
8587
#ifdef JEMALLOC_JET
8688
typedef void (witness_not_owner_error_t)(const witness_t *);
8789
extern witness_not_owner_error_t *witness_not_owner_error;
90+
#else
91+
void witness_not_owner_error(const witness_t *witness);
8892
#endif
89-
void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness);
9093
#ifdef JEMALLOC_JET
9194
typedef void (witness_lockless_error_t)(const witness_list_t *);
9295
extern witness_lockless_error_t *witness_lockless_error;
96+
#else
97+
void witness_lockless_error(const witness_list_t *witnesses);
9398
#endif
94-
void witness_assert_lockless(tsdn_t *tsdn);
9599

96100
void witnesses_cleanup(tsd_t *tsd);
97101
void witness_fork_cleanup(tsd_t *tsd);
@@ -103,5 +107,143 @@ void witness_postfork_child(tsd_t *tsd);
103107
/******************************************************************************/
104108
#ifdef JEMALLOC_H_INLINES
105109

110+
#ifndef JEMALLOC_ENABLE_INLINE
111+
void witness_assert_owner(tsdn_t *tsdn, const witness_t *witness);
112+
void witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness);
113+
void witness_assert_lockless(tsdn_t *tsdn);
114+
void witness_lock(tsdn_t *tsdn, witness_t *witness);
115+
void witness_unlock(tsdn_t *tsdn, witness_t *witness);
116+
#endif
117+
118+
#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_))
119+
JEMALLOC_INLINE void
120+
witness_assert_owner(tsdn_t *tsdn, const witness_t *witness)
121+
{
122+
tsd_t *tsd;
123+
witness_list_t *witnesses;
124+
witness_t *w;
125+
126+
if (!config_debug)
127+
return;
128+
129+
if (tsdn_null(tsdn))
130+
return;
131+
tsd = tsdn_tsd(tsdn);
132+
if (witness->rank == WITNESS_RANK_OMIT)
133+
return;
134+
135+
witnesses = tsd_witnessesp_get(tsd);
136+
ql_foreach(w, witnesses, link) {
137+
if (w == witness)
138+
return;
139+
}
140+
witness_owner_error(witness);
141+
}
142+
143+
JEMALLOC_INLINE void
144+
witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness)
145+
{
146+
tsd_t *tsd;
147+
witness_list_t *witnesses;
148+
witness_t *w;
149+
150+
if (!config_debug)
151+
return;
152+
153+
if (tsdn_null(tsdn))
154+
return;
155+
tsd = tsdn_tsd(tsdn);
156+
if (witness->rank == WITNESS_RANK_OMIT)
157+
return;
158+
159+
witnesses = tsd_witnessesp_get(tsd);
160+
ql_foreach(w, witnesses, link) {
161+
if (w == witness)
162+
witness_not_owner_error(witness);
163+
}
164+
}
165+
166+
JEMALLOC_INLINE void
167+
witness_assert_lockless(tsdn_t *tsdn)
168+
{
169+
tsd_t *tsd;
170+
witness_list_t *witnesses;
171+
witness_t *w;
172+
173+
if (!config_debug)
174+
return;
175+
176+
if (tsdn_null(tsdn))
177+
return;
178+
tsd = tsdn_tsd(tsdn);
179+
180+
witnesses = tsd_witnessesp_get(tsd);
181+
w = ql_last(witnesses, link);
182+
if (w != NULL)
183+
witness_lockless_error(witnesses);
184+
}
185+
186+
JEMALLOC_INLINE void
187+
witness_lock(tsdn_t *tsdn, witness_t *witness)
188+
{
189+
tsd_t *tsd;
190+
witness_list_t *witnesses;
191+
witness_t *w;
192+
193+
if (!config_debug)
194+
return;
195+
196+
if (tsdn_null(tsdn))
197+
return;
198+
tsd = tsdn_tsd(tsdn);
199+
if (witness->rank == WITNESS_RANK_OMIT)
200+
return;
201+
202+
witness_assert_not_owner(tsdn, witness);
203+
204+
witnesses = tsd_witnessesp_get(tsd);
205+
w = ql_last(witnesses, link);
206+
if (w == NULL) {
207+
/* No other locks; do nothing. */
208+
} else if (tsd_witness_fork_get(tsd) && w->rank <= witness->rank) {
209+
/* Forking, and relaxed ranking satisfied. */
210+
} else if (w->rank > witness->rank) {
211+
/* Not forking, rank order reversal. */
212+
witness_lock_error(witnesses, witness);
213+
} else if (w->rank == witness->rank && (w->comp == NULL || w->comp !=
214+
witness->comp || w->comp(w, witness) > 0)) {
215+
/*
216+
* Missing/incompatible comparison function, or comparison
217+
* function indicates rank order reversal.
218+
*/
219+
witness_lock_error(witnesses, witness);
220+
}
221+
222+
ql_elm_new(witness, link);
223+
ql_tail_insert(witnesses, witness, link);
224+
}
225+
226+
JEMALLOC_INLINE void
227+
witness_unlock(tsdn_t *tsdn, witness_t *witness)
228+
{
229+
tsd_t *tsd;
230+
witness_list_t *witnesses;
231+
232+
if (!config_debug)
233+
return;
234+
235+
if (tsdn_null(tsdn))
236+
return;
237+
tsd = tsdn_tsd(tsdn);
238+
if (witness->rank == WITNESS_RANK_OMIT)
239+
return;
240+
241+
witness_assert_owner(tsdn, witness);
242+
243+
witnesses = tsd_witnessesp_get(tsd);
244+
ql_remove(witnesses, witness, link);
245+
}
246+
#endif
247+
106248
#endif /* JEMALLOC_H_INLINES */
107249
/******************************************************************************/

0 commit comments

Comments
 (0)