Skip to content

Commit f9f38df

Browse files
udesouudesou
committed
Changes to support mmtk
1 parent 3f11a82 commit f9f38df

11 files changed

+137
-105
lines changed

‎src/gc-common.c

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
#include "gc.h"
44

55
jl_gc_num_t gc_num = {0};
6+
gc_heapstatus_t gc_heap_stats = {0};
67
size_t last_long_collect_interval;
78
int gc_n_threads;
89
jl_ptls_t* gc_all_tls_states;
@@ -484,7 +485,43 @@ void jl_gc_count_allocd(size_t sz) JL_NOTSAFEPOINT
484485
jl_batch_accum_heap_size(ptls, sz);
485486
}
486487

488+
void jl_batch_accum_free_size(jl_ptls_t ptls, uint64_t sz) JL_NOTSAFEPOINT
489+
{
490+
jl_atomic_store_relaxed(&ptls->gc_tls.gc_num.free_acc, jl_atomic_load_relaxed(&ptls->gc_tls.gc_num.free_acc) + sz);
491+
}
492+
493+
void jl_gc_count_freed(size_t sz) JL_NOTSAFEPOINT
494+
{
495+
jl_batch_accum_free_size(jl_current_task->ptls, sz);
496+
}
487497

498+
void jl_gc_free_memory(jl_value_t *v, int isaligned) JL_NOTSAFEPOINT
499+
{
500+
assert(jl_is_genericmemory(v));
501+
jl_genericmemory_t *m = (jl_genericmemory_t*)v;
502+
assert(jl_genericmemory_how(m) == 1 || jl_genericmemory_how(m) == 2);
503+
char *d = (char*)m->ptr;
504+
if (isaligned)
505+
jl_free_aligned(d);
506+
else
507+
free(d);
508+
jl_atomic_store_relaxed(&gc_heap_stats.heap_size,
509+
jl_atomic_load_relaxed(&gc_heap_stats.heap_size) - jl_genericmemory_nbytes(m));
510+
gc_num.freed += jl_genericmemory_nbytes(m);
511+
gc_num.freecall++;
512+
}
513+
514+
void jl_free_thread_gc_state(jl_ptls_t ptls)
515+
{
516+
jl_gc_markqueue_t *mq = &ptls->gc_tls.mark_queue;
517+
ws_queue_t *cq = &mq->chunk_queue;
518+
free_ws_array(jl_atomic_load_relaxed(&cq->array));
519+
jl_atomic_store_relaxed(&cq->array, NULL);
520+
ws_queue_t *q = &mq->ptr_queue;
521+
free_ws_array(jl_atomic_load_relaxed(&q->array));
522+
jl_atomic_store_relaxed(&q->array, NULL);
523+
arraylist_free(&mq->reclaim_set);
524+
}
488525

489526
// GCNum, statistics manipulation
490527
// ---

‎src/gc-page-profiler.c

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
// This file is a part of Julia. License is MIT: https://julialang.org/license
22

3+
#ifndef MMTK_GC
4+
35
#include "gc-page-profiler.h"
46

57
#ifdef __cplusplus
@@ -177,3 +179,5 @@ JL_DLLEXPORT void jl_gc_take_page_profile(ios_t *stream)
177179
#ifdef __cplusplus
178180
}
179181
#endif
182+
183+
#endif // !MMTK_GC

‎src/gc-stacks.c

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -325,6 +325,8 @@ void sweep_stack_pools(void) JL_NOTSAFEPOINT
325325
}
326326
}
327327

328+
extern int gc_first_tid;
329+
328330
JL_DLLEXPORT jl_array_t *jl_live_tasks(void)
329331
{
330332
size_t nthreads = jl_atomic_load_acquire(&jl_n_threads);

‎src/gc.c

Lines changed: 3 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -193,8 +193,6 @@ static _Atomic(int) support_conservative_marking = 0;
193193
* have proper support of GC transition in codegen, we should execute the
194194
* finalizers in unmanaged (GC safe) mode.
195195
*/
196-
197-
gc_heapstatus_t gc_heap_stats = {0};
198196
int next_sweep_full = 0;
199197

200198
// List of marked big objects. Not per-thread. Accessed only by master thread.
@@ -600,10 +598,7 @@ STATIC_INLINE void jl_batch_accum_heap_size(jl_ptls_t ptls, uint64_t sz) JL_NOTS
600598
}
601599
}
602600

603-
STATIC_INLINE void jl_batch_accum_free_size(jl_ptls_t ptls, uint64_t sz) JL_NOTSAFEPOINT
604-
{
605-
jl_atomic_store_relaxed(&ptls->gc_tls.gc_num.free_acc, jl_atomic_load_relaxed(&ptls->gc_tls.gc_num.free_acc) + sz);
606-
}
601+
607602

608603
// big value list
609604

@@ -697,28 +692,7 @@ static void sweep_big(jl_ptls_t ptls, int sweep_full) JL_NOTSAFEPOINT
697692
}
698693

699694
// tracking Memorys with malloc'd storage
700-
void jl_gc_count_freed(size_t sz) JL_NOTSAFEPOINT
701-
{
702-
jl_batch_accum_free_size(jl_current_task->ptls, sz);
703-
}
704-
705-
706-
static void jl_gc_free_memory(jl_value_t *v, int isaligned) JL_NOTSAFEPOINT
707-
{
708-
assert(jl_is_genericmemory(v));
709-
jl_genericmemory_t *m = (jl_genericmemory_t*)v;
710-
assert(jl_genericmemory_how(m) == 1 || jl_genericmemory_how(m) == 2);
711-
char *d = (char*)m->ptr;
712-
if (isaligned)
713-
jl_free_aligned(d);
714-
else
715-
free(d);
716-
jl_atomic_store_relaxed(&gc_heap_stats.heap_size,
717-
jl_atomic_load_relaxed(&gc_heap_stats.heap_size) - jl_genericmemory_nbytes(m));
718-
gc_num.freed += jl_genericmemory_nbytes(m);
719-
gc_num.freecall++;
720-
}
721-
695+
extern void jl_gc_free_memory(jl_value_t *v, int isaligned);
722696
static void sweep_malloced_memory(void) JL_NOTSAFEPOINT
723697
{
724698
gc_time_mallocd_memory_start();
@@ -3380,18 +3354,6 @@ void jl_init_thread_heap(jl_ptls_t ptls)
33803354
jl_atomic_store_relaxed(&ptls->gc_tls.gc_num.allocd, -(int64_t)gc_num.interval);
33813355
}
33823356

3383-
void jl_free_thread_gc_state(jl_ptls_t ptls)
3384-
{
3385-
jl_gc_markqueue_t *mq = &ptls->gc_tls.mark_queue;
3386-
ws_queue_t *cq = &mq->chunk_queue;
3387-
free_ws_array(jl_atomic_load_relaxed(&cq->array));
3388-
jl_atomic_store_relaxed(&cq->array, NULL);
3389-
ws_queue_t *q = &mq->ptr_queue;
3390-
free_ws_array(jl_atomic_load_relaxed(&q->array));
3391-
jl_atomic_store_relaxed(&q->array, NULL);
3392-
arraylist_free(&mq->reclaim_set);
3393-
}
3394-
33953357
void jl_deinit_thread_heap(jl_ptls_t ptls)
33963358
{
33973359
// Do nothing
@@ -3478,6 +3440,7 @@ JL_DLLEXPORT void *jl_gc_counted_calloc(size_t nm, size_t sz)
34783440
return data;
34793441
}
34803442

3443+
extern void jl_batch_accum_free_size(jl_ptls_t ptls, uint64_t sz) JL_NOTSAFEPOINT;
34813444
JL_DLLEXPORT void jl_gc_counted_free_with_size(void *p, size_t sz)
34823445
{
34833446
jl_gcframe_t **pgcstack = jl_get_pgcstack();

‎src/gc.h

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,14 @@ extern uint64_t finalizer_rngState[];
7171
extern int gc_n_threads;
7272
extern jl_ptls_t* gc_all_tls_states;
7373

74+
#ifdef GC_SMALL_PAGE
75+
#define GC_PAGE_LG2 12 // log2(size of a page)
76+
#else
77+
#define GC_PAGE_LG2 14 // log2(size of a page)
78+
#endif
79+
#define GC_PAGE_SZ (1 << GC_PAGE_LG2)
80+
#define GC_PAGE_OFFSET (JL_HEAP_ALIGNMENT - (sizeof(jl_taggedvalue_t) % JL_HEAP_ALIGNMENT))
81+
7482
// This struct must be kept in sync with the Julia type of the same name in base/timing.jl
7583
typedef struct {
7684
int64_t allocd;
@@ -99,6 +107,13 @@ typedef struct {
99107
uint64_t last_incremental_sweep;
100108
} jl_gc_num_t;
101109

110+
typedef struct {
111+
_Atomic(size_t) bytes_mapped;
112+
_Atomic(size_t) bytes_resident;
113+
_Atomic(size_t) heap_size;
114+
_Atomic(size_t) heap_target;
115+
} gc_heapstatus_t;
116+
102117
extern jl_gc_num_t gc_num;
103118

104119
// data structure for tracking malloc'd arrays.
@@ -192,14 +207,6 @@ typedef struct {
192207
extern "C" {
193208
#endif
194209

195-
#ifdef GC_SMALL_PAGE
196-
#define GC_PAGE_LG2 12 // log2(size of a page)
197-
#else
198-
#define GC_PAGE_LG2 14 // log2(size of a page)
199-
#endif
200-
#define GC_PAGE_SZ (1 << GC_PAGE_LG2)
201-
#define GC_PAGE_OFFSET (JL_HEAP_ALIGNMENT - (sizeof(jl_taggedvalue_t) % JL_HEAP_ALIGNMENT))
202-
203210
#define jl_malloc_tag ((void*)0xdeadaa01)
204211
#define jl_singleton_tag ((void*)0xdeadaa02)
205212

@@ -428,13 +435,6 @@ typedef struct {
428435
pagetable1_t *meta1[REGION2_PG_COUNT];
429436
} pagetable_t;
430437

431-
typedef struct {
432-
_Atomic(size_t) bytes_mapped;
433-
_Atomic(size_t) bytes_resident;
434-
_Atomic(size_t) heap_size;
435-
_Atomic(size_t) heap_target;
436-
} gc_heapstatus_t;
437-
438438
#define GC_PAGE_UNMAPPED 0
439439
#define GC_PAGE_ALLOCATED 1
440440
#define GC_PAGE_LAZILY_FREED 2

‎src/julia.h

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -646,12 +646,11 @@ typedef struct _jl_binding_t {
646646
_Atomic(struct _jl_binding_t*) owner; // for individual imported bindings (NULL until 'resolved')
647647
_Atomic(jl_value_t*) ty; // binding type
648648
uint8_t constp:1;
649-
uint8_t exportp:1; // `public foo` sets `publicp`, `export foo` sets both `publicp` and `exportp`
650-
uint8_t publicp:1; // exportp without publicp is not allowed.
649+
uint8_t exportp:1;
651650
uint8_t imported:1;
652651
uint8_t usingfailed:1;
653652
uint8_t deprecated:2; // 0=not deprecated, 1=renamed, 2=moved to another package
654-
uint8_t padding:1;
653+
uint8_t padding:2;
655654
} jl_binding_t;
656655

657656
typedef struct {

‎src/llvm-final-gc-lowering.cpp

Lines changed: 10 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,7 @@ void FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
307307
builder.CreateStore(new_cursor, cursor_ptr);
308308

309309
// ptls->gc_num.allocd += osize;
310-
auto pool_alloc_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), offsetof(jl_tls_states_t, gc_num));
310+
auto pool_alloc_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), offsetof(jl_tls_states_t, gc_tls) + offsetof(jl_gc_tls_states_t, gc_num));
311311
auto pool_alloc_i8 = builder.CreateGEP(Type::getInt8Ty(target->getContext()), ptls, pool_alloc_pos);
312312
auto pool_alloc_tls = builder.CreateBitCast(pool_alloc_i8, PointerType::get(Type::getInt64Ty(target->getContext()), 0), "pool_alloc");
313313
auto pool_allocd = builder.CreateLoad(Type::getInt64Ty(target->getContext()), pool_alloc_tls);
@@ -322,11 +322,13 @@ void FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
322322
phiNode->addIncoming(v_as_ptr, fastpath);
323323
phiNode->takeName(target);
324324

325-
return phiNode;
325+
target->replaceAllUsesWith(phiNode);
326+
target->eraseFromParent();
327+
return;
326328
} else {
327329
auto pool_offs = ConstantInt::get(Type::getInt32Ty(F.getContext()), 1);
328-
newI = builder.CreateCall(poolAllocFunc, { ptls, pool_offs, pool_osize_i32 });
329-
derefAttr = Attribute::getWithDereferenceableBytes(F.getContext(), osize);
330+
newI = builder.CreateCall(poolAllocFunc, { ptls, pool_offs, pool_osize_i32, type });
331+
derefBytes = sizeof(void*);
330332
}
331333
#endif // MMTK_GC
332334
}
@@ -368,13 +370,6 @@ bool FinalLowerGC::runOnFunction(Function &F)
368370
allocTypedFunc = getOrDeclare(jl_well_known::GCAllocTyped);
369371
T_size = F.getParent()->getDataLayout().getIntPtrType(F.getContext());
370372

371-
#ifdef MMTK_GC
372-
auto writeBarrier1Func = getOrNull(jl_intrinsics::writeBarrier1);
373-
auto writeBarrier2Func = getOrNull(jl_intrinsics::writeBarrier2);
374-
auto writeBarrier1SlowFunc = getOrNull(jl_intrinsics::writeBarrier1Slow);
375-
auto writeBarrier2SlowFunc = getOrNull(jl_intrinsics::writeBarrier2Slow);
376-
#endif
377-
378373
// Lower all calls to supported intrinsics.
379374
for (auto &BB : F) {
380375
for (auto &I : make_early_inc_range(BB)) {
@@ -403,10 +398,10 @@ bool FinalLowerGC::runOnFunction(Function &F)
403398

404399

405400
#ifdef MMTK_GC
406-
LOWER_INTRINSIC(writeBarrier1Func, lowerWriteBarrier1);
407-
LOWER_INTRINSIC(writeBarrier2Func, lowerWriteBarrier2);
408-
LOWER_INTRINSIC(writeBarrier1SlowFunc, lowerWriteBarrier1Slow);
409-
LOWER_INTRINSIC(writeBarrier2SlowFunc, lowerWriteBarrier2Slow);
401+
LOWER_INTRINSIC(writeBarrier1, lowerWriteBarrier1);
402+
LOWER_INTRINSIC(writeBarrier2, lowerWriteBarrier2);
403+
LOWER_INTRINSIC(writeBarrier1Slow, lowerWriteBarrier1Slow);
404+
LOWER_INTRINSIC(writeBarrier2Slow, lowerWriteBarrier2Slow);
410405
#endif
411406

412407

‎src/llvm-late-gc-lowering.cpp

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2575,15 +2575,15 @@ bool LateLowerGCFrame::CleanupIR(Function &F, State *S, bool *CFGModified) {
25752575
if (CFGModified) {
25762576
*CFGModified = true;
25772577
}
2578+
2579+
IRBuilder<> builder(CI);
2580+
builder.SetCurrentDebugLocation(CI->getDebugLoc());
2581+
#ifndef MMTK_GC
25782582
auto DebugInfoMeta = F.getParent()->getModuleFlag("julia.debug_level");
25792583
int debug_info = 1;
25802584
if (DebugInfoMeta != nullptr) {
25812585
debug_info = cast<ConstantInt>(cast<ConstantAsMetadata>(DebugInfoMeta)->getValue())->getZExtValue();
25822586
}
2583-
2584-
IRBuilder<> builder(CI);
2585-
builder.SetCurrentDebugLocation(CI->getDebugLoc());
2586-
#ifndef MMTK_GC
25872587
auto parBits = builder.CreateAnd(EmitLoadTag(builder, T_size, parent), GC_OLD_MARKED);
25882588
setName(parBits, "parent_bits", debug_info);
25892589
auto parOldMarked = builder.CreateICmpEQ(parBits, ConstantInt::get(T_size, GC_OLD_MARKED));

‎src/llvm-pass-helpers.cpp

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -275,7 +275,11 @@ namespace jl_intrinsics {
275275
false),
276276
Function::ExternalLinkage,
277277
WRITE_BARRIER_1_NAME);
278+
#if JL_LLVM_VERSION >= 160000
279+
intrinsic->setMemoryEffects(MemoryEffects::inaccessibleOrArgMemOnly());
280+
#else
278281
intrinsic->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
282+
#endif
279283
return intrinsic;
280284
});
281285
const IntrinsicDescription writeBarrier2(
@@ -290,7 +294,11 @@ namespace jl_intrinsics {
290294
false),
291295
Function::ExternalLinkage,
292296
WRITE_BARRIER_2_NAME);
297+
#if JL_LLVM_VERSION >= 160000
298+
intrinsic->setMemoryEffects(MemoryEffects::inaccessibleOrArgMemOnly());
299+
#else
293300
intrinsic->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
301+
#endif
294302
return intrinsic;
295303
});
296304
const IntrinsicDescription writeBarrier1Slow(
@@ -305,7 +313,11 @@ namespace jl_intrinsics {
305313
false),
306314
Function::ExternalLinkage,
307315
WRITE_BARRIER_1_SLOW_NAME);
316+
#if JL_LLVM_VERSION >= 160000
317+
intrinsic->setMemoryEffects(MemoryEffects::inaccessibleOrArgMemOnly());
318+
#else
308319
intrinsic->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
320+
#endif
309321
return intrinsic;
310322
});
311323
const IntrinsicDescription writeBarrier2Slow(
@@ -320,7 +332,11 @@ namespace jl_intrinsics {
320332
false),
321333
Function::ExternalLinkage,
322334
WRITE_BARRIER_2_SLOW_NAME);
335+
#if JL_LLVM_VERSION >= 160000
336+
intrinsic->setMemoryEffects(MemoryEffects::inaccessibleOrArgMemOnly());
337+
#else
323338
intrinsic->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
339+
#endif
324340
return intrinsic;
325341
});
326342
#endif
@@ -423,7 +439,11 @@ namespace jl_well_known {
423439
false),
424440
Function::ExternalLinkage,
425441
GC_WB_1_NAME);
442+
#if JL_LLVM_VERSION >= 160000
443+
func->setMemoryEffects(MemoryEffects::inaccessibleOrArgMemOnly());
444+
#else
426445
func->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
446+
#endif
427447
return func;
428448
});
429449

@@ -439,7 +459,11 @@ namespace jl_well_known {
439459
false),
440460
Function::ExternalLinkage,
441461
GC_WB_2_NAME);
462+
#if JL_LLVM_VERSION >= 160000
463+
func->setMemoryEffects(MemoryEffects::inaccessibleOrArgMemOnly());
464+
#else
442465
func->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
466+
#endif
443467
return func;
444468
});
445469

@@ -455,7 +479,11 @@ namespace jl_well_known {
455479
false),
456480
Function::ExternalLinkage,
457481
GC_WB_1_SLOW_NAME);
482+
#if JL_LLVM_VERSION >= 160000
483+
func->setMemoryEffects(MemoryEffects::inaccessibleOrArgMemOnly());
484+
#else
458485
func->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
486+
#endif
459487
return func;
460488
});
461489

@@ -471,7 +499,11 @@ namespace jl_well_known {
471499
false),
472500
Function::ExternalLinkage,
473501
GC_WB_2_SLOW_NAME);
502+
#if JL_LLVM_VERSION >= 160000
503+
func->setMemoryEffects(MemoryEffects::inaccessibleOrArgMemOnly());
504+
#else
474505
func->addFnAttr(Attribute::InaccessibleMemOrArgMemOnly);
506+
#endif
475507
return func;
476508
});
477509
#endif

0 commit comments

Comments
 (0)