From 3db5c6dd6bc501218b6ba1f3ef081d95a45ecb6b Mon Sep 17 00:00:00 2001 From: Ilija Tovilo Date: Fri, 29 Aug 2025 00:09:30 +0200 Subject: [PATCH 1/3] GC optimization for primitive arrays PoC for GH-19608 --- Zend/zend_gc.c | 32 ++++++++++++++++++++++++++++++-- Zend/zend_hash.c | 3 +++ Zend/zend_hash.h | 3 +++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/Zend/zend_gc.c b/Zend/zend_gc.c index e15f97ecfe802..a0a4508825e5b 100644 --- a/Zend/zend_gc.c +++ b/Zend/zend_gc.c @@ -983,6 +983,9 @@ static void gc_scan_black(zend_refcounted *ref, gc_stack *stack) handle_ht: n = ht->nNumUsed; zv = ht->arPacked; + if (GC_FLAGS(ht) & GC_NOT_COLLECTABLE) { + goto next; + } if (HT_IS_PACKED(ht)) { goto handle_zvals; } @@ -1041,7 +1044,7 @@ static void gc_scan_black(zend_refcounted *ref, gc_stack *stack) * counts and mark visited nodes grey. See MarkGray() in Bacon & Rajan. */ static void gc_mark_grey(zend_refcounted *ref, gc_stack *stack) { - HashTable *ht; + HashTable *ht, *zvals_ht = NULL; Bucket *p; zval *zv; uint32_t n; @@ -1129,9 +1132,11 @@ static void gc_mark_grey(zend_refcounted *ref, gc_stack *stack) goto handle_ht; } } -handle_zvals: +handle_zvals:; + bool is_primitive = true; for (; n != 0; n--) { if (Z_COLLECTABLE_P(zv)) { + is_primitive = false; ref = Z_COUNTED_P(zv); GC_DELREF(ref); if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) { @@ -1148,22 +1153,32 @@ static void gc_mark_grey(zend_refcounted *ref, gc_stack *stack) } zv++; } + zvals_ht = NULL; goto tail_call; } } zv++; } + if (is_primitive && zvals_ht) { + GC_ADD_FLAGS(zvals_ht, GC_NOT_COLLECTABLE); + } + zvals_ht = NULL; } } else if (GC_TYPE(ref) == IS_ARRAY) { ZEND_ASSERT(((zend_array*)ref) != &EG(symbol_table)); ht = (zend_array*)ref; handle_ht: n = ht->nNumUsed; + if (GC_FLAGS(ht) & GC_NOT_COLLECTABLE) { + goto next; + } if (HT_IS_PACKED(ht)) { zv = ht->arPacked; + zvals_ht = ht; goto handle_zvals; } + bool is_primitive = true; p = ht->arData; for (; n != 0; n--) { zv = &p->val; @@ -1171,6 +1186,7 @@ static void gc_mark_grey(zend_refcounted *ref, gc_stack *stack) zv = Z_INDIRECT_P(zv); } if (Z_COLLECTABLE_P(zv)) { + is_primitive = false; ref = Z_COUNTED_P(zv); GC_DELREF(ref); if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) { @@ -1196,6 +1212,9 @@ static void gc_mark_grey(zend_refcounted *ref, gc_stack *stack) } p++; } + if (is_primitive) { + GC_ADD_FLAGS(ht, GC_NOT_COLLECTABLE); + } } else if (GC_TYPE(ref) == IS_REFERENCE) { if (Z_COLLECTABLE(((zend_reference*)ref)->val)) { ref = Z_COUNTED(((zend_reference*)ref)->val); @@ -1378,6 +1397,9 @@ static void gc_scan(zend_refcounted *ref, gc_stack *stack) handle_ht: n = ht->nNumUsed; + if (GC_FLAGS(ht) & GC_NOT_COLLECTABLE) { + goto next; + } if (HT_IS_PACKED(ht)) { zv = ht->arPacked; goto handle_zvals; @@ -1623,6 +1645,9 @@ static int gc_collect_white(zend_refcounted *ref, uint32_t *flags, gc_stack *sta handle_ht: n = ht->nNumUsed; + if (GC_FLAGS(ht) & GC_NOT_COLLECTABLE) { + goto next; + } if (HT_IS_PACKED(ht)) { zv = ht->arPacked; goto handle_zvals; @@ -1811,6 +1836,9 @@ static int gc_remove_nested_data_from_buffer(zend_refcounted *ref, gc_root_buffe handle_ht: n = ht->nNumUsed; + if (GC_FLAGS(ht) & GC_NOT_COLLECTABLE) { + goto next; + } if (HT_IS_PACKED(ht)) { zv = ht->arPacked; goto handle_zvals; diff --git a/Zend/zend_hash.c b/Zend/zend_hash.c index f48c298f167e7..3111cda233cde 100644 --- a/Zend/zend_hash.c +++ b/Zend/zend_hash.c @@ -885,6 +885,7 @@ static zend_always_inline zval *_zend_hash_add_or_update_i(HashTable *ht, zend_s nIndex = h | ht->nTableMask; Z_NEXT(p->val) = HT_HASH_EX(arData, nIndex); HT_HASH_EX(arData, nIndex) = HT_IDX_TO_HASH(idx); + GC_DEL_FLAGS(ht, GC_NOT_COLLECTABLE); if (flag & HASH_LOOKUP) { ZVAL_NULL(&p->val); } else { @@ -970,6 +971,7 @@ static zend_always_inline zval *_zend_hash_str_add_or_update_i(HashTable *ht, co nIndex = h | ht->nTableMask; Z_NEXT(p->val) = HT_HASH(ht, nIndex); HT_HASH(ht, nIndex) = HT_IDX_TO_HASH(idx); + GC_DEL_FLAGS(ht, GC_NOT_COLLECTABLE); return &p->val; } @@ -1172,6 +1174,7 @@ static zend_always_inline zval *_zend_hash_index_add_or_update_i(HashTable *ht, p = ht->arData + idx; Z_NEXT(p->val) = HT_HASH(ht, nIndex); HT_HASH(ht, nIndex) = HT_IDX_TO_HASH(idx); + GC_DEL_FLAGS(ht, GC_NOT_COLLECTABLE); if ((zend_long)h >= ht->nNextFreeElement) { ht->nNextFreeElement = (zend_long)h < ZEND_LONG_MAX ? h + 1 : ZEND_LONG_MAX; } diff --git a/Zend/zend_hash.h b/Zend/zend_hash.h index 71206e61550bb..cd48c88e7d12e 100644 --- a/Zend/zend_hash.h +++ b/Zend/zend_hash.h @@ -1638,6 +1638,7 @@ static zend_always_inline zval *_zend_hash_append_ex(HashTable *ht, zend_string nIndex = (uint32_t)p->h | ht->nTableMask; Z_NEXT(p->val) = HT_HASH(ht, nIndex); HT_HASH(ht, nIndex) = HT_IDX_TO_HASH(idx); + GC_DEL_FLAGS(ht, GC_NOT_COLLECTABLE); ht->nNumOfElements++; return &p->val; } @@ -1664,6 +1665,7 @@ static zend_always_inline zval *_zend_hash_append_ptr_ex(HashTable *ht, zend_str nIndex = (uint32_t)p->h | ht->nTableMask; Z_NEXT(p->val) = HT_HASH(ht, nIndex); HT_HASH(ht, nIndex) = HT_IDX_TO_HASH(idx); + GC_DEL_FLAGS(ht, GC_NOT_COLLECTABLE); ht->nNumOfElements++; return &p->val; } @@ -1690,6 +1692,7 @@ static zend_always_inline void _zend_hash_append_ind(HashTable *ht, zend_string nIndex = (uint32_t)p->h | ht->nTableMask; Z_NEXT(p->val) = HT_HASH(ht, nIndex); HT_HASH(ht, nIndex) = HT_IDX_TO_HASH(idx); + GC_DEL_FLAGS(ht, GC_NOT_COLLECTABLE); ht->nNumOfElements++; } From 1ae410e2c10562d7fe29b764b3e74dc0301f7ed0 Mon Sep 17 00:00:00 2001 From: Ilija Tovilo Date: Fri, 29 Aug 2025 00:17:11 +0200 Subject: [PATCH 2/3] Simplify --- Zend/zend_gc.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Zend/zend_gc.c b/Zend/zend_gc.c index a0a4508825e5b..6c022073c23c8 100644 --- a/Zend/zend_gc.c +++ b/Zend/zend_gc.c @@ -1132,11 +1132,10 @@ static void gc_mark_grey(zend_refcounted *ref, gc_stack *stack) goto handle_ht; } } -handle_zvals:; - bool is_primitive = true; +handle_zvals: for (; n != 0; n--) { if (Z_COLLECTABLE_P(zv)) { - is_primitive = false; + zvals_ht = NULL; ref = Z_COUNTED_P(zv); GC_DELREF(ref); if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) { @@ -1153,16 +1152,15 @@ handle_zvals:; } zv++; } - zvals_ht = NULL; goto tail_call; } } zv++; } - if (is_primitive && zvals_ht) { + if (zvals_ht) { GC_ADD_FLAGS(zvals_ht, GC_NOT_COLLECTABLE); + zvals_ht = NULL; } - zvals_ht = NULL; } } else if (GC_TYPE(ref) == IS_ARRAY) { ZEND_ASSERT(((zend_array*)ref) != &EG(symbol_table)); From 123dcc565e2cded0db753499314f2d94e59f8de9 Mon Sep 17 00:00:00 2001 From: Ilija Tovilo Date: Fri, 29 Aug 2025 00:19:20 +0200 Subject: [PATCH 3/3] Simplify more --- Zend/zend_gc.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Zend/zend_gc.c b/Zend/zend_gc.c index 6c022073c23c8..169febbeb7ff9 100644 --- a/Zend/zend_gc.c +++ b/Zend/zend_gc.c @@ -1044,7 +1044,7 @@ static void gc_scan_black(zend_refcounted *ref, gc_stack *stack) * counts and mark visited nodes grey. See MarkGray() in Bacon & Rajan. */ static void gc_mark_grey(zend_refcounted *ref, gc_stack *stack) { - HashTable *ht, *zvals_ht = NULL; + HashTable *ht, *primitive_ht = NULL; Bucket *p; zval *zv; uint32_t n; @@ -1135,7 +1135,7 @@ static void gc_mark_grey(zend_refcounted *ref, gc_stack *stack) handle_zvals: for (; n != 0; n--) { if (Z_COLLECTABLE_P(zv)) { - zvals_ht = NULL; + primitive_ht = NULL; ref = Z_COUNTED_P(zv); GC_DELREF(ref); if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) { @@ -1157,9 +1157,9 @@ static void gc_mark_grey(zend_refcounted *ref, gc_stack *stack) } zv++; } - if (zvals_ht) { - GC_ADD_FLAGS(zvals_ht, GC_NOT_COLLECTABLE); - zvals_ht = NULL; + if (primitive_ht) { + GC_ADD_FLAGS(primitive_ht, GC_NOT_COLLECTABLE); + primitive_ht = NULL; } } } else if (GC_TYPE(ref) == IS_ARRAY) { @@ -1170,13 +1170,12 @@ static void gc_mark_grey(zend_refcounted *ref, gc_stack *stack) if (GC_FLAGS(ht) & GC_NOT_COLLECTABLE) { goto next; } + primitive_ht = ht; if (HT_IS_PACKED(ht)) { zv = ht->arPacked; - zvals_ht = ht; goto handle_zvals; } - bool is_primitive = true; p = ht->arData; for (; n != 0; n--) { zv = &p->val; @@ -1184,7 +1183,7 @@ static void gc_mark_grey(zend_refcounted *ref, gc_stack *stack) zv = Z_INDIRECT_P(zv); } if (Z_COLLECTABLE_P(zv)) { - is_primitive = false; + primitive_ht = NULL; ref = Z_COUNTED_P(zv); GC_DELREF(ref); if (!GC_REF_CHECK_COLOR(ref, GC_GREY)) { @@ -1210,8 +1209,9 @@ static void gc_mark_grey(zend_refcounted *ref, gc_stack *stack) } p++; } - if (is_primitive) { + if (primitive_ht) { GC_ADD_FLAGS(ht, GC_NOT_COLLECTABLE); + primitive_ht = NULL; } } else if (GC_TYPE(ref) == IS_REFERENCE) { if (Z_COLLECTABLE(((zend_reference*)ref)->val)) {