Skip to content

Commit 8ae51e7

Browse files
erthinkerthink
committed
mdbx: page-age histogram feature (v0.14.1-326-g72e0af44).
2026-01-24 mdbx: minor reflow internal comments. 2026-01-24 mdbx-make: add `--deduplicate` to `git ls-files`. 2026-01-24 mdbx: merge `page-age histogram` feature. 2026-01-22 mdbx-tool: add page-age histogram into chk-output. 2026-01-22 mdbx: add `histogram_acc_ex()` and refine `histogram_dist()`. 2026-01-22 mdbx: rename `nested_height_or_gc_span_length` inside `MDBX_chk_table_t.histograms`. 2026-01-22 mdbx: rework histogram internal merges. 2026-01-22 mdbx: add internal 128-bit integers. 2026-01-20 mdbx: split-out `histogram.c` 2026-01-20 mdbx: fix `histogram_reduce()`. 2026-01-20 mdbx: explicit `le1_` prefixes for `MDBX_chk_histogram` fields.
1 parent 9f4665f commit 8ae51e7

File tree

10 files changed

+1254
-202
lines changed

10 files changed

+1254
-202
lines changed

‎VERSION.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
{ "git_describe": "v0.14.1-315-g009726d3", "git_timestamp": "2026-01-25T00:49:59+03:00", "git_tree": "bd54d6e6082817e0becf5c4d6c52dd446bbeef4a", "git_commit": "009726d3b7659155118c8c2917a93c65f5ea12c2", "semver": "0.14.1.315" }
1+
{ "git_describe": "v0.14.1-326-g72e0af44", "git_timestamp": "2026-01-25T00:52:18+03:00", "git_tree": "32db7628fa7e56137220952cf3328075093e8efc", "git_commit": "72e0af44e90a9e678e21ab84ce4b7fe9275fc730", "semver": "0.14.1.326" }

‎mdbx.c‎

Lines changed: 434 additions & 176 deletions
Large diffs are not rendered by default.

‎mdbx.c++‎

Lines changed: 141 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
/// \author Леонид Юрьев aka Leonid Yuriev <[email protected]> \date 2015-2026
33
/* clang-format off */
44

5-
#define MDBX_BUILD_SOURCERY 9e316b99c2dd59a62dfecc0d27aab4fc009eb09071df9580efa6333960118e9d_v0_14_1_315_g009726d3
5+
#define MDBX_BUILD_SOURCERY d0c652016ea3e3d314181251772ed3ad35b30d8654a3e0e3f7a1dfc73f6406b4_v0_14_1_326_g72e0af44
66

77
#define LIBMDBX_INTERNALS
88
#define MDBX_DEPRECATED
@@ -1576,6 +1576,23 @@ typedef union bin128 {
15761576
__anonymous_struct_extension__ struct {
15771577
uint32_t a, b, c, d;
15781578
};
1579+
__anonymous_struct_extension__ struct {
1580+
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1581+
uint64_t l, h;
1582+
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
1583+
uint64_t h, l;
1584+
#else
1585+
#error "FIXME: Unsupported byte order"
1586+
#endif /* __BYTE_ORDER__ */
1587+
};
1588+
1589+
#if defined(__SIZEOF_INT128__)
1590+
#define MDBX_HAVE_NATIVE_U128 1
1591+
__int128_t i128;
1592+
__uint128_t u128;
1593+
#else
1594+
#define MDBX_HAVE_NATIVE_U128 0
1595+
#endif
15791596
} bin128_t;
15801597

15811598
MDBX_INTERNAL bin128_t osal_guid(const MDBX_env *);
@@ -1982,6 +1999,16 @@ MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t osal_bswap32
19821999
#define MDBX_USE_FALLOCATE_CONFIG MDBX_STRINGIFY(MDBX_USE_FALLOCATE)
19832000
#endif /* MDBX_USE_FALLOCATE */
19842001

2002+
#ifndef MDBX_HISTOGRAM_USING_128BIT
2003+
#if MDBX_WORDBITS >= 64
2004+
#define MDBX_HISTOGRAM_USING_128BIT 1
2005+
#else
2006+
#define MDBX_HISTOGRAM_USING_128BIT 0
2007+
#endif
2008+
#elif !(MDBX_HISTOGRAM_USING_128BIT == 0 || MDBX_HISTOGRAM_USING_128BIT == 1)
2009+
#error MDBX_HISTOGRAM_USING_128BIT must be defined as 0 or 1
2010+
#endif /* MDBX_HISTOGRAM_USING_128BIT */
2011+
19852012
//------------------------------------------------------------------------------
19862013

19872014
#ifndef MDBX_CPU_WRITEBACK_INCOHERENT
@@ -3110,8 +3137,90 @@ typedef struct ratio2digits_buffer {
31103137
char string[1 + 20 + 1 + 19 + 1];
31113138
} ratio2digits_buffer_t;
31123139

3113-
char *ratio2digits(const uint64_t v, const uint64_t d, ratio2digits_buffer_t *const buffer, int precision);
3114-
char *ratio2percent(const uint64_t v, const uint64_t d, ratio2digits_buffer_t *const buffer);
3140+
MDBX_INTERNAL char *ratio2digits(const uint64_t v, const uint64_t d, ratio2digits_buffer_t *const buffer,
3141+
int precision);
3142+
MDBX_INTERNAL char *ratio2percent(const uint64_t v, const uint64_t d, ratio2digits_buffer_t *const buffer);
3143+
3144+
MDBX_INTERNAL bin128_t mul64x64_128_fallback(uint64_t x, uint64_t y);
3145+
3146+
MDBX_MAYBE_UNUSED static inline bin128_t mul64x64_128(uint64_t x, uint64_t y) {
3147+
bin128_t r;
3148+
#if MDBX_HAVE_NATIVE_U128 && !MDBX_DEBUG
3149+
r.u128 = x;
3150+
r.u128 *= y;
3151+
#else
3152+
r = mul64x64_128_fallback(x, y);
3153+
#if MDBX_HAVE_NATIVE_U128
3154+
assert(r.u128 == (__uint128_t)x * (__uint128_t)y);
3155+
#endif
3156+
#endif
3157+
return r;
3158+
}
3159+
3160+
MDBX_MAYBE_UNUSED static inline bin128_t u128_add(bin128_t x, bin128_t y) {
3161+
bin128_t r;
3162+
#if MDBX_HAVE_NATIVE_U128 && !MDBX_DEBUG
3163+
r.u128 = x.u128 + y.u128;
3164+
#else
3165+
r.l = x.l + y.l;
3166+
r.h = x.h + y.h + /* carry */ (r.l < x.l);
3167+
#if MDBX_HAVE_NATIVE_U128
3168+
assert(r.u128 == x.u128 + y.u128);
3169+
#endif
3170+
#endif
3171+
return r;
3172+
}
3173+
3174+
MDBX_MAYBE_UNUSED static inline int u128_cmp(bin128_t x, bin128_t y) {
3175+
int r;
3176+
#if defined(__SIZEOF_INT128__) && !MDBX_DEBUG
3177+
r = CMP2INT(x.u128, y.u128);
3178+
#else
3179+
const uint64_t a = (x.h != y.h) ? x.h : x.l;
3180+
const uint64_t b = (x.h != y.h) ? y.h : y.l;
3181+
r = CMP2INT(a, b);
3182+
#if MDBX_HAVE_NATIVE_U128
3183+
assert(r == CMP2INT(x.u128, y.u128));
3184+
#endif
3185+
#endif
3186+
return r;
3187+
}
3188+
3189+
MDBX_MAYBE_UNUSED static inline bool u128_gt(bin128_t x, bin128_t y) {
3190+
bool r;
3191+
#if defined(__SIZEOF_INT128__) && !MDBX_DEBUG
3192+
r = x.u128 > y.u128;
3193+
#else
3194+
r = x.h > y.h || (x.h == y.h && x.l > y.l);
3195+
#if MDBX_HAVE_NATIVE_U128
3196+
assert(r == (x.u128 > y.u128));
3197+
#endif
3198+
#endif
3199+
return r;
3200+
}
3201+
3202+
MDBX_MAYBE_UNUSED static inline bool u128_lt(bin128_t x, bin128_t y) {
3203+
bool r;
3204+
#if defined(__SIZEOF_INT128__) && !MDBX_DEBUG
3205+
r = x.u128 < y.u128;
3206+
#else
3207+
r = x.h < y.h || (x.h == y.h && x.l > y.l);
3208+
#if MDBX_HAVE_NATIVE_U128
3209+
assert(r == (x.u128 < y.u128));
3210+
#endif
3211+
#endif
3212+
return r;
3213+
}
3214+
3215+
MDBX_MAYBE_UNUSED static inline bin128_t u128(uint64_t v) {
3216+
bin128_t r;
3217+
r.l = v;
3218+
r.h = 0;
3219+
#if defined(__SIZEOF_INT128__)
3220+
assert(r.u128 == v);
3221+
#endif
3222+
return r;
3223+
}
31153224

31163225
/* An PNL is an Page Number List, a sorted array of IDs.
31173226
*
@@ -3839,6 +3948,32 @@ MDBX_INTERNAL int coherency_check_written(const MDBX_env *env, const txnid_t txn
38393948
const intptr_t pgno, uint64_t *timestamp);
38403949
MDBX_INTERNAL int coherency_timeout(uint64_t *timestamp, intptr_t pgno, const MDBX_env *env);
38413950

3951+
/* histogram.c */
3952+
#define HISTOGRAM_LE0 1
3953+
MDBX_INTERNAL void histogram_acc_ex(const size_t value, struct MDBX_chk_histogram *histogram, unsigned options);
3954+
MDBX_MAYBE_UNUSED static inline void histogram_acc(const size_t value, struct MDBX_chk_histogram *histogram) {
3955+
histogram_acc_ex(value, histogram, 0);
3956+
}
3957+
MDBX_INTERNAL MDBX_chk_line_t *histogram_dist(MDBX_chk_line_t *line, const struct MDBX_chk_histogram *histogram,
3958+
const char *prefix, const char *first, bool amount);
3959+
MDBX_INTERNAL MDBX_chk_line_t *histogram_print(MDBX_chk_scope_t *scope, MDBX_chk_line_t *line,
3960+
const struct MDBX_chk_histogram *histogram, const char *prefix,
3961+
const char *first, bool amount);
3962+
3963+
/* print.c */
3964+
MDBX_INTERNAL void chk_line_end(MDBX_chk_line_t *line);
3965+
MDBX_INTERNAL __must_check_result MDBX_chk_line_t *chk_line_begin(MDBX_chk_scope_t *const scope,
3966+
enum MDBX_chk_severity severity);
3967+
MDBX_INTERNAL MDBX_chk_line_t *chk_line_feed(MDBX_chk_line_t *line);
3968+
MDBX_INTERNAL MDBX_chk_line_t *chk_flush(MDBX_chk_line_t *line);
3969+
MDBX_INTERNAL size_t chk_print_wanna(MDBX_chk_line_t *line, size_t need);
3970+
MDBX_INTERNAL MDBX_chk_line_t *chk_puts(MDBX_chk_line_t *line, const char *str);
3971+
MDBX_INTERNAL MDBX_chk_line_t *chk_print_va(MDBX_chk_line_t *line, const char *fmt, va_list args);
3972+
MDBX_INTERNAL MDBX_chk_line_t *MDBX_PRINTF_ARGS(2, 3) chk_print(MDBX_chk_line_t *line, const char *fmt, ...);
3973+
MDBX_INTERNAL void chk_println_va(MDBX_chk_scope_t *const scope, enum MDBX_chk_severity severity, const char *fmt,
3974+
va_list args);
3975+
MDBX_INTERNAL void chk_println(MDBX_chk_scope_t *const scope, enum MDBX_chk_severity severity, const char *fmt, ...);
3976+
38423977
/* Сортированный набор txnid, использующий внутри комбинацию непрерывного интервала и списка.
38433978
* Обеспечивает хранение id записей при переработке, очистку и обновлении GC, включая возврат остатков переработанных
38443979
* страниц.
@@ -6644,8 +6779,9 @@ typedef struct walk_tbl {
66446779

66456780
typedef int walk_func(const size_t pgno, const unsigned number, void *const ctx, const int deep,
66466781
const walk_tbl_t *table, const size_t page_size, const page_type_t page_type,
6647-
const MDBX_error_t err, const size_t nentries, const size_t payload_bytes,
6648-
const size_t header_bytes, const size_t unused_bytes, const size_t parent_pgno);
6782+
const txnid_t page_txnid, const MDBX_error_t err, const size_t nentries,
6783+
const size_t payload_bytes, const size_t header_bytes, const size_t unused_bytes,
6784+
const size_t parent_pgno);
66496785

66506786
typedef enum walk_options { dont_check_keys_ordering = 1 } walk_options_t;
66516787

‎mdbx.h‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6924,7 +6924,7 @@ typedef struct MDBX_chk_user_table_cookie MDBX_chk_user_table_cookie_t;
69246924
* собираемой при проверке целостности БД.
69256925
* \see mdbx_env_chk() */
69266926
struct MDBX_chk_histogram {
6927-
size_t amount, count, ones, pad;
6927+
size_t amount, count, le1_amount, le1_count;
69286928
struct {
69296929
size_t begin, end, amount, count;
69306930
} ranges[9];
@@ -6959,7 +6959,7 @@ typedef struct MDBX_chk_table {
69596959
/// Histogram of large/overflow pages length
69606960
struct MDBX_chk_histogram large_pages;
69616961
/// Histogram of nested trees height, span length for GC
6962-
struct MDBX_chk_histogram nested_height;
6962+
struct MDBX_chk_histogram nested_height_or_gc_span_length;
69636963
/// Keys length histogram
69646964
struct MDBX_chk_histogram key_len;
69656965
/// Values length histogram
@@ -6970,6 +6970,8 @@ typedef struct MDBX_chk_table {
69706970
struct MDBX_chk_histogram tree_density;
69716971
/// Histogram of nested tree(s) branch and leaf pages filling in percents
69726972
struct MDBX_chk_histogram large_or_nested_density;
6973+
/// Histogram of pages age
6974+
struct MDBX_chk_histogram page_age;
69736975
} histogram;
69746976
} MDBX_chk_table_t;
69756977

@@ -6988,6 +6990,8 @@ typedef struct MDBX_chk_context {
69886990
size_t processed_pages, reclaimable_pages, gc_pages, alloc_pages, backed_pages;
69896991
size_t problems_meta, tree_problems, gc_tree_problems, kv_tree_problems, problems_gc, problems_kv, total_problems;
69906992
uint64_t steady_txnid, recent_txnid;
6993+
/// Histogram of pages age
6994+
struct MDBX_chk_histogram histogram_page_age;
69916995
/** Указатель на массив размером table_total с указателями на экземпляры
69926996
* структур MDBX_chk_table_t с информацией о всех таблицах ключ-значение,
69936997
* включая MainDB и GC/FreeDB. */

‎mdbx_chk.c‎

Lines changed: 112 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
/// \copyright SPDX-License-Identifier: Apache-2.0
1919
/// \author Леонид Юрьев aka Leonid Yuriev <[email protected]> \date 2015-2026
2020

21-
#define MDBX_BUILD_SOURCERY 9e316b99c2dd59a62dfecc0d27aab4fc009eb09071df9580efa6333960118e9d_v0_14_1_315_g009726d3
21+
#define MDBX_BUILD_SOURCERY d0c652016ea3e3d314181251772ed3ad35b30d8654a3e0e3f7a1dfc73f6406b4_v0_14_1_326_g72e0af44
2222

2323
#define LIBMDBX_INTERNALS
2424
#define MDBX_DEPRECATED
@@ -1592,6 +1592,23 @@ typedef union bin128 {
15921592
__anonymous_struct_extension__ struct {
15931593
uint32_t a, b, c, d;
15941594
};
1595+
__anonymous_struct_extension__ struct {
1596+
#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
1597+
uint64_t l, h;
1598+
#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
1599+
uint64_t h, l;
1600+
#else
1601+
#error "FIXME: Unsupported byte order"
1602+
#endif /* __BYTE_ORDER__ */
1603+
};
1604+
1605+
#if defined(__SIZEOF_INT128__)
1606+
#define MDBX_HAVE_NATIVE_U128 1
1607+
__int128_t i128;
1608+
__uint128_t u128;
1609+
#else
1610+
#define MDBX_HAVE_NATIVE_U128 0
1611+
#endif
15951612
} bin128_t;
15961613

15971614
MDBX_INTERNAL bin128_t osal_guid(const MDBX_env *);
@@ -1998,6 +2015,16 @@ MDBX_MAYBE_UNUSED MDBX_NOTHROW_PURE_FUNCTION static inline uint32_t osal_bswap32
19982015
#define MDBX_USE_FALLOCATE_CONFIG MDBX_STRINGIFY(MDBX_USE_FALLOCATE)
19992016
#endif /* MDBX_USE_FALLOCATE */
20002017

2018+
#ifndef MDBX_HISTOGRAM_USING_128BIT
2019+
#if MDBX_WORDBITS >= 64
2020+
#define MDBX_HISTOGRAM_USING_128BIT 1
2021+
#else
2022+
#define MDBX_HISTOGRAM_USING_128BIT 0
2023+
#endif
2024+
#elif !(MDBX_HISTOGRAM_USING_128BIT == 0 || MDBX_HISTOGRAM_USING_128BIT == 1)
2025+
#error MDBX_HISTOGRAM_USING_128BIT must be defined as 0 or 1
2026+
#endif /* MDBX_HISTOGRAM_USING_128BIT */
2027+
20012028
//------------------------------------------------------------------------------
20022029

20032030
#ifndef MDBX_CPU_WRITEBACK_INCOHERENT
@@ -3126,8 +3153,90 @@ typedef struct ratio2digits_buffer {
31263153
char string[1 + 20 + 1 + 19 + 1];
31273154
} ratio2digits_buffer_t;
31283155

3129-
char *ratio2digits(const uint64_t v, const uint64_t d, ratio2digits_buffer_t *const buffer, int precision);
3130-
char *ratio2percent(const uint64_t v, const uint64_t d, ratio2digits_buffer_t *const buffer);
3156+
MDBX_INTERNAL char *ratio2digits(const uint64_t v, const uint64_t d, ratio2digits_buffer_t *const buffer,
3157+
int precision);
3158+
MDBX_INTERNAL char *ratio2percent(const uint64_t v, const uint64_t d, ratio2digits_buffer_t *const buffer);
3159+
3160+
MDBX_INTERNAL bin128_t mul64x64_128_fallback(uint64_t x, uint64_t y);
3161+
3162+
MDBX_MAYBE_UNUSED static inline bin128_t mul64x64_128(uint64_t x, uint64_t y) {
3163+
bin128_t r;
3164+
#if MDBX_HAVE_NATIVE_U128 && !MDBX_DEBUG
3165+
r.u128 = x;
3166+
r.u128 *= y;
3167+
#else
3168+
r = mul64x64_128_fallback(x, y);
3169+
#if MDBX_HAVE_NATIVE_U128
3170+
assert(r.u128 == (__uint128_t)x * (__uint128_t)y);
3171+
#endif
3172+
#endif
3173+
return r;
3174+
}
3175+
3176+
MDBX_MAYBE_UNUSED static inline bin128_t u128_add(bin128_t x, bin128_t y) {
3177+
bin128_t r;
3178+
#if MDBX_HAVE_NATIVE_U128 && !MDBX_DEBUG
3179+
r.u128 = x.u128 + y.u128;
3180+
#else
3181+
r.l = x.l + y.l;
3182+
r.h = x.h + y.h + /* carry */ (r.l < x.l);
3183+
#if MDBX_HAVE_NATIVE_U128
3184+
assert(r.u128 == x.u128 + y.u128);
3185+
#endif
3186+
#endif
3187+
return r;
3188+
}
3189+
3190+
MDBX_MAYBE_UNUSED static inline int u128_cmp(bin128_t x, bin128_t y) {
3191+
int r;
3192+
#if defined(__SIZEOF_INT128__) && !MDBX_DEBUG
3193+
r = CMP2INT(x.u128, y.u128);
3194+
#else
3195+
const uint64_t a = (x.h != y.h) ? x.h : x.l;
3196+
const uint64_t b = (x.h != y.h) ? y.h : y.l;
3197+
r = CMP2INT(a, b);
3198+
#if MDBX_HAVE_NATIVE_U128
3199+
assert(r == CMP2INT(x.u128, y.u128));
3200+
#endif
3201+
#endif
3202+
return r;
3203+
}
3204+
3205+
MDBX_MAYBE_UNUSED static inline bool u128_gt(bin128_t x, bin128_t y) {
3206+
bool r;
3207+
#if defined(__SIZEOF_INT128__) && !MDBX_DEBUG
3208+
r = x.u128 > y.u128;
3209+
#else
3210+
r = x.h > y.h || (x.h == y.h && x.l > y.l);
3211+
#if MDBX_HAVE_NATIVE_U128
3212+
assert(r == (x.u128 > y.u128));
3213+
#endif
3214+
#endif
3215+
return r;
3216+
}
3217+
3218+
MDBX_MAYBE_UNUSED static inline bool u128_lt(bin128_t x, bin128_t y) {
3219+
bool r;
3220+
#if defined(__SIZEOF_INT128__) && !MDBX_DEBUG
3221+
r = x.u128 < y.u128;
3222+
#else
3223+
r = x.h < y.h || (x.h == y.h && x.l > y.l);
3224+
#if MDBX_HAVE_NATIVE_U128
3225+
assert(r == (x.u128 < y.u128));
3226+
#endif
3227+
#endif
3228+
return r;
3229+
}
3230+
3231+
MDBX_MAYBE_UNUSED static inline bin128_t u128(uint64_t v) {
3232+
bin128_t r;
3233+
r.l = v;
3234+
r.h = 0;
3235+
#if defined(__SIZEOF_INT128__)
3236+
assert(r.u128 == v);
3237+
#endif
3238+
return r;
3239+
}
31313240

31323241
/* An PNL is an Page Number List, a sorted array of IDs.
31333242
*

0 commit comments

Comments
 (0)