Apply microsoft#1188

Author: Jarred-Sumner

include/mimalloc-stats.h

@@ -11,7 +11,7 @@ terms of the MIT license. A copy of the license can be found in the file
 #include <mimalloc.h>
 #include <stdint.h>
 
-#define MI_STAT_VERSION   2   // increased on every backward incompatible change
+#define MI_STAT_VERSION   3   // increased on every backward incompatible change
 
 // count allocation over time
 typedef struct mi_stat_count_s {
@@ -25,12 +25,18 @@ typedef struct mi_stat_counter_s {
   int64_t total;                              // total count
 } mi_stat_counter_t;
 
+// average over time
+typedef struct mi_stat_average_s {      
+  int64_t total;
+  int64_t count;
+} mi_stat_average_t;
+
 #define MI_STAT_FIELDS() \
   MI_STAT_COUNT(pages)                      /* count of mimalloc pages */ \
   MI_STAT_COUNT(reserved)                   /* reserved memory bytes */ \
   MI_STAT_COUNT(committed)                  /* committed bytes */ \
-  MI_STAT_COUNT(reset)                      /* reset bytes */ \
-  MI_STAT_COUNT(purged)                     /* purged bytes */ \
+  MI_STAT_COUNTER(reset)                    /* reset bytes */ \
+  MI_STAT_COUNTER(purged)                   /* purged bytes */ \
   MI_STAT_COUNT(page_committed)             /* committed memory inside pages */ \
   MI_STAT_COUNT(pages_abandoned)            /* abandonded pages count */ \
   MI_STAT_COUNT(threads)                    /* number of threads */ \
@@ -52,7 +58,7 @@ typedef struct mi_stat_counter_s {
   MI_STAT_COUNTER(arena_purges) \
   MI_STAT_COUNTER(pages_extended)           /* number of page extensions */ \
   MI_STAT_COUNTER(pages_retire)             /* number of pages that are retired */ \
-  MI_STAT_COUNTER(page_searches)            /* searches for a fresh page */ \
+  MI_STAT_AVERAGE(page_searches)            /* searches for a fresh page */ \
   /* only on v1 and v2 */ \
   MI_STAT_COUNT(segments) \
   MI_STAT_COUNT(segments_abandoned) \
@@ -81,6 +87,7 @@ typedef enum mi_chunkbin_e {
 #define MI_BIN_HUGE             (73U)   // see types.h
 #define MI_STAT_COUNT(stat)     mi_stat_count_t stat;
 #define MI_STAT_COUNTER(stat)   mi_stat_counter_t stat;
+#define MI_STAT_AVERAGE(stat)   mi_stat_average_t stat;
 
 typedef struct mi_stats_s
 {
@@ -100,6 +107,7 @@ typedef struct mi_stats_s
 
 #undef MI_STAT_COUNT
 #undef MI_STAT_COUNTER
+#undef MI_STAT_AVERAGE
 
 
 // Exported definitions

include/mimalloc/internal.h

@@ -350,13 +350,18 @@ void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount);
 void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount);
 void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount);
 
+// averages can just be increased
+void __mi_stat_average_increase(mi_stat_average_t* stat, size_t amount);
+void __mi_stat_average_increase_mt(mi_stat_average_t* stat, size_t amount);
+
 #define mi_subproc_stat_counter_increase(subproc,stat,amount)   __mi_stat_counter_increase_mt( &(subproc)->stats.stat, amount)
 #define mi_subproc_stat_increase(subproc,stat,amount)           __mi_stat_increase_mt( &(subproc)->stats.stat, amount)
 #define mi_subproc_stat_decrease(subproc,stat,amount)           __mi_stat_decrease_mt( &(subproc)->stats.stat, amount)
 #define mi_subproc_stat_adjust_increase(subproc,stat,amnt)      __mi_stat_adjust_increase_mt( &(subproc)->stats.stat, amnt)
 #define mi_subproc_stat_adjust_decrease(subproc,stat,amnt)      __mi_stat_adjust_decrease_mt( &(subproc)->stats.stat, amnt)
 
 #define mi_tld_stat_counter_increase(tld,stat,amount)           __mi_stat_counter_increase( &(tld)->stats.stat, amount)
+#define mi_tld_stat_average_increase(tld,stat,amount)           __mi_stat_average_increase( &(tld)->stats.stat, amount)
 #define mi_tld_stat_increase(tld,stat,amount)                   __mi_stat_increase( &(tld)->stats.stat, amount)
 #define mi_tld_stat_decrease(tld,stat,amount)                   __mi_stat_decrease( &(tld)->stats.stat, amount)
 #define mi_tld_stat_adjust_increase(tld,stat,amnt)              __mi_stat_adjust_increase( &(tld)->stats.stat, amnt)
@@ -367,6 +372,7 @@ void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount);
 #define mi_os_stat_decrease(stat,amount)                        mi_subproc_stat_decrease(_mi_subproc(),stat,amount)
 
 #define mi_heap_stat_counter_increase(heap,stat,amount)         mi_tld_stat_counter_increase(heap->tld, stat, amount)
+#define mi_heap_stat_average_increase(heap,stat,amount)         mi_tld_stat_average_increase(heap->tld, stat, amount)
 #define mi_heap_stat_increase(heap,stat,amount)                 mi_tld_stat_increase( heap->tld, stat, amount)
 #define mi_heap_stat_decrease(heap,stat,amount)                 mi_tld_stat_decrease( heap->tld, stat, amount)
 #define mi_heap_stat_adjust_decrease(heap,stat,amount)          mi_tld_stat_adjust_decrease( heap->tld, stat, amount)

src/init.c

@@ -65,16 +65,19 @@ const mi_page_t _mi_page_empty = {
     QNULL(MI_LARGE_MAX_OBJ_WSIZE + 2) /* Full queue */ }
 
 #define MI_STAT_COUNT_NULL()  {0,0,0}
+#define MI_STAT_AVERAGE_NULL() {0,0}
 
 // Empty statistics
 #define MI_STATS_NULL  \
-  MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
-  MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
+  MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
+  { 0 }, { 0 }, \
+  MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
   MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), MI_STAT_COUNT_NULL(), \
   { 0 }, { 0 }, { 0 }, { 0 }, \
   { 0 }, { 0 }, { 0 }, { 0 }, \
   \
-  { 0 }, { 0 }, { 0 }, { 0 }, { 0 }, \
+  { 0 }, { 0 }, { 0 }, { 0 }, \
+  MI_STAT_AVERAGE_NULL(), \
   MI_INIT4(MI_STAT_COUNT_NULL), \
   { 0 }, { 0 }, { 0 }, { 0 },  \
   \

src/os.c

@@ -551,7 +551,7 @@ bool _mi_os_reset(void* addr, size_t size) {
   size_t csize;
   void* start = mi_os_page_align_area_conservative(addr, size, &csize);
   if (csize == 0) return true;  // || _mi_os_is_huge_reserved(addr)
-  mi_os_stat_increase(reset, csize);
+  mi_os_stat_counter_increase(reset, csize);
   mi_os_stat_counter_increase(reset_calls, 1);
 
   #if (MI_DEBUG>1) && !MI_SECURE && !MI_TRACK_ENABLED // && !MI_TSAN
@@ -583,7 +583,7 @@ bool _mi_os_purge_ex(void* p, size_t size, bool allow_reset, size_t stat_size, m
 {
   if (mi_option_get(mi_option_purge_delay) < 0) return false;  // is purging allowed?
   mi_os_stat_counter_increase(purge_calls, 1);
-  mi_os_stat_increase(purged, size);
+  mi_os_stat_counter_increase(purged, size);
 
   if (commit_fun != NULL) {
     bool decommitted = (*commit_fun)(false, p, size, NULL, commit_fun_arg);

src/page.c

@@ -789,7 +789,7 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_heap_t* heap, m
     page = next;
   } // for each page
 
-  mi_heap_stat_counter_increase(heap, page_searches, count);
+  mi_heap_stat_average_increase(heap, page_searches, count);
 
   // set the page to the best candidate
   if (page_candidate != NULL) {

src/stats.c

@@ -46,6 +46,16 @@ void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) {
   stat->total += amount;
 }
 
+void __mi_stat_average_increase_mt(mi_stat_average_t* stat, size_t amount) {
+  mi_atomic_addi64_relaxed( &stat->count, (int64_t)1 );
+  mi_atomic_addi64_relaxed( &stat->total, (int64_t)amount );
+}
+
+void __mi_stat_average_increase(mi_stat_average_t* stat, size_t amount) {
+  stat->count++;
+  stat->total += amount;
+}
+
 void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount) {
   mi_stat_update_mt(stat, (int64_t)amount);
 }
@@ -111,8 +121,15 @@ static void mi_stat_counter_add_mt(mi_stat_counter_t* stat, const mi_stat_counte
   mi_atomic_void_addi64_relaxed(&stat->total, &src->total);
 }
 
+static void mi_stat_average_add_mt(mi_stat_average_t* stat, const mi_stat_average_t* src) {
+  if (stat==src) return;
+  mi_atomic_void_addi64_relaxed(&stat->count, &src->count);
+  mi_atomic_void_addi64_relaxed(&stat->total, &src->total);
+}
+
 #define MI_STAT_COUNT(stat)    mi_stat_count_add_mt(&stats->stat, &src->stat);
 #define MI_STAT_COUNTER(stat)  mi_stat_counter_add_mt(&stats->stat, &src->stat);
+#define MI_STAT_AVERAGE(stat)  mi_stat_average_add_mt(&stats->stat, &src->stat);
 
 // must be thread safe as it is called from stats_merge
 static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {
@@ -133,6 +150,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) {
 
 #undef MI_STAT_COUNT
 #undef MI_STAT_COUNTER
+#undef MI_STAT_AVERAGE
 
 /* -----------------------------------------------------------
   Display statistics
@@ -225,12 +243,6 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t
   mi_stat_print_ex(stat, msg, unit, out, arg, NULL);
 }
 
-static void mi_stat_peak_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) {
-  _mi_fprintf(out, arg, "%10s:", msg);
-  mi_print_amount(stat->peak, unit, out, arg);
-  _mi_fprintf(out, arg, "\n");
-}
-
 #if MI_STAT>1
 static void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) {
   _mi_fprintf(out, arg, "%10s:", msg);
@@ -246,15 +258,13 @@ static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg
   _mi_fprintf(out, arg, "\n");
 }
 
-
-static void mi_stat_counter_print_avg(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg) {
-  const int64_t avg_tens = (stat->total == 0 ? 0 : (stat->total*10 / stat->total));
+static void mi_stat_average_print(const mi_stat_average_t* stat, const char* msg, mi_output_fun* out, void* arg) {
+  const int64_t avg_tens = (stat->count == 0 ? 0 : (stat->total*10 / stat->count));
   const long avg_whole = (long)(avg_tens/10);
   const long avg_frac1 = (long)(avg_tens%10);
   _mi_fprintf(out, arg, "%10s: %5ld.%ld avg\n", msg, avg_whole, avg_frac1);
 }
 
-
 static void mi_print_header(mi_output_fun* out, void* arg ) {
   _mi_fprintf(out, arg, "%10s: %11s %11s %11s %11s %11s\n", "heap stats", "peak   ", "total   ", "current   ", "block   ", "total#   ");
 }
@@ -341,8 +351,8 @@ void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_attr
   #endif
   mi_stat_print_ex(&stats->reserved, "reserved", 1, out, arg, "");
   mi_stat_print_ex(&stats->committed, "committed", 1, out, arg, "");
-  mi_stat_peak_print(&stats->reset, "reset", 1, out, arg );
-  mi_stat_peak_print(&stats->purged, "purged", 1, out, arg );
+  mi_stat_counter_print(&stats->reset, "reset", out, arg );
+  mi_stat_counter_print(&stats->purged, "purged", out, arg );
   mi_stat_print_ex(&stats->page_committed, "touched", 1, out, arg, "");
   // mi_stat_print(&stats->segments, "segments", -1, out, arg);
   // mi_stat_print(&stats->segments_abandoned, "-abandoned", -1, out, arg);
@@ -365,7 +375,7 @@ void _mi_stats_print(mi_stats_t* stats, mi_output_fun* out0, void* arg0) mi_attr
   mi_stat_counter_print(&stats->purge_calls, "purges", out, arg);
   mi_stat_counter_print(&stats->malloc_guarded_count, "guarded", out, arg);
   mi_stat_print(&stats->threads, "threads", -1, out, arg);
-  mi_stat_counter_print_avg(&stats->page_searches, "searches", out, arg);
+  mi_stat_average_print(&stats->page_searches, "searches", out, arg);
   _mi_fprintf(out, arg, "%10s: %5i\n", "numa nodes", _mi_os_numa_node_count());
 
   size_t elapsed;
@@ -619,8 +629,16 @@ static void mi_heap_buf_print_counter_value(mi_heap_buf_t* hbuf, const char* nam
   mi_heap_buf_print_value(hbuf, name, stat->total);
 }
 
+static void mi_heap_buf_print_average_value(mi_heap_buf_t* hbuf, const char* name, mi_stat_average_t* stat) {
+  char buf[128];
+  _mi_snprintf(buf, 128, "  \"%s\": { \"count\": %lld, \"total\": %lld },\n", name, stat->count, stat->total);
+  buf[127] = 0;
+  mi_heap_buf_print(hbuf, buf);
+}
+
 #define MI_STAT_COUNT(stat)    mi_heap_buf_print_count_value(&hbuf, #stat, &stats->stat);
 #define MI_STAT_COUNTER(stat)  mi_heap_buf_print_counter_value(&hbuf, #stat, &stats->stat);
+#define MI_STAT_AVERAGE(stat)  mi_heap_buf_print_average_value(&hbuf, #stat, &stats->stat);
 
 char* mi_stats_get_json(size_t output_size, char* output_buf) mi_attr_noexcept {
   mi_stats_merge();