[add] Enable full percentile spectrum analysis on latency

Author: filipecosta90

.gitignore

@@ -2,9 +2,11 @@ aclocal.m4
 autom4te.cache
 config.guess
 config.h.in
+config.h.in~
 config.h
 config.sub
 configure
+compile
 depcomp
 install-sh
 ltmain.sh
@@ -19,6 +21,11 @@ config.log
 config.status
 libtool
 stamp-h1
-.idea
-.vscode
+*.DS_Store
+.vscode/*
+.idea/*
+
+# memtier outputs
+*.hgrm
+*.txt
 __pycache__

Makefile.am

@@ -40,7 +40,11 @@ memtier_benchmark_SOURCES = \
 	obj_gen.cpp obj_gen.h \
 	item.cpp item.h \
 	file_io.cpp file_io.h \
-	config_types.cpp config_types.h
+	config_types.cpp config_types.h \
+	deps/hdr_histogram/hdr_histogram_log.c deps/hdr_histogram/hdr_histogram_log.h deps/hdr_histogram/byteorder.h \
+	deps/hdr_histogram/hdr_histogram.c deps/hdr_histogram/hdr_histogram.h \
+	deps/hdr_histogram/hdr_time.c deps/hdr_histogram/hdr_time.h deps/hdr_histogram/hdr_encoding.c deps/hdr_histogram/hdr_encoding.h
+
 memtier_benchmark_LDADD = \
 	$(LIBEVENT_LIBS) \
 	$(LIBEVENT_OPENSSL_LIBS) \

README.md

@@ -156,3 +156,24 @@ Also, the ratio and the key generator is per client (and not connection).
 In this case, setting the ratio to 1:1 does not guarantee 100% hits because
 the keys spread to different connections/nodes.
 
+### Full latency spectrum analysis
+
+For distributions that are non-normal, such as the latency, many “basic rules” of normally distributed statistics are violated.  Instead of computing just the mean, which tries to express the whole distribution in a single result, we can use a sampling of the distribution at intervals -- percentiles, which tell you how many requests actually would experience that delay. 
+
+
+When used for normally distributed data, the samples are usually taken at regular intervals. However, since the data does not obey to a normal distribution it would be very expensive to keep equally spaced intervals of latency records while enabling large value ranges. We can apply algorithms that can calculate a good approximation of percentiles at minimal CPU and memory cost, such as [t-digest](https://github.com/tdunning/t-digest) or [HdrHistogram](https://github.com/HdrHistogram/HdrHistogram_c). On memtier_benchmark we’ve decided to use the  HdrHistogram due to its low memory footprint, high precision, zero allocation during the benchmark and constant access time.
+
+
+By default Memtier will output the 50th, 99th, and 99.9th percentiles. They are the latency thresholds at which 50%, 99%, and 99.9% of commands are faster than that particular presented value. 
+To output different percentiles you should use the --print-percentiles option followed by the comma separated list of values ( example: `--print-percentiles 90,99,99.9,99.99` ).
+
+#### Saving the full latency spectrum
+To save the full latencies you should use the --hdr-file-prefix option followed by the prefix name you wish the filenames to have. 
+Each distinct command will be saved into two different files - one in .txt (textual format) and another in .hgrm (HistogramLogProcessor format).
+The textual format can be hard to analyze solely, but you can use an [online formatter](http://hdrhistogram.github.io/HdrHistogram/plotFiles.html) to generate visual histograms from it. The .hgrm format will be later added as input to Redislabs [mbdirector](https://github.com/redislabs/mbdirector) to enable visualization of time-domain results.
+
+Sample Visual Feel of the full latency spectrum using an [online formatter](http://hdrhistogram.github.io/HdrHistogram/plotFiles.html):
+![alt text][sample_visual_histogram]
+
+
+[sample_visual_histogram]: ./docs/sample_visual_histogram.png "Sample Full Latency Spectrum Histogram"

client.cpp

@@ -56,6 +56,7 @@ bool client::setup_client(benchmark_config *config, abstract_protocol *protocol,
 {
     m_config = config;
     assert(m_config != NULL);
+    unsigned long long total_num_of_clients = config->clients*config->threads;
 
     // create main connection
     shard_connection* conn = new shard_connection(m_connections.size(), this, m_config, m_event_base, protocol);
@@ -74,7 +75,6 @@ bool client::setup_client(benchmark_config *config, abstract_protocol *protocol,
     // Parallel key-pattern determined according to the first command
     if ((config->arbitrary_commands->is_defined() && config->arbitrary_commands->at(0).key_pattern == 'P') ||
         (config->key_pattern[key_pattern_set]=='P')) {
-        unsigned long long total_num_of_clients = config->clients*config->threads;
         unsigned long long client_index = config->next_client_idx % total_num_of_clients;
 
         unsigned long long range = (config->key_maximum - config->key_minimum)/total_num_of_clients + 1;

client.h

@@ -61,17 +61,17 @@ class client : public connections_manager {
     object_generator* m_obj_gen;
     run_stats m_stats;
 
-    unsigned long long m_reqs_processed;      // requests processed (responses received)
-    unsigned long long m_reqs_generated;      // requests generated (wait for responses)
-    unsigned int m_set_ratio_count;     // number of sets counter (overlaps on ratio)
-    unsigned int m_get_ratio_count;     // number of gets counter (overlaps on ratio)
+    unsigned long long m_reqs_processed;          // requests processed (responses received)
+    unsigned long long m_reqs_generated;          // requests generated (wait for responses)
+    unsigned int m_set_ratio_count;               // number of sets counter (overlaps on ratio)
+    unsigned int m_get_ratio_count;               // number of gets counter (overlaps on ratio)
     unsigned int m_arbitrary_command_ratio_count; // number of arbitrary commands counter (overlaps on ratio)
-    unsigned int m_executed_command_index; // current arbitrary command executed
+    unsigned int m_executed_command_index;        // current arbitrary command executed
 
-    unsigned long long m_tot_set_ops;        // Total number of SET ops
-    unsigned long long m_tot_wait_ops;       // Total number of WAIT ops
+    unsigned long long m_tot_set_ops;             // Total number of SET ops
+    unsigned long long m_tot_wait_ops;            // Total number of WAIT ops
 
-    keylist *m_keylist;                 // used to construct multi commands
+    keylist *m_keylist;                           // used to construct multi commands
 
 public:
     client(client_group* group);

config_types.cpp

@@ -35,6 +35,7 @@
 #include <netdb.h>
 
 #include <string>
+#include <iostream>
 #include <stdexcept>
 #include <climits>
 #include <algorithm>
@@ -87,6 +88,34 @@ config_ratio::config_ratio(const char *ratio_str) :
     }
 }
 
+config_quantiles::config_quantiles(){
+
+}
+
+config_quantiles::config_quantiles(const char *str)
+{
+    assert(str != NULL);
+
+    do {
+        float quantile;
+        char *p = NULL;
+        quantile = strtof(str, &p);
+        if (!p || (*p != ',' && *p != '\0')) {
+            quantile_list.clear();
+            return;
+        }
+        str = p;
+        if (*str) str++;
+        quantile_list.push_back(quantile);
+    } while (*str);
+}
+
+bool config_quantiles::is_defined(void)
+{
+    return quantile_list.size() > 0;
+}
+
+
 config_weight_list::config_weight_list() :
     next_size_weight(0)
 {

config_types.h

@@ -46,6 +46,15 @@ struct config_ratio {
     bool is_defined(void) { return (a > 0 || b > 0); }
 };
 
+struct config_quantiles {
+    std::vector<float> quantile_list;
+    config_quantiles();
+    config_quantiles(const char *ratio_str);
+    bool is_defined(void);
+    inline std::vector<float>::iterator begin()  { return quantile_list.begin(); }
+    inline std::vector<float>::iterator end()  { return quantile_list.end(); }
+};
+
 struct config_weight_list {
     struct weight_item {
         unsigned int size;

docs/sample_visual_histogram.png

@@ -275,6 +275,10 @@ static void config_init_defaults(struct benchmark_config *cfg)
     }
     if (!cfg->requests && !cfg->test_time)
         cfg->requests = 10000;
+    if (!cfg->hdr_prefix)
+        cfg->hdr_prefix = "";
+    if (!cfg->print_percentiles.is_defined())
+        cfg->print_percentiles = config_quantiles("50,99,99.9");
 }
 
 static int generate_random_seed()
@@ -363,6 +367,7 @@ static int config_parse_args(int argc, char *argv[], struct benchmark_config *cf
         o_key_median,
         o_show_config,
         o_hide_histogram,
+        o_print_percentiles,
         o_distinct_client_seed,
         o_randomize,
         o_client_stats,
@@ -384,7 +389,8 @@ static int config_parse_args(int argc, char *argv[], struct benchmark_config *cf
         o_tls_key,
         o_tls_cacert,
         o_tls_skip_verify,
-        o_tls_sni
+        o_tls_sni,
+        o_hdr_file_prefix
     };
 
     static struct option long_options[] = {
@@ -401,11 +407,13 @@ static int config_parse_args(int argc, char *argv[], struct benchmark_config *cf
         { "sni",                        1, 0, o_tls_sni },
 #endif
         { "out-file",                   1, 0, 'o' },
+        { "hdr-file-prefix",            1, 0, o_hdr_file_prefix },
         { "client-stats",               1, 0, o_client_stats },
         { "run-count",                  1, 0, 'x' },
         { "debug",                      0, 0, 'D' },
         { "show-config",                0, 0, o_show_config },
         { "hide-histogram",             0, 0, o_hide_histogram },
+        { "print-percentiles",          1, 0, o_print_percentiles },
         { "distinct-client-seed",       0, 0, o_distinct_client_seed },
         { "randomize",                  0, 0, o_randomize },
         { "requests",                   1, 0, 'n' },
@@ -493,6 +501,9 @@ static int config_parse_args(int argc, char *argv[], struct benchmark_config *cf
                 case 'o':
                     cfg->out_file = optarg;
                     break;
+                case o_hdr_file_prefix:
+                    cfg->hdr_prefix = optarg;
+                    break;
                 case o_client_stats:
                     cfg->client_stats = optarg;
                     break;
@@ -513,6 +524,13 @@ static int config_parse_args(int argc, char *argv[], struct benchmark_config *cf
                 case o_hide_histogram:
                     cfg->hide_histogram++;
                     break;
+                case o_print_percentiles:
+                    cfg->print_percentiles = config_quantiles(optarg);
+                    if (!cfg->print_percentiles.is_defined()) {
+                        fprintf(stderr, "error: quantiles must be expressed as [0.0-100.0],[0.0-100.0](,...) .\n");
+                        return -1;
+                    }
+                    break;
                 case o_distinct_client_seed:
                     cfg->distinct_client_seed++;
                     break;
@@ -858,8 +876,10 @@ void usage() {
             "      --client-stats=FILE        Produce per-client stats file\n"
             "      --out-file=FILE            Name of output file (default: stdout)\n"
             "      --json-out-file=FILE       Name of JSON output file, if not set, will not print to json\n"
+            "      --hdr-file-prefix=FILE     Prefix of HDR Latency Histogram output files, if not set, will not save latency histogram files\n"
             "      --show-config              Print detailed configuration before running\n"
             "      --hide-histogram           Don't print detailed latency histogram\n"
+            "      --print-percentiles        Specify which percentiles info to print on the results table (by default prints percentiles: 50,99,99.9)\n"
             "      --cluster-mode             Run client in cluster mode\n"
             "      --help                     Display this help\n"
             "      --version                  Display version information\n"
@@ -1455,6 +1475,7 @@ int main(int argc, char *argv[])
             perror(cfg.out_file);
         }
     } else {
+        fprintf(stderr, "Writing results to stdout\n");
         outfile = stdout;
     }
 
@@ -1468,6 +1489,10 @@ int main(int argc, char *argv[])
 
             run_stats stats = run_benchmark(run_id, &cfg, obj_gen);
             all_stats.push_back(stats);
+            stats.save_hdr_full_run( &cfg,run_id );
+            stats.save_hdr_get_command( &cfg,run_id );
+            stats.save_hdr_set_command( &cfg,run_id );
+            stats.save_hdr_arbitrary_commands( &cfg,run_id );
         }
         //
         // Print some run information
@@ -1485,7 +1510,7 @@ int main(int argc, char *argv[])
             jsonhandler->write_obj("Connections per thread","%u",cfg.clients);
             jsonhandler->write_obj(cfg.requests > 0 ? "Requests per client"  : "Seconds","%llu",
                                    cfg.requests > 0 ? cfg.requests : (unsigned long long)cfg.test_time);
-
+            jsonhandler->write_obj("Format version","%d",2);
             jsonhandler->close_nesting();
         }
 

memtier_benchmark.cpp

@@ -53,6 +53,7 @@ struct benchmark_config {
     int debug;
     int show_config;
     int hide_histogram;
+    config_quantiles print_percentiles;
     int distinct_client_seed;
     int randomize;
     int next_client_idx;
@@ -93,6 +94,7 @@ struct benchmark_config {
     const char *json_out_file;
     bool cluster_mode;
     struct arbitrary_command_list* arbitrary_commands;
+    const char *hdr_prefix;
 #ifdef USE_TLS
     bool tls;
     const char *tls_cert;