Merge pull request #44 from cloudant/107-add-ops-col-view-query

Add extra col to represent no. of ops

Author: ManwithaYarradoddi

src/basho_bench_stats.erl

@@ -72,7 +72,9 @@ op_complete(Op, {ok, Units}, ElapsedUs) ->
             folsom_metrics:notify_existing_metric({latencies, Op}, ElapsedUs, histogram),
             folsom_metrics:notify_existing_metric({overall_latencies, Op}, ElapsedUs, histogram),
             folsom_metrics:notify_existing_metric({overall_units, Op}, {inc, Units}, counter),
-            folsom_metrics:notify_existing_metric({units, Op}, {inc, Units}, counter)
+            folsom_metrics:notify_existing_metric({units, Op}, {inc, Units}, counter),
+            folsom_metrics:notify_existing_metric({overall_ops, Op}, {inc, 1}, counter),
+            folsom_metrics:notify_existing_metric({ops, Op}, {inc, 1}, counter)
     end,
     ok;
 op_complete(Op, Result, ElapsedUs) ->
@@ -114,7 +116,9 @@ init([]) ->
          folsom_metrics:new_histogram({latencies, Op}, slide, basho_bench_config:get(report_interval)),
          folsom_metrics:new_histogram({overall_latencies, Op}, uniform, 16384),
          folsom_metrics:new_counter({overall_units, Op}),
-         folsom_metrics:new_counter({units, Op})
+         folsom_metrics:new_counter({units, Op}),
+         folsom_metrics:new_counter({overall_ops, Op}),
+         folsom_metrics:new_counter({ops, Op})
      end || Op <- Ops ++ Measurements],
 
     StatsWriter = basho_bench_config:get(stats, csv),
@@ -162,10 +166,13 @@ handle_cast({Op, {ok, Units}, ElapsedUs}, State = #state{last_write_time = LWT,
     folsom_metrics:notify_existing_metric({overall_latencies, Op}, ElapsedUs, histogram),
     folsom_metrics:notify_existing_metric({overall_units, Op}, {inc, Units}, counter),
     folsom_metrics:notify_existing_metric({units, Op}, {inc, Units}, counter),
+    folsom_metrics:notify_existing_metric({overall_ops, Op}, {inc, 1}, counter),
+    folsom_metrics:notify_existing_metric({ops, Op}, {inc, 1}, counter),
     {noreply, NewState};
 handle_cast(_, State) ->
     {noreply, State}.
 
+
 handle_info(report, State) ->
     consume_report_msgs(),
     Now = os:timestamp(),
@@ -251,19 +258,21 @@ process_stats(Now, #state{stats_writer=Module}=State) ->
     Window = timer:now_diff(Now, State#state.last_write_time) / 1000000,
 
     %% Time to report latency data to our CSV files
-    {Oks, Errors, OkOpsRes} =
-        lists:foldl(fun(Op, {TotalOks, TotalErrors, OpsResAcc}) ->
-                            {Oks, Errors} = report_latency(State, Elapsed, Window, Op),
-                            {TotalOks + Oks, TotalErrors + Errors,
-                             [{Op, Oks}|OpsResAcc]}
-                    end, {0,0,[]}, State#state.ops),
+    {Ops, Oks, Errors, OkOpsRes} =
+        lists:foldl(fun(Op, {TotalOps, TotalOks, TotalErrors, OpsResAcc}) ->
+                            {Ops, Oks, Errors} = report_latency(State, Elapsed, Window, Op),
+                            {TotalOps + Ops, TotalOks + Oks, TotalErrors + Errors,
+                             [{Op, Oks, Ops}|OpsResAcc]}
+                    end, {0,0,0,[]}, State#state.ops),
 
     %% Reset units
-    [folsom_metrics_counter:dec({units, Op}, OpAmount) || {Op, OpAmount} <- OkOpsRes],
+    [folsom_metrics_counter:dec({units, Op}, OksAmount) || {Op, OksAmount, _} <- OkOpsRes],
+    %% Reset Ops
+    [folsom_metrics_counter:dec({ops, Op}, OpsAmount) || {Op, _, OpsAmount} <- OkOpsRes],
 
     %% Write summary
     Module:process_summary(State#state.stats_writer_data,
-                           Elapsed, Window, Oks, Errors),
+                           Elapsed, Window, Ops, Oks, Errors),
 
     %% Dump current error counts to console
     case (State#state.errors_since_last_report) of
@@ -285,7 +294,8 @@ process_global_stats(#state{stats_writer=Module}=State) ->
         Stats = folsom_metrics:get_histogram_statistics({overall_latencies, Op}),
         Errors = error_counter(Op),
         Units = folsom_metrics:get_metric_value({overall_units, Op}),
-        Module:report_global_stats(Op, Stats, Errors, Units)
+        Ops = folsom_metrics:get_metric_value({overall_ops, Op}),
+        Module:report_global_stats(Op, Stats, Errors, Units, Ops)
     end, State#state.ops).
 
 %%
@@ -296,12 +306,13 @@ report_latency(#state{stats_writer=Module}=State, Elapsed, Window, Op) ->
     Stats = folsom_metrics:get_histogram_statistics({latencies, Op}),
     Errors = error_counter(Op),
     Units = folsom_metrics:get_metric_value({units, Op}),
+    Ops = folsom_metrics:get_metric_value({ops, Op}),
 
     Module:report_latency({State#state.stats_writer,
                                              State#state.stats_writer_data},
                                             Elapsed, Window, Op,
-                                            Stats, Errors, Units),
-    {Units, Errors}.
+                                            Stats, Errors, Units, Ops),
+    {Ops, Units, Errors}.
 
 report_total_errors(#state{stats_writer=Module}=State) ->
     case ets:tab2list(basho_bench_total_errors) of
@@ -349,14 +360,14 @@ get_worker_ops() ->
    Workers = basho_bench_config:get(workers, []),
    case Workers of
        %% No workers reverts to original case is fine as long as we stick with 2-tuples
-       [] -> 
+       [] ->
             lists:map(
                 fun({OpTag, _Count}) -> {OpTag, OpTag};
                     ({Label, OpTag, _Count}) -> {Label, OpTag};
                     ({Label, OpTag, _Count, _OptionsList}) -> {Label, OpTag}
                 end, basho_bench_config:get(operations, []));
        %% Use workers to determine active worker types for current config
-        _ -> 
+        _ ->
             get_worker_ops(Workers, basho_bench_config:get(worker_types), [])
     end.
 
@@ -374,7 +385,7 @@ get_worker_ops([{WorkerType, _Count} | RestWorkers], WorkerTypes, Acc) ->
     end,
 
      Ops = lists:map(
-        fun({OpTag, _Count2}) -> 
+        fun({OpTag, _Count2}) ->
                 { worker_op_name(WorkerType, OpTag), worker_op_name(WorkerType, OpTag)};
            ({Label, OpTag, _Count2}) ->
                 { worker_op_name(WorkerType, Label), worker_op_name(WorkerType, OpTag)}
@@ -383,7 +394,7 @@ get_worker_ops([{WorkerType, _Count} | RestWorkers], WorkerTypes, Acc) ->
 
 
 worker_op_name(Worker,Op) ->
-    case Worker of 
+    case Worker of
         single_worker ->
             Op;
         _ ->

src/basho_bench_stats_writer_csv.erl

@@ -33,10 +33,10 @@
 
 -export([new/2,
          terminate/1,
-         process_summary/5,
+         process_summary/6,
          report_error/3,
-         report_global_stats/4,
-         report_latency/7]).
+         report_global_stats/5,
+         report_latency/8]).
 
 -include("basho_bench.hrl").
 
@@ -57,7 +57,7 @@ new(Ops, Measurements) ->
         filename:join([TestDir, "summary.csv"]),
         [raw, binary, write]
     ),
-    file:write(SummaryFile, <<"elapsed, window, total, successful, failed\n">>),
+    file:write(SummaryFile, <<"elapsed, window, total_operations, total_units, successful, failed\n">>),
 
     %% Setup errors file w/counters for each error.  Embedded commas likely
     %% in the error messages so quote the columns.
@@ -82,13 +82,14 @@ terminate({SummaryFile, ErrorsFile}) ->
     ok.
 
 process_summary({SummaryFile, _ErrorsFile},
-                Elapsed, Window, Oks, Errors) ->
+                Elapsed, Window, Ops, Oks, Errors) ->
     file:write(SummaryFile,
-               io_lib:format("~w, ~w, ~w, ~w, ~w\n",
+               io_lib:format("~w, ~w, ~w, ~w, ~w, ~w\n",
                              [Elapsed,
                               Window,
-                              Oks + Errors,
+                              Ops + Errors,
                               Oks,
+                              Ops,
                               Errors])).
 
 report_error({_SummaryFile, ErrorsFile},
@@ -97,8 +98,8 @@ report_error({_SummaryFile, ErrorsFile},
                io_lib:format("\"~w\",\"~w\"\n",
                              [Key, Count])).
 
-report_global_stats({Op,_}, Stats, Errors, Units) ->
-    %% Build up JSON structure representing statistices collected in folsom
+report_global_stats({Op,_}, Stats, Errors, Units, Ops) ->
+    %% Build up JSON structure representing statistics collected in folsom
     P = proplists:get_value(percentile, Stats),
     JsonElements0 = lists:foldl(fun(K, Acc) ->
         case K of
@@ -132,21 +133,25 @@ report_global_stats({Op,_}, Stats, Errors, Units) ->
     %% insert error counts
     JsonElements2 = [{basho_errors, Errors} | JsonElements1],
 
+    %% insert Ops counts
+    JsonElements3 = [{'ops', Ops} | JsonElements2],
+
     JsonMetrics0 = erlang:get(run_metrics),
-    JsonMetrics = lists:keyreplace(Op, 1, JsonMetrics0, {Op, {JsonElements2}}),
+    JsonMetrics = lists:keyreplace(Op, 1, JsonMetrics0, {Op, {JsonElements3}}),
     %?DEBUG("Generated Json:\n~w",[JsonElements]),
     erlang:put(run_metrics, JsonMetrics).
 
 report_latency({_SummaryFile, _ErrorsFile},
                Elapsed, Window, Op,
-               Stats, Errors, Units) ->
+               Stats, Errors, Units, Ops) ->
     case proplists:get_value(n, Stats) > 0 of
         true ->
             P = proplists:get_value(percentile, Stats),
-            Line = io_lib:format("~w, ~w, ~w, ~w, ~.1f, ~w, ~w, ~w, ~w, ~w, ~w\n",
+            Line = io_lib:format("~w, ~w, ~w, ~w, ~w, ~.1f, ~w, ~w, ~w, ~w, ~w, ~w\n",
                                  [Elapsed,
                                   Window,
                                   Units,
+                                  Ops,
                                   proplists:get_value(min, Stats),
                                   proplists:get_value(arithmetic_mean, Stats),
                                   proplists:get_value(median, Stats),
@@ -157,7 +162,7 @@ report_latency({_SummaryFile, _ErrorsFile},
                                   Errors]);
         false ->
             ?WARN("No data for op: ~p\n", [Op]),
-            Line = io_lib:format("~w, ~w, 0, 0, 0, 0, 0, 0, 0, 0, ~w\n",
+            Line = io_lib:format("~w, ~w, 0, 0, 0, 0, 0, 0, 0, 0, 0, ~w\n",
                                  [Elapsed,
                                   Window,
                                   Errors])
@@ -172,7 +177,7 @@ op_csv_file({Label, _Op}) ->
     TestDir = basho_bench:get_test_dir(),
     Fname = filename:join([TestDir, normalize_label(Label) ++ "_latencies.csv"]),
     {ok, F} = file:open(Fname, [raw, binary, write]),
-    ok = file:write(F, <<"elapsed, window, n, min, mean, median, 95th, 99th, 99_9th, max, errors\n">>),
+    ok = file:write(F, <<"elapsed, window, n, ops, min, mean, median, 95th, 99th, 99_9th, max, errors\n">>),
     F.
 
 measurement_csv_file({Label, _Op}) ->
@@ -194,7 +199,7 @@ stringify_stats(_RunStatsType=json, RunMetrics) ->
     [ jiffy:encode( {[{recordedMetrics, {RunMetrics}}]}, [pretty] ) ];
 stringify_stats(_RunStatsType=csv, RunMetrics) ->
     % Ordered output of fields
-    OrderedFields = [n, mean, geometric_mean, harmonic_mean,
+    OrderedFields = [n, ops, mean, geometric_mean, harmonic_mean,
         variance, standard_deviation, skewness, kurtosis,
         median, p75, p90, p95, p99, p999,
         min, max, basho_errors],
@@ -223,7 +228,7 @@ write_run_statistics(FileName, Lines)->
         FileName,
         [raw, binary, write]
     ),
-    
+
     %% Write lines to file
     lists:foreach(fun(Line) ->
         ok = file:write(GlobalMetricsFile, Line)
@@ -251,4 +256,3 @@ replace_special_chars([_|T]) ->
     [$-|replace_special_chars(T)];
 replace_special_chars([]) ->
     [].
-