[delta_q] add extractor scripts for simulation logs (not yet complete)

Author: rkuhn

delta_q/comparison_hs.txt

@@ -0,0 +1,68 @@
+#!/usr/bin/env jq -crnf
+
+def fmt: . * 1000 | round | . / 1000;
+
+def hist($max; $bins; src): reduce src as $in (
+  { bins: $bins, max: $max, hist: [range(0; $bins) | 0] };
+  .hist[fmin($in / $max * $bins | floor; $bins - 1)] += 1
+);
+
+# takes a hist and transforms it into a CDF string
+def cdf: last(.hist[]) as $sum |
+  . as { bins: $b, max: $m, hist: $h } |
+  ($h | $m / length) as $bin_size |
+  $h | keys | reduce .[] as $idx (
+    [0, []];
+    ($h[$idx] / $sum|fmt) as $y |
+    if $y == .[0] then .
+    else (.[1] += ["(\($idx * $bin_size|fmt),\($y))"]) | .[0] = $y
+    end
+  ) | .[1] | "CDF[\(join(","))]";
+
+# `gen`: extract [type, id, time] of data generation, or empty
+# `recv`: extract [type, id, time] of data reception, or empty
+def read_log(in; gen; recv): reduce in as $in (
+  {}; # { [type]: { [id]: [start, latency[]] }}
+  def g: ($in | gen) as [$type, $id, $time] |
+    setpath([$type, $id]; [$time, []]);
+  def r: ($in | recv) as [$type, $id, $time] |
+    .[$type][$id] |= . as [$t, $l] | [$t, $l + [$time - $t]];
+  g // r // .
+) | map_values(
+  (map(.[1]|length)|max) as $max |
+  map_values(
+    .[1] | select(length == $max) | hist(10; 100; .[])
+  )
+);
+
+def cumulative: .hist |= [foreach .[] as $in (0; . + $in; .)];
+
+# turns an object with hist values into a single hist
+def hist_combine(combine): reduce .[] as $in (
+  null;
+  if . == null then $in
+  else .hist |= . as $h | keys | [.[] | [$h[.], $in.hist[.] // 0] | combine]
+  end
+);
+
+def hist_min: hist_combine(fmin(.[0]; .[1]));
+def hist_max: hist_combine(fmax(.[0]; .[1]));
+def hist_avg: (1 / length) as $scale | hist_combine(.[0] + $scale * .[1]);
+
+def min_max: map(cumulative) | {
+  min: (hist_min | cdf),
+  max: (hist_max | cdf),
+  avg: (hist_avg | cdf)
+};
+
+def print: . as $all | [
+  paths(type == "string") |
+  . as $path |
+  "\(join("_")) := \($all | getpath($path))"
+] | join("\n");
+
+read_log(limit(1000000; inputs);
+  select(.event.tag=="generated")|[.event.kind, .event.id, .time_s];
+  select(.event.tag=="received")|[.event.kind, .event.id, .time_s])
+| map_values(min_max)
+| print

delta_q/diffusion.jq

@@ -0,0 +1,74 @@
+#!/usr/bin/env jq -cnf
+
+def read_rs($emit; key_in; key_out): foreach (range(1; 1000000) | input) as $in (
+  [{}];
+  (.[0] as $state |
+    ($in.message.type | capture("(?<obj>.*)(?<dir>Sent|Received)$")) as { obj: $obj, dir: $dir } |
+    ($in.message.id | tostring) as $id |
+    if $dir == $emit then
+      $state |
+      ($in.message | key_out) as $key |
+      (getpath([$obj, $key.[], $id]) // empty) as $time |
+      delpaths([[$obj, $key.[], $id]]) |
+      [., [$obj, ($in.time - $time) / 1000000]]
+    else
+      $state |
+      ($in.message | key_in) as $key |
+      setpath([$obj, $key.[], $id]; $in.time) |
+      [.]
+    end
+  ) // [.[0]];
+  .[1] // empty
+);
+
+def read_hs($store; key_in; $emit; key_out): foreach inputs as $in (
+  [{}];
+  (.[0] as $state |
+    $in.event as { tag: $dir, kind: $obj } |
+    select($dir == $store or $dir == $emit) |
+    ($in.event.id // $in.event.ids[0]) as $id |
+    if $dir == $emit then
+      $state |
+      ($in.event | key_out) as $key |
+      (getpath([$obj, $key.[], $id]) // empty) as $time |
+      delpaths([[$obj, $key.[], $id]]) |
+      [., [$obj, ($in.time_s - $time) * 1000]]
+    else
+      $state |
+      ($in.event | key_in) as $key |
+      setpath([$obj, $key.[], $id]; $in.time_s) |
+      [.]
+    end
+  ) // [.[0]];
+  .[1] // empty
+);
+
+def histogram(src): reduce src as $in (
+  {};
+  .[$in[0]][$in[1] | tostring] += 1
+);
+
+def histogram_binned($max; $bins; src): reduce src as $in (
+  {};
+  .[$in[0]][fmin($in[1] / $max * $bins | floor; $bins - 1)] += 1
+);
+
+########
+# Rust #
+########
+
+# delay between receiving a message and sending it on
+#histogram(read_rs("Sent"; [.recipient]; [.sender]))
+
+# message propagation time
+#histogram(read_rs("Received"; [.sender, .recipient]; [.sender, .recipient]))
+
+###########
+# Haskell #
+###########
+
+# delay between receiving a message and sending it on
+#histogram_binned(1000; 20; read_hs("received"; [.node]; "Sent"; [.sender]))
+
+# message propagation time
+histogram_binned(1000; 20; read_hs("Sent"; [.receipient]; "received"; [.node]))