Merge pull request #2233 from metanivek/lower_layer_for_tree_bench

Author: metanivek

bench/irmin-pack/trace_replay.ml

@@ -159,6 +159,7 @@ module Make (Store : Store) = struct
     mutable commits_since_start_or_gc : int;
     mutable latest_commit_idx : int;
         (** the most recent commit idx to be replayed. initial value is -1 *)
+    mutable gc_count : int;
     key_per_commit_idx : (int, Store.commit_key) Hashtbl.t;
   }
 
@@ -322,7 +323,7 @@ module Make (Store : Store) = struct
     in
     aux operations 0
 
-  let gc_actions config i commits_since_start_or_gc =
+  let gc_actions config i commits_since_start_or_gc gc_count =
     let gc_enabled =
       (* Is GC enabled at all? *)
       config.gc_every > 0
@@ -357,10 +358,18 @@ module Make (Store : Store) = struct
       else false
     in
 
+    let time_to_add_volume =
+      config.add_volume_every > 0
+      && time_to_split
+      && gc_count > 0
+      && gc_count mod config.add_volume_every = 0
+    in
+
     let really_split = gc_enabled && time_to_split in
     let really_start_gc = gc_enabled && time_to_start in
     let really_wait_gc = gc_wait_enabled && time_to_wait in
-    (really_wait_gc, really_start_gc, really_split)
+    let really_add_volume = time_to_add_volume in
+    (really_wait_gc, really_start_gc, really_split, really_add_volume)
 
   let add_commits config repo commit_seq on_commit on_end stats check_hash
       empty_blobs =
@@ -372,6 +381,7 @@ module Make (Store : Store) = struct
         contexts = Hashtbl.create 3;
         hash_corresps = Hashtbl.create 3;
         commits_since_start_or_gc = 0;
+        gc_count = 0;
         latest_commit_idx = -1;
         key_per_commit_idx = Hashtbl.create 3;
       }
@@ -384,11 +394,12 @@ module Make (Store : Store) = struct
       match commit_seq () with
       | Seq.Nil -> on_end () >|= fun () -> i
       | Cons (ops, commit_seq) ->
-          let really_wait_gc, really_start_gc, really_split =
-            gc_actions config i t.commits_since_start_or_gc
+          let really_wait_gc, really_start_gc, really_split, really_add_volume =
+            gc_actions config i t.commits_since_start_or_gc t.gc_count
           in
           (* Split before GC to simulate how it is inteded to be used. *)
           let () = if really_split then Store.split repo in
+          let () = if really_add_volume then Store.add_volume repo in
           let* () =
             if really_wait_gc then (
               [%logs.app
@@ -418,6 +429,7 @@ module Make (Store : Store) = struct
                   (Hashtbl.find t.key_per_commit_idx t.latest_commit_idx)];
               let finished = function
                 | Ok stats ->
+                    t.gc_count <- t.gc_count + 1;
                     let commit_idx = t.latest_commit_idx in
                     let commit_duration = commit_idx - gc_start_commit_idx in
                     let duration =

bench/irmin-pack/trace_replay_intf.ml

@@ -33,6 +33,7 @@ module Config = struct
     gc_every : int;
     gc_distance_in_the_past : int;
     gc_wait_after : int;
+    add_volume_every : int;
   }
   (** Replay configuration
 
@@ -102,6 +103,7 @@ module type Store = sig
     root:string -> store_config -> (Repo.t * on_commit * on_end) Lwt.t
 
   val split : repo -> unit
+  val add_volume : repo -> unit
   val gc_wait : repo -> unit Lwt.t
 
   type stats := Irmin_pack_unix.Stats.Latest_gc.stats

bench/irmin-pack/tree.ml

@@ -39,6 +39,7 @@ type config = {
   gc_every : int;
   gc_distance_in_the_past : int;
   gc_wait_after : int;
+  add_volume_every : int;
 }
 
 module type Store = sig
@@ -61,6 +62,7 @@ module type Store = sig
   type stats := Irmin_pack_unix.Stats.Latest_gc.stats
 
   val split : repo -> unit
+  val add_volume : repo -> unit
 
   val gc_run :
     ?finished:((stats, string) result -> unit Lwt.t) ->
@@ -181,6 +183,7 @@ module Bench_suite (Store : Store) = struct
         gc_every = config.gc_every;
         gc_distance_in_the_past = config.gc_distance_in_the_past;
         gc_wait_after = config.gc_wait_after;
+        add_volume_every = config.add_volume_every;
       }
     in
     if config.no_summary then
@@ -224,6 +227,7 @@ module Make_store_mem (Conf : Irmin_pack.Conf.S) = struct
     Lwt.return (repo, on_commit, on_end)
 
   let split _repo = ()
+  let add_volume _repo = ()
   let gc_wait _repo = Lwt.return_unit
   let gc_run ?finished:_ _repo _key = Lwt.return_unit
 end
@@ -242,9 +246,14 @@ module Make_store_pack (Conf : Irmin_pack.Conf.S) = struct
 
   let indexing_strategy = Irmin_pack.Indexing_strategy.minimal
 
-  let create_repo ~root _config =
+  let create_repo ~root (config : store_config) =
+    let lower_root =
+      if config.add_volume_every > 0 then Some (Filename.concat root "lower")
+      else None
+    in
     let conf =
-      Irmin_pack.config ~readonly:false ~fresh:true ~indexing_strategy root
+      Irmin_pack.config ~readonly:false ~fresh:true ~indexing_strategy
+        ~lower_root root
     in
     prepare_artefacts_dir root;
     let* repo = Store.Repo.v conf in
@@ -253,6 +262,7 @@ module Make_store_pack (Conf : Irmin_pack.Conf.S) = struct
     Lwt.return (repo, on_commit, on_end)
 
   let split = Store.split
+  let add_volume = Store.add_volume
 
   let gc_wait repo =
     let* r = Store.Gc.wait repo in
@@ -395,7 +405,8 @@ let get_suite suite_filter =
 let main () ncommits number_of_commits_to_replay suite_filter inode_config
     store_type freeze_commit path_conversion depth width nchain_trees
     nlarge_trees replay_trace_path artefacts_path keep_store keep_stat_trace
-    no_summary empty_blobs gc_every gc_distance_in_the_past gc_wait_after =
+    no_summary empty_blobs gc_every gc_distance_in_the_past gc_wait_after
+    add_volume_every =
   let default = match suite_filter with `Quick -> 10000 | _ -> 13315 in
   let number_of_commits_to_replay =
     Option.value ~default number_of_commits_to_replay
@@ -422,6 +433,7 @@ let main () ncommits number_of_commits_to_replay suite_filter inode_config
       gc_every;
       gc_distance_in_the_past;
       gc_wait_after;
+      add_volume_every;
     }
   in
   Printexc.record_backtrace true;
@@ -602,6 +614,10 @@ let gc_wait_after =
   in
   Arg.(value @@ opt int 0 doc)
 
+let add_volume_every =
+  let doc = Arg.info ~doc:"Add volume ever N GCs" [ "add-volume-every" ] in
+  Arg.(value @@ opt int 0 doc)
+
 let main_term =
   Term.(
     const main
@@ -625,7 +641,8 @@ let main_term =
     $ empty_blobs
     $ gc_every
     $ gc_distance_in_the_past
-    $ gc_wait_after)
+    $ gc_wait_after
+    $ add_volume_every)
 
 let deprecated_info = (Term.info [@alert "-deprecated"])
 let deprecated_exit = (Term.exit [@alert "-deprecated"])

test/irmin-bench/replay.ml

@@ -95,6 +95,7 @@ let replay_1_commit () =
       gc_every = 0;
       gc_distance_in_the_past = 0;
       gc_wait_after = 0;
+      add_volume_every = 0;
     }
   in
   let+ summary = Replay.run () replay_config in
@@ -141,6 +142,7 @@ module Store_mem = struct
     Lwt.return (repo, on_commit, on_end)
 
   let split _repo = ()
+  let add_volume _repo = ()
   let gc_wait _repo = Lwt.return_unit
   let gc_run ?finished:_ _repo _key = Lwt.return_unit
 end
@@ -164,6 +166,7 @@ let replay_1_commit_mem () =
       gc_every = 0;
       gc_distance_in_the_past = 0;
       gc_wait_after = 0;
+      add_volume_every = 0;
     }
   in
   let+ summary = Replay_mem.run () replay_config in