Add a suite of benchmark scripts

CHANGELOG.md

@@ -1,10 +1,11 @@
 # Changelog
 
-## v1.13.0
+## v1.13.0 (unreleased)
 
 * Add support for Elixir 1.18. Drop support for Elixir 1.15. Elixir >= 1.16 is now required. Dropping support for older versions of Elixir simply means that this package is no longer tested with them in CI, and that compatibility issues are not considered bugs.
 * Drop support for Erlang/OTP 24, and Erlang/OTP >= 25 is now required. Dropping support for older versions of Erlang/OTP simply means that this package is not tested with them in CI, and that no compatibility issues are considered bugs.
 * Improve how the change-notification Phoenix.PubSub adapter manages its connection and readiness status. ([pull/191](https://github.com/tompave/fun_with_flags/pull/191))
+* Adding a suite of synthetic benchmark scripts for the package. ([pull/193](https://github.com/tompave/fun_with_flags/pull/193))
 
 ## v1.12.0
 

README.md

@@ -46,6 +46,7 @@ It stores flag information in Redis or a relational DB (PostgreSQL, MySQL, or SQ
 * [Testing](#testing)
 * [Development](#development)
   - [Working with PubSub Locally](#working-with-pubsub-locally)
+  - [Benchmarks](#benchmarks)
 
 ## What's a Feature Flag?
 
@@ -839,3 +840,7 @@ Steps:
 5. In either terminal, run `Node.list()` to check that there is a connection.
 
 Done that, modifying any flag data in either terminal will notify the other one via PubSub.
+
+### Benchmarks
+
+The package comes with [a suite of synthetic benchmark scripts](https://github.com/tompave/fun_with_flags/tree/master/benchmarks). Their use is recommended when working on the internals of the package.

benchmarks/README.md

@@ -0,0 +1,9 @@
+# Benchmarks
+
+Some simple benchmark scripts for the package. Use them as they are or modify them to test specific scenarios.
+
+Example:
+
+```
+mix run benchmarks/flag.exs
+```

benchmarks/cache.exs

@@ -0,0 +1,85 @@
+# :observer.start
+
+# Start the ecto repo if running the benchmarks with ecto.
+# {:ok, _pid} = FunWithFlags.Dev.EctoRepo.start_link()
+
+FunWithFlags.clear(:one)
+FunWithFlags.clear(:two)
+FunWithFlags.clear(:three)
+FunWithFlags.clear(:four)
+
+alias PlainUser, as: User
+
+u1 = %User{id: 1, group: "foo"}
+u2 = %User{id: 2, group: "foo"}
+u3 = %User{id: 3, group: "bar"}
+u4 = %User{id: 4, group: "bar"}
+
+FunWithFlags.enable(:one)
+
+FunWithFlags.enable(:two)
+FunWithFlags.enable(:two, for_actor: u4)
+FunWithFlags.disable(:two, for_group: "nope")
+
+FunWithFlags.disable(:three)
+FunWithFlags.enable(:three, for_actor: u2)
+FunWithFlags.enable(:three, for_actor: u3)
+FunWithFlags.enable(:three, for_actor: u4)
+FunWithFlags.disable(:three, for_group: "nope")
+FunWithFlags.disable(:three, for_group: "nope2")
+
+
+FunWithFlags.disable(:four)
+FunWithFlags.enable(:four, for_actor: u2)
+FunWithFlags.enable(:four, for_actor: u3)
+FunWithFlags.enable(:four, for_actor: u4)
+FunWithFlags.enable(:four, for_actor: "a")
+FunWithFlags.enable(:four, for_actor: "b")
+FunWithFlags.enable(:four, for_actor: "c")
+FunWithFlags.enable(:four, for_actor: "d")
+FunWithFlags.enable(:four, for_actor: "e")
+FunWithFlags.disable(:four, for_group: "nope")
+FunWithFlags.disable(:four, for_group: "nope2")
+FunWithFlags.disable(:four, for_group: "nope3")
+FunWithFlags.disable(:four, for_group: "nope4")
+FunWithFlags.enable(:four, for_percentage_of: {:actors, 0.99})
+
+# warm up the cache
+FunWithFlags.enabled?(:one)
+FunWithFlags.enabled?(:two)
+FunWithFlags.enabled?(:three)
+FunWithFlags.enabled?(:four)
+
+
+alias FunWithFlags.Store.Cache
+
+# -----------------------------------
+one = fn() ->
+  Cache.get(:one)
+end
+
+two = fn() ->
+  Cache.get(:two)
+end
+
+three = fn() ->
+  Cache.get(:three)
+end
+
+four = fn() ->
+  Cache.get(:four)
+end
+
+
+Benchee.run(
+  %{
+    "one" => one,
+    "two" => two,
+    "three" => three,
+    "four" => four,
+  }#,
+  # formatters: [
+  #   Benchee.Formatters.HTML,
+  #   Benchee.Formatters.Console
+  # ]
+)

benchmarks/flag.exs

@@ -0,0 +1,101 @@
+# :observer.start
+
+# Start the ecto repo if running the benchmarks with ecto.
+# {:ok, _pid} = FunWithFlags.Dev.EctoRepo.start_link()
+
+FunWithFlags.clear(:one)
+FunWithFlags.clear(:two)
+FunWithFlags.clear(:three)
+FunWithFlags.clear(:four)
+
+alias PlainUser, as: User
+
+u1 = %User{id: 1, group: "foo"}
+u2 = %User{id: 2, group: "foo"}
+u3 = %User{id: 3, group: "bar"}
+u4 = %User{id: 4, group: "bar"}
+
+FunWithFlags.enable(:one)
+
+FunWithFlags.enable(:two)
+FunWithFlags.enable(:two, for_actor: u4)
+FunWithFlags.disable(:two, for_group: "nope")
+
+FunWithFlags.disable(:three)
+FunWithFlags.enable(:three, for_actor: u2)
+FunWithFlags.enable(:three, for_actor: u3)
+FunWithFlags.enable(:three, for_actor: u4)
+FunWithFlags.disable(:three, for_group: "nope")
+FunWithFlags.disable(:three, for_group: "nope2")
+
+
+FunWithFlags.disable(:four)
+FunWithFlags.enable(:four, for_actor: u2)
+FunWithFlags.enable(:four, for_actor: u3)
+FunWithFlags.enable(:four, for_actor: u4)
+FunWithFlags.enable(:four, for_actor: "a")
+FunWithFlags.enable(:four, for_actor: "b")
+FunWithFlags.enable(:four, for_actor: "c")
+FunWithFlags.enable(:four, for_actor: "d")
+FunWithFlags.enable(:four, for_actor: "e")
+FunWithFlags.disable(:four, for_group: "nope")
+FunWithFlags.disable(:four, for_group: "nope2")
+FunWithFlags.disable(:four, for_group: "nope3")
+FunWithFlags.disable(:four, for_group: "nope4")
+FunWithFlags.enable(:four, for_percentage_of: {:actors, 0.99})
+
+# warm up the cache
+FunWithFlags.enabled?(:one)
+FunWithFlags.enabled?(:two)
+FunWithFlags.enabled?(:three)
+FunWithFlags.enabled?(:four)
+
+# -----------------------------------
+one_a = fn() ->
+  FunWithFlags.enabled?(:one)
+end
+
+one_b = fn() ->
+  FunWithFlags.enabled?(:one, for: u1)
+end
+
+two_a = fn() ->
+  FunWithFlags.enabled?(:two)
+end
+
+two_b = fn() ->
+  FunWithFlags.enabled?(:two, for: u1)
+end
+
+three_a = fn() ->
+  FunWithFlags.enabled?(:three)
+end
+
+three_b = fn() ->
+  FunWithFlags.enabled?(:three, for: u1)
+end
+
+four_a = fn() ->
+  FunWithFlags.enabled?(:four)
+end
+
+four_b = fn() ->
+  FunWithFlags.enabled?(:four, for: u1)
+end
+
+Benchee.run(
+  %{
+    "one_a" => one_a,
+    "one_b" => one_b,
+    "two_a" => two_a,
+    "two_b" => two_b,
+    "three_a" => three_a,
+    "three_b" => three_b,
+    "four_a" => four_a,
+    "four_b" => four_b,
+  }#,
+#   formatters: [
+#     Benchee.Formatters.HTML,
+#     Benchee.Formatters.Console
+#   ]
+)

benchmarks/persistence.exs

@@ -0,0 +1,79 @@
+# Test the performance of the persistence adapters.
+# This benchmark is mostly affected by the performance of the underlying datastore.
+# However, it's also useful to assess how the store is accessed in Elixir. For example,
+# when switching from compiled-in config to just straight calls to the config module.
+
+# :observer.start
+
+# Start the ecto repo if running the benchmarks with ecto.
+# {:ok, _pid} = FunWithFlags.Dev.EctoRepo.start_link()
+
+FunWithFlags.clear(:one)
+FunWithFlags.clear(:two)
+FunWithFlags.clear(:three)
+FunWithFlags.clear(:four)
+
+alias PlainUser, as: User
+
+u1 = %User{id: 1, group: "foo"}
+u2 = %User{id: 2, group: "foo"}
+u3 = %User{id: 3, group: "bar"}
+u4 = %User{id: 4, group: "bar"}
+
+FunWithFlags.enable(:one)
+
+FunWithFlags.enable(:two)
+FunWithFlags.enable(:two, for_actor: u4)
+FunWithFlags.disable(:two, for_group: "nope")
+
+FunWithFlags.disable(:three)
+FunWithFlags.enable(:three, for_actor: u2)
+FunWithFlags.enable(:three, for_actor: u3)
+FunWithFlags.enable(:three, for_actor: u4)
+FunWithFlags.disable(:three, for_group: "nope")
+FunWithFlags.disable(:three, for_group: "nope2")
+
+
+FunWithFlags.disable(:four)
+FunWithFlags.enable(:four, for_actor: u2)
+FunWithFlags.enable(:four, for_actor: u3)
+FunWithFlags.enable(:four, for_actor: u4)
+FunWithFlags.enable(:four, for_actor: "a")
+FunWithFlags.enable(:four, for_actor: "b")
+FunWithFlags.enable(:four, for_actor: "c")
+FunWithFlags.enable(:four, for_actor: "d")
+FunWithFlags.enable(:four, for_actor: "e")
+FunWithFlags.disable(:four, for_group: "nope")
+FunWithFlags.disable(:four, for_group: "nope2")
+FunWithFlags.disable(:four, for_group: "nope3")
+FunWithFlags.disable(:four, for_group: "nope4")
+FunWithFlags.enable(:four, for_percentage_of: {:actors, 0.99})
+
+alias FunWithFlags.SimpleStore
+
+# -----------------------------------
+one = fn() ->
+  SimpleStore.lookup(:one)
+end
+
+two = fn() ->
+  SimpleStore.lookup(:two)
+end
+
+three = fn() ->
+  SimpleStore.lookup(:three)
+end
+
+four = fn() ->
+  SimpleStore.lookup(:four)
+end
+
+
+Benchee.run(
+  %{
+    "one" => one,
+    "two" => two,
+    "three" => three,
+    "four" => four,
+  }
+)

dev_support/protocols.ex

@@ -31,3 +31,18 @@ defimpl FunWithFlags.Group, for: Map do
   def in?(_, _), do: false
 end
 
+
+defmodule PlainUser do
+  defstruct [:id, :group]
+end
+
+defimpl FunWithFlags.Actor, for: PlainUser do
+  def id(%{id: id}) do
+    "user:#{id}"
+  end
+end
+
+defimpl FunWithFlags.Group, for: PlainUser do
+  def in?(%{group: group}, group), do: true
+  def in?(_, _), do: false
+end

mix.exs

@@ -79,7 +79,10 @@ defmodule FunWithFlags.Mixfile do
 
       {:ex_doc, "~> 0.21", only: :dev, runtime: false},
       {:credo, "~> 1.7", only: :dev, runtime: false},
-      {:dialyxir, "~> 1.0", only: :dev, runtime: false}
+      {:dialyxir, "~> 1.0", only: :dev, runtime: false},
+
+      {:benchee, "~> 1.0", only: :dev, runtime: false},
+      {:benchee_html, "~> 1.0", only: :dev, runtime: false},
     ]
   end
 

mix.lock

@@ -1,10 +1,14 @@
 %{
+  "benchee": {:hex, :benchee, "1.3.1", "c786e6a76321121a44229dde3988fc772bca73ea75170a73fd5f4ddf1af95ccf", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "76224c58ea1d0391c8309a8ecbfe27d71062878f59bd41a390266bf4ac1cc56d"},
+  "benchee_html": {:hex, :benchee_html, "1.0.1", "1e247c0886c3fdb0d3f4b184b653a8d6fb96e4ad0d0389267fe4f36968772e24", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:benchee_json, "~> 1.0", [hex: :benchee_json, repo: "hexpm", optional: false]}], "hexpm", "b00a181af7152431901e08f3fc9f7197ed43ff50421a8347b0c80bf45d5b3fef"},
+  "benchee_json": {:hex, :benchee_json, "1.0.0", "cc661f4454d5995c08fe10dd1f2f72f229c8f0fb1c96f6b327a8c8fc96a91fe5", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "da05d813f9123505f870344d68fb7c86a4f0f9074df7d7b7e2bb011a63ec231c"},
   "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
   "cc_precompiler": {:hex, :cc_precompiler, "0.1.10", "47c9c08d8869cf09b41da36538f62bc1abd3e19e41701c2cea2675b53c704258", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "f6e046254e53cd6b41c6bacd70ae728011aa82b2742a80d6e2214855c6e06b22"},
   "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"},
   "credo": {:hex, :credo, "1.7.11", "d3e805f7ddf6c9c854fd36f089649d7cf6ba74c42bc3795d587814e3c9847102", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "56826b4306843253a66e47ae45e98e7d284ee1f95d53d1612bb483f88a8cf219"},
   "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"},
   "decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"},
+  "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
   "dialyxir": {:hex, :dialyxir, "1.4.5", "ca1571ac18e0f88d4ab245f0b60fa31ff1b12cbae2b11bd25d207f865e8ae78a", [:mix], [{:erlex, ">= 0.2.7", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b0fb08bb8107c750db5c0b324fa2df5ceaa0f9307690ee3c1f6ba5b9eb5d35c3"},
   "earmark": {:hex, :earmark, "1.4.7", "7b5f0474469688f5514948a4e0c42955a5690d165deaf19c8eb950a3443a40f3", [:mix], [], "hexpm", "0a55a49ee6fa8bc8678f894dfa8a882af6fe8deb65aed6856122226db5b0fe5b"},
   "earmark_parser": {:hex, :earmark_parser, "1.4.43", "34b2f401fe473080e39ff2b90feb8ddfeef7639f8ee0bbf71bb41911831d77c5", [:mix], [], "hexpm", "970a3cd19503f5e8e527a190662be2cee5d98eed1ff72ed9b3d1a3d466692de8"},
@@ -28,5 +32,6 @@
   "phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"},
   "postgrex": {:hex, :postgrex, "0.20.0", "363ed03ab4757f6bc47942eff7720640795eb557e1935951c1626f0d303a3aed", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "d36ef8b36f323d29505314f704e21a1a038e2dc387c6409ee0cd24144e187c0f"},
   "redix": {:hex, :redix, "1.5.2", "ab854435a663f01ce7b7847f42f5da067eea7a3a10c0a9d560fa52038fd7ab48", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "78538d184231a5d6912f20567d76a49d1be7d3fca0e1aaaa20f4df8e1142dcb8"},
+  "statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"},
   "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},
 }