Skip to content

Commit 7b9af98

Browse files
tompavetompave
authored
Merge pull request #193 from tompave/benchmarks
Add a suite of benchmark scripts
2 parents 4237c67 + 7196969 commit 7b9af98

File tree

9 files changed

+305
-2
lines changed

9 files changed

+305
-2
lines changed

‎CHANGELOG.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
# Changelog
22

3-
## v1.13.0
3+
## v1.13.0 (unreleased)
44

55
* Add support for Elixir 1.18. Drop support for Elixir 1.15. Elixir >= 1.16 is now required. Dropping support for older versions of Elixir simply means that this package is no longer tested with them in CI, and that compatibility issues are not considered bugs.
66
* Drop support for Erlang/OTP 24, and Erlang/OTP >= 25 is now required. Dropping support for older versions of Erlang/OTP simply means that this package is not tested with them in CI, and that no compatibility issues are considered bugs.
77
* Improve how the change-notification Phoenix.PubSub adapter manages its connection and readiness status. ([pull/191](https://github.com/tompave/fun_with_flags/pull/191))
8+
* Adding a suite of synthetic benchmark scripts for the package. ([pull/193](https://github.com/tompave/fun_with_flags/pull/193))
89

910
## v1.12.0
1011

‎README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ It stores flag information in Redis or a relational DB (PostgreSQL, MySQL, or SQ
4646
* [Testing](#testing)
4747
* [Development](#development)
4848
- [Working with PubSub Locally](#working-with-pubsub-locally)
49+
- [Benchmarks](#benchmarks)
4950

5051
## What's a Feature Flag?
5152

@@ -839,3 +840,7 @@ Steps:
839840
5. In either terminal, run `Node.list()` to check that there is a connection.
840841

841842
Done that, modifying any flag data in either terminal will notify the other one via PubSub.
843+
844+
### Benchmarks
845+
846+
The package comes with [a suite of synthetic benchmark scripts](https://github.com/tompave/fun_with_flags/tree/master/benchmarks). Their use is recommended when working on the internals of the package.

‎benchmarks/README.md‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
# Benchmarks
2+
3+
Some simple benchmark scripts for the package. Use them as they are or modify them to test specific scenarios.
4+
5+
Example:
6+
7+
```
8+
mix run benchmarks/flag.exs
9+
```

‎benchmarks/cache.exs‎

Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# :observer.start
2+
3+
# Start the ecto repo if running the benchmarks with ecto.
4+
# {:ok, _pid} = FunWithFlags.Dev.EctoRepo.start_link()
5+
6+
FunWithFlags.clear(:one)
7+
FunWithFlags.clear(:two)
8+
FunWithFlags.clear(:three)
9+
FunWithFlags.clear(:four)
10+
11+
alias PlainUser, as: User
12+
13+
u1 = %User{id: 1, group: "foo"}
14+
u2 = %User{id: 2, group: "foo"}
15+
u3 = %User{id: 3, group: "bar"}
16+
u4 = %User{id: 4, group: "bar"}
17+
18+
FunWithFlags.enable(:one)
19+
20+
FunWithFlags.enable(:two)
21+
FunWithFlags.enable(:two, for_actor: u4)
22+
FunWithFlags.disable(:two, for_group: "nope")
23+
24+
FunWithFlags.disable(:three)
25+
FunWithFlags.enable(:three, for_actor: u2)
26+
FunWithFlags.enable(:three, for_actor: u3)
27+
FunWithFlags.enable(:three, for_actor: u4)
28+
FunWithFlags.disable(:three, for_group: "nope")
29+
FunWithFlags.disable(:three, for_group: "nope2")
30+
31+
32+
FunWithFlags.disable(:four)
33+
FunWithFlags.enable(:four, for_actor: u2)
34+
FunWithFlags.enable(:four, for_actor: u3)
35+
FunWithFlags.enable(:four, for_actor: u4)
36+
FunWithFlags.enable(:four, for_actor: "a")
37+
FunWithFlags.enable(:four, for_actor: "b")
38+
FunWithFlags.enable(:four, for_actor: "c")
39+
FunWithFlags.enable(:four, for_actor: "d")
40+
FunWithFlags.enable(:four, for_actor: "e")
41+
FunWithFlags.disable(:four, for_group: "nope")
42+
FunWithFlags.disable(:four, for_group: "nope2")
43+
FunWithFlags.disable(:four, for_group: "nope3")
44+
FunWithFlags.disable(:four, for_group: "nope4")
45+
FunWithFlags.enable(:four, for_percentage_of: {:actors, 0.99})
46+
47+
# warm up the cache
48+
FunWithFlags.enabled?(:one)
49+
FunWithFlags.enabled?(:two)
50+
FunWithFlags.enabled?(:three)
51+
FunWithFlags.enabled?(:four)
52+
53+
54+
alias FunWithFlags.Store.Cache
55+
56+
# -----------------------------------
57+
one = fn() ->
58+
Cache.get(:one)
59+
end
60+
61+
two = fn() ->
62+
Cache.get(:two)
63+
end
64+
65+
three = fn() ->
66+
Cache.get(:three)
67+
end
68+
69+
four = fn() ->
70+
Cache.get(:four)
71+
end
72+
73+
74+
Benchee.run(
75+
%{
76+
"one" => one,
77+
"two" => two,
78+
"three" => three,
79+
"four" => four,
80+
}#,
81+
# formatters: [
82+
# Benchee.Formatters.HTML,
83+
# Benchee.Formatters.Console
84+
# ]
85+
)

‎benchmarks/flag.exs‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
# :observer.start
2+
3+
# Start the ecto repo if running the benchmarks with ecto.
4+
# {:ok, _pid} = FunWithFlags.Dev.EctoRepo.start_link()
5+
6+
FunWithFlags.clear(:one)
7+
FunWithFlags.clear(:two)
8+
FunWithFlags.clear(:three)
9+
FunWithFlags.clear(:four)
10+
11+
alias PlainUser, as: User
12+
13+
u1 = %User{id: 1, group: "foo"}
14+
u2 = %User{id: 2, group: "foo"}
15+
u3 = %User{id: 3, group: "bar"}
16+
u4 = %User{id: 4, group: "bar"}
17+
18+
FunWithFlags.enable(:one)
19+
20+
FunWithFlags.enable(:two)
21+
FunWithFlags.enable(:two, for_actor: u4)
22+
FunWithFlags.disable(:two, for_group: "nope")
23+
24+
FunWithFlags.disable(:three)
25+
FunWithFlags.enable(:three, for_actor: u2)
26+
FunWithFlags.enable(:three, for_actor: u3)
27+
FunWithFlags.enable(:three, for_actor: u4)
28+
FunWithFlags.disable(:three, for_group: "nope")
29+
FunWithFlags.disable(:three, for_group: "nope2")
30+
31+
32+
FunWithFlags.disable(:four)
33+
FunWithFlags.enable(:four, for_actor: u2)
34+
FunWithFlags.enable(:four, for_actor: u3)
35+
FunWithFlags.enable(:four, for_actor: u4)
36+
FunWithFlags.enable(:four, for_actor: "a")
37+
FunWithFlags.enable(:four, for_actor: "b")
38+
FunWithFlags.enable(:four, for_actor: "c")
39+
FunWithFlags.enable(:four, for_actor: "d")
40+
FunWithFlags.enable(:four, for_actor: "e")
41+
FunWithFlags.disable(:four, for_group: "nope")
42+
FunWithFlags.disable(:four, for_group: "nope2")
43+
FunWithFlags.disable(:four, for_group: "nope3")
44+
FunWithFlags.disable(:four, for_group: "nope4")
45+
FunWithFlags.enable(:four, for_percentage_of: {:actors, 0.99})
46+
47+
# warm up the cache
48+
FunWithFlags.enabled?(:one)
49+
FunWithFlags.enabled?(:two)
50+
FunWithFlags.enabled?(:three)
51+
FunWithFlags.enabled?(:four)
52+
53+
# -----------------------------------
54+
one_a = fn() ->
55+
FunWithFlags.enabled?(:one)
56+
end
57+
58+
one_b = fn() ->
59+
FunWithFlags.enabled?(:one, for: u1)
60+
end
61+
62+
two_a = fn() ->
63+
FunWithFlags.enabled?(:two)
64+
end
65+
66+
two_b = fn() ->
67+
FunWithFlags.enabled?(:two, for: u1)
68+
end
69+
70+
three_a = fn() ->
71+
FunWithFlags.enabled?(:three)
72+
end
73+
74+
three_b = fn() ->
75+
FunWithFlags.enabled?(:three, for: u1)
76+
end
77+
78+
four_a = fn() ->
79+
FunWithFlags.enabled?(:four)
80+
end
81+
82+
four_b = fn() ->
83+
FunWithFlags.enabled?(:four, for: u1)
84+
end
85+
86+
Benchee.run(
87+
%{
88+
"one_a" => one_a,
89+
"one_b" => one_b,
90+
"two_a" => two_a,
91+
"two_b" => two_b,
92+
"three_a" => three_a,
93+
"three_b" => three_b,
94+
"four_a" => four_a,
95+
"four_b" => four_b,
96+
}#,
97+
# formatters: [
98+
# Benchee.Formatters.HTML,
99+
# Benchee.Formatters.Console
100+
# ]
101+
)

‎benchmarks/persistence.exs‎

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
# Test the performance of the persistence adapters.
2+
# This benchmark is mostly affected by the performance of the underlying datastore.
3+
# However, it's also useful to assess how the store is accessed in Elixir. For example,
4+
# when switching from compiled-in config to just straight calls to the config module.
5+
6+
# :observer.start
7+
8+
# Start the ecto repo if running the benchmarks with ecto.
9+
# {:ok, _pid} = FunWithFlags.Dev.EctoRepo.start_link()
10+
11+
FunWithFlags.clear(:one)
12+
FunWithFlags.clear(:two)
13+
FunWithFlags.clear(:three)
14+
FunWithFlags.clear(:four)
15+
16+
alias PlainUser, as: User
17+
18+
u1 = %User{id: 1, group: "foo"}
19+
u2 = %User{id: 2, group: "foo"}
20+
u3 = %User{id: 3, group: "bar"}
21+
u4 = %User{id: 4, group: "bar"}
22+
23+
FunWithFlags.enable(:one)
24+
25+
FunWithFlags.enable(:two)
26+
FunWithFlags.enable(:two, for_actor: u4)
27+
FunWithFlags.disable(:two, for_group: "nope")
28+
29+
FunWithFlags.disable(:three)
30+
FunWithFlags.enable(:three, for_actor: u2)
31+
FunWithFlags.enable(:three, for_actor: u3)
32+
FunWithFlags.enable(:three, for_actor: u4)
33+
FunWithFlags.disable(:three, for_group: "nope")
34+
FunWithFlags.disable(:three, for_group: "nope2")
35+
36+
37+
FunWithFlags.disable(:four)
38+
FunWithFlags.enable(:four, for_actor: u2)
39+
FunWithFlags.enable(:four, for_actor: u3)
40+
FunWithFlags.enable(:four, for_actor: u4)
41+
FunWithFlags.enable(:four, for_actor: "a")
42+
FunWithFlags.enable(:four, for_actor: "b")
43+
FunWithFlags.enable(:four, for_actor: "c")
44+
FunWithFlags.enable(:four, for_actor: "d")
45+
FunWithFlags.enable(:four, for_actor: "e")
46+
FunWithFlags.disable(:four, for_group: "nope")
47+
FunWithFlags.disable(:four, for_group: "nope2")
48+
FunWithFlags.disable(:four, for_group: "nope3")
49+
FunWithFlags.disable(:four, for_group: "nope4")
50+
FunWithFlags.enable(:four, for_percentage_of: {:actors, 0.99})
51+
52+
alias FunWithFlags.SimpleStore
53+
54+
# -----------------------------------
55+
one = fn() ->
56+
SimpleStore.lookup(:one)
57+
end
58+
59+
two = fn() ->
60+
SimpleStore.lookup(:two)
61+
end
62+
63+
three = fn() ->
64+
SimpleStore.lookup(:three)
65+
end
66+
67+
four = fn() ->
68+
SimpleStore.lookup(:four)
69+
end
70+
71+
72+
Benchee.run(
73+
%{
74+
"one" => one,
75+
"two" => two,
76+
"three" => three,
77+
"four" => four,
78+
}
79+
)

‎dev_support/protocols.ex‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,3 +31,18 @@ defimpl FunWithFlags.Group, for: Map do
3131
def in?(_, _), do: false
3232
end
3333

34+
35+
defmodule PlainUser do
36+
defstruct [:id, :group]
37+
end
38+
39+
defimpl FunWithFlags.Actor, for: PlainUser do
40+
def id(%{id: id}) do
41+
"user:#{id}"
42+
end
43+
end
44+
45+
defimpl FunWithFlags.Group, for: PlainUser do
46+
def in?(%{group: group}, group), do: true
47+
def in?(_, _), do: false
48+
end

‎mix.exs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,10 @@ defmodule FunWithFlags.Mixfile do
7979

8080
{:ex_doc, "~> 0.21", only: :dev, runtime: false},
8181
{:credo, "~> 1.7", only: :dev, runtime: false},
82-
{:dialyxir, "~> 1.0", only: :dev, runtime: false}
82+
{:dialyxir, "~> 1.0", only: :dev, runtime: false},
83+
84+
{:benchee, "~> 1.0", only: :dev, runtime: false},
85+
{:benchee_html, "~> 1.0", only: :dev, runtime: false},
8386
]
8487
end
8588

‎mix.lock‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,14 @@
11
%{
2+
"benchee": {:hex, :benchee, "1.3.1", "c786e6a76321121a44229dde3988fc772bca73ea75170a73fd5f4ddf1af95ccf", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "76224c58ea1d0391c8309a8ecbfe27d71062878f59bd41a390266bf4ac1cc56d"},
3+
"benchee_html": {:hex, :benchee_html, "1.0.1", "1e247c0886c3fdb0d3f4b184b653a8d6fb96e4ad0d0389267fe4f36968772e24", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:benchee_json, "~> 1.0", [hex: :benchee_json, repo: "hexpm", optional: false]}], "hexpm", "b00a181af7152431901e08f3fc9f7197ed43ff50421a8347b0c80bf45d5b3fef"},
4+
"benchee_json": {:hex, :benchee_json, "1.0.0", "cc661f4454d5995c08fe10dd1f2f72f229c8f0fb1c96f6b327a8c8fc96a91fe5", [:mix], [{:benchee, ">= 0.99.0 and < 2.0.0", [hex: :benchee, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "da05d813f9123505f870344d68fb7c86a4f0f9074df7d7b7e2bb011a63ec231c"},
25
"bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
36
"cc_precompiler": {:hex, :cc_precompiler, "0.1.10", "47c9c08d8869cf09b41da36538f62bc1abd3e19e41701c2cea2675b53c704258", [:mix], [{:elixir_make, "~> 0.7", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "f6e046254e53cd6b41c6bacd70ae728011aa82b2742a80d6e2214855c6e06b22"},
47
"connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"},
58
"credo": {:hex, :credo, "1.7.11", "d3e805f7ddf6c9c854fd36f089649d7cf6ba74c42bc3795d587814e3c9847102", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "56826b4306843253a66e47ae45e98e7d284ee1f95d53d1612bb483f88a8cf219"},
69
"db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"},
710
"decimal": {:hex, :decimal, "2.3.0", "3ad6255aa77b4a3c4f818171b12d237500e63525c2fd056699967a3e7ea20f62", [:mix], [], "hexpm", "a4d66355cb29cb47c3cf30e71329e58361cfcb37c34235ef3bf1d7bf3773aeac"},
11+
"deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"},
812
"dialyxir": {:hex, :dialyxir, "1.4.5", "ca1571ac18e0f88d4ab245f0b60fa31ff1b12cbae2b11bd25d207f865e8ae78a", [:mix], [{:erlex, ">= 0.2.7", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b0fb08bb8107c750db5c0b324fa2df5ceaa0f9307690ee3c1f6ba5b9eb5d35c3"},
913
"earmark": {:hex, :earmark, "1.4.7", "7b5f0474469688f5514948a4e0c42955a5690d165deaf19c8eb950a3443a40f3", [:mix], [], "hexpm", "0a55a49ee6fa8bc8678f894dfa8a882af6fe8deb65aed6856122226db5b0fe5b"},
1014
"earmark_parser": {:hex, :earmark_parser, "1.4.43", "34b2f401fe473080e39ff2b90feb8ddfeef7639f8ee0bbf71bb41911831d77c5", [:mix], [], "hexpm", "970a3cd19503f5e8e527a190662be2cee5d98eed1ff72ed9b3d1a3d466692de8"},
@@ -28,5 +32,6 @@
2832
"phoenix_pubsub": {:hex, :phoenix_pubsub, "2.1.3", "3168d78ba41835aecad272d5e8cd51aa87a7ac9eb836eabc42f6e57538e3731d", [:mix], [], "hexpm", "bba06bc1dcfd8cb086759f0edc94a8ba2bc8896d5331a1e2c2902bf8e36ee502"},
2933
"postgrex": {:hex, :postgrex, "0.20.0", "363ed03ab4757f6bc47942eff7720640795eb557e1935951c1626f0d303a3aed", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "d36ef8b36f323d29505314f704e21a1a038e2dc387c6409ee0cd24144e187c0f"},
3034
"redix": {:hex, :redix, "1.5.2", "ab854435a663f01ce7b7847f42f5da067eea7a3a10c0a9d560fa52038fd7ab48", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:nimble_options, "~> 0.5.0 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "78538d184231a5d6912f20567d76a49d1be7d3fca0e1aaaa20f4df8e1142dcb8"},
35+
"statistex": {:hex, :statistex, "1.0.0", "f3dc93f3c0c6c92e5f291704cf62b99b553253d7969e9a5fa713e5481cd858a5", [:mix], [], "hexpm", "ff9d8bee7035028ab4742ff52fc80a2aa35cece833cf5319009b52f1b5a86c27"},
3136
"telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"},
3237
}

0 commit comments

Comments
 (0)