From d61a61efecde84a01b6d63f76b0b261a56c6a376 Mon Sep 17 00:00:00 2001 From: Aaron Tavistock Date: Sun, 29 Sep 2024 18:19:24 -0700 Subject: [PATCH 1/7] Implementing suggestions from @josevalim --- .../Elixir/phoenix/lib/hello/world_cache.ex | 2 +- frameworks/Elixir/phoenix/lib/hello_web.ex | 4 +- .../hello_web/controllers/page_controller.ex | 68 ++++++++++--------- .../Elixir/phoenix/lib/hello_web/endpoint.ex | 1 - 4 files changed, 40 insertions(+), 35 deletions(-) diff --git a/frameworks/Elixir/phoenix/lib/hello/world_cache.ex b/frameworks/Elixir/phoenix/lib/hello/world_cache.ex index b4943e14a35..e6cde7ab58e 100644 --- a/frameworks/Elixir/phoenix/lib/hello/world_cache.ex +++ b/frameworks/Elixir/phoenix/lib/hello/world_cache.ex @@ -23,9 +23,9 @@ defmodule Hello.WorldCache do world = Repo.get(World, id) :ok = __MODULE__.put(id, world) world + world -> world end end - end diff --git a/frameworks/Elixir/phoenix/lib/hello_web.ex b/frameworks/Elixir/phoenix/lib/hello_web.ex index 1193aab23a5..1114ed22e51 100644 --- a/frameworks/Elixir/phoenix/lib/hello_web.ex +++ b/frameworks/Elixir/phoenix/lib/hello_web.ex @@ -95,8 +95,8 @@ defmodule HelloWeb do end @doc """ - When used, dispatch to the appropriate controller/view/etc. - """ + When used, dispatch to the appropriate controller/view/etc. + """ defmacro __using__(which) when is_atom(which) do apply(__MODULE__, which, []) end diff --git a/frameworks/Elixir/phoenix/lib/hello_web/controllers/page_controller.ex b/frameworks/Elixir/phoenix/lib/hello_web/controllers/page_controller.ex index 34c1fc120fd..2ce36b38551 100644 --- a/frameworks/Elixir/phoenix/lib/hello_web/controllers/page_controller.ex +++ b/frameworks/Elixir/phoenix/lib/hello_web/controllers/page_controller.ex @@ -1,5 +1,4 @@ defmodule HelloWeb.PageController do - use HelloWeb, :controller alias Hello.Models.Fortune @@ -25,13 +24,12 @@ defmodule HelloWeb.PageController do end def queries(conn, params) do - :rand.seed(:exsp) - worlds = - Stream.repeatedly(&random_id/0) - |> Stream.uniq() - |> Stream.map(&Repo.get(World, &1)) - |> Enum.take(size(params["queries"])) + Repo.checkout(fn -> + params["queries"] + |> random_ids_sample() + |> Enum.map(&Repo.get(World, &1)) + end) json(conn, worlds) end @@ -44,34 +42,38 @@ defmodule HelloWeb.PageController do fortunes = [additional_fortune | Repo.all(Fortune)] - |> Enum.sort_by(& &1.message) + |> Enum.sort(fn a, b -> a.message < b.message end) render(conn, :fortunes, fortunes: fortunes) end def updates(conn, params) do - :rand.seed(:exsp) - - worlds = - Stream.repeatedly(&random_id/0) - |> Stream.uniq() - |> Stream.map(&Repo.get(World, &1)) - |> Stream.map(fn world -> %{id: world.id, randomnumber: :rand.uniform(@random_max)} end) - |> Enum.take(size(params["queries"])) - # If this is not sorted it sometimes generates - # FAIL for http://tfb-server:8080/updates/20 - # Only 20470 executed queries in the database out of roughly 20480 expected. - |> Enum.sort_by(& &1.id) + world_updates = + Repo.checkout(fn -> + params["queries"] + |> random_ids_sample() + |> Enum.sort() + # + # If this is not sorted it will intermittently generate: + # + # FAIL for http://tfb-server:8080/updates/20 + # Only 20470 executed queries in the database out of roughly 20480 expected. + # + |> Enum.map(fn id -> + world = Repo.get(World, id) + %{id: world.id, randomnumber: :rand.uniform(@random_max)} + end) + end) Repo.insert_all( World, - worlds, + world_updates, on_conflict: {:replace_all_except, [:id]}, conflict_target: [:id], returning: false ) - json(conn, worlds) + json(conn, world_updates) end def plaintext(conn, _params) do @@ -79,14 +81,12 @@ defmodule HelloWeb.PageController do end def cached(conn, params) do - :rand.seed(:exsp) WorldCache.seed() worlds = - Stream.repeatedly(&random_id/0) - |> Stream.uniq() - |> Stream.map(&WorldCache.fetch(&1)) - |> Enum.take(size(params["count"])) + params["count"] + |> random_ids_sample() + |> Enum.map(&WorldCache.fetch(&1)) json(conn, worlds) end @@ -95,11 +95,17 @@ defmodule HelloWeb.PageController do :rand.uniform(@random_max) end - defp size(nil), do: 1 - defp size(""), do: 1 + defp random_ids_sample(count) do + # Use the fastest rand algorithm + :rand.seed(:exsp) + + Stream.repeatedly(&random_id/0) + |> Stream.uniq() + |> Enum.take(size(count)) + end - defp size(queries) when is_bitstring(queries) do - case Integer.parse(queries) do + defp size(param_count) when is_bitstring(param_count) do + case Integer.parse(param_count) do {count, _} -> max(1, min(500, count)) _ -> 1 end diff --git a/frameworks/Elixir/phoenix/lib/hello_web/endpoint.ex b/frameworks/Elixir/phoenix/lib/hello_web/endpoint.ex index f709d00a2e8..a764bbf653e 100644 --- a/frameworks/Elixir/phoenix/lib/hello_web/endpoint.ex +++ b/frameworks/Elixir/phoenix/lib/hello_web/endpoint.ex @@ -20,4 +20,3 @@ defmodule HelloWeb.Endpoint do plug HelloWeb.HeadersPlug plug HelloWeb.Router end - From 55170e0dad6ded32b7e7d6d481aee9e92f65fef2 Mon Sep 17 00:00:00 2001 From: Aaron Tavistock Date: Sun, 29 Sep 2024 18:29:50 -0700 Subject: [PATCH 2/7] We don't need to worry about 'accepts' plug in this case --- frameworks/Elixir/phoenix/lib/hello_web/router.ex | 6 ------ 1 file changed, 6 deletions(-) diff --git a/frameworks/Elixir/phoenix/lib/hello_web/router.ex b/frameworks/Elixir/phoenix/lib/hello_web/router.ex index 1634f3403fd..7e3caed3b8f 100755 --- a/frameworks/Elixir/phoenix/lib/hello_web/router.ex +++ b/frameworks/Elixir/phoenix/lib/hello_web/router.ex @@ -1,13 +1,7 @@ defmodule HelloWeb.Router do use HelloWeb, :router - pipeline :browser do - plug :accepts, ~w(html json) - end - scope "/", HelloWeb do - pipe_through [:browser] - get "/json", PageController, :_json get "/db", PageController, :db get "/queries", PageController, :queries From 954b5700576b451a18f7a8854ea2e7ebe448a9ef Mon Sep 17 00:00:00 2001 From: Aaron Tavistock Date: Sun, 29 Sep 2024 20:53:08 -0700 Subject: [PATCH 3/7] Prematurely removed accepts plug (still needed for 'fortunes' test) --- .../Elixir/phoenix/lib/hello_web/controllers/page_controller.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frameworks/Elixir/phoenix/lib/hello_web/controllers/page_controller.ex b/frameworks/Elixir/phoenix/lib/hello_web/controllers/page_controller.ex index 2ce36b38551..4a83bb67b68 100644 --- a/frameworks/Elixir/phoenix/lib/hello_web/controllers/page_controller.ex +++ b/frameworks/Elixir/phoenix/lib/hello_web/controllers/page_controller.ex @@ -8,6 +8,8 @@ defmodule HelloWeb.PageController do @random_max 10_000 + plug :accepts, ~w(html json) when action in [:fortunes] + def index(conn, _params) do json(conn, %{"TE Benchmarks\n" => "Started"}) end From 5691623f091aeba9a5ead9e35cf839cc649ec443 Mon Sep 17 00:00:00 2001 From: Aaron Tavistock Date: Sun, 29 Sep 2024 21:24:28 -0700 Subject: [PATCH 4/7] Avoid checking a list we don't need to --- .../Elixir/phoenix/lib/hello_web/controllers/page_controller.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frameworks/Elixir/phoenix/lib/hello_web/controllers/page_controller.ex b/frameworks/Elixir/phoenix/lib/hello_web/controllers/page_controller.ex index 4a83bb67b68..d3cb32504e2 100644 --- a/frameworks/Elixir/phoenix/lib/hello_web/controllers/page_controller.ex +++ b/frameworks/Elixir/phoenix/lib/hello_web/controllers/page_controller.ex @@ -8,7 +8,7 @@ defmodule HelloWeb.PageController do @random_max 10_000 - plug :accepts, ~w(html json) when action in [:fortunes] + plug :accepts, ~w(html json) when action == :fortunes def index(conn, _params) do json(conn, %{"TE Benchmarks\n" => "Started"}) From 426829b2444ee22183c5be81368e143075792853 Mon Sep 17 00:00:00 2001 From: Aaron Tavistock Date: Thu, 10 Oct 2024 11:29:00 -0700 Subject: [PATCH 5/7] Using explicit content-type for plain-text --- .../phoenix/lib/hello_web/controllers/page_controller.ex | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/frameworks/Elixir/phoenix/lib/hello_web/controllers/page_controller.ex b/frameworks/Elixir/phoenix/lib/hello_web/controllers/page_controller.ex index d3cb32504e2..59f2f5c2f35 100644 --- a/frameworks/Elixir/phoenix/lib/hello_web/controllers/page_controller.ex +++ b/frameworks/Elixir/phoenix/lib/hello_web/controllers/page_controller.ex @@ -78,8 +78,10 @@ defmodule HelloWeb.PageController do json(conn, world_updates) end - def plaintext(conn, _params) do - text(conn, "Hello, World!") + def plaintext(conn, _params) do + conn + |> put_resp_header("content-type", "text/plain") + |> send_resp(200, "Hello, World!") end def cached(conn, params) do From d7430486dfa2921ddfa8c3567a0c727590bfee70 Mon Sep 17 00:00:00 2001 From: Aaron Tavistock Date: Thu, 10 Oct 2024 13:06:12 -0700 Subject: [PATCH 6/7] Updating ecto to use pool count (also tested different count/size groupings) --- frameworks/Elixir/phoenix/config/prod.exs | 17 ++--------------- frameworks/Elixir/phoenix/mix.lock | 4 ++-- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/frameworks/Elixir/phoenix/config/prod.exs b/frameworks/Elixir/phoenix/config/prod.exs index 9ad458a09be..c1180b8e30e 100755 --- a/frameworks/Elixir/phoenix/config/prod.exs +++ b/frameworks/Elixir/phoenix/config/prod.exs @@ -21,7 +21,8 @@ config :hello, Hello.Repo, password: "benchmarkdbpass", database: "hello_world", hostname: "tfb-database", - pool_size: 50, + pool_count: 24, + pool_size: 64, queue_target: 5000, log: false @@ -33,17 +34,3 @@ config :logger, ], level: :error, backends: [] - -# ## SSL Support -# -# To get SSL working, you will need to add the `https` key -# to the previous section: -# -# config:hello, Hello.Endpoint, -# ... -# https: [port: 443, -# keyfile: System.get_env("SOME_APP_SSL_KEY_PATH"), -# certfile: System.get_env("SOME_APP_SSL_CERT_PATH")] -# -# Where those two env variables point to a file on -# disk for the key and cert. diff --git a/frameworks/Elixir/phoenix/mix.lock b/frameworks/Elixir/phoenix/mix.lock index 4c59758f12a..0c47cab2458 100644 --- a/frameworks/Elixir/phoenix/mix.lock +++ b/frameworks/Elixir/phoenix/mix.lock @@ -6,8 +6,8 @@ "cowlib": {:hex, :cowlib, "2.13.0", "db8f7505d8332d98ef50a3ef34b34c1afddec7506e4ee4dd4a3a266285d282ca", [:make, :rebar3], [], "hexpm", "e1e1284dc3fc030a64b1ad0d8382ae7e99da46c3246b815318a4b848873800a4"}, "db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"}, "decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"}, - "ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"}, - "ecto_sql": {:hex, :ecto_sql, "3.11.3", "4eb7348ff8101fbc4e6bbc5a4404a24fecbe73a3372d16569526b0cf34ebc195", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.11.0", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.6", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.16 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "e5f36e3d736b99c7fee3e631333b8394ade4bafe9d96d35669fca2d81c2be928"}, + "ecto": {:hex, :ecto, "3.12.4", "267c94d9f2969e6acc4dd5e3e3af5b05cdae89a4d549925f3008b2b7eb0b93c3", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ef04e4101688a67d061e1b10d7bc1fbf00d1d13c17eef08b71d070ff9188f747"}, + "ecto_sql": {:hex, :ecto_sql, "3.12.1", "c0d0d60e85d9ff4631f12bafa454bc392ce8b9ec83531a412c12a0d415a3a4d0", [:mix], [{:db_connection, "~> 2.4.1 or ~> 2.5", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.12", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.7", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.19 or ~> 1.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.1 or ~> 2.2", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "aff5b958a899762c5f09028c847569f7dfb9cc9d63bdb8133bff8a5546de6bf5"}, "expo": {:hex, :expo, "1.0.0", "647639267e088717232f4d4451526e7a9de31a3402af7fcbda09b27e9a10395a", [:mix], [], "hexpm", "18d2093d344d97678e8a331ca0391e85d29816f9664a25653fd7e6166827827c"}, "gettext": {:hex, :gettext, "0.25.0", "98a95a862a94e2d55d24520dd79256a15c87ea75b49673a2e2f206e6ebc42e5d", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "38e5d754e66af37980a94fb93bb20dcde1d2361f664b0a19f01e87296634051f"}, "hpax": {:hex, :hpax, "1.0.0", "28dcf54509fe2152a3d040e4e3df5b265dcb6cb532029ecbacf4ce52caea3fd2", [:mix], [], "hexpm", "7f1314731d711e2ca5fdc7fd361296593fc2542570b3105595bb0bc6d0fad601"}, From cf7babfa11ceae27dd42fb7201a2a0703e66dda4 Mon Sep 17 00:00:00 2001 From: Aaron Tavistock Date: Thu, 10 Oct 2024 17:02:35 -0700 Subject: [PATCH 7/7] db pools set for the target machine --- frameworks/Elixir/phoenix/config/prod.exs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frameworks/Elixir/phoenix/config/prod.exs b/frameworks/Elixir/phoenix/config/prod.exs index c1180b8e30e..c920950ee04 100755 --- a/frameworks/Elixir/phoenix/config/prod.exs +++ b/frameworks/Elixir/phoenix/config/prod.exs @@ -21,8 +21,8 @@ config :hello, Hello.Repo, password: "benchmarkdbpass", database: "hello_world", hostname: "tfb-database", - pool_count: 24, - pool_size: 64, + pool_count: 56, + pool_size: 15, queue_target: 5000, log: false