From d6bf5f752ff49451ee87d754fddba777ca0a3159 Mon Sep 17 00:00:00 2001 From: Jan-Pieter Baert Date: Thu, 15 Jul 2021 17:44:25 +0200 Subject: [PATCH 1/6] Fix error when logging Co-authored-by: madnificent --- lib/manipulators/cache_key_logger.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/manipulators/cache_key_logger.ex b/lib/manipulators/cache_key_logger.ex index cdc5e54..7f35965 100644 --- a/lib/manipulators/cache_key_logger.ex +++ b/lib/manipulators/cache_key_logger.ex @@ -38,7 +38,9 @@ defmodule Manipulators.CacheKeyLogger do end defp header_value(headers, header_name) do - header = Enum.find(headers, header_name) + header = + headers + |> Enum.find({nil, "[]"}, &match?({header_name, _}, &1)) if header do elem(header, 1) From 90d14eda3ca23698c117f3d8bce98abd1cfb125a Mon Sep 17 00:00:00 2001 From: Jan-Pieter Baert Date: Fri, 16 Jul 2021 15:48:00 +0200 Subject: [PATCH 2/6] Add support file This support file is from the mu-elixir-template and should be replaced by integrating the template in here. --- lib/support.ex | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++ mix.exs | 4 +++- 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 lib/support.ex diff --git a/lib/support.ex b/lib/support.ex new file mode 100644 index 0000000..2f1e4ac --- /dev/null +++ b/lib/support.ex @@ -0,0 +1,62 @@ +defmodule Support do + # environment variables + def sparql_endpoint() do + System.get_env("MU_SPARQL_ENDPOINT") + end + def application_graph() do + System.get_env("MU_APPLICATION_GRAPH") + end + def sparql_timeout() do + System.get_env("MU_SPARQL_TIMEOUT") + end + def log_level() do + System.get_env("LOG_LEVEL") + end + + # uuid helper + def generate_uuid() do + UUID.uuid4() + end + + # graph helper + def graph() do + application_graph() + end + + # query helper + def _query_get_URL(escaped_query_string) do + if sparql_timeout() > 0 do + "#{sparql_endpoint()}?query=#{escaped_query_string}&timeout=#{sparql_timeout()}" + else + "#{sparql_endpoint()}?query=#{escaped_query_string}" + end + end + + def query(query_string) do + query_string + |> URI.encode + |> _query_get_URL + |> HTTPoison.get!([{"Accept", "application/json"}]) + |> ( &( &1.body ) ).() + |> Poison.decode! + end + + def update(query_string) do + HTTPoison.post!( + sparql_endpoint(), + [ + "query=" <> + URI.encode_www_form(query_string) <> + "&format=" <> URI.encode_www_form("application/sparql-results+json") + ], + ["Content-Type": "application/x-www-form-urlencoded"], + [] + ) + |> ( &( &1.body ) ).() + |> Poison.decode! + end + + def sparql_escape(value) do + String.replace(value, "\"", "\\\"") + end +end diff --git a/mix.exs b/mix.exs index 41e47b0..8103c71 100644 --- a/mix.exs +++ b/mix.exs @@ -48,7 +48,9 @@ defmodule UsePlugProxy.Mixfile do {:plug_cowboy, "~> 2.4.1"}, {:dialyxir, "~> 1.0", only: [:dev], runtime: false}, {:credo, "~> 1.5", only: [:dev, :test], runtime: false}, - {:poison, "~> 2.0"} + {:poison, "~> 2.0"}, + {:uuid, "~> 1.1"}, + {:httpoison, "~> 1.5"} ] end end From 61f26c8f7b55f76b1637d05eb6f06699b47d540e Mon Sep 17 00:00:00 2001 From: Jan-Pieter Baert Date: Fri, 16 Jul 2021 15:51:07 +0200 Subject: [PATCH 3/6] Add cache clear events to the database --- lib/cache/cache.ex | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/lib/cache/cache.ex b/lib/cache/cache.ex index 201360e..273dc97 100644 --- a/lib/cache/cache.ex +++ b/lib/cache/cache.ex @@ -93,13 +93,18 @@ defmodule Cache.Registry do # We have multiple clear_keys and need to update the state for it. %{cache: cache, caches_by_key: caches_by_key} = state + # IO.inspect(clear_keys, label: "Clearing these keys") + # IO.inspect(cache, label: "Cache before clearing the keys") + cache = Enum.reduce(clear_keys, cache, fn clear_key, cache -> keys_to_remove = Map.get(caches_by_key, clear_key, []) + log_key_to_remove!(keys_to_remove) cache = Map.drop(cache, keys_to_remove) cache end) + # IO.inspect(cache, label: "Cache after clearing the keys") caches_by_key = Map.drop(caches_by_key, clear_keys) %{state | cache: cache, caches_by_key: caches_by_key} @@ -135,4 +140,29 @@ defmodule Cache.Registry do %{state | cache: cache} end + + defp log_key_to_remove!(request_key_to_remove) do + # Write to the database when a cache for a url is cleared + Enum.map(request_key_to_remove, fn req_key -> + {method, path, query, auth} = req_key + uuid = Support.generate_uuid() + auth_escaped = Support.sparql_escape(auth) + query = """ + PREFIX mu: + PREFIX mucache: + INSERT DATA { + GRAPH { + a mucache:CacheClear ; + mu:uuid "#{uuid}"; + mucache:path "#{path}"; + mucache:method "#{method}"; + mucache:query "#{query}"; + mucache:muAuthAllowedGroups \"\"\"#{auth_escaped}\"\"\"; + mucache:muAuthUsedGroups \"\"\"#{auth_escaped}\"\"\". + } + } + """ + sparql_request = Support.update(query) + end) + end end From 348779711ce4144e31a170d08be2ed94c3ea42ca Mon Sep 17 00:00:00 2001 From: Jan-Pieter Baert Date: Sun, 18 Jul 2021 23:27:48 +0200 Subject: [PATCH 4/6] Remove unused variable --- lib/cache/cache.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cache/cache.ex b/lib/cache/cache.ex index 273dc97..04d6336 100644 --- a/lib/cache/cache.ex +++ b/lib/cache/cache.ex @@ -162,7 +162,7 @@ defmodule Cache.Registry do } } """ - sparql_request = Support.update(query) + Support.update(query) end) end end From 8427bf4e03ab9367a989830f9e28aca076b18d9f Mon Sep 17 00:00:00 2001 From: Jan-Pieter Baert Date: Thu, 22 Jul 2021 18:14:27 +0200 Subject: [PATCH 5/6] Add note to support file --- lib/support.ex | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/support.ex b/lib/support.ex index 2f1e4ac..75e40fa 100644 --- a/lib/support.ex +++ b/lib/support.ex @@ -1,3 +1,4 @@ +# Note: this is copied from https://github.com/mu-semtech/mu-elixir-template should be replaced by integrating the template defmodule Support do # environment variables def sparql_endpoint() do From 193c99fc4afc02b563d436f34a15c31291d0cb26 Mon Sep 17 00:00:00 2001 From: Jan-Pieter Baert Date: Thu, 22 Jul 2021 18:15:04 +0200 Subject: [PATCH 6/6] Make logging the cache clear events optional --- README.md | 1 + config/config.exs | 1 + lib/cache/cache.ex | 46 ++++++++++++++++++++++++---------------------- 3 files changed, 26 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 233aef6..a540b14 100644 --- a/README.md +++ b/README.md @@ -30,5 +30,6 @@ These services may need to be configured in order to enable mu-cache support. Ch ## Debugging Debugging of cache keys is helped by following environment variables: + - `LOG_CACHE_CLEAR_EVENT`: Store cache clear events in the database - `LOG_CACHE_KEYS`: Logs received cache key to a response - `LOG_CLEAR_KEYS`: Logs received clear keys either as a response, or explicitly received through ./mu/delta. diff --git a/config/config.exs b/config/config.exs index 7e440ca..b777a4e 100644 --- a/config/config.exs +++ b/config/config.exs @@ -21,6 +21,7 @@ end # 3rd-party users, it should be done in your "mix.exs" file. config :mu_cache, + log_cache_clear_event: CH.system_boolean("LOG_CACHE_CLEAR_EVENT"), log_cache_keys: CH.system_boolean("LOG_CACHE_KEYS"), log_clear_keys: CH.system_boolean("LOG_CLEAR_KEYS") diff --git a/lib/cache/cache.ex b/lib/cache/cache.ex index 04d6336..7c2ad22 100644 --- a/lib/cache/cache.ex +++ b/lib/cache/cache.ex @@ -99,7 +99,7 @@ defmodule Cache.Registry do cache = Enum.reduce(clear_keys, cache, fn clear_key, cache -> keys_to_remove = Map.get(caches_by_key, clear_key, []) - log_key_to_remove!(keys_to_remove) + maybe_send_cleared_key_to_database!(keys_to_remove) cache = Map.drop(cache, keys_to_remove) cache end) @@ -141,28 +141,30 @@ defmodule Cache.Registry do %{state | cache: cache} end - defp log_key_to_remove!(request_key_to_remove) do + defp maybe_send_cleared_key_to_database!(request_key_to_remove) do # Write to the database when a cache for a url is cleared - Enum.map(request_key_to_remove, fn req_key -> - {method, path, query, auth} = req_key - uuid = Support.generate_uuid() - auth_escaped = Support.sparql_escape(auth) - query = """ - PREFIX mu: - PREFIX mucache: - INSERT DATA { - GRAPH { - a mucache:CacheClear ; - mu:uuid "#{uuid}"; - mucache:path "#{path}"; - mucache:method "#{method}"; - mucache:query "#{query}"; - mucache:muAuthAllowedGroups \"\"\"#{auth_escaped}\"\"\"; - mucache:muAuthUsedGroups \"\"\"#{auth_escaped}\"\"\". + if Application.get_env(:mu_cache, :log_cache_clear_event) do + Enum.map(request_key_to_remove, fn req_key -> + {method, path, query, auth} = req_key + uuid = Support.generate_uuid() + auth_escaped = Support.sparql_escape(auth) + query = """ + PREFIX mu: + PREFIX mucache: + INSERT DATA { + GRAPH { + a mucache:CacheClear ; + mu:uuid "#{uuid}"; + mucache:path "#{path}"; + mucache:method "#{method}"; + mucache:query "#{query}"; + mucache:muAuthAllowedGroups \"\"\"#{auth_escaped}\"\"\"; + mucache:muAuthUsedGroups \"\"\"#{auth_escaped}\"\"\". + } } - } - """ - Support.update(query) - end) + """ + Support.update(query) + end) + end end end