DashboardQueryParser & DashboardQuerySerializer (#5938)

* rename query_parser_test to api_query_parser_test

* allow metrics to be nil in ParsedQueryParams

* swap now with relative_date in ParsedQueryParams

* add DashboardQueryParser

* stop defining defaults in ParsedQueryParams

* add DashboardQuerySerializer

* make sure parse -> serialize is a reversible transformation

* fix codespell

* fix test and silence credo

* fix another test

* parse and serialize with_imported

* cleaner decode_filters

* precompile do_not_url_encode_map and simplify uri_encode_permissive

* remove prepending ? logic

Author: RobertJoonas

assets/js/dashboard/util/url-search-params.ts

@@ -3,7 +3,7 @@ import { v1 } from './url-search-params-v1'
 import { v2 } from './url-search-params-v2'
 
 /**
- * These charcters are not URL encoded to have more readable URLs.
+ * These characters are not URL encoded to have more readable URLs.
  * Browsers seem to handle this just fine.
  * `?f=is,page,/my/page/:some_param` vs `?f=is,page,%2Fmy%2Fpage%2F%3Asome_param``
  */

lib/plausible/stats/api_query_parser.ex

@@ -5,8 +5,23 @@ defmodule Plausible.Stats.ApiQueryParser do
 
   alias Plausible.Stats.{Filters, Metrics, DateTimeRange, JSONSchema}
 
+  @default_include %{
+    imports: false,
+    imports_meta: false,
+    time_labels: false,
+    total_rows: false,
+    trim_relative_date_range: false,
+    comparisons: nil,
+    legacy_time_on_page_cutoff: nil
+  }
+
+  def default_include(), do: @default_include
+
+  @default_pagination %{limit: 10_000, offset: 0}
+
+  def default_pagination(), do: @default_pagination
+
   def parse(schema_type, params) when is_map(params) do
-    now = Plausible.Stats.Query.Test.get_fixed_now()
     input_date_range = Map.get(params, "date_range")
 
     with :ok <- JSONSchema.validate(schema_type, params),
@@ -19,7 +34,6 @@ defmodule Plausible.Stats.ApiQueryParser do
          {:ok, include} <- parse_include(params["include"]) do
       {:ok,
        Plausible.Stats.ParsedQueryParams.new!(%{
-         now: now,
          input_date_range: input_date_range,
          metrics: metrics,
          filters: filters,
@@ -61,7 +75,7 @@ defmodule Plausible.Stats.ApiQueryParser do
     parse_list(filters, &parse_filter/1)
   end
 
-  def parse_filters(nil), do: {:ok, nil}
+  def parse_filters(nil), do: {:ok, []}
 
   defp parse_filter(filter) do
     with {:ok, operator} <- parse_operator(filter),
@@ -208,7 +222,7 @@ defmodule Plausible.Stats.ApiQueryParser do
     )
   end
 
-  defp parse_dimensions(nil), do: {:ok, nil}
+  defp parse_dimensions(nil), do: {:ok, []}
 
   def parse_order_by(order_by) when is_list(order_by) do
     parse_list(order_by, &parse_order_by_entry/1)
@@ -261,17 +275,18 @@ defmodule Plausible.Stats.ApiQueryParser do
   defp parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{i(entry)}'."}
 
   def parse_include(include) when is_map(include) do
-    with {:ok, include} <- atomize_include_keys(include) do
-      parse_comparison_date_range(include)
+    with {:ok, include} <- atomize_include_keys(include),
+         {:ok, include} <- parse_comparison_date_range(include) do
+      {:ok, Map.merge(@default_include, include)}
     end
   end
 
-  def parse_include(nil), do: {:ok, nil}
+  def parse_include(nil), do: {:ok, @default_include}
   def parse_include(include), do: {:error, "Invalid include '#{i(include)}'."}
 
   defp atomize_include_keys(map) do
     expected_keys =
-      Plausible.Stats.ParsedQueryParams.default_include()
+      @default_include
       |> Map.keys()
       |> Enum.map(&Atom.to_string/1)
 
@@ -291,11 +306,10 @@ defmodule Plausible.Stats.ApiQueryParser do
   defp parse_comparison_date_range(include), do: {:ok, include}
 
   defp parse_pagination(pagination) when is_map(pagination) do
-    {:ok,
-     Map.merge(Plausible.Stats.ParsedQueryParams.default_pagination(), atomize_keys(pagination))}
+    {:ok, Map.merge(@default_pagination, atomize_keys(pagination))}
   end
 
-  defp parse_pagination(nil), do: {:ok, nil}
+  defp parse_pagination(nil), do: {:ok, @default_pagination}
 
   defp atomize_keys(map) when is_map(map) do
     Map.new(map, fn {key, value} ->

lib/plausible/stats/dashboard_query_parser.ex

@@ -0,0 +1,129 @@
+defmodule Plausible.Stats.DashboardQueryParser do
+  @moduledoc """
+  Parses a dashboard query string into `%ParsedQueryParams{}`. Note that
+  `metrics` and `dimensions` do not exist at this step yet, and are expected
+  to be filled in by each specific report.
+  """
+
+  alias Plausible.Stats.{ParsedQueryParams}
+
+  @default_include %{
+    imports: true,
+    # `include.imports_meta` can be true even when `include.imports`
+    # is false. Even if we don't want to include imported data, we
+    # might still want to know whether imported data can be toggled
+    # on/off on the dashboard.
+    imports_meta: true,
+    time_labels: true,
+    total_rows: false,
+    trim_relative_date_range: true,
+    comparisons: nil,
+    legacy_time_on_page_cutoff: nil
+  }
+
+  def default_include(), do: @default_include
+
+  @default_pagination nil
+
+  def default_pagination(), do: @default_pagination
+
+  def parse(query_string) when is_binary(query_string) do
+    query_string = String.trim_leading(query_string, "?")
+    params_map = URI.decode_query(query_string)
+
+    with {:ok, filters} <- parse_filters(query_string),
+         {:ok, relative_date} <- parse_relative_date(params_map) do
+      include_imports? = parse_include_imports(params_map)
+
+      {:ok,
+       ParsedQueryParams.new!(%{
+         input_date_range: parse_input_date_range(params_map),
+         relative_date: relative_date,
+         filters: filters,
+         include: Map.merge(@default_include, %{imports: include_imports?})
+       })}
+    end
+  end
+
+  defp parse_input_date_range(%{"period" => "realtime"}), do: :realtime
+  defp parse_input_date_range(%{"period" => "day"}), do: :day
+  defp parse_input_date_range(%{"period" => "month"}), do: :month
+  defp parse_input_date_range(%{"period" => "year"}), do: :year
+  defp parse_input_date_range(%{"period" => "all"}), do: :all
+  defp parse_input_date_range(%{"period" => "7d"}), do: {:last_n_days, 7}
+  defp parse_input_date_range(%{"period" => "28d"}), do: {:last_n_days, 28}
+  defp parse_input_date_range(%{"period" => "30d"}), do: {:last_n_days, 30}
+  defp parse_input_date_range(%{"period" => "91d"}), do: {:last_n_days, 91}
+  defp parse_input_date_range(%{"period" => "6mo"}), do: {:last_n_months, 6}
+  defp parse_input_date_range(%{"period" => "12mo"}), do: {:last_n_months, 12}
+
+  defp parse_input_date_range(%{"period" => "custom", "from" => from, "to" => to}) do
+    from_date = Date.from_iso8601!(String.trim(from))
+    to_date = Date.from_iso8601!(String.trim(to))
+    {:date_range, from_date, to_date}
+  end
+
+  defp parse_input_date_range(_), do: nil
+
+  defp parse_relative_date(%{"date" => date}) do
+    case Date.from_iso8601(date) do
+      {:ok, date} -> {:ok, date}
+      _ -> {:error, :invalid_date}
+    end
+  end
+
+  defp parse_relative_date(_), do: {:ok, nil}
+
+  defp parse_include_imports(%{"with_imported" => "false"}), do: false
+  defp parse_include_imports(_), do: true
+
+  defp parse_filters(query_string) do
+    with {:ok, filters} <- decode_filters(query_string) do
+      Plausible.Stats.ApiQueryParser.parse_filters(filters)
+    end
+  end
+
+  defp decode_filters(query_string) do
+    query_string
+    |> URI.query_decoder()
+    |> Enum.filter(fn {key, _value} -> key == "f" end)
+    |> Enum.reduce_while({:ok, []}, fn {_, filter_expression}, {:ok, acc} ->
+      case decode_filter(filter_expression) do
+        {:ok, filter} -> {:cont, {:ok, acc ++ [filter]}}
+        {:error, _} -> {:halt, {:error, :invalid_filters}}
+      end
+    end)
+  end
+
+  defp decode_filter(filter_expression) do
+    case String.split(filter_expression, ",") do
+      [operator, dimension | clauses] ->
+        {:ok,
+         [
+           operator,
+           with_prefix(dimension),
+           Enum.map(clauses, &URI.decode_www_form/1)
+         ]}
+
+      _ ->
+        {:error, :invalid_filter}
+    end
+  end
+
+  @event_prefix "event:"
+  @visit_prefix "visit:"
+  @no_prefix_dimensions ["segment"]
+  defp with_prefix(dimension) do
+    cond do
+      dimension in @no_prefix_dimensions -> dimension
+      event_dimension?(dimension) -> @event_prefix <> dimension
+      true -> @visit_prefix <> dimension
+    end
+  end
+
+  @event_props_prefix "props:"
+  @event_dimensions ["name", "page", "goal", "hostname"]
+  defp event_dimension?(dimension) do
+    dimension in @event_dimensions or String.starts_with?(dimension, @event_props_prefix)
+  end
+end

lib/plausible/stats/dashboard_query_serializer.ex

@@ -0,0 +1,85 @@
+defmodule Plausible.Stats.DashboardQuerySerializer do
+  @moduledoc """
+  Takes a `%ParsedQueryParams{}` struct and turns it into a query
+  string.
+  """
+
+  alias Plausible.Stats.{ParsedQueryParams, DashboardQueryParser}
+
+  def serialize(%ParsedQueryParams{} = params) do
+    params
+    |> Map.to_list()
+    |> Enum.flat_map(&get_serialized_fields/1)
+    |> Enum.sort_by(&elem(&1, 0))
+    |> Enum.map_join("&", fn {key, value} -> "#{key}=#{value}" end)
+  end
+
+  defp get_serialized_fields({_, nil}), do: []
+  defp get_serialized_fields({_, []}), do: []
+
+  defp get_serialized_fields({:input_date_range, {:date_range, from_date, to_date}}) do
+    [
+      {"period", "custom"},
+      {"from", Date.to_iso8601(from_date)},
+      {"to", Date.to_iso8601(to_date)}
+    ]
+  end
+
+  defp get_serialized_fields({:input_date_range, input_date_range}) do
+    period =
+      case input_date_range do
+        :realtime -> "realtime"
+        :day -> "day"
+        :month -> "month"
+        :year -> "year"
+        :all -> "all"
+        {:last_n_days, 7} -> "7d"
+        {:last_n_days, 28} -> "28d"
+        {:last_n_days, 30} -> "30d"
+        {:last_n_days, 91} -> "91d"
+        {:last_n_months, 6} -> "6mo"
+        {:last_n_months, 12} -> "12mo"
+      end
+
+    [{"period", period}]
+  end
+
+  defp get_serialized_fields({:relative_date, date}) do
+    [{"date", Date.to_iso8601(date)}]
+  end
+
+  defp get_serialized_fields({:filters, [_ | _] = filters}) do
+    filters
+    |> Enum.map(fn [operator, dimension, clauses] ->
+      clauses = Enum.map_join(clauses, ",", &uri_encode_permissive/1)
+      dimension = String.split(dimension, ":", parts: 2) |> List.last()
+      {"f", "#{operator},#{dimension},#{clauses}"}
+    end)
+  end
+
+  defp get_serialized_fields({:include, include}) do
+    if include.imports == DashboardQueryParser.default_include().imports do
+      []
+    else
+      [{"with_imported", to_string(include.imports)}]
+    end
+  end
+
+  defp get_serialized_fields(_) do
+    []
+  end
+
+  # These characters are not URL encoded to have more readable URLs.
+  # Browsers seem to handle this just fine. `?f=is,page,/my/page/:some_param`
+  # vs `?f=is,page,%2Fmy%2Fpage%2F%3Asome_param`
+  @do_not_url_encode [":", "/"]
+  @do_not_url_encode_map Enum.into(@do_not_url_encode, %{}, fn char ->
+                           {URI.encode_www_form(char), char}
+                         end)
+
+  defp uri_encode_permissive(input) do
+    input
+    |> URI.encode_www_form()
+    |> String.replace(Map.keys(@do_not_url_encode_map), &@do_not_url_encode_map[&1])
+  end
+end

lib/plausible/stats/legacy/legacy_query_builder.ex

@@ -281,10 +281,10 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
   @doc """
   ### Examples:
     iex> Plausible.Stats.Legacy.QueryBuilder.parse_include(nil)
-    Plausible.Stats.ParsedQueryParams.default_include()
+    Plausible.Stats.ApiQueryParser.default_include()
 
     iex> Plausible.Stats.Legacy.QueryBuilder.parse_include(~s({"total_rows": true}))
-    Map.merge(Plausible.Stats.ParsedQueryParams.default_include(), %{total_rows: true})
+    Map.merge(Plausible.Stats.ApiQueryParser.default_include(), %{total_rows: true})
   """
   def parse_include(include) do
     include =
@@ -296,7 +296,7 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do
         _ -> %{}
       end
 
-    Plausible.Stats.ParsedQueryParams.default_include()
+    Plausible.Stats.ApiQueryParser.default_include()
     |> Map.merge(include)
   end
 

lib/plausible/stats/parsed_query_params.ex

@@ -2,8 +2,13 @@ defmodule Plausible.Stats.ParsedQueryParams do
   @moduledoc false
 
   defstruct [
-    :now,
     :input_date_range,
+    # `relative_date` is a convenience currently exclusive to the internal
+    # dashboard API for constructing datetime ranges. It adds the ability
+    # to use `day`, `month` and `year` periods relative to a specific date.
+    # E.g.: `?period=month&date=2021-01-15` will query the entire month of
+    # January 2021. In the public API it is always today in site.timezone.
+    :relative_date,
     :metrics,
     :filters,
     :dimensions,
@@ -12,41 +17,7 @@ defmodule Plausible.Stats.ParsedQueryParams do
     :include
   ]
 
-  @default_include %{
-    imports: false,
-    # `include.imports_meta` can be true even when `include.imports`
-    # is false. Even if we don't want to include imported data, we
-    # might still want to know whether imported data can be toggled
-    # on/off on the dashboard.
-    imports_meta: false,
-    time_labels: false,
-    total_rows: false,
-    trim_relative_date_range: false,
-    comparisons: nil,
-    legacy_time_on_page_cutoff: nil
-  }
-
-  def default_include(), do: @default_include
-
-  @default_pagination %{
-    limit: 10_000,
-    offset: 0
-  }
-
-  def default_pagination(), do: @default_pagination
-
   def new!(params) when is_map(params) do
-    [_ | _] = metrics = Map.fetch!(params, :metrics)
-
-    %__MODULE__{
-      now: params[:now],
-      input_date_range: Map.fetch!(params, :input_date_range),
-      metrics: metrics,
-      filters: params[:filters] || [],
-      dimensions: params[:dimensions] || [],
-      order_by: params[:order_by],
-      pagination: Map.merge(@default_pagination, params[:pagination] || %{}),
-      include: Map.merge(@default_include, params[:include] || %{})
-    }
+    struct!(__MODULE__, Map.to_list(params))
   end
 end