diff --git a/extra/lib/plausible/stats/consolidated_view.ex b/extra/lib/plausible/stats/consolidated_view.ex index fa98b06931b9..376cb49be5b4 100644 --- a/extra/lib/plausible/stats/consolidated_view.ex +++ b/extra/lib/plausible/stats/consolidated_view.ex @@ -56,7 +56,7 @@ defmodule Plausible.Stats.ConsolidatedView do |> DateTime.to_iso8601() stats_query = - Stats.Query.build!(view, :internal, %{ + Stats.Query.parse_and_build!(view, :internal, %{ "site_id" => view.domain, "metrics" => ["visitors", "visits", "pageviews", "views_per_visit"], "include" => %{"comparisons" => %{"mode" => "custom", "date_range" => [c_from, c_to]}}, @@ -91,7 +91,7 @@ defmodule Plausible.Stats.ConsolidatedView do defp query_24h_intervals(view, now) do graph_query = - Stats.Query.build!( + Stats.Query.parse_and_build!( view, :internal, %{ diff --git a/extra/lib/plausible_web/live/funnel_settings/form.ex b/extra/lib/plausible_web/live/funnel_settings/form.ex index fa18248be046..7e4036a1e0db 100644 --- a/extra/lib/plausible_web/live/funnel_settings/form.ex +++ b/extra/lib/plausible_web/live/funnel_settings/form.ex @@ -350,7 +350,7 @@ defmodule PlausibleWeb.Live.FunnelSettings.Form do ) query = - Plausible.Stats.Query.build!( + Plausible.Stats.Query.parse_and_build!( site, :internal, %{ diff --git a/lib/plausible/segments/filters.ex b/lib/plausible/segments/filters.ex index bd62b8dd58b7..92d9034fc8f2 100644 --- a/lib/plausible/segments/filters.ex +++ b/lib/plausible/segments/filters.ex @@ -3,7 +3,7 @@ defmodule Plausible.Segments.Filters do This module contains functions that enable resolving segments in filters. """ alias Plausible.Segments - alias Plausible.Stats.Filters + alias Plausible.Stats.{Filters, QueryParser} @max_segment_filters_count 10 @@ -48,7 +48,7 @@ defmodule Plausible.Segments.Filters do segments, %{}, fn %Segments.Segment{id: id, segment_data: segment_data} -> - case Filters.QueryParser.parse_filters(segment_data["filters"]) do + case QueryParser.parse_filters(segment_data["filters"]) do {:ok, filters} -> {id, filters} _ -> {id, nil} end diff --git a/lib/plausible/segments/segment.ex b/lib/plausible/segments/segment.ex index 22cd86b4d152..5a6520d6835c 100644 --- a/lib/plausible/segments/segment.ex +++ b/lib/plausible/segments/segment.ex @@ -131,7 +131,7 @@ defmodule Plausible.Segments.Segment do """ def build_naive_query_from_segment_data(%Plausible.Site{} = site, filters), do: - Plausible.Stats.Query.build( + Plausible.Stats.Query.parse_and_build( site, :internal, %{ diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.ex index b693528b7654..f99f32b74a8b 100644 --- a/lib/plausible/stats/filters/filters.ex +++ b/lib/plausible/stats/filters/filters.ex @@ -4,8 +4,8 @@ defmodule Plausible.Stats.Filters do """ alias Plausible.Stats.Query - alias Plausible.Stats.Filters.QueryParser - alias Plausible.Stats.Filters.StatsAPIFilterParser + alias Plausible.Stats.QueryParser + alias Plausible.Stats.Filters.LegacyStatsAPIFilterParser @visit_props [ :source, @@ -70,7 +70,7 @@ defmodule Plausible.Stats.Filters do case Jason.decode(filters) do {:ok, filters} when is_list(filters) -> parse(filters) {:ok, _} -> [] - {:error, err} -> StatsAPIFilterParser.parse_filter_expression(err.data) + {:error, err} -> LegacyStatsAPIFilterParser.parse_filter_expression(err.data) end end diff --git a/lib/plausible/stats/filters/stats_api_filter_parser.ex b/lib/plausible/stats/filters/legacy_stats_api_filter_parser.ex similarity index 94% rename from lib/plausible/stats/filters/stats_api_filter_parser.ex rename to lib/plausible/stats/filters/legacy_stats_api_filter_parser.ex index 341fe9049cdd..189553aeaf35 100644 --- a/lib/plausible/stats/filters/stats_api_filter_parser.ex +++ b/lib/plausible/stats/filters/legacy_stats_api_filter_parser.ex @@ -1,5 +1,7 @@ -defmodule Plausible.Stats.Filters.StatsAPIFilterParser do - @moduledoc false +defmodule Plausible.Stats.Filters.LegacyStatsAPIFilterParser do + @moduledoc """ + Parser for legacy filter format used in Stats API v1. + """ @non_escaped_pipe_regex ~r/(? put_consolidated_site_ids(site) |> put_order_by(params) |> put_include(site, params) - |> Query.put_comparison_utc_time_range() + |> QueryBuilder.put_comparison_utc_time_range() |> Query.put_imported_opts(site) - |> Query.set_time_on_page_data(site) + |> QueryBuilder.set_time_on_page_data(site) on_ee do query = Plausible.Stats.Sampling.put_threshold(query, site, params) @@ -68,7 +68,7 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do defp preload_goals_and_revenue(query, site) do {preloaded_goals, revenue_warning, revenue_currencies} = - Plausible.Stats.Filters.QueryParser.preload_goals_and_revenue( + Plausible.Stats.QueryBuilder.preload_goals_and_revenue( site, query.metrics, query.filters, @@ -269,36 +269,37 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do [{:visitors, :asc}, {"visit:source", :desc}] """ def parse_order_by(order_by) do - json_decode(order_by) - |> unwrap([]) - |> Filters.QueryParser.parse_order_by() - |> unwrap([]) + with true <- is_binary(order_by), + {:ok, order_by} <- JSON.decode(order_by), + {:ok, order_by} <- QueryParser.parse_order_by(order_by) do + order_by + else + _ -> [] + end end @doc """ ### Examples: iex> QueryBuilder.parse_include(%{}, nil) - QueryParser.default_include() + Plausible.Stats.ParsedQueryParams.default_include() iex> QueryBuilder.parse_include(%{}, ~s({"total_rows": true})) - Map.merge(QueryParser.default_include(), %{total_rows: true}) + Map.merge(Plausible.Stats.ParsedQueryParams.default_include(), %{total_rows: true}) """ def parse_include(site, include) do - json_decode(include) - |> unwrap(%{}) - |> Filters.QueryParser.parse_include(site) - |> unwrap(Filters.QueryParser.default_include()) - end + include = + with true <- is_binary(include), + {:ok, include} <- JSON.decode(include), + {:ok, include} <- QueryParser.parse_include(include, site) do + include + else + _ -> %{} + end - defp json_decode(string) when is_binary(string) do - Jason.decode(string) + Plausible.Stats.ParsedQueryParams.default_include() + |> Map.merge(include) end - defp json_decode(_other), do: :error - - defp unwrap({:ok, result}, _default), do: result - defp unwrap(_, default), do: default - defp put_order_by(query, %{} = params) do struct!(query, order_by: parse_order_by(params["order_by"])) end @@ -342,7 +343,7 @@ defmodule Plausible.Stats.Legacy.QueryBuilder do def parse_comparison_params(site, %{"comparison" => "custom"} = params) do {:ok, date_range} = - Filters.QueryParser.parse_date_range_pair(site, [ + QueryParser.parse_date_range_pair(site, [ params["compare_from"], params["compare_to"] ]) diff --git a/lib/plausible/stats/parsed_query_params.ex b/lib/plausible/stats/parsed_query_params.ex new file mode 100644 index 000000000000..f1488b4ef8ca --- /dev/null +++ b/lib/plausible/stats/parsed_query_params.ex @@ -0,0 +1,55 @@ +defmodule Plausible.Stats.ParsedQueryParams do + @moduledoc false + + defstruct [ + :now, + :utc_time_range, + :metrics, + :filters, + :dimensions, + :order_by, + :pagination, + :include + ] + + alias Plausible.Stats.DateTimeRange + + @default_include %{ + imports: false, + # `include.imports_meta` can be true even when `include.imports` + # is false. Even if we don't want to include imported data, we + # might still want to know whether imported data can be toggled + # on/off on the dashboard. + imports_meta: false, + time_labels: false, + total_rows: false, + trim_relative_date_range: false, + comparisons: nil, + legacy_time_on_page_cutoff: nil + } + + def default_include(), do: @default_include + + @default_pagination %{ + limit: 10_000, + offset: 0 + } + + def default_pagination(), do: @default_pagination + + def new!(params) when is_map(params) do + %DateTimeRange{} = utc_time_range = Map.fetch!(params, :utc_time_range) + [_ | _] = metrics = Map.fetch!(params, :metrics) + + %__MODULE__{ + now: params[:now], + utc_time_range: utc_time_range, + metrics: metrics, + filters: params[:filters] || [], + dimensions: params[:dimensions] || [], + order_by: params[:order_by], + pagination: Map.merge(@default_pagination, params[:pagination] || %{}), + include: Map.merge(@default_include, params[:include] || %{}) + } + end +end diff --git a/lib/plausible/stats/query.ex b/lib/plausible/stats/query.ex index 89f0fa72cacc..6f99cdd8f8d9 100644 --- a/lib/plausible/stats/query.ex +++ b/lib/plausible/stats/query.ex @@ -18,7 +18,7 @@ defmodule Plausible.Stats.Query do timezone: nil, legacy_breakdown: false, preloaded_goals: [], - include: Plausible.Stats.Filters.QueryParser.default_include(), + include: Plausible.Stats.ParsedQueryParams.default_include(), debug_metadata: %{}, pagination: nil, # Revenue metric specific metadata @@ -34,45 +34,40 @@ defmodule Plausible.Stats.Query do smear_session_metrics: false require OpenTelemetry.Tracer, as: Tracer - alias Plausible.Stats.{DateTimeRange, Filters, Imported, Legacy, Comparisons} + + alias Plausible.Stats.{ + DateTimeRange, + Imported, + Legacy, + Comparisons, + QueryParser, + ParsedQueryParams, + QueryBuilder + } @type t :: %__MODULE__{} - def build( + def parse_and_build( %Plausible.Site{domain: domain} = site, schema_type, %{"site_id" => domain} = params, debug_metadata \\ %{} ) do - with {:ok, query_data} <- Filters.QueryParser.parse(site, schema_type, params) do - query = - %__MODULE__{ - debug_metadata: debug_metadata, - site_id: site.id, - site_native_stats_start_at: site.native_stats_start_at - } - |> struct!(Map.to_list(query_data)) - |> set_time_on_page_data(site) - |> put_comparison_utc_time_range() - |> put_imported_opts(site) - - on_ee do - query = Plausible.Stats.Sampling.put_threshold(query, site, params) - end - - {:ok, query} + with {:ok, %ParsedQueryParams{} = parsed_query_params} <- + QueryParser.parse(site, schema_type, params) do + QueryBuilder.build(site, parsed_query_params, params, debug_metadata) end end - def build!(site, schema_type, params, debug_metadata \\ %{}) do - case build(site, schema_type, params, debug_metadata) do + def parse_and_build!(site, schema_type, params, debug_metadata \\ %{}) do + case parse_and_build(site, schema_type, params, debug_metadata) do {:ok, query} -> query {:error, reason} -> raise "Failed to build query: #{inspect(reason)}" end end @doc """ - Builds query from old-style stats APIv1 params. New code should use `Query.build`. + Builds query from old-style stats APIv1 params. New code should use `Query.parse_and_build`. """ def from(site, params, debug_metadata \\ %{}, now \\ nil) do Legacy.QueryBuilder.from(site, params, debug_metadata, now) @@ -143,13 +138,6 @@ defmodule Plausible.Stats.Query do put_imported_opts(query, nil) end - def put_comparison_utc_time_range(%__MODULE__{include: %{comparisons: nil}} = query), do: query - - def put_comparison_utc_time_range(%__MODULE__{include: %{comparisons: comparison_opts}} = query) do - datetime_range = Comparisons.get_comparison_utc_time_range(query, comparison_opts) - struct!(query, comparison_utc_time_range: datetime_range) - end - def put_imported_opts(query, site) do requested? = query.include.imports @@ -190,15 +178,6 @@ defmodule Plausible.Stats.Query do in_comparison_range ++ in_range end - def set_time_on_page_data(query, site) do - struct!(query, - time_on_page_data: %{ - new_metric_visible: Plausible.Stats.TimeOnPage.new_time_on_page_visible?(site), - cutoff_date: site.legacy_time_on_page_cutoff - } - ) - end - @spec get_skip_imported_reason(t()) :: nil | :no_imported_data | :out_of_range | :unsupported_query def get_skip_imported_reason(query) do diff --git a/lib/plausible/stats/query_builder.ex b/lib/plausible/stats/query_builder.ex new file mode 100644 index 000000000000..498f6665a4e5 --- /dev/null +++ b/lib/plausible/stats/query_builder.ex @@ -0,0 +1,363 @@ +defmodule Plausible.Stats.QueryBuilder do + @moduledoc """ + A module used for building the Query struct from already parsed params. + """ + + use Plausible + alias Plausible.Segments + alias Plausible.Stats.{Query, ParsedQueryParams, Comparisons, Filters, Time, TableDecider} + + def build(site, parsed_query_params, params, debug_metadata \\ %{}) do + with {:ok, parsed_query_params} <- resolve_segments_in_filters(parsed_query_params, site), + query = do_build(parsed_query_params, site, params, debug_metadata), + :ok <- validate_order_by(query), + :ok <- validate_custom_props_access(site, query), + :ok <- validate_toplevel_only_filter_dimension(query), + :ok <- validate_special_metrics_filters(query), + :ok <- validate_behavioral_filters(query), + :ok <- validate_filtered_goals_exist(query), + :ok <- validate_revenue_metrics_access(site, query), + :ok <- validate_metrics(query), + :ok <- validate_include(query) do + query = + query + |> set_time_on_page_data(site) + |> put_comparison_utc_time_range() + |> Query.put_imported_opts(site) + + on_ee do + # NOTE: The Query API schema does not allow the sample_threshold param + # and it looks like it's not used as a parameter anymore. We might want + # to clean this up. + query = Plausible.Stats.Sampling.put_threshold(query, site, %{}) + end + + {:ok, query} + end + end + + defp resolve_segments_in_filters(%ParsedQueryParams{} = parsed_query_params, site) do + with {:ok, preloaded_segments} <- + Segments.Filters.preload_needed_segments(site, parsed_query_params.filters), + {:ok, filters} <- + Segments.Filters.resolve_segments(parsed_query_params.filters, preloaded_segments) do + {:ok, struct!(parsed_query_params, filters: filters)} + end + end + + defp do_build(parsed_query_params, site, params, debug_metadata) do + %ParsedQueryParams{metrics: metrics, filters: filters, dimensions: dimensions} = + parsed_query_params + + {preloaded_goals, revenue_warning, revenue_currencies} = + preload_goals_and_revenue(site, metrics, filters, dimensions) + + consolidated_site_ids = get_consolidated_site_ids(site) + + all_params = + parsed_query_params + |> Map.to_list() + |> Keyword.merge( + site_id: site.id, + site_native_stats_start_at: site.native_stats_start_at, + consolidated_site_ids: consolidated_site_ids, + timezone: site.timezone, + preloaded_goals: preloaded_goals, + revenue_warning: revenue_warning, + revenue_currencies: revenue_currencies, + input_date_range: Map.get(params, "date_range"), + debug_metadata: debug_metadata + ) + + struct!(%Query{}, all_params) + end + + on_ee do + def get_consolidated_site_ids(%Plausible.Site{} = site) do + if Plausible.Sites.consolidated?(site) do + Plausible.ConsolidatedView.Cache.get(site.domain) + end + end + else + def get_consolidated_site_ids(_site), do: nil + end + + def set_time_on_page_data(query, site) do + struct!(query, + time_on_page_data: %{ + new_metric_visible: Plausible.Stats.TimeOnPage.new_time_on_page_visible?(site), + cutoff_date: site.legacy_time_on_page_cutoff + } + ) + end + + def put_comparison_utc_time_range(%Query{include: %{comparisons: nil}} = query), do: query + + def put_comparison_utc_time_range(%Query{include: %{comparisons: comparison_opts}} = query) do + datetime_range = Comparisons.get_comparison_utc_time_range(query, comparison_opts) + struct!(query, comparison_utc_time_range: datetime_range) + end + + def preload_goals_and_revenue(site, metrics, filters, dimensions) do + preloaded_goals = + Plausible.Stats.Goals.preload_needed_goals(site, dimensions, filters) + + {revenue_warning, revenue_currencies} = + preload_revenue(site, preloaded_goals, metrics, dimensions) + + { + preloaded_goals, + revenue_warning, + revenue_currencies + } + end + + on_ee do + alias Plausible.Stats.Goal.Revenue + + def preload_revenue(site, preloaded_goals, metrics, dimensions) do + Revenue.preload(site, preloaded_goals, metrics, dimensions) + end + + defp validate_revenue_metrics_access(site, query) do + if Revenue.requested?(query.metrics) and not Revenue.available?(site) do + {:error, "The owner of this site does not have access to the revenue metrics feature."} + else + :ok + end + end + else + defp preload_revenue(_site, _preloaded_goals, _metrics, _dimensions), do: {nil, %{}} + + defp validate_revenue_metrics_access(_site, _query), do: :ok + end + + defp validate_order_by(query) do + if query.order_by do + valid_values = query.metrics ++ query.dimensions + + invalid_entry = + Enum.find(query.order_by, fn {value, _direction} -> + not Enum.member?(valid_values, value) + end) + + case invalid_entry do + nil -> + :ok + + _ -> + {:error, + "Invalid order_by entry '#{i(invalid_entry)}'. Entry is not a queried metric or dimension."} + end + else + :ok + end + end + + @only_toplevel ["event:goal", "event:hostname"] + defp validate_toplevel_only_filter_dimension(query) do + not_toplevel = + query.filters + |> Filters.dimensions_used_in_filters(min_depth: 1, behavioral_filters: :ignore) + |> Enum.filter(&(&1 in @only_toplevel)) + + if Enum.count(not_toplevel) > 0 do + {:error, + "Invalid filters. Dimension `#{List.first(not_toplevel)}` can only be filtered at the top level."} + else + :ok + end + end + + @special_metrics [:conversion_rate, :group_conversion_rate] + defp validate_special_metrics_filters(query) do + special_metric? = Enum.any?(@special_metrics, &(&1 in query.metrics)) + + deep_custom_property? = + query.filters + |> Filters.dimensions_used_in_filters(min_depth: 1) + |> Enum.any?(fn dimension -> String.starts_with?(dimension, "event:props:") end) + + if special_metric? and deep_custom_property? do + {:error, + "Invalid filters. When `conversion_rate` or `group_conversion_rate` metrics are used, custom property filters can only be used on top level."} + else + :ok + end + end + + defp validate_behavioral_filters(query) do + query.filters + |> Filters.traverse(0, fn behavioral_depth, operator -> + if operator in [:has_done, :has_not_done] do + behavioral_depth + 1 + else + behavioral_depth + end + end) + |> Enum.reduce_while(:ok, fn {[_operator, dimension | _rest], behavioral_depth}, :ok -> + cond do + behavioral_depth == 0 -> + # ignore non-behavioral filters + {:cont, :ok} + + behavioral_depth > 1 -> + {:halt, + {:error, + "Invalid filters. Behavioral filters (has_done, has_not_done) cannot be nested."}} + + not String.starts_with?(dimension, "event:") -> + {:halt, + {:error, + "Invalid filters. Behavioral filters (has_done, has_not_done) can only be used with event dimension filters."}} + + true -> + {:cont, :ok} + end + end) + end + + defp validate_filtered_goals_exist(query) do + # Note: We don't check :contains goal filters since it's acceptable if they match nothing. + goal_filter_clauses = + query.filters + |> Filters.all_leaf_filters() + |> Enum.flat_map(fn + [:is, "event:goal", clauses] -> clauses + _ -> [] + end) + + if length(goal_filter_clauses) > 0 do + configured_goal_names = + query.preloaded_goals.all + |> Enum.map(&Plausible.Goal.display_name/1) + + validate_list(goal_filter_clauses, &validate_goal_filter(&1, configured_goal_names)) + else + :ok + end + end + + defp validate_goal_filter(clause, configured_goal_names) do + if Enum.member?(configured_goal_names, clause) do + :ok + else + {:error, + "Invalid filters. The goal `#{clause}` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals"} + end + end + + defp validate_custom_props_access(site, query) do + allowed_props = Plausible.Props.allowed_for(site, bypass_setup?: true) + + validate_custom_props_access(site, query, allowed_props) + end + + defp validate_custom_props_access(_site, _query, :all), do: :ok + + defp validate_custom_props_access(_site, query, allowed_props) do + valid? = + query.filters + |> Filters.dimensions_used_in_filters() + |> Enum.concat(query.dimensions) + |> Enum.all?(fn + "event:props:" <> prop -> prop in allowed_props + _ -> true + end) + + if valid? do + :ok + else + {:error, "The owner of this site does not have access to the custom properties feature."} + end + end + + defp validate_metrics(query) do + with :ok <- validate_list(query.metrics, &validate_metric(&1, query)) do + TableDecider.validate_no_metrics_dimensions_conflict(query) + end + end + + defp validate_metric(metric, query) when metric in [:conversion_rate, :group_conversion_rate] do + if Enum.member?(query.dimensions, "event:goal") or + Filters.filtering_on_dimension?(query, "event:goal", behavioral_filters: :ignore) do + :ok + else + {:error, "Metric `#{metric}` can only be queried with event:goal filters or dimensions."} + end + end + + defp validate_metric(:scroll_depth = metric, query) do + page_dimension? = Enum.member?(query.dimensions, "event:page") + toplevel_page_filter? = not is_nil(Filters.get_toplevel_filter(query, "event:page")) + + if page_dimension? or toplevel_page_filter? do + :ok + else + {:error, "Metric `#{metric}` can only be queried with event:page filters or dimensions."} + end + end + + defp validate_metric(:exit_rate = metric, query) do + case {query.dimensions, TableDecider.sessions_join_events?(query)} do + {["visit:exit_page"], false} -> + :ok + + {["visit:exit_page"], true} -> + {:error, "Metric `#{metric}` cannot be queried when filtering on event dimensions."} + + _ -> + {:error, + "Metric `#{metric}` requires a `\"visit:exit_page\"` dimension. No other dimensions are allowed."} + end + end + + defp validate_metric(:views_per_visit = metric, query) do + cond do + Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore) -> + {:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`."} + + length(query.dimensions) > 0 -> + {:error, "Metric `#{metric}` cannot be queried with `dimensions`."} + + true -> + :ok + end + end + + defp validate_metric(:time_on_page = metric, query) do + cond do + Enum.member?(query.dimensions, "event:page") -> + :ok + + Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore) -> + :ok + + true -> + {:error, "Metric `#{metric}` can only be queried with event:page filters or dimensions."} + end + end + + defp validate_metric(_, _), do: :ok + + defp validate_include(query) do + time_dimension? = Enum.any?(query.dimensions, &Time.time_dimension?/1) + + if query.include.time_labels and not time_dimension? do + {:error, "Invalid include.time_labels: requires a time dimension."} + else + :ok + end + end + + defp i(value), do: inspect(value, charlists: :as_lists) + + defp validate_list(list, parser_function) do + Enum.reduce_while(list, :ok, fn value, :ok -> + case parser_function.(value) do + :ok -> {:cont, :ok} + {:error, _} = error -> {:halt, error} + end + end) + end +end diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/query_parser.ex similarity index 51% rename from lib/plausible/stats/filters/query_parser.ex rename to lib/plausible/stats/query_parser.ex index 1187e5c29448..8baf36b49195 100644 --- a/lib/plausible/stats/filters/query_parser.ex +++ b/lib/plausible/stats/query_parser.ex @@ -1,30 +1,9 @@ -defmodule Plausible.Stats.Filters.QueryParser do +defmodule Plausible.Stats.QueryParser do @moduledoc false use Plausible - alias Plausible.Stats.{TableDecider, Filters, Metrics, DateTimeRange, JSONSchema, Time} - - @default_include %{ - imports: false, - # `include.imports_meta` can be true even when `include.imports` - # is false. Even if we don't want to include imported data, we - # might still want to know whether imported data can be toggled - # on/off on the dashboard. - imports_meta: false, - time_labels: false, - total_rows: false, - trim_relative_date_range: false, - comparisons: nil, - legacy_time_on_page_cutoff: nil - } - - @default_pagination %{ - limit: 10_000, - offset: 0 - } - - def default_include(), do: @default_include + alias Plausible.Stats.{Filters, Metrics, DateTimeRange, JSONSchema} def parse(site, schema_type, params, now \\ nil) when is_map(params) do now = now || Plausible.Stats.Query.Test.get_fixed_now() @@ -35,58 +14,26 @@ defmodule Plausible.Stats.Filters.QueryParser do {:ok, raw_time_range} <- parse_time_range(site, Map.get(params, "date_range"), date, now), utc_time_range = raw_time_range |> DateTimeRange.to_timezone("Etc/UTC"), - {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])), - {:ok, filters} <- parse_filters(Map.get(params, "filters", [])), - {:ok, preloaded_segments} <- - Plausible.Segments.Filters.preload_needed_segments(site, filters), - {:ok, filters} <- - Plausible.Segments.Filters.resolve_segments(filters, preloaded_segments), - {:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])), - {:ok, order_by} <- parse_order_by(Map.get(params, "order_by")), - {:ok, include} <- parse_include(Map.get(params, "include", %{}), site), - {:ok, pagination} <- parse_pagination(Map.get(params, "pagination", %{})), - {preloaded_goals, revenue_warning, revenue_currencies} <- - preload_goals_and_revenue(site, metrics, filters, dimensions), - consolidated_site_ids = get_consolidated_site_ids(site), - query = %{ - now: now, - consolidated_site_ids: consolidated_site_ids, - input_date_range: Map.get(params, "date_range"), - metrics: metrics, - filters: filters, - utc_time_range: utc_time_range, - dimensions: dimensions, - order_by: order_by, - timezone: site.timezone, - include: include, - pagination: pagination, - preloaded_goals: preloaded_goals, - revenue_warning: revenue_warning, - revenue_currencies: revenue_currencies - }, - :ok <- validate_order_by(query), - :ok <- validate_custom_props_access(site, query), - :ok <- validate_toplevel_only_filter_dimension(query), - :ok <- validate_special_metrics_filters(query), - :ok <- validate_behavioral_filters(query), - :ok <- validate_filtered_goals_exist(query), - :ok <- validate_revenue_metrics_access(site, query), - :ok <- validate_metrics(query), - :ok <- validate_include(query) do - {:ok, query} + {:ok, metrics} <- parse_metrics(Map.fetch!(params, "metrics")), + {:ok, filters} <- parse_filters(params["filters"]), + {:ok, dimensions} <- parse_dimensions(params["dimensions"]), + {:ok, order_by} <- parse_order_by(params["order_by"]), + {:ok, pagination} <- parse_pagination(params["pagination"]), + {:ok, include} <- parse_include(params["include"], site) do + {:ok, + Plausible.Stats.ParsedQueryParams.new!(%{ + now: now, + utc_time_range: utc_time_range, + metrics: metrics, + filters: filters, + dimensions: dimensions, + order_by: order_by, + pagination: pagination, + include: include + })} end end - on_ee do - def get_consolidated_site_ids(%Plausible.Site{} = site) do - if Plausible.Sites.consolidated?(site) do - Plausible.ConsolidatedView.Cache.get(site.domain) - end - end - else - def get_consolidated_site_ids(_site), do: nil - end - def parse_date_range_pair(site, [from, to]) when is_binary(from) and is_binary(to) do with {:ok, date_range} <- date_range_from_date_strings(site, from, to) do {:ok, date_range |> DateTimeRange.to_timezone("Etc/UTC")} @@ -110,7 +57,7 @@ defmodule Plausible.Stats.Filters.QueryParser do parse_list(filters, &parse_filter/1) end - def parse_filters(_invalid_metrics), do: {:error, "Invalid filters passed."} + def parse_filters(nil), do: {:ok, nil} defp parse_filter(filter) do with {:ok, operator} <- parse_operator(filter), @@ -308,6 +255,8 @@ defmodule Plausible.Stats.Filters.QueryParser do ) end + defp parse_dimensions(nil), do: {:ok, nil} + def parse_order_by(order_by) when is_list(order_by) do parse_list(order_by, &parse_order_by_entry/1) end @@ -359,16 +308,19 @@ defmodule Plausible.Stats.Filters.QueryParser do defp parse_order_direction(entry), do: {:error, "Invalid order_by entry '#{i(entry)}'."} def parse_include(include, site) when is_map(include) do - with {:ok, include} <- atomize_include_keys(include), - {:ok, include} <- update_comparisons_date_range(include, site) do - {:ok, Map.merge(@default_include, include)} + with {:ok, include} <- atomize_include_keys(include) do + update_comparisons_date_range(include, site) end end + def parse_include(nil, _site), do: {:ok, nil} def parse_include(include, _site), do: {:error, "Invalid include '#{i(include)}'."} defp atomize_include_keys(map) do - expected_keys = @default_include |> Map.keys() |> Enum.map(&Atom.to_string/1) + expected_keys = + Plausible.Stats.ParsedQueryParams.default_include() + |> Map.keys() + |> Enum.map(&Atom.to_string/1) if Map.keys(map) |> Enum.all?(&(&1 in expected_keys)) do {:ok, atomize_keys(map)} @@ -386,9 +338,12 @@ defmodule Plausible.Stats.Filters.QueryParser do defp update_comparisons_date_range(include, _site), do: {:ok, include} defp parse_pagination(pagination) when is_map(pagination) do - {:ok, Map.merge(@default_pagination, atomize_keys(pagination))} + {:ok, + Map.merge(Plausible.Stats.ParsedQueryParams.default_pagination(), atomize_keys(pagination))} end + defp parse_pagination(nil), do: {:ok, nil} + defp atomize_keys(map) when is_map(map) do Map.new(map, fn {key, value} -> key = String.to_existing_atom(key) @@ -429,258 +384,6 @@ defmodule Plausible.Stats.Filters.QueryParser do end end - defp validate_order_by(query) do - if query.order_by do - valid_values = query.metrics ++ query.dimensions - - invalid_entry = - Enum.find(query.order_by, fn {value, _direction} -> - not Enum.member?(valid_values, value) - end) - - case invalid_entry do - nil -> - :ok - - _ -> - {:error, - "Invalid order_by entry '#{i(invalid_entry)}'. Entry is not a queried metric or dimension."} - end - else - :ok - end - end - - def preload_goals_and_revenue(site, metrics, filters, dimensions) do - preloaded_goals = - Plausible.Stats.Goals.preload_needed_goals(site, dimensions, filters) - - {revenue_warning, revenue_currencies} = - preload_revenue(site, preloaded_goals, metrics, dimensions) - - { - preloaded_goals, - revenue_warning, - revenue_currencies - } - end - - @only_toplevel ["event:goal", "event:hostname"] - defp validate_toplevel_only_filter_dimension(query) do - not_toplevel = - query.filters - |> Filters.dimensions_used_in_filters(min_depth: 1, behavioral_filters: :ignore) - |> Enum.filter(&(&1 in @only_toplevel)) - - if Enum.count(not_toplevel) > 0 do - {:error, - "Invalid filters. Dimension `#{List.first(not_toplevel)}` can only be filtered at the top level."} - else - :ok - end - end - - @special_metrics [:conversion_rate, :group_conversion_rate] - defp validate_special_metrics_filters(query) do - special_metric? = Enum.any?(@special_metrics, &(&1 in query.metrics)) - - deep_custom_property? = - query.filters - |> Filters.dimensions_used_in_filters(min_depth: 1) - |> Enum.any?(fn dimension -> String.starts_with?(dimension, "event:props:") end) - - if special_metric? and deep_custom_property? do - {:error, - "Invalid filters. When `conversion_rate` or `group_conversion_rate` metrics are used, custom property filters can only be used on top level."} - else - :ok - end - end - - defp validate_behavioral_filters(query) do - query.filters - |> Filters.traverse(0, fn behavioral_depth, operator -> - if operator in [:has_done, :has_not_done] do - behavioral_depth + 1 - else - behavioral_depth - end - end) - |> Enum.reduce_while(:ok, fn {[_operator, dimension | _rest], behavioral_depth}, :ok -> - cond do - behavioral_depth == 0 -> - # ignore non-behavioral filters - {:cont, :ok} - - behavioral_depth > 1 -> - {:halt, - {:error, - "Invalid filters. Behavioral filters (has_done, has_not_done) cannot be nested."}} - - not String.starts_with?(dimension, "event:") -> - {:halt, - {:error, - "Invalid filters. Behavioral filters (has_done, has_not_done) can only be used with event dimension filters."}} - - true -> - {:cont, :ok} - end - end) - end - - defp validate_filtered_goals_exist(query) do - # Note: We don't check :contains goal filters since it's acceptable if they match nothing. - goal_filter_clauses = - query.filters - |> Filters.all_leaf_filters() - |> Enum.flat_map(fn - [:is, "event:goal", clauses] -> clauses - _ -> [] - end) - - if length(goal_filter_clauses) > 0 do - configured_goal_names = - query.preloaded_goals.all - |> Enum.map(&Plausible.Goal.display_name/1) - - validate_list(goal_filter_clauses, &validate_goal_filter(&1, configured_goal_names)) - else - :ok - end - end - - on_ee do - alias Plausible.Stats.Goal.Revenue - - def preload_revenue(site, preloaded_goals, metrics, dimensions) do - Revenue.preload(site, preloaded_goals, metrics, dimensions) - end - - defp validate_revenue_metrics_access(site, query) do - if Revenue.requested?(query.metrics) and not Revenue.available?(site) do - {:error, "The owner of this site does not have access to the revenue metrics feature."} - else - :ok - end - end - else - defp preload_revenue(_site, _preloaded_goals, _metrics, _dimensions), do: {nil, %{}} - - defp validate_revenue_metrics_access(_site, _query), do: :ok - end - - defp validate_goal_filter(clause, configured_goal_names) do - if Enum.member?(configured_goal_names, clause) do - :ok - else - {:error, - "Invalid filters. The goal `#{clause}` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals"} - end - end - - defp validate_custom_props_access(site, query) do - allowed_props = Plausible.Props.allowed_for(site, bypass_setup?: true) - - validate_custom_props_access(site, query, allowed_props) - end - - defp validate_custom_props_access(_site, _query, :all), do: :ok - - defp validate_custom_props_access(_site, query, allowed_props) do - valid? = - query.filters - |> Filters.dimensions_used_in_filters() - |> Enum.concat(query.dimensions) - |> Enum.all?(fn - "event:props:" <> prop -> prop in allowed_props - _ -> true - end) - - if valid? do - :ok - else - {:error, "The owner of this site does not have access to the custom properties feature."} - end - end - - defp validate_metrics(query) do - with :ok <- validate_list(query.metrics, &validate_metric(&1, query)) do - TableDecider.validate_no_metrics_dimensions_conflict(query) - end - end - - defp validate_metric(metric, query) when metric in [:conversion_rate, :group_conversion_rate] do - if Enum.member?(query.dimensions, "event:goal") or - Filters.filtering_on_dimension?(query, "event:goal", behavioral_filters: :ignore) do - :ok - else - {:error, "Metric `#{metric}` can only be queried with event:goal filters or dimensions."} - end - end - - defp validate_metric(:scroll_depth = metric, query) do - page_dimension? = Enum.member?(query.dimensions, "event:page") - toplevel_page_filter? = not is_nil(Filters.get_toplevel_filter(query, "event:page")) - - if page_dimension? or toplevel_page_filter? do - :ok - else - {:error, "Metric `#{metric}` can only be queried with event:page filters or dimensions."} - end - end - - defp validate_metric(:exit_rate = metric, query) do - case {query.dimensions, TableDecider.sessions_join_events?(query)} do - {["visit:exit_page"], false} -> - :ok - - {["visit:exit_page"], true} -> - {:error, "Metric `#{metric}` cannot be queried when filtering on event dimensions."} - - _ -> - {:error, - "Metric `#{metric}` requires a `\"visit:exit_page\"` dimension. No other dimensions are allowed."} - end - end - - defp validate_metric(:views_per_visit = metric, query) do - cond do - Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore) -> - {:error, "Metric `#{metric}` cannot be queried with a filter on `event:page`."} - - length(query.dimensions) > 0 -> - {:error, "Metric `#{metric}` cannot be queried with `dimensions`."} - - true -> - :ok - end - end - - defp validate_metric(:time_on_page = metric, query) do - cond do - Enum.member?(query.dimensions, "event:page") -> - :ok - - Filters.filtering_on_dimension?(query, "event:page", behavioral_filters: :ignore) -> - :ok - - true -> - {:error, "Metric `#{metric}` can only be queried with event:page filters or dimensions."} - end - end - - defp validate_metric(_, _), do: :ok - - defp validate_include(query) do - time_dimension? = Enum.any?(query.dimensions, &Time.time_dimension?/1) - - if query.include.time_labels and not time_dimension? do - {:error, "Invalid include.time_labels: requires a time dimension."} - else - :ok - end - end - defp i(value), do: inspect(value, charlists: :as_lists) defp parse_list(list, parser_function) do @@ -691,13 +394,4 @@ defmodule Plausible.Stats.Filters.QueryParser do end end) end - - defp validate_list(list, parser_function) do - Enum.reduce_while(list, :ok, fn value, :ok -> - case parser_function.(value) do - :ok -> {:cont, :ok} - {:error, _} = error -> {:halt, error} - end - end) - end end diff --git a/lib/plausible_web/controllers/api/external_query_api_controller.ex b/lib/plausible_web/controllers/api/external_query_api_controller.ex index 7fc17e0e64ee..9a5e9d2341de 100644 --- a/lib/plausible_web/controllers/api/external_query_api_controller.ex +++ b/lib/plausible_web/controllers/api/external_query_api_controller.ex @@ -9,7 +9,7 @@ defmodule PlausibleWeb.Api.ExternalQueryApiController do def query(conn, params) do site = Repo.preload(conn.assigns.site, :owners) - case Query.build(site, conn.assigns.schema_type, params, debug_metadata(conn)) do + case Query.parse_and_build(site, conn.assigns.schema_type, params, debug_metadata(conn)) do {:ok, query} -> results = Plausible.Stats.query(site, query) json(conn, results) diff --git a/lib/plausible_web/live/goal_settings/form.ex b/lib/plausible_web/live/goal_settings/form.ex index 3abcaf53d695..5f083871c8f2 100644 --- a/lib/plausible_web/live/goal_settings/form.ex +++ b/lib/plausible_web/live/goal_settings/form.ex @@ -546,7 +546,7 @@ defmodule PlausibleWeb.Live.GoalSettings.Form do def suggest_page_paths(input, site) do query = - Plausible.Stats.Query.build!( + Plausible.Stats.Query.parse_and_build!( site, :internal, %{ diff --git a/lib/plausible_web/live/shields/hostname_rules.ex b/lib/plausible_web/live/shields/hostname_rules.ex index dbb4565bdf84..0b301c0af486 100644 --- a/lib/plausible_web/live/shields/hostname_rules.ex +++ b/lib/plausible_web/live/shields/hostname_rules.ex @@ -228,7 +228,7 @@ defmodule PlausibleWeb.Live.Shields.HostnameRules do def suggest_hostnames(input, _options, site) do query = - Plausible.Stats.Query.build!( + Plausible.Stats.Query.parse_and_build!( site, :internal, %{ diff --git a/lib/plausible_web/live/shields/page_rules.ex b/lib/plausible_web/live/shields/page_rules.ex index 3047fa0c5ef0..e175e3e50731 100644 --- a/lib/plausible_web/live/shields/page_rules.ex +++ b/lib/plausible_web/live/shields/page_rules.ex @@ -221,7 +221,7 @@ defmodule PlausibleWeb.Live.Shields.PageRules do def suggest_page_paths(input, _options, site, page_rules) do query = - Plausible.Stats.Query.build!( + Plausible.Stats.Query.parse_and_build!( site, :internal, %{ diff --git a/lib/workers/send_email_report.ex b/lib/workers/send_email_report.ex index 0c1aa6d81c49..5cfcc7ca1fc4 100644 --- a/lib/workers/send_email_report.ex +++ b/lib/workers/send_email_report.ex @@ -88,7 +88,7 @@ defmodule Plausible.Workers.SendEmailReport do defp stats_aggregates(site, date_range) do query = - Query.build!( + Query.parse_and_build!( site, :internal, %{ @@ -120,7 +120,7 @@ defmodule Plausible.Workers.SendEmailReport do defp pages(site, date_range) do query = - Query.build!( + Query.parse_and_build!( site, :internal, %{ @@ -145,7 +145,7 @@ defmodule Plausible.Workers.SendEmailReport do defp sources(site, date_range) do query = - Query.build!( + Query.parse_and_build!( site, :internal, %{ @@ -171,7 +171,7 @@ defmodule Plausible.Workers.SendEmailReport do defp goals(site, date_range) do query = - Query.build!( + Query.parse_and_build!( site, :internal, %{ diff --git a/lib/workers/traffic_change_notifier.ex b/lib/workers/traffic_change_notifier.ex index 4d14d987cb5f..fccab08b5dda 100644 --- a/lib/workers/traffic_change_notifier.ex +++ b/lib/workers/traffic_change_notifier.ex @@ -137,7 +137,7 @@ defmodule Plausible.Workers.TrafficChangeNotifier do defp put_sources(stats, site) do query = - Query.build!( + Query.parse_and_build!( site, :internal, Map.merge(@base_query_params, %{ @@ -154,7 +154,7 @@ defmodule Plausible.Workers.TrafficChangeNotifier do defp put_pages(stats, site) do query = - Query.build!( + Query.parse_and_build!( site, :internal, Map.merge(@base_query_params, %{ diff --git a/test/plausible/stats/comparisons_test.exs b/test/plausible/stats/comparisons_test.exs index e80a785c3102..ef77ade32bf8 100644 --- a/test/plausible/stats/comparisons_test.exs +++ b/test/plausible/stats/comparisons_test.exs @@ -428,7 +428,7 @@ defmodule Plausible.Stats.ComparisonsTest do defp build_comparison_query(site, params) do query = - Query.build!( + Query.parse_and_build!( site, :internal, Map.merge( diff --git a/test/plausible/stats/query_test.exs b/test/plausible/stats/query/query_from_test.exs similarity index 99% rename from test/plausible/stats/query_test.exs rename to test/plausible/stats/query/query_from_test.exs index d1fd76fb5b10..8ab9f0037f47 100644 --- a/test/plausible/stats/query_test.exs +++ b/test/plausible/stats/query/query_from_test.exs @@ -1,9 +1,8 @@ -defmodule Plausible.Stats.QueryTest do +defmodule Plausible.Stats.Query.QueryFromTest do use Plausible.DataCase, async: true use Plausible.Teams.Test alias Plausible.Stats.Query alias Plausible.Stats.Legacy.QueryBuilder - alias Plausible.Stats.Filters.QueryParser alias Plausible.Stats.DateTimeRange doctest Plausible.Stats.Legacy.QueryBuilder diff --git a/test/plausible/stats/query_optimizer_test.exs b/test/plausible/stats/query/query_optimizer_test.exs similarity index 91% rename from test/plausible/stats/query_optimizer_test.exs rename to test/plausible/stats/query/query_optimizer_test.exs index 036ae9fbc9e9..1d72656b07a9 100644 --- a/test/plausible/stats/query_optimizer_test.exs +++ b/test/plausible/stats/query/query_optimizer_test.exs @@ -1,7 +1,7 @@ -defmodule Plausible.Stats.QueryOptimizerTest do +defmodule Plausible.Stats.Query.QueryOptimizerTest do use Plausible.DataCase, async: true - alias Plausible.Stats.{Query, QueryOptimizer, DateTimeRange} + alias Plausible.Stats.{Query, QueryOptimizer, DateTimeRange, ParsedQueryParams} @default_params %{metrics: [:visitors]} @@ -154,8 +154,6 @@ defmodule Plausible.Stats.QueryOptimizerTest do end describe "trim_relative_date_range" do - alias Plausible.Stats.Filters.QueryParser - test "trims current month period when flag is set" do now = DateTime.new!(~D[2024-01-15], ~T[12:00:00], "UTC") @@ -165,7 +163,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do input_date_range: "month", now: now, timezone: "UTC", - include: Map.put(QueryParser.default_include(), :trim_relative_date_range, true) + include: Map.put(ParsedQueryParams.default_include(), :trim_relative_date_range, true) }) assert result.utc_time_range.first == ~U[2024-01-01 00:00:00Z] @@ -181,7 +179,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do input_date_range: "year", now: now, timezone: "UTC", - include: Map.put(QueryParser.default_include(), :trim_relative_date_range, true) + include: Map.put(ParsedQueryParams.default_include(), :trim_relative_date_range, true) }) assert result.utc_time_range.first == ~U[2024-01-01 00:00:00Z] @@ -197,7 +195,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do input_date_range: "day", now: now, timezone: "UTC", - include: Map.put(QueryParser.default_include(), :trim_relative_date_range, true) + include: Map.put(ParsedQueryParams.default_include(), :trim_relative_date_range, true) }) assert result.utc_time_range.first == ~U[2024-01-15 00:00:00Z] @@ -214,7 +212,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do input_date_range: "month", now: now, timezone: "UTC", - include: Map.put(QueryParser.default_include(), :trim_relative_date_range, true) + include: Map.put(ParsedQueryParams.default_include(), :trim_relative_date_range, true) }) assert result.utc_time_range == original_range @@ -230,7 +228,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do input_date_range: "year", now: now, timezone: "UTC", - include: Map.put(QueryParser.default_include(), :trim_relative_date_range, true) + include: Map.put(ParsedQueryParams.default_include(), :trim_relative_date_range, true) }) assert result.utc_time_range == original_range @@ -246,7 +244,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do input_date_range: "day", now: now, timezone: "UTC", - include: Map.put(QueryParser.default_include(), :trim_relative_date_range, true) + include: Map.put(ParsedQueryParams.default_include(), :trim_relative_date_range, true) }) assert result.utc_time_range == original_range @@ -264,7 +262,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do timezone: "UTC", include: Map.merge( - QueryParser.default_include(), + ParsedQueryParams.default_include(), %{comparisons: %{mode: "previous_period"}, trim_relative_date_range: true} ) }) @@ -283,7 +281,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do timezone: "UTC", include: Map.merge( - QueryParser.default_include(), + ParsedQueryParams.default_include(), %{comparisons: %{mode: "previous_period"}, trim_relative_date_range: true} ) }) @@ -302,7 +300,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do input_date_range: "month", now: now, timezone: "UTC", - include: Map.put(QueryParser.default_include(), :trim_relative_date_range, false) + include: Map.put(ParsedQueryParams.default_include(), :trim_relative_date_range, false) }) assert result.utc_time_range == original_range @@ -318,7 +316,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do input_date_range: "month", now: now, timezone: "UTC", - include: QueryParser.default_include() + include: ParsedQueryParams.default_include() }) assert result.utc_time_range == original_range @@ -335,7 +333,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do input_date_range: "7d", now: now, timezone: "UTC", - include: Map.put(QueryParser.default_include(), :trim_relative_date_range, true) + include: Map.put(ParsedQueryParams.default_include(), :trim_relative_date_range, true) }) assert result.utc_time_range == original_range @@ -352,7 +350,7 @@ defmodule Plausible.Stats.QueryOptimizerTest do input_date_range: "year", now: now, timezone: "America/New_York", - include: Map.put(QueryParser.default_include(), :trim_relative_date_range, true) + include: Map.put(ParsedQueryParams.default_include(), :trim_relative_date_range, true) }) nyc_mar_15_end = diff --git a/test/plausible/stats/query/query_parse_and_build_test.exs b/test/plausible/stats/query/query_parse_and_build_test.exs new file mode 100644 index 000000000000..a2361bf17907 --- /dev/null +++ b/test/plausible/stats/query/query_parse_and_build_test.exs @@ -0,0 +1,3056 @@ +defmodule Plausible.Stats.Query.QueryParseAndBuildTest do + use Plausible.DataCase + use Plausible.Teams.Test + import Plausible.AssertMatches + + alias Plausible.Stats.{Query, DateTimeRange, Filters} + + @now DateTime.new!(~D[2021-05-05], ~T[12:30:00], "Etc/UTC") + @date_range_realtime %DateTimeRange{ + first: DateTime.new!(~D[2021-05-05], ~T[12:25:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "Etc/UTC") + } + @date_range_30m %DateTimeRange{ + first: DateTime.new!(~D[2021-05-05], ~T[12:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "Etc/UTC") + } + @date_range_day %DateTimeRange{ + first: DateTime.new!(~D[2021-05-05], ~T[00:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "Etc/UTC") + } + @date_range_7d %DateTimeRange{ + first: DateTime.new!(~D[2021-04-28], ~T[00:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-05-04], ~T[23:59:59], "Etc/UTC") + } + @date_range_10d %DateTimeRange{ + first: DateTime.new!(~D[2021-04-25], ~T[00:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-05-04], ~T[23:59:59], "Etc/UTC") + } + @date_range_30d %DateTimeRange{ + first: DateTime.new!(~D[2021-04-05], ~T[00:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-05-04], ~T[23:59:59], "Etc/UTC") + } + @date_range_month %DateTimeRange{ + first: DateTime.new!(~D[2021-05-01], ~T[00:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "Etc/UTC") + } + @date_range_3mo %DateTimeRange{ + first: DateTime.new!(~D[2021-02-01], ~T[00:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-04-30], ~T[23:59:59], "Etc/UTC") + } + @date_range_6mo %DateTimeRange{ + first: DateTime.new!(~D[2020-11-01], ~T[00:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-04-30], ~T[23:59:59], "Etc/UTC") + } + @date_range_year %DateTimeRange{ + first: DateTime.new!(~D[2021-01-01], ~T[00:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-12-31], ~T[23:59:59], "Etc/UTC") + } + @date_range_12mo %DateTimeRange{ + first: DateTime.new!(~D[2020-05-01], ~T[00:00:00], "Etc/UTC"), + last: DateTime.new!(~D[2021-04-30], ~T[23:59:59], "Etc/UTC") + } + + @default_include %{ + imports: false, + imports_meta: false, + time_labels: false, + total_rows: false, + comparisons: nil, + legacy_time_on_page_cutoff: nil, + trim_relative_date_range: false + } + + setup [:create_user, :create_site] + + setup do + Plausible.Stats.Query.Test.fix_now(@now) + :ok + end + + def check_goals(query, opts) do + assert %Query{ + preloaded_goals: preloaded_goals, + revenue_warning: revenue_warning, + revenue_currencies: revenue_currencies + } = query + + assert goal_names(preloaded_goals[:all]) == + Enum.sort(Keyword.get(opts, :preloaded_goals)[:all]) + + assert goal_names(preloaded_goals[:matching_toplevel_filters]) == + Enum.sort(Keyword.get(opts, :preloaded_goals)[:matching_toplevel_filters]) + + assert revenue_warning == Keyword.get(opts, :revenue_warning) + assert revenue_currencies == Keyword.get(opts, :revenue_currencies) + end + + defp goal_names(goals), do: Enum.map(goals, & &1.display_name) |> Enum.sort() + + describe "metrics" do + test "valid metrics passed", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + "date_range" => "all" + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors, :events], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "fuller list of metrics", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => [ + "visitors", + "pageviews", + "visits", + "events", + "bounce_rate", + "visit_duration" + ], + "date_range" => "all" + } + + assert {:ok, query} = Query.parse_and_build(site, :internal, params) + + assert_matches %Query{ + metrics: [ + :visitors, + :pageviews, + :visits, + :events, + :bounce_rate, + :visit_duration + ], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "same metric queried multiple times", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["events", "visitors", "visitors"], + "date_range" => "all" + } + + assert {:error, "#/metrics: Expected items to be unique but they were not."} = + Query.parse_and_build(site, :public, params) + end + + test "no metrics passed", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => [], + "date_range" => "all" + } + + assert {:error, "#/metrics: Expected a minimum of 1 items but got 0."} = + Query.parse_and_build(site, :public, params) + end + end + + describe "filters validation" do + for operation <- [ + :is, + :is_not, + :matches_wildcard, + :matches_wildcard_not, + :matches, + :matches_not, + :contains, + :contains_not + ] do + test "#{operation} filter", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + [Atom.to_string(unquote(operation)), "event:name", ["foo"]] + ] + } + + assert {:ok, query} = Query.parse_and_build(site, :internal, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [ + [^unquote(operation), "event:name", ["foo"]] + ], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "#{operation} filter with invalid clause", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + [Atom.to_string(unquote(operation)), "event:name", "foo"] + ] + } + + assert {:error, error} = Query.parse_and_build(site, :internal, params) + + assert error == + "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:name\", \"foo\"]" + end + end + + for operation <- [:matches_wildcard, :matches_wildcard_not] do + test "#{operation} is not a valid filter operation in public API", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + [Atom.to_string(unquote(operation)), "event:name", ["foo"]] + ] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:name\", [\"foo\"]]" + end + end + + for too_short_filter <- [ + [], + ["and"], + ["or"], + ["and", []], + ["or", []], + ["not"], + ["is_not"], + ["is_not", "event:name"], + ["has_done"], + ["has_not_done"] + ] do + test "errors on too short filter #{inspect(too_short_filter)}", %{ + site: site + } do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + unquote(too_short_filter) + ] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == ~s(#/filters/0: Invalid filter #{inspect(unquote(too_short_filter))}) + end + end + + valid_filter = ["is", "event:props:foobar", ["value"]] + + for too_long_filter <- [ + ["and", [valid_filter], "extra"], + ["or", [valid_filter], []], + ["not", valid_filter, 1], + Enum.concat(valid_filter, [true]) + ] do + test "errors on too long filter #{inspect(too_long_filter)}", %{ + site: site + } do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + unquote(too_long_filter) + ] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == ~s(#/filters/0: Invalid filter #{inspect(unquote(too_long_filter))}) + end + end + + test "filtering by invalid operation", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["exists?", "event:name", ["foo"]] + ] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "#/filters/0: Invalid filter [\"exists?\", \"event:name\", [\"foo\"]]" + end + + test "filtering by custom properties", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "event:props:foobar", ["value"]] + ] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [ + [:is, "event:props:foobar", ["value"]] + ], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + for dimension <- Filters.event_props() do + if dimension != "goal" do + test "filtering by event:#{dimension} filter", %{site: site} do + prefixed_dimension = "event:#{unquote(dimension)}" + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", prefixed_dimension, ["foo"]] + ] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [ + [:is, ^prefixed_dimension, ["foo"]] + ], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + end + end + + for dimension <- Filters.visit_props() do + test "filtering by visit:#{dimension} filter", %{site: site} do + prefixed_dimension = "visit:#{unquote(dimension)}" + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", prefixed_dimension, ["ab"]] + ] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [ + [:is, ^prefixed_dimension, ["ab"]] + ], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + end + + test "invalid event filter", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "event:device", ["foo"]] + ] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == "#/filters/0: Invalid filter [\"is\", \"event:device\", [\"foo\"]]" + end + + test "invalid visit filter", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "visit:name", ["foo"]] + ] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == "#/filters/0: Invalid filter [\"is\", \"visit:name\", [\"foo\"]]" + end + + test "invalid filter", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => "foobar" + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "#/filters: Type mismatch. Expected Array but got String." + end + + test "numeric filter is invalid", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "visit:os_version", [123]]] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "Invalid filter '[\"is\", \"visit:os_version\", [123]]'." + end + + test "numbers are valid for visit:city", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "visit:city", [123, 456]]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [ + [:is, "visit:city", [123, 456]] + ], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "strings are valid for visit:city", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "visit:city", ["123", "456"]]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [ + [:is, "visit:city", ["123", "456"]] + ], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "invalid visit:country filter", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "visit:country", ["USA"]]] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "Invalid visit:country filter, visit:country needs to be a valid 2-letter country code." + end + + test "valid nested `not`, `and` and `or`", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + [ + "or", + [ + [ + "and", + [ + ["is", "visit:city_name", ["Tallinn"]], + ["is", "visit:country_name", ["Estonia"]] + ] + ], + ["not", ["is", "visit:country_name", ["Estonia"]]] + ] + ] + ] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [ + [ + :or, + [ + [ + :and, + [ + [:is, "visit:city_name", ["Tallinn"]], + [:is, "visit:country_name", ["Estonia"]] + ] + ], + [:not, [:is, "visit:country_name", ["Estonia"]]] + ] + ] + ], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "valid has_done and has_not_done filters", %{site: site} do + insert(:goal, %{site: site, event_name: "Signup"}) + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["has_done", ["is", "event:name", ["Signup"]]], + [ + "has_not_done", + [ + "or", + [ + ["is", "event:goal", ["Signup"]], + ["is", "event:page", ["/signup"]] + ] + ] + ] + ] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [ + [:has_done, [:is, "event:name", ["Signup"]]], + [ + :has_not_done, + [ + :or, + [[:is, "event:goal", ["Signup"]], [:is, "event:page", ["/signup"]]] + ] + ] + ], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "fails when using visit filters within has_done filters", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["has_done", ["is", "visit:browser", ["Chrome"]]] + ] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "Invalid filters. Behavioral filters (has_done, has_not_done) can only be used with event dimension filters." + end + + test "fails when nesting behavioral filters", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["has_done", ["has_not_done", ["is", "visit:browser", ["Chrome"]]]] + ] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "Invalid filters. Behavioral filters (has_done, has_not_done) cannot be nested." + end + + for operator <- ["not", "or", "has_done", "has_not_done"] do + test "invalid `#{operator}` clause", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [[unquote(operator), []]] + } + + assert {:error, error} = Query.parse_and_build(site, :internal, params) + assert error == "#/filters/0: Invalid filter [\"#{unquote(operator)}\", []]" + end + end + + test "event:hostname filter", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "event:hostname", ["a.plausible.io"]]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [ + [:is, "event:hostname", ["a.plausible.io"]] + ], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "event:hostname filter not at top level is invalid", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["not", ["is", "event:hostname", ["a.plausible.io"]]]] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "Invalid filters. Dimension `event:hostname` can only be filtered at the top level." + end + + for operation <- [:is, :contains, :is_not, :contains_not] do + test "#{operation} allows case_sensitive modifier", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + [ + Atom.to_string(unquote(operation)), + "event:page", + ["/foo"], + %{"case_sensitive" => false} + ], + [ + Atom.to_string(unquote(operation)), + "event:name", + ["/foo"], + %{"case_sensitive" => true} + ] + ] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [ + [ + ^unquote(operation), + "event:page", + ["/foo"], + %{case_sensitive: false} + ], + [^unquote(operation), "event:name", ["/foo"], %{case_sensitive: true}] + ], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + end + + for operation <- [:matches, :matches_not, :matches_wildcard, :matches_wildcard_not] do + test "case_sensitive modifier is not valid for #{operation}", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + [ + Atom.to_string(unquote(operation)), + "event:hostname", + ["a.plausible.io"], + %{"case_sensitive" => false} + ] + ] + } + + assert {:error, error} = Query.parse_and_build(site, :internal, params) + + assert error == + "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:hostname\", [\"a.plausible.io\"], %{\"case_sensitive\" => false}]" + end + end + end + + describe "preloading goals" do + setup %{site: site} do + insert(:goal, %{site: site, event_name: "Signup"}) + insert(:goal, %{site: site, event_name: "Purchase"}) + insert(:goal, %{site: site, event_name: "Contact"}) + + :ok + end + + test "with exact match", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "event:goal", ["Signup", "Purchase"]]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [[:is, "event:goal", ["Signup", "Purchase"]]], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + + check_goals(query, + preloaded_goals: %{ + all: ["Contact", "Purchase", "Signup"], + matching_toplevel_filters: ["Purchase", "Signup"] + }, + revenue_currencies: %{} + ) + end + + test "with case insensitive match", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "event:goal", ["signup", "purchase"], %{"case_sensitive" => false}]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [ + [:is, "event:goal", ["signup", "purchase"], %{case_sensitive: false}] + ], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + + check_goals(query, + preloaded_goals: %{ + all: ["Contact", "Purchase", "Signup"], + matching_toplevel_filters: ["Purchase", "Signup"] + }, + revenue_currencies: %{} + ) + end + + test "with contains match", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["contains", "event:goal", ["Sign", "pur"]]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [[:contains, "event:goal", ["Sign", "pur"]]], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + + check_goals(query, + preloaded_goals: %{ + all: ["Contact", "Purchase", "Signup"], + matching_toplevel_filters: ["Signup"] + }, + revenue_currencies: %{} + ) + end + + test "with case insensitive contains match", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["contains", "event:goal", ["sign", "CONT"], %{"case_sensitive" => false}]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [ + [:contains, "event:goal", ["sign", "CONT"], %{case_sensitive: false}] + ], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + + check_goals(query, + preloaded_goals: %{ + all: ["Contact", "Purchase", "Signup"], + matching_toplevel_filters: ["Contact", "Signup"] + }, + revenue_currencies: %{} + ) + end + end + + describe "include validation" do + test "setting include values", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["time"], + "include" => %{"imports" => true, "time_labels" => true, "total_rows" => true} + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: ["time"], + order_by: nil, + timezone: ^site.timezone, + include: %{ + imports: true, + imports_meta: false, + time_labels: true, + total_rows: true, + comparisons: nil, + legacy_time_on_page_cutoff: nil, + trim_relative_date_range: false + }, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "setting invalid imports value", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "include" => "foobar" + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "#/include: Type mismatch. Expected Object but got String." + end + + test "setting include.time_labels without time dimension", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "include" => %{"time_labels" => true} + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "Invalid include.time_labels: requires a time dimension." + end + end + + describe "include.comparisons" do + test "not allowed in public API", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "include" => %{"comparisons" => %{"mode" => "previous_period"}} + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "#/include/comparisons: Schema does not allow additional properties." + end + + test "mode=previous_period", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "include" => %{"comparisons" => %{"mode" => "previous_period"}} + } + + assert {:ok, query} = Query.parse_and_build(site, :internal, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: %{ + comparisons: %{ + mode: "previous_period" + }, + imports: false, + imports_meta: false, + time_labels: false, + total_rows: false, + legacy_time_on_page_cutoff: nil, + trim_relative_date_range: false + }, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "mode=year_over_year", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "include" => %{"comparisons" => %{"mode" => "year_over_year"}} + } + + assert {:ok, query} = Query.parse_and_build(site, :internal, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: %{ + comparisons: %{ + mode: "year_over_year" + }, + imports: false, + imports_meta: false, + time_labels: false, + total_rows: false, + legacy_time_on_page_cutoff: nil, + trim_relative_date_range: false + }, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "mode=custom", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "include" => %{ + "comparisons" => %{"mode" => "custom", "date_range" => ["2021-04-05", "2021-05-04"]} + } + } + + assert {:ok, query} = Query.parse_and_build(site, :internal, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: %{ + comparisons: %{ + mode: "custom", + date_range: ^@date_range_30d + }, + imports_meta: false, + imports: false, + time_labels: false, + total_rows: false, + legacy_time_on_page_cutoff: nil, + trim_relative_date_range: false + }, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "mode=custom without date_range is invalid", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "include" => %{"comparisons" => %{"mode" => "custom"}} + } + + assert {:error, error} = Query.parse_and_build(site, :internal, params) + + assert error == + "#/include/comparisons: Expected exactly one of the schemata to match, but none of them did." + end + + test "mode=previous_period with date_range is invalid", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "include" => %{ + "comparisons" => %{ + "mode" => "previous_period", + "date_range" => ["2024-01-01", "2024-01-31"] + } + } + } + + assert {:error, error} = Query.parse_and_build(site, :internal, params) + + assert error == + "#/include/comparisons: Expected exactly one of the schemata to match, but none of them did." + end + end + + describe "pagination validation" do + test "setting pagination values", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["time"], + "pagination" => %{"limit" => 100, "offset" => 200} + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: ["time"], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 100, offset: 200} + } = query + end + + test "out of range limit value", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "pagination" => %{"limit" => 100_000} + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "#/pagination/limit: Expected the value to be <= 10000" + end + + test "out of range offset value", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "pagination" => %{"offset" => -5} + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "#/pagination/offset: Expected the value to be >= 0" + end + end + + describe "event:goal filter validation" do + test "valid filters", %{site: site} do + insert(:goal, %{site: site, event_name: "Signup"}) + insert(:goal, %{site: site, page_path: "/thank-you"}) + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "event:goal", ["Signup", "Visit /thank-you"]] + ] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [ + [:is, "event:goal", ["Signup", "Visit /thank-you"]] + ], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + + check_goals(query, + preloaded_goals: %{ + all: ["Signup", "Visit /thank-you"], + matching_toplevel_filters: ["Signup", "Visit /thank-you"] + }, + revenue_warning: nil, + revenue_currencies: %{} + ) + end + + test "invalid event filter", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "event:goal", ["Signup"]] + ] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "Invalid filters. The goal `Signup` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals" + end + + test "invalid page filter", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "event:goal", ["Visit /thank-you"]] + ] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "Invalid filters. The goal `Visit /thank-you` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals" + end + + test "unsupported filter", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is_not", "event:goal", ["Signup"]] + ] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == "#/filters/0: Invalid filter [\"is_not\", \"event:goal\", [\"Signup\"]]" + end + + test "not top-level filter", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + [ + "or", + [ + ["is", "event:goal", ["Signup"]], + ["is", "event:name", ["pageview"]] + ] + ] + ] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "Invalid filters. Dimension `event:goal` can only be filtered at the top level." + end + + test "allowed within behavioral filters has_done", %{site: site} do + insert(:goal, %{site: site, event_name: "Signup"}) + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + [ + "has_done", + [ + "or", + [ + ["is", "event:goal", ["Signup"]], + ["is", "event:name", ["pageview"]] + ] + ] + ] + ] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [ + [ + :has_done, + [ + :or, + [ + [:is, "event:goal", ["Signup"]], + [:is, "event:name", ["pageview"]] + ] + ] + ] + ], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + + check_goals(query, + preloaded_goals: %{all: ["Signup"], matching_toplevel_filters: ["Signup"]}, + revenue_warning: nil, + revenue_currencies: %{} + ) + end + + test "name is checked even within behavioral filters", %{site: site} do + insert(:goal, %{site: site, event_name: "Signup"}) + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["has_done", ["is", "event:goal", ["Unknown"]]]] + } + + assert {:error, error} = Query.parse_and_build(site, :internal, params) + + assert error == + "Invalid filters. The goal `Unknown` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals" + end + end + + describe "date range validation" do + for {shortcut, expected_date_range} <- [ + {"day", @date_range_day}, + {"7d", @date_range_7d}, + {"10d", @date_range_10d}, + {"30d", @date_range_30d}, + {"month", @date_range_month}, + {"3mo", @date_range_3mo}, + {"6mo", @date_range_6mo}, + {"12mo", @date_range_12mo}, + {"year", @date_range_year} + ] do + test "parses '#{shortcut}' date_range shortcut ", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + "date_range" => unquote(shortcut) + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors, :events], + utc_time_range: ^unquote(Macro.escape(expected_date_range)), + filters: [], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + end + + for {shortcut, expected_date_range} <- [ + {"30m", @date_range_30m}, + {"realtime", @date_range_realtime} + ] do + test "'#{shortcut}' shortcut is available only in the internal API schema", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + "date_range" => unquote(shortcut) + } + + assert {:ok, query} = Query.parse_and_build(site, :internal, params) + + assert_matches %Query{ + metrics: [:visitors, :events], + utc_time_range: ^unquote(Macro.escape(expected_date_range)), + filters: [], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "#/date_range: Invalid date range \"#{unquote(shortcut)}\"" + end + end + + test "parsing `all` with previous data", %{site: site} do + site = Map.put(site, :stats_start_date, ~D[2020-01-01]) + expected_date_range = DateTimeRange.new!(~D[2020-01-01], ~D[2021-05-05], "Etc/UTC") + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + "date_range" => "all" + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors, :events], + utc_time_range: ^expected_date_range, + filters: [], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "parsing `all` with no previous data", %{site: site} do + site = Map.put(site, :stats_start_date, nil) + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + "date_range" => "all" + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors, :events], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "parsing custom date range from simple date strings", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + "date_range" => ["2021-05-05", "2021-05-05"] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors, :events], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "parsing custom date range from iso8601 timestamps", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + "date_range" => ["2024-01-01T00:00:00Z", "2024-01-02T23:59:59Z"] + } + + expected_utc_time_range = + DateTimeRange.new!( + DateTime.new!(~D[2024-01-01], ~T[00:00:00], "Etc/UTC"), + DateTime.new!(~D[2024-01-02], ~T[23:59:59], "Etc/UTC") + ) + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors, :events], + utc_time_range: ^expected_utc_time_range, + filters: [], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "parsing custom date range from iso8601 timestamps with non-UTC timezone", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + "date_range" => ["2024-08-29T07:12:34-07:00", "2024-08-29T10:12:34-07:00"] + } + + expected_utc_time_range = + DateTimeRange.new!(~U[2024-08-29 14:12:34Z], ~U[2024-08-29 17:12:34Z]) + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors, :events], + utc_time_range: ^expected_utc_time_range, + filters: [], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + for invalid_value <- ["-1d", "foo", ["21415-00", "eee"]] do + test "errors on invalid date range value (#{inspect(invalid_value)})", %{site: site} do + params = %{ + "site_id" => site.domain, + "date_range" => unquote(invalid_value), + "metrics" => ["visitors"] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == "#/date_range: Invalid date range #{inspect(unquote(invalid_value))}" + end + end + + test "999999999mo is invalid date range", %{site: site} do + params = %{ + "site_id" => site.domain, + "date_range" => "999999999mo", + "metrics" => ["visitors"] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == "Invalid date_range \"999999999mo\"" + end + + test "custom date range is invalid when timestamps do not include timezone info", %{ + site: site + } do + params = %{ + "site_id" => site.domain, + "date_range" => ["2021-02-03T00:00:00", "2021-02-03T23:59:59"], + "metrics" => ["visitors"] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == "Invalid date_range '[\"2021-02-03T00:00:00\", \"2021-02-03T23:59:59\"]'." + end + + test "custom date range is invalid when timestamp timezone is invalid", %{site: site} do + params = %{ + "site_id" => site.domain, + "date_range" => ["2021-02-03T00:00:00-25:00", "2021-02-03T23:59:59-25:00"], + "metrics" => ["visitors"] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "#/date_range: Invalid date range [\"2021-02-03T00:00:00-25:00\", \"2021-02-03T23:59:59-25:00\"]" + end + + test "custom date range is invalid when date and timestamp are combined", %{site: site} do + params = %{ + "site_id" => site.domain, + "date_range" => ["2021-02-03T00:00:00Z", "2021-02-04"], + "metrics" => ["visitors"] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "#/date_range: Invalid date range [\"2021-02-03T00:00:00Z\", \"2021-02-04\"]" + end + + test "parses date_range relative to date param", %{site: site} do + date = @now |> DateTime.to_date() |> Date.to_string() + + for {date_range_shortcut, expected_date_range} <- [ + {"day", @date_range_day}, + {"7d", @date_range_7d}, + {"10d", @date_range_10d}, + {"30d", @date_range_30d}, + {"month", @date_range_month}, + {"3mo", @date_range_3mo}, + {"6mo", @date_range_6mo}, + {"12mo", @date_range_12mo}, + {"year", @date_range_year} + ] do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => date_range_shortcut, + "date" => date + } + + assert {:ok, query} = Query.parse_and_build(site, :internal, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^expected_date_range, + filters: [], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + end + + test "date parameter is not available in the public API", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + "date_range" => "month", + "date" => "2021-05-05" + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "#/date: Schema does not allow additional properties." + end + + test "parses date_range.first into a datetime right after the gap in site.timezone", %{ + site: site + } do + site = %{site | timezone: "America/Santiago"} + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => ["2022-09-11", "2022-09-11"] + } + + expected_utc_time_range = + DateTimeRange.new!(~U[2022-09-11 04:00:00Z], ~U[2022-09-12 02:59:59Z]) + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^expected_utc_time_range, + filters: [], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "parses date_range.first into the latest of ambiguous datetimes in site.timezone", %{ + site: site + } do + site = %{site | timezone: "America/Havana"} + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => ["2023-11-05", "2023-11-05"] + } + + expected_utc_time_range = + DateTimeRange.new!(~U[2023-11-05 05:00:00Z], ~U[2023-11-06 04:59:59Z]) + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^expected_utc_time_range, + filters: [], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "parses date_range.last into the earliest of ambiguous datetimes in site.timezone", %{ + site: site + } do + site = %{site | timezone: "America/Asuncion"} + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => ["2024-03-23", "2024-03-23"] + } + + expected_utc_time_range = + DateTimeRange.new!(~U[2024-03-23 03:00:00Z], ~U[2024-03-24 02:59:59Z]) + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^expected_utc_time_range, + filters: [], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + end + + describe "dimensions validation" do + for dimension <- Filters.event_props() do + test "event:#{dimension} dimension", %{site: site} do + prefixed_dimension = "event:#{unquote(dimension)}" + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => [prefixed_dimension] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: [^prefixed_dimension], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + end + + for dimension <- Filters.visit_props() do + test "visit:#{dimension} dimension", %{site: site} do + prefixed_dimension = "visit:#{unquote(dimension)}" + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => [prefixed_dimension] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: [^prefixed_dimension], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + end + + test "time:minute dimension fails public schema validation", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["time:minute"] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "#/dimensions/0: Invalid dimension \"time:minute\"" + end + + test "time:minute dimension passes internal schema validation", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["time:minute"] + } + + assert {:ok, query} = Query.parse_and_build(site, :internal, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: ["time:minute"], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "custom properties dimension", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:foobar"] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: ["event:props:foobar"], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "invalid custom property dimension", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:"] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "#/dimensions/0: Invalid dimension \"event:props:\"" + end + + test "invalid dimension name passed", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["visitors"] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "#/dimensions/0: Invalid dimension \"visitors\"" + end + + test "invalid dimension", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => "foobar" + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "#/dimensions: Type mismatch. Expected Array but got String." + end + + test "dimensions are not unique", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:name", "event:name"] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "#/dimensions: Expected items to be unique but they were not." + end + end + + describe "order_by validation" do + test "ordering by metric", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + "date_range" => "all", + "order_by" => [["events", "desc"], ["visitors", "asc"]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors, :events], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: [], + order_by: [{:events, :desc}, {:visitors, :asc}], + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "ordering by dimension", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:name"], + "order_by" => [["event:name", "desc"]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: ["event:name"], + order_by: [{"event:name", :desc}], + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "ordering by invalid value", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "order_by" => [["visssss", "desc"]] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "#/order_by/0/0: Invalid value in order_by \"visssss\"" + end + + test "ordering by not queried metric", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "order_by" => [["events", "desc"]] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "Invalid order_by entry '{:events, :desc}'. Entry is not a queried metric or dimension." + end + + test "ordering by not queried dimension", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "order_by" => [["event:name", "desc"]] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "Invalid order_by entry '{\"event:name\", :desc}'. Entry is not a queried metric or dimension." + end + end + + describe "custom props access" do + test "filters - no access", %{site: site, user: user} do + subscribe_to_enterprise_plan(user, features: [Plausible.Billing.Feature.StatsAPI]) + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["not", ["is", "event:props:foobar", ["foo"]]]] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "The owner of this site does not have access to the custom properties feature." + end + + test "dimensions - no access", %{site: site, user: user} do + subscribe_to_enterprise_plan(user, features: [Plausible.Billing.Feature.StatsAPI]) + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:props:foobar"] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "The owner of this site does not have access to the custom properties feature." + end + end + + describe "conversion_rate metric" do + test "fails validation on its own", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["conversion_rate"], + "date_range" => "all" + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "Metric `conversion_rate` can only be queried with event:goal filters or dimensions." + end + + test "succeeds with event:goal filter", %{site: site} do + insert(:goal, %{site: site, event_name: "Signup"}) + insert(:goal, %{site: site, event_name: "Purchase", currency: "USD"}) + + params = %{ + "site_id" => site.domain, + "metrics" => ["conversion_rate"], + "date_range" => "all", + "filters" => [["is", "event:goal", ["Signup"]]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:conversion_rate], + utc_time_range: ^@date_range_day, + filters: [[:is, "event:goal", ["Signup"]]], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + + check_goals(query, + preloaded_goals: %{ + all: ["Purchase", "Signup"], + matching_toplevel_filters: ["Signup"] + }, + revenue_currencies: %{} + ) + end + + test "succeeds with event:goal dimension", %{site: site} do + insert(:goal, %{site: site, event_name: "Purchase", currency: "USD"}) + insert(:goal, %{site: site, event_name: "Signup"}) + + params = %{ + "site_id" => site.domain, + "metrics" => ["conversion_rate"], + "date_range" => "all", + "dimensions" => ["event:goal"] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:conversion_rate], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: ["event:goal"], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + + check_goals(query, + preloaded_goals: %{ + all: ["Purchase", "Signup"], + matching_toplevel_filters: ["Purchase", "Signup"] + }, + revenue_currencies: %{} + ) + end + + test "custom properties filter with special metric", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["conversion_rate", "group_conversion_rate"], + "date_range" => "all", + "filters" => [["is", "event:props:foo", ["bar"]]], + "dimensions" => ["event:goal"] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:conversion_rate, :group_conversion_rate], + utc_time_range: ^@date_range_day, + filters: [ + [:is, "event:props:foo", ["bar"]] + ], + dimensions: ["event:goal"], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "not top level custom properties filter with special metric is invalid", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["conversion_rate", "group_conversion_rate"], + "date_range" => "all", + "filters" => [["not", ["is", "event:props:foo", ["bar"]]]], + "dimensions" => ["event:goal"] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "Invalid filters. When `conversion_rate` or `group_conversion_rate` metrics are used, custom property filters can only be used on top level." + end + end + + describe "exit_rate metric" do + test "fails validation without visit:exit_page dimension", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["exit_rate"], + "date_range" => "all" + } + + assert {:error, error} = Query.parse_and_build(site, :internal, params) + + assert error == + "Metric `exit_rate` requires a `\"visit:exit_page\"` dimension. No other dimensions are allowed." + end + + test "fails validation with event only filters", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["exit_rate"], + "dimensions" => ["visit:exit_page"], + "filters" => [["is", "event:page", ["/"]]], + "date_range" => "all" + } + + assert {:error, error} = Query.parse_and_build(site, :internal, params) + assert error == "Metric `exit_rate` cannot be queried when filtering on event dimensions." + end + + test "fails validation with event metrics", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["exit_rate", "pageviews"], + "dimensions" => ["visit:exit_page"], + "date_range" => "all" + } + + assert {:error, error} = Query.parse_and_build(site, :internal, params) + + assert error == + "Event metric(s) `pageviews` cannot be queried along with session dimension(s) `visit:exit_page`" + end + + test "passes validation", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["exit_rate"], + "dimensions" => ["visit:exit_page"], + "date_range" => "all" + } + + assert {:ok, query} = Query.parse_and_build(site, :internal, params) + + assert_matches %Query{ + metrics: [:exit_rate], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: ["visit:exit_page"], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + end + + describe "scroll_depth metric" do + test "fails validation on its own", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["scroll_depth"], + "date_range" => "all" + } + + assert {:error, error} = Query.parse_and_build(site, :internal, params) + + assert error == + "Metric `scroll_depth` can only be queried with event:page filters or dimensions." + end + + test "fails with only a non-top-level event:page filter", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["scroll_depth"], + "date_range" => "all", + "filters" => [["not", ["is", "event:page", ["/"]]]] + } + + assert {:error, error} = Query.parse_and_build(site, :internal, params) + + assert error == + "Metric `scroll_depth` can only be queried with event:page filters or dimensions." + end + + test "succeeds with top-level event:page filter", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["scroll_depth"], + "date_range" => "all", + "filters" => [["is", "event:page", ["/"]]] + } + + assert {:ok, query} = Query.parse_and_build(site, :internal, params) + + assert_matches %Query{ + metrics: [:scroll_depth], + utc_time_range: ^@date_range_day, + filters: [[:is, "event:page", ["/"]]], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "succeeds with event:page dimension", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["scroll_depth"], + "date_range" => "all", + "dimensions" => ["event:page"] + } + + assert {:ok, query} = Query.parse_and_build(site, :internal, params) + + assert_matches %Query{ + metrics: [:scroll_depth], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: ["event:page"], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + end + + describe "views_per_visit metric" do + test "succeeds with normal filters", %{site: site} do + insert(:goal, %{site: site, event_name: "Signup"}) + + params = %{ + "site_id" => site.domain, + "metrics" => ["views_per_visit"], + "date_range" => "all", + "filters" => [["is", "event:goal", ["Signup"]]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:views_per_visit], + utc_time_range: ^@date_range_day, + filters: [[:is, "event:goal", ["Signup"]]], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + + check_goals(query, + preloaded_goals: %{all: ["Signup"], matching_toplevel_filters: ["Signup"]}, + revenue_currencies: %{} + ) + end + + test "fails validation if event:page filter specified", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["views_per_visit"], + "date_range" => "all", + "filters" => [["is", "event:page", ["/"]]] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "Metric `views_per_visit` cannot be queried with a filter on `event:page`." + end + + test "fails validation with dimensions", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["views_per_visit"], + "date_range" => "all", + "dimensions" => ["event:name"] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "Metric `views_per_visit` cannot be queried with `dimensions`." + end + end + + describe "time_on_page metric" do + test "fails validation on its own", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["time_on_page"], + "date_range" => "all" + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "Metric `time_on_page` can only be queried with event:page filters or dimensions." + end + + test "succeeds with event:page dimension", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["time_on_page"], + "date_range" => "all", + "dimensions" => ["time", "event:page"] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:time_on_page], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: ["time", "event:page"], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "succeeds with event:page filter", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["time_on_page"], + "date_range" => "all", + "filters" => [["is", "event:page", ["/"]]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:time_on_page], + utc_time_range: ^@date_range_day, + filters: [[:is, "event:page", ["/"]]], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "fails when using only a behavioral filter", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["time_on_page"], + "date_range" => "all", + "filters" => [ + ["has_done", ["is", "event:page", ["/"]]] + ] + } + + assert {:error, error} = Query.parse_and_build(site, :internal, params) + + assert error == + "Metric `time_on_page` can only be queried with event:page filters or dimensions." + end + end + + describe "revenue metrics" do + @describetag :ee_only + + setup %{user: user} do + subscribe_to_enterprise_plan(user, features: [Plausible.Billing.Feature.RevenueGoals]) + :ok + end + + test "can request", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["total_revenue", "average_revenue"], + "date_range" => "all" + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:total_revenue, :average_revenue], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + + check_goals(query, + preloaded_goals: %{ + all: [], + matching_toplevel_filters: [] + }, + revenue_warning: :no_revenue_goals_matching, + revenue_currencies: %{} + ) + end + + test "no access" do + user = new_user() + site = new_site(owner: user) + + subscribe_to_enterprise_plan(user, features: [Plausible.Billing.Feature.StatsAPI]) + + params = %{ + "site_id" => site.domain, + "metrics" => ["total_revenue", "average_revenue"], + "date_range" => "all" + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "The owner of this site does not have access to the revenue metrics feature." + end + + test "with event:goal filters with same currency", %{site: site} do + insert(:goal, + site: site, + event_name: "Purchase", + currency: "USD", + display_name: "PurchaseUSD" + ) + + insert(:goal, site: site, event_name: "Subscription", currency: "USD") + insert(:goal, site: site, event_name: "Signup") + insert(:goal, site: site, event_name: "Logout") + + params = %{ + "site_id" => site.domain, + "metrics" => ["total_revenue", "average_revenue"], + "date_range" => "all", + "filters" => [["is", "event:goal", ["PurchaseUSD", "Signup", "Subscription"]]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:total_revenue, :average_revenue], + utc_time_range: ^@date_range_day, + filters: [[:is, "event:goal", ["PurchaseUSD", "Signup", "Subscription"]]], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + + check_goals(query, + preloaded_goals: %{ + all: ["PurchaseUSD", "Signup", "Subscription", "Logout"], + matching_toplevel_filters: ["PurchaseUSD", "Signup", "Subscription"] + }, + revenue_warning: nil, + revenue_currencies: %{default: :USD} + ) + end + + test "with event:goal filters with different currencies", %{site: site} do + insert(:goal, site: site, event_name: "Purchase", currency: "USD") + insert(:goal, site: site, event_name: "Subscription", currency: "EUR") + insert(:goal, site: site, event_name: "Signup") + + params = %{ + "site_id" => site.domain, + "metrics" => ["total_revenue", "average_revenue"], + "date_range" => "all", + "filters" => [["is", "event:goal", ["Purchase", "Signup", "Subscription"]]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:total_revenue, :average_revenue], + utc_time_range: ^@date_range_day, + filters: [[:is, "event:goal", ["Purchase", "Signup", "Subscription"]]], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + + check_goals(query, + preloaded_goals: %{ + all: ["Purchase", "Signup", "Subscription"], + matching_toplevel_filters: ["Purchase", "Signup", "Subscription"] + }, + revenue_warning: :no_single_revenue_currency, + revenue_currencies: %{} + ) + end + + test "with event:goal filters with no revenue currencies", %{site: site} do + insert(:goal, site: site, event_name: "Purchase", currency: "USD") + insert(:goal, site: site, event_name: "Subscription", currency: "EUR") + insert(:goal, site: site, event_name: "Signup") + + params = %{ + "site_id" => site.domain, + "metrics" => ["total_revenue", "average_revenue"], + "date_range" => "all", + "filters" => [["is", "event:goal", ["Signup"]]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:total_revenue, :average_revenue], + utc_time_range: ^@date_range_day, + filters: [[:is, "event:goal", ["Signup"]]], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + + check_goals(query, + preloaded_goals: %{ + all: ["Purchase", "Subscription", "Signup"], + matching_toplevel_filters: ["Signup"] + }, + revenue_warning: :no_revenue_goals_matching, + revenue_currencies: %{} + ) + end + + test "with event:goal dimension, different currencies", %{site: site} do + insert(:goal, site: site, event_name: "Purchase", currency: "USD") + insert(:goal, site: site, event_name: "Donation", currency: "EUR") + insert(:goal, site: site, event_name: "Signup") + + params = %{ + "site_id" => site.domain, + "metrics" => ["total_revenue", "average_revenue"], + "date_range" => "all", + "dimensions" => ["event:goal"] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:total_revenue, :average_revenue], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: ["event:goal"], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + + check_goals(query, + preloaded_goals: %{ + all: ["Donation", "Purchase", "Signup"], + matching_toplevel_filters: ["Donation", "Purchase", "Signup"] + }, + revenue_warning: nil, + revenue_currencies: %{"Donation" => :EUR, "Purchase" => :USD} + ) + end + + test "with event:goal dimension and filters", %{site: site} do + insert(:goal, site: site, event_name: "Purchase", currency: "USD") + insert(:goal, site: site, event_name: "Subscription", currency: "EUR") + insert(:goal, site: site, event_name: "Signup") + insert(:goal, site: site, event_name: "Logout") + + params = %{ + "site_id" => site.domain, + "metrics" => ["total_revenue", "average_revenue"], + "date_range" => "all", + "dimensions" => ["event:goal"], + "filters" => [["is", "event:goal", ["Purchase", "Signup", "Subscription"]]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:total_revenue, :average_revenue], + utc_time_range: ^@date_range_day, + filters: [[:is, "event:goal", ["Purchase", "Signup", "Subscription"]]], + dimensions: ["event:goal"], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + + check_goals(query, + preloaded_goals: %{ + all: ["Logout", "Purchase", "Signup", "Subscription"], + matching_toplevel_filters: ["Purchase", "Signup", "Subscription"] + }, + revenue_warning: nil, + revenue_currencies: %{"Purchase" => :USD, "Subscription" => :EUR} + ) + end + + test "with event:goal dimension and filters with no revenue goals matching", %{ + site: site + } do + insert(:goal, site: site, event_name: "Purchase", currency: "USD") + insert(:goal, site: site, event_name: "Subscription", currency: "USD") + insert(:goal, site: site, event_name: "Signup") + insert(:goal, site: site, event_name: "Logout") + + params = %{ + "site_id" => site.domain, + "metrics" => ["total_revenue", "average_revenue"], + "date_range" => "all", + "dimensions" => ["event:goal"], + "filters" => [["is", "event:goal", ["Signup"]]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:total_revenue, :average_revenue], + utc_time_range: ^@date_range_day, + filters: [[:is, "event:goal", ["Signup"]]], + dimensions: ["event:goal"], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + + check_goals(query, + preloaded_goals: %{ + all: ["Logout", "Signup", "Subscription", "Purchase"], + matching_toplevel_filters: ["Signup"] + }, + revenue_warning: :no_revenue_goals_matching, + revenue_currencies: %{} + ) + end + end + + @tag :ce_build_only + test "revenue metrics are not available on CE", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["total_revenue", "average_revenue"], + "date_range" => "all" + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "#/metrics/0: Invalid metric \"total_revenue\"\n#/metrics/1: Invalid metric \"average_revenue\"" + end + + describe "session metrics" do + test "single session metric succeeds", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["bounce_rate"], + "date_range" => "all", + "dimensions" => ["visit:device"] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:bounce_rate], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: ["visit:device"], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "fails if using session metric with event dimension", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["bounce_rate"], + "date_range" => "all", + "dimensions" => ["event:props:foo"] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "Session metric(s) `bounce_rate` cannot be queried along with event dimension(s) `event:props:foo`" + end + + test "fails if using event metric with session-only dimension", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["events"], + "date_range" => "all", + "dimensions" => ["visit:exit_page"] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "Event metric(s) `events` cannot be queried along with session dimension(s) `visit:exit_page`" + end + + test "does not fail if using session metric with event:page dimension", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["bounce_rate"], + "date_range" => "all", + "dimensions" => ["event:page"] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:bounce_rate], + utc_time_range: ^@date_range_day, + filters: [], + dimensions: ["event:page"], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "does not fail if using session metric with event filter", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["bounce_rate"], + "date_range" => "all", + "filters" => [["is", "event:props:foo", ["(none)"]]] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:bounce_rate], + utc_time_range: ^@date_range_day, + filters: [[:is, "event:props:foo", ["(none)"]]], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + end + + describe "filtering with segments" do + test "parsing fails when too many segments in query", %{ + user: user, + site: site + } do + segments = + insert_list(11, :segment, + type: :site, + owner: user, + site: site, + name: "any" + ) + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["and", segments |> Enum.map(fn segment -> ["is", "segment", [segment.id]] end)] + ] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == "Invalid filters. You can only use up to 10 segment filters in a query." + end + + test "parsing fails when segment filter is used, but segment is from another site", %{ + site: site + } do + other_user = new_user() + other_site = new_site(owner: other_user) + + segment = + insert(:segment, + type: :site, + owner: other_user, + site: other_site, + name: "any" + ) + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "segment", [segment.id]]] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == "Invalid filters. Some segments don't exist or aren't accessible." + end + + test "hiding custom properties filters in segments doesn't allow bypasssing feature check", + %{ + site: site, + user: user + } do + subscribe_to_enterprise_plan(user, features: [Plausible.Billing.Feature.StatsAPI]) + + segment = + insert(:segment, + type: :site, + owner: user, + site: site, + name: "segment with custom props filter", + segment_data: %{"filters" => [["is", "event:props:foobar", ["foo"]]]} + ) + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "segment", [segment.id]]] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "The owner of this site does not have access to the custom properties feature." + end + + test "querying conversion rate is illegal if the complex event:goal filter is within a segment", + %{ + site: site, + user: user + } do + segment = + insert(:segment, + type: :site, + owner: user, + site: site, + name: "any", + segment_data: %{ + "filters" => [ + [ + "or", + [ + ["is", "event:goal", ["Signup"]], + ["contains", "event:page", ["/"]] + ] + ] + ] + } + ) + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors", "conversion_rate"], + "date_range" => "all", + "filters" => [["is", "segment", [segment.id]]] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + + assert error == + "Invalid filters. Dimension `event:goal` can only be filtered at the top level." + end + + test "resolves segments correctly", %{site: site, user: user} do + emea_segment = + insert(:segment, + type: :site, + owner: user, + site: site, + name: "EMEA", + segment_data: %{ + "filters" => [["is", "visit:country", ["FR", "DE"]]], + "labels" => %{"FR" => "France", "DE" => "Germany"} + } + ) + + apac_segment = + insert(:segment, + type: :site, + owner: user, + site: site, + name: "APAC", + segment_data: %{ + "filters" => [["is", "visit:country", ["AU", "NZ"]]], + "labels" => %{"AU" => "Australia", "NZ" => "New Zealand"} + } + ) + + firefox_segment = + insert(:segment, + type: :site, + owner: user, + site: site, + name: "APAC", + segment_data: %{ + "filters" => [ + ["is", "visit:browser", ["Firefox"]], + ["is", "visit:os", ["Linux"]] + ] + } + ) + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + "date_range" => "all", + "filters" => [ + [ + "and", + [ + ["is", "segment", [apac_segment.id, emea_segment.id]], + ["is", "segment", [firefox_segment.id]] + ] + ] + ] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors, :events], + utc_time_range: ^@date_range_day, + filters: [ + [ + :or, + [ + [:and, [[:is, "visit:country", ["AU", "NZ"]]]], + [:and, [[:is, "visit:country", ["FR", "DE"]]]] + ] + ], + [ + :and, + [ + [:is, "visit:browser", ["Firefox"]], + [:is, "visit:os", ["Linux"]] + ] + ] + ], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "resolves segments containing otherwise internal features", %{site: site, user: user} do + insert(:goal, %{site: site, event_name: "Signup"}) + + segment_from_dashboard = + insert(:segment, + name: "A segment that contains :internal features", + type: :site, + owner: user, + site: site, + segment_data: %{ + "filters" => [["has_not_done", ["is", "event:goal", ["Signup"]]]] + } + ) + + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + "date_range" => "all", + "filters" => [ + ["is", "segment", [segment_from_dashboard.id]] + ] + } + + assert {:ok, query} = Query.parse_and_build(site, :public, params) + + assert_matches %Query{ + metrics: [:visitors, :events], + utc_time_range: ^@date_range_day, + filters: [ + [:has_not_done, [:is, "event:goal", ["Signup"]]] + ], + dimensions: [], + order_by: nil, + timezone: ^site.timezone, + include: ^@default_include, + pagination: %{limit: 10_000, offset: 0} + } = query + end + + test "validation fails with string segment ids", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "segment", ["123"]]] + } + + assert {:error, error} = Query.parse_and_build(site, :public, params) + assert error == "Invalid filter '[\"is\", \"segment\", [\"123\"]]'." + end + end + + on_ee do + describe "query.consolidated_site_ids" do + test "is set to nil when site is regular", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all" + } + + {:ok, %{consolidated_site_ids: nil}} = Query.parse_and_build(site, :public, params) + {:ok, %{consolidated_site_ids: nil}} = Query.parse_and_build(site, :internal, params) + end + + test "is set to a list of site_ids when site is consolidated", %{site: site} do + new_site(team: site.team) + cv = new_consolidated_view(site.team) + + params = %{ + "site_id" => cv.domain, + "metrics" => ["visitors"], + "date_range" => "all" + } + + assert {:ok, %{consolidated_site_ids: site_ids}} = + Query.parse_and_build(cv, :public, params) + + assert length(site_ids) == 2 + assert site.id in site_ids + + assert {:ok, %{consolidated_site_ids: site_ids}} = + Query.parse_and_build(cv, :internal, params) + + assert length(site_ids) == 2 + assert site.id in site_ids + end + end + end +end diff --git a/test/plausible/stats/query/query_parser_test.exs b/test/plausible/stats/query/query_parser_test.exs new file mode 100644 index 000000000000..34a21aebc76d --- /dev/null +++ b/test/plausible/stats/query/query_parser_test.exs @@ -0,0 +1,22 @@ +defmodule Plausible.Stats.Query.QueryParserTest do + use Plausible.DataCase + import Plausible.Stats.QueryParser + + setup [:create_user, :create_site] + + test "parsing empty map fails", %{site: site} do + assert {:error, "#: Required properties site_id, metrics, date_range were not present."} = + parse(site, :public, %{}) + end + + test "invalid metric passed", %{site: site} do + params = %{ + "site_id" => site.domain, + "metrics" => ["visitors", "event:name"], + "date_range" => "all" + } + + assert {:error, "#/metrics/1: Invalid metric \"event:name\""} = + parse(site, :public, params) + end +end diff --git a/test/plausible/stats/query_result_test.exs b/test/plausible/stats/query/query_result_test.exs similarity index 94% rename from test/plausible/stats/query_result_test.exs rename to test/plausible/stats/query/query_result_test.exs index 485adfc9fa64..a11eceaf7b93 100644 --- a/test/plausible/stats/query_result_test.exs +++ b/test/plausible/stats/query/query_result_test.exs @@ -1,4 +1,4 @@ -defmodule Plausible.Stats.QueryResultTest do +defmodule Plausible.Stats.Query.QueryResultTest do use Plausible.DataCase, async: true use Plausible.Teams.Test alias Plausible.Stats.{Query, QueryRunner, QueryResult, QueryOptimizer} @@ -18,7 +18,7 @@ defmodule Plausible.Stats.QueryResultTest do test "query!/3 raises on error on site_id mismatch", %{site: site} do assert_raise FunctionClauseError, fn -> - Query.build!( + Query.parse_and_build!( site, :public, %{ @@ -32,7 +32,7 @@ defmodule Plausible.Stats.QueryResultTest do assert_raise RuntimeError, ~s/Failed to build query: "#: Required properties metrics, date_range were not present."/, fn -> - Query.build!( + Query.parse_and_build!( site, :public, %{ @@ -44,7 +44,7 @@ defmodule Plausible.Stats.QueryResultTest do test "serializing query to JSON keeps keys ordered", %{site: site} do query = - Query.build!( + Query.parse_and_build!( site, :public, %{ diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exs deleted file mode 100644 index ab4eb5b4d809..000000000000 --- a/test/plausible/stats/query_parser_test.exs +++ /dev/null @@ -1,2740 +0,0 @@ -defmodule Plausible.Stats.Filters.QueryParserTest do - use Plausible - use Plausible.DataCase - use Plausible.Teams.Test - import Plausible.Stats.Filters.QueryParser - doctest Plausible.Stats.Filters.QueryParser - - alias Plausible.Stats.DateTimeRange - alias Plausible.Stats.Filters - - setup [:create_user, :create_site] - - @now DateTime.new!(~D[2021-05-05], ~T[12:30:00], "Etc/UTC") - @date_range_realtime %DateTimeRange{ - first: DateTime.new!(~D[2021-05-05], ~T[12:25:00], "Etc/UTC"), - last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "Etc/UTC") - } - @date_range_30m %DateTimeRange{ - first: DateTime.new!(~D[2021-05-05], ~T[12:00:00], "Etc/UTC"), - last: DateTime.new!(~D[2021-05-05], ~T[12:30:05], "Etc/UTC") - } - @date_range_day %DateTimeRange{ - first: DateTime.new!(~D[2021-05-05], ~T[00:00:00], "Etc/UTC"), - last: DateTime.new!(~D[2021-05-05], ~T[23:59:59], "Etc/UTC") - } - @date_range_7d %DateTimeRange{ - first: DateTime.new!(~D[2021-04-28], ~T[00:00:00], "Etc/UTC"), - last: DateTime.new!(~D[2021-05-04], ~T[23:59:59], "Etc/UTC") - } - @date_range_10d %DateTimeRange{ - first: DateTime.new!(~D[2021-04-25], ~T[00:00:00], "Etc/UTC"), - last: DateTime.new!(~D[2021-05-04], ~T[23:59:59], "Etc/UTC") - } - @date_range_30d %DateTimeRange{ - first: DateTime.new!(~D[2021-04-05], ~T[00:00:00], "Etc/UTC"), - last: DateTime.new!(~D[2021-05-04], ~T[23:59:59], "Etc/UTC") - } - @date_range_month %DateTimeRange{ - first: DateTime.new!(~D[2021-05-01], ~T[00:00:00], "Etc/UTC"), - last: DateTime.new!(~D[2021-05-31], ~T[23:59:59], "Etc/UTC") - } - @date_range_3mo %DateTimeRange{ - first: DateTime.new!(~D[2021-02-01], ~T[00:00:00], "Etc/UTC"), - last: DateTime.new!(~D[2021-04-30], ~T[23:59:59], "Etc/UTC") - } - @date_range_6mo %DateTimeRange{ - first: DateTime.new!(~D[2020-11-01], ~T[00:00:00], "Etc/UTC"), - last: DateTime.new!(~D[2021-04-30], ~T[23:59:59], "Etc/UTC") - } - @date_range_year %DateTimeRange{ - first: DateTime.new!(~D[2021-01-01], ~T[00:00:00], "Etc/UTC"), - last: DateTime.new!(~D[2021-12-31], ~T[23:59:59], "Etc/UTC") - } - @date_range_12mo %DateTimeRange{ - first: DateTime.new!(~D[2020-05-01], ~T[00:00:00], "Etc/UTC"), - last: DateTime.new!(~D[2021-04-30], ~T[23:59:59], "Etc/UTC") - } - - @default_include %{ - imports: false, - imports_meta: false, - time_labels: false, - total_rows: false, - comparisons: nil, - legacy_time_on_page_cutoff: nil, - trim_relative_date_range: false - } - - def check_success(params, site, expected_result, schema_type \\ :public) do - assert {:ok, result} = parse(site, schema_type, params, @now) - - return_value = Map.take(result, [:preloaded_goals, :revenue_warning, :revenue_currencies]) - - result = - Map.drop(result, [ - :now, - :input_date_range, - :preloaded_goals, - :revenue_warning, - :revenue_currencies, - :consolidated_site_ids - ]) - - assert result == expected_result - - return_value - end - - def check_error(params, site, expected_error_message, schema_type \\ :public) do - {:error, message} = parse(site, schema_type, params, @now) - assert message == expected_error_message - end - - def check_date_range(date_params, site, expected_date_range, schema_type \\ :public) do - params = - %{"site_id" => site.domain, "metrics" => ["visitors", "events"]} - |> Map.merge(date_params) - - expected_parsed = - %{ - metrics: [:visitors, :events], - utc_time_range: expected_date_range, - filters: [], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - } - - check_success(params, site, expected_parsed, schema_type) - end - - def check_goals(actual, opts) do - assert goal_names(actual[:preloaded_goals][:all]) == - Enum.sort(Keyword.get(opts, :preloaded_goals)[:all]) - - assert goal_names(actual[:preloaded_goals][:matching_toplevel_filters]) == - Enum.sort(Keyword.get(opts, :preloaded_goals)[:matching_toplevel_filters]) - - assert actual[:revenue_warning] == Keyword.get(opts, :revenue_warning) - assert actual[:revenue_currencies] == Keyword.get(opts, :revenue_currencies) - end - - defp goal_names(goals), do: Enum.map(goals, & &1.display_name) |> Enum.sort() - - test "parsing empty map fails", %{site: site} do - %{} - |> check_error(site, "#: Required properties site_id, metrics, date_range were not present.") - end - - describe "metrics validation" do - test "valid metrics passed", %{site: site} do - %{"site_id" => site.domain, "metrics" => ["visitors", "events"], "date_range" => "all"} - |> check_success(site, %{ - metrics: [:visitors, :events], - utc_time_range: @date_range_day, - filters: [], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - - test "invalid metric passed", %{site: site} do - %{"site_id" => site.domain, "metrics" => ["visitors", "event:name"], "date_range" => "all"} - |> check_error(site, "#/metrics/1: Invalid metric \"event:name\"") - end - - test "fuller list of metrics", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => [ - "visitors", - "pageviews", - "visits", - "events", - "bounce_rate", - "visit_duration" - ], - "date_range" => "all" - } - |> check_success( - site, - %{ - metrics: [ - :visitors, - :pageviews, - :visits, - :events, - :bounce_rate, - :visit_duration - ], - utc_time_range: @date_range_day, - filters: [], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }, - :internal - ) - end - - test "same metric queried multiple times", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["events", "visitors", "visitors"], - "date_range" => "all" - } - |> check_error(site, "#/metrics: Expected items to be unique but they were not.") - end - - test "no metrics passed", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => [], - "date_range" => "all" - } - |> check_error(site, "#/metrics: Expected a minimum of 1 items but got 0.") - end - end - - describe "filters validation" do - for operation <- [ - :is, - :is_not, - :matches_wildcard, - :matches_wildcard_not, - :matches, - :matches_not, - :contains, - :contains_not - ] do - test "#{operation} filter", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - [Atom.to_string(unquote(operation)), "event:name", ["foo"]] - ] - } - |> check_success( - site, - %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [ - [unquote(operation), "event:name", ["foo"]] - ], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }, - :internal - ) - end - - test "#{operation} filter with invalid clause", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - [Atom.to_string(unquote(operation)), "event:name", "foo"] - ] - } - |> check_error( - site, - "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:name\", \"foo\"]", - :internal - ) - end - end - - for operation <- [:matches_wildcard, :matches_wildcard_not] do - test "#{operation} is not a valid filter operation in public API", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - [Atom.to_string(unquote(operation)), "event:name", ["foo"]] - ] - } - |> check_error( - site, - "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:name\", [\"foo\"]]" - ) - end - end - - for too_short_filter <- [ - [], - ["and"], - ["or"], - ["and", []], - ["or", []], - ["not"], - ["is_not"], - ["is_not", "event:name"], - ["has_done"], - ["has_not_done"] - ] do - test "errors on too short filter #{inspect(too_short_filter)}", %{ - site: site - } do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - unquote(too_short_filter) - ] - } - |> check_error( - site, - ~s(#/filters/0: Invalid filter #{inspect(unquote(too_short_filter))}) - ) - end - end - - valid_filter = ["is", "event:props:foobar", ["value"]] - - for too_long_filter <- [ - ["and", [valid_filter], "extra"], - ["or", [valid_filter], []], - ["not", valid_filter, 1], - Enum.concat(valid_filter, [true]) - ] do - test "errors on too long filter #{inspect(too_long_filter)}", %{ - site: site - } do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - unquote(too_long_filter) - ] - } - |> check_error( - site, - ~s(#/filters/0: Invalid filter #{inspect(unquote(too_long_filter))}) - ) - end - end - - test "filtering by invalid operation", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - ["exists?", "event:name", ["foo"]] - ] - } - |> check_error(site, "#/filters/0: Invalid filter [\"exists?\", \"event:name\", [\"foo\"]]") - end - - test "filtering by custom properties", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - ["is", "event:props:foobar", ["value"]] - ] - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [ - [:is, "event:props:foobar", ["value"]] - ], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - - for dimension <- Filters.event_props() do - if dimension != "goal" do - test "filtering by event:#{dimension} filter", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - ["is", "event:#{unquote(dimension)}", ["foo"]] - ] - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [ - [:is, "event:#{unquote(dimension)}", ["foo"]] - ], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - end - end - - for dimension <- Filters.visit_props() do - test "filtering by visit:#{dimension} filter", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - ["is", "visit:#{unquote(dimension)}", ["ab"]] - ] - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [ - [:is, "visit:#{unquote(dimension)}", ["ab"]] - ], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - end - - test "invalid event filter", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - ["is", "event:device", ["foo"]] - ] - } - |> check_error(site, "#/filters/0: Invalid filter [\"is\", \"event:device\", [\"foo\"]]") - end - - test "invalid visit filter", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - ["is", "visit:name", ["foo"]] - ] - } - |> check_error(site, "#/filters/0: Invalid filter [\"is\", \"visit:name\", [\"foo\"]]") - end - - test "invalid filter", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => "foobar" - } - |> check_error(site, "#/filters: Type mismatch. Expected Array but got String.") - end - - test "numeric filter is invalid", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [["is", "visit:os_version", [123]]] - } - |> check_error(site, "Invalid filter '[\"is\", \"visit:os_version\", [123]]'.") - end - - test "numbers and strings are valid for visit:city", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [["is", "visit:city", [123, 456]]] - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [ - [:is, "visit:city", [123, 456]] - ], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [["is", "visit:city", ["123", "456"]]] - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [ - [:is, "visit:city", ["123", "456"]] - ], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - - test "invalid visit:country filter", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [["is", "visit:country", ["USA"]]] - } - |> check_error( - site, - "Invalid visit:country filter, visit:country needs to be a valid 2-letter country code." - ) - end - - test "valid nested `not`, `and` and `or`", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - [ - "or", - [ - [ - "and", - [ - ["is", "visit:city_name", ["Tallinn"]], - ["is", "visit:country_name", ["Estonia"]] - ] - ], - ["not", ["is", "visit:country_name", ["Estonia"]]] - ] - ] - ] - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [ - [ - :or, - [ - [ - :and, - [ - [:is, "visit:city_name", ["Tallinn"]], - [:is, "visit:country_name", ["Estonia"]] - ] - ], - [:not, [:is, "visit:country_name", ["Estonia"]]] - ] - ] - ], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - - test "valid has_done and has_not_done filters", %{site: site} do - insert(:goal, %{site: site, event_name: "Signup"}) - - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - ["has_done", ["is", "event:name", ["Signup"]]], - [ - "has_not_done", - [ - "or", - [ - ["is", "event:goal", ["Signup"]], - ["is", "event:page", ["/signup"]] - ] - ] - ] - ] - } - |> check_success( - site, - %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [ - [:has_done, [:is, "event:name", ["Signup"]]], - [ - :has_not_done, - [:or, [[:is, "event:goal", ["Signup"]], [:is, "event:page", ["/signup"]]]] - ] - ], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - } - ) - end - - test "fails when using visit filters within has_done filters", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - ["has_done", ["is", "visit:browser", ["Chrome"]]] - ] - } - |> check_error( - site, - "Invalid filters. Behavioral filters (has_done, has_not_done) can only be used with event dimension filters." - ) - end - - test "fails when nesting behavioral filters", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - ["has_done", ["has_not_done", ["is", "visit:browser", ["Chrome"]]]] - ] - } - |> check_error( - site, - "Invalid filters. Behavioral filters (has_done, has_not_done) cannot be nested." - ) - end - - for operator <- ["not", "or", "has_done", "has_not_done"] do - test "invalid `#{operator}` clause", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [[unquote(operator), []]] - } - |> check_error( - site, - "#/filters/0: Invalid filter [\"#{unquote(operator)}\", []]", - :internal - ) - end - end - - test "event:hostname filter", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [["is", "event:hostname", ["a.plausible.io"]]] - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [ - [:is, "event:hostname", ["a.plausible.io"]] - ], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - - test "event:hostname filter not at top level is invalid", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [["not", ["is", "event:hostname", ["a.plausible.io"]]]] - } - |> check_error( - site, - "Invalid filters. Dimension `event:hostname` can only be filtered at the top level." - ) - end - - for operation <- [:is, :contains, :is_not, :contains_not] do - test "#{operation} allows case_sensitive modifier", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - [ - Atom.to_string(unquote(operation)), - "event:page", - ["/foo"], - %{"case_sensitive" => false} - ], - [ - Atom.to_string(unquote(operation)), - "event:name", - ["/foo"], - %{"case_sensitive" => true} - ] - ] - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [ - [unquote(operation), "event:page", ["/foo"], %{case_sensitive: false}], - [unquote(operation), "event:name", ["/foo"], %{case_sensitive: true}] - ], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - end - - for operation <- [:matches, :matches_not, :matches_wildcard, :matches_wildcard_not] do - test "case_sensitive modifier is not valid for #{operation}", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - [ - Atom.to_string(unquote(operation)), - "event:hostname", - ["a.plausible.io"], - %{"case_sensitive" => false} - ] - ] - } - |> check_error( - site, - "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:hostname\", [\"a.plausible.io\"], %{\"case_sensitive\" => false}]", - :internal - ) - end - end - end - - describe "preloading goals" do - setup %{site: site} do - insert(:goal, %{site: site, event_name: "Signup"}) - insert(:goal, %{site: site, event_name: "Purchase"}) - insert(:goal, %{site: site, event_name: "Contact"}) - - :ok - end - - test "with exact match", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [["is", "event:goal", ["Signup", "Purchase"]]] - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [[:is, "event:goal", ["Signup", "Purchase"]]], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - |> check_goals( - preloaded_goals: %{ - all: ["Contact", "Purchase", "Signup"], - matching_toplevel_filters: ["Purchase", "Signup"] - }, - revenue_currencies: %{} - ) - end - - test "with case insensitive match", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [["is", "event:goal", ["signup", "purchase"], %{"case_sensitive" => false}]] - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [[:is, "event:goal", ["signup", "purchase"], %{case_sensitive: false}]], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - |> check_goals( - preloaded_goals: %{ - all: ["Contact", "Purchase", "Signup"], - matching_toplevel_filters: ["Purchase", "Signup"] - }, - revenue_currencies: %{} - ) - end - - test "with contains match", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [["contains", "event:goal", ["Sign", "pur"]]] - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [[:contains, "event:goal", ["Sign", "pur"]]], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - |> check_goals( - preloaded_goals: %{ - all: ["Contact", "Purchase", "Signup"], - matching_toplevel_filters: ["Signup"] - }, - revenue_currencies: %{} - ) - end - - test "with case insensitive contains match", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [["contains", "event:goal", ["sign", "CONT"], %{"case_sensitive" => false}]] - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [[:contains, "event:goal", ["sign", "CONT"], %{case_sensitive: false}]], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - |> check_goals( - preloaded_goals: %{ - all: ["Contact", "Purchase", "Signup"], - matching_toplevel_filters: ["Contact", "Signup"] - }, - revenue_currencies: %{} - ) - end - end - - describe "include validation" do - test "setting include values", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["time"], - "include" => %{"imports" => true, "time_labels" => true, "total_rows" => true} - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [], - dimensions: ["time"], - order_by: nil, - timezone: site.timezone, - include: %{ - imports: true, - imports_meta: false, - time_labels: true, - total_rows: true, - comparisons: nil, - legacy_time_on_page_cutoff: nil, - trim_relative_date_range: false - }, - pagination: %{limit: 10_000, offset: 0} - }) - end - - test "setting invalid imports value", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "include" => "foobar" - } - |> check_error(site, "#/include: Type mismatch. Expected Object but got String.") - end - - test "setting include.time_labels without time dimension", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "include" => %{"time_labels" => true} - } - |> check_error(site, "Invalid include.time_labels: requires a time dimension.") - end - end - - describe "include.comparisons" do - test "not allowed in public API", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "include" => %{"comparisons" => %{"mode" => "previous_period"}} - } - |> check_error( - site, - "#/include/comparisons: Schema does not allow additional properties." - ) - end - - test "mode=previous_period", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "include" => %{"comparisons" => %{"mode" => "previous_period"}} - } - |> check_success( - site, - %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: %{ - comparisons: %{ - mode: "previous_period" - }, - imports: false, - imports_meta: false, - time_labels: false, - total_rows: false, - legacy_time_on_page_cutoff: nil, - trim_relative_date_range: false - }, - pagination: %{limit: 10_000, offset: 0} - }, - :internal - ) - end - - test "mode=year_over_year", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "include" => %{"comparisons" => %{"mode" => "year_over_year"}} - } - |> check_success( - site, - %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: %{ - comparisons: %{ - mode: "year_over_year" - }, - imports: false, - imports_meta: false, - time_labels: false, - total_rows: false, - legacy_time_on_page_cutoff: nil, - trim_relative_date_range: false - }, - pagination: %{limit: 10_000, offset: 0} - }, - :internal - ) - end - - test "mode=custom", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "include" => %{ - "comparisons" => %{"mode" => "custom", "date_range" => ["2021-04-05", "2021-05-04"]} - } - } - |> check_success( - site, - %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: %{ - comparisons: %{ - mode: "custom", - date_range: @date_range_30d - }, - imports_meta: false, - imports: false, - time_labels: false, - total_rows: false, - legacy_time_on_page_cutoff: nil, - trim_relative_date_range: false - }, - pagination: %{limit: 10_000, offset: 0} - }, - :internal - ) - end - - test "mode=custom without date_range is invalid", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "include" => %{"comparisons" => %{"mode" => "custom"}} - } - |> check_error( - site, - "#/include/comparisons: Expected exactly one of the schemata to match, but none of them did.", - :internal - ) - end - - test "mode=previous_period with date_range is invalid", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "include" => %{ - "comparisons" => %{ - "mode" => "previous_period", - "date_range" => ["2024-01-01", "2024-01-31"] - } - } - } - |> check_error( - site, - "#/include/comparisons: Expected exactly one of the schemata to match, but none of them did.", - :internal - ) - end - end - - describe "pagination validation" do - test "setting pagination values", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["time"], - "pagination" => %{"limit" => 100, "offset" => 200} - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [], - dimensions: ["time"], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 100, offset: 200} - }) - end - - test "out of range limit value", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "pagination" => %{"limit" => 100_000} - } - |> check_error(site, "#/pagination/limit: Expected the value to be <= 10000") - end - - test "out of range offset value", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "pagination" => %{"offset" => -5} - } - |> check_error(site, "#/pagination/offset: Expected the value to be >= 0") - end - end - - describe "event:goal filter validation" do - test "valid filters", %{site: site} do - insert(:goal, %{site: site, event_name: "Signup"}) - insert(:goal, %{site: site, page_path: "/thank-you"}) - - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - ["is", "event:goal", ["Signup", "Visit /thank-you"]] - ] - } - |> check_success( - site, - %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [ - [:is, "event:goal", ["Signup", "Visit /thank-you"]] - ], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - } - ) - |> check_goals( - preloaded_goals: %{ - all: ["Signup", "Visit /thank-you"], - matching_toplevel_filters: ["Signup", "Visit /thank-you"] - }, - revenue_warning: nil, - revenue_currencies: %{} - ) - end - - test "invalid event filter", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - ["is", "event:goal", ["Signup"]] - ] - } - |> check_error( - site, - "Invalid filters. The goal `Signup` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals" - ) - end - - test "invalid page filter", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - ["is", "event:goal", ["Visit /thank-you"]] - ] - } - |> check_error( - site, - "Invalid filters. The goal `Visit /thank-you` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals" - ) - end - - test "unsupported filter", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - ["is_not", "event:goal", ["Signup"]] - ] - } - |> check_error( - site, - "#/filters/0: Invalid filter [\"is_not\", \"event:goal\", [\"Signup\"]]" - ) - end - - test "not top-level filter", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - [ - "or", - [ - ["is", "event:goal", ["Signup"]], - ["is", "event:name", ["pageview"]] - ] - ] - ] - } - |> check_error( - site, - "Invalid filters. Dimension `event:goal` can only be filtered at the top level." - ) - end - - test "allowed within behavioral filters has_done", %{site: site} do - insert(:goal, %{site: site, event_name: "Signup"}) - - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - [ - "has_done", - [ - "or", - [ - ["is", "event:goal", ["Signup"]], - ["is", "event:name", ["pageview"]] - ] - ] - ] - ] - } - |> check_success( - site, - %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [ - [ - :has_done, - [ - :or, - [ - [:is, "event:goal", ["Signup"]], - [:is, "event:name", ["pageview"]] - ] - ] - ] - ], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - } - ) - |> check_goals( - preloaded_goals: %{all: ["Signup"], matching_toplevel_filters: ["Signup"]}, - revenue_warning: nil, - revenue_currencies: %{} - ) - end - - test "name is checked even within behavioral filters", %{site: site} do - insert(:goal, %{site: site, event_name: "Signup"}) - - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [["has_done", ["is", "event:goal", ["Unknown"]]]] - } - |> check_error( - site, - "Invalid filters. The goal `Unknown` is not configured for this site. Find out how to configure goals here: https://plausible.io/docs/stats-api#filtering-by-goals", - :internal - ) - end - end - - describe "date range validation" do - test "parsing shortcut options", %{site: site} do - check_date_range(%{"date_range" => "day"}, site, @date_range_day) - check_date_range(%{"date_range" => "7d"}, site, @date_range_7d) - check_date_range(%{"date_range" => "10d"}, site, @date_range_10d) - check_date_range(%{"date_range" => "30d"}, site, @date_range_30d) - check_date_range(%{"date_range" => "month"}, site, @date_range_month) - check_date_range(%{"date_range" => "3mo"}, site, @date_range_3mo) - check_date_range(%{"date_range" => "6mo"}, site, @date_range_6mo) - check_date_range(%{"date_range" => "12mo"}, site, @date_range_12mo) - check_date_range(%{"date_range" => "year"}, site, @date_range_year) - end - - test "30m and realtime are available in internal API", %{site: site} do - check_date_range(%{"date_range" => "30m"}, site, @date_range_30m, :internal) - - check_date_range( - %{"date_range" => "realtime"}, - site, - @date_range_realtime, - :internal - ) - end - - test "30m and realtime date_ranges are unavailable in public API", %{ - site: site - } do - for date_range <- ["realtime", "30m"] do - %{"site_id" => site.domain, "metrics" => ["visitors"], "date_range" => date_range} - |> check_error(site, "#/date_range: Invalid date range \"#{date_range}\"") - end - end - - test "parsing `all` with previous data", %{site: site} do - site = Map.put(site, :stats_start_date, ~D[2020-01-01]) - expected_date_range = DateTimeRange.new!(~D[2020-01-01], ~D[2021-05-05], "Etc/UTC") - check_date_range(%{"date_range" => "all"}, site, expected_date_range) - end - - test "parsing `all` with no previous data", %{site: site} do - site = Map.put(site, :stats_start_date, nil) - check_date_range(%{"date_range" => "all"}, site, @date_range_day) - end - - test "parsing custom date range from simple date strings", %{site: site} do - check_date_range(%{"date_range" => ["2021-05-05", "2021-05-05"]}, site, @date_range_day) - end - - test "parsing custom date range from iso8601 timestamps", %{site: site} do - check_date_range( - %{"date_range" => ["2024-01-01T00:00:00Z", "2024-01-02T23:59:59Z"]}, - site, - DateTimeRange.new!( - DateTime.new!(~D[2024-01-01], ~T[00:00:00], "Etc/UTC"), - DateTime.new!(~D[2024-01-02], ~T[23:59:59], "Etc/UTC") - ) - ) - - check_date_range( - %{ - "date_range" => [ - "2024-08-29T07:12:34-07:00", - "2024-08-29T10:12:34-07:00" - ] - }, - site, - DateTimeRange.new!( - ~U[2024-08-29 14:12:34Z], - ~U[2024-08-29 17:12:34Z] - ) - ) - end - - test "parsing invalid custom date range with invalid dates", %{site: site} do - %{"site_id" => site.domain, "date_range" => "-1d", "metrics" => ["visitors"]} - |> check_error(site, "#/date_range: Invalid date range \"-1d\"") - - %{"site_id" => site.domain, "date_range" => "foo", "metrics" => ["visitors"]} - |> check_error(site, "#/date_range: Invalid date range \"foo\"") - - %{"site_id" => site.domain, "date_range" => ["21415-00", "eee"], "metrics" => ["visitors"]} - |> check_error(site, "#/date_range: Invalid date range [\"21415-00\", \"eee\"]") - - %{"site_id" => site.domain, "date_range" => "999999999mo", "metrics" => ["visitors"]} - |> check_error(site, "Invalid date_range \"999999999mo\"") - end - - test "custom date range is invalid when timestamps do not include timezone info", %{ - site: site - } do - %{ - "site_id" => site.domain, - "date_range" => ["2021-02-03T00:00:00", "2021-02-03T23:59:59"], - "metrics" => ["visitors"] - } - |> check_error( - site, - "Invalid date_range '[\"2021-02-03T00:00:00\", \"2021-02-03T23:59:59\"]'." - ) - end - - test "custom date range is invalid when timestamp timezone is invalid", %{site: site} do - %{ - "site_id" => site.domain, - "date_range" => ["2021-02-03T00:00:00-25:00", "2021-02-03T23:59:59-25:00"], - "metrics" => ["visitors"] - } - |> check_error( - site, - "#/date_range: Invalid date range [\"2021-02-03T00:00:00-25:00\", \"2021-02-03T23:59:59-25:00\"]" - ) - end - - test "custom date range is invalid when date and timestamp are combined", %{site: site} do - %{ - "site_id" => site.domain, - "date_range" => ["2021-02-03T00:00:00Z", "2021-02-04"], - "metrics" => ["visitors"] - } - |> check_error( - site, - "#/date_range: Invalid date range [\"2021-02-03T00:00:00Z\", \"2021-02-04\"]" - ) - end - - test "parses date_range relative to date param", %{site: site} do - date = @now |> DateTime.to_date() |> Date.to_string() - - for {date_range_shortcut, expected_date_range} <- [ - {"day", @date_range_day}, - {"7d", @date_range_7d}, - {"10d", @date_range_10d}, - {"30d", @date_range_30d}, - {"month", @date_range_month}, - {"3mo", @date_range_3mo}, - {"6mo", @date_range_6mo}, - {"12mo", @date_range_12mo}, - {"year", @date_range_year} - ] do - %{"date_range" => date_range_shortcut, "date" => date} - |> check_date_range(site, expected_date_range, :internal) - end - end - - test "date parameter is not available in the public API", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors", "events"], - "date_range" => "month", - "date" => "2021-05-05" - } - |> check_error(site, "#/date: Schema does not allow additional properties.") - end - - test "parses date_range.first into a datetime right after the gap in site.timezone", %{ - site: site - } do - site = %{site | timezone: "America/Santiago"} - - %{"date_range" => ["2022-09-11", "2022-09-11"]} - |> check_date_range( - site, - DateTimeRange.new!(~U[2022-09-11 04:00:00Z], ~U[2022-09-12 02:59:59Z]) - ) - end - - test "parses date_range.first into the latest of ambiguous datetimes in site.timezone", %{ - site: site - } do - site = %{site | timezone: "America/Havana"} - - %{"date_range" => ["2023-11-05", "2023-11-05"]} - |> check_date_range( - site, - DateTimeRange.new!(~U[2023-11-05 05:00:00Z], ~U[2023-11-06 04:59:59Z]) - ) - end - - test "parses date_range.last into the earliest of ambiguous datetimes in site.timezone", %{ - site: site - } do - site = %{site | timezone: "America/Asuncion"} - - %{"date_range" => ["2024-03-23", "2024-03-23"]} - |> check_date_range( - site, - DateTimeRange.new!(~U[2024-03-23 03:00:00Z], ~U[2024-03-24 02:59:59Z]) - ) - end - end - - describe "dimensions validation" do - for dimension <- Filters.event_props() do - test "event:#{dimension} dimension", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["event:#{unquote(dimension)}"] - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [], - dimensions: ["event:#{unquote(dimension)}"], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - end - - for dimension <- Filters.visit_props() do - test "visit:#{dimension} dimension", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["visit:#{unquote(dimension)}"] - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [], - dimensions: ["visit:#{unquote(dimension)}"], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - end - - test "time:minute dimension fails public schema validation", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["time:minute"] - } - |> check_error(site, "#/dimensions/0: Invalid dimension \"time:minute\"") - end - - test "time:minute dimension passes internal schema validation", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["time:minute"] - } - |> check_success( - site, - %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [], - dimensions: ["time:minute"], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }, - :internal - ) - end - - test "custom properties dimension", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["event:props:foobar"] - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [], - dimensions: ["event:props:foobar"], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - - test "invalid custom property dimension", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["event:props:"] - } - |> check_error(site, "#/dimensions/0: Invalid dimension \"event:props:\"") - end - - test "invalid dimension name passed", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["visitors"] - } - |> check_error(site, "#/dimensions/0: Invalid dimension \"visitors\"") - end - - test "invalid dimension", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => "foobar" - } - |> check_error(site, "#/dimensions: Type mismatch. Expected Array but got String.") - end - - test "dimensions are not unique", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["event:name", "event:name"] - } - |> check_error(site, "#/dimensions: Expected items to be unique but they were not.") - end - end - - describe "order_by validation" do - test "ordering by metric", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors", "events"], - "date_range" => "all", - "order_by" => [["events", "desc"], ["visitors", "asc"]] - } - |> check_success(site, %{ - metrics: [:visitors, :events], - utc_time_range: @date_range_day, - filters: [], - dimensions: [], - order_by: [{:events, :desc}, {:visitors, :asc}], - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - - test "ordering by dimension", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["event:name"], - "order_by" => [["event:name", "desc"]] - } - |> check_success(site, %{ - metrics: [:visitors], - utc_time_range: @date_range_day, - filters: [], - dimensions: ["event:name"], - order_by: [{"event:name", :desc}], - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - - test "ordering by invalid value", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "order_by" => [["visssss", "desc"]] - } - |> check_error(site, "#/order_by/0/0: Invalid value in order_by \"visssss\"") - end - - test "ordering by not queried metric", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "order_by" => [["events", "desc"]] - } - |> check_error( - site, - "Invalid order_by entry '{:events, :desc}'. Entry is not a queried metric or dimension." - ) - end - - test "ordering by not queried dimension", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "order_by" => [["event:name", "desc"]] - } - |> check_error( - site, - "Invalid order_by entry '{\"event:name\", :desc}'. Entry is not a queried metric or dimension." - ) - end - end - - describe "custom props access" do - test "filters - no access", %{site: site, user: user} do - subscribe_to_enterprise_plan(user, features: [Plausible.Billing.Feature.StatsAPI]) - - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [["not", ["is", "event:props:foobar", ["foo"]]]] - } - |> check_error( - site, - "The owner of this site does not have access to the custom properties feature." - ) - end - - test "dimensions - no access", %{site: site, user: user} do - subscribe_to_enterprise_plan(user, features: [Plausible.Billing.Feature.StatsAPI]) - - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "dimensions" => ["event:props:foobar"] - } - |> check_error( - site, - "The owner of this site does not have access to the custom properties feature." - ) - end - end - - describe "conversion_rate metric" do - test "fails validation on its own", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["conversion_rate"], - "date_range" => "all" - } - |> check_error( - site, - "Metric `conversion_rate` can only be queried with event:goal filters or dimensions." - ) - end - - test "succeeds with event:goal filter", %{site: site} do - insert(:goal, %{site: site, event_name: "Signup"}) - insert(:goal, %{site: site, event_name: "Purchase", currency: "USD"}) - - %{ - "site_id" => site.domain, - "metrics" => ["conversion_rate"], - "date_range" => "all", - "filters" => [["is", "event:goal", ["Signup"]]] - } - |> check_success(site, %{ - metrics: [:conversion_rate], - utc_time_range: @date_range_day, - filters: [[:is, "event:goal", ["Signup"]]], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - |> check_goals( - preloaded_goals: %{ - all: ["Purchase", "Signup"], - matching_toplevel_filters: ["Signup"] - }, - revenue_currencies: %{} - ) - end - - test "succeeds with event:goal dimension", %{site: site} do - insert(:goal, %{site: site, event_name: "Purchase", currency: "USD"}) - insert(:goal, %{site: site, event_name: "Signup"}) - - %{ - "site_id" => site.domain, - "metrics" => ["conversion_rate"], - "date_range" => "all", - "dimensions" => ["event:goal"] - } - |> check_success(site, %{ - metrics: [:conversion_rate], - utc_time_range: @date_range_day, - filters: [], - dimensions: ["event:goal"], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - |> check_goals( - preloaded_goals: %{ - all: ["Purchase", "Signup"], - matching_toplevel_filters: ["Purchase", "Signup"] - }, - revenue_currencies: %{} - ) - end - - test "custom properties filter with special metric", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["conversion_rate", "group_conversion_rate"], - "date_range" => "all", - "filters" => [["is", "event:props:foo", ["bar"]]], - "dimensions" => ["event:goal"] - } - |> check_success(site, %{ - metrics: [:conversion_rate, :group_conversion_rate], - utc_time_range: @date_range_day, - filters: [ - [:is, "event:props:foo", ["bar"]] - ], - dimensions: ["event:goal"], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - - test "not top level custom properties filter with special metric is invalid", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["conversion_rate", "group_conversion_rate"], - "date_range" => "all", - "filters" => [["not", ["is", "event:props:foo", ["bar"]]]], - "dimensions" => ["event:goal"] - } - |> check_error( - site, - "Invalid filters. When `conversion_rate` or `group_conversion_rate` metrics are used, custom property filters can only be used on top level." - ) - end - end - - describe "exit_rate metric" do - test "fails validation without visit:exit_page dimension", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["exit_rate"], - "date_range" => "all" - } - |> check_error( - site, - "Metric `exit_rate` requires a `\"visit:exit_page\"` dimension. No other dimensions are allowed.", - :internal - ) - end - - test "fails validation with event only filters", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["exit_rate"], - "dimensions" => ["visit:exit_page"], - "filters" => [["is", "event:page", ["/"]]], - "date_range" => "all" - } - |> check_error( - site, - "Metric `exit_rate` cannot be queried when filtering on event dimensions.", - :internal - ) - end - - test "fails validation with event metrics", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["exit_rate", "pageviews"], - "dimensions" => ["visit:exit_page"], - "date_range" => "all" - } - |> check_error( - site, - "Event metric(s) `pageviews` cannot be queried along with session dimension(s) `visit:exit_page`", - :internal - ) - end - - test "passes validation", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["exit_rate"], - "dimensions" => ["visit:exit_page"], - "date_range" => "all" - } - |> check_success( - site, - %{ - metrics: [:exit_rate], - utc_time_range: @date_range_day, - filters: [], - dimensions: ["visit:exit_page"], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }, - :internal - ) - end - end - - describe "scroll_depth metric" do - test "fails validation on its own", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["scroll_depth"], - "date_range" => "all" - } - |> check_error( - site, - "Metric `scroll_depth` can only be queried with event:page filters or dimensions.", - :internal - ) - end - - test "fails with only a non-top-level event:page filter", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["scroll_depth"], - "date_range" => "all", - "filters" => [["not", ["is", "event:page", ["/"]]]] - } - |> check_error( - site, - "Metric `scroll_depth` can only be queried with event:page filters or dimensions.", - :internal - ) - end - - test "succeeds with top-level event:page filter", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["scroll_depth"], - "date_range" => "all", - "filters" => [["is", "event:page", ["/"]]] - } - |> check_success( - site, - %{ - metrics: [:scroll_depth], - utc_time_range: @date_range_day, - filters: [[:is, "event:page", ["/"]]], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }, - :internal - ) - end - - test "succeeds with event:page dimension", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["scroll_depth"], - "date_range" => "all", - "dimensions" => ["event:page"] - } - |> check_success( - site, - %{ - metrics: [:scroll_depth], - utc_time_range: @date_range_day, - filters: [], - dimensions: ["event:page"], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }, - :internal - ) - end - end - - describe "views_per_visit metric" do - test "succeeds with normal filters", %{site: site} do - insert(:goal, %{site: site, event_name: "Signup"}) - - %{ - "site_id" => site.domain, - "metrics" => ["views_per_visit"], - "date_range" => "all", - "filters" => [["is", "event:goal", ["Signup"]]] - } - |> check_success(site, %{ - metrics: [:views_per_visit], - utc_time_range: @date_range_day, - filters: [[:is, "event:goal", ["Signup"]]], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - |> check_goals( - preloaded_goals: %{all: ["Signup"], matching_toplevel_filters: ["Signup"]}, - revenue_currencies: %{} - ) - end - - test "fails validation if event:page filter specified", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["views_per_visit"], - "date_range" => "all", - "filters" => [["is", "event:page", ["/"]]] - } - |> check_error( - site, - "Metric `views_per_visit` cannot be queried with a filter on `event:page`." - ) - end - - test "fails validation with dimensions", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["views_per_visit"], - "date_range" => "all", - "dimensions" => ["event:name"] - } - |> check_error( - site, - "Metric `views_per_visit` cannot be queried with `dimensions`." - ) - end - end - - describe "time_on_page metric" do - test "fails validation on its own", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["time_on_page"], - "date_range" => "all" - } - |> check_error( - site, - "Metric `time_on_page` can only be queried with event:page filters or dimensions." - ) - end - - test "succeeds with event:page dimension", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["time_on_page"], - "date_range" => "all", - "dimensions" => ["time", "event:page"] - } - |> check_success(site, %{ - metrics: [:time_on_page], - utc_time_range: @date_range_day, - filters: [], - dimensions: ["time", "event:page"], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - - test "succeeds with event:page filter", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["time_on_page"], - "date_range" => "all", - "filters" => [["is", "event:page", ["/"]]] - } - |> check_success(site, %{ - metrics: [:time_on_page], - utc_time_range: @date_range_day, - filters: [[:is, "event:page", ["/"]]], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - - test "fails when using only a behavioral filter", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["time_on_page"], - "date_range" => "all", - "filters" => [ - ["has_done", ["is", "event:page", ["/"]]] - ] - } - |> check_error( - site, - "Metric `time_on_page` can only be queried with event:page filters or dimensions.", - :internal - ) - end - end - - describe "revenue metrics" do - @describetag :ee_only - - setup %{user: user} do - subscribe_to_enterprise_plan(user, features: [Plausible.Billing.Feature.RevenueGoals]) - :ok - end - - test "can request", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["total_revenue", "average_revenue"], - "date_range" => "all" - } - |> check_success( - site, - %{ - metrics: [:total_revenue, :average_revenue], - utc_time_range: @date_range_day, - filters: [], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - } - ) - |> check_goals( - preloaded_goals: %{ - all: [], - matching_toplevel_filters: [] - }, - revenue_warning: :no_revenue_goals_matching, - revenue_currencies: %{} - ) - end - - test "no access" do - user = new_user() - site = new_site(owner: user) - - subscribe_to_enterprise_plan(user, features: [Plausible.Billing.Feature.StatsAPI]) - - %{ - "site_id" => site.domain, - "metrics" => ["total_revenue", "average_revenue"], - "date_range" => "all" - } - |> check_error( - site, - "The owner of this site does not have access to the revenue metrics feature." - ) - end - - test "with event:goal filters with same currency", %{site: site} do - insert(:goal, - site: site, - event_name: "Purchase", - currency: "USD", - display_name: "PurchaseUSD" - ) - - insert(:goal, site: site, event_name: "Subscription", currency: "USD") - insert(:goal, site: site, event_name: "Signup") - insert(:goal, site: site, event_name: "Logout") - - %{ - "site_id" => site.domain, - "metrics" => ["total_revenue", "average_revenue"], - "date_range" => "all", - "filters" => [["is", "event:goal", ["PurchaseUSD", "Signup", "Subscription"]]] - } - |> check_success( - site, - %{ - metrics: [:total_revenue, :average_revenue], - utc_time_range: @date_range_day, - filters: [[:is, "event:goal", ["PurchaseUSD", "Signup", "Subscription"]]], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - } - ) - |> check_goals( - preloaded_goals: %{ - all: ["PurchaseUSD", "Signup", "Subscription", "Logout"], - matching_toplevel_filters: ["PurchaseUSD", "Signup", "Subscription"] - }, - revenue_warning: nil, - revenue_currencies: %{default: :USD} - ) - end - - test "with event:goal filters with different currencies", %{site: site} do - insert(:goal, site: site, event_name: "Purchase", currency: "USD") - insert(:goal, site: site, event_name: "Subscription", currency: "EUR") - insert(:goal, site: site, event_name: "Signup") - - %{ - "site_id" => site.domain, - "metrics" => ["total_revenue", "average_revenue"], - "date_range" => "all", - "filters" => [["is", "event:goal", ["Purchase", "Signup", "Subscription"]]] - } - |> check_success( - site, - %{ - metrics: [:total_revenue, :average_revenue], - utc_time_range: @date_range_day, - filters: [[:is, "event:goal", ["Purchase", "Signup", "Subscription"]]], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - } - ) - |> check_goals( - preloaded_goals: %{ - all: ["Purchase", "Signup", "Subscription"], - matching_toplevel_filters: ["Purchase", "Signup", "Subscription"] - }, - revenue_warning: :no_single_revenue_currency, - revenue_currencies: %{} - ) - end - - test "with event:goal filters with no revenue currencies", %{site: site} do - insert(:goal, site: site, event_name: "Purchase", currency: "USD") - insert(:goal, site: site, event_name: "Subscription", currency: "EUR") - insert(:goal, site: site, event_name: "Signup") - - %{ - "site_id" => site.domain, - "metrics" => ["total_revenue", "average_revenue"], - "date_range" => "all", - "filters" => [["is", "event:goal", ["Signup"]]] - } - |> check_success( - site, - %{ - metrics: [:total_revenue, :average_revenue], - utc_time_range: @date_range_day, - filters: [[:is, "event:goal", ["Signup"]]], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - } - ) - |> check_goals( - preloaded_goals: %{ - all: ["Purchase", "Subscription", "Signup"], - matching_toplevel_filters: ["Signup"] - }, - revenue_warning: :no_revenue_goals_matching, - revenue_currencies: %{} - ) - end - - test "with event:goal dimension, different currencies", %{site: site} do - insert(:goal, site: site, event_name: "Purchase", currency: "USD") - insert(:goal, site: site, event_name: "Donation", currency: "EUR") - insert(:goal, site: site, event_name: "Signup") - - %{ - "site_id" => site.domain, - "metrics" => ["total_revenue", "average_revenue"], - "date_range" => "all", - "dimensions" => ["event:goal"] - } - |> check_success( - site, - %{ - metrics: [:total_revenue, :average_revenue], - utc_time_range: @date_range_day, - filters: [], - dimensions: ["event:goal"], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - } - ) - |> check_goals( - preloaded_goals: %{ - all: ["Donation", "Purchase", "Signup"], - matching_toplevel_filters: ["Donation", "Purchase", "Signup"] - }, - revenue_warning: nil, - revenue_currencies: %{"Donation" => :EUR, "Purchase" => :USD} - ) - end - - test "with event:goal dimension and filters", %{site: site} do - insert(:goal, site: site, event_name: "Purchase", currency: "USD") - insert(:goal, site: site, event_name: "Subscription", currency: "EUR") - insert(:goal, site: site, event_name: "Signup") - insert(:goal, site: site, event_name: "Logout") - - %{ - "site_id" => site.domain, - "metrics" => ["total_revenue", "average_revenue"], - "date_range" => "all", - "dimensions" => ["event:goal"], - "filters" => [["is", "event:goal", ["Purchase", "Signup", "Subscription"]]] - } - |> check_success( - site, - %{ - metrics: [:total_revenue, :average_revenue], - utc_time_range: @date_range_day, - filters: [[:is, "event:goal", ["Purchase", "Signup", "Subscription"]]], - dimensions: ["event:goal"], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - } - ) - |> check_goals( - preloaded_goals: %{ - all: ["Logout", "Purchase", "Signup", "Subscription"], - matching_toplevel_filters: ["Purchase", "Signup", "Subscription"] - }, - revenue_warning: nil, - revenue_currencies: %{"Purchase" => :USD, "Subscription" => :EUR} - ) - end - - test "with event:goal dimension and filters with no revenue goals matching", %{ - site: site - } do - insert(:goal, site: site, event_name: "Purchase", currency: "USD") - insert(:goal, site: site, event_name: "Subscription", currency: "USD") - insert(:goal, site: site, event_name: "Signup") - insert(:goal, site: site, event_name: "Logout") - - %{ - "site_id" => site.domain, - "metrics" => ["total_revenue", "average_revenue"], - "date_range" => "all", - "dimensions" => ["event:goal"], - "filters" => [["is", "event:goal", ["Signup"]]] - } - |> check_success( - site, - %{ - metrics: [:total_revenue, :average_revenue], - utc_time_range: @date_range_day, - filters: [[:is, "event:goal", ["Signup"]]], - dimensions: ["event:goal"], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - } - ) - |> check_goals( - preloaded_goals: %{ - all: ["Logout", "Signup", "Subscription", "Purchase"], - matching_toplevel_filters: ["Signup"] - }, - revenue_warning: :no_revenue_goals_matching, - revenue_currencies: %{} - ) - end - end - - @tag :ce_build_only - test "revenue metrics are not available on CE", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["total_revenue", "average_revenue"], - "date_range" => "all" - } - |> check_error( - site, - "#/metrics/0: Invalid metric \"total_revenue\"\n#/metrics/1: Invalid metric \"average_revenue\"" - ) - end - - describe "session metrics" do - test "single session metric succeeds", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["bounce_rate"], - "date_range" => "all", - "dimensions" => ["visit:device"] - } - |> check_success(site, %{ - metrics: [:bounce_rate], - utc_time_range: @date_range_day, - filters: [], - dimensions: ["visit:device"], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - - test "fails if using session metric with event dimension", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["bounce_rate"], - "date_range" => "all", - "dimensions" => ["event:props:foo"] - } - |> check_error( - site, - "Session metric(s) `bounce_rate` cannot be queried along with event dimension(s) `event:props:foo`" - ) - end - - test "fails if using event metric with session-only dimension", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["events"], - "date_range" => "all", - "dimensions" => ["visit:exit_page"] - } - |> check_error( - site, - "Event metric(s) `events` cannot be queried along with session dimension(s) `visit:exit_page`" - ) - end - - test "does not fail if using session metric with event:page dimension", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["bounce_rate"], - "date_range" => "all", - "dimensions" => ["event:page"] - } - |> check_success(site, %{ - metrics: [:bounce_rate], - utc_time_range: @date_range_day, - filters: [], - dimensions: ["event:page"], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - - test "does not fail if using session metric with event filter", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["bounce_rate"], - "date_range" => "all", - "filters" => [["is", "event:props:foo", ["(none)"]]] - } - |> check_success(site, %{ - metrics: [:bounce_rate], - utc_time_range: @date_range_day, - filters: [[:is, "event:props:foo", ["(none)"]]], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - }) - end - end - - describe "filtering with segments" do - test "parsing fails when too many segments in query", %{ - user: user, - site: site - } do - segments = - insert_list(11, :segment, - type: :site, - owner: user, - site: site, - name: "any" - ) - - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [ - ["and", segments |> Enum.map(fn segment -> ["is", "segment", [segment.id]] end)] - ] - } - |> check_error( - site, - "Invalid filters. You can only use up to 10 segment filters in a query." - ) - end - - test "parsing fails when segment filter is used, but segment is from another site", %{ - site: site - } do - other_user = new_user() - other_site = new_site(owner: other_user) - - segment = - insert(:segment, - type: :site, - owner: other_user, - site: other_site, - name: "any" - ) - - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [["is", "segment", [segment.id]]] - } - |> check_error( - site, - "Invalid filters. Some segments don't exist or aren't accessible." - ) - end - - test "hiding custom properties filters in segments doesn't allow bypasssing feature check", - %{ - site: site, - user: user - } do - subscribe_to_enterprise_plan(user, features: [Plausible.Billing.Feature.StatsAPI]) - - segment = - insert(:segment, - type: :site, - owner: user, - site: site, - name: "segment with custom props filter", - segment_data: %{"filters" => [["is", "event:props:foobar", ["foo"]]]} - ) - - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [["is", "segment", [segment.id]]] - } - |> check_error( - site, - "The owner of this site does not have access to the custom properties feature." - ) - end - - test "querying conversion rate is illegal if the complex event:goal filter is within a segment", - %{ - site: site, - user: user - } do - segment = - insert(:segment, - type: :site, - owner: user, - site: site, - name: "any", - segment_data: %{ - "filters" => [ - [ - "or", - [ - ["is", "event:goal", ["Signup"]], - ["contains", "event:page", ["/"]] - ] - ] - ] - } - ) - - %{ - "site_id" => site.domain, - "metrics" => ["visitors", "conversion_rate"], - "date_range" => "all", - "filters" => [["is", "segment", [segment.id]]] - } - |> check_error( - site, - "Invalid filters. Dimension `event:goal` can only be filtered at the top level." - ) - end - - test "resolves segments correctly", %{site: site, user: user} do - emea_segment = - insert(:segment, - type: :site, - owner: user, - site: site, - name: "EMEA", - segment_data: %{ - "filters" => [["is", "visit:country", ["FR", "DE"]]], - "labels" => %{"FR" => "France", "DE" => "Germany"} - } - ) - - apac_segment = - insert(:segment, - type: :site, - owner: user, - site: site, - name: "APAC", - segment_data: %{ - "filters" => [["is", "visit:country", ["AU", "NZ"]]], - "labels" => %{"AU" => "Australia", "NZ" => "New Zealand"} - } - ) - - firefox_segment = - insert(:segment, - type: :site, - owner: user, - site: site, - name: "APAC", - segment_data: %{ - "filters" => [ - ["is", "visit:browser", ["Firefox"]], - ["is", "visit:os", ["Linux"]] - ] - } - ) - - %{ - "site_id" => site.domain, - "metrics" => ["visitors", "events"], - "date_range" => "all", - "filters" => [ - [ - "and", - [ - ["is", "segment", [apac_segment.id, emea_segment.id]], - ["is", "segment", [firefox_segment.id]] - ] - ] - ] - } - |> check_success( - site, - %{ - metrics: [:visitors, :events], - utc_time_range: @date_range_day, - filters: [ - [ - :or, - [ - [:and, [[:is, "visit:country", ["AU", "NZ"]]]], - [:and, [[:is, "visit:country", ["FR", "DE"]]]] - ] - ], - [ - :and, - [ - [:is, "visit:browser", ["Firefox"]], - [:is, "visit:os", ["Linux"]] - ] - ] - ], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - } - ) - end - - test "resolves segments containing otherwise internal features", %{site: site, user: user} do - insert(:goal, %{site: site, event_name: "Signup"}) - - segment_from_dashboard = - insert(:segment, - name: "A segment that contains :internal features", - type: :site, - owner: user, - site: site, - segment_data: %{ - "filters" => [["has_not_done", ["is", "event:goal", ["Signup"]]]] - } - ) - - %{ - "site_id" => site.domain, - "metrics" => ["visitors", "events"], - "date_range" => "all", - "filters" => [ - ["is", "segment", [segment_from_dashboard.id]] - ] - } - |> check_success( - site, - %{ - metrics: [:visitors, :events], - utc_time_range: @date_range_day, - filters: [ - [:has_not_done, [:is, "event:goal", ["Signup"]]] - ], - dimensions: [], - order_by: nil, - timezone: site.timezone, - include: @default_include, - pagination: %{limit: 10_000, offset: 0} - } - ) - end - - test "validation fails with string segment ids", %{site: site} do - %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all", - "filters" => [["is", "segment", ["123"]]] - } - |> check_error( - site, - "Invalid filter '[\"is\", \"segment\", [\"123\"]]'." - ) - end - end - - on_ee do - describe "query.consolidated_site_ids" do - test "is set to nil when site is regular", %{site: site} do - params = %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all" - } - - {:ok, %{consolidated_site_ids: nil}} = parse(site, :public, params) - {:ok, %{consolidated_site_ids: nil}} = parse(site, :internal, params) - end - - test "is set to a list of site_ids when site is consolidated", %{site: site} do - new_site(team: site.team) - cv = new_consolidated_view(site.team) - - params = %{ - "site_id" => site.domain, - "metrics" => ["visitors"], - "date_range" => "all" - } - - assert {:ok, %{consolidated_site_ids: site_ids}} = parse(cv, :public, params) - assert length(site_ids) == 2 - assert site.id in site_ids - - assert {:ok, %{consolidated_site_ids: site_ids}} = parse(cv, :internal, params) - assert length(site_ids) == 2 - assert site.id in site_ids - end - end - end -end