diff --git a/assets/js/types/query-api.d.ts b/assets/js/types/query-api.d.ts index 2b77acf19216..51b52f505099 100644 --- a/assets/js/types/query-api.d.ts +++ b/assets/js/types/query-api.d.ts @@ -64,7 +64,7 @@ export type CustomPropertyFilterDimensions = string; export type GoalDimension = "event:goal"; export type TimeDimensions = "time" | "time:month" | "time:week" | "time:day" | "time:hour"; export type FilterTree = FilterEntry | FilterAndOr | FilterNot; -export type FilterEntry = FilterWithoutGoals | FilterWithGoals | FilterWithPattern; +export type FilterEntry = FilterWithoutGoals | FilterWithGoals | FilterWithPattern | FilterForSegment; /** * @minItems 3 * @maxItems 4 @@ -115,6 +115,11 @@ export type FilterWithPattern = [ * filter operation */ export type FilterOperationRegex = "matches" | "matches_not"; +/** + * @minItems 3 + * @maxItems 3 + */ +export type FilterForSegment = ["is", "segment", number[]]; /** * @minItems 2 * @maxItems 2 diff --git a/lib/plausible/segments/filters.ex b/lib/plausible/segments/filters.ex new file mode 100644 index 000000000000..a18cf33e787b --- /dev/null +++ b/lib/plausible/segments/filters.ex @@ -0,0 +1,108 @@ +defmodule Plausible.Segments.Filters do + @moduledoc """ + This module contains functions that enable resolving segments in filters. + """ + alias Plausible.Segments + alias Plausible.Stats.Filters + + @max_segment_filters_count 10 + + @doc """ + Finds unique segment IDs used in query filters. + + ## Examples + iex> get_segment_ids([[:not, [:is, "segment", [10, 20]]], [:contains, "visit:entry_page", ["blog"]]]) + {:ok, [10, 20]} + + iex> get_segment_ids([[:and, [[:is, "segment", Enum.to_list(1..6)], [:is, "segment", Enum.to_list(1..6)]]]]) + {:error, "Invalid filters. You can only use up to 10 segment filters in a query."} + """ + def get_segment_ids(filters) do + ids = + filters + |> Filters.traverse() + |> Enum.flat_map(fn + {[_operation, "segment", clauses], _depth} -> clauses + _ -> [] + end) + + if length(ids) > @max_segment_filters_count do + {:error, + "Invalid filters. You can only use up to #{@max_segment_filters_count} segment filters in a query."} + else + {:ok, Enum.uniq(ids)} + end + end + + def preload_needed_segments(%Plausible.Site{} = site, filters) do + with {:ok, segment_ids} <- get_segment_ids(filters), + {:ok, segments} <- + Segments.get_many( + site, + segment_ids, + fields: [:id, :segment_data] + ), + {:ok, segments_by_id} <- + {:ok, + Enum.into( + segments, + %{}, + fn %Segments.Segment{id: id, segment_data: segment_data} -> + case Filters.QueryParser.parse_filters(segment_data["filters"]) do + {:ok, filters} -> {id, filters} + _ -> {id, nil} + end + end + )}, + :ok <- + if(Enum.any?(segment_ids, fn id -> is_nil(Map.get(segments_by_id, id)) end), + do: {:error, "Invalid filters. Some segments don't exist or aren't accessible."}, + else: :ok + ) do + {:ok, segments_by_id} + end + end + + defp replace_segment_with_filter_tree([_, "segment", clauses], preloaded_segments) do + if length(clauses) === 1 do + [[:and, Map.get(preloaded_segments, Enum.at(clauses, 0))]] + else + [[:or, Enum.map(clauses, fn id -> [:and, Map.get(preloaded_segments, id)] end)]] + end + end + + defp replace_segment_with_filter_tree(_filter, _preloaded_segments) do + nil + end + + @doc """ + ## Examples + + iex> resolve_segments([[:is, "visit:entry_page", ["/home"]]], %{}) + {:ok, [[:is, "visit:entry_page", ["/home"]]]} + + iex> resolve_segments([[:is, "visit:entry_page", ["/home"]], [:is, "segment", [1]]], %{1 => [[:contains, "visit:entry_page", ["blog"]], [:is, "visit:country", ["PL"]]]}) + {:ok, [ + [:is, "visit:entry_page", ["/home"]], + [:and, [[:contains, "visit:entry_page", ["blog"]], [:is, "visit:country", ["PL"]]]] + ]} + + iex> resolve_segments([[:is, "segment", [1, 2]]], %{1 => [[:contains, "event:goal", ["Singup"]], [:is, "visit:country", ["PL"]]], 2 => [[:contains, "event:goal", ["Sauna"]], [:is, "visit:country", ["EE"]]]}) + {:ok, [ + [:or, [ + [:and, [[:contains, "event:goal", ["Singup"]], [:is, "visit:country", ["PL"]]]], + [:and, [[:contains, "event:goal", ["Sauna"]], [:is, "visit:country", ["EE"]]]]] + ] + ]} + """ + def resolve_segments(original_filters, preloaded_segments) do + if map_size(preloaded_segments) > 0 do + {:ok, + Filters.transform_filters(original_filters, fn f -> + replace_segment_with_filter_tree(f, preloaded_segments) + end)} + else + {:ok, original_filters} + end + end +end diff --git a/lib/plausible/segments/segments.ex b/lib/plausible/segments/segments.ex index 6284bd6345ad..37f420e6d604 100644 --- a/lib/plausible/segments/segments.ex +++ b/lib/plausible/segments/segments.ex @@ -37,6 +37,21 @@ defmodule Plausible.Segments do end end + @spec get_many(Plausible.Site.t(), list(pos_integer()), Keyword.t()) :: + {:ok, [Segment.t()]} + def get_many(%Plausible.Site{} = site, segment_ids, opts) when is_list(segment_ids) do + fields = Keyword.get(opts, :fields, [:id]) + + query = + from(segment in Segment, + select: ^fields, + where: segment.site_id == ^site.id, + where: segment.id in ^segment_ids + ) + + {:ok, Repo.all(query)} + end + @spec get_one(pos_integer(), Plausible.Site.t(), atom(), pos_integer() | nil) :: {:ok, Segment.t()} | error_not_enough_permissions() diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.ex index 9f4116f2c8a0..4c828fef1e50 100644 --- a/lib/plausible/stats/filters/filters.ex +++ b/lib/plausible/stats/filters/filters.ex @@ -57,7 +57,7 @@ defmodule Plausible.Stats.Filters do Returns an empty list when argument type is unexpected (e.g. `nil`). - ### Examples: + ## Examples: iex> Filters.parse("visit:browser!=Chrome") [[:is_not, "visit:browser", ["Chrome"]]] @@ -128,6 +128,13 @@ defmodule Plausible.Stats.Filters do Transformer will receive each node (filter, and/or/not subtree) of query and must return a list of nodes to replace it with or nil to ignore and look deeper. + + ## Examples + iex> Filters.transform_filters([[:is, "visit:os", ["Linux"]], [:and, [[:is, "segment", [1]], [:is, "segment", [2]]]]], fn + ...> [_, "segment", _] -> [[:is, "segment", ["changed"]]] + ...> _ -> nil + ...> end) + [[:is, "visit:os", ["Linux"]], [:and, [[:is, "segment", ["changed"]], [:is, "segment", ["changed"]]]]] """ def transform_filters(filters, transformer) do filters @@ -146,7 +153,7 @@ defmodule Plausible.Stats.Filters do # Reached a leaf node, return existing value {nil, filter} -> - [[filter]] + [filter] # Transformer returned a value - don't transform that subtree {transformed_filters, _filter} -> @@ -154,7 +161,7 @@ defmodule Plausible.Stats.Filters do end end - defp traverse(filters, depth \\ -1) do + def traverse(filters, depth \\ -1) do filters |> Enum.flat_map(&traverse_tree(&1, depth + 1)) end diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex index 048e61f98845..0dd6a976c8ac 100644 --- a/lib/plausible/stats/filters/query_parser.ex +++ b/lib/plausible/stats/filters/query_parser.ex @@ -34,6 +34,10 @@ defmodule Plausible.Stats.Filters.QueryParser do utc_time_range = raw_time_range |> DateTimeRange.to_timezone("Etc/UTC"), {:ok, metrics} <- parse_metrics(Map.get(params, "metrics", [])), {:ok, filters} <- parse_filters(Map.get(params, "filters", [])), + {:ok, preloaded_segments} <- + Plausible.Segments.Filters.preload_needed_segments(site, filters), + {:ok, filters} <- + Plausible.Segments.Filters.resolve_segments(filters, preloaded_segments), {:ok, dimensions} <- parse_dimensions(Map.get(params, "dimensions", [])), {:ok, order_by} <- parse_order_by(Map.get(params, "order_by")), {:ok, include} <- parse_include(site, Map.get(params, "include", %{})), @@ -161,7 +165,10 @@ defmodule Plausible.Stats.Filters.QueryParser do "Invalid visit:country filter, visit:country needs to be a valid 2-letter country code."} end - {_, true} -> + {"segment", _} when all_integers? -> + {:ok, list} + + {_, true} when filter_key !== "segment" -> {:ok, list} _ -> @@ -396,6 +403,9 @@ defmodule Plausible.Stats.Filters.QueryParser do {:error, error_message} end + "segment" -> + {:ok, filter_key} + _ -> {:error, error_message} end diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.json index df18503283fa..6a325c8d8e0e 100644 --- a/priv/json-schemas/query-api-schema.json +++ b/priv/json-schemas/query-api-schema.json @@ -98,13 +98,11 @@ "minItems": 2, "maxItems": 2, "items": { - "type": "string", - "format": "date" + "type": "string", + "format": "date" }, "description": "If custom period. A list of two ISO8601 dates or timestamps to compare against.", - "examples": [ - ["2024-01-01", "2024-01-31"] - ] + "examples": [["2024-01-01", "2024-01-31"]] } }, "required": ["mode", "date_range"], @@ -439,11 +437,35 @@ } ] }, + "filter_for_segment": { + "type": "array", + "additionalItems": false, + "minItems": 3, + "maxItems": 3, + "items": [ + { + "const": "is" + }, + { + "const": "segment" + }, + { + "type": "array", + "items": { + "type": ["integer"] + } + } + ] + }, "filter_entry": { "oneOf": [ { "$ref": "#/definitions/filter_without_goals" }, { "$ref": "#/definitions/filter_with_goals" }, - { "$ref": "#/definitions/filter_with_pattern" } + { "$ref": "#/definitions/filter_with_pattern" }, + { + "$ref": "#/definitions/filter_for_segment", + "$comment": "only :internal" + } ] }, "filter_tree": { diff --git a/test/plausible/segments/filters_test.exs b/test/plausible/segments/filters_test.exs new file mode 100644 index 000000000000..e19b58df73b0 --- /dev/null +++ b/test/plausible/segments/filters_test.exs @@ -0,0 +1,5 @@ +defmodule Plausible.Segments.FiltersTest do + use ExUnit.Case, async: true + + doctest Plausible.Segments.Filters, import: true +end diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exs index ba7c8e8b818c..968d04fa0fcf 100644 --- a/test/plausible/stats/query_parser_test.exs +++ b/test/plausible/stats/query_parser_test.exs @@ -2,10 +2,11 @@ defmodule Plausible.Stats.Filters.QueryParserTest do use Plausible use Plausible.DataCase use Plausible.Teams.Test + import Plausible.Stats.Filters.QueryParser + doctest Plausible.Stats.Filters.QueryParser alias Plausible.Stats.DateTimeRange alias Plausible.Stats.Filters - import Plausible.Stats.Filters.QueryParser setup [:create_user, :create_site] @@ -2028,4 +2029,215 @@ defmodule Plausible.Stats.Filters.QueryParserTest do }) end end + + describe "filtering with segments" do + test "parsing fails when too many segments in query", %{ + user: user, + site: site + } do + segments = + insert_list(11, :segment, + type: :site, + owner: user, + site: site, + name: "any" + ) + + %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["and", segments |> Enum.map(fn segment -> ["is", "segment", [segment.id]] end)] + ] + } + |> check_error( + site, + "Invalid filters. You can only use up to 10 segment filters in a query.", + :internal + ) + end + + test "parsing fails when segment filter is used, but segment is from another site", %{ + site: site + } do + other_user = new_user() + other_site = new_site(owner: other_user) + + segment = + insert(:segment, + type: :site, + owner: other_user, + site: other_site, + name: "any" + ) + + %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "segment", [segment.id]]] + } + |> check_error( + site, + "Invalid filters. Some segments don't exist or aren't accessible.", + :internal + ) + end + + test "hiding custom properties filters in segments doesn't allow bypasssing feature check", + %{ + site: site, + user: user + } do + subscribe_to_enterprise_plan(user, features: [Plausible.Billing.Feature.StatsAPI]) + + segment = + insert(:segment, + type: :site, + owner: user, + site: site, + name: "segment with custom props filter", + segment_data: %{"filters" => [["is", "event:props:foobar", ["foo"]]]} + ) + + %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "segment", [segment.id]]] + } + |> check_error( + site, + "The owner of this site does not have access to the custom properties feature.", + :internal + ) + end + + test "querying conversion rate is illegal if the complex event:goal filter is within a segment", + %{ + site: site, + user: user + } do + segment = + insert(:segment, + type: :site, + owner: user, + site: site, + name: "any", + segment_data: %{ + "filters" => [ + [ + "or", + [ + ["is", "event:goal", ["Signup"]], + ["contains", "event:page", ["/"]] + ] + ] + ] + } + ) + + %{ + "site_id" => site.domain, + "metrics" => ["visitors", "conversion_rate"], + "date_range" => "all", + "filters" => [["is", "segment", [segment.id]]] + } + |> check_error( + site, + "Invalid filters. Dimension `event:goal` can only be filtered at the top level.", + :internal + ) + end + + test "resolves segments correctly", %{site: site, user: user} do + emea_segment = + insert(:segment, + type: :site, + owner: user, + site: site, + name: "EMEA", + segment_data: %{ + "filters" => [["is", "visit:country", ["FR", "DE"]]], + "labels" => %{"FR" => "France", "DE" => "Germany"} + } + ) + + apac_segment = + insert(:segment, + type: :site, + owner: user, + site: site, + name: "APAC", + segment_data: %{ + "filters" => [["is", "visit:country", ["AU", "NZ"]]], + "labels" => %{"AU" => "Australia", "NZ" => "New Zealand"} + } + ) + + firefox_segment = + insert(:segment, + type: :site, + owner: user, + site: site, + name: "APAC", + segment_data: %{ + "filters" => [ + ["is", "visit:browser", ["Firefox"]], + ["is", "visit:os", ["Linux"]] + ] + } + ) + + %{ + "site_id" => site.domain, + "metrics" => ["visitors", "events"], + "date_range" => "all", + "filters" => [ + [ + "and", + [ + ["is", "segment", [apac_segment.id, emea_segment.id]], + ["is", "segment", [firefox_segment.id]] + ] + ] + ] + } + |> check_success( + site, + %{ + metrics: [:visitors, :events], + utc_time_range: @date_range_day, + filters: [ + [ + :and, + [ + [ + :or, + [ + [:and, [[:is, "visit:country", ["AU", "NZ"]]]], + [:and, [[:is, "visit:country", ["FR", "DE"]]]] + ] + ], + [ + :and, + [ + [:is, "visit:browser", ["Firefox"]], + [:is, "visit:os", ["Linux"]] + ] + ] + ] + ] + ], + dimensions: [], + order_by: nil, + timezone: site.timezone, + include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil}, + pagination: %{limit: 10_000, offset: 0} + }, + :internal + ) + end + end end diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs index 1681bb208f88..4697b94c7154 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs @@ -4577,4 +4577,110 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do assert "metric_warnings" not in json_response(conn, 200)["meta"] end end + + describe "segment filters" do + setup [:create_user, :create_site, :create_api_key, :use_api_key] + + test "segment filters are (not yet) available in public API", %{conn: conn, site: site} do + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "filters" => [["is", "segment", [1]]], + "date_range" => "all", + "metrics" => ["visitors"] + }) + + assert json_response(conn, 400) == %{ + "error" => "#/filters/0: Invalid filter [\"is\", \"segment\", [1]]" + } + end + + test "site segments of other sites don't resolve", %{ + conn: conn, + site: site + } do + other_site = new_site() + other_user = add_guest(other_site, role: :editor) + + segment = + insert(:segment, + name: "Segment of another site", + type: :site, + owner: other_user, + site: other_site, + segment_data: %{ + "filters" => [["is", "event:page", ["/blog"]]] + } + ) + + conn = + post(conn, "/api/v2/query-internal-test", %{ + "site_id" => site.domain, + "filters" => [["is", "segment", [segment.id]]], + "date_range" => "all", + "metrics" => ["events"] + }) + + assert json_response(conn, 400) == %{ + "error" => "Invalid filters. Some segments don't exist or aren't accessible." + } + end + + test "even personal segments of other users of the same site resolve to filters, with segments expanded in response", + %{ + conn: conn, + site: site + } do + other_user = add_guest(site, role: :editor) + + segment = + insert(:segment, + type: :personal, + owner: other_user, + site: site, + name: "Signups", + segment_data: %{ + "filters" => [["is", "event:name", ["Signup"]]] + } + ) + + insert(:goal, %{site: site, event_name: "Signup"}) + + populate_stats(site, [ + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "AnyOtherEvent", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query-internal-test", %{ + "site_id" => site.domain, + "filters" => [["is", "segment", [segment.id]]], + "date_range" => "all", + "metrics" => ["events"] + }) + + assert json_response(conn, 200)["results"] == [%{"dimensions" => [], "metrics" => [3]}] + + # response shows what filters the segment was resolved to + assert json_response(conn, 200)["query"]["filters"] == [ + ["and", [["is", "event:name", ["Signup"]]]] + ] + end + end end