diff --git a/CHANGELOG.md b/CHANGELOG.md index dc0f3056f8bb..00bb32b5c9a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,9 +6,13 @@ All notable changes to this project will be documented in this file. ### Added - Dashboard shows comparisons for all reports - UTM Medium report and API shows (gclid) and (msclkid) for paid searches when no explicit utm medium present. +- Support for `case_sensitive: false` modifiers in Stats API V2 filters for case-insensitive searches. ### Removed ### Changed + +- Details modal search inputs are now case-insensitive. + ### Fixed - Fix returning filter suggestions for multiple custom property values in the dashboard Filter modal diff --git a/assets/js/dashboard/stats/modals/conversions.js b/assets/js/dashboard/stats/modals/conversions.js index 3514bd852de8..cc1a00d94782 100644 --- a/assets/js/dashboard/stats/modals/conversions.js +++ b/assets/js/dashboard/stats/modals/conversions.js @@ -28,7 +28,7 @@ function ConversionsModal() { }, [reportInfo.dimension]) const addSearchFilter = useCallback((query, searchString) => { - return addFilter(query, ['contains', reportInfo.dimension, [searchString]]) + return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }]) }, [reportInfo.dimension]) function chooseMetrics() { diff --git a/assets/js/dashboard/stats/modals/devices/browser-versions-modal.js b/assets/js/dashboard/stats/modals/devices/browser-versions-modal.js index 369b4dce9f9c..d0e11e299b97 100644 --- a/assets/js/dashboard/stats/modals/devices/browser-versions-modal.js +++ b/assets/js/dashboard/stats/modals/devices/browser-versions-modal.js @@ -29,7 +29,7 @@ function BrowserVersionsModal() { }, [reportInfo.dimension]) const addSearchFilter = useCallback((query, searchString) => { - return addFilter(query, ['contains', reportInfo.dimension, [searchString]]) + return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }]) }, [reportInfo.dimension]) const renderIcon = useCallback((listItem) => browserIconFor(listItem.browser), []) diff --git a/assets/js/dashboard/stats/modals/devices/browsers-modal.js b/assets/js/dashboard/stats/modals/devices/browsers-modal.js index 1160b8ebf5f8..9b600514bd99 100644 --- a/assets/js/dashboard/stats/modals/devices/browsers-modal.js +++ b/assets/js/dashboard/stats/modals/devices/browsers-modal.js @@ -29,7 +29,7 @@ function BrowsersModal() { }, [reportInfo.dimension]) const addSearchFilter = useCallback((query, searchString) => { - return addFilter(query, ['contains', reportInfo.dimension, [searchString]]) + return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }]) }, [reportInfo.dimension]) const renderIcon = useCallback((listItem) => browserIconFor(listItem.name), []) diff --git a/assets/js/dashboard/stats/modals/devices/operating-system-versions-modal.js b/assets/js/dashboard/stats/modals/devices/operating-system-versions-modal.js index 6c7d5c72cc61..9c5622b48ded 100644 --- a/assets/js/dashboard/stats/modals/devices/operating-system-versions-modal.js +++ b/assets/js/dashboard/stats/modals/devices/operating-system-versions-modal.js @@ -29,7 +29,7 @@ function OperatingSystemVersionsModal() { }, [reportInfo.dimension]) const addSearchFilter = useCallback((query, searchString) => { - return addFilter(query, ['contains', reportInfo.dimension, [searchString]]) + return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }]) }, [reportInfo.dimension]) const renderIcon = useCallback((listItem) => osIconFor(listItem.os), []) diff --git a/assets/js/dashboard/stats/modals/devices/operating-systems-modal.js b/assets/js/dashboard/stats/modals/devices/operating-systems-modal.js index 2a92795648ad..9df8a6c7c7e4 100644 --- a/assets/js/dashboard/stats/modals/devices/operating-systems-modal.js +++ b/assets/js/dashboard/stats/modals/devices/operating-systems-modal.js @@ -29,7 +29,7 @@ function OperatingSystemsModal() { }, [reportInfo.dimension]) const addSearchFilter = useCallback((query, searchString) => { - return addFilter(query, ['contains', reportInfo.dimension, [searchString]]) + return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }]) }, [reportInfo.dimension]) const renderIcon = useCallback((listItem) => osIconFor(listItem.name), []) diff --git a/assets/js/dashboard/stats/modals/entry-pages.js b/assets/js/dashboard/stats/modals/entry-pages.js index 6d5a72d693ad..79963046fb64 100644 --- a/assets/js/dashboard/stats/modals/entry-pages.js +++ b/assets/js/dashboard/stats/modals/entry-pages.js @@ -29,7 +29,7 @@ function EntryPagesModal() { }, [reportInfo.dimension]) const addSearchFilter = useCallback((query, searchString) => { - return addFilter(query, ['contains', reportInfo.dimension, [searchString]]) + return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }]) }, [reportInfo.dimension]) function chooseMetrics() { diff --git a/assets/js/dashboard/stats/modals/exit-pages.js b/assets/js/dashboard/stats/modals/exit-pages.js index 2be05f016868..413c89056877 100644 --- a/assets/js/dashboard/stats/modals/exit-pages.js +++ b/assets/js/dashboard/stats/modals/exit-pages.js @@ -29,7 +29,7 @@ function ExitPagesModal() { }, [reportInfo.dimension]) const addSearchFilter = useCallback((query, searchString) => { - return addFilter(query, ['contains', reportInfo.dimension, [searchString]]) + return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }]) }, [reportInfo.dimension]) function chooseMetrics() { diff --git a/assets/js/dashboard/stats/modals/locations-modal.js b/assets/js/dashboard/stats/modals/locations-modal.js index 615f6e9e3d42..bfdd471b9720 100644 --- a/assets/js/dashboard/stats/modals/locations-modal.js +++ b/assets/js/dashboard/stats/modals/locations-modal.js @@ -32,7 +32,7 @@ function LocationsModal({ currentView }) { }, [reportInfo.dimension]) const addSearchFilter = useCallback((query, searchString) => { - return addFilter(query, ['contains', `${reportInfo.dimension}_name`, [searchString]]) + return addFilter(query, ['contains', `${reportInfo.dimension}_name`, [searchString], { case_sensitive: false }]) }, [reportInfo.dimension]) function chooseMetrics() { diff --git a/assets/js/dashboard/stats/modals/pages.js b/assets/js/dashboard/stats/modals/pages.js index 26bf287c6cb1..e9521f9774be 100644 --- a/assets/js/dashboard/stats/modals/pages.js +++ b/assets/js/dashboard/stats/modals/pages.js @@ -29,7 +29,7 @@ function PagesModal() { }, [reportInfo.dimension]) const addSearchFilter = useCallback((query, searchString) => { - return addFilter(query, ['contains', reportInfo.dimension, [searchString]]) + return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }]) }, [reportInfo.dimension]) function chooseMetrics() { @@ -46,14 +46,14 @@ function PagesModal() { metrics.createVisitors({renderLabel: (_query) => 'Current visitors', width: 'w-36'}) ] } - + const defaultMetrics = [ metrics.createVisitors({renderLabel: (_query) => "Visitors" }), metrics.createPageviews(), metrics.createBounceRate(), metrics.createTimeOnPage() ] - + return site.flags.scroll_depth ? [...defaultMetrics, metrics.createScrollDepth()] : defaultMetrics } diff --git a/assets/js/dashboard/stats/modals/props.js b/assets/js/dashboard/stats/modals/props.js index 0d691abe8d59..a131e8a43837 100644 --- a/assets/js/dashboard/stats/modals/props.js +++ b/assets/js/dashboard/stats/modals/props.js @@ -36,7 +36,7 @@ function PropsModal() { }, [propKey]) const addSearchFilter = useCallback((query, searchString) => { - return addFilter(query, ['contains', `${EVENT_PROPS_PREFIX}${propKey}`, [searchString]]) + return addFilter(query, ['contains', `${EVENT_PROPS_PREFIX}${propKey}`, [searchString], { case_sensitive: false }]) }, [propKey]) function chooseMetrics() { diff --git a/assets/js/dashboard/stats/modals/referrer-drilldown.js b/assets/js/dashboard/stats/modals/referrer-drilldown.js index 6077bb737e2b..264e1d54547b 100644 --- a/assets/js/dashboard/stats/modals/referrer-drilldown.js +++ b/assets/js/dashboard/stats/modals/referrer-drilldown.js @@ -33,7 +33,7 @@ function ReferrerDrilldownModal() { }, [reportInfo.dimension]) const addSearchFilter = useCallback((query, searchString) => { - return addFilter(query, ['contains', reportInfo.dimension, [searchString]]) + return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }]) }, [reportInfo.dimension]) function chooseMetrics() { diff --git a/assets/js/dashboard/stats/modals/sources.js b/assets/js/dashboard/stats/modals/sources.js index d4be4b30b951..3fd0006c71d1 100644 --- a/assets/js/dashboard/stats/modals/sources.js +++ b/assets/js/dashboard/stats/modals/sources.js @@ -57,7 +57,7 @@ function SourcesModal({ currentView }) { }, [reportInfo.dimension]) const addSearchFilter = useCallback((query, searchString) => { - return addFilter(query, ['contains', reportInfo.dimension, [searchString]]) + return addFilter(query, ['contains', reportInfo.dimension, [searchString], { case_sensitive: false }]) }, [reportInfo.dimension]) function chooseMetrics() { diff --git a/assets/js/dashboard/util/filters.js b/assets/js/dashboard/util/filters.js index d8d13a70dab5..12e9a60c604e 100644 --- a/assets/js/dashboard/util/filters.js +++ b/assets/js/dashboard/util/filters.js @@ -229,16 +229,18 @@ export function cleanLabels(filters, labels, mergedFilterKey, mergedLabels) { const EVENT_FILTER_KEYS = new Set(['name', 'page', 'goal', 'hostname']) export function serializeApiFilters(filters) { - const apiFilters = filters.map(([operation, filterKey, clauses]) => { - let apiFilterKey = `visit:${filterKey}` - if ( - filterKey.startsWith(EVENT_PROPS_PREFIX) || - EVENT_FILTER_KEYS.has(filterKey) - ) { - apiFilterKey = `event:${filterKey}` + const apiFilters = filters.map( + ([operation, filterKey, clauses, ...modifiers]) => { + let apiFilterKey = `visit:${filterKey}` + if ( + filterKey.startsWith(EVENT_PROPS_PREFIX) || + EVENT_FILTER_KEYS.has(filterKey) + ) { + apiFilterKey = `event:${filterKey}` + } + return [operation, apiFilterKey, clauses, ...modifiers] } - return [operation, apiFilterKey, clauses] - }) + ) return JSON.stringify(apiFilters) } diff --git a/assets/js/types/query-api.d.ts b/assets/js/types/query-api.d.ts index 06121a8166e3..2b77acf19216 100644 --- a/assets/js/types/query-api.d.ts +++ b/assets/js/types/query-api.d.ts @@ -64,34 +64,57 @@ export type CustomPropertyFilterDimensions = string; export type GoalDimension = "event:goal"; export type TimeDimensions = "time" | "time:month" | "time:week" | "time:day" | "time:hour"; export type FilterTree = FilterEntry | FilterAndOr | FilterNot; -export type FilterEntry = FilterWithoutGoals | FilterWithGoals; +export type FilterEntry = FilterWithoutGoals | FilterWithGoals | FilterWithPattern; /** * @minItems 3 - * @maxItems 3 + * @maxItems 4 */ -export type FilterWithoutGoals = [ - FilterOperationWithoutGoals | ("matches_wildcard" | "matches_wildcard_not"), - SimpleFilterDimensions | CustomPropertyFilterDimensions, - Clauses -]; +export type FilterWithoutGoals = + | [FilterOperationWithoutGoals, SimpleFilterDimensions | CustomPropertyFilterDimensions, Clauses] + | [ + FilterOperationWithoutGoals, + SimpleFilterDimensions | CustomPropertyFilterDimensions, + Clauses, + { + case_sensitive?: boolean; + } + ]; /** * filter operation */ -export type FilterOperationWithoutGoals = "is_not" | "contains_not" | "matches" | "matches_not"; +export type FilterOperationWithoutGoals = "is_not" | "contains_not"; export type Clauses = (string | number)[]; +/** + * @minItems 3 + * @maxItems 4 + */ +export type FilterWithGoals = + | [FilterOperationContains, GoalDimension | SimpleFilterDimensions | CustomPropertyFilterDimensions, Clauses] + | [ + FilterOperationContains, + GoalDimension | SimpleFilterDimensions | CustomPropertyFilterDimensions, + Clauses, + { + case_sensitive?: boolean; + } + ]; +/** + * filter operation + */ +export type FilterOperationContains = "is" | "contains"; /** * @minItems 3 * @maxItems 3 */ -export type FilterWithGoals = [ - FilterOperationWithGoals, - GoalDimension | SimpleFilterDimensions | CustomPropertyFilterDimensions, +export type FilterWithPattern = [ + FilterOperationRegex | ("matches_wildcard" | "matches_wildcard_not"), + SimpleFilterDimensions | CustomPropertyFilterDimensions, Clauses ]; /** * filter operation */ -export type FilterOperationWithGoals = "is" | "contains"; +export type FilterOperationRegex = "matches" | "matches_not"; /** * @minItems 2 * @maxItems 2 diff --git a/lib/plausible/goals/filters.ex b/lib/plausible/goals/filters.ex index 142095e0488e..b9d7d5a74935 100644 --- a/lib/plausible/goals/filters.ex +++ b/lib/plausible/goals/filters.ex @@ -20,14 +20,14 @@ defmodule Plausible.Goals.Filters do * `imported?` - when `true`, builds conditions on the `page` db field rather than `pathname`, and also skips the `e.name == "pageview"` check. """ - def add_filter(query, [operation, "event:goal", clauses], opts \\ []) + def add_filter(query, [operation, "event:goal", clauses | _] = filter, opts \\ []) when operation in [:is, :contains] do imported? = Keyword.get(opts, :imported?, false) Enum.reduce(clauses, false, fn clause, dynamic_statement -> condition = query.preloaded_goals - |> filter_preloaded(operation, clause) + |> filter_preloaded(filter, clause) |> build_condition(imported?) dynamic([e], ^condition or ^dynamic_statement) @@ -38,32 +38,46 @@ defmodule Plausible.Goals.Filters do goals = Plausible.Goals.for_site(site) Enum.reduce(filters, goals, fn - [operation, "event:goal", clauses], goals -> - goals_matching_any_clause(goals, operation, clauses) + [_, "event:goal" | _] = filter, goals -> + goals_matching_any_clause(goals, filter) _filter, goals -> goals end) end - def filter_preloaded(preloaded_goals, operation, clause) when operation in [:is, :contains] do - Enum.filter(preloaded_goals, fn goal -> matches?(goal, operation, clause) end) + defp filter_preloaded(preloaded_goals, filter, clause) do + Enum.filter(preloaded_goals, fn goal -> matches?(goal, filter, clause) end) end - defp goals_matching_any_clause(goals, operation, clauses) do + defp goals_matching_any_clause(goals, [_, _, clauses | _] = filter) do goals |> Enum.filter(fn goal -> - Enum.any?(clauses, fn clause -> matches?(goal, operation, clause) end) + Enum.any?(clauses, fn clause -> matches?(goal, filter, clause) end) end) end - defp matches?(goal, operation, clause) do + defp matches?(goal, [operation | _rest] = filter, clause) do + goal_name = + goal + |> Plausible.Goal.display_name() + |> mod(filter) + + clause = mod(clause, filter) + case operation do :is -> - Plausible.Goal.display_name(goal) == clause + goal_name == clause :contains -> - String.contains?(Plausible.Goal.display_name(goal), clause) + String.contains?(goal_name, clause) + end + end + + defp mod(str, filter) do + case filter do + [_, _, _, %{case_sensitive: false}] -> String.downcase(str) + _ -> str end end diff --git a/lib/plausible/google/search_console/filters.ex b/lib/plausible/google/search_console/filters.ex index d010e013b5d4..1874e1640351 100644 --- a/lib/plausible/google/search_console/filters.ex +++ b/lib/plausible/google/search_console/filters.ex @@ -24,14 +24,15 @@ defmodule Plausible.Google.SearchConsole.Filters do transform_filter(property, [op, "visit:entry_page" | rest]) end - defp transform_filter(property, [:is, "visit:entry_page", pages]) when is_list(pages) do + # :TODO: Should also work case-insensitive, if not, blacklist. + defp transform_filter(property, [:is, "visit:entry_page", pages | _]) when is_list(pages) do expression = Enum.map_join(pages, "|", fn page -> property_url(property, Regex.escape(page)) end) %{dimension: "page", operator: "includingRegex", expression: expression} end - defp transform_filter(property, [:matches_wildcard, "visit:entry_page", pages]) + defp transform_filter(property, [:matches_wildcard, "visit:entry_page", pages | _]) when is_list(pages) do expression = Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end) @@ -39,12 +40,12 @@ defmodule Plausible.Google.SearchConsole.Filters do %{dimension: "page", operator: "includingRegex", expression: expression} end - defp transform_filter(_property, [:is, "visit:screen", devices]) when is_list(devices) do + defp transform_filter(_property, [:is, "visit:screen", devices | _]) when is_list(devices) do expression = Enum.map_join(devices, "|", &search_console_device/1) %{dimension: "device", operator: "includingRegex", expression: expression} end - defp transform_filter(_property, [:is, "visit:country", countries]) + defp transform_filter(_property, [:is, "visit:country", countries | _]) when is_list(countries) do expression = Enum.map_join(countries, "|", &search_console_country/1) %{dimension: "country", operator: "includingRegex", expression: expression} diff --git a/lib/plausible/stats/filters/filters.ex b/lib/plausible/stats/filters/filters.ex index c68e2b3f8651..0c16ccb3f21e 100644 --- a/lib/plausible/stats/filters/filters.ex +++ b/lib/plausible/stats/filters/filters.ex @@ -121,8 +121,8 @@ defmodule Plausible.Stats.Filters do def rename_dimensions_used_in_filter(filters, renames) do transform_filters(filters, fn - [operation, dimension, clauses] -> - [[operation, Map.get(renames, dimension, dimension), clauses]] + [operation, dimension | rest] -> + [[operation, Map.get(renames, dimension, dimension) | rest]] _subtree -> nil diff --git a/lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex b/lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex index 8e3ef1ccd27c..a0a6cf0401c6 100644 --- a/lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex +++ b/lib/plausible/stats/filters/legacy_dashboard_filter_parser.ex @@ -37,7 +37,6 @@ defmodule Plausible.Stats.Filters.LegacyDashboardFilterParser do is_negated && is_wildcard && is_list -> [:matches_wildcard_not, key, val] - # TODO is_negated && is_contains && is_list -> [:matches_wildcard_not, key, Enum.map(val, &"**#{&1}**")] diff --git a/lib/plausible/stats/filters/query_parser.ex b/lib/plausible/stats/filters/query_parser.ex index b7410964b854..684d6e1310f4 100644 --- a/lib/plausible/stats/filters/query_parser.ex +++ b/lib/plausible/stats/filters/query_parser.ex @@ -133,31 +133,35 @@ defmodule Plausible.Stats.Filters.QueryParser do :matches_wildcard_not, :contains, :contains_not - ], - do: parse_clauses_list(filter) + ] do + with {:ok, clauses} <- parse_clauses_list(filter), + {:ok, modifiers} <- parse_filter_modifiers(Enum.at(filter, 3)) do + {:ok, [clauses | modifiers]} + end + end defp parse_filter_rest(operator, _filter) when operator in [:not, :and, :or], do: {:ok, []} - defp parse_clauses_list([operation, filter_key, list] = filter) when is_list(list) do + defp parse_clauses_list([operator, filter_key, list | _rest] = filter) when is_list(list) do all_strings? = Enum.all?(list, &is_binary/1) all_integers? = Enum.all?(list, &is_integer/1) case {filter_key, all_strings?} do {"visit:city", false} when all_integers? -> - {:ok, [list]} + {:ok, list} - {"visit:country", true} when operation in ["is", "is_not"] -> + {"visit:country", true} when operator in ["is", "is_not"] -> if Enum.all?(list, &(String.length(&1) == 2)) do - {:ok, [list]} + {:ok, list} else {:error, "Invalid visit:country filter, visit:country needs to be a valid 2-letter country code."} end {_, true} -> - {:ok, [list]} + {:ok, list} _ -> {:error, "Invalid filter '#{i(filter)}'."} @@ -166,6 +170,14 @@ defmodule Plausible.Stats.Filters.QueryParser do defp parse_clauses_list(filter), do: {:error, "Invalid filter '#{i(filter)}'"} + defp parse_filter_modifiers(modifiers) when is_map(modifiers) do + {:ok, [atomize_keys(modifiers)]} + end + + defp parse_filter_modifiers(nil) do + {:ok, []} + end + defp parse_date(_site, date_string, _date) when is_binary(date_string) do case Date.from_iso8601(date_string) do {:ok, date} -> {:ok, date} diff --git a/lib/plausible/stats/filters/stats_api_filter_parser.ex b/lib/plausible/stats/filters/stats_api_filter_parser.ex index f7168a674194..ef3b03cdaa01 100644 --- a/lib/plausible/stats/filters/stats_api_filter_parser.ex +++ b/lib/plausible/stats/filters/stats_api_filter_parser.ex @@ -40,7 +40,7 @@ defmodule Plausible.Stats.Filters.StatsAPIFilterParser do end end - defp reject_invalid_country_codes([_op, "visit:country", code_or_codes] = filter) do + defp reject_invalid_country_codes([_op, "visit:country", code_or_codes | _rest] = filter) do code_or_codes |> List.wrap() |> Enum.reduce_while(filter, fn diff --git a/lib/plausible/stats/imported/base.ex b/lib/plausible/stats/imported/base.ex index 831314e6dcb8..2328a4497f65 100644 --- a/lib/plausible/stats/imported/base.ex +++ b/lib/plausible/stats/imported/base.ex @@ -117,8 +117,8 @@ defmodule Plausible.Stats.Imported.Base do has_required_name_filter? = query.filters |> Enum.flat_map(fn - [:is, "event:name", names] -> names - [:is, "event:goal", names] -> names + [:is, "event:name", names | _rest] -> names + [:is, "event:goal", names | _rest] -> names _ -> [] end) |> Enum.any?(&(&1 in special_goals_for(property))) @@ -144,7 +144,7 @@ defmodule Plausible.Stats.Imported.Base do defp do_decide_tables(%Query{dimensions: ["event:goal"]} = query) do filter_dimensions = dimensions_used_in_filters(query.filters) - filter_goals = get_filter_goals(query) + filter_goals = query.preloaded_goals any_event_goals? = Enum.any?(filter_goals, fn goal -> Plausible.Goal.type(goal) == :event end) @@ -177,8 +177,7 @@ defmodule Plausible.Stats.Imported.Base do |> Enum.map(&@property_to_table_mappings[&1]) filter_goal_table_candidates = - query - |> get_filter_goals() + query.preloaded_goals |> Enum.map(&Plausible.Goal.type/1) |> Enum.map(fn :event -> "imported_custom_events" @@ -193,17 +192,6 @@ defmodule Plausible.Stats.Imported.Base do end end - defp get_filter_goals(query) do - query.filters - |> Enum.filter(fn [_, dimension | _rest] -> dimension == "event:goal" end) - |> Enum.flat_map(fn [operation, _dimension, clauses] -> - Enum.flat_map(clauses, fn clause -> - query.preloaded_goals - |> Plausible.Goals.Filters.filter_preloaded(operation, clause) - end) - end) - end - def special_goals_for("event:props:url"), do: Imported.goals_with_url() def special_goals_for("event:props:path"), do: Imported.goals_with_path() end diff --git a/lib/plausible/stats/imported/sql/where_builder.ex b/lib/plausible/stats/imported/sql/where_builder.ex index 638fdf270c2c..93d847621032 100644 --- a/lib/plausible/stats/imported/sql/where_builder.ex +++ b/lib/plausible/stats/imported/sql/where_builder.ex @@ -49,7 +49,7 @@ defmodule Plausible.Stats.Imported.SQL.WhereBuilder do |> Enum.reduce(fn condition, acc -> dynamic([], ^acc or ^condition) end) end - defp add_filter(query, [_operation, dimension, _clauses] = filter) do + defp add_filter(query, [_operation, dimension, _clauses | _rest] = filter) do db_field = Plausible.Stats.Filters.without_prefix(dimension) if db_field == :goal do diff --git a/lib/plausible/stats/sql/special_metrics.ex b/lib/plausible/stats/sql/special_metrics.ex index 1bf77d50b6b1..e6b3aeba9065 100644 --- a/lib/plausible/stats/sql/special_metrics.ex +++ b/lib/plausible/stats/sql/special_metrics.ex @@ -56,7 +56,8 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do |> remove_filters_ignored_in_totals_query() |> Query.set( dimensions: [], - include_imported: query.include_imported + include_imported: query.include_imported, + preloaded_goals: [] ) q @@ -98,7 +99,8 @@ defmodule Plausible.Stats.SQL.SpecialMetrics do |> Query.set( metrics: [:visitors], order_by: [], - include_imported: query.include_imported + include_imported: query.include_imported, + preloaded_goals: [] ) from(e in subquery(q), diff --git a/lib/plausible/stats/sql/where_builder.ex b/lib/plausible/stats/sql/where_builder.ex index 2463a889e230..443983521a4e 100644 --- a/lib/plausible/stats/sql/where_builder.ex +++ b/lib/plausible/stats/sql/where_builder.ex @@ -93,11 +93,11 @@ defmodule Plausible.Stats.SQL.WhereBuilder do |> Enum.reduce(fn condition, acc -> dynamic([], ^acc or ^condition) end) end - defp add_filter(:events, _query, [:is, "event:name", list]) do - dynamic([e], e.name in ^list) + defp add_filter(:events, _query, [:is, "event:name" | _rest] = filter) do + in_clause(col_value(:name), filter) end - defp add_filter(:events, query, [_, "event:goal", _] = filter) do + defp add_filter(:events, query, [_, "event:goal" | _rest] = filter) do Plausible.Goals.Filters.add_filter(query, filter) end @@ -154,43 +154,49 @@ defmodule Plausible.Stats.SQL.WhereBuilder do false end - defp filter_custom_prop(prop_name, column_name, [:is, _, clauses]) do + defp filter_custom_prop(prop_name, column_name, [:is, _, clauses | _rest] = filter) do none_value_included = Enum.member?(clauses, "(none)") + prop_value_expr = custom_prop_value(column_name, prop_name) dynamic( [t], - (has_key(t, column_name, ^prop_name) and get_by_key(t, column_name, ^prop_name) in ^clauses) or + (has_key(t, column_name, ^prop_name) and ^in_clause(prop_value_expr, filter)) or (^none_value_included and not has_key(t, column_name, ^prop_name)) ) end - defp filter_custom_prop(prop_name, column_name, [:is_not, _, clauses]) do + defp filter_custom_prop(prop_name, column_name, [:is_not, _, clauses | _rest] = filter) do none_value_included = Enum.member?(clauses, "(none)") + prop_value_expr = custom_prop_value(column_name, prop_name) dynamic( [t], (has_key(t, column_name, ^prop_name) and - get_by_key(t, column_name, ^prop_name) not in ^clauses) or + not (^in_clause(prop_value_expr, filter))) or (^none_value_included and has_key(t, column_name, ^prop_name) and - get_by_key(t, column_name, ^prop_name) not in ^clauses) or + not (^in_clause(prop_value_expr, filter))) or (not (^none_value_included) and not has_key(t, column_name, ^prop_name)) ) end - defp filter_custom_prop(prop_name, column_name, [:matches_wildcard, dimension, clauses]) do + defp filter_custom_prop(prop_name, column_name, [:matches_wildcard, dimension, clauses | rest]) do regexes = Enum.map(clauses, &page_regex/1) - filter_custom_prop(prop_name, column_name, [:matches, dimension, regexes]) + filter_custom_prop(prop_name, column_name, [:matches, dimension, regexes | rest]) end - defp filter_custom_prop(prop_name, column_name, [:matches_wildcard_not, dimension, clauses]) do + defp filter_custom_prop(prop_name, column_name, [ + :matches_wildcard_not, + dimension, + clauses | rest + ]) do regexes = Enum.map(clauses, &page_regex/1) - filter_custom_prop(prop_name, column_name, [:matches_not, dimension, regexes]) + filter_custom_prop(prop_name, column_name, [:matches_not, dimension, regexes | rest]) end - defp filter_custom_prop(prop_name, column_name, [:matches, _dimension, clauses]) do + defp filter_custom_prop(prop_name, column_name, [:matches, _dimension, clauses | _rest]) do dynamic( [t], has_key(t, column_name, ^prop_name) and @@ -202,7 +208,7 @@ defmodule Plausible.Stats.SQL.WhereBuilder do ) end - defp filter_custom_prop(prop_name, column_name, [:matches_not, _dimension, clauses]) do + defp filter_custom_prop(prop_name, column_name, [:matches_not, _dimension, clauses | _rest]) do dynamic( [t], has_key(t, column_name, ^prop_name) and @@ -214,31 +220,23 @@ defmodule Plausible.Stats.SQL.WhereBuilder do ) end - defp filter_custom_prop(prop_name, column_name, [:contains, _dimension, clauses]) do + defp filter_custom_prop(prop_name, column_name, [:contains | _rest] = filter) do dynamic( [t], has_key(t, column_name, ^prop_name) and - fragment( - "multiSearchAny(?, ?)", - get_by_key(t, column_name, ^prop_name), - ^clauses - ) + ^contains_clause(custom_prop_value(column_name, prop_name), filter) ) end - defp filter_custom_prop(prop_name, column_name, [:contains_not, _dimension, clauses]) do + defp filter_custom_prop(prop_name, column_name, [:contains_not | _] = filter) do dynamic( [t], has_key(t, column_name, ^prop_name) and - fragment( - "not(multiSearchAny(?, ?))", - get_by_key(t, column_name, ^prop_name), - ^clauses - ) + not (^contains_clause(custom_prop_value(column_name, prop_name), filter)) ) end - defp filter_field(db_field, [:matches_wildcard, _dimension, glob_exprs]) do + defp filter_field(db_field, [:matches_wildcard, _dimension, glob_exprs | _rest]) do page_regexes = Enum.map(glob_exprs, &page_regex/1) dynamic( @@ -247,36 +245,36 @@ defmodule Plausible.Stats.SQL.WhereBuilder do ) end - defp filter_field(db_field, [:matches_wildcard_not, dimension, clauses]) do - dynamic([], not (^filter_field(db_field, [:matches_wildcard, dimension, clauses]))) + defp filter_field(db_field, [:matches_wildcard_not | rest]) do + dynamic([], not (^filter_field(db_field, [:matches_wildcard | rest]))) end - defp filter_field(db_field, [:contains, _dimension, values]) do - dynamic([x], fragment("multiSearchAny(?, ?)", type(field(x, ^db_field), :string), ^values)) + defp filter_field(db_field, [:contains | _rest] = filter) do + contains_clause(col_value_string(db_field), filter) end - defp filter_field(db_field, [:contains_not, dimension, clauses]) do - dynamic([], not (^filter_field(db_field, [:contains, dimension, clauses]))) + defp filter_field(db_field, [:contains_not | rest]) do + dynamic([], not (^filter_field(db_field, [:contains | rest]))) end - defp filter_field(db_field, [:matches, _dimension, clauses]) do + defp filter_field(db_field, [:matches, _dimension, clauses | _rest]) do dynamic( [x], fragment("multiMatchAny(?, ?)", type(field(x, ^db_field), :string), ^clauses) ) end - defp filter_field(db_field, [:matches_not, dimension, clauses]) do - dynamic([], not (^filter_field(db_field, [:matches, dimension, clauses]))) + defp filter_field(db_field, [:matches_not | rest]) do + dynamic([], not (^filter_field(db_field, [:matches | rest]))) end - defp filter_field(db_field, [:is, _dimension, clauses]) do - list = Enum.map(clauses, &db_field_val(db_field, &1)) - dynamic([x], field(x, ^db_field) in ^list) + defp filter_field(db_field, [:is, _dimension, clauses | _rest] = filter) do + list = clauses |> Enum.map(&db_field_val(db_field, &1)) + in_clause(col_value(db_field), filter, list) end - defp filter_field(db_field, [:is_not, dimension, clauses]) do - dynamic([], not (^filter_field(db_field, [:is, dimension, clauses]))) + defp filter_field(db_field, [:is_not | rest]) do + dynamic([], not (^filter_field(db_field, [:is | rest]))) end @no_ref "Direct / None" @@ -294,4 +292,45 @@ defmodule Plausible.Stats.SQL.WhereBuilder do defp db_field_val(:utm_term, @no_ref), do: "" defp db_field_val(_, @not_set), do: "" defp db_field_val(_, val), do: val + + defp col_value(column_name) do + dynamic([t], field(t, ^column_name)) + end + + # Needed for string functions to work properly + defp col_value_string(column_name) do + dynamic([t], type(field(t, ^column_name), :string)) + end + + defp custom_prop_value(column_name, prop_name) do + dynamic([t], get_by_key(t, column_name, ^prop_name)) + end + + defp in_clause(value_expression, [_, _, clauses | _] = filter, values \\ nil) do + values = values || clauses + + if case_sensitive?(filter) do + dynamic([t], ^value_expression in ^values) + else + values = values |> Enum.map(&String.downcase/1) + dynamic([t], fragment("lower(?)", ^value_expression) in ^values) + end + end + + defp contains_clause(value_expression, [_, _, clauses | _] = filter) do + if case_sensitive?(filter) do + dynamic( + [x], + fragment("multiSearchAny(?, ?)", ^value_expression, ^clauses) + ) + else + dynamic( + [x], + fragment("multiSearchAnyCaseInsensitive(?, ?)", ^value_expression, ^clauses) + ) + end + end + + defp case_sensitive?([_, _, _, %{case_sensitive: false}]), do: false + defp case_sensitive?(_), do: true end diff --git a/lib/plausible_web/controllers/api/external_stats_controller.ex b/lib/plausible_web/controllers/api/external_stats_controller.ex index 0adebbd506d0..8e121b817557 100644 --- a/lib/plausible_web/controllers/api/external_stats_controller.ex +++ b/lib/plausible_web/controllers/api/external_stats_controller.ex @@ -351,7 +351,7 @@ defmodule PlausibleWeb.Api.ExternalStatsController do end) end - defp validate_filter(site, [_type, "event:goal", goal_filter]) do + defp validate_filter(site, [_type, "event:goal", goal_filter | _rest]) do configured_goals = site |> Plausible.Goals.for_site() diff --git a/priv/json-schemas/query-api-schema.json b/priv/json-schemas/query-api-schema.json index e715c5957184..593544d1ecc7 100644 --- a/priv/json-schemas/query-api-schema.json +++ b/priv/json-schemas/query-api-schema.json @@ -341,17 +341,22 @@ "enum": ["matches_wildcard", "matches_wildcard_not"], "description": "filter operation" }, + "filter_operation_regex": { + "type": "string", + "enum": ["matches", "matches_not"], + "description": "filter operation" + }, "filter_operation_without_goals": { "type": "string", - "enum": ["is_not", "contains_not", "matches", "matches_not"], + "enum": ["is_not", "contains_not"], "description": "filter operation" }, - "filter_operation_with_goals": { + "filter_operation_contains": { "type": "string", "enum": ["is", "contains"], "description": "filter operation" }, - "filter_without_goals": { + "filter_with_pattern": { "type": "array", "additionalItems": false, "minItems": 3, @@ -359,7 +364,7 @@ "items": [ { "oneOf": [ - { "$ref": "#/definitions/filter_operation_without_goals" }, + { "$ref": "#/definitions/filter_operation_regex" }, { "$ref": "#/definitions/filter_operation_wildcard", "$comment": "only :internal" @@ -375,14 +380,40 @@ { "$ref": "#/definitions/clauses" } ] }, + "filter_without_goals": { + "type": "array", + "additionalItems": false, + "minItems": 3, + "maxItems": 4, + "items": [ + { "$ref": "#/definitions/filter_operation_without_goals" }, + { + "oneOf": [ + { "$ref": "#/definitions/simple_filter_dimensions" }, + { "$ref": "#/definitions/custom_property_filter_dimensions" } + ] + }, + { "$ref": "#/definitions/clauses" }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "case_sensitive": { + "type": "boolean", + "default": true + } + } + } + ] + }, "filter_with_goals": { "type": "array", "additionalItems": false, "minItems": 3, - "maxItems": 3, + "maxItems": 4, "items": [ { - "$ref": "#/definitions/filter_operation_with_goals" + "$ref": "#/definitions/filter_operation_contains" }, { "oneOf": [ @@ -393,13 +424,24 @@ }, { "$ref": "#/definitions/clauses" + }, + { + "type": "object", + "additionalProperties": false, + "properties": { + "case_sensitive": { + "type": "boolean", + "default": true + } + } } ] }, "filter_entry": { "oneOf": [ { "$ref": "#/definitions/filter_without_goals" }, - { "$ref": "#/definitions/filter_with_goals" } + { "$ref": "#/definitions/filter_with_goals" }, + { "$ref": "#/definitions/filter_with_pattern" } ] }, "filter_tree": { diff --git a/test/plausible/stats/query_parser_test.exs b/test/plausible/stats/query_parser_test.exs index 53e998d23a99..e59dd28ddfa5 100644 --- a/test/plausible/stats/query_parser_test.exs +++ b/test/plausible/stats/query_parser_test.exs @@ -598,6 +598,156 @@ defmodule Plausible.Stats.Filters.QueryParserTest do "Invalid filters. Dimension `event:hostname` can only be filtered at the top level." ) end + + for operation <- [:is, :contains, :is_not, :contains_not] do + test "#{operation} allows case_sensitive modifier", %{site: site} do + %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + [ + Atom.to_string(unquote(operation)), + "event:page", + ["/foo"], + %{"case_sensitive" => false} + ], + [ + Atom.to_string(unquote(operation)), + "event:name", + ["/foo"], + %{"case_sensitive" => true} + ] + ] + } + |> check_success(site, %{ + metrics: [:visitors], + utc_time_range: @date_range_day, + filters: [ + [unquote(operation), "event:page", ["/foo"], %{case_sensitive: false}], + [unquote(operation), "event:name", ["/foo"], %{case_sensitive: true}] + ], + dimensions: [], + order_by: nil, + timezone: site.timezone, + include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil}, + pagination: %{limit: 10_000, offset: 0} + }) + end + end + + for operation <- [:matches, :matches_not, :matches_wildcard, :matches_wildcard_not] do + test "case_sensitive modifier is not valid for #{operation}", %{site: site} do + %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + [ + Atom.to_string(unquote(operation)), + "event:hostname", + ["a.plausible.io"], + %{"case_sensitive" => false} + ] + ] + } + |> check_error( + site, + "#/filters/0: Invalid filter [\"#{unquote(operation)}\", \"event:hostname\", [\"a.plausible.io\"], %{\"case_sensitive\" => false}]", + :internal + ) + end + end + end + + describe "preloading goals" do + setup %{site: site} do + insert(:goal, %{site: site, event_name: "Signup"}) + insert(:goal, %{site: site, event_name: "Purchase"}) + insert(:goal, %{site: site, event_name: "Contact"}) + + :ok + end + + test "with exact match", %{site: site} do + %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "event:goal", ["Signup", "Purchase"]]] + } + |> check_success(site, %{ + metrics: [:visitors], + utc_time_range: @date_range_day, + filters: [[:is, "event:goal", ["Signup", "Purchase"]]], + dimensions: [], + order_by: nil, + timezone: site.timezone, + include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil}, + pagination: %{limit: 10_000, offset: 0} + }) + |> check_goals(preloaded_goals: ["Purchase", "Signup"], revenue_currencies: %{}) + end + + test "with case insensitive match", %{site: site} do + %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["is", "event:goal", ["signup", "purchase"], %{"case_sensitive" => false}]] + } + |> check_success(site, %{ + metrics: [:visitors], + utc_time_range: @date_range_day, + filters: [[:is, "event:goal", ["signup", "purchase"], %{case_sensitive: false}]], + dimensions: [], + order_by: nil, + timezone: site.timezone, + include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil}, + pagination: %{limit: 10_000, offset: 0} + }) + |> check_goals(preloaded_goals: ["Purchase", "Signup"], revenue_currencies: %{}) + end + + test "with contains match", %{site: site} do + %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["contains", "event:goal", ["Sign", "pur"]]] + } + |> check_success(site, %{ + metrics: [:visitors], + utc_time_range: @date_range_day, + filters: [[:contains, "event:goal", ["Sign", "pur"]]], + dimensions: [], + order_by: nil, + timezone: site.timezone, + include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil}, + pagination: %{limit: 10_000, offset: 0} + }) + |> check_goals(preloaded_goals: ["Signup"], revenue_currencies: %{}) + end + + test "with case insensitive contains match", %{site: site} do + %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [["contains", "event:goal", ["sign", "CONT"], %{"case_sensitive" => false}]] + } + |> check_success(site, %{ + metrics: [:visitors], + utc_time_range: @date_range_day, + filters: [[:contains, "event:goal", ["sign", "CONT"], %{case_sensitive: false}]], + dimensions: [], + order_by: nil, + timezone: site.timezone, + include: %{imports: false, time_labels: false, total_rows: false, comparisons: nil}, + pagination: %{limit: 10_000, offset: 0} + }) + |> check_goals(preloaded_goals: ["Contact", "Signup"], revenue_currencies: %{}) + end end describe "include validation" do diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs index 3c7c1a41c37f..cd887fb7418a 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/query_imported_test.exs @@ -34,6 +34,301 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryImportedTest do assert json_response(conn2, 200)["meta"]["imports_included"] refute json_response(conn2, 200)["meta"]["imports_warning"] end + + test "filters correctly with 'is' operator", %{ + conn: conn, + site: site, + site_import: site_import + } do + populate_stats(site, site_import.id, [ + build(:pageview, pathname: "/blog", timestamp: ~N[2023-01-01 00:00:00]), + build(:pageview, pathname: "/blog", timestamp: ~N[2023-01-01 00:00:00]), + build(:pageview, pathname: "/blog/post/1", timestamp: ~N[2023-01-01 00:00:00]), + build(:pageview, pathname: "/about", timestamp: ~N[2023-01-01 00:00:00]), + build(:imported_pages, + page: "/blog", + pageviews: 5, + visitors: 3, + date: ~D[2023-01-01] + ), + build(:imported_pages, + page: "/blog/post/1", + pageviews: 2, + visitors: 2, + date: ~D[2023-01-01] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "pageviews"], + "filters" => [ + ["is", "event:page", ["/blog"]] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [5, 7], "dimensions" => []} + ] + end + + test "filters correctly with 'is' operator (case insensitive)", %{ + conn: conn, + site: site, + site_import: site_import + } do + populate_stats(site, site_import.id, [ + build(:pageview, pathname: "/BLOG", timestamp: ~N[2023-01-01 00:00:00]), + build(:pageview, pathname: "/blog", timestamp: ~N[2023-01-01 00:00:00]), + build(:pageview, pathname: "/blog/post/1", timestamp: ~N[2023-01-01 00:00:00]), + build(:pageview, pathname: "/about", timestamp: ~N[2023-01-01 00:00:00]), + build(:imported_pages, + page: "/BLOG", + pageviews: 5, + visitors: 3, + date: ~D[2023-01-01] + ), + build(:imported_pages, + page: "/blog/post/1", + pageviews: 2, + visitors: 2, + date: ~D[2023-01-01] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "pageviews"], + "filters" => [ + ["is", "event:page", ["/blOG"], %{"case_sensitive" => false}] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [5, 7], "dimensions" => []} + ] + end + + test "filters correctly with 'contains' operator", %{ + conn: conn, + site: site, + site_import: site_import + } do + populate_stats(site, site_import.id, [ + build(:pageview, pathname: "/blog", timestamp: ~N[2023-01-01 00:00:00]), + build(:pageview, pathname: "/blog/post/1", timestamp: ~N[2023-01-01 00:00:00]), + build(:pageview, pathname: "/blog/post/2", timestamp: ~N[2023-01-01 00:00:00]), + build(:pageview, pathname: "/about", timestamp: ~N[2023-01-01 00:00:00]), + build(:imported_pages, + page: "/blog", + pageviews: 5, + visitors: 3, + date: ~D[2023-01-01] + ), + build(:imported_pages, + page: "/blog/post/1", + pageviews: 2, + visitors: 2, + date: ~D[2023-01-01] + ), + build(:imported_pages, + page: "/blog/POST/2", + pageviews: 3, + visitors: 1, + date: ~D[2023-01-01] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "pageviews"], + "filters" => [ + ["contains", "event:page", ["blog/post"]] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [4, 4], "dimensions" => []} + ] + end + + test "filters correctly with 'contains' operator (case insensitive)", %{ + conn: conn, + site: site, + site_import: site_import + } do + populate_stats(site, site_import.id, [ + build(:pageview, pathname: "/BLOG/post/1", timestamp: ~N[2023-01-01 00:00:00]), + build(:pageview, pathname: "/blog/POST/2", timestamp: ~N[2023-01-01 00:00:00]), + build(:pageview, pathname: "/about", timestamp: ~N[2023-01-01 00:00:00]), + build(:imported_pages, + page: "/BLOG/POST/1", + pageviews: 5, + visitors: 3, + date: ~D[2023-01-01] + ), + build(:imported_pages, + page: "/blog/post/2", + pageviews: 2, + visitors: 2, + date: ~D[2023-01-01] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "pageviews"], + "filters" => [ + ["contains", "event:page", ["blog/POST"], %{"case_sensitive" => false}] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [7, 9], "dimensions" => []} + ] + end + + test "aggregates custom event goals with 'is' and 'contains' operators", %{ + conn: conn, + site: site, + site_import: site_import + } do + insert(:goal, event_name: "Purchase", site: site) + + populate_stats(site, site_import.id, [ + build(:event, + name: "Purchase", + timestamp: ~N[2023-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + timestamp: ~N[2023-01-01 00:00:00] + ), + build(:event, + name: "Signup", + timestamp: ~N[2023-01-01 00:00:00] + ), + build(:imported_custom_events, + name: "Purchase", + visitors: 3, + events: 5, + date: ~D[2023-01-01] + ), + build(:imported_custom_events, + name: "Signup", + visitors: 2, + events: 2, + date: ~D[2023-01-01] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "events"], + "filters" => [ + ["is", "event:goal", ["Purchase"]] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => [], "metrics" => [5, 7]} + ] + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "events"], + "filters" => [ + ["contains", "event:goal", ["Purch", "sign"]] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => [], "metrics" => [5, 7]} + ] + end + + test "aggregates custom event goals with 'is' and 'contains' operators (case insensitive)", %{ + conn: conn, + site: site, + site_import: site_import + } do + insert(:goal, event_name: "Purchase", site: site) + + populate_stats(site, site_import.id, [ + build(:event, + name: "Purchase", + timestamp: ~N[2023-01-01 00:00:00] + ), + build(:event, + name: "Purchase", + timestamp: ~N[2023-01-01 00:00:00] + ), + build(:event, + name: "Signup", + timestamp: ~N[2023-01-01 00:00:00] + ), + build(:imported_custom_events, + name: "Purchase", + visitors: 3, + events: 5, + date: ~D[2023-01-01] + ), + build(:imported_custom_events, + name: "Signup", + visitors: 2, + events: 2, + date: ~D[2023-01-01] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "events"], + "filters" => [ + ["is", "event:goal", ["purchase"], %{"case_sensitive" => false}] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => [], "metrics" => [5, 7]} + ] + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "events"], + "filters" => [ + ["contains", "event:goal", ["PURCH"], %{"case_sensitive" => false}] + ], + "include" => %{"imports" => true} + }) + + assert json_response(conn, 200)["results"] == [ + %{"dimensions" => [], "metrics" => [5, 7]} + ] + end end test "breaks down all metrics by visit:referrer with imported data", %{conn: conn, site: site} do diff --git a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs index 087004aa72b6..fd06c558bdfd 100644 --- a/test/plausible_web/controllers/api/external_stats_controller/query_test.exs +++ b/test/plausible_web/controllers/api/external_stats_controller/query_test.exs @@ -271,6 +271,62 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do ] end + test "can filter by utm_medium case insensitively", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + utm_medium: "Social", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + utm_medium: "SOCIAL", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:utm_medium", ["sociaL"], %{"case_sensitive" => false}]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end + + test "can filter by is_not utm_medium case insensitively", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + utm_medium: "Social", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + utm_medium: "SOCIAL", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews"], + "filters" => [["is_not", "visit:utm_medium", ["sociaL"], %{"case_sensitive" => false}]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [1], "dimensions" => []} + ] + end + test "can filter by utm_source", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, @@ -621,6 +677,43 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}] end + test "filtering by a custom event goal (case insensitive)", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "Signup", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:event, + name: "NotConfigured", + timestamp: ~N[2021-01-01 00:25:00] + ) + ]) + + insert(:goal, %{site: site, event_name: "Signup"}) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors", "events"], + "filters" => [ + ["is", "event:goal", ["signup"], %{"case_sensitive" => false}] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2, 3], "dimensions" => []}] + end + test "filtering by a revenue goal", %{conn: conn, site: site} do populate_stats(site, [ build(:event, @@ -819,6 +912,46 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do assert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}] end + test "contains page filter case insensitive", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/en/page1"), + build(:pageview, pathname: "/EN/page2"), + build(:pageview, pathname: "/pl/page1") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "filters" => [ + ["contains", "event:page", ["/En/"], %{"case_sensitive" => false}] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [2], "dimensions" => []}] + end + + test "contains_not page filter case insensitive", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, pathname: "/en/page1"), + build(:pageview, pathname: "/EN/page2"), + build(:pageview, pathname: "/pl/page1") + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "filters" => [ + ["contains_not", "event:page", ["/En/"], %{"case_sensitive" => false}] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}] + end + test "contains_not page filter", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, pathname: "/en/page1"), @@ -1009,6 +1142,88 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do assert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}] end + test "`contains` operator with custom properties case insensitive", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + "meta.key": ["name"], + "meta.value": ["large-1"] + ), + build(:pageview, + "meta.key": ["name"], + "meta.value": ["Small-1"] + ), + build(:pageview, + "meta.key": ["name"], + "meta.value": ["mall-1"] + ), + build(:pageview, + "meta.key": ["name"], + "meta.value": ["SMALL-2"] + ), + build(:pageview, + "meta.key": ["name"], + "meta.value": ["small-2"] + ), + build(:pageview) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "filters" => [ + ["contains", "event:props:name", ["maLL"], %{"case_sensitive" => false}] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [4], "dimensions" => []}] + end + + test "`contains_not` operator with custom properties case insensitive", %{ + conn: conn, + site: site + } do + populate_stats(site, [ + build(:pageview, + "meta.key": ["name"], + "meta.value": ["large-1"] + ), + build(:pageview, + "meta.key": ["name"], + "meta.value": ["Small-1"] + ), + build(:pageview, + "meta.key": ["name"], + "meta.value": ["mall-1"] + ), + build(:pageview, + "meta.key": ["name"], + "meta.value": ["SMALL-2"] + ), + build(:pageview, + "meta.key": ["name"], + "meta.value": ["small-2"] + ), + build(:pageview) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["visitors"], + "filters" => [ + ["contains_not", "event:props:name", ["maLL"], %{"case_sensitive" => false}] + ] + }) + + assert json_response(conn, 200)["results"] == [%{"metrics" => [1], "dimensions" => []}] + end + test "`matches` and `matches_not` operator with custom properties", %{ conn: conn, site: site @@ -2587,6 +2802,39 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do ] end + test "goal contains filter for goal breakdown (case insensitive)", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, name: "Onboarding conversion: Step 1"), + build(:event, name: "Onboarding conversion: Step 1"), + build(:event, name: "Onboarding conversion: Step 2"), + build(:event, name: "Unrelated"), + build(:pageview, pathname: "/conversion") + ]) + + insert(:goal, site: site, event_name: "Onboarding conversion: Step 1") + insert(:goal, site: site, event_name: "Onboarding conversion: Step 2") + insert(:goal, site: site, event_name: "Unrelated") + insert(:goal, site: site, page_path: "/conversion") + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "dimensions" => ["event:goal"], + "filters" => [ + ["contains", "event:goal", ["step"], %{"case_sensitive" => false}] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => ["Onboarding conversion: Step 1"], "metrics" => [2]}, + %{"dimensions" => ["Onboarding conversion: Step 2"], "metrics" => [1]} + ] + end + test "mixed multi-goal filter for breakdown by visit:country", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, country_code: "EE", pathname: "/en/register"), @@ -2813,6 +3061,43 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do ] end + test "IN filter for event:name case insensitive", %{conn: conn, site: site} do + populate_stats(site, [ + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Signup", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Login", + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:event, + name: "Irrelevant", + timestamp: ~N[2021-01-01 00:00:00] + ) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "metrics" => ["visitors"], + "date_range" => "all", + "filters" => [ + ["is", "event:name", ["signup", "LOGIN"], %{"case_sensitive" => false}] + ] + }) + + %{"results" => results} = json_response(conn, 200) + + assert results == [ + %{"dimensions" => [], "metrics" => [3]} + ] + end + test "IN filter for event:props:*", %{conn: conn, site: site} do populate_stats(site, [ build(:pageview, @@ -2880,6 +3165,12 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do "meta.value": ["Safari", "target_value"], timestamp: ~N[2021-01-01 00:00:00] ), + build(:pageview, + browser: "Chrome", + "meta.key": ["browser", "prop"], + "meta.value": ["Chrome", "target_value"], + timestamp: ~N[2021-01-01 00:00:00] + ), build(:pageview, browser: "Firefox", "meta.key": ["browser", "prop"], @@ -2895,7 +3186,9 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do "date_range" => "all", "dimensions" => ["visit:browser"], "filters" => [ - ["is", "event:props:browser", ["Chrome", "Safari"]], + ["is", "event:props:browser", ["CHROME", "sAFari"], %{"case_sensitive" => false}], + # Negate a previously set filter + ["is_not", "event:props:browser", ["Chrome"], %{"case_sensitive" => false}], ["is", "event:props:prop", ["target_value"]] ] }) @@ -3720,4 +4013,32 @@ defmodule PlausibleWeb.Api.ExternalStatsController.QueryTest do ] end end + + test "can filter by utm_medium case insensitively", %{conn: conn, site: site} do + populate_stats(site, [ + build(:pageview, + utm_medium: "Social", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:00:00] + ), + build(:pageview, + utm_medium: "SOCIAL", + user_id: @user_id, + timestamp: ~N[2021-01-01 00:25:00] + ), + build(:pageview, timestamp: ~N[2021-01-01 00:00:00]) + ]) + + conn = + post(conn, "/api/v2/query", %{ + "site_id" => site.domain, + "date_range" => "all", + "metrics" => ["pageviews", "visitors", "bounce_rate", "visit_duration"], + "filters" => [["is", "visit:utm_medium", ["social"], %{"case_sensitive" => false}]] + }) + + assert json_response(conn, 200)["results"] == [ + %{"metrics" => [2, 1, 0, 1500], "dimensions" => []} + ] + end end