Skip to content

Commit 60eaaaf

Browse files
mikecaomikecao
authored
Merge pull request umami-software#3528 from umami-software/analytics
v2.19.0
2 parents abc0246 + 827c107 commit 60eaaaf

File tree

107 files changed

+4496
-2958
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

107 files changed

+4496
-2958
lines changed

‎cypress/e2e/api-website.cy.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
import { uuid } from '../../src/lib/crypto';
2+
13
describe('Website API tests', () => {
24
Cypress.session.clearAllSavedSessions();
35

@@ -65,6 +67,37 @@ describe('Website API tests', () => {
6567
});
6668
});
6769

70+
it('Creates a website with a fixed ID.', () => {
71+
cy.fixture('websites').then(data => {
72+
const websiteCreate = data.websiteCreate;
73+
const fixedId = uuid();
74+
cy.request({
75+
method: 'POST',
76+
url: '/api/websites',
77+
headers: {
78+
'Content-Type': 'application/json',
79+
Authorization: Cypress.env('authorization'),
80+
},
81+
body: { ...websiteCreate, id: fixedId },
82+
}).then(response => {
83+
expect(response.status).to.eq(200);
84+
expect(response.body).to.have.property('id', fixedId);
85+
expect(response.body).to.have.property('name', 'Cypress Website');
86+
expect(response.body).to.have.property('domain', 'cypress.com');
87+
88+
// cleanup
89+
cy.request({
90+
method: 'DELETE',
91+
url: `/api/websites/${fixedId}`,
92+
headers: {
93+
'Content-Type': 'application/json',
94+
Authorization: Cypress.env('authorization'),
95+
},
96+
});
97+
});
98+
});
99+
});
100+
68101
it('Returns all tracked websites.', () => {
69102
cy.request({
70103
method: 'GET',

‎db/clickhouse/migrations/08_update_hostname_view.sql‎

Lines changed: 253 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,253 @@
1+
-- create new hourly table
2+
CREATE TABLE umami.website_event_stats_hourly_new
3+
(
4+
website_id UUID,
5+
session_id UUID,
6+
visit_id UUID,
7+
hostname SimpleAggregateFunction(groupArrayArray, Array(String)),
8+
browser LowCardinality(String),
9+
os LowCardinality(String),
10+
device LowCardinality(String),
11+
screen LowCardinality(String),
12+
language LowCardinality(String),
13+
country LowCardinality(String),
14+
region LowCardinality(String),
15+
city String,
16+
entry_url AggregateFunction(argMin, String, DateTime('UTC')),
17+
exit_url AggregateFunction(argMax, String, DateTime('UTC')),
18+
url_path SimpleAggregateFunction(groupArrayArray, Array(String)),
19+
url_query SimpleAggregateFunction(groupArrayArray, Array(String)),
20+
utm_source SimpleAggregateFunction(groupArrayArray, Array(String)),
21+
utm_medium SimpleAggregateFunction(groupArrayArray, Array(String)),
22+
utm_campaign SimpleAggregateFunction(groupArrayArray, Array(String)),
23+
utm_content SimpleAggregateFunction(groupArrayArray, Array(String)),
24+
utm_term SimpleAggregateFunction(groupArrayArray, Array(String)),
25+
referrer_domain SimpleAggregateFunction(groupArrayArray, Array(String)),
26+
page_title SimpleAggregateFunction(groupArrayArray, Array(String)),
27+
gclid SimpleAggregateFunction(groupArrayArray, Array(String)),
28+
fbclid SimpleAggregateFunction(groupArrayArray, Array(String)),
29+
msclkid SimpleAggregateFunction(groupArrayArray, Array(String)),
30+
ttclid SimpleAggregateFunction(groupArrayArray, Array(String)),
31+
li_fat_id SimpleAggregateFunction(groupArrayArray, Array(String)),
32+
twclid SimpleAggregateFunction(groupArrayArray, Array(String)),
33+
event_type UInt32,
34+
event_name SimpleAggregateFunction(groupArrayArray, Array(String)),
35+
views SimpleAggregateFunction(sum, UInt64),
36+
min_time SimpleAggregateFunction(min, DateTime('UTC')),
37+
max_time SimpleAggregateFunction(max, DateTime('UTC')),
38+
tag SimpleAggregateFunction(groupArrayArray, Array(String)),
39+
distinct_id String,
40+
created_at Datetime('UTC')
41+
)
42+
ENGINE = AggregatingMergeTree
43+
PARTITION BY toYYYYMM(created_at)
44+
ORDER BY (
45+
website_id,
46+
event_type,
47+
toStartOfHour(created_at),
48+
cityHash64(visit_id),
49+
visit_id
50+
)
51+
SAMPLE BY cityHash64(visit_id);
52+
53+
-- create view
54+
CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv_new
55+
TO umami.website_event_stats_hourly_new
56+
AS
57+
SELECT
58+
website_id,
59+
session_id,
60+
visit_id,
61+
hostnames as hostname,
62+
browser,
63+
os,
64+
device,
65+
screen,
66+
language,
67+
country,
68+
region,
69+
city,
70+
entry_url,
71+
exit_url,
72+
url_paths as url_path,
73+
url_query,
74+
utm_source,
75+
utm_medium,
76+
utm_campaign,
77+
utm_content,
78+
utm_term,
79+
referrer_domain,
80+
page_title,
81+
gclid,
82+
fbclid,
83+
msclkid,
84+
ttclid,
85+
li_fat_id,
86+
twclid,
87+
event_type,
88+
event_name,
89+
views,
90+
min_time,
91+
max_time,
92+
tag,
93+
distinct_id,
94+
timestamp as created_at
95+
FROM (SELECT
96+
website_id,
97+
session_id,
98+
visit_id,
99+
arrayFilter(x -> x != '', groupArray(hostname)) hostnames,
100+
browser,
101+
os,
102+
device,
103+
screen,
104+
language,
105+
country,
106+
region,
107+
city,
108+
argMinState(url_path, created_at) entry_url,
109+
argMaxState(url_path, created_at) exit_url,
110+
arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
111+
arrayFilter(x -> x != '', groupArray(url_query)) url_query,
112+
arrayFilter(x -> x != '', groupArray(utm_source)) utm_source,
113+
arrayFilter(x -> x != '', groupArray(utm_medium)) utm_medium,
114+
arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
115+
arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
116+
arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
117+
arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,
118+
arrayFilter(x -> x != '', groupArray(page_title)) page_title,
119+
arrayFilter(x -> x != '', groupArray(gclid)) gclid,
120+
arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
121+
arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,
122+
arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,
123+
arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,
124+
arrayFilter(x -> x != '', groupArray(twclid)) twclid,
125+
event_type,
126+
if(event_type = 2, groupArray(event_name), []) event_name,
127+
sumIf(1, event_type = 1) views,
128+
min(created_at) min_time,
129+
max(created_at) max_time,
130+
arrayFilter(x -> x != '', groupArray(tag)) tag,
131+
distinct_id,
132+
toStartOfHour(created_at) timestamp
133+
FROM umami.website_event
134+
GROUP BY website_id,
135+
session_id,
136+
visit_id,
137+
hostname,
138+
browser,
139+
os,
140+
device,
141+
screen,
142+
language,
143+
country,
144+
region,
145+
city,
146+
event_type,
147+
distinct_id,
148+
timestamp);
149+
150+
-- rename tables
151+
RENAME TABLE umami.website_event_stats_hourly TO umami.website_event_stats_hourly_old;
152+
RENAME TABLE umami.website_event_stats_hourly_new TO umami.website_event_stats_hourly;
153+
154+
-- drop views
155+
DROP TABLE umami.website_event_stats_hourly_mv;
156+
DROP TABLE umami.website_event_stats_hourly_mv_new;
157+
158+
-- recreate view
159+
CREATE MATERIALIZED VIEW umami.website_event_stats_hourly_mv
160+
TO umami.website_event_stats_hourly
161+
AS
162+
SELECT
163+
website_id,
164+
session_id,
165+
visit_id,
166+
hostnames as hostname,
167+
browser,
168+
os,
169+
device,
170+
screen,
171+
language,
172+
country,
173+
region,
174+
city,
175+
entry_url,
176+
exit_url,
177+
url_paths as url_path,
178+
url_query,
179+
utm_source,
180+
utm_medium,
181+
utm_campaign,
182+
utm_content,
183+
utm_term,
184+
referrer_domain,
185+
page_title,
186+
gclid,
187+
fbclid,
188+
msclkid,
189+
ttclid,
190+
li_fat_id,
191+
twclid,
192+
event_type,
193+
event_name,
194+
views,
195+
min_time,
196+
max_time,
197+
tag,
198+
distinct_id,
199+
timestamp as created_at
200+
FROM (SELECT
201+
website_id,
202+
session_id,
203+
visit_id,
204+
arrayFilter(x -> x != '', groupArray(hostname)) hostnames,
205+
browser,
206+
os,
207+
device,
208+
screen,
209+
language,
210+
country,
211+
region,
212+
city,
213+
argMinState(url_path, created_at) entry_url,
214+
argMaxState(url_path, created_at) exit_url,
215+
arrayFilter(x -> x != '', groupArray(url_path)) as url_paths,
216+
arrayFilter(x -> x != '', groupArray(url_query)) url_query,
217+
arrayFilter(x -> x != '', groupArray(utm_source)) utm_source,
218+
arrayFilter(x -> x != '', groupArray(utm_medium)) utm_medium,
219+
arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
220+
arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
221+
arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
222+
arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,
223+
arrayFilter(x -> x != '', groupArray(page_title)) page_title,
224+
arrayFilter(x -> x != '', groupArray(gclid)) gclid,
225+
arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
226+
arrayFilter(x -> x != '', groupArray(msclkid)) msclkid,
227+
arrayFilter(x -> x != '', groupArray(ttclid)) ttclid,
228+
arrayFilter(x -> x != '', groupArray(li_fat_id)) li_fat_id,
229+
arrayFilter(x -> x != '', groupArray(twclid)) twclid,
230+
event_type,
231+
if(event_type = 2, groupArray(event_name), []) event_name,
232+
sumIf(1, event_type = 1) views,
233+
min(created_at) min_time,
234+
max(created_at) max_time,
235+
arrayFilter(x -> x != '', groupArray(tag)) tag,
236+
distinct_id,
237+
toStartOfHour(created_at) timestamp
238+
FROM umami.website_event
239+
GROUP BY website_id,
240+
session_id,
241+
visit_id,
242+
hostname,
243+
browser,
244+
os,
245+
device,
246+
screen,
247+
language,
248+
country,
249+
region,
250+
city,
251+
event_type,
252+
distinct_id,
253+
timestamp);

‎db/clickhouse/schema.sql‎

Lines changed: 39 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@ CREATE TABLE umami.website_event_stats_hourly
9090
website_id UUID,
9191
session_id UUID,
9292
visit_id UUID,
93-
hostname LowCardinality(String),
93+
hostname SimpleAggregateFunction(groupArrayArray, Array(String)),
9494
browser LowCardinality(String),
9595
os LowCardinality(String),
9696
device LowCardinality(String),
@@ -143,7 +143,7 @@ SELECT
143143
website_id,
144144
session_id,
145145
visit_id,
146-
hostname,
146+
hostnames as hostname,
147147
browser,
148148
os,
149149
device,
@@ -181,7 +181,7 @@ FROM (SELECT
181181
website_id,
182182
session_id,
183183
visit_id,
184-
hostname,
184+
arrayFilter(x -> x != '', groupArray(hostname)) hostnames,
185185
browser,
186186
os,
187187
device,
@@ -199,7 +199,7 @@ FROM (SELECT
199199
arrayFilter(x -> x != '', groupArray(utm_campaign)) utm_campaign,
200200
arrayFilter(x -> x != '', groupArray(utm_content)) utm_content,
201201
arrayFilter(x -> x != '', groupArray(utm_term)) utm_term,
202-
arrayFilter(x -> x != '', groupArray(referrer_domain)) referrer_domain,
202+
arrayFilter(x -> x != '' and x != hostname, groupArray(referrer_domain)) referrer_domain,
203203
arrayFilter(x -> x != '', groupArray(page_title)) page_title,
204204
arrayFilter(x -> x != '', groupArray(gclid)) gclid,
205205
arrayFilter(x -> x != '', groupArray(fbclid)) fbclid,
@@ -246,3 +246,38 @@ SELECT * ORDER BY toStartOfDay(created_at), website_id, referrer_domain, created
246246
);
247247

248248
ALTER TABLE umami.website_event MATERIALIZE PROJECTION website_event_referrer_domain_projection;
249+
250+
-- revenue
251+
CREATE TABLE umami.website_revenue
252+
(
253+
website_id UUID,
254+
session_id UUID,
255+
event_id UUID,
256+
event_name String,
257+
currency String,
258+
revenue DECIMAL(18,4),
259+
created_at DateTime('UTC')
260+
)
261+
ENGINE = MergeTree
262+
PARTITION BY toYYYYMM(created_at)
263+
ORDER BY (website_id, session_id, created_at)
264+
SETTINGS index_granularity = 8192;
265+
266+
267+
CREATE MATERIALIZED VIEW umami.website_revenue_mv
268+
TO umami.website_revenue
269+
AS
270+
SELECT DISTINCT
271+
ed.website_id,
272+
ed.session_id,
273+
ed.event_id,
274+
ed.event_name,
275+
c.currency,
276+
coalesce(toDecimal64(ed.number_value, 2), toDecimal64(ed.string_value, 2)) revenue,
277+
ed.created_at
278+
FROM umami.event_data ed
279+
JOIN (SELECT event_id, string_value as currency
280+
FROM umami.event_data
281+
WHERE positionCaseInsensitive(data_key, 'currency') > 0) c
282+
ON c.event_id = ed.event_id
283+
WHERE positionCaseInsensitive(data_key, 'revenue') > 0;

‎db/mysql/migrations/11_add_segment/migration.sql‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
-- CreateTable
2+
CREATE TABLE `segment` (
3+
`segment_id` VARCHAR(36) NOT NULL,
4+
`website_id` VARCHAR(36) NOT NULL,
5+
`type` VARCHAR(200) NOT NULL,
6+
`name` VARCHAR(200) NOT NULL,
7+
`parameters` JSON NOT NULL,
8+
`created_at` TIMESTAMP(0) NULL DEFAULT CURRENT_TIMESTAMP(0),
9+
`updated_at` TIMESTAMP(0) NULL,
10+
11+
UNIQUE INDEX `segment_segment_id_key`(`segment_id`),
12+
INDEX `segment_website_id_idx`(`website_id`),
13+
PRIMARY KEY (`segment_id`)
14+
) DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;

‎db/mysql/migrations/12_update_report_parameter/migration.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
-- AlterTable
2+
ALTER TABLE `report` MODIFY `parameters` JSON NOT NULL;

0 commit comments

Comments
 (0)