Skip to content

Commit 809b4f3

Browse files
author
Andy Hattemer
committed
update
1 parent d9a4b3c commit 809b4f3

File tree

10 files changed

+277
-7
lines changed

10 files changed

+277
-7
lines changed

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
cube/.cubestore
2+
cube/.env

‎compose.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ services:
2121
- MYSQL_USER=mysqluser
2222
- MYSQL_PASSWORD=mysqlpw
2323
volumes:
24-
- mysql/mysql.cnf:/etc/mysql/conf.d
25-
- mysql/mysql_bootstrap.sql:/docker-entrypoint-initdb.d/mysql_bootstrap.sql
24+
- ${PWD}/mysql/mysql.cnf:/etc/mysql/conf.d
25+
- ${PWD}/mysql/mysql_bootstrap.sql:/docker-entrypoint-initdb.d/mysql_bootstrap.sql
2626
healthcheck: {test: mysql -pdebezium -e 'select 1', interval: 1s, start_period: 60s}
2727
redpanda:
2828
image: docker.vectorized.io/vectorized/redpanda:v21.11.2
@@ -71,10 +71,10 @@ services:
7171
environment:
7272
KAFKA_ADDR: redpanda:9092
7373
volumes:
74-
- mysql/mysql_dbz.sh:/mysql_dbz.sh
74+
- ${PWD}/mysql/mysql_dbz.sh:/mysql_dbz.sh
7575
entrypoint: [bash, -c, /mysql_dbz.sh]
7676
loadgen:
77-
build: ../loadgen
77+
build: ${PWD}/loadgen
7878
init: true
7979
environment:
8080
KAFKA_ADDR: redpanda:9092

‎cube/cube.js‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
module.exports = {
2+
3+
};

‎cube/schema/.ItemSummary.js.swp‎

1 KB
Binary file not shown.

‎cube/schema/ItemSummary.js‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
cube(`ItemSummary`, {
2+
refreshKey: {
3+
every: `1 second`
4+
},
5+
sql: `SELECT * FROM public.item_summary`,
6+
measures: {
7+
count: {
8+
type: `count`,
9+
drillMembers: [CUBE.name]
10+
},
11+
totalRevenue: {
12+
type: `sum`,
13+
sql: `revenue`,
14+
format: `currency`
15+
},
16+
totalOrders: {
17+
type: `sum`,
18+
sql: `orders`
19+
},
20+
totalPageviews: {
21+
type: `sum`,
22+
sql: `pageviews`
23+
},
24+
totalItemsSold: {
25+
type: `sum`,
26+
sql: `items_sold`
27+
},
28+
conversionRate: {
29+
type: `number`,
30+
sql: `SUM(orders)/SUM(pageviews)`,
31+
format: `percent`
32+
},
33+
conversionRateLastHour: {
34+
type: `number`,
35+
sql: `SUM(last_hour_orders)/SUM(last_hour_pageviews)`,
36+
format: `percent`
37+
}
38+
},
39+
dimensions: {
40+
id: {
41+
sql: `item_id`,
42+
type: `number`
43+
},
44+
name: {
45+
sql: `name`,
46+
type: `string`
47+
},
48+
category: {
49+
sql: `category`,
50+
type: `string`
51+
}
52+
},
53+
dataSource: `default`,
54+
preAggregations: {
55+
main: {
56+
measures: [ItemSummary.count, ItemSummary.totalRevenue, ItemSummary.totalOrders, ItemSummary.totalPageviews, ItemSummary.totalItemsSold, ItemSummary.conversionRate, ItemSummary.conversionRateLastHour],
57+
dimensions: [ItemSummary.category, ItemSummary.id, ItemSummary.name]
58+
}
59+
}
60+
});

‎cube/schema/ProfileViewsEnriched.js‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
cube(`ProfileViewsEnriched`, {
2+
sql: `SELECT * FROM public.profile_views_enriched`,
3+
4+
preAggregations: {
5+
// Pre-Aggregations definitions go here
6+
// Learn more here: https://cube.dev/docs/caching/pre-aggregations/getting-started
7+
},
8+
9+
joins: {
10+
11+
},
12+
13+
measures: {
14+
count: {
15+
type: `count`,
16+
drillMembers: []
17+
}
18+
},
19+
20+
dimensions: {
21+
ownerEmail: {
22+
sql: `owner_email`,
23+
type: `string`
24+
},
25+
26+
viewerEmail: {
27+
sql: `viewer_email`,
28+
type: `string`
29+
}
30+
},
31+
32+
dataSource: `default`
33+
});

‎cube/schema/ProfileViewsPerMinuteLast10.js‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
cube(`ProfileViewsPerMinuteLast10`, {
2+
sql: `SELECT * FROM public.profile_views_per_minute_last_10`,
3+
4+
preAggregations: {
5+
// Pre-Aggregations definitions go here
6+
// Learn more here: https://cube.dev/docs/caching/pre-aggregations/getting-started
7+
},
8+
9+
joins: {
10+
11+
},
12+
13+
measures: {
14+
pageviews: {
15+
sql: `pageviews`,
16+
type: `sum`
17+
}
18+
},
19+
20+
dimensions: {
21+
user: {
22+
sql: `user_id`,
23+
type: `number`,
24+
},
25+
receivedAtMinute: {
26+
sql: `received_at_minute`,
27+
type: `time`
28+
}
29+
},
30+
31+
dataSource: `default`
32+
});

‎materialize/create.sql‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
-- CREATE SOURCES
2+
3+
CREATE SOURCE purchases
4+
FROM KAFKA BROKER 'redpanda:9092' TOPIC 'mysql.shop.purchases'
5+
FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://redpanda:8081'
6+
ENVELOPE DEBEZIUM;
7+
8+
CREATE SOURCE items
9+
FROM KAFKA BROKER 'redpanda:9092' TOPIC 'mysql.shop.items'
10+
FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://redpanda:8081'
11+
ENVELOPE DEBEZIUM;
12+
13+
CREATE SOURCE users
14+
FROM KAFKA BROKER 'redpanda:9092' TOPIC 'mysql.shop.users'
15+
FORMAT AVRO USING CONFLUENT SCHEMA REGISTRY 'http://redpanda:8081'
16+
ENVELOPE DEBEZIUM;
17+
18+
CREATE SOURCE json_pageviews
19+
FROM KAFKA BROKER 'redpanda:9092' TOPIC 'pageviews'
20+
FORMAT BYTES;
21+
22+
23+
24+
-- CREATE NON-MATERIALIZED VIEW TO PARSE JSON PAGEVIEWS
25+
26+
CREATE VIEW pageview_stg AS
27+
SELECT
28+
*,
29+
regexp_match(url, '/(products|profiles)/')[1] AS pageview_type,
30+
(regexp_match(url, '/(?:products|profiles)/(\d+)')[1])::INT AS target_id
31+
FROM (
32+
SELECT
33+
(data->'user_id')::INT AS user_id,
34+
data->>'url' AS url,
35+
data->>'channel' AS channel,
36+
(data->>'received_at')::double AS received_at
37+
FROM (
38+
SELECT CAST(data AS jsonb) AS data
39+
FROM (
40+
SELECT convert_from(data, 'utf8') AS data
41+
FROM json_pageviews
42+
)
43+
)
44+
);
45+
46+
47+
-- CREATE ANALYTICAL VIEWS
48+
49+
CREATE VIEW purchases_by_item AS
50+
SELECT
51+
item_id,
52+
SUM(purchase_price) as revenue,
53+
COUNT(id) AS orders,
54+
SUM(quantity) AS items_sold
55+
FROM purchases GROUP BY 1;
56+
57+
CREATE VIEW pageviews_by_item AS
58+
SELECT
59+
target_id as item_id,
60+
COUNT(*) AS pageviews
61+
FROM pageview_stg
62+
WHERE pageview_type = 'products'
63+
GROUP BY 1;
64+
65+
CREATE VIEW purchases_by_item_last_hour AS
66+
SELECT
67+
item_id,
68+
SUM(purchase_price) as revenue,
69+
COUNT(id) AS orders,
70+
SUM(quantity) AS items_sold
71+
FROM purchases
72+
WHERE mz_logical_timestamp() < (extract(epoch from created_at)*1000 + 3600000)::numeric
73+
GROUP BY 1;
74+
75+
CREATE VIEW pageviews_by_item_last_hour AS
76+
SELECT
77+
target_id as item_id,
78+
COUNT(*) AS pageviews
79+
FROM pageview_stg
80+
WHERE pageview_type = 'products' AND mz_logical_timestamp() < (received_at*1000 + 3600000)::numeric
81+
GROUP BY 1;
82+
83+
CREATE MATERIALIZED VIEW item_summary AS
84+
SELECT
85+
items.id as item_id,
86+
items.name,
87+
items.category,
88+
SUM(purchases_by_item.items_sold) as items_sold,
89+
SUM(purchases_by_item.orders) as orders,
90+
SUM(purchases_by_item.revenue) as revenue,
91+
SUM(pageviews_by_item.pageviews) as pageviews,
92+
SUM(purchases_by_item_last_hour.orders) as last_hour_orders,
93+
SUM(pageviews_by_item_last_hour.pageviews) as last_hour_pageviews
94+
FROM items
95+
JOIN purchases_by_item ON purchases_by_item.item_id = items.id
96+
JOIN pageviews_by_item ON pageviews_by_item.item_id = items.id
97+
JOIN purchases_by_item_last_hour ON purchases_by_item_last_hour.item_id = items.id
98+
JOIN pageviews_by_item_last_hour ON pageviews_by_item_last_hour.item_id = items.id
99+
GROUP BY 1, 2, 3;
100+
101+
102+
-- CREATE USER-FACING ANALYTICS VIEWS
103+
104+
CREATE MATERIALIZED VIEW profile_views_per_minute_last_10 AS
105+
SELECT
106+
target_id as user_id,
107+
date_trunc('minute', to_timestamp(received_at)) as received_at_minute,
108+
COUNT(*) as pageviews
109+
FROM pageview_stg
110+
WHERE
111+
pageview_type = 'profiles' AND
112+
mz_logical_timestamp() < (received_at*1000 + 600000)::numeric
113+
GROUP BY 1, 2;
114+
115+
CREATE MATERIALIZED VIEW profile_views AS
116+
SELECT
117+
target_id AS owner_id,
118+
user_id AS viewer_id,
119+
received_at AS received_at
120+
FROM (SELECT DISTINCT target_id FROM pageview_stg) grp,
121+
LATERAL (
122+
SELECT user_id, received_at FROM pageview_stg
123+
WHERE target_id = grp.target_id
124+
ORDER BY received_at DESC LIMIT 10
125+
);
126+
127+
CREATE MATERIALIZED VIEW profile_views_enriched AS
128+
SELECT
129+
owner.id as owner_id,
130+
owner.email as owner_email,
131+
viewers.id as viewer_id,
132+
viewers.email as viewer_email,
133+
profile_views.received_at
134+
FROM profile_views
135+
JOIN users owner ON profile_views.owner_id = owner.id
136+
JOIN users viewers ON profile_views.viewer_id = viewers.id;

‎materialize/drop.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
DROP SOURCE purchases CASCADE;
2+
DROP SOURCE items CASCADE;
3+
DROP SOURCE users CASCADE;
4+
DROP SOURCE json_pageviews CASCADE;

‎mysql/mysql_bootstrap.sql‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ CREATE TABLE IF NOT EXISTS shop.users
1414
id SERIAL PRIMARY KEY,
1515
email VARCHAR(255),
1616
is_vip BOOLEAN DEFAULT FALSE,
17-
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
17+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
1818
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
1919
);
2020

@@ -26,7 +26,7 @@ CREATE TABLE IF NOT EXISTS shop.items
2626
price DECIMAL(7,2),
2727
inventory INT,
2828
inventory_updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
29-
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
29+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
3030
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
3131
);
3232

@@ -38,6 +38,6 @@ CREATE TABLE IF NOT EXISTS shop.purchases
3838
status TINYINT UNSIGNED DEFAULT 1,
3939
quantity INT UNSIGNED DEFAULT 1,
4040
purchase_price DECIMAL(12,2),
41-
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
41+
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
4242
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
4343
);

0 commit comments

Comments
 (0)