Skip to content

Commit ff4ef09

Browse files
lio-plio-p
committed
add one table per service
1 parent 883a8b5 commit ff4ef09

File tree

1 file changed

+167
-0
lines changed

1 file changed

+167
-0
lines changed

‎blog-examples/log_clustering/one_table_service.sql‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
-- Create table for nginx logs
2+
CREATE TABLE logs_nginx
3+
(
4+
`remote_addr` IPv4,
5+
`remote_user` LowCardinality(String),
6+
`time_local` DateTime CODEC(Delta(4), ZSTD(1)),
7+
`request_type` LowCardinality(String),
8+
`request_path` String CODEC(ZSTD(6)),
9+
`request_protocol` LowCardinality(String),
10+
`status` UInt16,
11+
`size` UInt32,
12+
`referer` String CODEC(ZSTD(6)),
13+
`user_agent` LowCardinality(String),
14+
Body ALIAS format('{0} - {1} [{2}] "{3} {4} {5}" {6} {7} "{8}" "{9}"',remote_addr, remote_user, time_local, request_type, request_path, request_protocol, status, size, referer, user_agent)
15+
)
16+
ORDER BY (user_agent, remote_user, referer, request_path)
17+
18+
19+
-- Create table for recommendation service logs
20+
CREATE TABLE logs_service_recommendation
21+
(
22+
TemplateNumber UInt8,
23+
`date` String,
24+
`time` String,
25+
`service_name` Nullable(UUID),
26+
`trace_sampled` Nullable(UUID),
27+
`prod_1` LowCardinality(String),
28+
`prod_2` LowCardinality(String),
29+
`prod_3` LowCardinality(String),
30+
`prod_4` LowCardinality(String),
31+
`prod_5` LowCardinality(String),
32+
Body ALIAS multiIf(TemplateNumber=1, format('{0} {1} INFO [main] [recommendation_server.py:47] resource.service.name={2} trace_sampled={3}] - Receive ListRecommendations for product {4} {5} {6} {7} {8}',date,time,service_name,trace_sampled,prod_1,prod_2,prod_3,prod_4,prod_5), TemplateNumber=2, format('Receive ListRecommendations for product {0} {1} {2} {3} {4}',prod_1,prod_2,prod_3,prod_4,prod_5),''))
33+
ORDER BY (date, prod_1, prod_2, prod_3, prod_4, prod_5)
34+
35+
-- Create table for cart service logs
36+
CREATE TABLE logs_service_cart
37+
(
38+
TemplateNumber UInt8,
39+
`user_id` Nullable(UUID),
40+
`product_id` String,
41+
`quantity` String,
42+
Body ALIAS multiIf(
43+
TemplateNumber=1, format('GetCartAsync called with userId={0}',user_id),
44+
TemplateNumber=2, 'info: cart.cartstore.ValkeyCartStore[0]',
45+
TemplateNumber=3, format('AddItemAsync called with userId={0}, productId={1}, quantity={2}', user_id, product_id, quantity),
46+
TemplateNumber=4, format('EmptyCartAsync called with userId={0}',user_id),
47+
'')
48+
)
49+
ORDER BY (TemplateNumber, product_id, quantity)
50+
51+
-- Create materialized view for nginx logs
52+
CREATE MATERIALIZED VIEW IF NOT EXISTS mv_logs_nginx
53+
TO logs_service_nginx
54+
AS
55+
SELECT
56+
remote_address,
57+
remote_user,
58+
time_local,
59+
request_type,
60+
request_path,
61+
request_protocol,
62+
status,
63+
size,
64+
referer,
65+
user_agent
66+
FROM
67+
(
68+
WITH
69+
'^(\S+) - (\S+) \[([^\]]+)\] "([A-Z]+)?\s*(.*?)\s*(HTTP\S+)?" (\d{3}) (\d+) "([^"]*)" "([^"]*)"$' AS pattern
70+
SELECT
71+
*,
72+
match(Body, pattern) AS m1,
73+
extractAllGroups(Body, pattern) AS g,
74+
arrayElement(arrayElement(g, 1), 1) AS remote_address,
75+
arrayElement(arrayElement(g, 1), 2) AS remote_user,
76+
parseDateTimeBestEffort(arrayElement(arrayElement(g, 1), 3)) AS time_local,
77+
arrayElement(arrayElement(g, 1), 4) AS request_type,
78+
arrayElement(arrayElement(g, 1), 5) AS request_path,
79+
arrayElement(arrayElement(g, 1), 6) AS request_protocol,
80+
arrayElement(arrayElement(g, 1), 7) AS status,
81+
arrayElement(arrayElement(g, 1), 8) AS size,
82+
arrayElement(arrayElement(g, 1), 9) AS referer,
83+
arrayElement(arrayElement(g, 1), 10) AS user_agent
84+
FROM raw_logs where ServiceName='nginx'
85+
) WHERE m1;
86+
87+
-- Create materialized view for recommendation service logs
88+
CREATE MATERIALIZED VIEW IF NOT EXISTS mv_logs_recommendation
89+
TO logs_service_recommendation
90+
AS
91+
SELECT
92+
CASE WHEN m1 THEN 1 WHEN m2 THEN 2 ELSE 0 END AS TemplateNumber,
93+
CASE when m1 THEN g1_1 ELSE NULL END AS date,
94+
CASE when m1 THEN g1_2 ELSE NULL END AS time,
95+
CASE when m1 THEN g1_3 ELSE NULL END AS service_name,
96+
CASE when m1 THEN g1_4 ELSE NULL END AS trace_sampled,
97+
CASE when m1 THEN g1_5 ELSE g2_1 END AS prod_1,
98+
CASE when m1 THEN g1_6 ELSE g2_2 END AS prod_2,
99+
CASE when m1 THEN g1_7 ELSE g2_3 END AS prod_3,
100+
CASE when m1 THEN g1_8 ELSE g2_4 END AS prod_4,
101+
CASE when m1 THEN g1_9 ELSE g2_5 END AS prod_5
102+
103+
FROM
104+
(
105+
WITH
106+
'^([^\\s]+) ([^\\s]+) INFO \[main\] \[recommendation_server.py:47\] \[trace_id=([^\\s]+) span_id=([^\\s]+) resource\.service\.name=recommendation trace_sampled=True\] - Receive ListRecommendations for product ids:\[([^\\s]+) ([^\\s]+) ([^\\s]+) ([^\\s]+) ([^\\s]+)\]$' AS pattern1,
107+
'^Receive ListRecommendations for product ([^\\s]+) ([^\\s]+) ([^\\s]+) ([^\\s]+) ([^\\s]+)$' AS pattern2
108+
SELECT
109+
*,
110+
match(Body, pattern1) AS m1,
111+
match(Body, pattern2) AS m2,
112+
extractAllGroups(Body, pattern1) AS g1,
113+
extractAllGroups(Body, pattern2) AS g2,
114+
115+
arrayElement(arrayElement(g1, 1), 1) AS g1_1,
116+
arrayElement(arrayElement(g1, 1), 2) AS g1_2,
117+
arrayElement(arrayElement(g1, 1), 3) AS g1_3,
118+
arrayElement(arrayElement(g1, 1), 4) AS g1_4,
119+
arrayElement(arrayElement(g1, 1), 5) AS g1_5,
120+
arrayElement(arrayElement(g1, 1), 6) AS g1_6,
121+
arrayElement(arrayElement(g1, 1), 7) AS g1_7,
122+
arrayElement(arrayElement(g1, 1), 7) AS g1_8,
123+
arrayElement(arrayElement(g1, 1), 7) AS g1_9,
124+
125+
arrayElement(arrayElement(g2, 1), 1) AS g2_1,
126+
arrayElement(arrayElement(g2, 1), 2) AS g2_2,
127+
arrayElement(arrayElement(g2, 1), 3) AS g2_3,
128+
arrayElement(arrayElement(g2, 1), 4) AS g2_4,
129+
arrayElement(arrayElement(g2, 1), 5) AS g2_5
130+
FROM raw_logs where ServiceName='recommendation'
131+
);
132+
133+
-- Create materialized view for cart service logs
134+
CREATE MATERIALIZED VIEW IF NOT EXISTS mv_logs_cart
135+
TO logs_service_cart
136+
AS
137+
SELECT
138+
multiIf(m1, 1, m2, 2, m3, 3, 0) AS TemplateNumber,
139+
multiIf(m1, g1_1, m2, Null, m3, g3_1, m4, g4_1, Null) AS user_id,
140+
multiIf(m1, '', m2, '', m3, g3_2, '') AS product_id,
141+
multiIf(m1, '', m2, '', m3, g3_3, '') AS quantity
142+
143+
FROM
144+
(
145+
WITH
146+
'^[\\s]*GetCartAsync called with userId=([^\\s]*)$' AS pattern1,
147+
'^info\: cart.cartstore.ValkeyCartStore\[0\]$' AS pattern2,
148+
'^[\\s]*AddItemAsync called with userId=([^\\s]+), productId=([^\\s]+), quantity=([^\\s]+)$' AS pattern3,
149+
'^[\\s]*EmptyCartAsync called with userId=([^\\s]*)$' AS pattern4
150+
SELECT
151+
*,
152+
match(Body, pattern1) AS m1,
153+
match(Body, pattern2) AS m2,
154+
match(Body, pattern3) AS m3,
155+
match(Body, pattern4) AS m4,
156+
extractAllGroups(Body, pattern1) AS g1,
157+
extractAllGroups(Body, pattern2) AS g2,
158+
extractAllGroups(Body, pattern3) AS g3,
159+
extractAllGroups(Body, pattern4) AS g4,
160+
161+
arrayElement(arrayElement(g1, 1), 1) AS g1_1,
162+
arrayElement(arrayElement(g3, 1), 1) AS g3_1,
163+
arrayElement(arrayElement(g3, 1), 2) AS g3_2,
164+
arrayElement(arrayElement(g3, 1), 3) AS g3_3,
165+
arrayElement(arrayElement(g4, 1), 1) AS g4_1
166+
FROM raw_logs where ServiceName='cart'
167+
);

0 commit comments

Comments
 (0)