Skip to content

Commit 5fbc3e2

Browse files
alrocaralrocar
authored
add logs explorer matview (#20)
* add logs explorer matview * add logs explorer endpoint
1 parent 1651ccf commit 5fbc3e2

File tree

6 files changed

+156
-2
lines changed

6 files changed

+156
-2
lines changed

‎tinybird/datasources/logs_range_15m.datasource‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
SCHEMA >
2+
`start_ts` DateTime64(3) `json:$.start_ts`,
3+
`end_ts` DateTime64(3) `json:$.end_ts`,
4+
`environment` LowCardinality(String) `json:$.environment`,
5+
`service` LowCardinality(String) `json:$.service`,
6+
`level` LowCardinality(String) `json:$.level`,
7+
`n_rows` AggregateFunction(count, UInt64) `json:$.n_rows`
8+
9+
ENGINE "AggregatingMergeTree"
10+
ENGINE_PARTITION_KEY "toYYYYMM(start_ts)"
11+
ENGINE_SORTING_KEY "start_ts, end_ts, environment, service, level"
12+
ENGINE_PRIMARY_KEY "start_ts, end_ts, environment"

‎tinybird/endpoints/log_analysis.pipe‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ SQL >
4343
AND user_agent in {{Array(user_agent)}}
4444
{% end %}
4545
{% if defined(message) and message != '' %}
46-
AND message like concat('%', {{String(message)}}, '%')
46+
AND message ilike concat('%', {{String(message)}}, '%')
4747
{% end %}
4848
ORDER BY
4949
{% if defined(sort_by) and sort_by != '' %}

‎tinybird/endpoints/log_explorer.pipe‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
NODE cummulative_rows
2+
DESCRIPTION >
3+
Filter data and calculate the running sum of rows in the result set to determine the time range.
4+
5+
SQL >
6+
7+
%
8+
SELECT end_ts, start_ts, countMerge(n_rows) OVER (ORDER BY start_ts DESC) AS n_rows
9+
FROM logs_range_15m
10+
WHERE 1=1
11+
{% if defined(environment) and environment != [''] %}
12+
AND environment in {{ Array(environment) }}
13+
{% end %}
14+
{% if defined(service) and service != [''] %}
15+
AND service in {{ Array(service) }}
16+
{% end %}
17+
{% if defined(level) and level != [''] %}
18+
AND level in {{ Array(level) }}
19+
{% end %}
20+
AND start_ts >= {{ DateTime(start, '2024-03-08 00:00:00') }} - INTERVAL {{ UInt8(days_param, 30) }} DAY
21+
AND end_ts <= {{ DateTime(start, '2024-03-08 00:00:00') }}
22+
ORDER BY end_ts DESC
23+
24+
25+
26+
NODE ts_range
27+
DESCRIPTION >
28+
What is the time period, starting at X, and up to Y, that I need to query in order to get Z amount of rows that I need.
29+
30+
SQL >
31+
32+
%
33+
SELECT max(end_ts) AS max_end_ts, min(start_ts) AS min_start_ts
34+
FROM cummulative_rows
35+
WHERE
36+
n_rows > {{ Int32(page, 0) * Int32(page_size, 100) }}
37+
AND n_rows <= {{ (Int32(page, 0) + 1) * Int32(page_size, 100) }}
38+
39+
40+
41+
NODE logs_result
42+
DESCRIPTION >
43+
Rreturn the logs within the time range.
44+
NOTE: this does not guarantee that you get the exact page size. To avoid repeating records, you need to add extra conditions to check the last value on the previous page.
45+
46+
SQL >
47+
48+
%
49+
WITH
50+
(SELECT min_start_ts FROM ts_range) AS start_range,
51+
(SELECT max_end_ts FROM ts_range) AS end_range
52+
53+
SELECT
54+
timestamp,
55+
request_id,
56+
request_method,
57+
status_code,
58+
service,
59+
request_path,
60+
level,
61+
message
62+
FROM logs
63+
WHERE 1=1
64+
{% if defined(service) and service != [''] %}
65+
AND service in {{Array(service)}}
66+
{% end %}
67+
{% if defined(level) and level != [''] %}
68+
AND level in {{Array(level)}}
69+
{% end %}
70+
{% if defined(environment) and environment != [''] %}
71+
AND environment in {{Array(environment)}}
72+
{% end %}
73+
{% if defined(request_method) and request_method != [''] %}
74+
AND request_method in {{Array(request_method)}}
75+
{% end %}
76+
{% if defined(status_code) and status_code != [''] %}
77+
AND status_code in {{Array(status_code)}}
78+
{% end %}
79+
{% if defined(request_path) and request_path != [''] %}
80+
AND request_path in {{Array(request_path)}}
81+
{% end %}
82+
{% if defined(user_agent) and user_agent != [''] %}
83+
AND user_agent in {{Array(user_agent)}}
84+
{% end %}
85+
{% if defined(message) and message != '' %}
86+
AND message ilike concat('%', {{String(message)}}, '%')
87+
{% end %}
88+
AND timestamp >= start_range
89+
AND timestamp <= end_range
90+
ORDER BY timestamp DESC
91+
92+
TYPE endpoint

‎tinybird/endpoints/log_timeseries.pipe‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ SQL >
114114
{% end %}
115115
{% if date_diff_in_hours(start_date, end_date) <= 24 * 7 %}
116116
{% if defined(message) and message != '' %}
117-
AND message like concat('%', {{String(message)}}, '%')
117+
AND message ilike concat('%', {{String(message)}}, '%')
118118
{% end %}
119119
{% end %}
120120
GROUP BY date

‎tinybird/materializations/logs_range_15m_mv.pipe‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
NODE mv
2+
SQL >
3+
4+
SELECT
5+
toStartOfFifteenMinutes(timestamp) AS start_ts,
6+
start_ts + toIntervalMinute(15) AS end_ts,
7+
environment,
8+
service,
9+
level,
10+
countState() AS n_rows
11+
FROM logs
12+
GROUP BY
13+
start_ts,
14+
end_ts,
15+
environment,
16+
service,
17+
level
18+
19+
TYPE materialized
20+
DATASOURCE logs_range_15m

‎tinybird/tests/log_explorer.yaml‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
2+
- name: default_log_explorer
3+
description: Test with default parameters for last 1 day
4+
parameters: start=2025-02-04%2000:00:00&page=0&page_size=3&days_param=1
5+
expected_result: |
6+
{"timestamp":"2025-02-03 20:03:14.000","request_id":"req-6296819fe0433ec7","request_method":"POST","status_code":373,"service":"database","request_path":"\/orders\/373","level":"ERROR","message":"Log message 373"}
7+
{"timestamp":"2025-02-03 19:34:02.000","request_id":"req-6f65658010123820","request_method":"POST","status_code":525,"service":"api","request_path":"\/orders\/925","level":"ERROR","message":"Log message 925"}
8+
{"timestamp":"2025-02-03 19:23:42.000","request_id":"req-ab50f632c7544b42","request_method":"POST","status_code":345,"service":"api","request_path":"\/orders\/945","level":"ERROR","message":"Log message 945"}
9+
10+
- name: filtered_by_service_and_level
11+
description: Test filtered by specific service and error level
12+
parameters: start=2025-02-04%2000:00:00&service=api&level=ERROR&page=0&page_size=5&days_param=1
13+
expected_result: |
14+
{"timestamp":"2025-02-03 19:34:02.000","request_id":"req-6f65658010123820","request_method":"POST","status_code":525,"service":"api","request_path":"\/orders\/925","level":"ERROR","message":"Log message 925"}
15+
{"timestamp":"2025-02-03 19:23:42.000","request_id":"req-ab50f632c7544b42","request_method":"POST","status_code":345,"service":"api","request_path":"\/orders\/945","level":"ERROR","message":"Log message 945"}
16+
{"timestamp":"2025-02-03 19:13:22.000","request_id":"req-41cfd4707d5d9189","request_method":"POST","status_code":565,"service":"api","request_path":"\/orders\/365","level":"ERROR","message":"Log message 365"}
17+
{"timestamp":"2025-02-03 11:04:22.000","request_id":"req-1be7850effe31e0a","request_method":"POST","status_code":305,"service":"api","request_path":"\/orders\/105","level":"ERROR","message":"Log message 105"}
18+
{"timestamp":"2025-02-03 08:02:42.000","request_id":"req-e80695b4b6b1f89f","request_method":"POST","status_code":405,"service":"api","request_path":"\/orders\/205","level":"ERROR","message":"Log message 205"}
19+
20+
- name: search_by_message
21+
description: Test searching for specific message content
22+
parameters: start=2025-02-04%2000:00:00&message=945&page=0&page_size=3&days_param=1
23+
expected_result: |
24+
{"timestamp":"2025-02-03 19:23:42.000","request_id":"req-ab50f632c7544b42","request_method":"POST","status_code":345,"service":"api","request_path":"\/orders\/945","level":"ERROR","message":"Log message 945"}
25+
26+
- name: filtered_by_request_method
27+
description: Test filtered by HTTP method and status code
28+
parameters: start=2025-02-04%2000:00:00&request_method=POST&status_code=345&page=0&page_size=25&days_param=1
29+
expected_result: |
30+
{"timestamp":"2025-02-03 19:23:42.000","request_id":"req-ab50f632c7544b42","request_method":"POST","status_code":345,"service":"api","request_path":"\/orders\/945","level":"ERROR","message":"Log message 945"}

0 commit comments

Comments
 (0)