Skip to content

Commit 354b7d3

Browse files
izadoesdevizadoesdev
committed
spoiler
1 parent fe09b4f commit 354b7d3

File tree

1 file changed

+65
-87
lines changed

1 file changed

+65
-87
lines changed

‎packages/db/src/clickhouse/schema.ts‎

Lines changed: 65 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { clickHouse } from "./client";
22

33
const ANALYTICS_DATABASE = "analytics";
4-
const OBSERVABILITY_DATABASE = "observability";
4+
const UPTIME_DATABASE = "uptime";
55

66
const CREATE_DATABASE = `
77
CREATE DATABASE IF NOT EXISTS ${ANALYTICS_DATABASE}
@@ -279,80 +279,6 @@ ORDER BY (domain, event_time)
279279
SETTINGS index_granularity = 8192
280280
`;
281281

282-
const CREATE_OTEL_TRACES_TABLE = `
283-
CREATE TABLE IF NOT EXISTS ${OBSERVABILITY_DATABASE}.otel_traces (
284-
Timestamp DateTime64(9) CODEC(Delta(8), ZSTD(1)),
285-
TraceId String CODEC(ZSTD(1)),
286-
TenantId String CODEC(ZSTD(1)),
287-
SpanId String CODEC(ZSTD(1)),
288-
ParentSpanId String CODEC(ZSTD(1)),
289-
TraceState String CODEC(ZSTD(1)),
290-
SpanName LowCardinality(String) CODEC(ZSTD(1)),
291-
SpanKind LowCardinality(String) CODEC(ZSTD(1)),
292-
ServiceName LowCardinality(String) CODEC(ZSTD(1)),
293-
ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
294-
ScopeName String CODEC(ZSTD(1)),
295-
ScopeVersion String CODEC(ZSTD(1)),
296-
SpanAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
297-
Duration Int64 CODEC(ZSTD(1)),
298-
StatusCode LowCardinality(String) CODEC(ZSTD(1)),
299-
StatusMessage String CODEC(ZSTD(1)),
300-
Events Nested(
301-
Timestamp DateTime64(9),
302-
Name LowCardinality(String),
303-
Attributes Map(LowCardinality(String), String)
304-
),
305-
Links Nested(
306-
TraceId String,
307-
SpanId String,
308-
TraceState String,
309-
Attributes Map(LowCardinality(String), String)
310-
),
311-
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
312-
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
313-
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
314-
INDEX idx_span_attr_key mapKeys(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
315-
INDEX idx_span_attr_value mapValues(SpanAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
316-
INDEX idx_duration Duration TYPE minmax GRANULARITY 1
317-
) ENGINE = MergeTree
318-
PARTITION BY toDate(Timestamp)
319-
ORDER BY (ServiceName, SpanName, toUnixTimestamp(Timestamp), TraceId)
320-
TTL toDateTime(Timestamp) + toIntervalDay(3)
321-
SETTINGS ttl_only_drop_parts = 1
322-
`;
323-
324-
const CREATE_OTEL_LOGS_TABLE = `
325-
CREATE TABLE IF NOT EXISTS ${OBSERVABILITY_DATABASE}.otel_logs (
326-
Timestamp DateTime64(9) CODEC(Delta(8), ZSTD(1)),
327-
TraceId String CODEC(ZSTD(1)),
328-
TenantId String CODEC(ZSTD(1)),
329-
SpanId String CODEC(ZSTD(1)),
330-
TraceFlags UInt32 CODEC(ZSTD(1)),
331-
SeverityText LowCardinality(String) CODEC(ZSTD(1)),
332-
SeverityNumber Int32 CODEC(ZSTD(1)),
333-
ServiceName LowCardinality(String) CODEC(ZSTD(1)),
334-
Body String CODEC(ZSTD(1)),
335-
ResourceSchemaUrl String CODEC(ZSTD(1)),
336-
ResourceAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
337-
ScopeSchemaUrl String CODEC(ZSTD(1)),
338-
ScopeName String CODEC(ZSTD(1)),
339-
ScopeVersion String CODEC(ZSTD(1)),
340-
ScopeAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
341-
LogAttributes Map(LowCardinality(String), String) CODEC(ZSTD(1)),
342-
INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1,
343-
INDEX idx_res_attr_key mapKeys(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
344-
INDEX idx_res_attr_value mapValues(ResourceAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
345-
INDEX idx_scope_attr_key mapKeys(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
346-
INDEX idx_scope_attr_value mapValues(ScopeAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
347-
INDEX idx_log_attr_key mapKeys(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
348-
INDEX idx_log_attr_value mapValues(LogAttributes) TYPE bloom_filter(0.01) GRANULARITY 1,
349-
INDEX idx_body Body TYPE tokenbf_v1(32768, 3, 0) GRANULARITY 1
350-
) ENGINE = MergeTree
351-
PARTITION BY toDate(Timestamp)
352-
ORDER BY (ServiceName, SeverityText, toUnixTimestamp(Timestamp), TraceId)
353-
TTL toDateTime(Timestamp) + toIntervalDay(3)
354-
SETTINGS ttl_only_drop_parts = 1
355-
`;
356282

357283
/**
358284
* Lean custom event spans table
@@ -433,6 +359,37 @@ ORDER BY (client_id, timestamp, id)
433359
SETTINGS index_granularity = 8192
434360
`;
435361

362+
const CREATE_UPTIME_TABLE = `
363+
CREATE TABLE IF NOT EXISTS ${UPTIME_DATABASE}.uptime_monitor (
364+
site_id String CODEC(ZSTD(1)),
365+
url String CODEC(ZSTD(1)),
366+
timestamp DateTime64(3, 'UTC') CODEC(Delta(8), ZSTD(1)),
367+
status UInt8 CODEC(Delta(1)),
368+
http_code UInt16 CODEC(Delta(16)),
369+
ttfb_ms UInt32 CODEC(Delta(32)),
370+
total_ms UInt32 CODEC(Delta(32)),
371+
attempt UInt8 DEFAULT 1 CODEC(Delta(1)),
372+
failure_streak UInt16 DEFAULT 0 CODEC(Delta(16)),
373+
response_bytes UInt32 DEFAULT 0 CODEC(Delta(32)),
374+
content_hash String CODEC(ZSTD(1)),
375+
redirect_count UInt8 DEFAULT 0 CODEC(Delta(1)),
376+
probe_region LowCardinality(String) DEFAULT 'default',
377+
probe_ip String CODEC(ZSTD(1)),
378+
ssl_expiry DateTime64(3, 'UTC') DEFAULT NULL,
379+
ssl_valid UInt8 DEFAULT 1 CODEC(Delta(1)),
380+
env LowCardinality(String) DEFAULT 'prod',
381+
check_type LowCardinality(String) DEFAULT 'http',
382+
user_agent String DEFAULT 'uptime-monitor',
383+
error String DEFAULT '' CODEC(ZSTD(1)),
384+
INDEX idx_site_id site_id TYPE bloom_filter(0.01) GRANULARITY 1,
385+
INDEX idx_status status TYPE minmax GRANULARITY 1,
386+
INDEX idx_timestamp timestamp TYPE minmax GRANULARITY 1
387+
) ENGINE = MergeTree
388+
PARTITION BY toYYYYMM(timestamp)
389+
ORDER BY (site_id, timestamp)
390+
SETTINGS index_granularity = 8192
391+
`;
392+
436393
/**
437394
* Lean error span type
438395
*/
@@ -577,6 +534,29 @@ export type CustomOutgoingLink = {
577534
timestamp: number;
578535
};
579536

537+
export type UptimeMonitor = {
538+
site_id: string;
539+
url: string;
540+
timestamp: number;
541+
status: number;
542+
http_code: number;
543+
ttfb_ms: number;
544+
total_ms: number;
545+
attempt: number;
546+
failure_streak: number;
547+
response_bytes: number;
548+
content_hash: string;
549+
redirect_count: number;
550+
probe_region: string;
551+
probe_ip: string;
552+
ssl_expiry?: number;
553+
ssl_valid: number;
554+
env: string;
555+
check_type: string;
556+
user_agent: string;
557+
error: string;
558+
};
559+
580560
export type AnalyticsEvent = {
581561
id: string;
582562
client_id: string;
@@ -656,11 +636,10 @@ export async function initClickHouseSchema() {
656636
});
657637
console.info(`Created database: ${ANALYTICS_DATABASE}`);
658638

659-
// Create the observability database
660639
await clickHouse.command({
661-
query: `CREATE DATABASE IF NOT EXISTS ${OBSERVABILITY_DATABASE}`,
640+
query: `CREATE DATABASE IF NOT EXISTS ${UPTIME_DATABASE}`,
662641
});
663-
console.info(`Created database: ${OBSERVABILITY_DATABASE}`);
642+
console.info(`Created database: ${UPTIME_DATABASE}`);
664643

665644
// Create base tables first
666645
const tables = [
@@ -689,10 +668,9 @@ export async function initClickHouseSchema() {
689668
},
690669
];
691670

692-
// Create observability tables
693-
const observabilityTables = [
694-
{ name: "otel_traces", query: CREATE_OTEL_TRACES_TABLE },
695-
{ name: "otel_logs", query: CREATE_OTEL_LOGS_TABLE },
671+
// Uptime tables
672+
const uptimeTables = [
673+
{ name: "uptime_monitor", query: CREATE_UPTIME_TABLE },
696674
];
697675

698676
// Create base tables
@@ -701,10 +679,10 @@ export async function initClickHouseSchema() {
701679
console.info(`Created table: ${ANALYTICS_DATABASE}.${table.name}`);
702680
}
703681

704-
// Create observability tables
705-
for (const table of observabilityTables) {
682+
// Create uptime tables
683+
for (const table of uptimeTables) {
706684
await clickHouse.command({ query: table.query });
707-
console.info(`Created table: ${OBSERVABILITY_DATABASE}.${table.name}`);
685+
console.info(`Created table: ${UPTIME_DATABASE}.${table.name}`);
708686
}
709687

710688
// Create materialized views (after target tables exist)
@@ -723,8 +701,8 @@ export async function initClickHouseSchema() {
723701
database: ANALYTICS_DATABASE,
724702
tables: tables.map((t) => t.name),
725703
materialized_views: materializedViews.map((mv) => mv.name),
726-
observability_database: OBSERVABILITY_DATABASE,
727-
observability_tables: observabilityTables.map((t) => t.name),
704+
uptime_database: UPTIME_DATABASE,
705+
uptime_tables: uptimeTables.map((t) => t.name),
728706
},
729707
};
730708
} catch (error) {

0 commit comments

Comments
 (0)