Skip to content

Commit c95e231

Browse files
izadoesdevizadoesdev
committed
email schema
1 parent 51b701c commit c95e231

File tree

10 files changed

+303
-0
lines changed

10 files changed

+303
-0
lines changed

‎packages/db/src/clickhouse/sql/blocked-traffic.sql‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
-- Blocked traffic table for tracking filtered requests
2+
CREATE TABLE IF NOT EXISTS analytics.blocked_traffic (
3+
id UUID,
4+
client_id String,
5+
timestamp DateTime64(3, 'UTC'),
6+
7+
path Nullable(String),
8+
url Nullable(String),
9+
referrer Nullable(String),
10+
method LowCardinality(String) DEFAULT 'POST',
11+
origin Nullable(String),
12+
13+
ip String,
14+
user_agent Nullable(String),
15+
accept_header Nullable(String),
16+
language Nullable(String),
17+
18+
block_reason LowCardinality(String),
19+
block_category LowCardinality(String),
20+
bot_name Nullable(String),
21+
22+
country Nullable(String),
23+
region Nullable(String),
24+
browser_name Nullable(String),
25+
browser_version Nullable(String),
26+
os_name Nullable(String),
27+
os_version Nullable(String),
28+
device_type Nullable(String),
29+
30+
payload_size Nullable(UInt32),
31+
32+
created_at DateTime64(3, 'UTC') DEFAULT now()
33+
) ENGINE = MergeTree()
34+
PARTITION BY toYYYYMM(timestamp)
35+
ORDER BY (timestamp, client_id, id)
36+
TTL toDateTime(timestamp) + INTERVAL 6 MONTH
37+
SETTINGS index_granularity = 8192;

‎packages/db/src/clickhouse/sql/database.sql‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
-- Create the analytics database
2+
CREATE DATABASE IF NOT EXISTS analytics;

‎packages/db/src/clickhouse/sql/email-events.sql‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
CREATE TABLE IF NOT EXISTS analytics.email_events (
2+
event_id UUID DEFAULT generateUUIDv4(),
3+
email_hash String,
4+
domain String,
5+
labels Array(LowCardinality(String)),
6+
event_time DateTime,
7+
received_at DateTime,
8+
ingestion_time DateTime DEFAULT now(),
9+
metadata_json JSON
10+
) ENGINE = MergeTree()
11+
PARTITION BY toYYYYMM(event_time)
12+
ORDER BY (domain, event_time)
13+
SETTINGS index_granularity = 8192;

‎packages/db/src/clickhouse/sql/errors.sql‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
-- Dedicated errors table for error events
2+
CREATE TABLE IF NOT EXISTS analytics.errors (
3+
id UUID,
4+
client_id String,
5+
event_id Nullable(String),
6+
7+
anonymous_id String,
8+
session_id String,
9+
timestamp DateTime64(3, 'UTC'),
10+
11+
path String,
12+
13+
message String,
14+
filename Nullable(String),
15+
lineno Nullable(Int32),
16+
colno Nullable(Int32),
17+
stack Nullable(String),
18+
error_type Nullable(String),
19+
20+
ip Nullable(String),
21+
user_agent Nullable(String),
22+
browser_name Nullable(String),
23+
browser_version Nullable(String),
24+
os_name Nullable(String),
25+
os_version Nullable(String),
26+
device_type Nullable(String),
27+
country Nullable(String),
28+
region Nullable(String),
29+
30+
created_at DateTime64(3, 'UTC')
31+
) ENGINE = MergeTree()
32+
PARTITION BY toYYYYMM(timestamp)
33+
ORDER BY (client_id, timestamp, id)
34+
SETTINGS index_granularity = 8192;

‎packages/db/src/clickhouse/sql/events.sql‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
-- Events table stores all raw events
2+
-- Optimizations:
3+
-- 1. Use LowCardinality(String) for fields with limited distinct values
4+
-- 2. Reorder ORDER BY to prioritize time-based queries
5+
-- 3. Add materialized views for common aggregations
6+
-- 4. Remove Nullable where 0 is a sensible default
7+
8+
CREATE TABLE IF NOT EXISTS analytics.events (
9+
id UUID,
10+
client_id String,
11+
event_name String,
12+
anonymous_id String,
13+
time DateTime64(3, 'UTC'),
14+
session_id String,
15+
16+
event_type LowCardinality(String) DEFAULT 'track', -- 'track', 'error', 'web_vitals'
17+
event_id Nullable(String), -- UUID from client for deduplication
18+
session_start_time Nullable(DateTime64(3, 'UTC')), -- New session tracking
19+
timestamp DateTime64(3, 'UTC') DEFAULT time, -- Alias for new format
20+
21+
referrer Nullable(String),
22+
url String,
23+
path String,
24+
title Nullable(String),
25+
26+
ip String,
27+
user_agent String,
28+
browser_name Nullable(String),
29+
browser_version Nullable(String),
30+
os_name Nullable(String),
31+
os_version Nullable(String),
32+
device_type Nullable(String),
33+
device_brand Nullable(String),
34+
device_model Nullable(String),
35+
country Nullable(String),
36+
region Nullable(String),
37+
city Nullable(String),
38+
39+
screen_resolution Nullable(String),
40+
viewport_size Nullable(String),
41+
language Nullable(String),
42+
timezone Nullable(String),
43+
44+
connection_type Nullable(String),
45+
rtt Nullable(Int16),
46+
downlink Nullable(Float32), -- New field
47+
48+
time_on_page Nullable(Float32),
49+
scroll_depth Nullable(Float32),
50+
interaction_count Nullable(Int16),
51+
exit_intent UInt8,
52+
page_count UInt8 DEFAULT 1,
53+
is_bounce UInt8 DEFAULT 1,
54+
has_exit_intent Nullable(UInt8), -- New field
55+
page_size Nullable(Int32),
56+
57+
utm_source Nullable(String),
58+
utm_medium Nullable(String),
59+
utm_campaign Nullable(String),
60+
utm_term Nullable(String),
61+
utm_content Nullable(String),
62+
63+
load_time Nullable(Int32),
64+
dom_ready_time Nullable(Int32),
65+
dom_interactive Nullable(Int32), -- New field
66+
ttfb Nullable(Int32),
67+
connection_time Nullable(Int32),
68+
request_time Nullable(Int32),
69+
render_time Nullable(Int32),
70+
redirect_time Nullable(Int32), -- New field
71+
domain_lookup_time Nullable(Int32), -- New field
72+
73+
fcp Nullable(Int32),
74+
lcp Nullable(Int32),
75+
cls Nullable(Float32),
76+
fid Nullable(Int32), -- New field
77+
inp Nullable(Int32), -- New field
78+
79+
href Nullable(String),
80+
text Nullable(String),
81+
82+
value Nullable(String),
83+
84+
error_message Nullable(String),
85+
error_filename Nullable(String),
86+
error_lineno Nullable(Int32),
87+
error_colno Nullable(Int32),
88+
error_stack Nullable(String),
89+
error_type Nullable(String),
90+
91+
properties String,
92+
93+
created_at DateTime64(3, 'UTC')
94+
) ENGINE = MergeTree()
95+
PARTITION BY toYYYYMM(time)
96+
ORDER BY (client_id, time, id)
97+
SETTINGS index_granularity = 8192;

‎packages/db/src/clickhouse/sql/stripe-charges.sql‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
-- Stripe Charges table
2+
CREATE TABLE IF NOT EXISTS analytics.stripe_charges (
3+
id String,
4+
client_id String,
5+
webhook_token String,
6+
created DateTime64(3, 'UTC'),
7+
status LowCardinality(String),
8+
currency LowCardinality(String),
9+
amount UInt64,
10+
amount_captured UInt64,
11+
amount_refunded UInt64,
12+
paid UInt8,
13+
refunded UInt8,
14+
livemode UInt8,
15+
failure_code Nullable(String),
16+
failure_message Nullable(String),
17+
outcome_type Nullable(String),
18+
risk_level LowCardinality(String),
19+
card_brand LowCardinality(String),
20+
payment_intent_id Nullable(String),
21+
session_id Nullable(String),
22+
created_at DateTime64(3, 'UTC') DEFAULT now()
23+
) ENGINE = MergeTree()
24+
PARTITION BY toYYYYMM(created)
25+
ORDER BY (client_id, webhook_token, created, id)
26+
SETTINGS index_granularity = 8192;

‎packages/db/src/clickhouse/sql/stripe-payment-intents.sql‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
-- Stripe Payment Intents table
2+
CREATE TABLE IF NOT EXISTS analytics.stripe_payment_intents (
3+
id String,
4+
client_id String,
5+
webhook_token String,
6+
created DateTime64(3, 'UTC'),
7+
status LowCardinality(String),
8+
currency LowCardinality(String),
9+
amount UInt64,
10+
amount_received UInt64,
11+
amount_capturable UInt64,
12+
livemode UInt8,
13+
metadata JSON,
14+
payment_method_types Array(String),
15+
failure_reason Nullable(String),
16+
canceled_at Nullable(DateTime64(3, 'UTC')),
17+
cancellation_reason Nullable(String),
18+
description Nullable(String),
19+
application_fee_amount Nullable(UInt64),
20+
setup_future_usage Nullable(String),
21+
session_id Nullable(String),
22+
created_at DateTime64(3, 'UTC') DEFAULT now()
23+
) ENGINE = MergeTree()
24+
PARTITION BY toYYYYMM(created)
25+
ORDER BY (client_id, webhook_token, created, id)
26+
SETTINGS index_granularity = 8192;

‎packages/db/src/clickhouse/sql/stripe-refunds.sql‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
-- Stripe Refunds table
2+
CREATE TABLE IF NOT EXISTS analytics.stripe_refunds (
3+
id String,
4+
client_id String,
5+
webhook_token String,
6+
created DateTime64(3, 'UTC'),
7+
amount UInt64,
8+
status LowCardinality(String),
9+
reason LowCardinality(String),
10+
currency LowCardinality(String),
11+
charge_id String,
12+
payment_intent_id Nullable(String),
13+
metadata JSON,
14+
session_id Nullable(String),
15+
created_at DateTime64(3, 'UTC') DEFAULT now()
16+
) ENGINE = MergeTree()
17+
PARTITION BY toYYYYMM(created)
18+
ORDER BY (client_id, webhook_token, created, id)
19+
SETTINGS index_granularity = 8192;

‎packages/db/src/clickhouse/sql/web-vitals.sql‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
-- Dedicated web vitals table for performance metrics
2+
CREATE TABLE IF NOT EXISTS analytics.web_vitals (
3+
id UUID,
4+
client_id String,
5+
event_id Nullable(String),
6+
7+
anonymous_id String,
8+
session_id String,
9+
timestamp DateTime64(3, 'UTC'),
10+
11+
path String,
12+
13+
fcp Nullable(Int32),
14+
lcp Nullable(Int32),
15+
cls Nullable(Float32),
16+
fid Nullable(Int32),
17+
inp Nullable(Int32),
18+
19+
ip Nullable(String),
20+
user_agent Nullable(String),
21+
browser_name Nullable(String),
22+
browser_version Nullable(String),
23+
os_name Nullable(String),
24+
os_version Nullable(String),
25+
device_type Nullable(String),
26+
country Nullable(String),
27+
region Nullable(String),
28+
29+
created_at DateTime64(3, 'UTC')
30+
) ENGINE = MergeTree()
31+
PARTITION BY toYYYYMM(timestamp)
32+
ORDER BY (client_id, timestamp, id)
33+
SETTINGS index_granularity = 8192;

‎packages/validation/src/schemas/email.ts‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
import { z } from 'zod';
2+
3+
// Email event validation schema
4+
export const emailEventSchema = z.object({
5+
email_id: z.string().min(1).max(255),
6+
domain: z.string().min(1).max(255),
7+
labels: z.array(z.string().max(100)).optional().default([]),
8+
event_time: z.number().int().positive().optional(),
9+
received_at: z.number().int().positive().optional(),
10+
metadata: z.record(z.string(), z.unknown()).optional().default({}),
11+
});
12+
13+
export const batchEmailEventSchema = z.array(emailEventSchema).max(100);
14+
15+
export type EmailEventInput = z.infer<typeof emailEventSchema>;
16+
export type BatchEmailEventInput = z.infer<typeof batchEmailEventSchema>;

0 commit comments

Comments
 (0)