Skip to content

Commit 0cd912f

Browse files
izadoesdevizadoesdev
committed
feat: new error tracking, web vitals, span based
1 parent caf71a6 commit 0cd912f

File tree

13 files changed

+464
-81
lines changed

13 files changed

+464
-81
lines changed

‎apps/basket/src/lib/event-service.ts‎

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,11 @@ import type {
44
CustomEvent,
55
CustomOutgoingLink,
66
ErrorEvent,
7+
ErrorSpanRow,
78
WebVitalsEvent,
9+
WebVitalsSpan,
810
} from "@databuddy/db";
11+
import type { ErrorSpan, IndividualVital } from "@databuddy/validation";
912
import { getGeo } from "../utils/ip-geo";
1013
import { parseUserAgent } from "../utils/user-agent";
1114
import {
@@ -471,6 +474,54 @@ export function insertErrorsBatch(events: ErrorEvent[]): Promise<void> {
471474
});
472475
}
473476

477+
/**
478+
* Insert lean error spans (v2.x format)
479+
*/
480+
export async function insertErrorSpans(
481+
errors: ErrorSpan[],
482+
clientId: string
483+
): Promise<void> {
484+
if (errors.length === 0) {
485+
return;
486+
}
487+
488+
const now = Date.now();
489+
const spans: ErrorSpanRow[] = [];
490+
491+
for (const error of errors) {
492+
// Use message hash as dedup key
493+
const dedupKey = `error_${clientId}_${error.message.slice(0, 50)}_${error.path}`;
494+
const isDuplicate = await checkDuplicate(dedupKey, "error");
495+
if (isDuplicate) {
496+
continue;
497+
}
498+
499+
const errorSpan: ErrorSpanRow = {
500+
client_id: clientId,
501+
anonymous_id: sanitizeString(error.anonymousId, VALIDATION_LIMITS.SHORT_STRING_MAX_LENGTH),
502+
session_id: validateSessionId(error.sessionId),
503+
timestamp: typeof error.timestamp === "number" ? error.timestamp : now,
504+
path: sanitizeString(error.path, VALIDATION_LIMITS.STRING_MAX_LENGTH),
505+
message: sanitizeString(error.message, VALIDATION_LIMITS.STRING_MAX_LENGTH),
506+
filename: sanitizeString(error.filename, VALIDATION_LIMITS.STRING_MAX_LENGTH),
507+
lineno: error.lineno ?? undefined,
508+
colno: error.colno ?? undefined,
509+
stack: sanitizeString(error.stack, VALIDATION_LIMITS.STRING_MAX_LENGTH),
510+
error_type: sanitizeString(error.errorType, VALIDATION_LIMITS.SHORT_STRING_MAX_LENGTH) || "Error",
511+
};
512+
513+
spans.push(errorSpan);
514+
}
515+
516+
if (spans.length > 0) {
517+
try {
518+
await sendEventBatch("analytics-error-spans", spans);
519+
} catch (error) {
520+
captureError(error, { count: spans.length });
521+
}
522+
}
523+
}
524+
474525
export function insertWebVitalsBatch(events: WebVitalsEvent[]): Promise<void> {
475526
return record("insertWebVitalsBatch", async () => {
476527
if (events.length === 0) {
@@ -490,6 +541,54 @@ export function insertWebVitalsBatch(events: WebVitalsEvent[]): Promise<void> {
490541
});
491542
}
492543

544+
/**
545+
* Insert individual vital metrics (v2.x format)
546+
* Transforms and groups metrics before storing
547+
*/
548+
/**
549+
* Insert individual vital metrics (v2.x format) as spans
550+
* Each metric is stored as a separate row - no aggregation
551+
*/
552+
export async function insertIndividualVitals(
553+
vitals: IndividualVital[],
554+
clientId: string
555+
): Promise<void> {
556+
if (vitals.length === 0) {
557+
return;
558+
}
559+
560+
const now = Date.now();
561+
const spans: WebVitalsSpan[] = [];
562+
563+
for (const vital of vitals) {
564+
// Dedup by client+session+path+metric
565+
const dedupKey = `vital_${clientId}_${vital.sessionId}_${vital.path}_${vital.metricName}`;
566+
const isDuplicate = await checkDuplicate(dedupKey, "web_vitals");
567+
if (isDuplicate) {
568+
continue;
569+
}
570+
571+
const span: WebVitalsSpan = {
572+
client_id: clientId,
573+
session_id: validateSessionId(vital.sessionId),
574+
timestamp: typeof vital.timestamp === "number" ? vital.timestamp : now,
575+
path: sanitizeString(vital.path, VALIDATION_LIMITS.STRING_MAX_LENGTH),
576+
metric_name: vital.metricName,
577+
metric_value: vital.metricValue,
578+
};
579+
580+
spans.push(span);
581+
}
582+
583+
if (spans.length > 0) {
584+
try {
585+
await sendEventBatch("analytics-vitals-spans", spans);
586+
} catch (error) {
587+
captureError(error, { count: spans.length });
588+
}
589+
}
590+
}
591+
493592
export function insertCustomEventsBatch(events: CustomEvent[]): Promise<void> {
494593
return record("insertCustomEventsBatch", async () => {
495594
if (events.length === 0) {

‎apps/basket/src/routes/basket.ts‎

Lines changed: 43 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,18 @@ import type {
66
ErrorEvent,
77
WebVitalsEvent,
88
} from "@databuddy/db";
9+
import {
10+
batchedErrorsSchema,
11+
batchedVitalsSchema,
12+
} from "@databuddy/validation";
913
import { Elysia } from "elysia";
1014
import {
1115
insertCustomEvent,
1216
insertCustomEventsBatch,
1317
insertError,
18+
insertErrorSpans,
1419
insertErrorsBatch,
20+
insertIndividualVitals,
1521
insertOutgoingLink,
1622
insertOutgoingLinksBatch,
1723
insertTrackEvent,
@@ -321,8 +327,8 @@ function processOutgoingLinkData(
321327
const app = new Elysia()
322328
.post("/vitals", async (context) => {
323329
const { body, query, request } = context as {
324-
body: any;
325-
query: any;
330+
body: unknown;
331+
query: Record<string, string>;
326332
request: Request;
327333
};
328334

@@ -332,45 +338,42 @@ const app = new Elysia()
332338
return validation.error;
333339
}
334340

335-
const { clientId, userAgent, ip } = validation;
336-
337-
const [botError, parseResult] = await Promise.all([
338-
checkForBot(request, body, query, clientId, userAgent),
339-
validateEventSchema(
340-
webVitalsEventSchema,
341-
body,
342-
request,
343-
query,
344-
clientId
345-
),
346-
]);
341+
const { clientId, userAgent } = validation;
347342

348-
if (botError) {
349-
return botError.error;
350-
}
343+
// v2.x tracker sends batched individual vital metrics to /vitals
344+
const parseResult = batchedVitalsSchema.safeParse(body);
351345

352346
if (!parseResult.success) {
353347
return createSchemaErrorResponse(parseResult.error.issues);
354348
}
355349

356-
const vitalsEvent = await processWebVitalsEventData(
350+
const botError = await checkForBot(
351+
request,
357352
body,
353+
query,
358354
clientId,
359-
userAgent,
360-
ip
355+
userAgent
361356
);
357+
if (botError) {
358+
return botError.error;
359+
}
360+
361+
await insertIndividualVitals(parseResult.data, clientId);
362362

363-
await insertWebVitals(vitalsEvent, clientId, userAgent, ip);
364-
return { status: "success", type: "web_vitals" };
363+
return {
364+
status: "success",
365+
type: "web_vitals",
366+
count: parseResult.data.length,
367+
};
365368
} catch (error) {
366369
captureError(error, { message: "Error processing vitals" });
367370
return { status: "error", message: "Internal server error" };
368371
}
369372
})
370373
.post("/errors", async (context) => {
371374
const { body, query, request } = context as {
372-
body: any;
373-
query: any;
375+
body: unknown;
376+
query: Record<string, string>;
374377
request: Request;
375378
};
376379

@@ -380,38 +383,32 @@ const app = new Elysia()
380383
return validation.error;
381384
}
382385

383-
const { clientId, userAgent, ip } = validation;
384-
385-
if (FILTERED_ERROR_MESSAGES.has(body.message)) {
386-
return {
387-
status: "ignored",
388-
type: "error",
389-
reason: "filtered_message",
390-
};
391-
}
386+
const { clientId, userAgent } = validation;
392387

393-
const [botError, parseResult] = await Promise.all([
394-
checkForBot(request, body, query, clientId, userAgent),
395-
validateEventSchema(errorEventSchema, body, request, query, clientId),
396-
]);
397-
398-
if (botError) {
399-
return botError.error;
400-
}
388+
const parseResult = batchedErrorsSchema.safeParse(body);
401389

402390
if (!parseResult.success) {
403391
return createSchemaErrorResponse(parseResult.error.issues);
404392
}
405393

406-
const errorEvent = await processErrorEventData(
394+
const botError = await checkForBot(
395+
request,
407396
body,
397+
query,
408398
clientId,
409-
userAgent,
410-
ip
399+
userAgent
411400
);
401+
if (botError) {
402+
return botError.error;
403+
}
412404

413-
await insertError(errorEvent, clientId, userAgent, ip);
414-
return { status: "success", type: "error" };
405+
await insertErrorSpans(parseResult.data, clientId);
406+
407+
return {
408+
status: "success",
409+
type: "error",
410+
count: parseResult.data.length,
411+
};
415412
} catch (error) {
416413
captureError(error, { message: "Error processing error" });
417414
return { status: "error", message: "Internal server error" };

‎infra/ingest/vector.yaml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ sources:
66
topics:
77
- analytics-events
88
- analytics-errors
9+
- analytics-error-spans
910
- analytics-web-vitals
11+
- analytics-vitals-spans
1012
- analytics-custom-events
1113
- analytics-outgoing-links
1214
- analytics-email-events
@@ -35,7 +37,9 @@ transforms:
3537
route:
3638
events: '.topic == "analytics-events"'
3739
errors: '.topic == "analytics-errors"'
40+
error_spans: '.topic == "analytics-error-spans"'
3841
web_vitals: '.topic == "analytics-web-vitals"'
42+
vitals_spans: '.topic == "analytics-vitals-spans"'
3943
custom_events: '.topic == "analytics-custom-events"'
4044
outgoing_links: '.topic == "analytics-outgoing-links"'
4145
email_events: '.topic == "analytics-email-events"'
@@ -90,6 +94,26 @@ sinks:
9094
encoding:
9195
timestamp_format: unix_ms
9296

97+
clickhouse_error_spans:
98+
type: clickhouse
99+
inputs:
100+
- route_analytics.error_spans
101+
endpoint: "${CLICKHOUSE_URL}"
102+
database: analytics
103+
table: error_spans
104+
auth:
105+
strategy: basic
106+
user: "${CLICKHOUSE_USER}"
107+
password: "${CLICKHOUSE_PASSWORD}"
108+
batch:
109+
max_events: 5000
110+
max_bytes: 5000000
111+
timeout_secs: 5
112+
date_time_best_effort: true
113+
skip_unknown_fields: true
114+
encoding:
115+
timestamp_format: unix_ms
116+
93117
clickhouse_web_vitals:
94118
type: clickhouse
95119
inputs:
@@ -110,6 +134,26 @@ sinks:
110134
encoding:
111135
timestamp_format: unix_ms
112136

137+
clickhouse_vitals_spans:
138+
type: clickhouse
139+
inputs:
140+
- route_analytics.vitals_spans
141+
endpoint: "${CLICKHOUSE_URL}"
142+
database: analytics
143+
table: web_vitals_spans
144+
auth:
145+
strategy: basic
146+
user: "${CLICKHOUSE_USER}"
147+
password: "${CLICKHOUSE_PASSWORD}"
148+
batch:
149+
max_events: 5000
150+
max_bytes: 5000000
151+
timeout_secs: 5
152+
date_time_best_effort: true
153+
skip_unknown_fields: true
154+
encoding:
155+
timestamp_format: unix_ms
156+
113157
clickhouse_custom_events:
114158
type: clickhouse
115159
inputs:

0 commit comments

Comments
 (0)