Skip to content

Commit a059441

Browse files
izadoesdevizadoesdev
committed
feat: data export
1 parent 23caedb commit a059441

File tree

12 files changed

+1241
-7
lines changed

12 files changed

+1241
-7
lines changed

‎apps/api/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
"autumn-js": "^0.1.10",
2222
"dayjs": "^1.11.13",
2323
"elysia": "^1.3.6",
24+
"jszip": "^3.10.1",
2425
"openai": "^5.9.0"
2526
}
2627
}

‎apps/api/src/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import { autumnHandler } from 'autumn-js/elysia';
77
import { Elysia } from 'elysia';
88
import { logger } from './lib/logger';
99
import { assistant } from './routes/assistant';
10+
import { exportRoute } from './routes/export';
1011
import { health } from './routes/health';
1112
import { query } from './routes/query';
1213

@@ -42,6 +43,7 @@ const app = new Elysia()
4243
)
4344
.use(query)
4445
.use(assistant)
46+
.use(exportRoute)
4547
.all('/trpc/*', ({ request }) => {
4648
return fetchRequestHandler({
4749
endpoint: '/trpc',

‎apps/api/src/lib/export/data-fetcher.ts‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
// Data fetching logic for exports
2+
3+
import { chQuery } from '@databuddy/db';
4+
import type {
5+
SanitizedEvent,
6+
SanitizedError,
7+
SanitizedWebVitals,
8+
ExportRequest
9+
} from './types';
10+
import {
11+
buildDateFilter,
12+
getEventsQuery,
13+
getErrorsQuery,
14+
getWebVitalsQuery
15+
} from './queries';
16+
17+
export interface ExportData {
18+
events: SanitizedEvent[];
19+
errors: SanitizedError[];
20+
webVitals: SanitizedWebVitals[];
21+
}
22+
23+
export async function fetchExportData(request: ExportRequest): Promise<ExportData> {
24+
const { website_id: websiteId, start_date: startDate, end_date: endDate } = request;
25+
26+
// Build secure date filter with parameters
27+
const { filter: dateFilter, params: dateParams } = buildDateFilter(startDate, endDate);
28+
29+
// Prepare queries
30+
const eventsQuery = getEventsQuery(dateFilter);
31+
const errorsQuery = getErrorsQuery(dateFilter);
32+
const webVitalsQuery = getWebVitalsQuery(dateFilter);
33+
34+
// Combine parameters: websiteId + date parameters
35+
const queryParams = { websiteId, ...dateParams };
36+
37+
// Execute queries in parallel with secure parameters
38+
const [events, errors, webVitals] = await Promise.all([
39+
chQuery<SanitizedEvent>(eventsQuery, queryParams),
40+
chQuery<SanitizedError>(errorsQuery, queryParams),
41+
chQuery<SanitizedWebVitals>(webVitalsQuery, queryParams),
42+
]);
43+
44+
return {
45+
events,
46+
errors,
47+
webVitals,
48+
};
49+
}

‎apps/api/src/lib/export/file-generator.ts‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
// File generation and ZIP creation
2+
3+
import JSZip from 'jszip';
4+
import type {
5+
ExportFile,
6+
ExportFormat,
7+
ExportMetadata,
8+
ExportRequest
9+
} from './types';
10+
import type { ExportData } from './data-fetcher';
11+
import { formatData, getFileExtension } from './formatters';
12+
13+
export function generateExportFiles(
14+
data: ExportData,
15+
format: ExportFormat
16+
): ExportFile[] {
17+
const extension = getFileExtension(format);
18+
19+
return [
20+
{
21+
name: `events.${extension}`,
22+
content: formatData(data.events, format, 'Event'),
23+
},
24+
{
25+
name: `errors.${extension}`,
26+
content: formatData(data.errors, format, 'Error'),
27+
},
28+
{
29+
name: `web_vitals.${extension}`,
30+
content: formatData(data.webVitals, format, 'WebVital'),
31+
},
32+
];
33+
}
34+
35+
export function generateMetadataFile(
36+
request: ExportRequest,
37+
data: ExportData
38+
): ExportFile {
39+
const metadata: ExportMetadata = {
40+
export_date: new Date().toISOString(),
41+
website_id: request.website_id,
42+
date_range: {
43+
start: request.start_date || 'all_time',
44+
end: request.end_date || 'all_time',
45+
},
46+
format: request.format || 'json',
47+
counts: {
48+
events: data.events.length,
49+
errors: data.errors.length,
50+
web_vitals: data.webVitals.length,
51+
},
52+
};
53+
54+
return {
55+
name: 'metadata.json',
56+
content: JSON.stringify(metadata, null, 2),
57+
};
58+
}
59+
60+
export async function createZipBuffer(files: ExportFile[]): Promise<Buffer> {
61+
const zip = new JSZip();
62+
63+
for (const file of files) {
64+
zip.file(file.name, file.content);
65+
}
66+
67+
return await zip.generateAsync({ type: 'nodebuffer' });
68+
}
69+
70+
export function generateExportFilename(websiteId: string): string {
71+
const date = new Date().toISOString().slice(0, 10);
72+
return `databuddy_export_${websiteId}_${date}.zip`;
73+
}

‎apps/api/src/lib/export/formatters.ts‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
// Data formatting functions for different export formats
2+
3+
import type { ExportFormat } from './types';
4+
5+
export function convertToCSV<T extends Record<string, unknown>>(data: T[]): string {
6+
if (data.length === 0) return '';
7+
8+
const headers = Object.keys(data[0] || {}).join(',');
9+
const rows = data
10+
.map((row) =>
11+
Object.values(row)
12+
.map((value) => {
13+
if (value === null || value === undefined) {
14+
return '';
15+
}
16+
const stringValue = String(value);
17+
// Escape commas, quotes, and newlines in CSV
18+
if (
19+
stringValue.includes(',') ||
20+
stringValue.includes('"') ||
21+
stringValue.includes('\n')
22+
) {
23+
return `"${stringValue.replace(/"/g, '""')}"`;
24+
}
25+
return stringValue;
26+
})
27+
.join(',')
28+
)
29+
.join('\n');
30+
31+
return `${headers}\n${rows}`;
32+
}
33+
34+
export function convertToTXT<T extends Record<string, unknown>>(data: T[]): string {
35+
if (data.length === 0) return '';
36+
37+
const headers = Object.keys(data[0] || {}).join('\t');
38+
const rows = data
39+
.map((row) =>
40+
Object.values(row)
41+
.map((value) => {
42+
if (value === null || value === undefined) {
43+
return '';
44+
}
45+
// Replace tabs and newlines to maintain format
46+
return String(value).replace(/[\t\n\r]/g, ' ');
47+
})
48+
.join('\t')
49+
)
50+
.join('\n');
51+
52+
return `${headers}\n${rows}`;
53+
}
54+
55+
export function convertToProto<T extends Record<string, unknown>>(
56+
data: T[],
57+
typeName: string
58+
): string {
59+
if (data.length === 0) return '';
60+
61+
let protoContent = `# Protocol Buffer Text Format\n# Type: ${typeName}\n\n`;
62+
63+
for (const [index, row] of data.entries()) {
64+
protoContent += `${typeName} {\n`;
65+
66+
for (const [key, value] of Object.entries(row)) {
67+
if (value !== null && value !== undefined) {
68+
const fieldName = key.toLowerCase().replace(/[^a-z0-9_]/g, '_');
69+
70+
if (typeof value === 'string') {
71+
// Escape quotes in string values
72+
const escapedValue = value.replace(/"/g, '\\"').replace(/\n/g, '\\n');
73+
protoContent += ` ${fieldName}: "${escapedValue}"\n`;
74+
} else if (typeof value === 'number') {
75+
protoContent += ` ${fieldName}: ${value}\n`;
76+
} else if (typeof value === 'boolean') {
77+
protoContent += ` ${fieldName}: ${value}\n`;
78+
} else {
79+
// Convert other types to string
80+
const stringValue = String(value).replace(/"/g, '\\"').replace(/\n/g, '\\n');
81+
protoContent += ` ${fieldName}: "${stringValue}"\n`;
82+
}
83+
}
84+
}
85+
86+
protoContent += '}\n';
87+
if (index < data.length - 1) {
88+
protoContent += '\n';
89+
}
90+
}
91+
92+
return protoContent;
93+
}
94+
95+
export function formatData<T extends Record<string, unknown>>(
96+
data: T[],
97+
format: ExportFormat,
98+
typeName: string
99+
): string {
100+
switch (format) {
101+
case 'csv':
102+
return convertToCSV(data);
103+
case 'txt':
104+
return convertToTXT(data);
105+
case 'proto':
106+
return convertToProto(data, typeName);
107+
case 'json':
108+
default:
109+
return JSON.stringify(data, null, 2);
110+
}
111+
}
112+
113+
export function getFileExtension(format: ExportFormat): string {
114+
switch (format) {
115+
case 'csv':
116+
return 'csv';
117+
case 'txt':
118+
return 'txt';
119+
case 'proto':
120+
return 'proto.txt';
121+
case 'json':
122+
default:
123+
return 'json';
124+
}
125+
}

‎apps/api/src/lib/export/index.ts‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
// Main export orchestrator
2+
3+
import { logger } from '../logger';
4+
import type { ExportRequest } from './types';
5+
import { fetchExportData } from './data-fetcher';
6+
import {
7+
generateExportFiles,
8+
generateMetadataFile,
9+
createZipBuffer,
10+
generateExportFilename
11+
} from './file-generator';
12+
13+
export interface ExportResult {
14+
buffer: Buffer;
15+
filename: string;
16+
metadata: {
17+
websiteId: string;
18+
format: string;
19+
totalRecords: number;
20+
fileSize: number;
21+
};
22+
}
23+
24+
export async function processExport(request: ExportRequest): Promise<ExportResult> {
25+
const { website_id: websiteId, format = 'json' } = request;
26+
27+
logger.info('Starting data export', {
28+
websiteId,
29+
startDate: request.start_date,
30+
endDate: request.end_date,
31+
format,
32+
});
33+
34+
// Fetch data from ClickHouse
35+
const data = await fetchExportData(request);
36+
37+
logger.info('Data export queries completed', {
38+
websiteId,
39+
eventsCount: data.events.length,
40+
errorsCount: data.errors.length,
41+
webVitalsCount: data.webVitals.length,
42+
});
43+
44+
// Generate export files
45+
const exportFiles = generateExportFiles(data, format);
46+
const metadataFile = generateMetadataFile(request, data);
47+
const allFiles = [...exportFiles, metadataFile];
48+
49+
// Create ZIP buffer
50+
const buffer = await createZipBuffer(allFiles);
51+
const filename = generateExportFilename(websiteId);
52+
53+
const totalRecords = data.events.length + data.errors.length + data.webVitals.length;
54+
55+
logger.info('Data export completed successfully', {
56+
websiteId,
57+
filename,
58+
totalSize: buffer.length,
59+
totalRecords,
60+
});
61+
62+
return {
63+
buffer,
64+
filename,
65+
metadata: {
66+
websiteId,
67+
format,
68+
totalRecords,
69+
fileSize: buffer.length,
70+
},
71+
};
72+
}
73+
74+
// Re-export types for convenience
75+
export type {
76+
ExportRequest,
77+
ExportFormat,
78+
SanitizedEvent,
79+
SanitizedError,
80+
SanitizedWebVitals
81+
} from './types';

0 commit comments

Comments
 (0)