fix: perf issues with high memory usage in summary time series queries

Author: izadoesdev

apps/api/src/query/builders/summary.ts

@@ -85,15 +85,7 @@ export const SummaryBuilders: Record<string, SimpleQueryConfig> = {
 				e.session_id,
 				e.anonymous_id,
 				e.event_name,
-				toTimeZone(e.time, {timezone:String}) as normalized_time,
-				sa.session_referrer as referrer,
-				sa.session_utm_source as utm_source,
-				sa.session_utm_medium as utm_medium,
-				sa.session_utm_campaign as utm_campaign,
-				sa.session_country as country,
-				sa.session_device_type as device_type,
-				sa.session_browser_name as browser_name,
-				sa.session_os_name as os_name
+				toTimeZone(e.time, {timezone:String}) as normalized_time
 			FROM analytics.events e
 			${helpers.sessionAttributionJoin('e')}
 			WHERE 
@@ -318,15 +310,7 @@ export const SummaryBuilders: Record<string, SimpleQueryConfig> = {
                     e.session_id,
                     e.anonymous_id,
                     e.event_name,
-                    toTimeZone(e.time, {timezone:String}) as normalized_time,
-                    sa.session_referrer as referrer,
-                    sa.session_utm_source as utm_source,
-                    sa.session_utm_medium as utm_medium,
-                    sa.session_utm_campaign as utm_campaign,
-                    sa.session_country as country,
-                    sa.session_device_type as device_type,
-                    sa.session_browser_name as browser_name,
-                    sa.session_os_name as os_name
+                    toTimeZone(e.time, {timezone:String}) as normalized_time
                   FROM analytics.events e
                   ${helpers.sessionAttributionJoin('e')}
                   WHERE 
@@ -356,12 +340,6 @@ export const SummaryBuilders: Record<string, SimpleQueryConfig> = {
 					sql: `
                 WITH ${sessionAttributionCTE}
                 ${baseEventsQuery}
-                hour_range AS (
-                  SELECT arrayJoin(arrayMap(
-                    h -> toStartOfHour(toTimeZone(toDateTime(concat({startDate:String}, ' 00:00:00')) + (h * 3600), {timezone:String})),
-                    range(toUInt32(dateDiff('hour', toDateTime(concat({startDate:String}, ' 00:00:00')), toDateTime(concat({endDate:String}, ' 23:59:59'))) + 1))
-                  )) AS datetime
-                ),
                 session_details AS (
                   SELECT
                     session_id,
@@ -389,9 +367,9 @@ export const SummaryBuilders: Record<string, SimpleQueryConfig> = {
                   GROUP BY event_hour
                 )
                 SELECT
-                  formatDateTime(hr.datetime, '%Y-%m-%d %H:00:00') as date,
-                  COALESCE(hem.pageviews, 0) as pageviews,
-                  COALESCE(hem.unique_visitors, 0) as visitors,
+                  formatDateTime(hem.event_hour, '%Y-%m-%d %H:00:00') as date,
+                  hem.pageviews as pageviews,
+                  hem.unique_visitors as visitors,
                   COALESCE(hsm.sessions, 0) as sessions,
                   ROUND(CASE 
                     WHEN COALESCE(hsm.sessions, 0) > 0 
@@ -401,13 +379,12 @@ export const SummaryBuilders: Record<string, SimpleQueryConfig> = {
                   ROUND(COALESCE(hsm.median_session_duration, 0), 2) as avg_session_duration,
                   ROUND(CASE 
                     WHEN COALESCE(hsm.sessions, 0) > 0 
-                    THEN COALESCE(hem.pageviews, 0) / COALESCE(hsm.sessions, 0) 
+                    THEN hem.pageviews / COALESCE(hsm.sessions, 0) 
                     ELSE 0 
                   END, 2) as pages_per_session
-                FROM hour_range hr
-                LEFT JOIN hourly_session_metrics hsm ON hr.datetime = hsm.event_hour
-                LEFT JOIN hourly_event_metrics hem ON hr.datetime = hem.event_hour
-                ORDER BY hr.datetime ASC
+                FROM hourly_event_metrics hem
+                LEFT JOIN hourly_session_metrics hsm ON hem.event_hour = hsm.event_hour
+                ORDER BY hem.event_hour ASC
             `,
 					params: {
 						websiteId,
@@ -431,15 +408,7 @@ export const SummaryBuilders: Record<string, SimpleQueryConfig> = {
                     e.session_id,
                     e.anonymous_id,
                     e.event_name,
-                    toTimeZone(e.time, {timezone:String}) as normalized_time,
-                    sa.session_referrer as referrer,
-                    sa.session_utm_source as utm_source,
-                    sa.session_utm_medium as utm_medium,
-                    sa.session_utm_campaign as utm_campaign,
-                    sa.session_country as country,
-                    sa.session_device_type as device_type,
-                    sa.session_browser_name as browser_name,
-                    sa.session_os_name as os_name
+                    toTimeZone(e.time, {timezone:String}) as normalized_time
                   FROM analytics.events e
                   ${helpers.sessionAttributionJoin('e')}
                   WHERE
@@ -469,12 +438,6 @@ export const SummaryBuilders: Record<string, SimpleQueryConfig> = {
 				sql: `
                 WITH ${sessionAttributionCTE}
                 ${baseEventsQuery}
-                date_range AS (
-                  SELECT arrayJoin(arrayMap(
-                    d -> toDate({startDate:String}) + d,
-                    range(toUInt32(dateDiff('day', toDate({startDate:String}), toDate({endDate:String})) + 1))
-                  )) AS date
-                ),
                 session_details AS (
                   SELECT
                     session_id,
@@ -502,9 +465,9 @@ export const SummaryBuilders: Record<string, SimpleQueryConfig> = {
                   GROUP BY event_date
                 )
                 SELECT
-                  dr.date,
-                  COALESCE(dem.pageviews, 0) as pageviews,
-                  COALESCE(dem.unique_visitors, 0) as visitors,
+                  dem.event_date as date,
+                  dem.pageviews as pageviews,
+                  dem.unique_visitors as visitors,
                   COALESCE(dsm.sessions, 0) as sessions,
                   ROUND(CASE 
                     WHEN COALESCE(dsm.sessions, 0) > 0 
@@ -514,13 +477,12 @@ export const SummaryBuilders: Record<string, SimpleQueryConfig> = {
                   ROUND(COALESCE(dsm.median_session_duration, 0), 2) as avg_session_duration,
                   ROUND(CASE 
                     WHEN COALESCE(dsm.sessions, 0) > 0 
-                    THEN COALESCE(dem.pageviews, 0) / COALESCE(dsm.sessions, 0) 
+                    THEN dem.pageviews / COALESCE(dsm.sessions, 0) 
                     ELSE 0 
                   END, 2) as pages_per_session
-                FROM date_range dr
-                LEFT JOIN daily_session_metrics dsm ON dr.date = dsm.session_start_date
-                LEFT JOIN daily_event_metrics dem ON dr.date = dem.event_date
-                ORDER BY dr.date ASC
+                FROM daily_event_metrics dem
+                LEFT JOIN daily_session_metrics dsm ON dem.event_date = dsm.session_start_date
+                ORDER BY dem.event_date ASC
             `,
 				params: {
 					websiteId,

apps/dashboard/app/(main)/websites/[id]/_components/tabs/overview-tab.tsx

@@ -340,11 +340,82 @@ export function WebsiteOverviewTab({
 		[dateRange.granularity]
 	);
 
-	const chartData = useMemo(() => {
-		if (!analytics.events_by_date?.length) return [];
+	const fillMissingDates = useCallback((data: MetricPoint[]) => {
+		const userTimezone = getUserTimezone();
+		const isHourly = dateRange.granularity === 'hourly';
+		
+		// Create a map of existing data for quick lookup
+		const dataMap = new Map<string, MetricPoint>();
+		for (const item of data) {
+			const key = isHourly ? item.date : item.date.slice(0, 10);
+			dataMap.set(key, item);
+		}
+
+		// Generate all dates in range
+		const startDate = dayjs(dateRange.start_date).tz(userTimezone);
+		const endDate = dayjs(dateRange.end_date).tz(userTimezone);
+		const now = dayjs().tz(userTimezone);
+		
+		const filled: MetricPoint[] = [];
+		let current = startDate;
+
+		while (current.isBefore(endDate) || current.isSame(endDate, 'day')) {
+			if (isHourly) {
+				// Hourly granularity - iterate through hours
+				for (let hour = 0; hour < 24; hour++) {
+					const hourDate = current.hour(hour);
+					
+					// Don't add future hours
+					if (hourDate.isAfter(now)) break;
+					
+					const key = hourDate.format('YYYY-MM-DD HH:00:00');
+					const existing = dataMap.get(key);
+					
+					filled.push(existing || {
+						date: key,
+						pageviews: 0,
+						visitors: 0,
+						unique_visitors: 0,
+						sessions: 0,
+						bounce_rate: 0,
+						avg_session_duration: 0,
+						pages_per_session: 0,
+					});
+				}
+				current = current.add(1, 'day');
+				
+				// If we've reached today, stop after processing current hour
+				if (current.isAfter(endDate, 'day')) break;
+			} else {
+				// Daily granularity
+				const key = current.format('YYYY-MM-DD');
+				const existing = dataMap.get(key);
+				
+				filled.push(existing || {
+					date: key,
+					pageviews: 0,
+					visitors: 0,
+					unique_visitors: 0,
+					sessions: 0,
+					bounce_rate: 0,
+					avg_session_duration: 0,
+					pages_per_session: 0,
+				});
+				
+				current = current.add(1, 'day');
+			}
+		}
+
+		return filled;
+	}, [dateRange.start_date, dateRange.end_date, dateRange.granularity]);
 
-		const filteredEvents = filterFutureEvents(analytics.events_by_date);
-		return filteredEvents.map((event: MetricPoint): ChartDataPoint => ({
+	const chartData = useMemo(() => {
+		const filteredEvents = analytics.events_by_date?.length 
+			? filterFutureEvents(analytics.events_by_date) 
+			: [];
+		const filledEvents = fillMissingDates(filteredEvents);
+		
+		return filledEvents.map((event: MetricPoint): ChartDataPoint => ({
 			date: formatDateByGranularity(event.date, dateRange.granularity),
 			rawDate: event.date,
 			...(visibleMetrics.pageviews && { pageviews: event.pageviews as number }),
@@ -360,19 +431,24 @@ export function WebsiteOverviewTab({
 	}, [
 		analytics.events_by_date,
 		dateRange.granularity,
+		dateRange.start_date,
+		dateRange.end_date,
 		visibleMetrics,
 		filterFutureEvents,
+		fillMissingDates,
 	]);
 
 	const miniChartData = useMemo(() => {
-		if (!analytics.events_by_date?.length) return {};
-
-		const filteredEvents = filterFutureEvents(analytics.events_by_date);
+		const filteredEvents = analytics.events_by_date?.length 
+			? filterFutureEvents(analytics.events_by_date) 
+			: [];
+		const filledEvents = fillMissingDates(filteredEvents);
+		
 		const createChartSeries = (
 			field: keyof MetricPoint,
 			transform?: (value: number) => number
 		) =>
-			filteredEvents.map((event: MetricPoint) => ({
+			filledEvents.map((event: MetricPoint) => ({
 				date: dateRange.granularity === 'hourly' ? event.date : event.date.slice(0, 10),
 				value: transform ? transform(event[field] as number) : (event[field] as number) || 0,
 			}));
@@ -392,7 +468,7 @@ export function WebsiteOverviewTab({
 			bounceRate: createChartSeries('bounce_rate'),
 			sessionDuration: createChartSeries('avg_session_duration', formatSessionDuration),
 		};
-	}, [analytics.events_by_date, dateRange.granularity, filterFutureEvents]);
+	}, [analytics.events_by_date, dateRange.granularity, filterFutureEvents, fillMissingDates]);
 
 	const createTechnologyCell = (type: 'browser' | 'os') => (info: CellInfo) => {
 		const entry = info.row.original as TechnologyData;

apps/dashboard/app/(main)/websites/[id]/_components/utils/date-filling.ts

@@ -0,0 +1 @@
+ 

apps/dashboard/package.json

@@ -11,8 +11,13 @@
   },
   "dependencies": {
     "@ai-sdk/react": "^2.0.68",
+    "@databuddy/auth": "workspace:*",
+    "@databuddy/db": "workspace:*",
     "@databuddy/env": "workspace:*",
+    "@databuddy/redis": "workspace:*",
+    "@databuddy/rpc": "workspace:*",
     "@databuddy/sdk": "workspace:*",
+    "@databuddy/shared": "workspace:*",
     "@databuddy/validation": "workspace:*",
     "@hello-pangea/dnd": "^18.0.1",
     "@hookform/resolvers": "^5.2.2",
@@ -39,8 +44,9 @@
     "@types/geojson": "^7946.0.16",
     "@types/leaflet": "^1.9.21",
     "ai": "^5.0.68",
-    "autumn-js": "^0.0.101",
+    "autumn-js": "^0.1.40",
     "babel-plugin-react-compiler": "^19.1.0-rc.1-rc-af1b7da-20250421",
+    "better-auth": "^1.3.27",
     "class-variance-authority": "^0.7.1",
     "classnames": "^2.5.1",
     "clsx": "^2.1.1",