Fixed stats endpoints inconsistently handling timezone (#24574)

ref https://linear.app/ghost/issue/PROD-2373/
- more declarative about passing timezone with date_from and date_to in
stats endpoint options
- centralized date calculations and date filter application to knex
query objects; this allows us to cleanly handle timezone adjustments if
someone is on the day before/after UTC's current date

__The Problem__
We do not consistently handle date translations w/r to timezones. We
pass the `date_from`, `date_to`, and `timezone` options to Tinybird,
which does handle timezone translations, and passes the data back in a
localized format. We __do not__ currently do that with Ghost data.

__Solution__
For the moment, we'll centralize utils for translating `date_from` and
`date_to` to UTC dates based on the provided `timezone`, and clean up
the endpoints by using a centralized filter, ensuring we're applying the
date filtering in the same way everywhere.

Ideally, we'll set standards around endpoints always being in UTC, and
letting the caller handle the conversion, which feels more appropriate
(the client knows its timezone, why does the server need to know it?).

Author: 9larsons

ghost/core/core/server/api/endpoints/stats.js

@@ -533,12 +533,7 @@ const controller = {
             };
         },
         async query(frame) {
-            return await statsService.api.getTopSourcesWithRange(
-                frame.options.date_from, 
-                frame.options.date_to, 
-                frame.options.order || 'free_members desc',
-                frame.options.limit || 50
-            );
+            return await statsService.api.getTopSourcesWithRange(frame.options);
         }
     }
 

ghost/core/core/server/services/stats/PostsStatsService.js

@@ -1,5 +1,8 @@
 const moment = require('moment');
 
+// Import centralized date utilities
+const {getDateBoundaries, applyDateFilter} = require('./utils/date-utils');
+
 // Source normalization mapping - consolidated from frontend apps
 const SOURCE_NORMALIZATION_MAP = new Map([
     // Social Media Consolidation
@@ -252,45 +255,44 @@ class ReferrersStatsService {
 
     /**
      * Fetch MRR sources with date range
-     * @param {string} startDate 
-     * @param {string} endDate 
+     * @param {Object} options
      * @returns {Promise<MrrCountStatDate[]>}
      **/
-    async fetchMrrSourcesWithRange(startDate, endDate) {
+    async fetchMrrSourcesWithRange(options) {
         const knex = this.knex;
-        const startDateTime = moment.utc(startDate).startOf('day').format('YYYY-MM-DD HH:mm:ss');
-        const endDateTime = moment.utc(endDate).endOf('day').format('YYYY-MM-DD HH:mm:ss');
+        const {dateFrom: startDateTime, dateTo: endDateTime} = getDateBoundaries(options);
 
         // Join subscription created events with paid subscription events to get MRR changes
-        const rows = await knex('members_subscription_created_events as msce')
+        let query = knex('members_subscription_created_events as msce')
             .join('members_paid_subscription_events as mpse', function () {
                 this.on('msce.member_id', '=', 'mpse.member_id')
                     .andOn('msce.subscription_id', '=', 'mpse.subscription_id');
             })
             .select(knex.raw(`DATE(msce.created_at) as date`))
             .select(knex.raw(`SUM(mpse.mrr_delta) as mrr`))
             .select(knex.raw(`msce.referrer_source as source`))
-            .where('msce.created_at', '>=', startDateTime)
-            .where('msce.created_at', '<=', endDateTime)
             .where('mpse.mrr_delta', '>', 0) // Only positive MRR changes (new subscriptions)
             .whereNotNull('msce.referrer_source') // Only entries with attribution
             .groupBy('date', 'msce.referrer_source')
             .orderBy('date');
+            
+        // Apply centralized date filtering
+        applyDateFilter(query, startDateTime, endDateTime, 'msce.created_at');
+        
+        const rows = await query;
 
         return rows;
     }
 
     /**
      * Fetch deduplicated member counts by source with date range
      * Returns both free signups (excluding those who converted) and paid conversions
-     * @param {string} startDate 
-     * @param {string} endDate 
+     * @param {Object} options
      * @returns {Promise<{source: string, signups: number, paid_conversions: number}[]>}
      **/
-    async fetchMemberCountsBySource(startDate, endDate) {
+    async fetchMemberCountsBySource(options) {
         const knex = this.knex;
-        const startDateTime = moment.utc(startDate).startOf('day').format('YYYY-MM-DD HH:mm:ss');
-        const endDateTime = moment.utc(endDate).endOf('day').format('YYYY-MM-DD HH:mm:ss');
+        const {dateFrom: startDateTime, dateTo: endDateTime} = getDateBoundaries(options);
 
         // Query 1: Free members who haven't converted to paid within the same time window
         const freeSignupsQuery = knex('members_created_events as mce')
@@ -302,18 +304,20 @@ class ReferrersStatsService {
                     .andOn('msce.created_at', '>=', knex.raw('?', [startDateTime]))
                     .andOn('msce.created_at', '<=', knex.raw('?', [endDateTime]));
             })
-            .where('mce.created_at', '>=', startDateTime)
-            .where('mce.created_at', '<=', endDateTime)
             .whereNull('msce.id')
             .groupBy('mce.referrer_source');
+            
+        // Apply date filtering to the main query
+        applyDateFilter(freeSignupsQuery, startDateTime, endDateTime, 'mce.created_at');
 
         // Query 2: Paid conversions
         const paidConversionsQuery = knex('members_subscription_created_events as msce')
             .select('msce.referrer_source as source')
             .select(knex.raw('COUNT(DISTINCT msce.member_id) as paid_conversions'))
-            .where('msce.created_at', '>=', startDateTime)
-            .where('msce.created_at', '<=', endDateTime)
             .groupBy('msce.referrer_source');
+            
+        // Apply date filtering to the paid conversions query
+        applyDateFilter(paidConversionsQuery, startDateTime, endDateTime, 'msce.created_at');
 
         // Execute both queries in parallel
         const [freeResults, paidResults] = await Promise.all([
@@ -353,17 +357,21 @@ class ReferrersStatsService {
     /**
      * Return aggregated attribution sources for a date range, grouped by source only (not by date)
      * This is used for "Top Sources" tables that need server-side sorting
-     * @param {string} startDate - Start date in YYYY-MM-DD format
-     * @param {string} endDate - End date in YYYY-MM-DD format
-     * @param {string} [orderBy='signups desc'] - Sort order: 'signups desc', 'paid_conversions desc', 'mrr desc', 'source desc'
-     * @param {number} [limit=50] - Maximum number of sources to return
+     * @param {Object} options
+     * @param {string} [options.date_from] - Start date in YYYY-MM-DD format
+     * @param {string} [options.date_to] - End date in YYYY-MM-DD format
+     * @param {string} [options.timezone] - Timezone to use for date interpretation
+     * @param {string} [options.orderBy='signups desc'] - Sort order: 'signups desc', 'paid_conversions desc', 'mrr desc', 'source desc'
+     * @param {number} [options.limit=50] - Maximum number of sources to return
      * @returns {Promise<{data: AttributionCountStatWithMrr[], meta: {}}>}
      */
-    async getTopSourcesWithRange(startDate, endDate, orderBy = 'signups desc', limit = 50) {
+    async getTopSourcesWithRange(options = {}) {
+        const {orderBy = 'signups desc', limit = 50} = options;
+        
         // Get deduplicated member counts and MRR data in parallel
         const [memberCounts, mrrEntries] = await Promise.all([
-            this.fetchMemberCountsBySource(startDate, endDate),
-            this.fetchMrrSourcesWithRange(startDate, endDate)
+            this.fetchMemberCountsBySource(options),
+            this.fetchMrrSourcesWithRange(options) 
         ]);
 
         // Aggregate by source (not by date + source)

ghost/core/core/server/services/stats/ReferrersStatsService.js

@@ -123,6 +123,7 @@ class StatsService {
      * @param {number} [options.limit=20] - Max number of results to return
      * @param {string} [options.date_from] - Start date filter in YYYY-MM-DD format
      * @param {string} [options.date_to] - End date filter in YYYY-MM-DD format
+     * @param {string} [options.timezone] - Timezone to use for date interpretation
      * @returns {Promise<{data: import('./PostsStatsService').NewsletterStatResult[]}>}
      */
     async getNewsletterStats(options = {}) {
@@ -146,6 +147,7 @@ class StatsService {
      * @param {string} [options.newsletter_id] - ID of the specific newsletter to get stats for
      * @param {string} [options.date_from] - Start date filter in YYYY-MM-DD format
      * @param {string} [options.date_to] - End date filter in YYYY-MM-DD format
+     * @param {string} [options.timezone] - Timezone to use for date interpretation
      * @returns {Promise<{data: import('./PostsStatsService').NewsletterSubscriberStats[]}>}
      */
     async getNewsletterSubscriberStats(options = {}) {
@@ -190,6 +192,7 @@ class StatsService {
      * @param {string} [options.newsletter_id] - ID of the specific newsletter to get stats for
      * @param {string} [options.order='published_at desc'] - Order field and direction
      * @param {number} [options.limit=20] - Max number of results to return
+     * @param {string} [options.timezone] - Timezone to use for date interpretation
      * @param {string} [options.date_from] - Start date filter in YYYY-MM-DD format
      * @param {string} [options.date_to] - End date filter in YYYY-MM-DD format
      * @returns {Promise<{data: import('./PostsStatsService').NewsletterStatResult[]}>}
@@ -229,8 +232,16 @@ class StatsService {
         return result;
     }
 
-    async getTopSourcesWithRange(startDate, endDate, orderBy, limit) {
-        return this.referrers.getTopSourcesWithRange(startDate, endDate, orderBy, limit);
+    /**
+     * @param {Object} options
+     * @param {string} [options.date_from] - Start date in YYYY-MM-DD format
+     * @param {string} [options.date_to] - End date in YYYY-MM-DD format
+     * @param {string} [options.timezone] - Timezone to use for date interpretation
+     * @param {string} [options.orderBy='signups desc'] - Sort order: 'signups desc', 'paid_conversions desc', 'mrr desc', 'source desc'
+     * @param {number} [options.limit=50] - Maximum number of sources to return
+     */
+    async getTopSourcesWithRange(options = {}) {
+        return this.referrers.getTopSourcesWithRange(options);
     }
 
     /**

ghost/core/core/server/services/stats/StatsService.js

@@ -0,0 +1,37 @@
+const moment = require('moment-timezone');
+
+/**
+ * Get processed date boundaries with timezone support
+ * @param {Object} options - Options containing date and timezone info
+ * @param {string} [options.date_from] - Start date in YYYY-MM-DD format
+ * @param {string} [options.date_to] - End date in YYYY-MM-DD format
+ * @param {string} [options.timezone='UTC'] - Timezone for date interpretation
+ * @returns {{dateFrom: string|null, dateTo: string|null}} Processed dates in ISO format
+ */
+function getDateBoundaries(options) {
+    const timezone = options.timezone || 'UTC';
+    const dateFrom = options.date_from ? moment.tz(options.date_from, timezone).startOf('day').utc().toISOString() : null;
+    const dateTo = options.date_to ? moment.tz(options.date_to, timezone).endOf('day').utc().toISOString() : null;
+    return {dateFrom, dateTo};
+}
+
+/**
+ * Apply date filters to a query builder instance
+ * @param {import('knex').Knex.QueryBuilder} query - The query builder to apply filters to
+ * @param {string|null} dateFrom - Start date in ISO format
+ * @param {string|null} dateTo - End date in ISO format
+ * @param {string} dateColumn - The date column to filter on
+ */
+function applyDateFilter(query, dateFrom, dateTo, dateColumn) {
+    if (dateFrom) {
+        query.where(dateColumn, '>=', dateFrom);
+    }
+    if (dateTo) {
+        query.where(dateColumn, '<=', dateTo);
+    }
+}
+
+module.exports = {
+    getDateBoundaries,
+    applyDateFilter
+};

ghost/core/core/server/services/stats/utils/date-utils.js

@@ -1,5 +1,6 @@
 const knex = require('knex').default;
 const assert = require('assert/strict');
+const moment = require('moment-timezone');
 const PostsStatsService = require('../../../../../core/server/services/stats/PostsStatsService');
 
 /**
@@ -81,7 +82,7 @@ describe('PostsStatsService', function () {
             attribution_url: `/${postId.replace('post', 'post-')}/`,
             referrer_source: referrerSource,
             referrer_url: referrerSource ? `https://${referrerSource}` : null,
-            created_at: createdAt,
+            created_at: moment(createdAt).utc().format('YYYY-MM-DD HH:mm:ss'),
             source: 'member'
         });
     }
@@ -100,15 +101,15 @@ describe('PostsStatsService', function () {
             attribution_url: `/${postId.replace('post', 'post-')}/`,
             referrer_source: referrerSource,
             referrer_url: referrerSource ? `https://${referrerSource}` : null,
-            created_at: createdAt
+            created_at: moment(createdAt).utc().format('YYYY-MM-DD HH:mm:ss')
         });
 
         await db('members_paid_subscription_events').insert({
             id: paidEventId,
             member_id: memberId,
             subscription_id: subscriptionId,
             mrr_delta: mrr,
-            created_at: createdAt
+            created_at: moment(createdAt).utc().format('YYYY-MM-DD HH:mm:ss')
         });
     }
 
@@ -155,7 +156,7 @@ describe('PostsStatsService', function () {
             id: clickEventId,
             member_id: memberId,
             redirect_id: redirectId,
-            created_at: createdAt
+            created_at: moment(createdAt).utc().format('YYYY-MM-DD HH:mm:ss')
         });
     }
 
@@ -417,8 +418,9 @@ describe('PostsStatsService', function () {
             await _createFreeSignupEvent('post2', 'member_future', 'referrer_future', thirtyDaysAgo);
 
             const lastFifteenDaysResult = await service.getTopPosts({
-                date_from: fifteenDaysAgo,
-                date_to: new Date()
+                date_from: moment(fifteenDaysAgo).format('YYYY-MM-DD'),
+                date_to: moment().format('YYYY-MM-DD'),
+                timezone: 'UTC'
             });
 
             // Only post1 should be returned since post2 has no events in the date range
@@ -428,8 +430,9 @@ describe('PostsStatsService', function () {
 
             // Test filtering to include both dates
             const lastThirtyDaysResult = await service.getTopPosts({
-                date_from: sixtyDaysAgo,
-                date_to: new Date()
+                date_from: moment(sixtyDaysAgo).format('YYYY-MM-DD'),
+                date_to: moment().format('YYYY-MM-DD'),
+                timezone: 'UTC'
             });
 
             // Both posts should be returned
@@ -585,8 +588,9 @@ describe('PostsStatsService', function () {
             await _createFreeSignupEvent('post1', 'member_future', 'referrer_future', thirtyDaysAgo);
 
             const lastFifteenDaysResult = await service.getReferrersForPost('post1', {
-                date_from: fifteenDaysAgo,
-                date_to: new Date()
+                date_from: moment(fifteenDaysAgo).format('YYYY-MM-DD'),
+                date_to: moment().format('YYYY-MM-DD'),
+                timezone: 'UTC'
             });
 
             // Make sure we have the result for referrer_past
@@ -596,8 +600,9 @@ describe('PostsStatsService', function () {
 
             // Test filtering to include both dates
             const lastThirtyDaysResult = await service.getReferrersForPost('post1', {
-                date_from: sixtyDaysAgo,
-                date_to: new Date()
+                date_from: moment(sixtyDaysAgo).format('YYYY-MM-DD'),
+                date_to: moment().format('YYYY-MM-DD'),
+                timezone: 'UTC'
             });
 
             // Make sure we have results for both referrers