Skip to content

Commit b21770f

Browse files
EspressoTrip-v2EspressoTrip-v2
committed
scrape data
1 parent cb46003 commit b21770f

File tree

5 files changed

+124
-53
lines changed

5 files changed

+124
-53
lines changed

‎modules/module-mongodb-storage/src/storage/MongoReportStorage.ts‎

Lines changed: 106 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -2,28 +2,71 @@ import { mongo } from '@powersync/lib-service-mongodb';
22
import { storage } from '@powersync/service-core';
33
import { event_types } from '@powersync/service-types';
44
import { PowerSyncMongo } from './implementation/db.js';
5-
import {
6-
ListCurrentConnections,
7-
ListCurrentConnectionsResponse,
8-
SdkConnectBucketData,
9-
SdkDisconnectEventData
10-
} from '@powersync/service-types/dist/events.js';
115

12-
function dateFilter(userId: string, clientId: string): mongo.Filter<mongo.Document> {
13-
const date = new Date();
6+
function parseDate(date: Date) {
147
const year = date.getFullYear();
158
const month = date.getMonth();
16-
const day = date.getDate();
17-
const query = {
9+
const today = date.getDate();
10+
const day = date.getDay();
11+
return {
12+
year,
13+
month,
14+
today,
15+
day
16+
};
17+
}
18+
function updateDocFilter(userId: string, clientId: string): mongo.Filter<mongo.Document> {
19+
const { year, month, today } = parseDate(new Date());
20+
const nextDay = today + 1;
21+
return {
1822
user_id: userId,
1923
client_id: clientId,
2024
connect_at: {
21-
$gte: new Date(year, month, day, 0, 0, 0),
22-
$lt: new Date(year, month, day + 1, 0, 0, 0)
25+
$gte: new Date(year, month, today, 0, 0, 0),
26+
$lt: new Date(year, month, nextDay, 0, 0, 0)
2327
}
2428
};
25-
console.log({ query, date });
26-
return query;
29+
}
30+
31+
function timeSpan(timeframe: event_types.TimeFrames): mongo.Filter<mongo.Document> {
32+
const date = new Date();
33+
const { year, month, day, today } = parseDate(date);
34+
switch (timeframe) {
35+
case event_types.TimeFrames.MONTH: {
36+
// Cron should run the first day of the new month, this then retrieves from the 1st to the last day of the month
37+
const thisMonth = month;
38+
const nextMonth = month == 11 ? 0 : month + 1;
39+
return {
40+
connect_at: { $gte: new Date(year, thisMonth, 1, 0, 0, 0), $lte: new Date(year, nextMonth, 1, 0, 0, 0) }
41+
};
42+
}
43+
case event_types.TimeFrames.WEEK: {
44+
// Back tracks the date to the previous week Monday to Sunday
45+
const daysToSunday = 0 - day;
46+
const weekEndDate = new Date(date);
47+
weekEndDate.setDate(weekEndDate.getDate() + daysToSunday);
48+
const weekStartDate = new Date(weekEndDate);
49+
weekStartDate.setDate(weekStartDate.getDate() - 6);
50+
const weekStart = parseDate(weekStartDate);
51+
const weekEnd = parseDate(weekEndDate);
52+
return {
53+
connect_at: {
54+
$gte: new Date(weekStart.year, weekStart.month, weekStart.today, 0, 0, 0),
55+
$lte: new Date(weekEnd.year, weekEnd.month, weekEnd.today, 0, 0, 0)
56+
}
57+
};
58+
}
59+
default: {
60+
// Start from today to just before tomorrow
61+
const tomorrow = today + 1;
62+
return {
63+
connect_at: {
64+
$gte: new Date(date.getFullYear(), date.getMonth(), today, 0, 0, 0),
65+
$lte: new Date(date.getFullYear(), date.getMonth(), tomorrow, 0, 0, 0)
66+
}
67+
};
68+
}
69+
}
2770
}
2871

2972
export class MongoReportStorage implements storage.ReportStorageFactory {
@@ -39,37 +82,63 @@ export class MongoReportStorage implements storage.ReportStorageFactory {
3982
console.log(data);
4083
}
4184

42-
async scrapeSdkData(data: event_types.InstanceRequest): Promise<ListCurrentConnectionsResponse> {
85+
async scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise<event_types.ListCurrentConnectionsResponse> {
4386
const result = await this.db.sdk_report_events
44-
.aggregate<ListCurrentConnections>([
87+
.aggregate([
4588
{
46-
$group: {
47-
_id: null,
48-
user_ids: { $addToSet: '$user_id' },
49-
client_ids: { $addToSet: '$client_id' },
50-
sdks: { $addToSet: '$sdk' }
89+
$match: {
90+
connect_at: timeSpan(data.scrape_time)
5191
}
5292
},
5393
{
54-
$project: {
55-
_id: 0,
56-
users: '$user_ids',
57-
clients: '$client_ids',
58-
sdks: '$sdks'
94+
$facet: {
95+
unique_user_ids: [
96+
{
97+
$group: {
98+
_id: '$user_id'
99+
}
100+
},
101+
{
102+
$count: 'count'
103+
}
104+
],
105+
unique_user_sdk: [
106+
{
107+
$group: {
108+
_id: {
109+
user_id: '$user_id',
110+
sdk: '$sdk'
111+
}
112+
}
113+
},
114+
{
115+
$count: 'count'
116+
}
117+
],
118+
unique_user_client: [
119+
{
120+
$group: {
121+
_id: {
122+
user_id: '$user_id',
123+
client_id: '$client_id'
124+
}
125+
}
126+
},
127+
{
128+
$count: 'count'
129+
}
130+
]
59131
}
60132
}
61133
])
62134
.toArray();
63-
return {
64-
app_id: data.app_id,
65-
org_id: data.org_id,
66-
...result[0]
67-
};
135+
console.log(result[0]);
136+
return result[0] as event_types.ListCurrentConnectionsResponse;
68137
}
69138

70-
async reportSdkConnect(data: SdkConnectBucketData): Promise<void> {
139+
async reportSdkConnect(data: event_types.SdkConnectBucketData): Promise<void> {
71140
await this.db.sdk_report_events.findOneAndUpdate(
72-
dateFilter(data.user_id, data.client_id!),
141+
updateDocFilter(data.user_id, data.client_id!),
73142
{
74143
$set: data,
75144
$unset: {
@@ -81,8 +150,8 @@ export class MongoReportStorage implements storage.ReportStorageFactory {
81150
}
82151
);
83152
}
84-
async reportSdkDisconnect(data: SdkDisconnectEventData): Promise<void> {
85-
await this.db.sdk_report_events.findOneAndUpdate(dateFilter(data.user_id, data.client_id!), {
153+
async reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise<void> {
154+
await this.db.sdk_report_events.findOneAndUpdate(updateDocFilter(data.user_id, data.client_id!), {
86155
$set: {
87156
disconnect_at: data.disconnect_at
88157
},
@@ -91,9 +160,9 @@ export class MongoReportStorage implements storage.ReportStorageFactory {
91160
}
92161
});
93162
}
94-
async listCurrentConnections(data: event_types.InstanceRequest): Promise<ListCurrentConnectionsResponse> {
163+
async listCurrentConnections(data: event_types.InstanceRequest): Promise<event_types.ListCurrentConnectionsResponse> {
95164
const result = await this.db.sdk_report_events
96-
.aggregate<ListCurrentConnections>([
165+
.aggregate<event_types.ListCurrentConnections>([
97166
{
98167
$group: {
99168
_id: null,

‎packages/service-core/src/routes/endpoints/socket-route.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ export const syncStreamReactive: SocketRouteGenerator = (router) =>
2727
client_id: params.client_id,
2828
user_id: context.user_id!,
2929
user_agent: context.user_agent,
30-
jwt_exp: context.token_payload?.exp ? new Date(context.token_payload.exp) : undefined
30+
jwt_exp: context.token_payload?.exp ? new Date(context.token_payload.exp * 1000) : undefined
3131
};
3232

3333
const streamStart = Date.now();

‎packages/service-core/src/routes/endpoints/sync-stream.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ export const syncStreamed = routeDefinition({
3535
client_id: clientId,
3636
user_id: payload.context.user_id!,
3737
user_agent: userAgent as string,
38-
jwt_exp: token_payload?.exp ? new Date(token_payload?.exp) : undefined
38+
jwt_exp: token_payload?.exp ? new Date(token_payload?.exp * 1000) : undefined
3939
};
4040

4141
if (routerEngine.closed) {

‎packages/service-core/src/storage/ReportStorageFactory.ts‎

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,10 @@
1-
import {
2-
DeleteOldSdkData,
3-
InstanceRequest,
4-
ListCurrentConnectionsResponse,
5-
SdkConnectBucketData,
6-
SdkDisconnectEventData
7-
} from '@powersync/service-types/dist/events.js';
1+
import { event_types } from '@powersync/service-types';
82

93
// Interface for the ReportStorageFactory
104
export interface ReportStorageFactory extends AsyncDisposable {
11-
reportSdkConnect(data: SdkConnectBucketData): Promise<void>;
12-
reportSdkDisconnect(data: SdkDisconnectEventData): Promise<void>;
13-
listCurrentConnections(data: InstanceRequest): Promise<ListCurrentConnectionsResponse>;
14-
scrapeSdkData(data: InstanceRequest): Promise<ListCurrentConnectionsResponse>;
15-
deleteOldSdkData(data: DeleteOldSdkData): Promise<void>;
5+
reportSdkConnect(data: event_types.SdkConnectBucketData): Promise<void>;
6+
reportSdkDisconnect(data: event_types.SdkDisconnectEventData): Promise<void>;
7+
listCurrentConnections(data: event_types.InstanceRequest): Promise<event_types.ListCurrentConnectionsResponse>;
8+
scrapeSdkData(data: event_types.ScrapeSdkDataRequest): Promise<event_types.ListCurrentConnectionsResponse>;
9+
deleteOldSdkData(data: event_types.DeleteOldSdkData): Promise<void>;
1610
}

‎packages/types/src/events.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,12 @@ export enum EmitterEngineEvents {
44
SDK_DELETE_OLD = 'sdk-delete-old'
55
}
66

7+
export enum TimeFrames {
8+
DAY = 'day',
9+
WEEK = 'week',
10+
MONTH = 'month'
11+
}
12+
713
export type SubscribeEvents = {
814
[EmitterEngineEvents.SDK_CONNECT_EVENT]: SdkConnectEventData;
915
[EmitterEngineEvents.SDK_DISCONNECT_EVENT]: SdkDisconnectEventData;
@@ -19,7 +25,7 @@ export type SdkUserData = {
1925

2026
export type DeleteOldSdkData = {
2127
interval: number;
22-
timeframe: 'month' | 'week' | 'day';
28+
timeframe: TimeFrames;
2329
};
2430

2531
export type SdkConnectEventData = {
@@ -56,7 +62,9 @@ export type ListCurrentConnections = {
5662
[sdk_version: string]: number;
5763
};
5864
};
59-
65+
export type ScrapeSdkDataRequest = {
66+
scrape_time: TimeFrames;
67+
};
6068
export type ListCurrentConnectionsResponse = ListCurrentConnections & InstanceRequest;
6169

6270
export type EventHandlerFunc<K extends EmitterEngineEvents> = (data: SubscribeEvents[K]) => Promise<void> | void;

0 commit comments

Comments
 (0)