Skip to content

Commit 4d4dea4

Browse files
n-mattosn-mattos
authored
Merge pull request #34 from codam-coding-college/syncimprovement
Add checkpoints in synchronisation.
2 parents 97e185e + 83b83ec commit 4d4dea4

File tree

7 files changed

+115
-36
lines changed

7 files changed

+115
-36
lines changed

‎prisma/migrations/20251105120327_synccheckpoints/migration.sql‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
-- CreateTable
2+
CREATE TABLE "Campus" (
3+
"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
4+
"name" TEXT NOT NULL
5+
);
6+
7+
-- CreateTable
8+
CREATE TABLE "Project" (
9+
"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
10+
"name" TEXT NOT NULL,
11+
"slug" TEXT NOT NULL,
12+
"difficulty" INTEGER
13+
);
14+
15+
-- CreateTable
16+
CREATE TABLE "User" (
17+
"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
18+
"login" TEXT NOT NULL,
19+
"primary_campus_id" INTEGER,
20+
"image_url" TEXT,
21+
"pool" TEXT NOT NULL,
22+
"anonymize_date" TEXT,
23+
CONSTRAINT "User_primary_campus_id_fkey" FOREIGN KEY ("primary_campus_id") REFERENCES "Campus" ("id") ON DELETE SET NULL ON UPDATE CASCADE
24+
);
25+
26+
-- CreateTable
27+
CREATE TABLE "ProjectUser" (
28+
"project_id" INTEGER NOT NULL,
29+
"user_id" INTEGER NOT NULL,
30+
"created_at" TEXT NOT NULL,
31+
"updated_at" TEXT NOT NULL,
32+
"status" TEXT NOT NULL,
33+
34+
PRIMARY KEY ("project_id", "user_id"),
35+
CONSTRAINT "ProjectUser_project_id_fkey" FOREIGN KEY ("project_id") REFERENCES "Project" ("id") ON DELETE RESTRICT ON UPDATE CASCADE,
36+
CONSTRAINT "ProjectUser_user_id_fkey" FOREIGN KEY ("user_id") REFERENCES "User" ("id") ON DELETE RESTRICT ON UPDATE CASCADE
37+
);
38+
39+
-- CreateTable
40+
CREATE TABLE "Sync" (
41+
"id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
42+
"type" TEXT NOT NULL,
43+
"type_id" INTEGER NOT NULL,
44+
"last_pull" TEXT
45+
);
46+
47+
-- CreateIndex
48+
CREATE UNIQUE INDEX "User_login_key" ON "User"("login");
49+
50+
-- CreateIndex
51+
CREATE UNIQUE INDEX "Sync_type_type_id_key" ON "Sync"("type", "type_id");

‎prisma/migrations/migration_lock.toml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Please do not edit this file manually
2+
# It should be added in your version-control system (e.g., Git)
3+
provider = "sqlite"

‎prisma/schema.prisma‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,5 +54,9 @@ model ProjectUser {
5454

5555
model Sync {
5656
id Int @id @default(autoincrement())
57+
type String // campus_users, cursus_projects, projects_projects_users, full
58+
type_id Int // campus_id, cursus_id, project_id
5759
last_pull String?
58-
}
60+
61+
@@unique ([type, type_id])
62+
}

‎src/app.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ util.inspect.defaultOptions.depth = 10;
1515
* @returns How many milliseconds until the next pull request should be made.
1616
*/
1717
async function msUntilNextPull(): Promise<number> {
18-
const lastPullAgo = await DatabaseService.getLastSyncTimestamp().then(date => {
18+
const lastPullAgo = await DatabaseService.getLastSyncTimestamp("full", 1).then(date => {
1919
if (!date) {
2020
console.warn('No last sync timestamp found, assuming first pull.');
2121
return env.pullTimeout;

‎src/express.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -179,8 +179,8 @@ export async function startWebserver(port: number) {
179179
const userTimeZone = req.cookies.timezone || 'Europe/Amsterdam';
180180
const settings = {
181181
projects: await getProjects(campus.id, requestedStatus, showEmptyProjects),
182-
lastUpdate: await DatabaseService.getLastSyncTimestamp().then(date => date ? date.toLocaleString('en-NL', { timeZone: userTimeZone }).slice(0, -3) : 'N/A'),
183-
hoursAgo: (((Date.now()) - await DatabaseService.getLastSyncTimestamp().then(date => date ? date.getTime() : 0)) / (1000 * 60 * 60)).toFixed(2), // hours ago
182+
lastUpdate: await DatabaseService.getLastSyncTimestamp("full", 1).then(date => date ? date.toLocaleString('en-NL', { timeZone: userTimeZone }).slice(0, -3) : 'N/A'),
183+
hoursAgo: (((Date.now()) - await DatabaseService.getLastSyncTimestamp("full", 1).then(date => date ? date.getTime() : 0)) / (1000 * 60 * 60)).toFixed(2), // hours ago
184184
requestedStatus,
185185
projectStatuses: env.projectStatuses,
186186
campusName: campus.name,

‎src/services.ts‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,12 @@ export class DatabaseService {
2424
/**
2525
* @returns The last synchronization timestamp.
2626
*/
27-
static getLastSyncTimestamp = async function(): Promise<Date | null> {
27+
static getLastSyncTimestamp = async function(type: string, type_id: number): Promise<Date | null> {
2828
const sync = await prisma.sync.findUnique({
29-
where: { id: 1 },
29+
where: { type_type_id: { type, type_id } },
3030
select: { last_pull: true }
3131
});
3232
if (sync?.last_pull === null || sync?.last_pull === undefined) {
33-
log(2, `No last sync timestamp found, returning null.`);
3433
return null;
3534
}
3635
return new Date(sync.last_pull);
@@ -223,11 +222,15 @@ export class DatabaseService {
223222
* Save the synchronization timestamp.
224223
* @param timestamp The timestamp to save
225224
*/
226-
static saveSyncTimestamp = async function(timestamp: Date): Promise<void> {
225+
static saveSyncTimestamp = async function(type: string, type_id: number, timestamp: Date): Promise<void> {
227226
await prisma.sync.upsert({
228-
where: { id: 1 },
227+
where: { type_type_id: { type, type_id } },
229228
update: { last_pull: timestamp.toISOString() },
230-
create: { id: 1, last_pull: timestamp.toISOString() }
229+
create: {
230+
type,
231+
type_id,
232+
last_pull: timestamp.toISOString()
233+
}
231234
});
232235
}
233236

‎src/sync.ts‎

Lines changed: 44 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -49,19 +49,16 @@ export const syncWithIntra = async function(): Promise<void> {
4949
const now = new Date();
5050

5151
try {
52-
let lastSyncRaw = await DatabaseService.getLastSyncTimestamp();
53-
let lastSync: Date | undefined = lastSyncRaw === null ? undefined : lastSyncRaw;
54-
5552
// Syncs all data based on:
5653
// - last successful sync timestamp (lastSync)
5754
// - active campuses
5855
// - 42 and piscine cursus (cursus/21,9)
59-
await syncCampuses(fast42Api, lastSync);
60-
await syncCursusProjects(fast42Api, lastSync, '9');
61-
await syncCursusProjects(fast42Api, lastSync, '21');
62-
// await syncUsers(fast42Api, lastSync);
63-
await syncProjectUsers(fast42Api, lastSync);
64-
await DatabaseService.saveSyncTimestamp(now);
56+
await syncCampuses(fast42Api, now);
57+
await syncCursusProjects(fast42Api, '9', now);
58+
await syncCursusProjects(fast42Api, '21', now);
59+
await syncUsers(fast42Api, now);
60+
await syncProjectUsers(fast42Api, now);
61+
await DatabaseService.saveSyncTimestamp("full", 1, now);
6562

6663
console.info(`Intra synchronization completed at ${new Date().toISOString()}.`);
6764
}
@@ -77,11 +74,11 @@ export const syncWithIntra = async function(): Promise<void> {
7774
* @param lastPullDate The date of the last synchronization
7875
* @returns A promise that resolves when the synchronization is complete
7976
*/
80-
async function syncCampuses(fast42Api: Fast42, lastPullDate: Date | undefined): Promise<void> {
77+
async function syncCampuses(fast42Api: Fast42, lastSync: Date | undefined): Promise<void> {
8178
let campusesApi;
8279
try {
8380
log(2, `Syncing campuses...`);
84-
campusesApi = await syncData(fast42Api, new Date(), lastPullDate, `/campus`, { 'active': 'true' });
81+
campusesApi = await syncData(fast42Api, new Date(), lastSync, `/campus`, { 'active': 'true' });
8582
const dbCampuses = campusesApi.map(transformApiCampusToDb);
8683
await DatabaseService.insertManyCampuses(dbCampuses);
8784
log(2, `Finished syncing campuses`);
@@ -97,25 +94,32 @@ async function syncCampuses(fast42Api: Fast42, lastPullDate: Date | undefined):
9794
* @param lastPullDate The date of the last synchronization
9895
* @returns A promise that resolves when the synchronization is complete
9996
*/
100-
async function syncCursusProjects(fast42Api: Fast42, lastPullDate: Date | undefined, cursus: string): Promise<void> {
97+
async function syncCursusProjects(fast42Api: Fast42, cursus: string, syncDate: Date): Promise<void> {
10198
let pageIndex = 0;
10299
let hasMorePages = true;
103100
let params: { [key: string]: string } = { 'page[size]': '100' };
104-
if (lastPullDate) {
105-
let syncDate = new Date();
106-
params['range[updated_at]'] = `${lastPullDate.toISOString()},${syncDate.toISOString()}`;
107-
}
108101

109102
try {
110103
while (hasMorePages) {
111104
pageIndex++;
105+
106+
// Set pagination and last sync range
112107
params['page[number]'] = pageIndex.toString();
108+
let lastSyncRaw = await DatabaseService.getLastSyncTimestamp("cursus_projects", parseInt(cursus));
109+
let lastSync: Date | undefined = lastSyncRaw === null ? undefined : lastSyncRaw;
110+
if (lastSync) {
111+
params['range[updated_at]'] = `${lastSync.toISOString()},${syncDate.toISOString()}`;
112+
} else {
113+
params['range[updated_at]'] = `${new Date(0).toISOString()},${syncDate.toISOString()}`;
114+
}
115+
113116
log(2, `Fetching page ${pageIndex} of projects...`);
114117

115118
const projectsData = await fetchSingle42ApiPage(fast42Api, `/cursus/${cursus}/projects`, params);
116119
if (!projectsData || projectsData.length === 0) {
117120
log(2, `No more projects found on page ${pageIndex}. Stopping.`);
118121
hasMorePages = false;
122+
await DatabaseService.saveSyncTimestamp("cursus_projects", parseInt(cursus), syncDate);
119123
continue;
120124
}
121125

@@ -135,15 +139,11 @@ async function syncCursusProjects(fast42Api: Fast42, lastPullDate: Date | undefi
135139
* @param fast42Api The Fast42 API instance to use for fetching user data
136140
* @param lastPullDate The date of the last synchronization
137141
*/
138-
async function syncUsers(fast42Api: Fast42, lastPullDate: Date | undefined): Promise<void> {
142+
async function syncUsers(fast42Api: Fast42, syncDate: Date): Promise<void> {
139143
let pageIndex = 0;
140144
let hasMorePages = true;
141145
let params: { [key: string]: string } = {};
142146
params['page[size]'] = '100';
143-
if (lastPullDate) {
144-
let syncDate = new Date();
145-
params['range[updated_at]'] = `${lastPullDate.toISOString()},${syncDate.toISOString()}`;
146-
}
147147

148148
const campuses = await DatabaseService.getAllCampuses();
149149
let campusIds = campuses.map(c => c.id);
@@ -153,7 +153,17 @@ async function syncUsers(fast42Api: Fast42, lastPullDate: Date | undefined): Pro
153153
params['filter[primary_campus_id]'] = campusId.toString();
154154
while (hasMorePages) {
155155
pageIndex++;
156+
157+
// Set pagination and last sync range
156158
params['page[number]'] = pageIndex.toString();
159+
let lastSyncRaw = await DatabaseService.getLastSyncTimestamp("campus_users", campusId);
160+
let lastSync: Date | undefined = lastSyncRaw === null ? undefined : lastSyncRaw;
161+
if (lastSync) {
162+
params['range[updated_at]'] = `${lastSync.toISOString()},${syncDate.toISOString()}`;
163+
} else {
164+
params['range[updated_at]'] = `${new Date(0).toISOString()},${syncDate.toISOString()}`;
165+
}
166+
157167
log(2, `Fetching page ${pageIndex} of users for campus ${campusId} (${index + 1}/${totalCampuses})...`);
158168

159169
let usersData;
@@ -167,6 +177,7 @@ async function syncUsers(fast42Api: Fast42, lastPullDate: Date | undefined): Pro
167177
if (!usersData || usersData.length === 0) {
168178
log(2, `No more users found for campus ${campusId} on page ${pageIndex}. Stopping.`);
169179
hasMorePages = false;
180+
await DatabaseService.saveSyncTimestamp("campus_users", campusId, syncDate);
170181
break;
171182
}
172183

@@ -191,16 +202,12 @@ async function syncUsers(fast42Api: Fast42, lastPullDate: Date | undefined): Pro
191202
* @param lastPullDate The date of the last synchronization
192203
* @returns A promise that resolves when the synchronization is complete
193204
*/
194-
async function syncProjectUsers(fast42Api: Fast42, lastPullDate: Date | undefined): Promise<void> {
205+
async function syncProjectUsers(fast42Api: Fast42, syncDate: Date): Promise<void> {
195206
let pageIndex = 0;
196207
let hasMorePages = true;
197208
let params: { [key: string]: string } = {};
198209
params['page[size]'] = '100';
199210
params['filter[campus]'] = await DatabaseService.getAllCampuses().then(campuses => campuses.map(c => c.id).join(','));
200-
if (lastPullDate) {
201-
let syncDate = new Date();
202-
params['range[updated_at]'] = `${lastPullDate.toISOString()},${syncDate.toISOString()}`;
203-
}
204211

205212
const projects = await DatabaseService.getAllProjects();
206213
let projectIds = projects.map(p => p.id);
@@ -209,7 +216,17 @@ async function syncProjectUsers(fast42Api: Fast42, lastPullDate: Date | undefine
209216
for (let [index, projectId] of projectIds.entries()) {
210217
while (hasMorePages) {
211218
pageIndex++;
219+
220+
// Set pagination and last sync range
212221
params['page[number]'] = pageIndex.toString();
222+
let lastSyncRaw = await DatabaseService.getLastSyncTimestamp("projects_projects_users", projectId);
223+
let lastSync: Date | undefined = lastSyncRaw === null ? undefined : lastSyncRaw;
224+
if (lastSync) {
225+
params['range[updated_at]'] = `${lastSync.toISOString()},${syncDate.toISOString()}`;
226+
} else {
227+
params['range[updated_at]'] = `${new Date(0).toISOString()},${syncDate.toISOString()}`;
228+
}
229+
213230
log(2, `Fetching page ${pageIndex} of projectUsers for project ${projectId} (${index + 1}/${totalProjects})...`);
214231

215232
let projectUsersData;
@@ -223,6 +240,7 @@ async function syncProjectUsers(fast42Api: Fast42, lastPullDate: Date | undefine
223240
if (!projectUsersData || projectUsersData.length === 0) {
224241
log(2, `No more users found for project ${projectId} on page ${pageIndex}. Stopping.`);
225242
hasMorePages = false;
243+
await DatabaseService.saveSyncTimestamp("projects_projects_users", projectId, syncDate);
226244
break;
227245
}
228246

0 commit comments

Comments
 (0)