Skip to content

Commit 99a96c6

Browse files
needsneeds
committed
Save global counts incrementally on redis
1 parent ab8cabe commit 99a96c6

File tree

6 files changed

+256
-0
lines changed

6 files changed

+256
-0
lines changed

‎apps/worker/src/redis.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,3 +9,5 @@ export const redisClientPromise = createClient({
99
})
1010
.on('error', err => console.log('Redis Client Error', err))
1111
.connect();
12+
13+
export type RedisClient = Awaited<typeof redisClientPromise>;

‎apps/worker/src/workers/updateGlobalCounts.test.ts‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
import { updateCount } from "./updateGlobalCounts";
2+
import redis from "redis-mock";
3+
4+
describe("updateCounts", () => {
5+
test("create a new count", async () => {
6+
const redisClient = redis.createClient();
7+
8+
await updateCount({
9+
redisClient,
10+
getNewEntities: async () => [{ createdAt: new Date() }],
11+
lastUpdatedAtKey: "test-0-lastUpdatedAt",
12+
countKey: "test-0-count",
13+
});
14+
15+
expect(await redisClient.get("test-0-count")).toBe("1");
16+
expect(await redisClient.get("test-0-lastUpdatedAt")).toBe(new Date().toISOString());
17+
});
18+
19+
test("update an existing count", async () => {
20+
const redisClient = redis.createClient();
21+
22+
await redisClient.set("test-1-count", "1");
23+
await redisClient.set("test-1-lastUpdatedAt", new Date().toISOString());
24+
25+
await updateCount({
26+
redisClient,
27+
getNewEntities: async () => [{ createdAt: new Date() }],
28+
lastUpdatedAtKey: "test-1-lastUpdatedAt",
29+
countKey: "test-1-count",
30+
});
31+
32+
expect(await redisClient.get("test-1-count")).toBe("2");
33+
expect(await redisClient.get("test-1-lastUpdatedAt")).toBe(new Date().toISOString());
34+
});
35+
});

‎apps/worker/src/workers/updateGlobalCounts.ts‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
import { Worker } from "bullmq";
2+
import { prisma } from "../prisma";
3+
import { QUEUE_NAME_UPDATE_GLOBAL_COUNTS } from "@teerank/teerank";
4+
import { bullmqConnection } from "@teerank/teerank";
5+
import { RedisClient, redisClientPromise } from "../redis";
6+
import { minutesToSeconds } from "date-fns";
7+
8+
const GLOBAL_COUNTS_PLAYERS_LAST_UPDATED_AT_KEY = 'global-counts-players-last-updated-at';
9+
const GLOBAL_COUNTS_CLANS_LAST_UPDATED_AT_KEY = 'global-counts-clans-last-updated-at';
10+
const GLOBAL_COUNTS_MAPS_LAST_UPDATED_AT_KEY = 'global-counts-maps-last-updated-at';
11+
const GLOBAL_COUNTS_GAME_TYPES_LAST_UPDATED_AT_KEY = 'global-counts-game-types-last-updated-at';
12+
const GLOBAL_COUNTS_GAME_SERVERS_LAST_UPDATED_AT_KEY = 'global-counts-game-servers-last-updated-at';
13+
14+
const GLOBAL_COUNTS_PLAYERS_KEY = 'global-counts-players';
15+
const GLOBAL_COUNTS_CLANS_KEY = 'global-counts-clans';
16+
const GLOBAL_COUNTS_MAPS_KEY = 'global-counts-maps';
17+
const GLOBAL_COUNTS_GAME_TYPES_KEY = 'global-counts-game-types';
18+
const GLOBAL_COUNTS_GAME_SERVERS_KEY = 'global-counts-game-servers';
19+
20+
export async function updateCount({
21+
redisClient,
22+
getNewEntities,
23+
lastUpdatedAtKey,
24+
countKey,
25+
}: {
26+
redisClient: RedisClient;
27+
getNewEntities: (lastUpdateDate: Date) => Promise<{ createdAt: Date }[]>;
28+
lastUpdatedAtKey: string;
29+
countKey: string;
30+
}) {
31+
const lastUpdateDate = new Date(Number(await redisClient.get(lastUpdatedAtKey) || "0"));
32+
33+
console.log(`Last update date: ${lastUpdateDate}`);
34+
35+
const entities = await getNewEntities(lastUpdateDate);
36+
37+
const newCount = entities.length;
38+
const newLastUpdatedAt = Math.max(lastUpdateDate.getTime(), entities[entities.length - 1].createdAt.getTime());
39+
40+
console.log(`New count: ${newCount}`);
41+
console.log(`New last updated at: ${newLastUpdatedAt}`);
42+
43+
const pipeline = redisClient.multi();
44+
pipeline.incrBy(countKey, newCount);
45+
pipeline.set(lastUpdatedAtKey, newLastUpdatedAt);
46+
await pipeline.exec();
47+
48+
console.log(`Global counts saved`);
49+
}
50+
51+
async function updatePlayersCount(redisClient: RedisClient) {
52+
const getNewEntities = async (lastUpdateDate: Date) => {
53+
return await prisma.player.findMany({
54+
select: {
55+
createdAt: true,
56+
},
57+
where: {
58+
createdAt: {
59+
gt: lastUpdateDate,
60+
},
61+
},
62+
});
63+
}
64+
65+
console.log('Updating players count');
66+
await updateCount({
67+
redisClient,
68+
getNewEntities,
69+
lastUpdatedAtKey: GLOBAL_COUNTS_PLAYERS_LAST_UPDATED_AT_KEY,
70+
countKey: GLOBAL_COUNTS_PLAYERS_KEY
71+
});
72+
}
73+
74+
async function updateClansCount(redisClient: RedisClient) {
75+
const getNewEntities = async (lastUpdateDate: Date) => {
76+
return await prisma.clan.findMany({
77+
select: {
78+
createdAt: true,
79+
},
80+
where: {
81+
createdAt: {
82+
gt: lastUpdateDate,
83+
},
84+
},
85+
});
86+
}
87+
88+
console.log('Updating clans count');
89+
await updateCount({
90+
redisClient,
91+
getNewEntities,
92+
lastUpdatedAtKey: GLOBAL_COUNTS_CLANS_LAST_UPDATED_AT_KEY,
93+
countKey: GLOBAL_COUNTS_CLANS_KEY
94+
});
95+
}
96+
97+
async function updateGameServersCount(redisClient: RedisClient) {
98+
const getNewEntities = async (lastUpdateDate: Date) => {
99+
return await prisma.gameServer.findMany({
100+
select: {
101+
createdAt: true,
102+
},
103+
where: {
104+
createdAt: {
105+
gt: lastUpdateDate,
106+
},
107+
},
108+
});
109+
}
110+
111+
console.log('Updating game servers count');
112+
await updateCount({
113+
redisClient,
114+
getNewEntities,
115+
lastUpdatedAtKey: GLOBAL_COUNTS_GAME_SERVERS_LAST_UPDATED_AT_KEY,
116+
countKey: GLOBAL_COUNTS_GAME_SERVERS_KEY
117+
});
118+
}
119+
120+
async function updateMapsCount(redisClient: RedisClient) {
121+
const getNewEntities = async (lastUpdateDate: Date) => {
122+
return await prisma.map.findMany({
123+
select: {
124+
createdAt: true,
125+
},
126+
where: {
127+
createdAt: {
128+
gt: lastUpdateDate,
129+
},
130+
},
131+
});
132+
}
133+
134+
console.log('Updating maps count');
135+
await updateCount({
136+
redisClient,
137+
getNewEntities,
138+
lastUpdatedAtKey: GLOBAL_COUNTS_MAPS_LAST_UPDATED_AT_KEY,
139+
countKey: GLOBAL_COUNTS_MAPS_KEY
140+
});
141+
}
142+
143+
async function updateGameTypesCount(redisClient: RedisClient) {
144+
const getNewEntities = async (lastUpdateDate: Date) => {
145+
return await prisma.gameType.findMany({
146+
select: {
147+
createdAt: true,
148+
},
149+
where: {
150+
createdAt: {
151+
gt: lastUpdateDate,
152+
},
153+
},
154+
});
155+
}
156+
157+
console.log('Updating game types count');
158+
await updateCount({
159+
redisClient,
160+
getNewEntities,
161+
lastUpdatedAtKey: GLOBAL_COUNTS_GAME_TYPES_LAST_UPDATED_AT_KEY,
162+
countKey: GLOBAL_COUNTS_GAME_TYPES_KEY
163+
});
164+
}
165+
166+
async function processor() {
167+
const redisClient = await redisClientPromise;
168+
169+
await updatePlayersCount(redisClient);
170+
await updateClansCount(redisClient);
171+
await updateGameServersCount(redisClient);
172+
await updateMapsCount(redisClient);
173+
await updateGameTypesCount(redisClient);
174+
}
175+
176+
export async function startUpdateGlobalCountsWorker() {
177+
return new Worker(QUEUE_NAME_UPDATE_GLOBAL_COUNTS, processor, {
178+
connection: bullmqConnection,
179+
concurrency: 1,
180+
removeOnComplete: {
181+
age: minutesToSeconds(10),
182+
},
183+
removeOnFail: {
184+
count: 1000,
185+
}
186+
});
187+
}

‎libs/teerank/src/lib/bullmqConfig.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ export const QUEUE_NAME_RANK_PLAYER = 'rank-player';
1414
export const QUEUE_NAME_GAME_TYPE_COUNT = 'game-type-count';
1515
export const QUEUE_NAME_MAP_COUNT = 'map-count';
1616
export const QUEUE_NAME_FILL_CLAN_ACTIVE_PLAYER_COUNT = 'fill-clan-active-player-count';
17+
export const QUEUE_NAME_UPDATE_GLOBAL_COUNTS = 'update-global-counts';
1718

1819
export function getQueuePollMasterServer() {
1920
return new Queue(QUEUE_NAME_POLL_MASTER_SERVER, { connection: bullmqConnection });

‎package-lock.json‎

Lines changed: 29 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
"@types/node": "18.19.18",
3434
"@types/react": "18.3.1",
3535
"@types/react-dom": "18.3.0",
36+
"@types/redis-mock": "^0.17.3",
3637
"@typescript-eslint/eslint-plugin": "7.18.0",
3738
"@typescript-eslint/parser": "7.18.0",
3839
"autoprefixer": "^10.4.16",
@@ -53,6 +54,7 @@
5354
"postcss": "8.4.38",
5455
"prettier": "^2.6.2",
5556
"prisma": "^5.22.0",
57+
"redis-mock": "^0.56.3",
5658
"tailwindcss": "3.4.3",
5759
"ts-jest": "^29.1.0",
5860
"ts-node": "10.9.1",

0 commit comments

Comments
 (0)