Skip to content

Commit 030bdeb

Browse files
PerlkonigPerlkonig
committed
Break ttm into a separate lambda because of time constraints
1 parent 20560ef commit 030bdeb

File tree

3 files changed

+202
-29
lines changed

3 files changed

+202
-29
lines changed

‎serverless.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,18 @@ functions:
195195
enabled: ${self:custom.scheduleEnabled.${self:provider.stage}}
196196
# 3am UTC every sunday (giving the dump plenty of time to complete)
197197
schedule: cron(0 3 ? * 1 *)
198+
records-ttm:
199+
handler: utils/records-ttm.handler
200+
description: Generates static lists of game records
201+
timeout: 900
202+
memorySize: 10240
203+
events:
204+
- eventBridge:
205+
name: abstractplay-${self:provider.stage}-records
206+
description: Generates static lists of game records
207+
enabled: ${self:custom.scheduleEnabled.${self:provider.stage}}
208+
# 3am UTC every sunday (giving the dump plenty of time to complete)
209+
schedule: cron(0 3 ? * 1 *)
198210
summarize:
199211
handler: utils/summarize.handler
200212
description: Summarize generated game reports

‎utils/records-ttm.ts‎

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
'use strict';
2+
3+
import { S3Client, GetObjectCommand, ListObjectsV2Command, PutObjectCommand, type _Object } from "@aws-sdk/client-s3";
4+
import { Handler } from "aws-lambda";
5+
import { GameFactory } from '@abstractplay/gameslib';
6+
import { gunzipSync, strFromU8 } from "fflate";
7+
import { load } from "ion-js";
8+
9+
const REGION = "us-east-1";
10+
const s3 = new S3Client({region: REGION});
11+
const DUMP_BUCKET = "abstractplay-db-dump";
12+
const REC_BUCKET = "records.abstractplay.com";
13+
14+
type BasicRec = {
15+
Item: {
16+
pk: string;
17+
sk: string;
18+
[key: string]: any;
19+
}
20+
}
21+
22+
type GameRec = {
23+
pk: string;
24+
sk: string;
25+
id: string;
26+
metaGame: string;
27+
state: string;
28+
pieInvoked?: boolean;
29+
players: {
30+
name: string;
31+
id: string;
32+
time: number;
33+
}[];
34+
tournament?: string;
35+
event?: string;
36+
[key: string]: any;
37+
}
38+
39+
type Tournament = {
40+
pk: string;
41+
sk: string;
42+
id: string;
43+
metaGame: string;
44+
variants: string[];
45+
number: number;
46+
started: boolean;
47+
dateCreated: number;
48+
datePreviousEnded: number; // 0 means either the first tournament or a restart of the series (after it stopped because not enough participants), 3000000000000 means previous tournament still running.
49+
[key: string]: any;
50+
};
51+
52+
type OrgEvent = {
53+
pk: "ORGEVENT";
54+
sk: string; // <eventid>
55+
name: string;
56+
description: string;
57+
organizer: string;
58+
dateStart: number;
59+
dateEnd?: number;
60+
winner?: string[];
61+
visible: boolean;
62+
}
63+
64+
type OrgEventGame = {
65+
pk: "ORGEVENTGAME";
66+
sk: string; // <eventid>#<gameid>
67+
metaGame: string;
68+
variants?: string[];
69+
round: number;
70+
gameid: string;
71+
player1: string;
72+
player2: string;
73+
winner?: number[];
74+
arbitrated?: boolean;
75+
};
76+
77+
export const handler: Handler = async (event: any, context?: any) => {
78+
// scan bucket for data folder
79+
const command = new ListObjectsV2Command({
80+
Bucket: DUMP_BUCKET,
81+
});
82+
83+
const allContents: _Object[] = [];
84+
try {
85+
let isTruncatedOuter = true;
86+
87+
while (isTruncatedOuter) {
88+
const { Contents, IsTruncated: IsTruncatedInner, NextContinuationToken } =
89+
await s3.send(command);
90+
if (Contents === undefined) {
91+
throw new Error(`Could not list the bucket contents`);
92+
}
93+
allContents.push(...Contents);
94+
isTruncatedOuter = IsTruncatedInner || false;
95+
command.input.ContinuationToken = NextContinuationToken;
96+
}
97+
} catch (err) {
98+
console.error(err);
99+
}
100+
101+
// find the latest `manifest-summary.json` file
102+
const manifests = allContents.filter(c => c.Key?.includes("manifest-summary.json"));
103+
manifests.sort((a, b) => b.LastModified!.toISOString().localeCompare(a.LastModified!.toISOString()));
104+
const latest = manifests[0];
105+
const match = latest.Key!.match(/^AWSDynamoDB\/(\S+)\/manifest-summary.json$/);
106+
if (match === null) {
107+
throw new Error(`Could not extract uid from "${latest.Key}"`);
108+
}
109+
// from there, extract the UID and list of associated data files
110+
const uid = match[1];
111+
const dataFiles = allContents.filter(c => c.Key?.includes(`${uid}/data/`) && c.Key?.endsWith(".ion.gz"));
112+
console.log(`Found the following matching data files:\n${JSON.stringify(dataFiles, null, 2)}`);
113+
114+
// load the data from each data file, but only keep the GAME records
115+
const justGames: GameRec[] = [];
116+
for (const file of dataFiles) {
117+
const command = new GetObjectCommand({
118+
Bucket: DUMP_BUCKET,
119+
Key: file.Key,
120+
});
121+
122+
try {
123+
const response = await s3.send(command);
124+
// The Body object also has 'transformToByteArray' and 'transformToWebStream' methods.
125+
const bytes = await response.Body?.transformToByteArray();
126+
if (bytes !== undefined) {
127+
const ion = strFromU8(gunzipSync(bytes));
128+
for (const line of ion.split("\n")) {
129+
const outerRec = load(line);
130+
if (outerRec === null) {
131+
console.log(`Could not load ION record, usually because of an empty line.\nOffending line: "${line}"`)
132+
} else {
133+
const json = JSON.parse(JSON.stringify(outerRec)) as BasicRec;
134+
const rec = json.Item;
135+
if ( (rec.pk === "GAME") && (rec.sk.includes("#1#")) ) {
136+
justGames.push(rec as GameRec);
137+
}
138+
}
139+
}
140+
}
141+
} catch (err) {
142+
console.log(`An error occured while reading data files. The specific file was ${JSON.stringify(file)}`)
143+
console.error(err);
144+
}
145+
}
146+
console.log(`Found ${justGames.length} completed GAME records`);
147+
148+
// for each game, generate a game record and categorize it
149+
const pushToMap = (m: Map<string, any[]>, key: string, value: any) => {
150+
if (m.has(key)) {
151+
const current = m.get(key)!;
152+
m.set(key, [...current, value]);
153+
} else {
154+
m.set(key, [value]);
155+
}
156+
}
157+
const ttm = new Map<string, number[]>();
158+
for (const gdata of justGames) {
159+
const g = GameFactory(gdata.metaGame, gdata.state);
160+
if (g === undefined) {
161+
throw new Error(`Unable to instantiate ${gdata.metaGame} game ${gdata.id}:\n${JSON.stringify(gdata.state)}`);
162+
}
163+
// calculate response rates
164+
const times: number[] = [];
165+
for (let i = 0; i < g.stack.length - 1; i++) {
166+
const t1 = new Date(g.stack[i]._timestamp).getTime();
167+
const t2 = new Date(g.stack[i+1]._timestamp).getTime();
168+
times.push(t2 - t1);
169+
}
170+
times.forEach((t, i) => pushToMap(ttm, gdata.players[i % g.numplayers].id, t));
171+
}
172+
console.log(`ttm: ${ttm.size}`);
173+
174+
// write files to S3
175+
// response times
176+
for (const [player, lst] of ttm.entries()) {
177+
const cmd = new PutObjectCommand({
178+
Bucket: REC_BUCKET,
179+
Key: `ttm/${player}.json`,
180+
Body: JSON.stringify(lst),
181+
});
182+
const response = await s3.send(cmd);
183+
if (response["$metadata"].httpStatusCode !== 200) {
184+
console.log(response);
185+
}
186+
}
187+
console.log("Response times done");
188+
189+
console.log("ALL DONE");
190+
};

‎utils/records.ts‎

Lines changed: 0 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -168,7 +168,6 @@ export const handler: Handler = async (event: any, context?: any) => {
168168
const metaRecs = new Map<string, APGameRecord[]>();
169169
const userRecs = new Map<string, APGameRecord[]>();
170170
const eventRecs = new Map<string, APGameRecord[]>();
171-
const ttm = new Map<string, number[]>();
172171
for (const gdata of justGames) {
173172
const g = GameFactory(gdata.metaGame, gdata.state);
174173
if (g === undefined) {
@@ -223,21 +222,6 @@ export const handler: Handler = async (event: any, context?: any) => {
223222
pushToMap(eventRecs, id, rec);
224223
}
225224
}
226-
// calculate response rates
227-
const times: number[] = [];
228-
for (let i = 0; i < g.stack.length - 1; i++) {
229-
const t1 = new Date(g.stack[i]._timestamp).getTime();
230-
const t2 = new Date(g.stack[i+1]._timestamp).getTime();
231-
times.push(t2 - t1);
232-
}
233-
const interim = new Map<string, number[]>();
234-
times.forEach((t, i) => pushToMap(interim, gdata.players[i % g.numplayers].id, t));
235-
for (const [player, lst] of interim.entries()) {
236-
if (lst.length > 0) {
237-
const avg = lst.reduce((a, b) => a + b, 0) / lst.length;
238-
pushToMap(ttm, player, avg);
239-
}
240-
}
241225
}
242226
console.log(`allRecs: ${allRecs.length}, metaRecs: ${[...metaRecs.keys()].length}, userRecs: ${[...userRecs.keys()].length}, eventRecs: ${[...eventRecs.keys()].length}`);
243227

@@ -279,19 +263,6 @@ export const handler: Handler = async (event: any, context?: any) => {
279263
}
280264
}
281265
console.log("Player recs done");
282-
// response times
283-
for (const [player, lst] of ttm.entries()) {
284-
cmd = new PutObjectCommand({
285-
Bucket: REC_BUCKET,
286-
Key: `ttm/${player}.json`,
287-
Body: JSON.stringify(lst),
288-
});
289-
response = await s3.send(cmd);
290-
if (response["$metadata"].httpStatusCode !== 200) {
291-
console.log(response);
292-
}
293-
}
294-
console.log("Response times done");
295266
// events
296267
for (const [eventid, recs] of eventRecs.entries()) {
297268
cmd = new PutObjectCommand({

0 commit comments

Comments
 (0)