|
| 1 | +'use strict'; |
| 2 | + |
| 3 | +import { S3Client, GetObjectCommand, ListObjectsV2Command, PutObjectCommand, type _Object } from "@aws-sdk/client-s3"; |
| 4 | +import { Handler } from "aws-lambda"; |
| 5 | +import { GameFactory, addResource } from '@abstractplay/gameslib'; |
| 6 | +import { type APGameRecord } from '@abstractplay/recranks'; |
| 7 | +import { gunzipSync, strFromU8 } from "fflate"; |
| 8 | +import { load as loadIon } from "ion-js"; |
| 9 | +import { ReservoirSampler } from "../lib/ReservoirSampler"; |
| 10 | +import i18n from 'i18next'; |
| 11 | +import enGames from "@abstractplay/gameslib/locales/en/apgames.json"; |
| 12 | +import enBack from "../locales/en/apback.json"; |
| 13 | + |
| 14 | +const REGION = "us-east-1"; |
| 15 | +const s3 = new S3Client({region: REGION}); |
| 16 | +const DUMP_BUCKET = "abstractplay-db-dump"; |
| 17 | +const REC_BUCKET = "records.abstractplay.com"; |
| 18 | + |
| 19 | +type BasicRec = { |
| 20 | + Item: { |
| 21 | + pk: string; |
| 22 | + sk: string; |
| 23 | + [key: string]: any; |
| 24 | + } |
| 25 | +} |
| 26 | + |
| 27 | +type GameRec = { |
| 28 | + pk: string; |
| 29 | + sk: string; |
| 30 | + id: string; |
| 31 | + metaGame: string; |
| 32 | + state: string; |
| 33 | + pieInvoked?: boolean; |
| 34 | + players: { |
| 35 | + name: string; |
| 36 | + id: string; |
| 37 | + time: number; |
| 38 | + }[]; |
| 39 | + tournament?: string; |
| 40 | + event?: string; |
| 41 | + [key: string]: any; |
| 42 | +} |
| 43 | + |
| 44 | +type SamplerEntry = { |
| 45 | + active: ReservoirSampler<GameRec>; |
| 46 | + completed: ReservoirSampler<GameRec>; |
| 47 | +} |
| 48 | + |
| 49 | +export const handler: Handler = async (event: any, context?: any) => { |
| 50 | + await (i18n |
| 51 | + .init({ |
| 52 | + ns: ["apback"], |
| 53 | + defaultNS: "apback", |
| 54 | + lng: "en", |
| 55 | + fallbackLng: "en", |
| 56 | + debug: true, |
| 57 | + resources: { |
| 58 | + en: { |
| 59 | + apback: enBack, |
| 60 | + } |
| 61 | + } |
| 62 | + }) |
| 63 | + .then(async function() { |
| 64 | + if (!i18n.isInitialized) { |
| 65 | + throw new Error(`i18n is not initialized where it should be!`); |
| 66 | + } |
| 67 | + addResource("en"); |
| 68 | + // scan bucket for data folder |
| 69 | + const command = new ListObjectsV2Command({ |
| 70 | + Bucket: DUMP_BUCKET, |
| 71 | + }); |
| 72 | + |
| 73 | + const allContents: _Object[] = []; |
| 74 | + try { |
| 75 | + let isTruncatedOuter = true; |
| 76 | + |
| 77 | + while (isTruncatedOuter) { |
| 78 | + const { Contents, IsTruncated: IsTruncatedInner, NextContinuationToken } = |
| 79 | + await s3.send(command); |
| 80 | + if (Contents === undefined) { |
| 81 | + throw new Error(`Could not list the bucket contents`); |
| 82 | + } |
| 83 | + allContents.push(...Contents); |
| 84 | + isTruncatedOuter = IsTruncatedInner || false; |
| 85 | + command.input.ContinuationToken = NextContinuationToken; |
| 86 | + } |
| 87 | + } catch (err) { |
| 88 | + console.error(err); |
| 89 | + } |
| 90 | + |
| 91 | + // find the latest `manifest-summary.json` file |
| 92 | + const manifests = allContents.filter(c => c.Key?.includes("manifest-summary.json")); |
| 93 | + manifests.sort((a, b) => b.LastModified!.toISOString().localeCompare(a.LastModified!.toISOString())); |
| 94 | + const latest = manifests[0]; |
| 95 | + const match = latest.Key!.match(/^AWSDynamoDB\/(\S+)\/manifest-summary.json$/); |
| 96 | + if (match === null) { |
| 97 | + throw new Error(`Could not extract uid from "${latest.Key}"`); |
| 98 | + } |
| 99 | + // from there, extract the UID and list of associated data files |
| 100 | + const uid = match[1]; |
| 101 | + const dataFiles = allContents.filter(c => c.Key?.includes(`${uid}/data/`) && c.Key?.endsWith(".ion.gz")); |
| 102 | + console.log(`Found the following matching data files:\n${JSON.stringify(dataFiles, null, 2)}`); |
| 103 | + |
| 104 | + // load the data from each data file, but only keep the GAME records |
| 105 | + const samplerMap = new Map<string, SamplerEntry>(); |
| 106 | + for (const file of dataFiles) { |
| 107 | + console.log(`Loading ${file.Key}`); |
| 108 | + const command = new GetObjectCommand({ |
| 109 | + Bucket: DUMP_BUCKET, |
| 110 | + Key: file.Key, |
| 111 | + }); |
| 112 | + |
| 113 | + try { |
| 114 | + const response = await s3.send(command); |
| 115 | + // The Body object also has 'transformToByteArray' and 'transformToWebStream' methods. |
| 116 | + const bytes = await response.Body?.transformToByteArray(); |
| 117 | + if (bytes !== undefined) { |
| 118 | + const ion = gunzipSync(bytes); |
| 119 | + console.log(`Processing ${ion.length} bytes`); |
| 120 | + let sofar = ""; |
| 121 | + let ptr = 0; |
| 122 | + const chunk = 1000000; |
| 123 | + while (ptr < ion.length) { |
| 124 | + sofar += strFromU8(ion.slice(ptr, ptr + chunk)); |
| 125 | + while (sofar.includes("}}\n")) { |
| 126 | + const idx = sofar.indexOf("}}\n"); |
| 127 | + const line = sofar.substring(0, idx+2); |
| 128 | + sofar = sofar.substring(idx+3); |
| 129 | + try { |
| 130 | + const outerRec = loadIon(line); |
| 131 | + if (outerRec === null) { |
| 132 | + console.log(`Could not load ION record, usually because of an empty line.\nOffending line: "${line}"`) |
| 133 | + } else { |
| 134 | + const json = JSON.parse(JSON.stringify(outerRec)) as BasicRec; |
| 135 | + const rec = json.Item; |
| 136 | + if (rec.pk === "GAME") { |
| 137 | + const [meta, cbit,] = rec.sk.split("#"); |
| 138 | + if (samplerMap.has(meta)) { |
| 139 | + const sampler = samplerMap.get(meta)!; |
| 140 | + if (cbit === "1") { |
| 141 | + sampler.completed.add(rec as GameRec); |
| 142 | + } else { |
| 143 | + sampler.active.add(rec as GameRec); |
| 144 | + } |
| 145 | + } else { |
| 146 | + const sampler: SamplerEntry = { |
| 147 | + completed: new ReservoirSampler<GameRec>(), |
| 148 | + active: new ReservoirSampler<GameRec>(), |
| 149 | + }; |
| 150 | + if (cbit === "1") { |
| 151 | + sampler.completed.add(rec as GameRec); |
| 152 | + } else { |
| 153 | + sampler.active.add(rec as GameRec); |
| 154 | + } |
| 155 | + samplerMap.set(meta, sampler); |
| 156 | + } |
| 157 | + } |
| 158 | + } |
| 159 | + } catch (err) { |
| 160 | + console.log(`An error occurred while loading an ION record: ${line}`); |
| 161 | + console.error(err); |
| 162 | + } |
| 163 | + } |
| 164 | + ptr += chunk; |
| 165 | + } |
| 166 | + } else { |
| 167 | + throw new Error(`Could not load bytes from ${file.Key}`); |
| 168 | + } |
| 169 | + } catch (err) { |
| 170 | + console.log(`An error occured while reading data files. The specific file was ${JSON.stringify(file)}`) |
| 171 | + console.error(err); |
| 172 | + } |
| 173 | + } |
| 174 | + console.log(`GAME records processed`); |
| 175 | + |
| 176 | + // We now have a list of random records for each game. For each one: |
| 177 | + // - Instantiate |
| 178 | + // - Serialize it with the `strip` option to strip out hidden information |
| 179 | + // - Render and store the JSON |
| 180 | + const allRecs = new Map<string, string>(); |
| 181 | + for (const [meta, entry] of samplerMap.entries()) { |
| 182 | + const active = entry.active.getSample(); |
| 183 | + let rec: GameRec; |
| 184 | + if (active.length > 0) { |
| 185 | + rec = active[0]; |
| 186 | + } else { |
| 187 | + const completed = entry.completed.getSample(); |
| 188 | + if (completed.length === 0) { |
| 189 | + console.log(`No active or completed games found for meta "${meta}"! Failsafe needed.`); |
| 190 | + continue; |
| 191 | + } |
| 192 | + rec = completed[0]; |
| 193 | + } |
| 194 | + let g = GameFactory(meta, rec.state); |
| 195 | + if (g === undefined) { |
| 196 | + throw new Error(`Error instantiating the following game record:\n${rec}`); |
| 197 | + } |
| 198 | + const stripped = g.serialize({strip: true}); |
| 199 | + g = GameFactory(meta, stripped); |
| 200 | + if (g === undefined) { |
| 201 | + throw new Error(`Error instantiating the following game record AFTER STRIPPING:\n${rec}`); |
| 202 | + } |
| 203 | + const json = g.render({}); |
| 204 | + allRecs.set(meta, JSON.stringify(json)); |
| 205 | + } |
| 206 | + console.log(`Generated ${allRecs.size} thumbnails`); |
| 207 | + |
| 208 | + // write files to S3 |
| 209 | + // meta games |
| 210 | + for (const [meta, json] of allRecs.entries()) { |
| 211 | + const cmd = new PutObjectCommand({ |
| 212 | + Bucket: REC_BUCKET, |
| 213 | + Key: `${meta}.json`, |
| 214 | + Body: json, |
| 215 | + }); |
| 216 | + const response = await s3.send(cmd); |
| 217 | + if (response["$metadata"].httpStatusCode !== 200) { |
| 218 | + console.log(response); |
| 219 | + } |
| 220 | + } |
| 221 | + console.log("Thumbnails stored"); |
| 222 | + |
| 223 | + console.log("ALL DONE"); |
| 224 | + }) |
| 225 | + .catch(err => { |
| 226 | + throw new Error(`An error occurred while initializing i18next:\n${err}`); |
| 227 | + })); |
| 228 | +}; |
0 commit comments