Merge branch 'develop' of https://github.com/AbstractPlay/node-backend into develop

Author: Perlkonig

api/abstractplay.ts

@@ -6956,6 +6956,9 @@ async function sendPush(opts: PushOptions) {
     }
 
     if(subscription !== undefined) {
+      // treat both 410 and 404 as permanent. There are a LOT of 404 web push errors in the logs and apparently these are because since June 2024 the Chrome/FCM push service has changed the way it reports stale or deleted web‑push registrations.
+      const PERMANENT_FAILURES = new Set([404, 410]);
+
       try {
           const options: RequestOptions = {
             vapidDetails: {
@@ -6971,7 +6974,7 @@ async function sendPush(opts: PushOptions) {
           console.log(`Result of webpush:`);
           console.log(result);
       } catch (err: any) {
-          if ( ("statusCode" in err) && (err.statusCode === 410) ) {
+          if ( ("statusCode" in err) && PERMANENT_FAILURES.has(err.statusCode)) {
             await ddbDocClient.send(
                 new DeleteCommand({
                   TableName: process.env.ABSTRACT_PLAY_TABLE,

utils/starttournaments.ts

@@ -35,6 +35,32 @@ const headers = {
   'Access-Control-Allow-Origin': '*'
 };
 
+const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
+
+async function sendCommandWithRetry(command: any, maxRetries = 8, initialDelay = 100, maxDelay = 5000) {
+    let retries = 0;
+    while (retries < maxRetries) {
+        try {
+            // @ts-ignore
+            return await ddbDocClient.send(command);
+        } catch (err: any) {
+            if (['ThrottlingException', 'ProvisionedThroughputExceededException', 'InternalServerError', 'ServiceUnavailable'].includes(err.name)) {
+                retries++;
+                if (retries >= maxRetries) {
+                    console.error(`Command failed after ${maxRetries} retries.`);
+                    throw err;
+                }
+                const delay = Math.min(initialDelay * Math.pow(2, retries - 1), maxDelay);
+                const jitter = delay * 0.1 * Math.random();
+                console.log(`Retryable error (${err.name}) caught. Retrying in ${Math.round(delay + jitter)}ms...`);
+                await sleep(delay + jitter);
+            } else {
+                throw err;
+            }
+        }
+    }
+}
+
 // Types
 export type UserSettings = {
     [k: string]: any;
@@ -234,14 +260,14 @@ async function getPlayersSlowly(playerIDs: string[]) {
   const players: FullUser[] = [];
   for (const id of playerIDs) {
     try {
-      const playerData = await ddbDocClient.send(
+      const playerData = await sendCommandWithRetry(
         new GetCommand({
           TableName: process.env.ABSTRACT_PLAY_TABLE,
           Key: {
             "pk": "USER", "sk": id
           },
         })
-      );
+      ) as { Item?: any };
       players.push(playerData.Item as FullUser);
     } catch (error) {
       logGetItemError(error);
@@ -255,29 +281,29 @@ function addToGameLists(type: string, game: Game, now: number, keepgame: boolean
   const work: Promise<any>[] = [];
   const sk = now + "#" + game.id;
   if (type === "COMPLETEDGAMES" && keepgame) {
-    work.push(ddbDocClient.send(new PutCommand({
+    work.push(sendCommandWithRetry(new PutCommand({
       TableName: process.env.ABSTRACT_PLAY_TABLE,
         Item: {
           "pk": type,
           "sk": sk,
           ...game}
       })));
-    work.push(ddbDocClient.send(new PutCommand({
+    work.push(sendCommandWithRetry(new PutCommand({
       TableName: process.env.ABSTRACT_PLAY_TABLE,
         Item: {
           "pk": type + "#" + game.metaGame,
           "sk": sk,
           ...game}
       })));
     game.players.forEach((player: { id: string; }) => {
-      work.push(ddbDocClient.send(new PutCommand({
+      work.push(sendCommandWithRetry(new PutCommand({
         TableName: process.env.ABSTRACT_PLAY_TABLE,
           Item: {
             "pk": type + "#" + player.id,
             "sk": sk,
             ...game}
         })));
-      work.push(ddbDocClient.send(new PutCommand({
+      work.push(sendCommandWithRetry(new PutCommand({
         TableName: process.env.ABSTRACT_PLAY_TABLE,
           Item: {
             "pk": type + "#" + game.metaGame + "#" + player.id,
@@ -287,7 +313,7 @@ function addToGameLists(type: string, game: Game, now: number, keepgame: boolean
     });
   }
   if (type === "CURRENTGAMES") {
-    work.push(ddbDocClient.send(new UpdateCommand({
+    work.push(sendCommandWithRetry(new UpdateCommand({
       TableName: process.env.ABSTRACT_PLAY_TABLE,
       Key: { "pk": "METAGAMES", "sk": "COUNTS" },
       ExpressionAttributeNames: { "#g": game.metaGame },
@@ -301,7 +327,7 @@ function addToGameLists(type: string, game: Game, now: number, keepgame: boolean
         update += ", #g.completedgames :n";
         eavObj[":n"] = 1
     }
-    work.push(ddbDocClient.send(new UpdateCommand({
+    work.push(sendCommandWithRetry(new UpdateCommand({
       TableName: process.env.ABSTRACT_PLAY_TABLE,
       Key: { "pk": "METAGAMES", "sk": "COUNTS" },
       ExpressionAttributeNames: { "#g": game.metaGame },
@@ -352,7 +378,7 @@ async function startTournament(users: UserLastSeen[], tournament: Tournament) {
     // Cancel tournament. Everyone is gone.
     try {
       console.log(`Deleting tournament ${tournament.id}`);
-      await ddbDocClient.send(
+      await sendCommandWithRetry(
         new DeleteCommand({
           TableName: process.env.ABSTRACT_PLAY_TABLE,
           Key: {
@@ -361,7 +387,7 @@ async function startTournament(users: UserLastSeen[], tournament: Tournament) {
           },
         }));
       const sk = tournament.metaGame + "#" + tournament.variants.sort().join("|");
-      await ddbDocClient.send(
+      await sendCommandWithRetry(
         new UpdateCommand({
           TableName: process.env.ABSTRACT_PLAY_TABLE,
           Key: {"pk": "TOURNAMENTSCOUNTER", "sk": sk},
@@ -417,7 +443,7 @@ async function startTournament(users: UserLastSeen[], tournament: Tournament) {
     if (tournament.waiting !== true) {
       try {
         console.log(`Updating tournament ${tournament.id} to waiting`);
-        await ddbDocClient.send(new UpdateCommand({
+        await sendCommandWithRetry(new UpdateCommand({
           TableName: process.env.ABSTRACT_PLAY_TABLE,
           Key: { "pk": "TOURNAMENT", "sk": tournament.id },
           ExpressionAttributeValues: { ":t": true },
@@ -463,7 +489,7 @@ async function startTournament(users: UserLastSeen[], tournament: Tournament) {
       player.sk = tournament.id + "#" + division.toString() + '#' + player.playerid;
       try {
         console.log(`Adding player ${player.playerid} to tournament ${tournament.id} in division ${division}`);
-        await ddbDocClient.send(new PutCommand({
+        await sendCommandWithRetry(new PutCommand({
           TableName: process.env.ABSTRACT_PLAY_TABLE,
           Item: player
         }));
@@ -476,7 +502,7 @@ async function startTournament(users: UserLastSeen[], tournament: Tournament) {
       if (division > 1) {
         try {
           console.log(`Deleting player ${player.playerid} from tournament ${tournament.id} with division 1 (so they can be put in the right division)`);
-          await ddbDocClient.send(new DeleteCommand({
+          await sendCommandWithRetry(new DeleteCommand({
             TableName: process.env.ABSTRACT_PLAY_TABLE,
             Key: {
               "pk": "TOURNAMENTPLAYER", "sk": tournament.id + "#1#" + player.playerid
@@ -537,7 +563,7 @@ async function startTournament(users: UserLastSeen[], tournament: Tournament) {
           const state = engine.serialize();
           try {
             console.log(`Creating game ${gameId} for tournament ${tournament.id} with division ${division}`);
-            await ddbDocClient.send(new PutCommand({
+            await sendCommandWithRetry(new PutCommand({
               TableName: process.env.ABSTRACT_PLAY_TABLE,
                 Item: {
                   "pk": "GAME",
@@ -588,7 +614,7 @@ async function startTournament(users: UserLastSeen[], tournament: Tournament) {
             "player2": gamePlayers[1].id
           };
           console.log(`Adding game ${gameId} to TOURNAMENTGAME list`);
-          await ddbDocClient.send(new PutCommand({
+          await sendCommandWithRetry(new PutCommand({
             TableName: process.env.ABSTRACT_PLAY_TABLE,
             Item: tournamentGame
           }));
@@ -607,7 +633,7 @@ async function startTournament(users: UserLastSeen[], tournament: Tournament) {
     }
     const newTournamentid = uuid();
     console.log(`Updating tournament ${tournament.id} to started`);
-    await ddbDocClient.send(new UpdateCommand({
+    await sendCommandWithRetry(new UpdateCommand({
       TableName: process.env.ABSTRACT_PLAY_TABLE,
       Key: { "pk": "TOURNAMENT", "sk": tournament.id },
       ExpressionAttributeValues: { ":dt": now, ":t": true, ":nextid": newTournamentid, ":ds": divisions },
@@ -616,7 +642,7 @@ async function startTournament(users: UserLastSeen[], tournament: Tournament) {
     // open next tournament for sign-up.
     console.log(`Opening next tournament ${newTournamentid} for sign-up. Update TOURNAMENTSCOUNTER for '${tournament.metaGame}#${tournament.variants.sort().join("|")}'`);
     try {
-      await ddbDocClient.send(new UpdateCommand({
+      await sendCommandWithRetry(new UpdateCommand({
         TableName: process.env.ABSTRACT_PLAY_TABLE,
         Key: { "pk": "TOURNAMENTSCOUNTER", "sk": tournament.metaGame + "#" + tournament.variants.sort().join("|") },
         ExpressionAttributeValues: { ":inc": 1, ":f": false },
@@ -641,7 +667,7 @@ async function startTournament(users: UserLastSeen[], tournament: Tournament) {
     };
     console.log(`Creating new tournament ${newTournamentid}`);
     try {
-      await ddbDocClient.send(new PutCommand({
+      await sendCommandWithRetry(new PutCommand({
         TableName: process.env.ABSTRACT_PLAY_TABLE,
         Item: data
       }));
@@ -667,7 +693,7 @@ async function startTournament(users: UserLastSeen[], tournament: Tournament) {
         };
         try {
             console.log(`Adding player ${player.playerid} to new tournament ${newTournamentid}`);
-            await ddbDocClient.send(new PutCommand({
+        await sendCommandWithRetry(new PutCommand({
             TableName: process.env.ABSTRACT_PLAY_TABLE,
             Item: playerdata
             }));
@@ -704,7 +730,7 @@ async function startTournament(users: UserLastSeen[], tournament: Tournament) {
   if (remove.length > 0) {
     for (const player of remove) {
       console.log(`Deleting tournament player record for ${player.playerid} from tournament ${tournament.id}`);
-      await ddbDocClient.send(
+      await sendCommandWithRetry(
         new DeleteCommand({
           TableName: process.env.ABSTRACT_PLAY_TABLE,
           Key: {
@@ -754,7 +780,7 @@ async function updateUserGames(userId: string, gamesUpdate: undefined | number,
   gameIDsChanged.length = 0;
   if (gamesUpdate === undefined) {
     // Update "old" users. This is a one-time update.
-    return ddbDocClient.send(new UpdateCommand({
+    return sendCommandWithRetry(new UpdateCommand({
       TableName: process.env.ABSTRACT_PLAY_TABLE,
       Key: { "pk": "USER", "sk": userId },
       ExpressionAttributeValues: { ":val": 1, ":gs": games },
@@ -763,7 +789,7 @@ async function updateUserGames(userId: string, gamesUpdate: undefined | number,
   } else {
     console.log(`updateUserGames: optimistically updating games for ${userId}`);
     try {
-      await ddbDocClient.send(new UpdateCommand({
+      await sendCommandWithRetry(new UpdateCommand({
         TableName: process.env.ABSTRACT_PLAY_TABLE,
         Key: { "pk": "USER", "sk": userId },
         ExpressionAttributeValues: { ":val": gamesUpdate, ":inc": 1, ":gs": games },
@@ -777,14 +803,14 @@ async function updateUserGames(userId: string, gamesUpdate: undefined | number,
         console.log(`updateUserGames: games has been modified by another process for ${userId}`);
         let count = 0;
         while (count < 3) {
-          const userData = await ddbDocClient.send(
+          const userData = await sendCommandWithRetry(
             new GetCommand({
               TableName: process.env.ABSTRACT_PLAY_TABLE,
               Key: {
                 "pk": "USER",
                 "sk": userId
               },
-            }));
+            })) as { Item?: any };
           const user = userData.Item as FullUser;
           const dbGames = user.games;
           const gamesUpdate = user.gamesUpdate;
@@ -801,7 +827,7 @@ async function updateUserGames(userId: string, gamesUpdate: undefined | number,
           }
           try {
             console.log(`updateUserGames: Update ${count} of games for user`, userId, newgames);
-            await ddbDocClient.send(new UpdateCommand({
+            await sendCommandWithRetry(new UpdateCommand({
               TableName: process.env.ABSTRACT_PLAY_TABLE,
               Key: { "pk": "USER", "sk": userId },
               ExpressionAttributeValues: { ":val": gamesUpdate, ":inc": 1, ":gs": newgames },
@@ -810,7 +836,11 @@ async function updateUserGames(userId: string, gamesUpdate: undefined | number,
             }));
             return;
           } catch (err: any) {
-            count++;
+            if (err.name === 'ConditionalCheckFailedException') {
+                count++;
+            } else {
+                throw err;
+            }
           }
         }
         new Error(`updateUserGames: Unable to update games for user ${userId} after 3 retries`);