Harden PlayerSave with transient SQL retry logic

Author: FallenDev

Server.Configurations/ZolianDB/ZolianAbilities.sql

@@ -368,49 +368,57 @@ public async Task ServerSave(List<Aisling> players, CancellationToken ct = defau
 
     private async Task PlayerSaveRoutine(Aisling player, CancellationToken ct)
     {
-        await using var conn = new SqlConnection(ConnectionString);
-        await conn.OpenAsync(ct).ConfigureAwait(false);
-        await using var tx = (SqlTransaction)await conn.BeginTransactionAsync(IsolationLevel.ReadCommitted, ct).ConfigureAwait(false);
-
-        try
+        var ok = await ExecuteWithRetryAsync(async token =>
         {
-            player.Client.LastSave = DateTime.UtcNow;
-            var dt = PlayerDataTable();
-            var qDt = QuestDataTable();
-            var cDt = ComboScrollDataTable();
-            var iDt = ItemsDataTable();
-            var skillDt = SkillDataTable();
-            var spellDt = SpellDataTable();
-            var buffDt = BuffsDataTable();
-            var debuffDt = DeBuffsDataTable();
-            dt = PlayerStatSave(player, dt);
-            qDt = PlayerQuestSave(player, qDt);
-            cDt = PlayerComboSave(player, cDt);
-            iDt = PlayerItemSave(player, iDt);
-            skillDt = PlayerSkillSave(player, skillDt);
-            spellDt = PlayerSpellSave(player, spellDt);
-            buffDt = PlayerBuffSave(player, buffDt);
-            debuffDt = PlayerDebuffSave(player, debuffDt);
-
-            await ExecTvpAsync(conn, tx, "PlayerSave", "@Players", "dbo.PlayerType", dt, ct).ConfigureAwait(false);
-            await ExecTvpAsync(conn, tx, "PlayerQuestSave", "@Quests", "dbo.QuestType", qDt, ct).ConfigureAwait(false);
-            await ExecTvpAsync(conn, tx, "PlayerComboSave", "@Combos", "dbo.ComboType", cDt, ct).ConfigureAwait(false);
-            await ExecTvpAsync(conn, tx, "ItemUpsert", "@Items", "dbo.ItemType", iDt, ct).ConfigureAwait(false);
-            await ExecTvpAsync(conn, tx, "PlayerSaveSkills", "@Skills", "dbo.SkillType", skillDt, ct).ConfigureAwait(false);
-            await ExecTvpAsync(conn, tx, "PlayerSaveSpells", "@Spells", "dbo.SpellType", spellDt, ct).ConfigureAwait(false);
-            await ExecTvpAsync(conn, tx, "PlayerBuffSync", "@Buffs", "dbo.BuffType", buffDt, ct, new SqlParameter("@Serial", SqlDbType.BigInt) { Value = (long)player.Serial }).ConfigureAwait(false);
-            await ExecTvpAsync(conn, tx, "PlayerDeBuffSync", "@Debuffs", "dbo.DebuffType", debuffDt, ct, new SqlParameter("@Serial", SqlDbType.BigInt) { Value = (long)player.Serial }).ConfigureAwait(false);
-
-            await tx.CommitAsync(ct).ConfigureAwait(false);
-
-            // Player has a synced database state
-            player.PlayerSaveDirty = false;
-        }
-        catch (Exception e)
+            await using var conn = new SqlConnection(ConnectionString);
+            await conn.OpenAsync(token).ConfigureAwait(false);
+            await using var tx = (SqlTransaction)await conn.BeginTransactionAsync(IsolationLevel.ReadCommitted, token).ConfigureAwait(false);
+
+            try
+            {
+                player.Client.LastSave = DateTime.UtcNow;
+
+                var dt = PlayerStatSave(player, PlayerDataTable());
+                var qDt = PlayerQuestSave(player, QuestDataTable());
+                var cDt = PlayerComboSave(player, ComboScrollDataTable());
+                var iDt = PlayerItemSave(player, ItemsDataTable());
+                var skillDt = PlayerSkillSave(player, SkillDataTable());
+                var spellDt = PlayerSpellSave(player, SpellDataTable());
+                var buffDt = PlayerBuffSave(player, BuffsDataTable());
+                var debuffDt = PlayerDebuffSave(player, DeBuffsDataTable());
+
+                await ExecTvpAsync(conn, tx, "PlayerSave", "@Players", "dbo.PlayerType", dt, token).ConfigureAwait(false);
+                await ExecTvpAsync(conn, tx, "PlayerQuestSave", "@Quests", "dbo.QuestType", qDt, token).ConfigureAwait(false);
+                await ExecTvpAsync(conn, tx, "PlayerComboSave", "@Combos", "dbo.ComboType", cDt, token).ConfigureAwait(false);
+                await ExecTvpAsync(conn, tx, "ItemUpsert", "@Items", "dbo.ItemType", iDt, token).ConfigureAwait(false);
+                await ExecTvpAsync(conn, tx, "PlayerSaveSkills", "@Skills", "dbo.SkillType", skillDt, token).ConfigureAwait(false);
+                await ExecTvpAsync(conn, tx, "PlayerSaveSpells", "@Spells", "dbo.SpellType", spellDt, token).ConfigureAwait(false);
+                await ExecTvpAsync(
+                    conn, tx, "PlayerBuffSync", "@Buffs", "dbo.BuffType", buffDt, token,
+                    new SqlParameter("@Serial", SqlDbType.BigInt) { Value = (long)player.Serial }).ConfigureAwait(false);
+                await ExecTvpAsync(
+                    conn, tx, "PlayerDeBuffSync", "@Debuffs", "dbo.DebuffType", debuffDt, token,
+                    new SqlParameter("@Serial", SqlDbType.BigInt) { Value = (long)player.Serial }).ConfigureAwait(false);
+
+                await tx.CommitAsync(token).ConfigureAwait(false);
+                player.PlayerSaveDirty = false;
+            }
+            catch
+            {
+                try
+                {
+                    await tx.RollbackAsync(token).ConfigureAwait(false);
+                }
+                catch { }
+
+                // Throw and let ExecuteWithRetryAsync see the SqlException
+                throw;
+            }
+        }, ct).ConfigureAwait(false);
+
+        if (!ok)
         {
-            await tx.RollbackAsync(ct).ConfigureAwait(false);
-            ServerSetup.EventsLogger($"PlayerSave performed rollback!", LogLevel.Error);
-            SentrySdk.CaptureException(e);
+            ServerSetup.EventsLogger($"PlayerSave failed after retries for Serial={player.Serial}", LogLevel.Error);
         }
     }
 
@@ -419,10 +427,10 @@ private static async Task ExecTvpAsync(SqlConnection conn, SqlTransaction tx, st
     {
         await using var cmd = new SqlCommand(procName, conn, tx)
         {
-            CommandType = CommandType.StoredProcedure
+            CommandType = CommandType.StoredProcedure,
+            CommandTimeout = 5
         };
 
-        // Add scalar/extra parameters first
         if (extraParams is { Length: > 0 })
         {
             foreach (var ep in extraParams)
@@ -432,10 +440,9 @@ private static async Task ExecTvpAsync(SqlConnection conn, SqlTransaction tx, st
             }
         }
 
-        // Add TVP parameter
-        var p = cmd.Parameters.AddWithValue(tvpParamName, tvp);
-        p.SqlDbType = SqlDbType.Structured;
-        p.TypeName = typeName;
+        var tvpParam = cmd.Parameters.Add(tvpParamName, SqlDbType.Structured);
+        tvpParam.TypeName = typeName;
+        tvpParam.Value = tvp;
 
         await cmd.ExecuteNonQueryAsync(ct).ConfigureAwait(false);
     }

Zolian.Server.Base/Database/AislingStorage.cs

@@ -27,4 +27,40 @@ protected static void ExecuteAndCloseConnection(SqlCommand command, SqlConnectio
         command.ExecuteNonQuery();
         conn.Close();
     }
+
+    public static async Task<bool> ExecuteWithRetryAsync(Func<CancellationToken, Task> action, CancellationToken ct, int maxAttempts = 4, int baseDelayMs = 40)
+    {
+        for (var attempt = 1; attempt <= maxAttempts; attempt++)
+        {
+            try
+            {
+                await action(ct).ConfigureAwait(false);
+                return true;
+            }
+            catch (SqlException ex) when (IsTransient(ex) && attempt < maxAttempts)
+            {
+                var delayMs = Math.Min(500, baseDelayMs * (1 << (attempt - 1)));
+                // Randomized exponential backoff
+                delayMs += Random.Shared.Next(0, 25);
+
+                await Task.Delay(delayMs, ct).ConfigureAwait(false);
+            }
+        }
+
+        return false;
+    }
+
+    private static bool IsTransient(SqlException ex)
+    {
+        foreach (SqlError err in ex.Errors)
+        {
+            // -2   = Command timeout
+            // 1205 = Deadlock victim
+            // 1222 = Lock request timeout
+            if (err.Number is -2 or 1205 or 1222)
+                return true;
+        }
+
+        return false;
+    }
 }