refactor(backend): improve redis and json.parse type safety with zod (@byseif21)

backend/src/api/controllers/result.ts

@@ -606,9 +606,9 @@ export async function addResult(
           badgeId: selectedBadgeId,
           lastActivityTimestamp: Date.now(),
           isPremium,
+          timeTypedSeconds: totalDurationTypedSeconds,
         },
         xpGained: xpGained.xp,
-        timeTypedSeconds: totalDurationTypedSeconds,
       }
     );
   }

backend/src/dal/new-quotes.ts

@@ -8,20 +8,23 @@ import MonkeyError from "../utils/error";
 import { compareTwoStrings } from "string-similarity";
 import { ApproveQuote, Quote } from "@monkeytype/contracts/schemas/quotes";
 import { WithObjectId } from "../utils/misc";
-
-type JsonQuote = {
-  text: string;
-  britishText?: string;
-  source: string;
-  length: number;
-  id: number;
-};
-
-type QuoteData = {
-  language: string;
-  quotes: JsonQuote[];
-  groups: [number, number][];
-};
+import { parseWithSchema as parseJsonWithSchema } from "@monkeytype/util/json";
+import { z } from "zod";
+
+const JsonQuoteSchema = z.object({
+  text: z.string(),
+  britishText: z.string().optional(),
+  approvedBy: z.string().optional(),
+  source: z.string(),
+  length: z.number(),
+  id: z.number(),
+});
+
+const QuoteDataSchema = z.object({
+  language: z.string(),
+  quotes: z.array(JsonQuoteSchema),
+  groups: z.array(z.tuple([z.number(), z.number()])),
+});
 
 const PATH_TO_REPO = "../../../../monkeytype-new-quotes";
 
@@ -86,7 +89,10 @@ export async function add(
   let similarityScore = -1;
   if (existsSync(fileDir)) {
     const quoteFile = await readFile(fileDir);
-    const quoteFileJSON = JSON.parse(quoteFile.toString()) as QuoteData;
+    const quoteFileJSON = parseJsonWithSchema(
+      quoteFile.toString(),
+      QuoteDataSchema
+    );
     quoteFileJSON.quotes.every((old) => {
       if (compareTwoStrings(old.text, quote.text) > 0.9) {
         duplicateId = old.id;
@@ -156,6 +162,7 @@ export async function approve(
     source: editSource ?? targetQuote.source,
     length: targetQuote.text.length,
     approvedBy: name,
+    id: -1,
   };
   let message = "";
 
@@ -170,7 +177,10 @@ export async function approve(
   await git.pull("upstream", "master");
   if (existsSync(fileDir)) {
     const quoteFile = await readFile(fileDir);
-    const quoteObject = JSON.parse(quoteFile.toString()) as QuoteData;
+    const quoteObject = parseJsonWithSchema(
+      quoteFile.toString(),
+      QuoteDataSchema
+    );
     quoteObject.quotes.every((old) => {
       if (compareTwoStrings(old.text, quote.text) > 0.8) {
         throw new MonkeyError(409, "Duplicate quote");
@@ -183,7 +193,12 @@ export async function approve(
       }
     });
     quote.id = maxid + 1;
-    quoteObject.quotes.push(quote as JsonQuote);
+
+    if (quote.id === -1) {
+      throw new MonkeyError(500, "Failed to get max id");
+    }
+
+    quoteObject.quotes.push(quote);
     writeFileSync(fileDir, JSON.stringify(quoteObject, null, 2));
     message = `Added quote to ${language}.json.`;
   } else {

backend/src/dal/user.ts

@@ -1103,11 +1103,7 @@ export async function updateStreak(
   if (isYesterday(streak.lastResultTimestamp, streak.hourOffset ?? 0)) {
     streak.length += 1;
   } else if (!isToday(streak.lastResultTimestamp, streak.hourOffset ?? 0)) {
-    void addImportantLog(
-      "streak_lost",
-      JSON.parse(JSON.stringify(streak)) as Record<string, unknown>,
-      uid
-    );
+    void addImportantLog("streak_lost", streak, uid);
     streak.length = 1;
   }
 

backend/src/init/redis.ts

@@ -1,11 +1,47 @@
 import fs from "fs";
 import _ from "lodash";
 import { join } from "path";
-import IORedis from "ioredis";
+import IORedis, { Redis } from "ioredis";
 import Logger from "../utils/logger";
 import { isDevEnvironment } from "../utils/misc";
 import { getErrorMessage } from "../utils/error";
 
+// Define Redis connection with custom methods for type safety
+export type RedisConnectionWithCustomMethods = Redis & {
+  addResult: (
+    keyCount: number,
+    scoresKey: string,
+    resultsKey: string,
+    maxResults: number,
+    expirationTime: number,
+    uid: string,
+    score: number,
+    data: string
+  ) => Promise<number>;
+  addResultIncrement: (
+    keyCount: number,
+    scoresKey: string,
+    resultsKey: string,
+    expirationTime: number,
+    uid: string,
+    score: number,
+    data: string
+  ) => Promise<number>;
+  getResults: (
+    keyCount: number,
+    scoresKey: string,
+    resultsKey: string,
+    minRank: number,
+    maxRank: number,
+    withScores: string
+  ) => Promise<[string[], string[]]>;
+  purgeResults: (
+    keyCount: number,
+    uid: string,
+    namespace: string
+  ) => Promise<void>;
+};
+
 let connection: IORedis.Redis;
 let connected = false;
 
@@ -73,11 +109,11 @@ export function isConnected(): boolean {
   return connected;
 }
 
-export function getConnection(): IORedis.Redis | undefined {
+export function getConnection(): RedisConnectionWithCustomMethods | null {
   const status = connection?.status;
   if (connection === undefined || status !== "ready") {
-    return undefined;
+    return null;
   }
 
-  return connection;
+  return connection as RedisConnectionWithCustomMethods;
 }

backend/src/server.ts

@@ -47,7 +47,7 @@ async function bootServer(port: number): Promise<Server> {
 
       Logger.info("Initializing queues...");
       queues.forEach((queue) => {
-        queue.init(connection);
+        queue.init(connection ?? undefined);
       });
       Logger.success(
         `Queues initialized: ${queues
@@ -57,11 +57,11 @@ async function bootServer(port: number): Promise<Server> {
 
       Logger.info("Initializing workers...");
       workers.forEach(async (worker) => {
-        await worker(connection).run();
+        await worker(connection ?? undefined).run();
       });
       Logger.success(
         `Workers initialized: ${workers
-          .map((worker) => worker(connection).name)
+          .map((worker) => worker(connection ?? undefined).name)
           .join(", ")}`
       );
     }

backend/src/services/weekly-xp-leaderboard.ts

@@ -1,24 +1,20 @@
 import { Configuration } from "@monkeytype/contracts/schemas/configuration";
 import * as RedisClient from "../init/redis";
 import LaterQueue from "../queues/later-queue";
-import { XpLeaderboardEntry } from "@monkeytype/contracts/schemas/leaderboards";
+import {
+  RedisXpLeaderboardEntry,
+  RedisXpLeaderboardEntrySchema,
+  RedisXpLeaderboardScore,
+  XpLeaderboardEntry,
+} from "@monkeytype/contracts/schemas/leaderboards";
 import { getCurrentWeekTimestamp } from "@monkeytype/util/date-and-time";
 import MonkeyError from "../utils/error";
 import { omit } from "lodash";
+import { parseWithSchema as parseJsonWithSchema } from "@monkeytype/util/json";
 
 type AddResultOpts = {
-  entry: Pick<
-    XpLeaderboardEntry,
-    | "uid"
-    | "name"
-    | "discordId"
-    | "discordAvatar"
-    | "badgeId"
-    | "lastActivityTimestamp"
-    | "isPremium"
-  >;
-  xpGained: number;
-  timeTypedSeconds: number;
+  entry: RedisXpLeaderboardEntry;
+  xpGained: RedisXpLeaderboardScore;
 };
 
 const weeklyXpLeaderboardLeaderboardNamespace =
@@ -59,7 +55,7 @@ export class WeeklyXpLeaderboard {
     weeklyXpLeaderboardConfig: Configuration["leaderboards"]["weeklyXp"],
     opts: AddResultOpts
   ): Promise<number> {
-    const { entry, xpGained, timeTypedSeconds } = opts;
+    const { entry, xpGained } = opts;
 
     const connection = RedisClient.getConnection();
     if (!connection || !weeklyXpLeaderboardConfig.enabled) {
@@ -89,25 +85,28 @@ export class WeeklyXpLeaderboard {
 
     const currentEntryTimeTypedSeconds =
       currentEntry !== null
-        ? (JSON.parse(currentEntry) as { timeTypedSeconds: number | undefined })
+        ? parseJsonWithSchema(currentEntry, RedisXpLeaderboardEntrySchema)
             ?.timeTypedSeconds
         : undefined;
 
     const totalTimeTypedSeconds =
-      timeTypedSeconds + (currentEntryTimeTypedSeconds ?? 0);
+      entry.timeTypedSeconds + (currentEntryTimeTypedSeconds ?? 0);
 
     const [rank] = await Promise.all([
-      // @ts-expect-error we are doing some weird file to function mapping, thats why its any
-      // eslint-disable-next-line @typescript-eslint/no-unsafe-call
       connection.addResultIncrement(
         2,
         weeklyXpLeaderboardScoresKey,
         weeklyXpLeaderboardResultsKey,
         weeklyXpLeaderboardExpirationTimeInSeconds,
         entry.uid,
         xpGained,
-        JSON.stringify({ ...entry, timeTypedSeconds: totalTimeTypedSeconds })
-      ) as Promise<number>,
+        JSON.stringify(
+          RedisXpLeaderboardEntrySchema.parse({
+            ...entry,
+            timeTypedSeconds: totalTimeTypedSeconds,
+          })
+        )
+      ),
       LaterQueue.scheduleForNextWeek(
         "weekly-xp-leaderboard-results",
         "weekly-xp"
@@ -138,10 +137,8 @@ export class WeeklyXpLeaderboard {
     const { weeklyXpLeaderboardScoresKey, weeklyXpLeaderboardResultsKey } =
       this.getThisWeeksXpLeaderboardKeys();
 
-    // @ts-expect-error we are doing some weird file to function mapping, thats why its any
-    // eslint-disable-next-line @typescript-eslint/no-unsafe-call
     const [results, scores] = (await connection.getResults(
-      2, // How many of the arguments are redis keys (https://redis.io/docs/manual/programmability/lua-api/)
+      2,
       weeklyXpLeaderboardScoresKey,
       weeklyXpLeaderboardResultsKey,
       minRank,
@@ -163,14 +160,32 @@ export class WeeklyXpLeaderboard {
 
     const resultsWithRanks: XpLeaderboardEntry[] = results.map(
       (resultJSON: string, index: number) => {
-        //TODO parse with zod?
-        const parsed = JSON.parse(resultJSON) as XpLeaderboardEntry;
-
-        return {
-          ...parsed,
-          rank: minRank + index + 1,
-          totalXp: parseInt(scores[index] as string, 10),
-        };
+        try {
+          const parsed = parseJsonWithSchema(
+            resultJSON,
+            RedisXpLeaderboardEntrySchema
+          );
+          const scoreValue = scores[index];
+
+          if (typeof scoreValue !== "string") {
+            throw new Error(
+              `Invalid score value at index ${index}: ${scoreValue}`
+            );
+          }
+
+          return {
+            ...parsed,
+            rank: minRank + index + 1,
+            totalXp: parseInt(scoreValue, 10),
+          };
+        } catch (error) {
+          throw new MonkeyError(
+            500,
+            `Failed to parse leaderboard entry at index ${index}: ${
+              error instanceof Error ? error.message : String(error)
+            }`
+          );
+        }
       }
     );
 
@@ -187,15 +202,12 @@ export class WeeklyXpLeaderboard {
   ): Promise<XpLeaderboardEntry | null> {
     const connection = RedisClient.getConnection();
     if (!connection || !weeklyXpLeaderboardConfig.enabled) {
-      throw new MonkeyError(500, "Redis connnection is unavailable");
+      throw new MonkeyError(500, "Redis connection is unavailable");
     }
 
     const { weeklyXpLeaderboardScoresKey, weeklyXpLeaderboardResultsKey } =
       this.getThisWeeksXpLeaderboardKeys();
 
-    // eslint-disable-next-line @typescript-eslint/no-unused-expressions
-    connection.set;
-
     const [[, rank], [, totalXp], [, _count], [, result]] = (await connection
       .multi()
       .zrevrank(weeklyXpLeaderboardScoresKey, uid)
@@ -213,11 +225,21 @@ export class WeeklyXpLeaderboard {
       return null;
     }
 
-    //TODO parse with zod?
-    const parsed = JSON.parse((result as string) ?? "null") as Omit<
-      XpLeaderboardEntry,
-      "rank" | "count" | "totalXp"
-    >;
+    // safely parse the result with error handling
+    let parsed: RedisXpLeaderboardEntry;
+    try {
+      parsed = parseJsonWithSchema(
+        result ?? "null",
+        RedisXpLeaderboardEntrySchema
+      );
+    } catch (error) {
+      throw new MonkeyError(
+        500,
+        `Failed to parse leaderboard entry: ${
+          error instanceof Error ? error.message : String(error)
+        }`
+      );
+    }
 
     return {
       ...parsed,
@@ -261,8 +283,6 @@ export async function purgeUserFromXpLeaderboards(
     return;
   }
 
-  // @ts-expect-error we are doing some weird file to function mapping, thats why its any
-  // eslint-disable-next-line @typescript-eslint/no-unsafe-call
   await connection.purgeResults(
     0,
     uid,