test helpers 2 electric boogaloo

.github/workflows/test.yaml

@@ -60,13 +60,6 @@ jobs:
     needs: lint-build
     steps:
       - uses: actions/checkout@v3
-      - name: Build the docker-compose stack
-        env:
-          PG_USER: ci_db_user
-          PG_PASS: ci_db_pass
-        run: docker-compose -f docker/docker-compose-ci.yml up -d
-      - name: Check running containers
-        run: docker ps
       - uses: actions/setup-node@v3
         with:
           node-version: 18
@@ -90,6 +83,28 @@ jobs:
         with:
           key: nyc-postgres-${{ github.sha }}
           path: ${{ github.workspace }}/.nyc_output
+    services:
+      redis:
+        image: redis:alpine
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+      postgres:
+        image: postgres:alpine
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        env:
+          POSTGRES_USER: ci_db_user
+          POSTGRES_PASSWORD: ci_db_pass
   codecov:
     needs: [test-sqlite, test-postgres]
     name: Run Codecov

docker/docker-compose-ci.yml

@@ -4,8 +4,8 @@ services:
     container_name: database-co
     image: postgres:alpine
     environment:
-      - POSTGRES_USER=${PG_USER}
-      - POSTGRES_PASSWORD=${PG_PASS}
+      - POSTGRES_USER=ci_db_user
+      - POSTGRES_PASSWORD=ci_db_pass
     ports:
       - 5432:5432
   redis:

src/app.ts

@@ -27,7 +27,7 @@ import { corsMiddleware } from "./middleware/cors";
 import { apiCspMiddleware } from "./middleware/apiCsp";
 import { rateLimitMiddleware } from "./middleware/requestRateLimit";
 import dumpDatabase from "./routes/dumpDatabase";
-import { endpoint as getSegmentInfo } from "./routes/getSegmentInfo";
+import { getSegmentInfo } from "./routes/getSegmentInfo";
 import { postClearCache } from "./routes/postClearCache";
 import { addUnlistedVideo } from "./routes/addUnlistedVideo";
 import { postPurgeAllSegments } from "./routes/postPurgeAllSegments";

src/routes/getSegmentInfo.ts

@@ -1,8 +1,7 @@
 import { Request, Response } from "express";
 import { db } from "../databases/databases";
 import { DBSegment, SegmentUUID } from "../types/segments.model";
-
-const isValidSegmentUUID = (str: string): boolean => /^([a-f0-9]{64}|[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})/.test(str);
+import { parseUUIDs } from "../utils/parseParams";
 
 async function getSegmentFromDBByUUID(UUID: SegmentUUID): Promise<DBSegment> {
     try {
@@ -15,63 +14,34 @@ async function getSegmentFromDBByUUID(UUID: SegmentUUID): Promise<DBSegment> {
 async function getSegmentsByUUID(UUIDs: SegmentUUID[]): Promise<DBSegment[]> {
     const DBSegments: DBSegment[] = [];
     for (const UUID of UUIDs) {
-        // if UUID is invalid, skip
-        if (!isValidSegmentUUID(UUID)) continue;
         DBSegments.push(await getSegmentFromDBByUUID(UUID as SegmentUUID));
     }
     return DBSegments;
 }
 
-async function handleGetSegmentInfo(req: Request, res: Response): Promise<DBSegment[]> {
+async function getSegmentInfo(req: Request, res: Response): Promise<Response> {
     // If using params instead of JSON, only one UUID can be pulled
-    let UUIDs = req.query.UUIDs
-        ? JSON.parse(req.query.UUIDs as string)
-        : req.query.UUID
-            ? Array.isArray(req.query.UUID)
-                ? req.query.UUID
-                : [req.query.UUID]
-            : null;
-    // deduplicate with set
-    UUIDs = [ ...new Set(UUIDs)];
-    // if more than 10 entries, slice
+    let UUIDs = parseUUIDs(req);
+    // verify format
     if (!Array.isArray(UUIDs) || !UUIDs?.length) {
         res.status(400).send("UUIDs parameter does not match format requirements.");
         return;
     }
+    // deduplicate with set
+    UUIDs = [ ...new Set(UUIDs)];
+    // if more than 10 entries, slice
     if (UUIDs.length > 10) UUIDs = UUIDs.slice(0, 10);
     const DBSegments = await getSegmentsByUUID(UUIDs);
-    // all uuids failed lookup
-    if (!DBSegments?.length) {
-        res.sendStatus(400);
-        return;
-    }
     // uuids valid but not found
-    if (DBSegments[0] === null || DBSegments[0] === undefined) {
-        res.sendStatus(400);
+    if (!DBSegments?.length || DBSegments[0] === null || DBSegments[0] === undefined) {
+        res.status(404).send("UUIDs not found in database.");
         return;
     }
-    return DBSegments;
-}
-
-async function endpoint(req: Request, res: Response): Promise<Response> {
-    try {
-        const DBSegments = await handleGetSegmentInfo(req, res);
-
-        // If false, res.send has already been called
-        if (DBSegments) {
-            //send result
-            return res.send(DBSegments);
-        }
-    } catch (err) /* istanbul ignore next */ {
-        if (err instanceof SyntaxError) { // catch JSON.parse error
-            return res.status(400).send("UUIDs parameter does not match format requirements.");
-        } else return res.sendStatus(500);
-    }
+    return res.send(DBSegments);
 }
 
 export {
     getSegmentFromDBByUUID,
     getSegmentsByUUID,
-    handleGetSegmentInfo,
-    endpoint
+    getSegmentInfo
 };

src/routes/postSkipSegments.ts

@@ -187,7 +187,7 @@ async function checkUserActiveWarning(userID: HashedUserID): Promise<CheckResult
     return CHECK_PASS;
 }
 
-async function checkInvalidFields(videoID: VideoID, userID: UserID, hashedUserID: HashedUserID
+async function checkInvalidFields(videoID: VideoID, userID: string, hashedUserID: HashedUserID
     , segments: IncomingSegment[], videoDurationParam: number, userAgent: string, service: Service): Promise<CheckResult> {
     const invalidFields = [];
     const errors = [];
@@ -496,10 +496,10 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
     let { videoID, userID: paramUserID, service, videoDuration, videoDurationParam, segments, userAgent } = preprocessInput(req);
 
     //hash the userID
-    if (!paramUserID) {
+    if (!paramUserID || typeof paramUserID !== "string"){
         return res.status(400).send("No userID provided");
     }
-    const userID: HashedUserID = await getHashCache(paramUserID);
+    const userID: HashedUserID = await getHashCache(paramUserID) as HashedUserID;
 
     const invalidCheckResult = await checkInvalidFields(videoID, paramUserID, userID, segments, videoDurationParam, userAgent, service);
     if (!invalidCheckResult.pass) {
@@ -528,7 +528,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
         const { lockedCategoryList, apiVideoDetails } = newData;
 
         // Check if all submissions are correct
-        const segmentCheckResult = await checkEachSegmentValid(rawIP, paramUserID, userID, videoID, segments, service, isVIP, isTempVIP, lockedCategoryList);
+        const segmentCheckResult = await checkEachSegmentValid(rawIP, paramUserID as UserID, userID, videoID, segments, service, isVIP, isTempVIP, lockedCategoryList);
         if (!segmentCheckResult.pass) {
             lock.unlock();
             return res.status(segmentCheckResult.errorCode).send(segmentCheckResult.errorMessage);

src/utils/parseParams.ts

@@ -4,28 +4,55 @@ import { config } from "../config";
 
 type fn = (req: Request, fallback: any) => any[];
 
+// generic parsing handlers
 const syntaxErrorWrapper = (fn: fn, req: Request, fallback: any) => {
     try { return fn(req, fallback); }
     catch (e) {
         return undefined;
     }
 };
 
+/**
+ * This function acts as a parser for singular and plural query parameters
+ * either as arrays, strings as arrays or singular values with optional fallback
+ * The priority is to parse the plural parameter natively, then with JSON
+ * then the singular parameter as an array, then as a single value
+ * and then finally fall back to the fallback value
+ * @param req Axios Request object
+ * @param fallback fallback value in case all parsing fail
+ * @param param Name of singular parameter
+ * @param paramPlural Name of plural parameter
+ * @returns Array of values
+ */
 const getQueryList = <T>(req: Request, fallback: T[], param: string, paramPlural: string): string[] | T[] =>
     req.query[paramPlural]
-        ? JSON.parse(req.query[paramPlural] as string)
+        ? Array.isArray(req.query[paramPlural])
+            ? req.query[paramPlural]
+            : JSON.parse(req.query[paramPlural] as string)
         : req.query[param]
             ? Array.isArray(req.query[param])
                 ? req.query[param]
                 : [req.query[param]]
             : fallback;
 
+// specfic parsing handlers
 const getCategories = (req: Request, fallback: Category[] ): string[] | Category[] =>
     getQueryList(req, fallback, "category", "categories");
 
 const getDeArrowTypes = (req: Request, fallback: DeArrowType[] ): string[] | DeArrowType[] =>
     getQueryList(req, fallback, "deArrowType", "deArrowTypes");
 
+const getActionTypes = (req: Request, fallback: ActionType[]): string[] | ActionType[] =>
+    getQueryList(req, fallback, "actionType", "actionTypes");
+
+const getRequiredSegments = (req: Request): string[] | SegmentUUID[] =>
+    getQueryList(req, [], "requiredSegment", "requiredSegments");
+
+const getUUIDs = (req: Request): string[] | SegmentUUID[] =>
+    getQueryList(req, [], "UUID", "UUIDs");
+
+
+// validation handlers
 const validateString = (array: any[]): any[] => {
     if (!Array.isArray(array)) return undefined;
     return array
@@ -45,28 +72,17 @@ const filterActionType = (actionTypes: ActionType[]) => {
     return [...filterCategories];
 };
 
+const validateUUID = (array: string[]): SegmentUUID[] => {
+    if (!Array.isArray(array)) return undefined;
+    const filtered = array
+        .filter((item: string) => typeof item === "string")
+        .filter((item: string) => /^([a-f0-9]{64,65}|[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12})$/.test(item));
+    return filtered as SegmentUUID[];
+};
+
 export const filterInvalidCategoryActionType = (categories: Category[], actionTypes: ActionType[]): Category[] =>
     categories.filter((category: Category) => filterActionType(actionTypes).includes(category));
 
-const getActionTypes = (req: Request, fallback: ActionType[]): ActionType[] =>
-    req.query.actionTypes
-        ? JSON.parse(req.query.actionTypes as string)
-        : req.query.actionType
-            ? Array.isArray(req.query.actionType)
-                ? req.query.actionType
-                : [req.query.actionType]
-            : fallback;
-
-// fallback to empty array
-const getRequiredSegments = (req: Request): SegmentUUID[] =>
-    req.query.requiredSegments
-        ? JSON.parse(req.query.requiredSegments as string)
-        : req.query.requiredSegment
-            ? Array.isArray(req.query.requiredSegment)
-                ? req.query.requiredSegment
-                : [req.query.requiredSegment]
-            : [];
-
 export const parseCategories = (req: Request, fallback: Category[]): Category[] => {
     const categories = syntaxErrorWrapper(getCategories, req, fallback);
     return categories ? validateString(categories) : undefined;
@@ -82,8 +98,16 @@ export const parseDeArrowTypes = (req: Request, fallback: DeArrowType[]): DeArro
     return deArrowTypes ? validateString(deArrowTypes) : undefined;
 };
 
-export const parseRequiredSegments = (req: Request): SegmentUUID[] | undefined =>
-    syntaxErrorWrapper(getRequiredSegments, req, []); // never fall back
+export const parseRequiredSegments = (req: Request): SegmentUUID[] | undefined => {
+    // fall back to empty array
+    // we do not do regex validation since required segments can be partial UUIDs on videos
+    return syntaxErrorWrapper(getRequiredSegments, req, []);
+};
+
+export const parseUUIDs = (req: Request): SegmentUUID[] | undefined => {
+    const UUIDs = syntaxErrorWrapper(getUUIDs, req, []); // fall back to empty array
+    return UUIDs ? validateUUID(UUIDs) : undefined;
+};
 
 export const validateCategories = (categories: string[]): boolean =>
     categories.every((category: string) => config.categoryList.includes(category));

test/cases/addFeatures.ts

@@ -4,7 +4,7 @@ import { Feature } from "../../src/types/user.model";
 import { hasFeature } from "../../src/utils/features";
 import { client } from "../utils/httpClient";
 import { grantFeature, insertVip } from "../utils/queryGen";
-import { User, genUser, genUsers } from "../utils/genUser";
+import { User, genUser, genUsersProxy } from "../utils/genUser";
 
 const endpoint = "/api/feature";
 
@@ -19,12 +19,7 @@ const postAddFeatures = (userID: string, adminUserID: string, feature: Feature,
     }
 });
 
-const cases = [
-    "grant",
-    "remove",
-    "update"
-];
-const users = genUsers("addFeatures", cases);
+const users = genUsersProxy("addFeatures");
 const vipUser = genUser("addFeatures", "vip");
 
 const testedFeature = Feature.ChapterSubmitter;

test/cases/addUserAsVIP.ts

@@ -2,15 +2,12 @@ import { HashedUserID } from "../../src/types/user.model";
 import { client } from "../utils/httpClient";
 import { db } from "../../src/databases/databases";
 import assert from "assert";
-import { genAnonUser, genUsers } from "../utils/genUser";
+import { genAnonUser, genUsersProxy } from "../utils/genUser";
 
 // helpers
 const checkUserVIP = (publicID: string) => db.prepare("get", `SELECT "userID" FROM "vipUsers" WHERE "userID" = ?`, [publicID]);
 
-const cases = [
-    "vip-1",
-];
-const users = genUsers("endpoint", cases);
+const users = genUsersProxy("addUserAsVIP");
 
 // hardcoded into test code
 const adminPrivateUserID = "testUserId";