Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into more-coverage

mchangrh · mchangrh · commit 622511c8fd60 · 2022-10-27T01:38:08.000-04:00
diff --git a/Dockerfile b/Dockerfile
@@ -6,11 +6,11 @@ RUN npm ci && npm run tsc
 
 FROM node:16-alpine as app
 WORKDIR /usr/src/app
-RUN apk add git postgresql-client
+RUN apk add --no-cache git postgresql-client
 COPY --from=builder ./node_modules ./node_modules
 COPY --from=builder ./dist ./dist
 COPY ./.git ./.git
 COPY entrypoint.sh .
 COPY databases/*.sql databases/
 EXPOSE 8080
-CMD ./entrypoint.sh
+CMD ./entrypoint.sh
diff --git a/src/config.ts b/src/config.ts
@@ -76,8 +76,7 @@ addDefaults(config, {
         port: 5432,
         max: 10,
         idleTimeoutMillis: 10000,
-        maxTries: 3,
-        maxConcurrentRequests: 3500
+        maxTries: 3
     },
     postgresReadOnly: {
         enabled: false,
@@ -91,7 +90,7 @@ addDefaults(config, {
         idleTimeoutMillis: 10000,
         maxTries: 3,
         fallbackOnFail: true,
-        maxConcurrentRequests: 3500
+        stopRetryThreshold: 800
     },
     dumpDatabase: {
         enabled: false,
@@ -139,6 +138,15 @@ addDefaults(config, {
         expiryTime: 24 * 60 * 60,
         getTimeout: 40
     },
+    redisRead: {
+        enabled: false,
+        socket: {
+            host: "",
+            port: 0
+        },
+        disableOfflineQueue: true,
+        weight: 1
+    },
     patreon: {
         clientId: "",
         clientSecret: "",
diff --git a/src/databases/Postgres.ts b/src/databases/Postgres.ts
@@ -33,8 +33,7 @@ export class Postgres implements IDatabase {
     private poolRead: Pool;
     private lastPoolReadFail = 0;
 
-    private concurrentRequests = 0;
-    private concurrentReadRequests = 0;
+    activePostgresRequests = 0;
 
     constructor(private config: DatabaseConfig) {}
 
@@ -54,19 +53,23 @@ export class Postgres implements IDatabase {
         });
 
         if (this.config.postgresReadOnly && this.config.postgresReadOnly.enabled) {
-            this.poolRead = new Pool({
-                ...this.config.postgresReadOnly
-            });
-            this.poolRead.on("error", (err, client) => {
-                Logger.error(err.stack);
-                this.lastPoolReadFail = Date.now();
-
-                try {
-                    client.release(true);
-                } catch (err) {
-                    Logger.error(`poolRead (postgres): ${err}`);
-                }
-            });
+            try {
+                this.poolRead = new Pool({
+                    ...this.config.postgresReadOnly
+                });
+                this.poolRead.on("error", (err, client) => {
+                    Logger.error(err.stack);
+                    this.lastPoolReadFail = Date.now();
+
+                    try {
+                        client.release(true);
+                    } catch (err) {
+                        Logger.error(`poolRead (postgres): ${err}`);
+                    }
+                });
+            } catch (e) {
+                Logger.error(`poolRead (postgres): ${e}`);
+            }
         }
 
         if (!this.config.readOnly) {
@@ -102,22 +105,6 @@ export class Postgres implements IDatabase {
 
         Logger.debug(`prepare (postgres): type: ${type}, query: ${query}, params: ${params}`);
 
-        if (this.config.readOnly) {
-            if (this.concurrentReadRequests > this.config.postgresReadOnly?.maxConcurrentRequests) {
-                Logger.error(`prepare (postgres): cancelling read query because too many concurrent requests, query: ${query}`);
-                throw new Error("Too many concurrent requests");
-            }
-
-            this.concurrentReadRequests++;
-        } else {
-            if (this.concurrentRequests > this.config.postgres.maxConcurrentRequests) {
-                Logger.error(`prepare (postgres): cancelling query because too many concurrent requests, query: ${query}`);
-                throw new Error("Too many concurrent requests");
-            }
-
-            this.concurrentRequests++;
-        }
-
         const pendingQueries: PromiseWithState<QueryResult<any>>[] = [];
         let tries = 0;
         let lastPool: Pool = null;
@@ -127,19 +114,15 @@ export class Postgres implements IDatabase {
             tries++;
 
             try {
+                this.activePostgresRequests++;
                 lastPool = this.getPool(type, options);
 
                 pendingQueries.push(savePromiseState(lastPool.query({ text: query, values: params })));
                 const currentPromises = [...pendingQueries];
                 if (options.useReplica && maxTries() - tries > 1) currentPromises.push(savePromiseState(timeoutPomise(this.config.postgresReadOnly.readTimeout)));
                 const queryResult = await nextFulfilment(currentPromises);
 
-                if (this.config.readOnly) {
-                    this.concurrentReadRequests--;
-                } else {
-                    this.concurrentRequests--;
-                }
-
+                this.activePostgresRequests--;
                 switch (type) {
                     case "get": {
                         const value = queryResult.rows[0];
@@ -159,30 +142,30 @@ export class Postgres implements IDatabase {
                 if (lastPool === this.pool) {
                     // Only applies if it is get or all request
                     options.forceReplica = true;
-                } else if (lastPool === this.poolRead && maxTries() - tries <= 1) {
-                    options.useReplica = false;
+                } else if (lastPool === this.poolRead) {
+                    this.lastPoolReadFail = Date.now();
+
+                    if (maxTries() - tries <= 1) {
+                        options.useReplica = false;
+                    }
                 }
 
                 Logger.error(`prepare (postgres) try ${tries}: ${err}`);
             }
-        } while (this.isReadQuery(type) && tries < maxTries());
-
-        if (this.config.readOnly) {
-            this.concurrentReadRequests--;
-        } else {
-            this.concurrentRequests--;
-        }
+        } while (this.isReadQuery(type) && tries < maxTries()
+            && this.activePostgresRequests < this.config.postgresReadOnly.stopRetryThreshold);
 
+        this.activePostgresRequests--;
         throw new Error(`prepare (postgres): ${type} ${query} failed after ${tries} tries`);
     }
 
     private getPool(type: string, options: QueryOption): Pool {
         const readAvailable = this.poolRead && options.useReplica && this.isReadQuery(type);
-        const ignroreReadDueToFailure = this.config.postgresReadOnly.fallbackOnFail
+        const ignoreReadDueToFailure = this.config.postgresReadOnly.fallbackOnFail
             && this.lastPoolReadFail > Date.now() - 1000 * 30;
         const readDueToFailure = this.config.postgresReadOnly.fallbackOnFail
             && this.lastPoolFail > Date.now() - 1000 * 30;
-        if (readAvailable && !ignroreReadDueToFailure && (options.forceReplica || readDueToFailure ||
+        if (readAvailable && !ignoreReadDueToFailure && (options.forceReplica || readDueToFailure ||
                 Math.random() > 1 / (this.config.postgresReadOnly.weight + 1))) {
             return this.poolRead;
         } else {
diff --git a/src/index.ts b/src/index.ts
@@ -12,7 +12,13 @@ async function init() {
         process.exit(1);
     });
 
-    await initDb();
+    try {
+        await initDb();
+    } catch (e) {
+        Logger.error(`Init Db: ${e}`);
+        process.exit(1);
+    }
+
     // edge case clause for creating compatible .db files, do not enable
     if (config.mode === "init-db-and-exit") process.exit(0);
     // do not enable init-db-only mode for usage.
@@ -27,4 +33,4 @@ async function init() {
     }).setTimeout(15000);
 }
 
-init().catch((err) => Logger.error(err));
+init().catch((err) => Logger.error(`Index.js: ${err}`));
diff --git a/src/routes/generateToken.ts b/src/routes/generateToken.ts
@@ -1,7 +1,7 @@
 import { Request, Response } from "express";
 import { config } from "../config";
 import { createAndSaveToken, TokenType } from "../utils/tokenUtils";
-
+import { getHashCache } from "../utils/getHashCache";
 
 interface GenerateTokenRequest extends Request {
     query: {
@@ -15,12 +15,13 @@ interface GenerateTokenRequest extends Request {
 
 export async function generateTokenRequest(req: GenerateTokenRequest, res: Response): Promise<Response> {
     const { query: { code, adminUserID }, params: { type } } = req;
+    const adminUserIDHash = adminUserID ? (await getHashCache(adminUserID)) : null;
 
     if (!code || !type) {
         return res.status(400).send("Invalid request");
     }
 
-    if (type === TokenType.patreon || (type === TokenType.local && adminUserID === config.adminUserID)) {
+    if (type === TokenType.patreon || (type === TokenType.local && adminUserIDHash === config.adminUserID)) {
         const licenseKey = await createAndSaveToken(type, code);
 
         /* istanbul ignore else */
diff --git a/src/routes/getSearchSegments.ts b/src/routes/getSearchSegments.ts
@@ -14,7 +14,7 @@ type searchSegmentResponse = {
 function getSegmentsFromDBByVideoID(videoID: VideoID, service: Service): Promise<DBSegment[]> {
     return db.prepare(
         "all",
-        `SELECT "UUID", "timeSubmitted", "startTime", "endTime", "category", "actionType", "votes", "views", "locked", "hidden", "shadowHidden", "userID" FROM "sponsorTimes" 
+        `SELECT "UUID", "timeSubmitted", "startTime", "endTime", "category", "actionType", "votes", "views", "locked", "hidden", "shadowHidden", "userID", "description" FROM "sponsorTimes" 
         WHERE "videoID" = ? AND "service" = ? ORDER BY "timeSubmitted"`,
         [videoID, service]
     ) as Promise<DBSegment[]>;
diff --git a/src/routes/getStatus.ts b/src/routes/getStatus.ts
@@ -2,28 +2,31 @@ import { db } from "../databases/databases";
 import { Logger } from "../utils/logger";
 import { Request, Response } from "express";
 import os from "os";
-import redis from "../utils/redis";
+import redis, { getRedisActiveRequests } from "../utils/redis";
 import { promiseOrTimeout } from "../utils/promise";
+import { Postgres } from "../databases/Postgres";
 
 export async function getStatus(req: Request, res: Response): Promise<Response> {
     const startTime = Date.now();
     let value = req.params.value as string[] | string;
     value = Array.isArray(value) ? value[0] : value;
     let processTime, redisProcessTime = -1;
     try {
+        const dbStartTime = Date.now();
         const dbVersion = await promiseOrTimeout(db.prepare("get", "SELECT key, value FROM config where key = ?", ["version"]), 5000)
             .then(e => {
-                processTime = Date.now() - startTime;
+                processTime = Date.now() - dbStartTime;
                 return e.value;
             })
             .catch(e => /* istanbul ignore next */ {
                 Logger.error(`status: SQL query timed out: ${e}`);
                 return -1;
             });
         let statusRequests: unknown = 0;
+        const redisStartTime = Date.now();
         const numberRequests = await promiseOrTimeout(redis.increment("statusRequest"), 5000)
             .then(e => {
-                redisProcessTime = Date.now() - startTime;
+                redisProcessTime = Date.now() - redisStartTime;
                 return e;
             }).catch(e => /* istanbul ignore next */ {
                 Logger.error(`status: redis increment timed out ${e}`);
@@ -40,7 +43,9 @@ export async function getStatus(req: Request, res: Response): Promise<Response>
             redisProcessTime,
             loadavg: os.loadavg().slice(1), // only return 5 & 15 minute load average
             statusRequests,
-            hostname: os.hostname()
+            hostname: os.hostname(),
+            activePostgresRequests: (db as Postgres)?.activePostgresRequests,
+            activeRedisRequests: getRedisActiveRequests(),
         };
         return value ? res.send(JSON.stringify(statusValues[value])) : res.send(statusValues);
     } catch (err) /* istanbul ignore next */ {
diff --git a/src/routes/getTopCategoryUsers.ts b/src/routes/getTopCategoryUsers.ts
@@ -26,7 +26,7 @@ async function generateTopCategoryUsersStats(sortBy: string, category: string) {
         SUM("votes") as "userVotes", COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID"
         LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID"
         WHERE "sponsorTimes"."category" = ? AND "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL
-        GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20
+        GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 2
         ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds, category]);
 
     if (rows) {
diff --git a/src/routes/getTopUsers.ts b/src/routes/getTopUsers.ts
@@ -28,14 +28,15 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled = fals
             SUM(CASE WHEN category = 'poi_highlight' THEN 1 ELSE 0 END) as "categorySumHighlight",
             SUM(CASE WHEN category = 'filler' THEN 1 ELSE 0 END) as "categorySumFiller",
             SUM(CASE WHEN category = 'exclusive_access' THEN 1 ELSE 0 END) as "categorySumExclusiveAccess",
+            SUM(CASE WHEN category = 'chapter' THEN 1 ELSE 0 END) as "categorySumChapter",
         `;
     }
 
     const rows = await db.prepare("all", `SELECT COUNT(*) as "totalSubmissions", SUM(views) as "viewCount",
-        SUM(((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views") as "minutesSaved",
+        SUM(CASE WHEN "sponsorTimes"."actionType" = 'chapter' THEN 0 ELSE ((CASE WHEN "sponsorTimes"."endTime" - "sponsorTimes"."startTime" > ? THEN ? ELSE "sponsorTimes"."endTime" - "sponsorTimes"."startTime" END) / 60) * "sponsorTimes"."views" END) as "minutesSaved",
         SUM("votes") as "userVotes", ${additionalFields} COALESCE("userNames"."userName", "sponsorTimes"."userID") as "userName" FROM "sponsorTimes" LEFT JOIN "userNames" ON "sponsorTimes"."userID"="userNames"."userID"
         LEFT JOIN "shadowBannedUsers" ON "sponsorTimes"."userID"="shadowBannedUsers"."userID"
-        WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "sponsorTimes"."actionType" != 'chapter' AND "shadowBannedUsers"."userID" IS NULL
+        WHERE "sponsorTimes"."votes" > -1 AND "sponsorTimes"."shadowHidden" != 1 AND "shadowBannedUsers"."userID" IS NULL
         GROUP BY COALESCE("userName", "sponsorTimes"."userID") HAVING SUM("votes") > 20
         ORDER BY "${sortBy}" DESC LIMIT 100`, [maxRewardTimePerSegmentInSeconds, maxRewardTimePerSegmentInSeconds]);
 
@@ -55,7 +56,8 @@ async function generateTopUsersStats(sortBy: string, categoryStatsEnabled = fals
                 row.categorySumPreview,
                 row.categorySumHighlight,
                 row.categorySumFiller,
-                row.categorySumExclusiveAccess
+                row.categorySumExclusiveAccess,
+                row.categorySumChapter
             ]);
         }
     }
diff --git a/src/routes/getUserInfo.ts b/src/routes/getUserInfo.ts
@@ -118,8 +118,7 @@ async function getPermissions(userID: HashedUserID): Promise<Record<string, bool
 
 async function getFreeChaptersAccess(userID: HashedUserID): Promise<boolean> {
     return await oneOf([isUserVIP(userID),
-        (async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "reputation" > 0 AND "timeSubmitted" < 1663872563000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))(),
-        (async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "timeSubmitted" < 1590969600000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))()
+        (async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "timeSubmitted" < 1666126187000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))()
     ]);
 }
 
diff --git a/src/routes/postSkipSegments.ts b/src/routes/postSkipSegments.ts
@@ -334,7 +334,7 @@ async function checkByAutoModerator(videoID: any, userID: any, segments: Array<a
             return {
                 pass: false,
                 errorCode: 403,
-                errorMessage: `Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.`
+                errorMessage: `Hi, currently there are server issues and you might have not recieved segments even though they exist. Sorry about this, I'm working on it. Request rejected by auto moderator: ${autoModerateResult} If this is an issue, send a message on Discord.`
             };
         }
     }
diff --git a/src/routes/voteOnSponsorTime.ts b/src/routes/voteOnSponsorTime.ts
@@ -14,6 +14,7 @@ import { DBSegment, Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID
 import { QueryCacher } from "../utils/queryCacher";
 import axios from "axios";
 import { getVideoDetails, videoDetails } from "../utils/getVideoDetails";
+import { deleteLockCategories } from "./deleteLockCategories";
 
 const voteTypes = {
     normal: 0,
@@ -59,7 +60,7 @@ async function updateSegmentVideoDuration(UUID: SegmentUUID) {
     let apiVideoDetails: videoDetails = null;
     if (service == Service.YouTube) {
         // don't use cache since we have no information about the video length
-        apiVideoDetails = await getVideoDetails(videoID);
+        apiVideoDetails = await getVideoDetails(videoID, true);
     }
     const apiVideoDuration = apiVideoDetails?.duration as VideoDuration;
     if (videoDurationChanged(videoDuration, apiVideoDuration)) {
@@ -95,6 +96,7 @@ async function checkVideoDuration(UUID: SegmentUUID) {
             AND "hidden" = 0 AND "shadowHidden" = 0 AND 
             "actionType" != 'full' AND "votes" > -2`,
         [videoID, service, latestSubmission.timeSubmitted]);
+        deleteLockCategories(videoID, null, null, service).catch(Logger.error);
     }
 }
 
diff --git a/src/types/config.model.ts b/src/types/config.model.ts
@@ -7,16 +7,21 @@ interface RedisConfig extends redis.RedisClientOptions {
     getTimeout: number;
 }
 
+interface RedisReadOnlyConfig extends redis.RedisClientOptions {
+    enabled: boolean;
+    weight: number;
+}
+
 export interface CustomPostgresConfig extends PoolConfig {
     enabled: boolean;
     maxTries: number;
-    maxConcurrentRequests: number;
 }
 
 export interface CustomPostgresReadOnlyConfig extends CustomPostgresConfig {
     weight: number;
     readTimeout: number;
     fallbackOnFail: boolean;
+    stopRetryThreshold: number;
 }
 
 export interface SBSConfig {
@@ -61,6 +66,7 @@ export interface SBSConfig {
     minimumPrefix?: string;
     maximumPrefix?: string;
     redis?: RedisConfig;
+    redisRead?: RedisReadOnlyConfig;
     maxRewardTimePerSegmentInSeconds?: number;
     postgres?: CustomPostgresConfig;
     postgresReadOnly?: CustomPostgresReadOnlyConfig;
diff --git a/src/utils/permissions.ts b/src/utils/permissions.ts
diff --git a/src/utils/redis.ts b/src/utils/redis.ts
diff --git a/test/cases/getSearchSegments.ts b/test/cases/getSearchSegments.ts

Original file line number	Diff line number	Diff line change
`@@ -118,8 +118,7 @@ async function getPermissions(userID: HashedUserID): Promise<Record<string, bool`
`118`	`118`
`119`	`119`	`async function getFreeChaptersAccess(userID: HashedUserID): Promise<boolean> {`
`120`	`120`	`return await oneOf([isUserVIP(userID),`
`121`		- (async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "reputation" > 0 AND "timeSubmitted" < 1663872563000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))(),
`122`		- (async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "timeSubmitted" < 1590969600000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))()
	`121`	+ (async () => !!(await db.prepare("get", `SELECT "timeSubmitted" FROM "sponsorTimes" WHERE "timeSubmitted" < 1666126187000 AND "userID" = ? LIMIT 1`, [userID], { useReplica: true })))()
`123`	`122`	`]);`
`124`	`123`	`}`
`125`	`124`