Skip to content

Commit d80be9e

Browse files
mchangrhmchangrh
committed
Merge branch 'master' of https://github.com/ajayyy/SponsorBlockServer into more-coverage
2 parents a8ddae1 + 847f1bb commit d80be9e

17 files changed

+285
-100
lines changed

‎DatabaseSchema.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,6 @@
126126
| channelID | TEXT | not null |
127127
| title | TEXT | not null |
128128
| published | REAL | not null |
129-
| genreUrl | TEXT | not null |
130129

131130
| index | field |
132131
| -- | :--: |

‎databases/_upgrade_sponsorTimes_34.sql‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
BEGIN TRANSACTION;
2+
3+
ALTER TABLE "videoInfo" DROP COLUMN "genreUrl";
4+
5+
UPDATE "config" SET value = 34 WHERE key = 'version';
6+
7+
COMMIT;

‎src/routes/addUserAsTempVIP.ts‎

Lines changed: 4 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
import { VideoID } from "../types/segments.model";
2-
import { YouTubeAPI } from "../utils/youtubeApi";
3-
import { APIVideoInfo } from "../types/youtubeApi.model";
4-
import { config } from "../config";
2+
import { getVideoDetails } from "../utils/getVideoDetails";
53
import { getHashCache } from "../utils/getHashCache";
64
import { privateDB } from "../databases/databases";
75
import { Request, Response } from "express";
@@ -20,15 +18,11 @@ interface AddUserAsTempVIPRequest extends Request {
2018
}
2119
}
2220

23-
function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> {
24-
return (config.newLeafURLs) ? YouTubeAPI.listVideos(videoID, ignoreCache) : null;
25-
}
26-
2721
const getChannelInfo = async (videoID: VideoID): Promise<{id: string | null, name: string | null }> => {
28-
const videoInfo = await getYouTubeVideoInfo(videoID);
22+
const videoInfo = await getVideoDetails(videoID);
2923
return {
30-
id: videoInfo?.data?.authorId,
31-
name: videoInfo?.data?.author
24+
id: videoInfo?.authorId,
25+
name: videoInfo?.authorName
3226
};
3327
};
3428

‎src/routes/getStatus.ts‎

Lines changed: 22 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,25 +3,41 @@ import { Logger } from "../utils/logger";
33
import { Request, Response } from "express";
44
import os from "os";
55
import redis from "../utils/redis";
6+
import { promiseOrTimeout } from "../utils/promise";
67

78
export async function getStatus(req: Request, res: Response): Promise<Response> {
89
const startTime = Date.now();
910
let value = req.params.value as string[] | string;
1011
value = Array.isArray(value) ? value[0] : value;
12+
let processTime, redisProcessTime = -1;
1113
try {
12-
const dbVersion = (await db.prepare("get", "SELECT key, value FROM config where key = ?", ["version"])).value;
14+
const dbVersion = await promiseOrTimeout(db.prepare("get", "SELECT key, value FROM config where key = ?", ["version"]), 5000)
15+
.then(e => {
16+
processTime = Date.now() - startTime;
17+
return e.value;
18+
})
19+
.catch(e => {
20+
Logger.error(`status: SQL query timed out: ${e}`);
21+
return -1;
22+
});
1323
let statusRequests: unknown = 0;
14-
try {
15-
const numberRequests = await redis.increment("statusRequest");
16-
statusRequests = numberRequests?.[0];
17-
} catch (error) { } // eslint-disable-line no-empty
24+
const numberRequests = await promiseOrTimeout(redis.increment("statusRequest"), 5000)
25+
.then(e => {
26+
redisProcessTime = Date.now() - startTime;
27+
return e;
28+
}).catch(e => {
29+
Logger.error(`status: redis increment timed out ${e}`);
30+
return [-1];
31+
});
32+
statusRequests = numberRequests?.[0];
1833

1934
const statusValues: Record<string, any> = {
2035
uptime: process.uptime(),
2136
commit: (global as any).HEADCOMMIT || "unknown",
2237
db: Number(dbVersion),
2338
startTime,
24-
processTime: Date.now() - startTime,
39+
processTime,
40+
redisProcessTime,
2541
loadavg: os.loadavg().slice(1), // only return 5 & 15 minute load average
2642
statusRequests,
2743
hostname: os.hostname()

‎src/routes/postSkipSegments.ts‎

Lines changed: 24 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import { config } from "../config";
22
import { Logger } from "../utils/logger";
33
import { db, privateDB } from "../databases/databases";
4-
import { getMaxResThumbnail, YouTubeAPI } from "../utils/youtubeApi";
4+
import { getMaxResThumbnail } from "../utils/youtubeApi";
55
import { getSubmissionUUID } from "../utils/getSubmissionUUID";
66
import { getHash } from "../utils/getHash";
77
import { getHashCache } from "../utils/getHashCache";
@@ -13,7 +13,6 @@ import { ActionType, Category, IncomingSegment, IPAddress, SegmentUUID, Service,
1313
import { deleteLockCategories } from "./deleteLockCategories";
1414
import { QueryCacher } from "../utils/queryCacher";
1515
import { getReputation } from "../utils/reputation";
16-
import { APIVideoData, APIVideoInfo } from "../types/youtubeApi.model";
1716
import { HashedUserID, UserID } from "../types/user.model";
1817
import { isUserVIP } from "../utils/isUserVIP";
1918
import { isUserTempVIP } from "../utils/isUserTempVIP";
@@ -22,6 +21,7 @@ import { getService } from "../utils/getService";
2221
import axios from "axios";
2322
import { vote } from "./voteOnSponsorTime";
2423
import { canSubmit } from "../utils/permissions";
24+
import { getVideoDetails, videoDetails } from "../utils/getVideoDetails";
2525

2626
type CheckResult = {
2727
pass: boolean,
@@ -35,7 +35,7 @@ const CHECK_PASS: CheckResult = {
3535
errorCode: 0
3636
};
3737

38-
async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: APIVideoData, { submissionStart, submissionEnd }: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
38+
async function sendWebhookNotification(userID: string, videoID: string, UUID: string, submissionCount: number, youtubeData: videoDetails, { submissionStart, submissionEnd }: { submissionStart: number; submissionEnd: number; }, segmentInfo: any) {
3939
const row = await db.prepare("get", `SELECT "userName" FROM "userNames" WHERE "userID" = ?`, [userID]);
4040
const userName = row !== undefined ? row.userName : null;
4141

@@ -48,7 +48,7 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st
4848
"video": {
4949
"id": videoID,
5050
"title": youtubeData?.title,
51-
"thumbnail": getMaxResThumbnail(youtubeData) || null,
51+
"thumbnail": getMaxResThumbnail(videoID),
5252
"url": `https://www.youtube.com/watch?v=${videoID}`,
5353
},
5454
"submission": {
@@ -64,16 +64,13 @@ async function sendWebhookNotification(userID: string, videoID: string, UUID: st
6464
});
6565
}
6666

67-
async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) {
68-
if (apiVideoInfo && service == Service.YouTube) {
67+
async function sendWebhooks(apiVideoDetails: videoDetails, userID: string, videoID: string, UUID: string, segmentInfo: any, service: Service) {
68+
if (apiVideoDetails && service == Service.YouTube) {
6969
const userSubmissionCountRow = await db.prepare("get", `SELECT count(*) as "submissionCount" FROM "sponsorTimes" WHERE "userID" = ?`, [userID]);
7070

71-
const { data, err } = apiVideoInfo;
72-
if (err) return;
73-
7471
const startTime = parseFloat(segmentInfo.segment[0]);
7572
const endTime = parseFloat(segmentInfo.segment[1]);
76-
sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, data, {
73+
sendWebhookNotification(userID, videoID, UUID, userSubmissionCountRow.submissionCount, apiVideoDetails, {
7774
submissionStart: startTime,
7875
submissionEnd: endTime,
7976
}, segmentInfo).catch(Logger.error);
@@ -84,7 +81,7 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
8481

8582
axios.post(config.discordFirstTimeSubmissionsWebhookURL, {
8683
embeds: [{
87-
title: data?.title,
84+
title: apiVideoDetails.title,
8885
url: `https://www.youtube.com/watch?v=${videoID}&t=${(parseInt(startTime.toFixed(0)) - 2)}s#requiredSegment=${UUID}`,
8986
description: `Submission ID: ${UUID}\
9087
\n\nTimestamp: \
@@ -95,7 +92,7 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
9592
name: userID,
9693
},
9794
thumbnail: {
98-
url: getMaxResThumbnail(data) || "",
95+
url: getMaxResThumbnail(videoID),
9996
},
10097
}],
10198
})
@@ -120,18 +117,10 @@ async function sendWebhooks(apiVideoInfo: APIVideoInfo, userID: string, videoID:
120117
// Looks like this was broken for no defined youtube key - fixed but IMO we shouldn't return
121118
// false for a pass - it was confusing and lead to this bug - any use of this function in
122119
// the future could have the same problem.
123-
async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
120+
async function autoModerateSubmission(apiVideoDetails: videoDetails,
124121
submission: { videoID: VideoID; userID: UserID; segments: IncomingSegment[], service: Service, videoDuration: number }) {
125-
126-
const apiVideoDuration = (apiVideoInfo: APIVideoInfo) => {
127-
if (!apiVideoInfo) return undefined;
128-
const { err, data } = apiVideoInfo;
129-
// return undefined if API error
130-
if (err) return undefined;
131-
return data?.lengthSeconds;
132-
};
133122
// get duration from API
134-
const apiDuration = apiVideoDuration(apiVideoInfo);
123+
const apiDuration = apiVideoDetails.duration;
135124
// if API fail or returns 0, get duration from client
136125
const duration = apiDuration || submission.videoDuration;
137126
// return false on undefined or 0
@@ -165,14 +154,6 @@ async function autoModerateSubmission(apiVideoInfo: APIVideoInfo,
165154
return false;
166155
}
167156

168-
function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> {
169-
if (config.newLeafURLs !== null) {
170-
return YouTubeAPI.listVideos(videoID, ignoreCache);
171-
} else {
172-
return null;
173-
}
174-
}
175-
176157
async function checkUserActiveWarning(userID: string): Promise<CheckResult> {
177158
const MILLISECONDS_IN_HOUR = 3600000;
178159
const now = Date.now();
@@ -345,10 +326,10 @@ async function checkEachSegmentValid(rawIP: IPAddress, paramUserID: UserID, user
345326
return CHECK_PASS;
346327
}
347328

348-
async function checkByAutoModerator(videoID: any, userID: any, segments: Array<any>, service:string, apiVideoInfo: APIVideoInfo, videoDuration: number): Promise<CheckResult> {
329+
async function checkByAutoModerator(videoID: any, userID: any, segments: Array<any>, service:string, apiVideoDetails: videoDetails, videoDuration: number): Promise<CheckResult> {
349330
// Auto moderator check
350331
if (service == Service.YouTube) {
351-
const autoModerateResult = await autoModerateSubmission(apiVideoInfo, { userID, videoID, segments, service, videoDuration });
332+
const autoModerateResult = await autoModerateSubmission(apiVideoDetails, { userID, videoID, segments, service, videoDuration });
352333
if (autoModerateResult) {
353334
return {
354335
pass: false,
@@ -377,12 +358,13 @@ async function updateDataIfVideoDurationChange(videoID: VideoID, service: Servic
377358
const videoDurationChanged = (videoDuration: number) => videoDuration != 0
378359
&& previousSubmissions.length > 0 && !previousSubmissions.some((e) => Math.abs(videoDuration - e.videoDuration) < 2);
379360

380-
let apiVideoInfo: APIVideoInfo = null;
361+
let apiVideoDetails: videoDetails = null;
381362
if (service == Service.YouTube) {
382363
// Don't use cache if we don't know the video duration, or the client claims that it has changed
383-
apiVideoInfo = await getYouTubeVideoInfo(videoID, !videoDurationParam || previousSubmissions.length === 0 || videoDurationChanged(videoDurationParam));
364+
const ignoreCache = !videoDurationParam || previousSubmissions.length === 0 || videoDurationChanged(videoDurationParam);
365+
apiVideoDetails = await getVideoDetails(videoID, ignoreCache);
384366
}
385-
const apiVideoDuration = apiVideoInfo?.data?.lengthSeconds as VideoDuration;
367+
const apiVideoDuration = apiVideoDetails?.duration as VideoDuration;
386368
if (!videoDurationParam || (apiVideoDuration && Math.abs(videoDurationParam - apiVideoDuration) > 2)) {
387369
// If api duration is far off, take that one instead (it is only precise to seconds, not millis)
388370
videoDuration = apiVideoDuration || 0 as VideoDuration;
@@ -400,7 +382,7 @@ async function updateDataIfVideoDurationChange(videoID: VideoID, service: Servic
400382

401383
return {
402384
videoDuration,
403-
apiVideoInfo,
385+
apiVideoDetails,
404386
lockedCategoryList
405387
};
406388
}
@@ -501,10 +483,6 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
501483
//hash the userID
502484
const userID = await getHashCache(paramUserID || "");
503485

504-
if (userID === "a41d853c7328a86f8d712f910c4ef77f6c7a9e467f349781b1a7d405c37b681b") {
505-
return res.status(200);
506-
}
507-
508486
const invalidCheckResult = await checkInvalidFields(videoID, paramUserID, userID, segments, videoDurationParam, userAgent);
509487
if (!invalidCheckResult.pass) {
510488
return res.status(invalidCheckResult.errorCode).send(invalidCheckResult.errorMessage);
@@ -521,7 +499,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
521499

522500
const newData = await updateDataIfVideoDurationChange(videoID, service, videoDuration, videoDurationParam);
523501
videoDuration = newData.videoDuration;
524-
const { lockedCategoryList, apiVideoInfo } = newData;
502+
const { lockedCategoryList, apiVideoDetails } = newData;
525503

526504
// Check if all submissions are correct
527505
const segmentCheckResult = await checkEachSegmentValid(rawIP, paramUserID, userID, videoID, segments, service, isVIP, lockedCategoryList);
@@ -530,7 +508,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
530508
}
531509

532510
if (!isVIP) {
533-
const autoModerateCheckResult = await checkByAutoModerator(videoID, userID, segments, service, apiVideoInfo, videoDurationParam);
511+
const autoModerateCheckResult = await checkByAutoModerator(videoID, userID, segments, service, apiVideoDetails, videoDurationParam);
534512
if (!autoModerateCheckResult.pass) {
535513
return res.status(autoModerateCheckResult.errorCode).send(autoModerateCheckResult.errorMessage);
536514
}
@@ -583,10 +561,10 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
583561
//add to private db as well
584562
await privateDB.prepare("run", `INSERT INTO "sponsorTimes" VALUES(?, ?, ?, ?)`, [videoID, hashedIP, timeSubmitted, service]);
585563

586-
await db.prepare("run", `INSERT INTO "videoInfo" ("videoID", "channelID", "title", "published", "genreUrl")
587-
SELECT ?, ?, ?, ?, ?
564+
await db.prepare("run", `INSERT INTO "videoInfo" ("videoID", "channelID", "title", "published")
565+
SELECT ?, ?, ?, ?
588566
WHERE NOT EXISTS (SELECT 1 FROM "videoInfo" WHERE "videoID" = ?)`, [
589-
videoID, apiVideoInfo?.data?.authorId || "", apiVideoInfo?.data?.title || "", apiVideoInfo?.data?.published || 0, apiVideoInfo?.data?.genreUrl || "", videoID]);
567+
videoID, apiVideoDetails?.authorId || "", apiVideoDetails?.title || "", apiVideoDetails?.published || 0, videoID]);
590568

591569
// Clear redis cache for this video
592570
QueryCacher.clearSegmentCache({
@@ -614,7 +592,7 @@ export async function postSkipSegments(req: Request, res: Response): Promise<Res
614592
}
615593

616594
for (let i = 0; i < segments.length; i++) {
617-
sendWebhooks(apiVideoInfo, userID, videoID, UUIDs[i], segments[i], service).catch(Logger.error);
595+
sendWebhooks(apiVideoDetails, userID, videoID, UUIDs[i], segments[i], service).catch(Logger.error);
618596
}
619597
return res.json(newSegments);
620598
}

‎src/routes/voteOnSponsorTime.ts‎

Lines changed: 12 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@ import { Logger } from "../utils/logger";
33
import { isUserVIP } from "../utils/isUserVIP";
44
import { isUserTempVIP } from "../utils/isUserTempVIP";
55
import { getMaxResThumbnail, YouTubeAPI } from "../utils/youtubeApi";
6-
import { APIVideoInfo } from "../types/youtubeApi.model";
76
import { db, privateDB } from "../databases/databases";
87
import { dispatchEvent, getVoteAuthor, getVoteAuthorRaw } from "../utils/webhookUtils";
98
import { getFormattedTime } from "../utils/getFormattedTime";
@@ -14,6 +13,7 @@ import { UserID } from "../types/user.model";
1413
import { DBSegment, Category, HashedIP, IPAddress, SegmentUUID, Service, VideoID, VideoIDHash, VideoDuration, ActionType, VoteType } from "../types/segments.model";
1514
import { QueryCacher } from "../utils/queryCacher";
1615
import axios from "axios";
16+
import { getVideoDetails, videoDetails } from "../utils/getVideoDetails";
1717

1818
const voteTypes = {
1919
normal: 0,
@@ -52,20 +52,16 @@ interface VoteData {
5252
finalResponse: FinalResponse;
5353
}
5454

55-
function getYouTubeVideoInfo(videoID: VideoID, ignoreCache = false): Promise<APIVideoInfo> {
56-
return config.newLeafURLs ? YouTubeAPI.listVideos(videoID, ignoreCache) : null;
57-
}
58-
5955
const videoDurationChanged = (segmentDuration: number, APIDuration: number) => (APIDuration > 0 && Math.abs(segmentDuration - APIDuration) > 2);
6056

6157
async function updateSegmentVideoDuration(UUID: SegmentUUID) {
6258
const { videoDuration, videoID, service } = await db.prepare("get", `select "videoDuration", "videoID", "service" from "sponsorTimes" where "UUID" = ?`, [UUID]);
63-
let apiVideoInfo: APIVideoInfo = null;
59+
let apiVideoDetails: videoDetails = null;
6460
if (service == Service.YouTube) {
6561
// don't use cache since we have no information about the video length
66-
apiVideoInfo = await getYouTubeVideoInfo(videoID);
62+
apiVideoDetails = await getVideoDetails(videoID);
6763
}
68-
const apiVideoDuration = apiVideoInfo?.data?.lengthSeconds as VideoDuration;
64+
const apiVideoDuration = apiVideoDetails?.duration as VideoDuration;
6965
if (videoDurationChanged(videoDuration, apiVideoDuration)) {
7066
Logger.info(`Video duration changed for ${videoID} from ${videoDuration} to ${apiVideoDuration}`);
7167
await db.prepare("run", `UPDATE "sponsorTimes" SET "videoDuration" = ? WHERE "UUID" = ?`, [apiVideoDuration, UUID]);
@@ -74,12 +70,12 @@ async function updateSegmentVideoDuration(UUID: SegmentUUID) {
7470

7571
async function checkVideoDuration(UUID: SegmentUUID) {
7672
const { videoID, service } = await db.prepare("get", `select "videoID", "service" from "sponsorTimes" where "UUID" = ?`, [UUID]);
77-
let apiVideoInfo: APIVideoInfo = null;
73+
let apiVideoDetails: videoDetails = null;
7874
if (service == Service.YouTube) {
7975
// don't use cache since we have no information about the video length
80-
apiVideoInfo = await getYouTubeVideoInfo(videoID, true);
76+
apiVideoDetails = await getVideoDetails(videoID, true);
8177
}
82-
const apiVideoDuration = apiVideoInfo?.data?.lengthSeconds as VideoDuration;
78+
const apiVideoDuration = apiVideoDetails?.duration as VideoDuration;
8379
// if no videoDuration return early
8480
if (isNaN(apiVideoDuration)) return;
8581
// fetch latest submission
@@ -129,7 +125,8 @@ async function sendWebhooks(voteData: VoteData) {
129125
}
130126

131127
if (config.newLeafURLs !== null) {
132-
const { err, data } = await YouTubeAPI.listVideos(submissionInfoRow.videoID);
128+
const videoID = submissionInfoRow.videoID;
129+
const { err, data } = await YouTubeAPI.listVideos(videoID);
133130
if (err) return;
134131

135132
const isUpvote = voteData.incrementAmount > 0;
@@ -141,8 +138,8 @@ async function sendWebhooks(voteData: VoteData) {
141138
"video": {
142139
"id": submissionInfoRow.videoID,
143140
"title": data?.title,
144-
"url": `https://www.youtube.com/watch?v=${submissionInfoRow.videoID}`,
145-
"thumbnail": getMaxResThumbnail(data) || null,
141+
"url": `https://www.youtube.com/watch?v=${videoID}`,
142+
"thumbnail": getMaxResThumbnail(videoID),
146143
},
147144
"submission": {
148145
"UUID": voteData.UUID,
@@ -187,7 +184,7 @@ async function sendWebhooks(voteData: VoteData) {
187184
`${getVoteAuthor(userSubmissionCountRow.submissionCount, voteData.isTempVIP, voteData.isVIP, voteData.isOwnSubmission)}${voteData.row.locked ? " (Locked)" : ""}`,
188185
},
189186
"thumbnail": {
190-
"url": getMaxResThumbnail(data) || "",
187+
"url": getMaxResThumbnail(videoID),
191188
},
192189
}],
193190
})

0 commit comments

Comments
 (0)