add ETag to skipSegments byHash

Author: mchangrh

src/app.ts

@@ -50,18 +50,21 @@ import { getVideoLabelsByHash } from "./routes/getVideoLabelByHash";
 import { addFeature } from "./routes/addFeature";
 import { generateTokenRequest } from "./routes/generateToken";
 import { verifyTokenRequest } from "./routes/verifyToken";
+import { cacheMiddlware } from "./middleware/etag";
 
 export function createServer(callback: () => void): Server {
     // Create a service (the app object is just a callback).
     const app = express();
 
     const router = ExpressPromiseRouter();
     app.use(router);
+    app.set("etag", false); // disable built in etag
 
     //setup CORS correctly
     router.use(corsMiddleware);
     router.use(loggerMiddleware);
     router.use("/api/", apiCspMiddleware);
+    router.use(cacheMiddlware);
     router.use(express.json());
 
     if (config.userCounterURL) router.use(userCounter);

src/middleware/cors.ts

@@ -3,6 +3,6 @@ import { NextFunction, Request, Response } from "express";
 export function corsMiddleware(req: Request, res: Response, next: NextFunction): void {
     res.header("Access-Control-Allow-Origin", "*");
     res.header("Access-Control-Allow-Methods", "GET, POST, OPTIONS, DELETE");
-    res.header("Access-Control-Allow-Headers", "Content-Type");
+    res.header("Access-Control-Allow-Headers", "Content-Type, If-None-Match");
     next();
 }

src/middleware/etag.ts

@@ -0,0 +1,48 @@
+import { NextFunction, Request, Response } from "express";
+import { VideoID, VideoIDHash, Service } from "../types/segments.model";
+import { QueryCacher } from "../utils/queryCacher";
+import { skipSegmentsHashKey, skipSegmentsKey, videoLabelsHashKey, videoLabelsKey } from "../utils/redisKeys";
+
+type hashType = "skipSegments" | "skipSegmentsHash" | "videoLabel" | "videoLabelHash";
+type ETag = `${hashType};${VideoIDHash};${Service};${number}`;
+
+export function cacheMiddlware(req: Request, res: Response, next: NextFunction): void {
+    const reqEtag = req.get("If-None-Match") as string;
+    // if weak etag, do not handle
+    if (!reqEtag || reqEtag.startsWith("W/")) return next();
+    // split into components
+    const [hashType, hashKey, service, lastModified] = reqEtag.split(";");
+    // fetch last-modified
+    getLastModified(hashType as hashType, hashKey as VideoIDHash, service as Service)
+        .then(redisLastModified => {
+            if (redisLastModified <= new Date(Number(lastModified) + 1000)) {
+                // match cache, generate etag
+                const etag = `${hashType};${hashKey};${service};${redisLastModified.getTime()}` as ETag;
+                res.status(304).set("etag", etag).send();
+            }
+            else next();
+        })
+        .catch(next);
+}
+
+function getLastModified(hashType: hashType, hashKey: (VideoID | VideoIDHash), service: Service): Promise<Date | null> {
+    let redisKey: string | null;
+    if (hashType === "skipSegments") redisKey = skipSegmentsKey(hashKey as VideoID, service);
+    else if (hashType === "skipSegmentsHash") redisKey = skipSegmentsHashKey(hashKey as VideoIDHash, service);
+    else if (hashType === "videoLabel") redisKey = videoLabelsKey(hashKey as VideoID, service);
+    else if (hashType === "videoLabelHash") redisKey = videoLabelsHashKey(hashKey as VideoIDHash, service);
+    else return Promise.reject();
+    return QueryCacher.getKeyLastModified(redisKey);
+}
+
+export async function getEtag(hashType: hashType, hashKey: VideoIDHash, service: Service): Promise<ETag> {
+    const lastModified = await getLastModified(hashType, hashKey, service);
+    return `${hashType};${hashKey};${service};${lastModified.getTime()}` as ETag;
+}
+
+/* example usage
+import { getEtag } from "../middleware/etag";
+await getEtag(hashType, hashPrefix, service)
+    .then(etag => res.set("ETag", etag))
+    .catch(() => null);
+*/

src/routes/getSkipSegmentsByHash.ts

@@ -4,6 +4,7 @@ import { Request, Response } from "express";
 import { ActionType, Category, SegmentUUID, VideoIDHash, Service } from "../types/segments.model";
 import { getService } from "../utils/getService";
 import { Logger } from "../utils/logger";
+import { getEtag } from "../middleware/etag";
 
 export async function getSkipSegmentsByHash(req: Request, res: Response): Promise<Response> {
     let hashPrefix = req.params.prefix as VideoIDHash;
@@ -69,6 +70,9 @@ export async function getSkipSegmentsByHash(req: Request, res: Response): Promis
     const segments = await getSegmentsByHash(req, hashPrefix, categories, actionTypes, requiredSegments, service);
 
     try {
+        await getEtag("skipSegmentsHash", hashPrefix, service)
+            .then(etag => res.set("ETag", etag))
+            .catch(() => null);
         const output = Object.entries(segments).map(([videoID, data]) => ({
             videoID,
             segments: data.segments,

src/utils/queryCacher.ts

@@ -87,6 +87,17 @@ function clearSegmentCache(videoInfo: { videoID: VideoID; hashedVideoID: VideoID
     }
 }
 
+async function getKeyLastModified(key: string): Promise<Date> {
+    if (!config.redis?.enabled) return Promise.reject("ETag - Redis not enabled");
+    return await redis.ttl(key)
+        .then(ttl => {
+            const sinceLive = config.redis?.expiryTime - ttl;
+            const now = Math.floor(Date.now() / 1000);
+            return new Date((now-sinceLive) * 1000);
+        })
+        .catch(() => Promise.reject("ETag - Redis error"));
+}
+
 function clearRatingCache(videoInfo: { hashedVideoID: VideoIDHash; service: Service;}): void {
     if (videoInfo) {
         redis.del(ratingHashKey(videoInfo.hashedVideoID, videoInfo.service)).catch((err) => Logger.error(err));
@@ -101,6 +112,7 @@ export const QueryCacher = {
     get,
     getAndSplit,
     clearSegmentCache,
+    getKeyLastModified,
     clearRatingCache,
-    clearFeatureCache
+    clearFeatureCache,
 };

src/utils/redis.ts

@@ -19,6 +19,7 @@ interface RedisSB {
     del(...keys: [RedisCommandArgument]): Promise<number>;
     increment?(key: RedisCommandArgument): Promise<RedisCommandRawReply[]>;
     sendCommand(args: RedisCommandArguments, options?: RedisClientOptions): Promise<RedisReply>;
+    ttl(key: RedisCommandArgument): Promise<number>;
     quit(): Promise<void>;
 }
 
@@ -30,6 +31,7 @@ let exportClient: RedisSB = {
     increment: () => new Promise((resolve) => resolve(null)),
     sendCommand: () => new Promise((resolve) => resolve(null)),
     quit: () => new Promise((resolve) => resolve(null)),
+    ttl: () => new Promise((resolve) => resolve(null)),
 };
 
 let lastClientFail = 0;