fix(transcriptions): Only process mp4 videos. Before the video archive was ready, we were processing the in-progress livestream, which our transcription flow doesn't handle properly. Now, we should only process hearings after the video is finalized and we can process a complete transcription

Author: Mephistic

functions/src/events/helpers.test.ts

@@ -1,5 +1,5 @@
 import { addDays, subDays } from "date-fns"
-import { withinCutoff } from "./helpers"
+import { isValidVideoUrl, withinCutoff } from "./helpers"
 
 describe("withinCutoff true", () => {
   beforeEach(() => {
@@ -37,3 +37,26 @@ describe("withinCutoff true", () => {
     expect(result).toEqual(false)
   })
 })
+
+describe("isValidVideoUrl", () => {
+  it("should return true for a valid video URL", () => {
+    const validUrl = "https://example.com/video.mp4"
+    const result = isValidVideoUrl(validUrl)
+    expect(result).toEqual(true)
+  })
+
+  it("should return false for a missing URL", () => {
+    const result = isValidVideoUrl(null)
+    expect(result).toEqual(false)
+  })
+
+  it("should return false for a URL with no file format", () => {
+    const result = isValidVideoUrl("https://example.com/video")
+    expect(result).toEqual(false)
+  })
+
+  it("should return false for a URL with an unsupported format", () => {
+    const result = isValidVideoUrl("https://example.com/video.m3u8")
+    expect(result).toEqual(false)
+  })
+})

functions/src/events/helpers.ts

@@ -1,8 +1,32 @@
 import { isAfter, subDays } from "date-fns"
 
+const VIDEO_FORMAT_ALLOWLIST = ["mp4"]
+
 export const withinCutoff = (date: Date) => {
   const now = new Date()
   const cutoff = subDays(now, 8)
 
   return isAfter(date, cutoff) && !isAfter(date, now)
 }
+
+// This isn't perfect because it relies on the file extension,
+// but it's a reasonable heuristic for now and should differentiate
+// the livestreams we don't want from the archived video we do want.
+export const isValidVideoUrl = (url: string | null | undefined) => {
+  if (!url) return false
+
+  const fileFormat = url.split(".").pop()?.toLowerCase()
+  if (!fileFormat) {
+    console.log(`Could not find file format for video URL: {url}`)
+    return false
+  } else if (!VIDEO_FORMAT_ALLOWLIST.includes(fileFormat)) {
+    console.log(
+      `Url ${url} has unsupported video format: ${fileFormat}. Supported formats: ${VIDEO_FORMAT_ALLOWLIST.join(
+        ", "
+      )}`
+    )
+    return false
+  }
+
+  return true
+}

functions/src/events/scrapeEvents.ts

@@ -19,7 +19,7 @@ import {
 import { currentGeneralCourt } from "../shared"
 import { randomBytes } from "node:crypto"
 import { sha256 } from "js-sha256"
-import { withinCutoff } from "./helpers"
+import { isValidVideoUrl, withinCutoff } from "./helpers"
 import ffmpeg from "fluent-ffmpeg"
 import fs from "fs"
 abstract class EventScraper<ListItem, Event extends BaseEvent> {
@@ -238,7 +238,9 @@ const getHearingVideoUrl = async (EventId: number) => {
         dom.window.document.querySelectorAll("video source")
       if (maybeVideoSource.length && maybeVideoSource[0]) {
         const firstVideoSource = maybeVideoSource[0] as HTMLSourceElement
-        return firstVideoSource.src
+        const maybeVideoUrl = firstVideoSource.src
+
+        return isValidVideoUrl(maybeVideoUrl) ? maybeVideoUrl : null
       }
     }
   }