Merge pull request #1747 from boazsender/automated-transcriptions

WIP automated transcriptions

Author: Mephistic

functions/package.json

@@ -15,12 +15,15 @@
   "dependencies": {
     "@google-cloud/firestore": "^5.0.2",
     "@google-cloud/pubsub": "^3.0.1",
+    "assemblyai": "^4.9.0",
     "axios": "^0.25.0",
     "date-fns": "^2.30.0",
     "firebase-admin": "^10",
     "firebase-functions": "^3.22.0",
     "fuse.js": "6.5.3",
     "handlebars": "^4.7.8",
+    "js-sha256": "^0.11.0",
+    "jsdom": "^26.0.0",
     "lodash": "^4.17.21",
     "luxon": "^2.3.1",
     "nanoid": "^3.3.2",
@@ -32,6 +35,7 @@
   },
   "devDependencies": {
     "@types/jest": "^27.4.0",
+    "@types/jsdom": "^21.1.7",
     "@types/luxon": "^2.0.9",
     "@types/object-hash": "^2.2.1",
     "copyfiles": "^2.4.1",

functions/src/events/helpers.test.ts

@@ -0,0 +1,39 @@
+import { addDays, subDays } from "date-fns"
+import { withinCutoff } from "./helpers"
+
+describe("withinCutoff true", () => {
+  beforeEach(() => {
+    jest.useFakeTimers()
+  })
+
+  afterEach(() => {
+    jest.useRealTimers()
+  })
+
+  it("should return true for a date within three days", () => {
+    const now = new Date()
+
+    const threeDaysAgo = subDays(now, 3)
+
+    const result = withinCutoff(threeDaysAgo)
+    expect(result).toEqual(true)
+  })
+
+  it("should return false for a date that is 9 days ago", () => {
+    const now = new Date()
+
+    const threeDaysAgo = subDays(now, 9)
+
+    const result = withinCutoff(threeDaysAgo)
+    expect(result).toEqual(false)
+  })
+
+  it("should return false for a date that is 2 days in the future", () => {
+    const now = new Date()
+
+    const threeDaysAgo = addDays(now, 2)
+
+    const result = withinCutoff(threeDaysAgo)
+    expect(result).toEqual(false)
+  })
+})

functions/src/events/helpers.ts

@@ -0,0 +1,8 @@
+import { isAfter, subDays } from "date-fns"
+
+export const withinCutoff = (date: Date) => {
+  const now = new Date()
+  const cutoff = subDays(now, 8)
+
+  return isAfter(date, cutoff) && !isAfter(date, now)
+}

functions/src/events/scrapeEvents.ts

@@ -1,5 +1,7 @@
 import { runWith } from "firebase-functions"
 import { DateTime } from "luxon"
+import { JSDOM } from "jsdom"
+import { AssemblyAI } from "assemblyai"
 import { logFetchError } from "../common"
 import { db, Timestamp } from "../firebase"
 import * as api from "../malegislature"
@@ -15,6 +17,13 @@ import {
   SpecialEventContent
 } from "./types"
 import { currentGeneralCourt } from "../shared"
+import { randomBytes } from "node:crypto"
+import { sha256 } from "js-sha256"
+import { withinCutoff } from "./helpers"
+
+const assembly = new AssemblyAI({
+  apiKey: process.env.ASSEMBLY_API_KEY ? process.env.ASSEMBLY_API_KEY : ""
+})
 
 abstract class EventScraper<ListItem, Event extends BaseEvent> {
   private schedule
@@ -40,7 +49,7 @@ abstract class EventScraper<ListItem, Event extends BaseEvent> {
     if (!list) return
 
     const writer = db.bulkWriter()
-    const upcomingOrRecentCutoff = DateTime.now().minus({ days: 1 })
+    const upcomingOrRecentCutoff = DateTime.now().minus({ days: 8 })
 
     for (let item of list) {
       const id = (item as any)?.EventId,
@@ -138,12 +147,73 @@ class HearingScraper extends EventScraper<HearingListItem, Hearing> {
     return events.filter(HearingListItem.guard)
   }
 
-  async getEvent({ EventId }: HearingListItem) {
-    const content = HearingContent.check(await api.getHearing(EventId))
+  async getEvent({ EventId }: HearingListItem /* e.g. 4962 */) {
+    const data = await api.getHearing(EventId)
+    const content = HearingContent.check(data)
+    const eventInDb = await db
+      .collection("events")
+      .doc(`hearing-${String(EventId)}`)
+      .get()
+    const eventData = eventInDb.data()
+    const hearing = Hearing.check(eventData)
+    const shouldScrape = withinCutoff(hearing.startsAt.toDate())
+
+    let maybeVideoURL = null
+    let transcript = null
+    if (!hearing.videoFetchedAt && shouldScrape) {
+      const req = await fetch(
+        `https://malegislature.gov/Events/Hearings/Detail/${EventId}`
+      )
+      const res = await req.text()
+      if (res) {
+        const dom = new JSDOM(res)
+        if (dom) {
+          const maybeVideoSource =
+            dom.window.document.querySelectorAll("video source")
+          if (maybeVideoSource.length && maybeVideoSource[0]) {
+            const newToken = randomBytes(16).toString("hex")
+            const firstVideoSource = maybeVideoSource[0] as HTMLSourceElement
+            maybeVideoURL = firstVideoSource.src
+
+            transcript = await assembly.transcripts.submit({
+              webhook_url:
+                process.env.NODE_ENV === "development"
+                  ? "https://us-central1-digital-testimony-dev.cloudfunctions.net/transcription"
+                  : "https://us-central1-digital-testimony-prod.cloudfunctions.net/transcription",
+              webhook_auth_header_name: "X-Maple-Webhook",
+              webhook_auth_header_value: newToken,
+              audio: firstVideoSource.src,
+              auto_highlights: true,
+              custom_topics: true,
+              entity_detection: true,
+              iab_categories: false,
+              format_text: true,
+              punctuate: true,
+              speaker_labels: true,
+              summarization: true,
+              summary_model: "informative",
+              summary_type: "bullets"
+            })
+
+            await db
+              .collection("events")
+              .doc(`hearing-${String(EventId)}`)
+              .collection("private")
+              .doc("webhookAuth")
+              .set({
+                videoAssemblyWebhookToken: sha256(newToken)
+              })
+          }
+        }
+      }
+    }
     const event: Hearing = {
-      id: `hearing-${content.EventId}`,
+      id: `hearing-${EventId}`,
       type: "hearing",
       content,
+      videoURL: maybeVideoURL,
+      videoFetchedAt: maybeVideoURL ? Timestamp.now() : null,
+      videoAssemblyId: transcript ? transcript.id : null,
       ...this.timestamps(content)
     }
     return event

functions/src/events/types.ts

@@ -76,7 +76,10 @@ export const HearingListItem = Record({ EventId: Number })
 export type Hearing = Static<typeof Hearing>
 export const Hearing = BaseEvent.extend({
   type: L("hearing"),
-  content: HearingContent
+  content: HearingContent,
+  videoURL: Nullable(String),
+  videoAssemblyId: Nullable(String),
+  videoFetchedAt: Nullable(InstanceOf(Timestamp))
 })
 
 export type Event = Static<typeof Event>

functions/src/index.ts

@@ -48,6 +48,8 @@ export {
   unfollowUser
 } from "./subscriptions"
 
+export { transcription } from "./webhooks"
+
 export * from "./triggerPubsubFunction"
 
 // Export the health check last so it is loaded last.

functions/src/webhooks/index.ts

@@ -0,0 +1 @@
+export * from "./transcription"

functions/src/webhooks/transcription.ts

@@ -0,0 +1,66 @@
+import * as functions from "firebase-functions"
+import { AssemblyAI } from "assemblyai"
+import { db } from "../firebase"
+import { sha256 } from "js-sha256"
+
+const assembly = new AssemblyAI({
+  apiKey: process.env.ASSEMBLY_API_KEY ? process.env.ASSEMBLY_API_KEY : ""
+})
+
+export const transcription = functions.https.onRequest(async (req, res) => {
+  if (
+    req.headers["X-Maple-Webhook"] &&
+    req.headers["webhook_auth_header_value"]
+  ) {
+    if (req.body.status === "completed") {
+      const transcript = await assembly.transcripts.get(req.body.transcript_id)
+      if (transcript && transcript.webhook_auth) {
+        const maybeEventInDb = await db
+          .collection("events")
+          .where("videoAssemblyId", "==", transcript.id)
+          .get()
+        if (maybeEventInDb.docs.length) {
+          const authenticatedEventsInDb = maybeEventInDb.docs.filter(
+            async e => {
+              const hashedToken = sha256(
+                String(req.headers["webhook_auth_header_value"])
+              )
+
+              const tokenInDb = await db
+                .collection("events")
+                .doc(e.id)
+                .collection("private")
+                .doc("webhookAuth")
+                .get()
+              const tokenInDbData = tokenInDb.data()
+              if (tokenInDbData) {
+                return hashedToken === tokenInDbData.videoAssemblyWebhookToken
+              }
+              return false
+            }
+          )
+          if (authenticatedEventsInDb) {
+            try {
+              await db
+                .collection("transcriptions")
+                .doc(transcript.id)
+                .set({ _timestamp: new Date(), ...transcript })
+
+              authenticatedEventsInDb.forEach(async d => {
+                await d.ref.update({
+                  ["webhook_auth_header_value"]: null
+                })
+              })
+              console.log("transcript saved in db")
+            } catch (error) {
+              console.log(error)
+            }
+          }
+        } else {
+          res.status(404).send("Not Found")
+        }
+      }
+    }
+  }
+  res.status(200).send()
+})