Merge pull request #1813 from boazsender/automated-transcriptions

send only audio to assembly

Author: Mephistic

firebase.json

@@ -7,7 +7,8 @@
   "functions": {
     "predeploy": ["yarn build:functions"],
     "source": "functions",
-    "runtime": "nodejs18"
+    "runtime": "nodejs18",
+    "runtimeConfig": ".runtimeconfig.json"
   },
   "firestore": {
     "rules": "firestore.rules",

functions/.runtimeconfig.json

@@ -0,0 +1,9 @@
+{
+  "runtime": {
+    "nodejs18": {
+      "apt": {
+        "packages": ["ffmpeg"]
+      }
+    }
+  }
+}

functions/package.json

@@ -20,6 +20,7 @@
     "date-fns": "^2.30.0",
     "firebase-admin": "^10",
     "firebase-functions": "^3.22.0",
+    "fluent-ffmpeg": "^2.1.3",
     "fuse.js": "6.5.3",
     "handlebars": "^4.7.8",
     "js-sha256": "^0.11.0",
@@ -34,6 +35,7 @@
     "zod": "^3.20.2"
   },
   "devDependencies": {
+    "@types/fluent-ffmpeg": "^2.1.27",
     "@types/jest": "^27.4.0",
     "@types/jsdom": "^21.1.7",
     "@types/luxon": "^2.0.9",

functions/src/events/scrapeEvents.ts

@@ -3,7 +3,7 @@ import { DateTime } from "luxon"
 import { JSDOM } from "jsdom"
 import { AssemblyAI } from "assemblyai"
 import { logFetchError } from "../common"
-import { db, Timestamp } from "../firebase"
+import { admin, db, Timestamp } from "../firebase"
 import * as api from "../malegislature"
 import {
   BaseEvent,
@@ -20,7 +20,8 @@ import { currentGeneralCourt } from "../shared"
 import { randomBytes } from "node:crypto"
 import { sha256 } from "js-sha256"
 import { withinCutoff } from "./helpers"
-
+import ffmpeg from "fluent-ffmpeg"
+import fs from "fs"
 abstract class EventScraper<ListItem, Event extends BaseEvent> {
   private schedule
   private timeout
@@ -33,7 +34,8 @@ abstract class EventScraper<ListItem, Event extends BaseEvent> {
   get function() {
     return runWith({
       timeoutSeconds: this.timeout,
-      secrets: ["ASSEMBLY_API_KEY"]
+      secrets: ["ASSEMBLY_API_KEY"],
+      memory: "2GB"
     })
       .pubsub.schedule(this.schedule)
       .onRun(() => this.run())
@@ -94,7 +96,7 @@ class SpecialEventsScraper extends EventScraper<
   SpecialEvent
 > {
   constructor() {
-    super("every 60 minutes", 120)
+    super("every 60 minutes", 540)
   }
 
   async listEvents() {
@@ -136,6 +138,53 @@ class SessionScraper extends EventScraper<SessionContent, Session> {
   }
 }
 
+const extractAudioFromVideo = async (
+  EventId: number,
+  videoUrl: string
+): Promise<string> => {
+  const tmpFilePath = `/tmp/hearing-${EventId}-${Date.now()}.wav`
+
+  // Stream directly from URL to MP3
+  await new Promise<void>((resolve, reject) => {
+    ffmpeg(videoUrl)
+      .noVideo()
+      .audioCodec("copy")
+      .format("wav")
+      .on("end", () => resolve())
+      .on("error", reject)
+      .save(tmpFilePath)
+  })
+
+  // Upload the audio file
+  const bucket = admin.storage().bucket()
+  const audioFileName = `hearing-${EventId}-${Date.now()}.wav`
+  const file = bucket.file(audioFileName)
+  await file.save(tmpFilePath)
+
+  // Clean up temporary file
+  await fs.promises.unlink(tmpFilePath)
+
+  const [url] = await file.getSignedUrl({
+    action: "read",
+    expires: Date.now() + 24 * 60 * 60 * 1000
+  })
+
+  // Delete old files
+  const [files] = await bucket.getFiles({
+    prefix: "hearing-",
+    maxResults: 1000
+  })
+  const oneDayAgo = Date.now() - 24 * 60 * 60 * 1000
+  const oldFiles = files.filter(file => {
+    const timestamp = parseInt(file.name.split("-").pop()?.split(".")[0] || "0")
+    return timestamp < oneDayAgo
+  })
+  await Promise.all(oldFiles.map(file => file.delete()))
+
+  // Return the new audio url
+  return url
+}
+
 const submitTranscription = async ({
   EventId,
   maybeVideoUrl
@@ -148,11 +197,12 @@ const submitTranscription = async ({
   })
 
   const newToken = randomBytes(16).toString("hex")
+  const audioUrl = await extractAudioFromVideo(EventId, maybeVideoUrl)
 
   const transcript = await assembly.transcripts.submit({
     audio:
       // test with: "https://assemblyaiusercontent.com/playground/aKUqpEtmYmI.flac",
-      maybeVideoUrl,
+      audioUrl,
     webhook_url:
       // make sure process.env.FUNCTIONS_API_BASE equals
       // https://us-central1-digital-testimony-prod.cloudfunctions.net
@@ -226,25 +276,34 @@ class HearingScraper extends EventScraper<HearingListItem, Hearing> {
     const content = HearingContent.check(data)
 
     if (await shouldScrapeVideo(EventId)) {
-      const maybeVideoUrl = await getHearingVideoUrl(EventId)
-      if (maybeVideoUrl) {
-        const transcriptId = await submitTranscription({
-          maybeVideoUrl,
-          EventId
-        })
-
+      try {
+        const maybeVideoUrl = await getHearingVideoUrl(EventId)
+        if (maybeVideoUrl) {
+          const transcriptId = await submitTranscription({
+            maybeVideoUrl,
+            EventId
+          })
+
+          return {
+            id: `hearing-${EventId}`,
+            type: "hearing",
+            content,
+            ...this.timestamps(content),
+            videoURL: maybeVideoUrl,
+            videoFetchedAt: Timestamp.now(),
+            videoTranscriptionId: transcriptId // using the assembly Id as our transcriptionId
+          } as Hearing
+        }
+      } catch (error) {
+        console.error(`Failed to process audio for hearing ${EventId}:`, error)
         return {
           id: `hearing-${EventId}`,
           type: "hearing",
           content,
-          ...this.timestamps(content),
-          videoURL: maybeVideoUrl,
-          videoFetchedAt: Timestamp.now(),
-          videoTranscriptionId: transcriptId // using the assembly Id as our transcriptionId
+          ...this.timestamps(content)
         } as Hearing
       }
     }
-
     return {
       id: `hearing-${EventId}`,
       type: "hearing",