Add Scrape Hearing Feature to Admin Page (#2021)

* feature(admin): Add quick UI to scrape single hearing to admin page. This will make it easy to run one-offs in the future

* chore(admin): Update local Docker instance to run ffmpeg, adding dummy secret for assemblyai to avoid error

* fix(scraper): Use empty array instead of undefined as default for committeeChairs in hearing scraper - firebase does not like undefined values by default

Author: Mephistic

components/moderation/ScrapeHearing.tsx

@@ -0,0 +1,109 @@
+import { useState } from "react"
+import { Title } from "react-admin"
+import { Box, Button, TextField, Alert, CircularProgress } from "@mui/material"
+import { httpsCallable } from "firebase/functions"
+import { functions } from "components/firebase"
+
+type ScrapeHearingRequest = { eventId: number }
+type ScrapeHearingResponse = {
+  status: string
+  message: string
+  hearingId: string
+}
+
+const scrapeSingleHearing = httpsCallable<
+  ScrapeHearingRequest,
+  ScrapeHearingResponse
+>(functions, "scrapeSingleHearing")
+
+export const ScrapeHearingForm = () => {
+  const [eventId, setEventId] = useState("")
+  const [loading, setLoading] = useState(false)
+  const [result, setResult] = useState<{
+    type: "success" | "error"
+    message: string
+  } | null>(null)
+
+  const handleSubmit = async (e: React.FormEvent) => {
+    e.preventDefault()
+    setResult(null)
+
+    const parsedEventId = parseInt(eventId, 10)
+    if (isNaN(parsedEventId)) {
+      setResult({
+        type: "error",
+        message: "Please enter a valid numeric Event ID"
+      })
+      return
+    }
+
+    setLoading(true)
+    try {
+      const response = await scrapeSingleHearing({ eventId: parsedEventId })
+      setResult({
+        type: "success",
+        message: `${response.data.message} (ID: ${response.data.hearingId})`
+      })
+      setEventId("")
+    } catch (error: any) {
+      const errorMessage =
+        error?.message || error?.details?.message || "Failed to scrape hearing"
+      setResult({
+        type: "error",
+        message: errorMessage
+      })
+    } finally {
+      setLoading(false)
+    }
+  }
+
+  return (
+    <Box sx={{ padding: 2 }}>
+      <Title title="Scrape Hearing" />
+      <Box
+        component="form"
+        onSubmit={handleSubmit}
+        sx={{
+          display: "flex",
+          flexDirection: "column",
+          gap: 2,
+          maxWidth: 400
+        }}
+      >
+        <TextField
+          label="Hearing Event ID"
+          placeholder="e.g., 1234"
+          value={eventId}
+          onChange={e => setEventId(e.target.value)}
+          disabled={loading}
+          required
+          type="number"
+          helperText="Enter the EventId from the MA Legislature website"
+          fullWidth
+        />
+        <Button
+          type="submit"
+          variant="contained"
+          disabled={loading || !eventId}
+          sx={{ alignSelf: "flex-start" }}
+        >
+          {loading ? (
+            <>
+              <CircularProgress size={20} sx={{ mr: 1 }} color="inherit" />
+              Scraping...
+            </>
+          ) : (
+            "Scrape Hearing"
+          )}
+        </Button>
+        {result && (
+          <Alert severity={result.type} sx={{ mt: 2 }}>
+            {result.message}
+          </Alert>
+        )}
+      </Box>
+    </Box>
+  )
+}
+
+export const ScrapeHearingList = () => <ScrapeHearingForm />

components/moderation/index.ts

@@ -3,6 +3,7 @@ export * from "./ListPublishedTestimony"
 export * from "./ListReports"
 export * from "./EditReports"
 export * from "./ListProfiles"
+export * from "./ScrapeHearing"
 
 import dynamic from "next/dynamic"
 

components/moderation/moderation.tsx

@@ -4,6 +4,7 @@ import { FirebaseDataProvider } from "react-admin-firebase"
 import { QueryClient, QueryClientProvider } from "react-query"
 import { EditReports, ListReports } from "./"
 import { ListProfiles } from "./ListProfiles"
+import { ScrapeHearingList } from "./ScrapeHearing"
 import {
   createMyOne,
   getMyListGroup,
@@ -48,6 +49,11 @@ const App = () => {
           list={ListProfiles}
           options={{ label: "Upgrade Requests" }}
         />
+        <Resource
+          name="scrape-hearing"
+          list={ScrapeHearingList}
+          options={{ label: "Scrape Hearing" }}
+        />
       </Admin>
     </QueryClientProvider>
   )

functions/.secret.local

@@ -1 +1,2 @@
 TYPESENSE_API_KEY=test-api-key
+ASSEMBLY_API_KEY=test-api-key

functions/src/events/index.ts

@@ -1 +1,2 @@
 export * from "./scrapeEvents"
+export { scrapeSingleHearing } from "./scrapeEvents"

functions/src/events/scrapeEvents.ts

@@ -1,8 +1,9 @@
+import * as functions from "firebase-functions"
 import { RuntimeOptions, runWith } from "firebase-functions"
 import { DateTime } from "luxon"
 import { JSDOM } from "jsdom"
 import { AssemblyAI } from "assemblyai"
-import { logFetchError } from "../common"
+import { checkAuth, checkAdmin, logFetchError } from "../common"
 import { db, storage, Timestamp } from "../firebase"
 import * as api from "../malegislature"
 import {
@@ -287,7 +288,10 @@ export const getHearingVideoUrl = async (EventId: number) => {
   return null
 }
 
-const shouldScrapeVideo = async (EventId: number) => {
+const shouldScrapeVideo = async (
+  EventId: number,
+  ignoreCutoff: boolean = false
+) => {
   const eventInDb = await db
     .collection("events")
     .doc(`hearing-${String(EventId)}`)
@@ -300,7 +304,10 @@ const shouldScrapeVideo = async (EventId: number) => {
     return false
   }
   if (!eventData.videoURL) {
-    return withinCutoff(new Date(Hearing.check(eventData).startsAt.toDate()))
+    return (
+      ignoreCutoff ||
+      withinCutoff(new Date(Hearing.check(eventData).startsAt.toDate()))
+    )
   }
   return false
 }
@@ -346,7 +353,10 @@ class HearingScraper extends EventScraper<HearingListItem, Hearing> {
     return events.filter(HearingListItem.guard)
   }
 
-  async getEvent({ EventId }: HearingListItem /* e.g. 4962 */) {
+  async getEvent(
+    { EventId }: HearingListItem /* e.g. 4962 */,
+    { ignoreCutoff = false }: { ignoreCutoff?: boolean } = {}
+  ) {
     const data = await api.getHearing(EventId)
     const content = HearingContent.check(data)
 
@@ -359,9 +369,9 @@ class HearingScraper extends EventScraper<HearingListItem, Hearing> {
             host.GeneralCourtNumber,
             host.CommitteeCode
           )
-        : undefined
+        : []
 
-    if (await shouldScrapeVideo(EventId)) {
+    if (await shouldScrapeVideo(EventId, ignoreCutoff)) {
       try {
         const maybeVideoUrl = await getHearingVideoUrl(EventId)
         if (maybeVideoUrl) {
@@ -411,6 +421,61 @@ class HearingScraper extends EventScraper<HearingListItem, Hearing> {
   }
 }
 
+/**
+ * Callable cloud function to scrape a single hearing by EventId.
+ * Requires authentication to prevent abuse of API call limits.
+ *
+ * @param data - Object containing the EventId (e.g., 1234)
+ * @param context - Firebase callable context with auth information
+ */
+export const scrapeSingleHearing = functions
+  .runWith({
+    timeoutSeconds: 480,
+    secrets: ["ASSEMBLY_API_KEY"],
+    memory: "4GB"
+  })
+  .https.onCall(async (data: { eventId: number }, context) => {
+    // Require admin authentication
+    checkAuth(context, false)
+    checkAdmin(context)
+
+    const { eventId } = data
+
+    if (!eventId || typeof eventId !== "number") {
+      throw new functions.https.HttpsError(
+        "invalid-argument",
+        "The function must be called with a valid eventId (number)."
+      )
+    }
+
+    try {
+      // Create a temporary scraper instance to reuse the existing logic
+      const scraper = new HearingScraper()
+      const hearing = await scraper.getEvent(
+        { EventId: eventId },
+        { ignoreCutoff: true }
+      )
+
+      // Save the hearing to Firestore
+      await db.doc(`/events/${hearing.id}`).set(hearing, { merge: true })
+
+      console.log(`Successfully scraped hearing ${eventId}`, hearing)
+
+      return {
+        status: "success",
+        message: `Successfully scraped hearing ${eventId}`,
+        hearingId: hearing.id
+      }
+    } catch (error: any) {
+      console.error(`Failed to scrape hearing ${eventId}:`, error)
+      throw new functions.https.HttpsError(
+        "internal",
+        `Failed to scrape hearing ${eventId}`,
+        { details: error.message }
+      )
+    }
+  })
+
 export const scrapeSpecialEvents = new SpecialEventsScraper().function
 export const scrapeSessions = new SessionScraper().function
 export const scrapeHearings = new HearingScraper().function

functions/src/index.ts

@@ -15,7 +15,12 @@ export {
   startCommitteeBatches,
   updateCommitteeRosters
 } from "./committees"
-export { scrapeHearings, scrapeSessions, scrapeSpecialEvents } from "./events"
+export {
+  scrapeHearings,
+  scrapeSessions,
+  scrapeSpecialEvents,
+  scrapeSingleHearing
+} from "./events"
 export {
   syncHearingToSearchIndex,
   upgradeHearingSearchIndex

infra/Dockerfile.firebase

@@ -1,7 +1,7 @@
 FROM andreysenov/firebase-tools:latest-node-20
 
 USER root
-RUN apt update && apt install -y curl python3 python3-pip python3-venv
+RUN apt update && apt install -y curl python3 python3-pip python3-venv ffmpeg
 
 WORKDIR /app
 RUN chown -R node:node .