Move the Sched API functions to shared dir and add a sync script

hasparus · hasparus · commit 8c17d580582c · 2025-06-09T16:31:06.000+02:00
diff --git a/.github/workflows/conference-sync.yml b/.github/workflows/conference-sync.yml
@@ -0,0 +1,23 @@
+# Sched's API rate limits are very limited, so we sync non-critical part of the data on a cron.
+on:
+  schedule:
+    - cron: "*/1 * * * *" # every minute
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Sync conference data from Sched
+        run: |
+          node --experimental-transform-types scripts/sync-sched/sync.ts --year 2025
+
+      - name: Commit changes
+        uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+          file_pattern: "scripts/sync-sched/*.json"
+          commit_message: "Sync conference data from Sched"
diff --git a/scripts/sync-sched/schedule-2023.json b/scripts/sync-sched/schedule-2023.json
@@ -0,0 +1 @@
+[]
diff --git a/scripts/sync-sched/schedule-2024.json b/scripts/sync-sched/schedule-2024.json
@@ -0,0 +1 @@
+[]
diff --git a/scripts/sync-sched/schedule-2025.json b/scripts/sync-sched/schedule-2025.json
@@ -0,0 +1 @@
+[]
diff --git a/scripts/sync-sched/speakers.json b/scripts/sync-sched/speakers.json
@@ -0,0 +1 @@
+[]
diff --git a/scripts/sync-sched/sync.ts b/scripts/sync-sched/sync.ts
@@ -0,0 +1,83 @@
+#!/usr/bin/env tsx
+
+import assert from "node:assert"
+import { parseArgs } from "node:util"
+
+import { getSchedule, getSpeakers } from "@/app/conf/_api/sched-client"
+
+// Sched API rate limit is 30 requests per minute per token.
+// This scripts fires:
+// - one request for the entire schedule which overwritten
+// - one request for the list of speakers with partial details
+// - and N requests for the full details of each speaker
+const SPEAKER_DETAILS_REQUEST_QUOTA = 10
+
+const options = {
+  year: {
+    type: "string" as const,
+    short: "y",
+  },
+  help: {
+    type: "boolean" as const,
+    short: "h",
+  },
+}
+
+function main() {
+  try {
+    const { values } = parseArgs({ options })
+
+    if (values.help) {
+      help()
+      process.exit(0)
+    }
+
+    const year = parseInt(values.year || new Date().getFullYear().toString())
+
+    console.log(`Syncing schedule for year: ${year}`)
+
+    const token = process.env[`SCHED_ACCESS_TOKEN_${year}`]
+    assert(token, `SCHED_ACCESS_TOKEN_${year} is not set`)
+
+    // TODO: Implement sync logic here
+    // You can now use the `year` variable in your sync logic
+  } catch (error) {
+    if (error instanceof Error && error.message.includes("Unknown option")) {
+      console.error(`Error: ${error.message}`)
+      help()
+      process.exit(1)
+    }
+    throw error
+  }
+}
+
+if (require.main === module) {
+  main()
+}
+
+async function sync(year: number, token: string) {
+  const apiUrl = {
+    2023: "https://graphqlconf23.sched.com/api",
+    2024: "https://graphqlconf2024.sched.com/api",
+    2025: "https://graphqlconf2025.sched.com/api",
+  }[year]
+
+  assert(apiUrl, `API URL for year ${year} not found`)
+
+  const ctx = { apiUrl, token }
+
+  console.log("Getting schedule and speakers list...")
+  const [schedule, speakers] = await Promise.all([
+    getSchedule(ctx),
+    getSpeakers(ctx),
+  ])
+
+  // console.log("Getting speaker details...")
+  // const speakerDetails = await Promise.all(
+  //   speakers.map(speaker => getSpeakerDetails(ctx, speaker.username)),
+  // )
+}
+
+function help() {
+  return console.log("Usage: tsx sync.ts --year <year>")
+}
diff --git a/src/app/conf/2023/_data.ts b/src/app/conf/2023/_data.ts
@@ -1,48 +1,11 @@
 import "server-only"
-import { stripHtml } from "string-strip-html"
 
-import { SchedSpeaker, ScheduleSession } from "@/app/conf/2023/types"
-import { fetchSchedData } from "../_api/sched-client"
+import { getSchedule, getSpeakers } from "../_api/sched-client"
 
-const token = process.env.SCHED_ACCESS_TOKEN_2023
-
-async function getSpeakers(): Promise<SchedSpeaker[]> {
-  const users = await fetchSchedData<SchedSpeaker[]>(
-    `https://graphqlconf23.sched.com/api/user/list?api_key=${token}&format=json&fields=username,company,position,name,about,location,url,avatar,role,socialurls`,
-  )
-
-  const result = users
-    .filter(user => user.role.includes("speaker"))
-    .map(user => {
-      return {
-        ...user,
-        about: stripHtml(user.about).result,
-      }
-    })
-
-  return result
+const ctx = {
+  apiUrl: "https://graphqlconf23.sched.com/api",
+  token: process.env.SCHED_ACCESS_TOKEN_2023!,
 }
 
-async function getSchedule(): Promise<ScheduleSession[]> {
-  const sessions = await fetchSchedData<ScheduleSession[]>(
-    `https://graphqlconf23.sched.com/api/session/export?api_key=${token}&format=json`,
-  )
-
-  const result = sessions.map(session => {
-    const { description } = session
-    if (description?.includes("<")) {
-      // console.log(`Found HTML element in about field for session "${session.name}"`)
-    }
-
-    return {
-      ...session,
-      description: description && stripHtml(description).result,
-    }
-  })
-
-  return result
-}
-
-export const speakers = await getSpeakers()
-
-export const schedule = await getSchedule()
+export const speakers = await getSpeakers(ctx)
+export const schedule = await getSchedule(ctx)
diff --git a/src/app/conf/2023/types.d.ts b/src/app/conf/2023/types.d.ts
@@ -1,27 +1 @@
-export type ScheduleSession = {
-  id: string
-  audience: string
-  description: string
-  event_end: string
-  event_start: string
-  event_subtype: string
-  event_type: string
-  name: string
-  venue: string
-  speakers?: SchedSpeaker[]
-  files?: { name: string; path: string }[]
-}
-
-export type SchedSpeaker = {
-  username: string
-  name: string
-  about: string
-  company?: string
-  position?: string
-  avatar?: string
-  url?: string
-  role: string
-  location?: string
-  socialurls: { service: string; url: string }[]
-  year: "2025" | "2024" | "2023"
-}
+export * from "../_api/sched-types"
diff --git a/src/app/conf/2024/_data.ts b/src/app/conf/2024/_data.ts
@@ -1,48 +1,11 @@
 import "server-only"
-import { stripHtml } from "string-strip-html"
-import { SchedSpeaker, ScheduleSession } from "@/app/conf/2023/types"
 
-import { fetchSchedData } from "../_api/sched-client"
+import { getSchedule, getSpeakers } from "../_api/sched-client"
 
-const token = process.env.SCHED_ACCESS_TOKEN_2024
-
-async function getSpeakers(): Promise<SchedSpeaker[]> {
-  const users = await fetchSchedData<SchedSpeaker[]>(
-    `https://graphqlconf2024.sched.com/api/user/list?api_key=${token}&format=json&fields=username,company,position,name,about,location,url,avatar,role,socialurls`,
-  )
-
-  const result = users
-    .filter(user => user.role.includes("speaker"))
-    .map(user => {
-      return {
-        ...user,
-        about: stripHtml(user.about).result,
-      }
-    })
-
-  return result
+const ctx = {
+  apiUrl: "https://graphqlconf2024.sched.com/api",
+  token: process.env.SCHED_ACCESS_TOKEN_2024!,
 }
 
-async function getSchedule(): Promise<ScheduleSession[]> {
-  const sessions = await fetchSchedData<ScheduleSession[]>(
-    `https://graphqlconf2024.sched.com/api/session/export?api_key=${token}&format=json`,
-  )
-
-  const result = sessions.map(session => {
-    const { description } = session
-    if (description?.includes("<")) {
-      // console.log(`Found HTML element in about field for session "${session.name}"`)
-    }
-
-    return {
-      ...session,
-      description: description && stripHtml(description).result,
-    }
-  })
-
-  return result
-}
-
-export const speakers = await getSpeakers()
-
-export const schedule = await getSchedule()
+export const speakers = await getSpeakers(ctx)
+export const schedule = await getSchedule(ctx)
diff --git a/src/app/conf/2025/_data.ts b/src/app/conf/2025/_data.ts
@@ -1,8 +1,8 @@
 import "server-only"
-import { stripHtml } from "string-strip-html"
+
 import { SchedSpeaker, ScheduleSession } from "@/app/conf/2023/types"
 
-import { fetchSchedData } from "../_api/sched-client"
+import { getSpeakers, getSchedule } from "../_api/sched-client"
 import { speakers as speakers2024 } from "../2024/_data"
 import { speakers as speakers2023 } from "../2023/_data"
 
@@ -13,64 +13,16 @@ const apiUrl = USE_2025
   : "https://graphqlconf2024.sched.com/api"
 
 const token = USE_2025
-  ? process.env.SCHED_ACCESS_TOKEN_2025
-  : process.env.SCHED_ACCESS_TOKEN_2024
-
-async function getSpeakers(): Promise<SchedSpeaker[]> {
-  const users = await fetchSchedData<SchedSpeaker[]>(
-    `${apiUrl}/user/list?api_key=${token}&format=json&fields=username,company,position,name,about,location,url,avatar,role,socialurls`,
-  )
-
-  const result = users
-    .filter(speaker => speaker.role.includes("speaker"))
-    .map(user => {
-      return {
-        ...user,
-        socialurls: user.socialurls || [],
-        about: preprocessDescription(user.about),
-      }
-    })
-    .sort((a, b) => {
-      if (a.avatar && !b.avatar) return -1
-      if (!a.avatar && b.avatar) return 1
-      return 0
-    })
-
-  return result
-}
-
-async function getSchedule(): Promise<ScheduleSession[]> {
-  const sessions = await fetchSchedData<ScheduleSession[]>(
-    `${apiUrl}/session/export?api_key=${token}&format=json`,
-  )
-
-  const result = sessions.map(session => {
-    const { description } = session
-
-    return {
-      ...session,
-      description: preprocessDescription(description),
-    }
-  })
+  ? process.env.SCHED_ACCESS_TOKEN_2025!
+  : process.env.SCHED_ACCESS_TOKEN_2024!
 
-  return result
+const ctx = {
+  apiUrl,
+  token,
 }
 
-function preprocessDescription(description: string | undefined | null): string {
-  let res = description || ""
-
-  // we respect manual line breaks
-  res = res.replace(/<br\s*\/?>/g, "\n")
-
-  // respecting <li> and <a> tags doesn't make sense, because speakers don't use them consistently
-  // we'll improve how the descriptions look later down the tree in the session details page
-  return stripHtml(res).result
-}
-
-export const speakers = await getSpeakers()
-
-// TODO: Collect tags from schedule for speakers.
-export const schedule = await getSchedule()
+export const speakers = await getSpeakers(ctx)
+export const schedule = await getSchedule(ctx)
 
 type SpeakerUsername = SchedSpeaker["username"]
 
@@ -99,12 +51,3 @@ for (const { username } of speakers2023) {
     returningSpeakers.add(username)
   }
 }
-
-const longestSessionName = schedule.reduce((max, session) => {
-  if (session.name.length > max.length) {
-    return session.name
-  }
-  return max
-}, "")
-
-console.log({ longestSessionName })
diff --git a/src/app/conf/_api/sched-client.tsx b/src/app/conf/_api/sched-client.tsx
diff --git a/src/app/conf/_api/sched-types.ts b/src/app/conf/_api/sched-types.ts
