Skip to content

Commit 8c17d58

Browse files
hasparushasparus
committed
Move the Sched API functions to shared dir and add a sync script
1 parent 80dc2bf commit 8c17d58

File tree

12 files changed

+257
-187
lines changed

12 files changed

+257
-187
lines changed

‎.github/workflows/conference-sync.yml

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Sched's API rate limits are very limited, so we sync non-critical part of the data on a cron.
2+
on:
3+
schedule:
4+
- cron: "*/1 * * * *" # every minute
5+
6+
jobs:
7+
sync:
8+
runs-on: ubuntu-latest
9+
permissions:
10+
contents: write
11+
steps:
12+
- name: Checkout code
13+
uses: actions/checkout@v4
14+
15+
- name: Sync conference data from Sched
16+
run: |
17+
node --experimental-transform-types scripts/sync-sched/sync.ts --year 2025
18+
19+
- name: Commit changes
20+
uses: stefanzweifel/git-auto-commit-action@v5
21+
with:
22+
file_pattern: "scripts/sync-sched/*.json"
23+
commit_message: "Sync conference data from Sched"

‎scripts/sync-sched/schedule-2023.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[]

‎scripts/sync-sched/schedule-2024.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[]

‎scripts/sync-sched/schedule-2025.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[]

‎scripts/sync-sched/speakers.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
[]

‎scripts/sync-sched/sync.ts

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
#!/usr/bin/env tsx
2+
3+
import assert from "node:assert"
4+
import { parseArgs } from "node:util"
5+
6+
import { getSchedule, getSpeakers } from "@/app/conf/_api/sched-client"
7+
8+
// Sched API rate limit is 30 requests per minute per token.
9+
// This scripts fires:
10+
// - one request for the entire schedule which overwritten
11+
// - one request for the list of speakers with partial details
12+
// - and N requests for the full details of each speaker
13+
const SPEAKER_DETAILS_REQUEST_QUOTA = 10
14+
15+
const options = {
16+
year: {
17+
type: "string" as const,
18+
short: "y",
19+
},
20+
help: {
21+
type: "boolean" as const,
22+
short: "h",
23+
},
24+
}
25+
26+
function main() {
27+
try {
28+
const { values } = parseArgs({ options })
29+
30+
if (values.help) {
31+
help()
32+
process.exit(0)
33+
}
34+
35+
const year = parseInt(values.year || new Date().getFullYear().toString())
36+
37+
console.log(`Syncing schedule for year: ${year}`)
38+
39+
const token = process.env[`SCHED_ACCESS_TOKEN_${year}`]
40+
assert(token, `SCHED_ACCESS_TOKEN_${year} is not set`)
41+
42+
// TODO: Implement sync logic here
43+
// You can now use the `year` variable in your sync logic
44+
} catch (error) {
45+
if (error instanceof Error && error.message.includes("Unknown option")) {
46+
console.error(`Error: ${error.message}`)
47+
help()
48+
process.exit(1)
49+
}
50+
throw error
51+
}
52+
}
53+
54+
if (require.main === module) {
55+
main()
56+
}
57+
58+
async function sync(year: number, token: string) {
59+
const apiUrl = {
60+
2023: "https://graphqlconf23.sched.com/api",
61+
2024: "https://graphqlconf2024.sched.com/api",
62+
2025: "https://graphqlconf2025.sched.com/api",
63+
}[year]
64+
65+
assert(apiUrl, `API URL for year ${year} not found`)
66+
67+
const ctx = { apiUrl, token }
68+
69+
console.log("Getting schedule and speakers list...")
70+
const [schedule, speakers] = await Promise.all([
71+
getSchedule(ctx),
72+
getSpeakers(ctx),
73+
])
74+
75+
// console.log("Getting speaker details...")
76+
// const speakerDetails = await Promise.all(
77+
// speakers.map(speaker => getSpeakerDetails(ctx, speaker.username)),
78+
// )
79+
}
80+
81+
function help() {
82+
return console.log("Usage: tsx sync.ts --year <year>")
83+
}

‎src/app/conf/2023/_data.ts

Lines changed: 6 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,48 +1,11 @@
11
import "server-only"
2-
import { stripHtml } from "string-strip-html"
32

4-
import { SchedSpeaker, ScheduleSession } from "@/app/conf/2023/types"
5-
import { fetchSchedData } from "../_api/sched-client"
3+
import { getSchedule, getSpeakers } from "../_api/sched-client"
64

7-
const token = process.env.SCHED_ACCESS_TOKEN_2023
8-
9-
async function getSpeakers(): Promise<SchedSpeaker[]> {
10-
const users = await fetchSchedData<SchedSpeaker[]>(
11-
`https://graphqlconf23.sched.com/api/user/list?api_key=${token}&format=json&fields=username,company,position,name,about,location,url,avatar,role,socialurls`,
12-
)
13-
14-
const result = users
15-
.filter(user => user.role.includes("speaker"))
16-
.map(user => {
17-
return {
18-
...user,
19-
about: stripHtml(user.about).result,
20-
}
21-
})
22-
23-
return result
5+
const ctx = {
6+
apiUrl: "https://graphqlconf23.sched.com/api",
7+
token: process.env.SCHED_ACCESS_TOKEN_2023!,
248
}
259

26-
async function getSchedule(): Promise<ScheduleSession[]> {
27-
const sessions = await fetchSchedData<ScheduleSession[]>(
28-
`https://graphqlconf23.sched.com/api/session/export?api_key=${token}&format=json`,
29-
)
30-
31-
const result = sessions.map(session => {
32-
const { description } = session
33-
if (description?.includes("<")) {
34-
// console.log(`Found HTML element in about field for session "${session.name}"`)
35-
}
36-
37-
return {
38-
...session,
39-
description: description && stripHtml(description).result,
40-
}
41-
})
42-
43-
return result
44-
}
45-
46-
export const speakers = await getSpeakers()
47-
48-
export const schedule = await getSchedule()
10+
export const speakers = await getSpeakers(ctx)
11+
export const schedule = await getSchedule(ctx)

‎src/app/conf/2023/types.d.ts

Lines changed: 1 addition & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,27 +1 @@
1-
export type ScheduleSession = {
2-
id: string
3-
audience: string
4-
description: string
5-
event_end: string
6-
event_start: string
7-
event_subtype: string
8-
event_type: string
9-
name: string
10-
venue: string
11-
speakers?: SchedSpeaker[]
12-
files?: { name: string; path: string }[]
13-
}
14-
15-
export type SchedSpeaker = {
16-
username: string
17-
name: string
18-
about: string
19-
company?: string
20-
position?: string
21-
avatar?: string
22-
url?: string
23-
role: string
24-
location?: string
25-
socialurls: { service: string; url: string }[]
26-
year: "2025" | "2024" | "2023"
27-
}
1+
export * from "../_api/sched-types"

‎src/app/conf/2024/_data.ts

Lines changed: 6 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -1,48 +1,11 @@
11
import "server-only"
2-
import { stripHtml } from "string-strip-html"
3-
import { SchedSpeaker, ScheduleSession } from "@/app/conf/2023/types"
42

5-
import { fetchSchedData } from "../_api/sched-client"
3+
import { getSchedule, getSpeakers } from "../_api/sched-client"
64

7-
const token = process.env.SCHED_ACCESS_TOKEN_2024
8-
9-
async function getSpeakers(): Promise<SchedSpeaker[]> {
10-
const users = await fetchSchedData<SchedSpeaker[]>(
11-
`https://graphqlconf2024.sched.com/api/user/list?api_key=${token}&format=json&fields=username,company,position,name,about,location,url,avatar,role,socialurls`,
12-
)
13-
14-
const result = users
15-
.filter(user => user.role.includes("speaker"))
16-
.map(user => {
17-
return {
18-
...user,
19-
about: stripHtml(user.about).result,
20-
}
21-
})
22-
23-
return result
5+
const ctx = {
6+
apiUrl: "https://graphqlconf2024.sched.com/api",
7+
token: process.env.SCHED_ACCESS_TOKEN_2024!,
248
}
259

26-
async function getSchedule(): Promise<ScheduleSession[]> {
27-
const sessions = await fetchSchedData<ScheduleSession[]>(
28-
`https://graphqlconf2024.sched.com/api/session/export?api_key=${token}&format=json`,
29-
)
30-
31-
const result = sessions.map(session => {
32-
const { description } = session
33-
if (description?.includes("<")) {
34-
// console.log(`Found HTML element in about field for session "${session.name}"`)
35-
}
36-
37-
return {
38-
...session,
39-
description: description && stripHtml(description).result,
40-
}
41-
})
42-
43-
return result
44-
}
45-
46-
export const speakers = await getSpeakers()
47-
48-
export const schedule = await getSchedule()
10+
export const speakers = await getSpeakers(ctx)
11+
export const schedule = await getSchedule(ctx)

‎src/app/conf/2025/_data.ts

Lines changed: 9 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
11
import "server-only"
2-
import { stripHtml } from "string-strip-html"
2+
33
import { SchedSpeaker, ScheduleSession } from "@/app/conf/2023/types"
44

5-
import { fetchSchedData } from "../_api/sched-client"
5+
import { getSpeakers, getSchedule } from "../_api/sched-client"
66
import { speakers as speakers2024 } from "../2024/_data"
77
import { speakers as speakers2023 } from "../2023/_data"
88

@@ -13,64 +13,16 @@ const apiUrl = USE_2025
1313
: "https://graphqlconf2024.sched.com/api"
1414

1515
const token = USE_2025
16-
? process.env.SCHED_ACCESS_TOKEN_2025
17-
: process.env.SCHED_ACCESS_TOKEN_2024
18-
19-
async function getSpeakers(): Promise<SchedSpeaker[]> {
20-
const users = await fetchSchedData<SchedSpeaker[]>(
21-
`${apiUrl}/user/list?api_key=${token}&format=json&fields=username,company,position,name,about,location,url,avatar,role,socialurls`,
22-
)
23-
24-
const result = users
25-
.filter(speaker => speaker.role.includes("speaker"))
26-
.map(user => {
27-
return {
28-
...user,
29-
socialurls: user.socialurls || [],
30-
about: preprocessDescription(user.about),
31-
}
32-
})
33-
.sort((a, b) => {
34-
if (a.avatar && !b.avatar) return -1
35-
if (!a.avatar && b.avatar) return 1
36-
return 0
37-
})
38-
39-
return result
40-
}
41-
42-
async function getSchedule(): Promise<ScheduleSession[]> {
43-
const sessions = await fetchSchedData<ScheduleSession[]>(
44-
`${apiUrl}/session/export?api_key=${token}&format=json`,
45-
)
46-
47-
const result = sessions.map(session => {
48-
const { description } = session
49-
50-
return {
51-
...session,
52-
description: preprocessDescription(description),
53-
}
54-
})
16+
? process.env.SCHED_ACCESS_TOKEN_2025!
17+
: process.env.SCHED_ACCESS_TOKEN_2024!
5518

56-
return result
19+
const ctx = {
20+
apiUrl,
21+
token,
5722
}
5823

59-
function preprocessDescription(description: string | undefined | null): string {
60-
let res = description || ""
61-
62-
// we respect manual line breaks
63-
res = res.replace(/<br\s*\/?>/g, "\n")
64-
65-
// respecting <li> and <a> tags doesn't make sense, because speakers don't use them consistently
66-
// we'll improve how the descriptions look later down the tree in the session details page
67-
return stripHtml(res).result
68-
}
69-
70-
export const speakers = await getSpeakers()
71-
72-
// TODO: Collect tags from schedule for speakers.
73-
export const schedule = await getSchedule()
24+
export const speakers = await getSpeakers(ctx)
25+
export const schedule = await getSchedule(ctx)
7426

7527
type SpeakerUsername = SchedSpeaker["username"]
7628

@@ -99,12 +51,3 @@ for (const { username } of speakers2023) {
9951
returningSpeakers.add(username)
10052
}
10153
}
102-
103-
const longestSessionName = schedule.reduce((max, session) => {
104-
if (session.name.length > max.length) {
105-
return session.name
106-
}
107-
return max
108-
}, "")
109-
110-
console.log({ longestSessionName })

0 commit comments

Comments
 (0)