Refactor YouTube playlist configuration to use only playlist ID. Update environment variables and documentation accordingly.

Author: kentcdodds

.env.example

@@ -124,8 +124,7 @@ R2_SECRET_ACCESS_KEY=MOCK_R2_SECRET_ACCESS_KEY
 SEMANTIC_SEARCH_IGNORE_LIST_KEY=manifests/ignore-list.json
 
 # Feature: YouTube playlist semantic indexing + /youtube route
-# Optional: set either playlist URL or playlist ID.
-YOUTUBE_PLAYLIST_URL=https://www.youtube.com/watch?v=wSEUlS8WcQs&list=PLV5CVI1eNcJgNqzNwcs4UKrlJdhfDjshf
+# Optional
 YOUTUBE_PLAYLIST_ID=PLV5CVI1eNcJgNqzNwcs4UKrlJdhfDjshf
 # Optional: helps bypass YouTube anti-bot checks for captions in some envs.
 YOUTUBE_COOKIE=MOCK_YOUTUBE_COOKIE

.github/workflows/index-semantic-youtube.yml

@@ -47,9 +47,8 @@ jobs:
             ${{ secrets.CLOUDFLARE_AI_EMBEDDING_MODEL }}
           R2_BUCKET: ${{ secrets.R2_BUCKET }}
 
-          # Prefer setting one of these as repository variables.
-          # If neither is set, the script uses a built-in default playlist ID.
-          YOUTUBE_PLAYLIST_URL: ${{ vars.YOUTUBE_PLAYLIST_URL }}
+          # Prefer setting this as a repository variable.
+          # If it is not set, the script uses a built-in default playlist ID.
           YOUTUBE_PLAYLIST_ID: ${{ vars.YOUTUBE_PLAYLIST_ID }}
 
           # Optional: force-index specific videos. Can be set either as a

app/routes/youtube.tsx

@@ -48,9 +48,8 @@ function parsePlaylistId(value: string | undefined) {
 
 export async function loader() {
 	const env = getEnv()
-	const playlistInput = env.YOUTUBE_PLAYLIST_URL ?? env.YOUTUBE_PLAYLIST_ID
 	const configuredPlaylistId =
-		parsePlaylistId(playlistInput) ?? DEFAULT_PLAYLIST_ID
+		parsePlaylistId(env.YOUTUBE_PLAYLIST_ID) ?? DEFAULT_PLAYLIST_ID
 	return json({
 		playlistId: configuredPlaylistId,
 	})

app/utils/env.server.ts

@@ -97,10 +97,7 @@ const schemaBase = z.object({
 		GITHUB_REF: z.string().trim().optional().default('main'),
 
 		// Optional: /youtube route + indexing scripts.
-		YOUTUBE_PLAYLIST_URL: z.string().trim().optional(),
 		YOUTUBE_PLAYLIST_ID: z.string().trim().optional(),
-		YOUTUBE_COOKIE: z.string().trim().optional(),
-		YOUTUBE_USER_AGENT: z.string().trim().optional(),
 	})
 
 const schema = schemaBase.superRefine((values, ctx) => {
@@ -217,10 +214,21 @@ export function init() {
 				'❌ Invalid environment variables:',
 				z.flattenError(error).fieldErrors,
 			)
-		} else {
-			console.error('❌ Unexpected error while validating environment:', error)
+			throw new Error('Invalid environment variables')
 		}
-		throw new Error('Invalid environment variables')
+
+		// Preserve non-Zod failures from `getEnv()` so callers see the real cause.
+		if (error instanceof Error) {
+			console.error(
+				'❌ Unexpected error while validating environment:',
+				error.message,
+			)
+			if (error.stack) console.error(error.stack)
+			throw error
+		}
+
+		console.error('❌ Unexpected error while validating environment:', error)
+		throw new Error(`Unexpected error while validating environment: ${String(error)}`)
 	}
 	// Keep unused warning quiet (and make debugging easier if needed).
 	void parsedEnv

other/semantic-search/index-youtube-playlist.ts

@@ -830,7 +830,8 @@ async function fetchTranscriptFromTrack(track: CaptionTrack, label: string) {
 				? event.tStartMs
 				: 0
 		const durationMs =
-			typeof event.dDurationMs === 'number' && Number.isFinite(event.dDurationMs)
+			typeof event.dDurationMs === 'number' &&
+			Number.isFinite(event.dDurationMs)
 				? event.dDurationMs
 				: 0
 		const line = (event.segs ?? [])
@@ -904,10 +905,7 @@ function chunkTranscriptEvents(
 		}
 
 		if (startMs === null) startMs = Math.max(0, Math.floor(e.startMs))
-		const eventEnd = Math.max(
-			0,
-			Math.floor(e.startMs + (e.durationMs || 0)),
-		)
+		const eventEnd = Math.max(0, Math.floor(e.startMs + (e.durationMs || 0)))
 		endMs = Math.max(endMs, eventEnd)
 		currentLines.push(line)
 		currentLen = currentLen + (currentLines.length > 1 ? 1 : 0) + line.length
@@ -1198,10 +1196,7 @@ async function main() {
 		dryRun,
 	} = parseArgs()
 	const playlistInput =
-		playlistArg ??
-		process.env.YOUTUBE_PLAYLIST_URL ??
-		process.env.YOUTUBE_PLAYLIST_ID ??
-		DEFAULT_PLAYLIST_ID
+		playlistArg ?? process.env.YOUTUBE_PLAYLIST_ID ?? DEFAULT_PLAYLIST_ID
 	const playlistId = getPlaylistId(playlistInput)
 	if (!playlistId) {
 		throw new Error(
@@ -1275,8 +1270,9 @@ async function main() {
 	const appearancesSourceCount = videos.filter((video) =>
 		video.sources.includes('appearances'),
 	).length
-	const talksSourceCount = videos.filter((video) => video.sources.includes('talks'))
-		.length
+	const talksSourceCount = videos.filter((video) =>
+		video.sources.includes('talks'),
+	).length
 	console.log(
 		`Total unique videos to process: ${videos.length} (playlist: ${playlistSourceCount}, appearances: ${appearancesSourceCount}, talks: ${talksSourceCount})${maxVideos ? `, capped by --max-videos=${maxVideos}` : ''}`,
 	)
@@ -1515,12 +1511,14 @@ async function main() {
 			? chunkTranscriptEvents(details.transcriptEvents, {
 					targetChars: 3500,
 					maxChunkChars: 5500,
-				}).map((c): YoutubeChunkItem => ({
-					kind: 'transcript',
-					body: c.body,
-					startSeconds: Math.floor(c.startMs / 1000),
-					endSeconds: Math.ceil(c.endMs / 1000),
-				}))
+				}).map(
+					(c): YoutubeChunkItem => ({
+						kind: 'transcript',
+						body: c.body,
+						startSeconds: Math.floor(c.startMs / 1000),
+						endSeconds: Math.ceil(c.endMs / 1000),
+					}),
+				)
 			: transcript
 				? chunkText(transcript, {
 						targetChars: 3500,
@@ -1550,13 +1548,16 @@ async function main() {
 				chunkIndex: index,
 				chunkCount,
 				kind: item.kind,
-				startSeconds: item.kind === 'transcript' ? item.startSeconds : undefined,
+				startSeconds:
+					item.kind === 'transcript' ? item.startSeconds : undefined,
 				endSeconds: item.kind === 'transcript' ? item.endSeconds : undefined,
 			})
 			if (oldChunksById.get(vectorId)?.hash === hash) continue
 
-			const startSeconds = item.kind === 'transcript' ? item.startSeconds : undefined
-			const endSeconds = item.kind === 'transcript' ? item.endSeconds : undefined
+			const startSeconds =
+				item.kind === 'transcript' ? item.startSeconds : undefined
+			const endSeconds =
+				item.kind === 'transcript' ? item.endSeconds : undefined
 
 			toUpsert.push({
 				vectorId,

other/semantic-search/readme.md

@@ -35,7 +35,6 @@ For podcast indexing:
 
 For YouTube playlist indexing (optional but recommended as repo variables):
 
-- `YOUTUBE_PLAYLIST_URL` (full URL with `list=...`) **or**
 - `YOUTUBE_PLAYLIST_ID` (playlist ID only)
 - Optional (helps when YouTube returns anti-bot `LOGIN_REQUIRED`):
   - `YOUTUBE_COOKIE` (cookie header value from a logged-in browser session)
@@ -89,8 +88,8 @@ Script:
 
 Optional flags:
 
-- `--playlist "<url-or-id>"` (defaults to `YOUTUBE_PLAYLIST_URL`,
-  `YOUTUBE_PLAYLIST_ID`, or a built-in default playlist ID)
+- `--playlist "<url-or-id>"` (defaults to `YOUTUBE_PLAYLIST_ID` or a built-in
+  default playlist ID)
 - `--max-videos 50` (helpful for staged/backfill runs)
 - `--include-auto-captions false` (manual captions only)
 - `--manifest-key manifests/youtube-my-playlist.json`