Refactor loadYouTubeTranscript() to include YouTube Video Metadata in Content When parseOnly is true (#4552)

* Enhance YouTube transcript loading to include video metadata in parsed content when parseOnly is true

* extract to function

---------

Co-authored-by: timothycarambat <[email protected]>

Author: angelplusultra

collector/utils/extensions/YoutubeTranscript/index.js

@@ -14,7 +14,7 @@ const { validYoutubeVideoUrl } = require("../../url");
 /**
  * Fetch the transcript content for a YouTube video
  * @param {string} url - The URL of the YouTube video
- * @returns {Promise<{success: boolean, reason: string|null, content: string|null, metadata: Object}>} - The transcript content for the YouTube video
+ * @returns {Promise<{success: boolean, reason: string|null, content: string|null, metadata: TranscriptMetadata}>} - The transcript content for the YouTube video
  */
 async function fetchVideoTranscriptContent({ url }) {
   if (!validYoutubeVideoUrl(url)) {
@@ -64,11 +64,20 @@ async function fetchVideoTranscriptContent({ url }) {
   };
 }
 
+/**
+ * @typedef {Object} TranscriptMetadata
+ * @property {string} title - The title of the video
+ * @property {string} author - The author of the video
+ * @property {string} description - The description of the video
+ * @property {string} view_count - The view count of the video
+ * @property {string} source - The source of the video (videoId)
+ */
+
 /**
  * @typedef {Object} TranscriptAsDocument
  * @property {boolean} success - Whether the transcript was successful
  * @property {string|null} reason - The reason for the transcript
- * @property {{title: string, author: string, destination: string}} data - The data from the transcript
+ * @property {TranscriptMetadata} metadata - The metadata from the transcript
  */
 
 /**
@@ -104,11 +113,12 @@ async function loadYouTubeTranscript({ url }, options = { parseOnly: false }) {
   }
 
   const { content, metadata } = transcriptResults;
+
   if (options.parseOnly) {
     return {
       success: true,
       reason: null,
-      content,
+      content: buildTranscriptContentWithMetadata(content, metadata),
       documents: [],
       saveAsDocument: options.parseOnly,
       data: {},
@@ -154,6 +164,33 @@ async function loadYouTubeTranscript({ url }, options = { parseOnly: false }) {
   };
 }
 
+/**
+ * Generate the transcript content and metadata into a single string
+ *
+ * Why? For ephemeral documents where we just want the content, we want to include the metadata as keys in the content
+ * so that the LLM has context about the video, this gives it a better understanding of the video
+ * and allows it to use the metadata in the conversation if relevant.
+ * Examples:
+ * - How many views does <LINK> have?
+ * - Checkout <LINK> and tell me the key points and if it is performing well
+ * - Summarize this video <LINK>? -> description could have links and references
+ * @param {string} content - The content of the transcript
+ * @param {TranscriptMetadata} metadata - The metadata from the transcript
+ * @returns {string} - The concatenated transcript content and metadata
+ */
+function buildTranscriptContentWithMetadata(content = "", metadata = {}) {
+  const VALID_METADATA_KEYS = ["title", "author", "description", "view_count"];
+  if (!content || !metadata || Object.keys(metadata).length === 0)
+    return content;
+
+  let contentWithMetadata = "";
+  VALID_METADATA_KEYS.forEach((key) => {
+    if (!metadata[key]) return;
+    contentWithMetadata += `<${key}>${metadata[key]}</${key}>`;
+  });
+  return `${contentWithMetadata}\nTranscript:\n${content}`;
+}
+
 module.exports = {
   loadYouTubeTranscript,
   fetchVideoTranscriptContent,