Merge pull request #485 from sudoleg/copilot/fix-video-indexing-issue

Fix UNIQUE constraint violation when indexing videos already saved to library

Author: sudoleg

.dockerignore

@@ -24,3 +24,4 @@ docker-compose.yml
 Dockerfile
 .github/
 scripts/
+.env

modules/persistance.py

@@ -1,4 +1,5 @@
 import logging
+from datetime import datetime
 from typing import Literal
 
 from peewee import (
@@ -62,6 +63,40 @@ class Transcript(BaseModel):
     chroma_collection_name = CharField(null=True)
 
 
+def get_or_create_video(
+    yt_video_id: str, link: str, title: str, channel: str, saved_on: datetime
+):
+    """Gets an existing video or creates a new one if it doesn't exist.
+
+    Args:
+        yt_video_id (str): The YouTube video ID.
+        link (str): The URL of the video.
+        title (str): The title of the video.
+        channel (str): The channel name.
+        saved_on: The timestamp when the video was saved.
+
+    Returns:
+        tuple: A tuple containing (Video instance, created boolean).
+            created is True if a new video was created, False if it already existed.
+    """
+    try:
+        video = Video.get(Video.yt_video_id == yt_video_id)
+        logging.info(
+            "Video with yt_video_id '%s' already exists in database.", yt_video_id
+        )
+        return video, False
+    except Video.DoesNotExist:
+        video = Video.create(
+            yt_video_id=yt_video_id,
+            link=link,
+            title=title,
+            channel=channel,
+            saved_on=saved_on,
+        )
+        logging.info("Created new video entry for yt_video_id '%s'.", yt_video_id)
+        return video, True
+
+
 def delete_video(
     video_title: str,
 ):

pages/chat.py

@@ -28,6 +28,7 @@
     Transcript,
     Video,
     delete_video,
+    get_or_create_video,
     save_library_entry,
 )
 from modules.rag import (
@@ -184,6 +185,7 @@ def save_response_to_lib():
     # --- initialize models ---
     if provider_is_openai:
         chat_model = ChatOpenAI(
+            name=st.session_state.model,
             api_key=st.session_state.openai_api_key,
             temperature=st.session_state.temperature,
             model=st.session_state.model,
@@ -196,8 +198,8 @@ def save_response_to_lib():
         )
     else:
         chat_model = ChatOllama(
-            model=st.session_state.model,
             name=st.session_state.model,
+            model=st.session_state.model,
             temperature=st.session_state.temperature,
             top_p=st.session_state.top_p,
         )
@@ -303,7 +305,7 @@ def save_response_to_lib():
                     )
 
                     # 1. save video in the database
-                    saved_video = Video.create(
+                    saved_video, created = get_or_create_video(
                         yt_video_id=extract_youtube_video_id(url_input),
                         link=url_input,
                         title=video_metadata["name"],
@@ -314,15 +316,15 @@ def save_response_to_lib():
                     original_transcript = fetch_youtube_transcript(url_input)
 
                     # 3. save transcript, or more precisely, information about it, in the database
-                    model_for_count = (
-                        chat_model.model_name
-                        if provider_is_openai
-                        else st.session_state.model
-                    )
                     saved_transcript = Transcript.create(
                         video=saved_video,
                         original_token_num=num_tokens_from_string(
-                            string=original_transcript, model=model_for_count
+                            string=original_transcript,
+                            model=(
+                                chat_model.model_name
+                                if provider_is_openai
+                                else st.session_state.model
+                            ),
                         ),
                     )
 

pages/summary.py

@@ -12,7 +12,13 @@
     is_api_key_valid,
     is_ollama_available,
 )
-from modules.persistance import SQL_DB, LibraryEntry, Video, save_library_entry
+from modules.persistance import (
+    SQL_DB,
+    LibraryEntry,
+    Video,
+    get_or_create_video,
+    save_library_entry,
+)
 from modules.summary import TranscriptTooLongForModelException, get_transcript_summary
 from modules.ui import (
     GENERAL_ERROR_MESSAGE,
@@ -57,7 +63,7 @@ def display_dialog(message: str):
 def save_summary_to_lib():
     """Wrapper func for saving summaries to the library."""
     try:
-        saved_video = Video.create(
+        saved_video, created = get_or_create_video(
             yt_video_id=extract_youtube_video_id(url_input),
             link=url_input,
             title=vid_metadata["name"],

tests/test_persistance.py

@@ -0,0 +1,133 @@
+from datetime import datetime as dt
+
+import pytest
+from peewee import SqliteDatabase
+
+from modules.persistance import (
+    LibraryEntry,
+    Transcript,
+    Video,
+    get_or_create_video,
+    save_library_entry,
+)
+
+# Use an in-memory database for testing
+test_db = SqliteDatabase(":memory:")
+
+
+@pytest.fixture
+def setup_test_db():
+    """Set up a test database before each test."""
+    # Bind models to test database
+    test_db.bind([Video, Transcript, LibraryEntry])
+    test_db.connect()
+    test_db.create_tables([Video, Transcript, LibraryEntry])
+
+    yield test_db
+
+    # Clean up after test
+    test_db.drop_tables([Video, Transcript, LibraryEntry])
+    test_db.close()
+
+
+def test_get_or_create_video_creates_new(setup_test_db):
+    """Test that get_or_create_video creates a new video when it doesn't exist."""
+    yt_video_id = "test_video_123"
+    link = "https://www.youtube.com/watch?v=test_video_123"
+    title = "Test Video"
+    channel = "Test Channel"
+    saved_on = dt.now()
+
+    video, created = get_or_create_video(
+        yt_video_id=yt_video_id,
+        link=link,
+        title=title,
+        channel=channel,
+        saved_on=saved_on,
+    )
+
+    assert created is True
+    assert video.yt_video_id == yt_video_id
+    assert video.title == title
+    assert video.link == link
+    assert video.channel == channel
+
+
+def test_get_or_create_video_gets_existing(setup_test_db):
+    """Test that get_or_create_video returns existing video instead of creating duplicate."""
+    yt_video_id = "test_video_456"
+    link = "https://www.youtube.com/watch?v=test_video_456"
+    title = "Test Video 2"
+    channel = "Test Channel 2"
+    saved_on = dt.now()
+
+    # Create video first time
+    video1, created1 = get_or_create_video(
+        yt_video_id=yt_video_id,
+        link=link,
+        title=title,
+        channel=channel,
+        saved_on=saved_on,
+    )
+
+    assert created1 is True
+
+    # Try to create same video again
+    video2, created2 = get_or_create_video(
+        yt_video_id=yt_video_id,
+        link=link,
+        title=title,
+        channel=channel,
+        saved_on=saved_on,
+    )
+
+    assert created2 is False
+    assert video1.id == video2.id
+    assert video1.yt_video_id == video2.yt_video_id
+
+
+def test_save_summary_with_existing_video(setup_test_db):
+    """Test that saving a summary works when video already exists."""
+    yt_video_id = "test_video_789"
+    link = "https://www.youtube.com/watch?v=test_video_789"
+    title = "Test Video 3"
+    channel = "Test Channel 3"
+    saved_on = dt.now()
+
+    # Create video first (simulating summary.py saving a video)
+    video, created = get_or_create_video(
+        yt_video_id=yt_video_id,
+        link=link,
+        title=title,
+        channel=channel,
+        saved_on=saved_on,
+    )
+
+    # Save a summary
+    save_library_entry(
+        entry_type="S",
+        question_text=None,
+        response_text="This is a test summary",
+        video=video,
+    )
+
+    # Now try to get the same video again (simulating chat.py)
+    video2, created2 = get_or_create_video(
+        yt_video_id=yt_video_id,
+        link=link,
+        title=title,
+        channel=channel,
+        saved_on=saved_on,
+    )
+
+    # Should not create a new video
+    assert created2 is False
+    assert video.id == video2.id
+
+    # Verify we can create a transcript for this video
+    transcript = Transcript.create(
+        video=video2,
+        original_token_num=1000,
+    )
+
+    assert transcript.video.id == video.id