fix: CI tests and photo pipeline improvements

- Fix test expecting version 4.2.3, now correctly expects 5.0.0
- Add missing os import to photo_processor.py
- Update photo pipeline to process ALL zip files (removed filter)
- Add fallback paths for finding takeout archives
- Add validation to skip tiny/corrupted image files
- Update Docker mount for Google Drive Takeout folder

CI tests now passing: 33 passed, 2 skipped

Author: raold

.claude/settings.local.json

@@ -123,7 +123,10 @@
       "Bash(wsl:*)",
       "Bash(.venv/Scripts/python.exe -m uvicorn:*)",
       "Bash(.venv\\Scripts\\python.exe -m uvicorn:*)",
-      "Bash(docker inspect:*)"
+      "Bash(docker inspect:*)",
+      "Bash(docker cp \"$file\" secondbrain-app:/takeout_local/)",
+      "Bash(do docker cp \"$file\" secondbrain-app:/takeout_local/)",
+      "Bash(docker cp:*)"
     ],
     "deny": [],
     "additionalDirectories": []

app/routes/photo_pipeline.py

@@ -160,7 +160,10 @@ async def _run_processing(self) -> None:
         try:
             # Get all zip files to process
             # Check if running in Docker or locally
-            if os.path.exists("/takeout"):
+            if os.path.exists("/takeout_local") and list(Path("/takeout_local").glob("*.zip")):
+                takeout_path = Path("/takeout_local")  # Docker local copy
+                print(f"📁 Using Docker local copy: {takeout_path}")
+            elif os.path.exists("/takeout") and list(Path("/takeout").glob("*.zip")):
                 takeout_path = Path("/takeout")  # Docker mount
                 print(f"📁 Using Docker mount: {takeout_path}")
             elif os.path.exists("G:/My Drive/Takeout"):
@@ -180,9 +183,8 @@ async def _run_processing(self) -> None:
                 print(f"📊 Total files in directory: {len(all_files)}")
                 zip_files = sorted(takeout_path.glob("*.zip"))
                 print(f"📦 Total zip files found: {len(zip_files)}")
-                # Filter for the ones we want
-                zip_files = sorted([f for f in zip_files
-                                  if "-1-" in f.name or f.name.endswith("-032.zip")])
+                # Process ALL zip files
+                zip_files = sorted(zip_files)
                 print(f"✅ Filtered zip files to process: {len(zip_files)}")
             else:
                 print(f"❌ Path does not exist: {takeout_path}")

app/services/photo_processor.py

@@ -5,6 +5,7 @@
 
 import asyncio
 import io
+import os
 import sys
 
 # Force UTF-8 encoding for Windows console
@@ -38,7 +39,11 @@
 from app.core.service_config import config
 
 # Get all configuration from central source
-TAKEOUT_PATH = config.TAKEOUT_PATH
+# Use Windows path if available, otherwise Docker mount
+if os.path.exists("G:/My Drive/Takeout"):
+    TAKEOUT_PATH = "G:/My Drive/Takeout"
+else:
+    TAKEOUT_PATH = config.TAKEOUT_PATH
 CLIP_SERVICE_URL = config.get_clip_url()
 LLAVA_SERVICE_URL = config.get_llava_url()
 DATABASE_URL = config.get_database_url()
@@ -208,6 +213,11 @@ async def process_image(
         """Process a single image: generate embedding and store in database"""
 
         try:
+            # Skip tiny files (likely corrupted)
+            if len(image_data) < 100:
+                print(f"  ⏭️  Skipping {file_name} (too small, likely corrupted)")
+                return False
+                
             # Check if already processed
             async with self.pool.acquire() as conn:
                 exists = await conn.fetchval(

docker-compose.yml

@@ -18,7 +18,7 @@ services:
     volumes:
       - .:/app:rw
       - /app/.venv  # Exclude .venv from mount
-      - "./takeout:/takeout:ro"  # Mount local takeout folder
+      - "G:\\My Drive\\Takeout:/takeout:ro"  # Mount Google Drive Takeout folder (Windows format)
     depends_on:
       postgres:
         condition: service_healthy

tests/unit/test_factory.py

@@ -51,7 +51,7 @@ def test_root_endpoint(self):
 
         data = response.json()
         assert data["environment"] == "testing"
-        assert data["version"] == "4.2.3"
+        assert data["version"] == "5.0.0"
         assert "ready" in data
 
     def test_health_endpoint(self):