Enable Playwright by default for all scrape sources

CLAUDE_STATUS.md

@@ -349,7 +349,12 @@ cd backend && pytest tests/ -v
 - `backend/scraper/playwright_fetcher.py` - Python client for Playwright service
 - `backend/scraper/sources/generic.py` - `_fetch_page()` with Playwright/httpx logic
 - `backend/scraper/runner.py` - Always enables Playwright for all scrapers
-- `backend/alembic/versions/006_add_use_playwright.py` - Migration (legacy)
+- `backend/alembic/versions/019_enable_playwright_by_default.py` - Sets `use_playwright=True` for all sources
+
+**Database Default:**
+- `use_playwright` column defaults to `True` for new sources (migration 019)
+- All existing sources were updated to `use_playwright=True`
+- The toggle exists in admin for rare cases where httpx-only is needed
 
 **Interactive Page Features (Playwright):**
 - `selectActions` - Array of `{selector, value}` for dropdown selection before page extraction

backend/alembic/versions/019_enable_playwright_by_default.py

@@ -0,0 +1,52 @@
+"""Enable Playwright by default for all sources
+
+This migration:
+1. Sets use_playwright=True for all existing sources that have it False or NULL
+2. Changes the column default to True for new sources
+
+Playwright is required for most modern job sites that use JavaScript rendering.
+Without it, the scraper only gets the initial HTML before JS executes, missing
+dynamically loaded job listings.
+
+Revision ID: 019
+Revises: 018
+Create Date: 2025-12-01
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '019'
+down_revision = '018'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    # Enable Playwright for all existing sources
+    op.execute(
+        "UPDATE scrape_sources SET use_playwright = TRUE WHERE use_playwright = FALSE OR use_playwright IS NULL"
+    )
+
+    # Change the column default to True for new sources
+    op.alter_column(
+        'scrape_sources',
+        'use_playwright',
+        server_default=sa.text('1'),  # MySQL uses 1 for True
+        existing_type=sa.Boolean(),
+        existing_nullable=True
+    )
+
+
+def downgrade() -> None:
+    # Revert column default to False
+    op.alter_column(
+        'scrape_sources',
+        'use_playwright',
+        server_default=sa.text('0'),
+        existing_type=sa.Boolean(),
+        existing_nullable=True
+    )
+    # Note: We don't revert existing data as that could break working scrapers

backend/app/templates/admin/scraper_guide.html

@@ -507,7 +507,7 @@ <h4 class="font-semibold text-gray-900 dark:text-white mb-2">State Abbreviation<
 <!-- Playwright Features -->
 <div class="bg-white dark:bg-gray-800 rounded-lg shadow-sm border border-gray-200 dark:border-gray-700 p-6 mb-8">
     <h3 class="text-lg font-semibold text-gray-900 dark:text-white mb-2">Playwright Features</h3>
-    <p class="text-gray-600 dark:text-gray-400 mb-4">Playwright is a headless browser that renders JavaScript. It's used automatically for all scrapers but provides extra features for DynamicScrapers.</p>
+    <p class="text-gray-600 dark:text-gray-400 mb-4">Playwright is a headless browser that renders JavaScript. <strong>It's enabled by default for all sources</strong> to ensure JavaScript-rendered job listings are properly loaded. DynamicScrapers can also use these additional interactive features:</p>
 
     <div class="grid md:grid-cols-2 gap-4">
         <div class="p-4 bg-gray-50 dark:bg-gray-900 rounded">
@@ -551,7 +551,7 @@ <h3 class="text-lg font-semibold text-gray-900 dark:text-white mb-2">Special Fla
                 <span class="px-2 py-1 bg-blue-100 dark:bg-blue-900 text-blue-800 dark:text-blue-200 rounded text-xs font-medium">use_playwright</span>
             </div>
             <div>
-                <p class="text-sm text-gray-600 dark:text-gray-400">Force Playwright browser rendering. Enabled by default for all scrapers, but can be explicitly set for DynamicScrapers that need it.</p>
+                <p class="text-sm text-gray-600 dark:text-gray-400"><strong>Enabled by default.</strong> All new sources use Playwright browser rendering automatically. This ensures JavaScript-rendered content is properly loaded. Only disable for rare cases where httpx-only is specifically needed.</p>
             </div>
         </div>
         <div class="flex items-start gap-4 p-4 bg-gray-50 dark:bg-gray-900 rounded">
@@ -581,10 +581,10 @@ <h3 class="text-lg font-semibold text-gray-900 dark:text-white mb-4">Troubleshoo
         <div>
             <h4 class="font-medium text-gray-900 dark:text-white mb-1">No jobs found</h4>
             <ul class="text-sm text-gray-600 dark:text-gray-400 list-disc list-inside">
-                <li>Check if the page requires JavaScript - enable Playwright</li>
-                <li>Verify CSS selectors match actual page structure</li>
+                <li>Verify CSS selectors match actual page structure (use browser DevTools)</li>
                 <li>Check for robots.txt blocking in scrape history</li>
                 <li>Try "Analyze Page with AI" for selector suggestions</li>
+                <li>Playwright is enabled by default - if issues persist, check Playwright service logs</li>
             </ul>
         </div>
         <div>