server.js

import express from 'express';
import cors from 'cors';
import { spawn } from 'child_process';
import { WebSocketServer } from 'ws';
import { createServer } from 'http';
import fs from 'fs-extra';
import path from 'path';
import { fileURLToPath } from 'url';
import crypto from 'crypto';
import fetch from 'node-fetch';
import dotenv from 'dotenv';
import { v4 as uuidv4 } from 'uuid';
import { createLogger } from './logger.js';

// Load environment variables from frontend directory
dotenv.config();

const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);

const app = express();
const server = createServer(app);
const wss = new WebSocketServer({ server });

// Create component-specific loggers
const apiLogger = createLogger('api');
const wsLogger = createLogger('websocket');
const processingLogger = createLogger('processing');

// Request correlation ID middleware
app.use((req, res, next) => {
  req.correlationId = uuidv4();
  res.set('X-Correlation-ID', req.correlationId);
  
  const start = Date.now();
  apiLogger.apiRequest(req.method, req.path, req.correlationId, { 
    ip: req.ip, 
    userAgent: req.get('User-Agent') 
  });
  
  // Log response when request completes
  res.on('finish', () => {
    const duration = Date.now() - start;
    apiLogger.apiResponse(req.method, req.path, res.statusCode, duration, req.correlationId);
  });
  
  next();
});

// Middleware
app.use(cors());
app.use(express.json());
app.use(express.static('public'));

// Serve Vite build files in production
if (process.env.NODE_ENV === 'production') {
  const distPath = path.join(__dirname, 'dist');
  // Serve static assets with proper MIME types
  app.use(express.static(distPath, {
    setHeaders: (res, filePath) => {
      // Correct MIME type for ES modules
      if (filePath.endsWith('.js') || filePath.endsWith('.mjs')) {
        res.type('application/javascript');
      }
    }
  }));
}

// Serve generated podcast files - updated path to parent directory
app.use('/generated', express.static(path.join(__dirname, '..', 'generated_podcasts')));

// JSON validation middleware for POST endpoints
const validateJSON = (req, res, next) => {
  if (req.method === 'POST' && req.headers['content-type']?.includes('application/json')) {
    if (!req.body || typeof req.body !== 'object' || Array.isArray(req.body)) {
      return res.status(400).json({ 
        error: 'Invalid request',
        message: 'Request body must be a valid JSON object (not array, string, or null)'
      });
    }
  }
  next();
};

// Apply JSON validation to all routes
app.use(validateJSON);

// Store active processing sessions
const activeSessions = new Map();

// API Keys from environment variables
const GEMINI_API_KEY = process.env.GEMINI_API_KEY;
const OPENAI_API_KEY = process.env.OPENAI_API_KEY;

// Podcast Index API configuration
const PODCAST_API_KEY = process.env.PODCAST_API_KEY;
const PODCAST_API_SECRET = process.env.PODCAST_API_SECRET;

// Helper function to create Podcast Index API headers
function createPodcastApiHeaders() {
  const apiHeaderTime = Math.floor(Date.now() / 1000);
  const hashString = `${PODCAST_API_KEY}${PODCAST_API_SECRET}${apiHeaderTime}`;
  const authHash = crypto.createHash('sha1').update(hashString).digest('hex');
  
  return {
    'User-Agent': 'cast-dread-technology/1.0',
    'X-Auth-Key': PODCAST_API_KEY,
    'X-Auth-Date': apiHeaderTime.toString(),
    'Authorization': authHash
  };
}

// Helper function to check if URL is from YouTube
function isYouTubeUrl(url) {
  if (!url || typeof url !== 'string') return false;
  
  try {
    const urlObj = new URL(url.toLowerCase().trim());
    const hostname = urlObj.hostname;
    
    return (
      hostname === 'youtube.com' ||
      hostname === 'www.youtube.com' ||
      hostname === 'youtu.be' ||
      hostname === 'm.youtube.com'
    );
  } catch (error) {
    return false;
  }
}

// Helper function to check for existing YouTube URL
async function checkExistingYouTubeUrl(youtubeUrl, outputDir) {
  try {
    if (!await fs.pathExists(outputDir)) {
      return null;
    }
    
    const files = await fs.readdir(outputDir);
    const metadataFiles = files.filter(file => file.endsWith('_metadata.json'));
    
    for (const metadataFile of metadataFiles) {
      try {
        const metadataPath = path.join(outputDir, metadataFile);
        const metadata = await fs.readJson(metadataPath);
        
        // Check if this metadata file contains the same YouTube URL
        if (metadata.youtubeUrl === youtubeUrl) {
          const baseName = metadataFile.replace('_metadata.json', '');
          
          // Check if corresponding transcript file exists
          const transcriptPath = path.join(outputDir, `${baseName}_transcript.jsonl`);
          if (await fs.pathExists(transcriptPath)) {
            // Read transcript content
            const transcriptContent = await fs.readFile(transcriptPath, 'utf-8');
            const transcript = transcriptContent
              .split('\n')
              .filter(line => line.trim())
              .map(line => JSON.parse(line));
            
            return {
              id: baseName,
              baseName: baseName,
              ...metadata,
              transcript,
              cached: true
            };
          }
        }
      } catch (error) {
        processingLogger.error(`Error reading metadata for ${metadataFile}`, { 
          error: error.message,
          stack: error.stack,
          metadataFile 
        });
      }
    }
    
    return null;
  } catch (error) {
    processingLogger.error('Error checking for existing YouTube URL', { 
      error: error.message,
      stack: error.stack 
    });
    return null;
  }
}

// WebSocket connection for real-time progress updates
wss.on('connection', (ws) => {
  const wsConnectionId = uuidv4();
  wsLogger.info('Client connected for progress updates', { connectionId: wsConnectionId });
  
  ws.on('message', (message) => {
    try {
      const data = JSON.parse(message);
      if (data.type === 'subscribe' && data.sessionId) {
        ws.sessionId = data.sessionId;
        wsLogger.info('Client subscribed to session', { 
          connectionId: wsConnectionId,
          sessionId: data.sessionId 
        });
      }
    } catch (error) {
      wsLogger.error('Error parsing WebSocket message', { 
        connectionId: wsConnectionId,
        error: error.message,
        message: message.toString()
      });
    }
  });
  
  ws.on('close', () => {
    wsLogger.info('Client disconnected', { 
      connectionId: wsConnectionId,
      sessionId: ws.sessionId 
    });
  });
});

// Broadcast progress to all clients subscribed to a session
function broadcastProgress(sessionId, progress) {
  // Create a clean copy of progress data without circular references
  const cleanProgress = {
    type: 'progress',
    sessionId,
    progress: progress.progress || 0,
    message: progress.message || '',
    status: progress.status || 'processing',
    startTime: progress.startTime,
    lastUpdate: progress.lastUpdate,
    estimatedDuration: progress.estimatedDuration,
    audioLength: progress.audioLength
  };
  
  // Add result and error if they exist
  if (progress.result) cleanProgress.result = progress.result;
  if (progress.error) cleanProgress.error = progress.error;
  if (progress.completedAt) cleanProgress.completedAt = progress.completedAt;
  
  wss.clients.forEach((client) => {
    if (client.sessionId === sessionId && client.readyState === 1) {
      client.send(JSON.stringify(cleanProgress));
    }
  });
}

// Cache utility functions

// Generate safe title from episode metadata (matches Python script logic)
function generateSafeTitle(episodeTitle) {
  // Remove non-alphanumeric characters except spaces, hyphens, and underscores
  const safeTitle = episodeTitle.replace(/[^a-zA-Z0-9 \-_]/g, '').trim();
  // Replace spaces with underscores
  return safeTitle.replace(/\s+/g, '_');
}

// Check if cached files exist for an episode
async function checkCachedFiles(episodeMetadata, outputDir) {
  try {
    const episodeTitle = episodeMetadata?.title || 'Unknown_Episode';
    const safeTitle = generateSafeTitle(episodeTitle);
    
    const metadataPath = path.join(outputDir, `${safeTitle}_metadata.json`);
    const transcriptPath = path.join(outputDir, `${safeTitle}_transcript.jsonl`);
    
    // Check if both essential files exist
    const metadataExists = await fs.pathExists(metadataPath);
    const transcriptExists = await fs.pathExists(transcriptPath);
    
    if (metadataExists && transcriptExists) {
      // Return the cached data
      const metadata = await fs.readJson(metadataPath);
      const transcriptContent = await fs.readFile(transcriptPath, 'utf-8');
      
      // Parse JSONL to array
      const transcript = transcriptContent
        .split('\n')
        .filter(line => line.trim())
        .map(line => JSON.parse(line));
      
      return {
        id: safeTitle,
        ...metadata,
        transcript,
        cached: true
      };
    }
    
    return null;
  } catch (error) {
    processingLogger.error('Error checking cached files', { 
      error: error.message,
      stack: error.stack 
    });
    return null;
  }
}

// API Routes

// Search podcasts using Podcast Index API
app.post('/api/search-podcasts', async (req, res) => {
  try {
    const { searchTerm } = req.body;
    
    if (!searchTerm) {
      apiLogger.warn('Search podcasts request missing search term', { 
        correlationId: req.correlationId 
      });
      return res.status(400).json({ error: 'Search term is required' });
    }

    apiLogger.info('Searching for podcasts', { 
      correlationId: req.correlationId,
      searchTerm 
    });
    
    // Call Podcast Index API
    const headers = createPodcastApiHeaders();
    const searchUrl = `https://api.podcastindex.org/api/1.0/search/byterm?q=${encodeURIComponent(searchTerm)}`;
    
    const response = await fetch(searchUrl, { headers });
    
    if (!response.ok) {
      throw new Error(`Podcast API returned ${response.status}: ${response.statusText}`);
    }
    
    const data = await response.json();
    
    if (!data.feeds || data.feeds.length === 0) {
      apiLogger.info('No podcasts found for search term', { 
        correlationId: req.correlationId,
        searchTerm 
      });
      return res.json({ feeds: [] });
    }
    
    // Limit to top 10 results and clean up the data
    const cleanedFeeds = data.feeds.slice(0, 10).map(feed => ({
      id: feed.id,
      title: feed.title,
      author: feed.author,
      description: feed.description,
      image: feed.image,
      url: feed.url,
      categories: feed.categories
    }));
    
    apiLogger.info('Podcast search completed successfully', { 
      correlationId: req.correlationId,
      searchTerm,
      resultCount: cleanedFeeds.length 
    });
    res.json({ feeds: cleanedFeeds });
    
  } catch (error) {
    apiLogger.error('Error searching podcasts', { 
      correlationId: req.correlationId,
      searchTerm: req.body.searchTerm,
      error: error.message,
      stack: error.stack
    });
    res.status(500).json({ error: 'Failed to search podcasts. Please try again.' });
  }
});

// Get episodes for a selected podcast
app.post('/api/get-episodes', async (req, res) => {
  try {
    const { podcastId, page = 1, limit = 10, sortBy = 'recent', search = '' } = req.body;
    
    if (!podcastId) {
      return res.status(400).json({ error: 'Podcast ID is required' });
    }

    // Validate pagination parameters
    const currentPage = Math.max(1, parseInt(page));
    const itemsPerPage = Math.min(Math.max(parseInt(limit), 1), 100); // Max 100 items per page
    
    // For server-side pagination, we need to fetch more episodes than requested
    // to handle sorting and filtering on the server side
    const maxEpisodesToFetch = 150; // Fetch up to 150 episodes to enable proper server-side sorting

    apiLogger.info('Getting episodes for podcast', { 
      correlationId: req.correlationId,
      podcastId,
      page: currentPage,
      limit: itemsPerPage,
      sortBy,
      search: search.trim(),
      maxEpisodesToFetch
    });
    
    // Call Podcast Index API for episodes
    const headers = createPodcastApiHeaders();
    const episodesUrl = `https://api.podcastindex.org/api/1.0/episodes/byfeedid?id=${podcastId}&max=${maxEpisodesToFetch}`;
    
    const response = await fetch(episodesUrl, { headers });
    
    if (!response.ok) {
      throw new Error(`Podcast API returned ${response.status}: ${response.statusText}`);
    }
    
    const data = await response.json();
    
    if (!data.items || data.items.length === 0) {
      return res.json({ 
        items: [],
        pagination: {
          currentPage: 1,
          totalPages: 0,
          totalItems: 0,
          hasNext: false,
          hasPrevious: false,
          itemsPerPage
        }
      });
    }
    
    // Clean up episode data
    let cleanedEpisodes = data.items.map(episode => ({
      id: episode.id,
      title: episode.title,
      description: episode.description,
      enclosureUrl: episode.enclosureUrl,
      datePublished: episode.datePublished,
      duration: episode.duration,
      episodeType: episode.episodeType
    }));

    // Apply server-side search filter
    if (search.trim()) {
      const query = search.toLowerCase();
      cleanedEpisodes = cleanedEpisodes.filter(episode => 
        episode.title?.toLowerCase().includes(query) ||
        episode.description?.toLowerCase().includes(query)
      );
    }

    // Apply server-side sorting
    cleanedEpisodes.sort((a, b) => {
      switch (sortBy) {
        case 'recent':
          return new Date(b.datePublished || 0) - new Date(a.datePublished || 0);
        case 'oldest':
          return new Date(a.datePublished || 0) - new Date(b.datePublished || 0);
        case 'duration-desc':
          return (b.duration || 0) - (a.duration || 0);
        case 'duration-asc':
          return (a.duration || 0) - (b.duration || 0);
        case 'title-asc':
          return (a.title || '').localeCompare(b.title || '');
        default:
          return 0;
      }
    });

    // Calculate pagination
    const totalItems = cleanedEpisodes.length;
    const totalPages = Math.ceil(totalItems / itemsPerPage);
    const startIndex = (currentPage - 1) * itemsPerPage;
    const endIndex = startIndex + itemsPerPage;
    
    // Get the page slice
    const paginatedEpisodes = cleanedEpisodes.slice(startIndex, endIndex);
    
    // Create pagination metadata
    const pagination = {
      currentPage,
      totalPages,
      totalItems,
      hasNext: currentPage < totalPages,
      hasPrevious: currentPage > 1,
      itemsPerPage
    };
    
    apiLogger.info('Episodes retrieved successfully', { 
      correlationId: req.correlationId,
      podcastId,
      episodeCount: paginatedEpisodes.length,
      totalItems,
      currentPage,
      totalPages
    });
    
    res.json({ 
      items: paginatedEpisodes,
      pagination 
    });
    
  } catch (error) {
    apiLogger.error('Error getting episodes', { 
      correlationId: req.correlationId,
      error: error.message,
      stack: error.stack 
    });
    res.status(500).json({ error: 'Failed to get episodes. Please try again.' });
  }
});

// Process a selected podcast episode or YouTube video
app.post('/api/process-podcast', async (req, res) => {
  try {
    const { episodeUrl, podcastMetadata, sessionId, youtubeUrl, forceReprocess } = req.body;
    
    // Support both podcast episodes and YouTube URLs
    const contentUrl = youtubeUrl || episodeUrl;
    
    if (!contentUrl || !sessionId) {
      return res.status(400).json({ error: 'Content URL and session ID are required' });
    }

    // Check if session already exists
    if (activeSessions.has(sessionId)) {
      return res.status(409).json({ error: 'Session already in progress' });
    }

    // Validate YouTube URL if provided
    if (youtubeUrl && !isYouTubeUrl(youtubeUrl)) {
      return res.status(400).json({ error: 'Invalid YouTube URL format' });
    }

    const outputDir = path.join(__dirname, '..', 'generated_podcasts');

    // Check for existing YouTube URL processing (skip if forceReprocess is true)
    if (youtubeUrl && !forceReprocess) {
      processingLogger.info('Checking for existing YouTube URL', { 
        sessionId,
        youtubeUrl 
      });
      const existingData = await checkExistingYouTubeUrl(youtubeUrl, outputDir);
      
      if (existingData) {
        processingLogger.info('Found existing processed YouTube video', { 
          sessionId,
          title: existingData.title,
          cached: true 
        });
        
        // Set session as completed immediately with existing data
        activeSessions.set(sessionId, {
          status: 'completed',
          progress: 100,
          message: 'YouTube video already processed - loaded from cache!',
          result: existingData,
          completedAt: Date.now(),
          startTime: Date.now(),
          cached: true
        });

        // Send immediate response with cache indicator
        res.json({ 
          sessionId, 
          status: 'completed',
          message: 'YouTube video already processed - loaded from cache!',
          cached: true,
          result: existingData
        });

        // Broadcast completion to WebSocket clients
        broadcastProgress(sessionId, activeSessions.get(sessionId));
        
        // Clean up session after 5 minutes
        setTimeout(() => {
          activeSessions.delete(sessionId);
        }, 5 * 60 * 1000);

        return;
      }
    }

    // Check for cached files first (only for podcast episodes, not YouTube since we already checked above)
    // Skip cache check if forceReprocess is true
    if (!youtubeUrl && !forceReprocess) {
      const episodeMetadata = podcastMetadata?.episode;
      
      if (episodeMetadata) {
        processingLogger.info('Checking cache for episode', { 
          sessionId,
          episodeTitle: episodeMetadata.title 
        });
        const cachedData = await checkCachedFiles(episodeMetadata, outputDir);
        
        if (cachedData) {
          processingLogger.info('Found cached transcript for episode', { 
            sessionId,
            episodeTitle: episodeMetadata.title,
            cached: true 
          });
          
          // Set session as completed immediately with cached data
          activeSessions.set(sessionId, {
            status: 'completed',
            progress: 100,
            message: 'Loaded from cache instantly!',
            result: cachedData,
            completedAt: Date.now(),
            startTime: Date.now(),
            cached: true
          });

          // Send immediate response with cache indicator
          res.json({ 
            sessionId, 
            status: 'completed',
            message: 'Podcast loaded from cache instantly!',
            cached: true,
            result: cachedData
          });

          // Broadcast completion to WebSocket clients
          broadcastProgress(sessionId, activeSessions.get(sessionId));
          
          // Clean up session after 5 minutes
          setTimeout(() => {
            activeSessions.delete(sessionId);
          }, 5 * 60 * 1000);

          return;
        }
      }

      // No cache found, proceed with normal processing
      processingLogger.info('No cache found, starting processing', { 
        sessionId,
        episodeTitle: episodeMetadata?.title || 'Unknown Episode' 
      });
    }
    
    // Log if force reprocessing
    if (forceReprocess) {
      processingLogger.info('Force reprocessing requested', { 
        sessionId,
        content: youtubeUrl || podcastMetadata?.episode?.title || 'Unknown',
        forceReprocess: true 
      });
    }
    
    // Start processing
    activeSessions.set(sessionId, { status: 'starting', startTime: Date.now() });
    
    // Send immediate response
    const contentType = youtubeUrl ? 'YouTube video' : 'podcast episode';
    const processingMessage = forceReprocess 
      ? `Re-processing ${contentType}. This will replace the existing transcript.`
      : `${contentType} processing started. Monitor progress via WebSocket.`;
      
    res.json({ 
      sessionId, 
      status: 'started',
      message: processingMessage,
      cached: false
    });

    // Start the Python script asynchronously
    if (youtubeUrl) {
      processYouTubeAsync(youtubeUrl, podcastMetadata || {}, sessionId);
    } else {
      processEpisodeAsync(episodeUrl, podcastMetadata, sessionId);
    }

  } catch (error) {
    processingLogger.error('Error starting content processing', { 
      sessionId,
      error: error.message,
      stack: error.stack 
    });
    res.status(500).json({ error: 'Failed to start processing' });
  }
});

// Get processing status
app.get('/api/status/:sessionId', (req, res) => {
  const { sessionId } = req.params;
  const session = activeSessions.get(sessionId);
  
  if (!session) {
    return res.status(404).json({ error: 'Session not found' });
  }
  
  // Create a clean copy without circular references
  const cleanSession = {
    progress: session.progress || 0,
    message: session.message || '',
    status: session.status || 'processing',
    startTime: session.startTime,
    lastUpdate: session.lastUpdate,
    estimatedDuration: session.estimatedDuration,
    audioLength: session.audioLength
  };
  
  // Add result and error if they exist
  if (session.result) cleanSession.result = session.result;
  if (session.error) cleanSession.error = session.error;
  if (session.completedAt) cleanSession.completedAt = session.completedAt;
  
  res.json(cleanSession);
});

// Check if podcasts/episodes are already processed
app.post('/api/check-processed', async (req, res) => {
  try {
    const { podcasts } = req.body; // Array of podcast objects with titles
    
    if (!podcasts || !Array.isArray(podcasts)) {
      return res.status(400).json({ error: 'Podcasts array is required' });
    }

    const generatedDir = path.join(__dirname, '..', 'generated_podcasts');
    const processedStatus = {};

    if (await fs.pathExists(generatedDir)) {
      const files = await fs.readdir(generatedDir);
      const metadataFiles = files.filter(file => file.endsWith('_metadata.json'));

      // Load all processed podcasts
      const processedPodcasts = [];
      for (const metadataFile of metadataFiles) {
        try {
          const metadataPath = path.join(generatedDir, metadataFile);
          const metadata = await fs.readJson(metadataPath);
          const baseName = metadataFile.replace('_metadata.json', '');
          
          // Check if transcript exists
          const transcriptPath = path.join(generatedDir, `${baseName}_transcript.jsonl`);
          if (await fs.pathExists(transcriptPath)) {
            processedPodcasts.push({
              id: baseName,
              title: metadata.title,
              podcastTitle: metadata.podcast_info?.title,
              podcastAuthor: metadata.podcast_info?.author
            });
          }
        } catch (error) {
          processingLogger.error(`Error reading metadata for ${metadataFile}`, { 
            error: error.message,
            stack: error.stack,
            metadataFile 
          });
        }
      }

      // Check each requested podcast
      for (const podcast of podcasts) {
        const podcastKey = `${podcast.id}`;
        processedStatus[podcastKey] = {
          isProcessed: false,
          processedEpisodes: []
        };

        // Check if any episodes from this podcast are processed
        const matchingEpisodes = processedPodcasts.filter(processed => 
          processed.podcastTitle?.toLowerCase().includes(podcast.title?.toLowerCase()) ||
          processed.podcastAuthor?.toLowerCase().includes(podcast.author?.toLowerCase())
        );

        if (matchingEpisodes.length > 0) {
          processedStatus[podcastKey] = {
            isProcessed: true,
            processedEpisodes: matchingEpisodes,
            count: matchingEpisodes.length
          };
        }
      }
    }

    res.json(processedStatus);
  } catch (error) {
    apiLogger.error('Error checking processed podcasts', { 
      correlationId: req.correlationId,
      error: error.message,
      stack: error.stack 
    });
    res.status(500).json({ error: 'Failed to check processed podcasts' });
  }
});

// Get available generated podcasts
app.get('/api/generated-podcasts', async (req, res) => {
  try {
    const generatedDir = path.join(__dirname, '..', 'generated_podcasts');
    
    if (!await fs.pathExists(generatedDir)) {
      return res.json([]);
    }
    
    const files = await fs.readdir(generatedDir);
    const podcasts = [];
    
    // Look for metadata files directly in the directory
    const metadataFiles = files.filter(file => file.endsWith('_metadata.json'));
    
    for (const metadataFile of metadataFiles) {
      try {
        const metadataPath = path.join(generatedDir, metadataFile);
        const metadata = await fs.readJson(metadataPath);
        
        // Extract the base name from the filename (remove _metadata.json)
        const baseName = metadataFile.replace('_metadata.json', '');
        
        // Check if corresponding transcript file exists
        const transcriptPath = path.join(generatedDir, `${baseName}_transcript.jsonl`);
        const transcriptExists = await fs.pathExists(transcriptPath);
        
        if (transcriptExists) {
          const originalId = metadata.id;
          const podcastData = {
            ...metadata,
            baseName: baseName,
            originalId: originalId
          };
          // Override the ID with baseName for API compatibility
          podcastData.id = baseName;
          
          podcasts.push(podcastData);
        }
      } catch (error) {
        processingLogger.error(`Error reading metadata for ${metadataFile}`, { 
          error: error.message,
          stack: error.stack,
          metadataFile 
        });
      }
    }
    
    // Sort by generation date (newest first)
    podcasts.sort((a, b) => new Date(b.generated_at) - new Date(a.generated_at));
    
    res.json(podcasts);
  } catch (error) {
    apiLogger.error('Error getting generated podcasts', { 
      correlationId: req.correlationId,
      error: error.message,
      stack: error.stack 
    });
    res.status(500).json({ error: 'Failed to get generated podcasts' });
  }
});

// Get specific podcast data
app.get('/api/podcast/:id', async (req, res) => {
  try {
    const { id } = req.params;
    const generatedDir = path.join(__dirname, '..', 'generated_podcasts');
    
    // Files are saved directly in generated_podcasts directory with the pattern {id}_metadata.json
    const metadataPath = path.join(generatedDir, `${id}_metadata.json`);
    
    if (!await fs.pathExists(metadataPath)) {
      return res.status(404).json({ error: 'Podcast not found' });
    }
    
    // Read metadata
    const metadata = await fs.readJson(metadataPath);
    
    // Read transcript
    const transcriptPath = path.join(generatedDir, `${id}_transcript.jsonl`);
    const transcriptContent = await fs.readFile(transcriptPath, 'utf-8');
    
    // Parse JSONL to array
    const transcript = transcriptContent
      .split('\n')
      .filter(line => line.trim())
      .map(line => JSON.parse(line));
    
    res.json({
      ...metadata,
      transcript,
      id
    });
  } catch (error) {
    apiLogger.error('Error getting podcast data', { 
      correlationId: req.correlationId,
      error: error.message,
      stack: error.stack 
    });
    res.status(500).json({ error: 'Failed to get podcast data' });
  }
});

// Get system prompt for podcast with robust filename matching
app.get('/api/system-prompt/:podcastId', async (req, res) => {
  try {
    const { podcastId } = req.params;
    const generatedDir = path.join(__dirname, '..', 'generated_podcasts');
    
    apiLogger.info('System prompt request', { 
      correlationId: req.correlationId,
      podcastId: podcastId,
      originalPodcastId: podcastId
    });
    
    // Strategy 1: Direct match
    let filename = `${podcastId}_system_prompt.txt`;
    let filePath = path.join(generatedDir, filename);
    let matchStrategy = 'direct';
    
    if (await fs.pathExists(filePath)) {
      apiLogger.info('System prompt found via direct match', { 
        correlationId: req.correlationId,
        podcastId,
        filename,
        strategy: matchStrategy
      });
    } else {
      // Strategy 2: URL decode match (handles %20, %2D, etc.)
      try {
        const decodedPodcastId = decodeURIComponent(podcastId);
        if (decodedPodcastId !== podcastId) {
          filename = `${decodedPodcastId}_system_prompt.txt`;
          filePath = path.join(generatedDir, filename);
          matchStrategy = 'url-decoded';
          
          if (await fs.pathExists(filePath)) {
            apiLogger.info('System prompt found via URL decode match', { 
              correlationId: req.correlationId,
              originalPodcastId: podcastId,
              decodedPodcastId,
              filename,
              strategy: matchStrategy
            });
          }
        }
      } catch (decodeError) {
        apiLogger.warn('URL decode failed', { 
          correlationId: req.correlationId,
          podcastId,
          error: decodeError.message
        });
      }
    }
    
    // Strategy 3: Fuzzy search if direct methods failed
    if (!await fs.pathExists(filePath)) {
      const files = await fs.readdir(generatedDir);
      const systemPromptFiles = files.filter(f => f.endsWith('_system_prompt.txt'));
      
      // Look for partial matches
      let bestMatch = null;
      let bestScore = 0;
      
      for (const file of systemPromptFiles) {
        const baseName = file.replace('_system_prompt.txt', '');
        
        // Exact match after removing file extension
        if (baseName === podcastId) {
          bestMatch = file;
          bestScore = 1;
          break;
        }
        
        // Partial match scoring
        const podcastIdLower = podcastId.toLowerCase();
        const baseNameLower = baseName.toLowerCase();
        
        if (baseNameLower.includes(podcastIdLower) || podcastIdLower.includes(baseNameLower)) {
          const score = Math.min(podcastIdLower.length, baseNameLower.length) / 
                       Math.max(podcastIdLower.length, baseNameLower.length);
          if (score > bestScore) {
            bestMatch = file;
            bestScore = score;
          }
        }
      }
      
      if (bestMatch && bestScore > 0.5) {  // Require at least 50% similarity
        filename = bestMatch;
        filePath = path.join(generatedDir, filename);
        matchStrategy = 'fuzzy';
        
        apiLogger.info('System prompt found via fuzzy match', { 
          correlationId: req.correlationId,
          originalPodcastId: podcastId,
          matchedFile: bestMatch,
          similarity: bestScore,
          strategy: matchStrategy
        });
      }
    }
    
    // Check if file exists after all strategies
    if (!await fs.pathExists(filePath)) {
      apiLogger.warn('System prompt not found after all strategies', { 
        correlationId: req.correlationId,
        podcastId,
        searchedStrategies: ['direct', 'url-decoded', 'fuzzy'],
        generatedDir
      });
      
      return res.status(404).json({ 
        error: 'System prompt not found',
        podcastId,
        searchedStrategies: ['direct', 'url-decoded', 'fuzzy']
      });
    }
    
    // Read and return the system prompt content
    const content = await fs.readFile(filePath, 'utf-8');
    
    apiLogger.info('System prompt loaded successfully', { 
      correlationId: req.correlationId,
      podcastId,
      filename,
      strategy: matchStrategy,
      contentLength: content.length
    });
    
    res.json({ 
      content,
      filename,
      podcastId,
      matchStrategy,
      success: true
    });
    
  } catch (error) {
    apiLogger.error('Error loading system prompt', { 
      correlationId: req.correlationId,
      podcastId: req.params.podcastId,
      error: error.message,
      stack: error.stack 
    });
    res.status(500).json({ 
      error: 'Failed to load system prompt',
      podcastId: req.params.podcastId
    });
  }
});

// Serve React app for any unmatched routes (SPA fallback)
if (process.env.NODE_ENV === 'production') {
  app.get('*', (req, res) => {
    res.sendFile(path.join(__dirname, 'dist', 'index.html'));
  });
}

// Async function to process YouTube video
async function processYouTubeAsync(youtubeUrl, youtubeMetadata, sessionId) {
  try {
    processingLogger.processingStart(sessionId, 'YouTube video', { 
      youtubeUrl,
      metadata: youtubeMetadata 
    });
    
    const scriptPath = path.join(__dirname, '..', 'scripts', 'podcast2jsonl.py');
    const outputDir = path.join(__dirname, '..', 'generated_podcasts');
    
    // Initial status with enhanced tracking
    const initialSession = { 
      status: 'starting', 
      progress: 5,
      message: 'Initializing YouTube video processing...',
      startTime: activeSessions.get(sessionId).startTime,
      lastUpdate: Date.now()
    };
    
    activeSessions.set(sessionId, initialSession);
    broadcastProgress(sessionId, initialSession);
    
    // Build command args for YouTube processing
    const args = [
      scriptPath,
      youtubeUrl, // Direct YouTube URL
      '--output-dir', outputDir,
      '--session-id', sessionId,
    ];
    
    // Create YouTube-specific metadata for the Python script
    const youtubeMetadataForScript = {
      episode: {
        title: youtubeMetadata.title || 'YouTube Video',
        description: youtubeMetadata.description || 'YouTube video transcript',
        youtubeUrl: youtubeUrl,
        datePublished: Math.floor(Date.now() / 1000),
        duration: youtubeMetadata.duration || 0,
        episodeType: 'youtube'
      },
      podcast: {
        title: youtubeMetadata.title || 'YouTube Video',
        author: youtubeMetadata.author || youtubeMetadata.channelTitle || 'Unknown Channel',
        description: youtubeMetadata.description || 'YouTube video content',
        url: youtubeUrl
      }
    };
    
    // Add metadata as JSON string
    args.push('--episode-metadata', JSON.stringify(youtubeMetadataForScript));
    
    // Add API keys
    args.push('--gemini-api-key', GEMINI_API_KEY);
    args.push('--openai-api-key', OPENAI_API_KEY);
    
    processingLogger.subprocessStart(sessionId, 'python3', args.slice(1));
    
    const pythonProcess = spawn('python3', args, {
      cwd: __dirname,
      stdio: ['pipe', 'pipe', 'pipe']
    });
    
    let stdoutData = '';
    let stderrData = '';
    const processStartTime = Date.now();
    
    pythonProcess.stdout.on('data', (data) => {
      const output = data.toString();
      stdoutData += output;
      processingLogger.subprocessOutput(sessionId, 'python3', output, false);
      
      // Set status to processing when we start getting output
      const session = activeSessions.get(sessionId);
      if (session && session.status === 'starting') {
        activeSessions.set(sessionId, {
          ...session,
          status: 'processing'
        });
      }
      
      // Parse output for progress indicators
      updateProgressFromOutput(sessionId, output);
    });
    
    pythonProcess.stderr.on('data', (data) => {
      const output = data.toString();
      stderrData += output;
      processingLogger.subprocessOutput(sessionId, 'python3', output, true);
      
      // Update session with error info for debugging
      const session = activeSessions.get(sessionId);
      if (session) {
        activeSessions.set(sessionId, {
          ...session,
          lastError: output.trim(),
          lastUpdate: Date.now()
        });
      }
    });
    
    pythonProcess.on('close', async (code) => {
      const processDuration = Date.now() - processStartTime;
      processingLogger.subprocessComplete(sessionId, 'python3', code, processDuration, {
        stdoutLength: stdoutData.length,
        stderrLength: stderrData.length
      });
      
      if (code === 0) {
        // Success - find the generated files
        try {
          const generatedFiles = await findGeneratedFiles(outputDir, activeSessions.get(sessionId).startTime);
          
          activeSessions.set(sessionId, {
            status: 'completed',
            progress: 100,
            message: 'YouTube video processing completed successfully!',
            result: generatedFiles,
            completedAt: Date.now(),
            startTime: activeSessions.get(sessionId).startTime
          });
          
          broadcastProgress(sessionId, activeSessions.get(sessionId));
          processingLogger.processingComplete(sessionId, Date.now() - activeSessions.get(sessionId).startTime, {
            resultId: generatedFiles.id
          });
          
          // Clean up session after 5 minutes
          setTimeout(() => {
            activeSessions.delete(sessionId);
          }, 5 * 60 * 1000);
          
        } catch (error) {
          processingLogger.error('Error finding generated files', { 
            sessionId,
            error: error.message,
            stack: error.stack 
          });
          activeSessions.set(sessionId, {
            status: 'error',
            progress: 0,
            message: 'Processing completed but failed to locate generated files',
            error: error.message,
            completedAt: Date.now(),
            startTime: activeSessions.get(sessionId).startTime
          });
          broadcastProgress(sessionId, activeSessions.get(sessionId));
        }
      } else {
        // Error
        processingLogger.error('Python script failed', { 
          sessionId,
          stderrData,
          exitCode: code 
        });
        
        activeSessions.set(sessionId, {
          status: 'error',
          progress: 0,
          message: 'YouTube video processing failed',
          error: stderrData.trim() || `Process exited with code ${code}`,
          completedAt: Date.now(),
          startTime: activeSessions.get(sessionId).startTime
        });
        broadcastProgress(sessionId, activeSessions.get(sessionId));
      }
    });
    
    pythonProcess.on('error', (error) => {
      processingLogger.error('Failed to start Python process', { 
        sessionId,
        error: error.message,
        stack: error.stack 
      });
      activeSessions.set(sessionId, {
        status: 'error',
        progress: 0,
        message: 'Failed to start processing script',
        error: error.message,
        completedAt: Date.now(),
        startTime: activeSessions.get(sessionId).startTime
      });
      broadcastProgress(sessionId, activeSessions.get(sessionId));
    });
    
  } catch (error) {
    processingLogger.error('Error in processYouTubeAsync', { 
      sessionId,
      error: error.message,
      stack: error.stack 
    });
    activeSessions.set(sessionId, {
      status: 'error',
      progress: 0,
      message: 'Failed to start YouTube processing',
      error: error.message,
      completedAt: Date.now(),
      startTime: activeSessions.get(sessionId).startTime
    });
    broadcastProgress(sessionId, activeSessions.get(sessionId));
  }
}

// Async function to process episode
async function processEpisodeAsync(episodeUrl, podcastMetadata, sessionId) {
  try {
    processingLogger.info('Starting real processing for session', { 
      sessionId 
    });
    processingLogger.info('Processing episode', { 
      sessionId,
      episodeUrl 
    });
    processingLogger.info('Processing with metadata', { 
      sessionId,
      podcastMetadata 
    });
    
    const scriptPath = path.join(__dirname, '..', 'scripts', 'podcast2jsonl.py');
    const outputDir = path.join(__dirname, '..', 'generated_podcasts');
    
    // Initial status with enhanced tracking
    const initialSession = { 
      status: 'starting', 
      progress: 5,
      message: 'Initializing podcast processing...',
      startTime: activeSessions.get(sessionId).startTime,
      lastUpdate: Date.now()
    };
    
    activeSessions.set(sessionId, initialSession);
    broadcastProgress(sessionId, initialSession);
    
    // Removed artificial heartbeat mechanism - progress now only updates based on actual Python script output
    
    // Build command args for direct episode processing
    const args = [
      scriptPath,
      '--episode-url', episodeUrl,
      '--episode-metadata', JSON.stringify(podcastMetadata),
      '--output-dir', outputDir,
      '--session-id', sessionId,
    ];
    
    // Add test mode flag for development if needed
    // args.push('--test-mode');  // Uncomment to use test mode
    
    // Add API keys
    args.push('--gemini-api-key', GEMINI_API_KEY);
    args.push('--openai-api-key', OPENAI_API_KEY);
    
    processingLogger.info('Executing Python script', { 
      sessionId,
      args: args.slice(1) // Don't log the script path
    });
    
    const pythonProcess = spawn('python3', args, {
      cwd: __dirname,
      stdio: ['pipe', 'pipe', 'pipe']
    });
    
    let stdoutData = '';
    let stderrData = '';
    
    pythonProcess.stdout.on('data', (data) => {
      const output = data.toString();
      stdoutData += output;
      processingLogger.subprocessOutput(sessionId, 'python3', output, false);
      
      // Set status to processing when we start getting output
      const session = activeSessions.get(sessionId);
      if (session && session.status === 'starting') {
        activeSessions.set(sessionId, {
          ...session,
          status: 'processing'
        });
      }
      
      // Parse output for progress indicators
      updateProgressFromOutput(sessionId, output);
    });
    
    pythonProcess.stderr.on('data', (data) => {
      const output = data.toString();
      stderrData += output;
      processingLogger.subprocessOutput(sessionId, 'python3', output, true);
    });
    
    pythonProcess.on('close', async (code) => {
      processingLogger.info('Python process completed', { 
        sessionId,
        exitCode: code 
      });
      
      if (code === 0) {
        // Success - find the generated files
        try {
          const generatedFiles = await findGeneratedFiles(outputDir, activeSessions.get(sessionId).startTime);
          
          // Heartbeat cleanup removed - no longer using artificial heartbeat
          
          activeSessions.set(sessionId, {
            status: 'completed',
            progress: 100,
            message: 'Podcast processing completed successfully!',
            result: generatedFiles,
            completedAt: Date.now(),
            startTime: activeSessions.get(sessionId).startTime
          });
          
          broadcastProgress(sessionId, activeSessions.get(sessionId));
          processingLogger.info('Processing completed successfully', { 
            sessionId,
            resultId: generatedFiles?.id 
          });
          
          // Clean up session after 5 minutes
          setTimeout(() => {
            activeSessions.delete(sessionId);
          }, 5 * 60 * 1000);
          
        } catch (error) {
          processingLogger.error('Error finding generated files', { 
            sessionId,
            error: error.message,
            stack: error.stack 
          });
          activeSessions.set(sessionId, {
            status: 'error',
            progress: 0,
            message: 'Processing completed but failed to locate generated files',
            error: error.message,
            completedAt: Date.now(),
            startTime: activeSessions.get(sessionId).startTime
          });
          broadcastProgress(sessionId, activeSessions.get(sessionId));
        }
      } else {
        // Error
        processingLogger.error('Python script failed', { 
          sessionId,
          stderrData,
          exitCode: code 
        });
        
        // Heartbeat cleanup removed - no longer using artificial heartbeat
        
        activeSessions.set(sessionId, {
          status: 'error',
          progress: 0,
          message: 'Podcast processing failed',
          error: stderrData.trim() || `Process exited with code ${code}`,
          completedAt: Date.now(),
          startTime: activeSessions.get(sessionId).startTime
        });
        broadcastProgress(sessionId, activeSessions.get(sessionId));
      }
    });
    
    pythonProcess.on('error', (error) => {
      processingLogger.error('Failed to start Python process', { 
        sessionId,
        error: error.message,
        stack: error.stack 
      });
      activeSessions.set(sessionId, {
        status: 'error',
        progress: 0,
        message: 'Failed to start processing script',
        error: error.message,
        completedAt: Date.now(),
        startTime: activeSessions.get(sessionId).startTime
      });
      broadcastProgress(sessionId, activeSessions.get(sessionId));
    });
    
  } catch (error) {
    processingLogger.error('Error in processEpisodeAsync', { 
      sessionId,
      error: error.message,
      stack: error.stack 
    });
    activeSessions.set(sessionId, {
      status: 'error',
      progress: 0,
      message: 'Failed to start processing',
      error: error.message,
      completedAt: Date.now(),
      startTime: activeSessions.get(sessionId).startTime
    });
    broadcastProgress(sessionId, activeSessions.get(sessionId));
  }
}

// Enhanced progress tracking with better parsing and time-based estimation
// Buffer map to accumulate stdout by session so markers split across data events are handled
const stdoutBuffers = new Map();

function updateProgressFromOutput(sessionId, output) {
  // Append to buffer for this session
  const prev = stdoutBuffers.get(sessionId) || '';
  const combined = prev + output;
  const lines = combined.split(/\r?\n/);
  // Keep last partial line in buffer (it might be incomplete)
  stdoutBuffers.set(sessionId, lines.pop());

  lines.forEach((line) => {
    if (!line) return;
    // Process chunk markers
    const marker = '@@CHUNK_READY@@:';
    if (line.startsWith(marker)) {
      const jsonPart = line.slice(marker.length).trim();
      try {
        const chunkPayload = JSON.parse(jsonPart);
        const message = {
          type: 'chunkTranscript',
          sessionId,
          chunk: chunkPayload,
        };
        wss.clients.forEach((client) => {
          if (client.readyState === 1 && client.sessionId === sessionId) {
            client.send(JSON.stringify(message));
          }
        });
      } catch (err) {
        console.error('Failed to parse CHUNK_READY payload:', err, 'raw:', jsonPart);
      }
    }
  });

  // Detect overall completion marker
  if (output.includes('@@PROCESS_COMPLETE@@:')) {
    const completeMatch = output.match(/@@PROCESS_COMPLETE@@:([A-Za-z0-9_\-]+)/);
    if (completeMatch) {
      const baseName = completeMatch[1];
      try {
        const outputDir = path.join(__dirname, '..', 'generated_podcasts');
        const metadataPath = path.join(outputDir, `${baseName}_metadata.json`);
        const transcriptPath = path.join(outputDir, `${baseName}_transcript.jsonl`);

        if (fs.existsSync(metadataPath) && fs.existsSync(transcriptPath)) {
          const metadata = fs.readJsonSync(metadataPath);
          const transcriptContent = fs.readFileSync(transcriptPath, 'utf-8');
          const transcriptArr = transcriptContent.split('\n').filter(l => l.trim()).map(l => JSON.parse(l));

          const resultObj = {
            id: baseName,
            baseName,
            ...metadata,
            transcript: transcriptArr
          };

          const sessionData = {
            status: 'completed',
            progress: 100,
            message: 'Podcast processing completed successfully!',
            result: resultObj,
            completedAt: Date.now(),
            startTime: activeSessions.get(sessionId)?.startTime || Date.now()
          };

          activeSessions.set(sessionId, sessionData);
          broadcastProgress(sessionId, sessionData);
          return; // We can exit early; no need to parse further text
        }
      } catch (e) {
        processingLogger.error('Failed to finalize session from PROCESS_COMPLETE marker', { 
          sessionId,
          error: e.message,
          stack: e.stack 
        });
      }
    }
  }
  const session = activeSessions.get(sessionId);
  if (!session) return;
  
  let progress = session.progress || 0;
  let message = session.message || '';
  let estimatedDuration = session.estimatedDuration || 0;
  let audioLength = session.audioLength || 0;
  
  // Parse different stages from output with improved progress distribution
  if (output.includes('Downloading episode audio') || output.includes('Downloading audio from YouTube')) {
    progress = 8;
    message = 'Downloading episode audio...';
  } else if (output.includes('Normalizing audio') || output.includes('Normalizing audio to mono 16kHz')) {
    progress = 12;
    message = 'Normalizing audio to mono 16kHz...';
  } else if (output.includes('Uploading audio file to Gemini for speaker identification') || output.includes('Identifying speakers')) {
    progress = 16;
    message = 'Identifying speakers using AI...';
  } else if (output.includes('Uploading audio file to Gemini for transcription') || output.includes('Generating transcription')) {
    progress = 20;
    message = 'Starting AI transcription...';
  } else if (output.includes('AI is analyzing and transcribing the audio')) {
    progress = 22;
    message = 'AI is analyzing and transcribing the audio (this may take several minutes)...';
  } else if (output.includes('Processing chunk')) {
    const match = output.match(/Processing chunk (\d+)\/(\d+)/);
    if (match) {
      const current = parseInt(match[1]);
      const total = parseInt(match[2]);
      // Transcription takes 20-75% of total progress
      progress = 20 + ((current - 1) / total) * 55;
      message = `Transcribing audio (chunk ${current}/${total})...`;
    }
  } else if (output.includes('Completed chunk')) {
    const match = output.match(/Completed chunk (\d+)\/(\d+)/);
    if (match) {
      const current = parseInt(match[1]);
      const total = parseInt(match[2]);
      // Update progress after chunk completion
      progress = 20 + (current / total) * 55;
      message = `Completed transcription of chunk ${current}/${total}`;
    }
  } else if (output.includes('Transcription completed successfully')) {
    progress = 75;
    message = 'Transcription completed successfully!';
  } else if (output.includes('Merging chunk transcripts')) {
    progress = 77;
    message = 'Merging transcript chunks...';
  } else if (output.includes('Post-processing transcript')) {
    progress = 80;
    message = 'Post-processing transcript...';
  } else if (output.includes('Generating podcast summary')) {
    progress = 85;
    message = 'Generating episode summary...';
  } else if (output.includes('Generating system prompt')) {
    progress = 90;
    message = 'Creating AI voice interaction prompt...';
  } else if (output.includes('Podcast data saved')) {
    progress = 100;
    message = 'Processing completed successfully!';
  }
  
  // Extract audio duration information for better time estimation
  const durationMatch = output.match(/Audio duration: ([\d.]+) seconds \(([\d.]+) minutes\)/);
  if (durationMatch) {
    audioLength = parseFloat(durationMatch[1]);
    // Estimate total processing time based on audio length
    // Rule of thumb: 1 minute of audio = 2-3 minutes of processing
    estimatedDuration = audioLength * 2.5;
  }
  
  // Parse chunk splitting information
  const chunkMatch = output.match(/Splitting audio into (\d+) chunks/);
  if (chunkMatch) {
    const totalChunks = parseInt(chunkMatch[1]);
    progress = 18;
    message = `Splitting audio into ${totalChunks} chunks for processing...`;
  }
  
  // Only update progress if it's actually advancing (no artificial increments)
  if (progress > session.progress) {
    const sessionUpdate = {
      ...session,
      progress: Math.min(progress, 100),
      message,
      lastUpdate: Date.now(),
      estimatedDuration,
      audioLength
    };
    
    activeSessions.set(sessionId, sessionUpdate);
    broadcastProgress(sessionId, sessionUpdate);
  }
}

// Find the most recently generated files (within the last 5 minutes to avoid stale files)
async function findGeneratedFiles(outputDir, sessionStartTime) {
  processingLogger.debug('findGeneratedFiles called', { 
    sessionStartTime: new Date(sessionStartTime) 
  });
  
  const files = await fs.readdir(outputDir);
  processingLogger.debug('Found files in output directory', { 
    fileCount: files.length 
  });
  
  // Look for metadata files that indicate a completed podcast processing
  const metadataFiles = files.filter(file => file.endsWith('_metadata.json'));
  processingLogger.debug('Found metadata files', { 
    metadataFileCount: metadataFiles.length,
    metadataFiles: metadataFiles.join(', ') 
  });
  
  if (metadataFiles.length === 0) {
    throw new Error('No generated podcast metadata files found');
  }
  
  // Find metadata files created after session start (with 2 minute buffer for YouTube processing)
  const sessionStartBuffer = sessionStartTime - (2 * 60 * 1000); // 2 minute buffer for YouTube
  processingLogger.debug('Session start buffer calculated', { 
    sessionStartBuffer: new Date(sessionStartBuffer) 
  });
  
  let newestFile = null;
  let newestTime = 0;
  
  for (const file of metadataFiles) {
    const filePath = path.join(outputDir, file);
    const stat = await fs.stat(filePath);
    const fileTime = stat.mtime.getTime();
    
    processingLogger.debug('File timestamp check', { 
      file,
      fileTime: new Date(fileTime),
      isAfterBuffer: fileTime > sessionStartBuffer 
    });
    
    // Only consider files created after the session started
    if (fileTime > sessionStartBuffer && fileTime > newestTime) {
      newestTime = fileTime;
      newestFile = file;
      processingLogger.debug('New newest file found', { 
        file 
      });
    }
  }
  
  if (!newestFile) {
    processingLogger.error('No new generated files found for this session', {
      sessionStartTime: new Date(sessionStartTime),
      sessionStartBuffer: new Date(sessionStartBuffer)
    });
    const availableFiles = metadataFiles.map(f => {
      const filePath = path.join(outputDir, f);
      try {
        const stat = fs.statSync(filePath);
        return `${f} (${new Date(stat.mtime)})`;
      } catch (e) {
        return `${f} (error reading stat)`;
      }
    }).join(', ');
    
    processingLogger.error('Available metadata files', {
      availableFiles
    });
    throw new Error('No new generated files found for this session. Processing may have failed - check Python script output for errors.');
  }
  
  processingLogger.debug('Selected newest file', { 
    newestFile,
    newestTime: new Date(newestTime) 
  });
  
  // Read metadata
  const metadataPath = path.join(outputDir, newestFile);
  const metadata = await fs.readJson(metadataPath);
  
  // Extract the base name from the filename (remove _metadata.json)
  const baseName = newestFile.replace('_metadata.json', '');
  
  return {
    id: baseName,
    directory: outputDir, // Files are in the main directory, not subdirectory
    baseName: baseName,
    metadataFile: newestFile,
    ...metadata
  };
}

// Global error handling middleware
app.use((error, req, res, next) => {
  processingLogger.error('Global error', { 
    error: error.message,
    stack: error.stack 
  });
  
  // Handle JSON parsing errors specifically
  if (error instanceof SyntaxError && error.status === 400 && 'body' in error) {
    return res.status(400).json({ 
      error: 'Invalid JSON format',
      message: 'Please check your request body syntax'
    });
  }
  
  // Handle other errors
  res.status(500).json({ 
    error: 'Internal server error',
    message: 'Something went wrong processing your request'
  });
});

const PORT = process.env.PORT || 3001;

server.listen(PORT, () => {
  processingLogger.info('Server startup complete', { 
    port: PORT,
    websocketEnabled: true,
    mode: process.env.NODE_ENV || 'development',
    servingFrom: process.env.NODE_ENV === 'production' ? '/dist' : 'development' 
  });
});