✨ Fix sitemap index handling and add interactive progress

Sitemap:
- Fix sitemap index files returning child sitemap URLs as pages
- Now follows sitemap index to parse actual child sitemaps
- Extracts real page URLs instead of sitemap-0.xml etc.

Progress display:
- Interactive terminals: single updating progress line with ETA
- Non-interactive (CI): traditional per-task logging
- Shows completion summary with total time and error count

Author: Robdel12

clients/static-site/package-lock.json

@@ -128,6 +128,26 @@ function formatDuration(ms) {
   return `${seconds}s`;
 }
 
+/**
+ * Check if stdout is an interactive TTY
+ * @returns {boolean}
+ */
+function isInteractiveTTY() {
+  return process.stdout.isTTY && !process.env.CI;
+}
+
+/**
+ * Clear current line and write new content (for TTY)
+ * @param {string} text - Text to write
+ */
+function writeProgress(text) {
+  if (isInteractiveTTY()) {
+    process.stdout.clearLine(0);
+    process.stdout.cursorTo(0);
+    process.stdout.write(text);
+  }
+}
+
 /**
  * Process all tasks through the tab pool
  * @param {Array<Object>} tasks - Array of task objects
@@ -143,6 +163,7 @@ export async function processAllTasks(tasks, pool, config, logger, deps = {}) {
   let total = tasks.length;
   let startTime = Date.now();
   let taskTimes = [];
+  let interactive = isInteractiveTTY();
 
   // Merge deps for processTask
   let taskDeps = { ...defaultDeps, ...deps };
@@ -176,20 +197,35 @@ export async function processAllTasks(tasks, pool, config, logger, deps = {}) {
         let remaining = total - completed;
         // Divide by concurrency since tasks run in parallel
         let etaMs = (remaining * avgTime) / config.concurrency;
-        let eta = remaining > 0 ? ` ~${formatDuration(etaMs)} remaining` : '';
-
-        logger.info(
-          `   ✓ [${completed}/${total}] ${task.page.path}@${task.viewport.name}${eta}`
-        );
+        let eta = remaining > 0 ? `~${formatDuration(etaMs)} remaining` : '';
+        let percent = Math.round((completed / total) * 100);
+
+        if (interactive) {
+          // Update single progress line
+          writeProgress(
+            `   📸 [${completed}/${total}] ${percent}% ${eta} - ${task.page.path}@${task.viewport.name}`
+          );
+        } else {
+          // Non-interactive: log each completion
+          logger.info(
+            `   ✓ [${completed}/${total}] ${task.page.path}@${task.viewport.name} ${eta}`
+          );
+        }
       } catch (error) {
         completed++;
         errors.push({
           page: task.page.path,
           viewport: task.viewport.name,
           error: error.message,
         });
+
+        if (interactive) {
+          // Clear progress line and log error
+          writeProgress('');
+          process.stdout.write('\n');
+        }
         logger.error(
-          `   ✗ [${completed}/${total}] ${task.page.path}@${task.viewport.name}: ${error.message}`
+          `   ✗ ${task.page.path}@${task.viewport.name}: ${error.message}`
         );
       } finally {
         pool.release(tab);
@@ -198,9 +234,19 @@ export async function processAllTasks(tasks, pool, config, logger, deps = {}) {
     config.concurrency
   );
 
+  // Clear progress line and show completion
+  if (interactive) {
+    writeProgress('');
+    process.stdout.write('\n');
+  }
+
   // Log total time
   let totalTime = Date.now() - startTime;
-  logger.info(`   Completed in ${formatDuration(totalTime)}`);
+  logger.info(`   ✅ Completed ${total} screenshots in ${formatDuration(totalTime)}`);
+
+  if (errors.length > 0) {
+    logger.warn(`   ⚠️  ${errors.length} failed`);
+  }
 
   return errors;
 }

clients/static-site/src/tasks.js

@@ -9,13 +9,40 @@ import { XMLParser } from 'fast-xml-parser';
 
 /**
  * Parse sitemap XML file and extract URLs
+ * Follows sitemap index files to get all page URLs
  * @param {string} sitemapPath - Absolute path to sitemap.xml file
- * @returns {Promise<Array<string>>} Array of URLs from sitemap
+ * @returns {Promise<Array<string>>} Array of page URLs from sitemap
  */
 export async function parseSitemapFile(sitemapPath) {
+  let { dirname } = await import('node:path');
+  let { existsSync } = await import('node:fs');
+
   try {
     let content = await readFile(sitemapPath, 'utf-8');
-    return parseSitemapXML(content);
+    let { urls, childSitemaps } = parseSitemapXML(content);
+
+    // If this is a sitemap index, follow child sitemaps
+    if (childSitemaps.length > 0) {
+      let baseDir = dirname(sitemapPath);
+
+      for (let childUrl of childSitemaps) {
+        // Extract filename from URL (e.g., "sitemap-0.xml" from "https://example.com/sitemap-0.xml")
+        let filename = childUrl.split('/').pop();
+        let childPath = join(baseDir, filename);
+
+        if (existsSync(childPath)) {
+          try {
+            let childContent = await readFile(childPath, 'utf-8');
+            let childResult = parseSitemapXML(childContent);
+            urls.push(...childResult.urls);
+          } catch {
+            // Skip unreadable child sitemaps
+          }
+        }
+      }
+    }
+
+    return urls;
   } catch (error) {
     throw new Error(
       `Failed to read sitemap at ${sitemapPath}: ${error.message}`
@@ -26,7 +53,7 @@ export async function parseSitemapFile(sitemapPath) {
 /**
  * Parse sitemap XML content and extract URLs
  * @param {string} xmlContent - Sitemap XML content
- * @returns {Array<string>} Array of URLs from sitemap
+ * @returns {{ urls: Array<string>, childSitemaps: Array<string> }} URLs and child sitemap URLs
  */
 export function parseSitemapXML(xmlContent) {
   let parser = new XMLParser({
@@ -41,18 +68,24 @@ export function parseSitemapXML(xmlContent) {
     let urls = Array.isArray(result.urlset.url)
       ? result.urlset.url
       : [result.urlset.url];
-    return urls.map(entry => entry.loc).filter(Boolean);
+    return {
+      urls: urls.map(entry => entry.loc).filter(Boolean),
+      childSitemaps: [],
+    };
   }
 
   // Handle sitemap index format (sitemap of sitemaps)
   if (result.sitemapindex?.sitemap) {
     let sitemaps = Array.isArray(result.sitemapindex.sitemap)
       ? result.sitemapindex.sitemap
       : [result.sitemapindex.sitemap];
-    return sitemaps.map(entry => entry.loc).filter(Boolean);
+    return {
+      urls: [],
+      childSitemaps: sitemaps.map(entry => entry.loc).filter(Boolean),
+    };
   }
 
-  return [];
+  return { urls: [], childSitemaps: [] };
 }
 
 /**

clients/static-site/src/utils/sitemap.js

@@ -215,6 +215,7 @@ describe('processAllTasks', () => {
     let logger = {
       info: mock.fn(),
       error: mock.fn(),
+      warn: mock.fn(),
     };
 
     let deps = {
@@ -258,6 +259,7 @@ describe('processAllTasks', () => {
     let logger = {
       info: mock.fn(),
       error: mock.fn(),
+      warn: mock.fn(),
     };
 
     let deps = {