crawl-issues.js

#!/usr/bin/env node
/**
 * Crawl Workback issues and create markdown dossiers in issues/<id>/issue-<id>.md
 *
 * Uses curl + cheerio (no browser required). Pass cookies via --cookie flag.
 *
 * Usage:
 *   node crawl-issues.js --cookie "csrftoken=...; sessionid=..."
 *   node crawl-issues.js --from-csv --missing-only --cookie "csrftoken=...; sessionid=..."
 *   node crawl-issues.js --ids-file /tmp/workback-live-ids-uniq.txt --missing-only --cookie "csrftoken=...; sessionid=..."
 *   node crawl-issues.js --discover-new --discover-pages 8 --cookie "csrftoken=...; sessionid=..."
 *   node crawl-issues.js --from-csv --missing-only --dry-run
 *   node crawl-issues.js --fix-images --cookie "csrftoken=...; sessionid=..."
 */

import { execSync } from 'child_process';
import { writeFileSync, mkdirSync, existsSync, readdirSync, readFileSync } from 'fs';
import { join } from 'path';
import { fileURLToPath } from 'url';
import { dirname } from 'path';
import { load as cheerioLoad } from 'cheerio';

const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);

const BASE_URL = 'https://app.workback.ai';
const ISSUES_DIR = join(__dirname, 'issues');
const PROGRESS_FILE = join(ISSUES_DIR, 'crawl-progress.json');
const ERROR_LOG = join(ISSUES_DIR, 'crawl-errors.log');
const EXPORT_CSV = join(__dirname, 'export.csv');
const DEFAULT_DISCOVER_PAGES = 8;
const DEFAULT_STOP_AFTER_STALE_PAGES = 2;

const ISSUE_IDS = [
  4226, 4221, 4215, 4208, 4204, 4194, 4189, 4185, 4180, 4171, 4164, 4161, 4144,
  3938, 3935, 3934, 3933, 3932, 3931, 3930, 3929, 3928, 3927, 3919, 3917, 3909,
  3904, 3902, 3897, 3896, 3885, 3883, 3878, 3868, 3867, 3866, 3853, 3852, 3850,
  3834, 3828, 3748, 3744, 3742, 3739, 3737, 3731, 3727, 3717, 3716, 3715, 3712,
  3701, 3697, 3687, 3682, 3676, 3668, 3650, 3642, 3640, 3634, 3631, 3628, 3627,
  3626, 3621, 3620, 3619, 3618, 3617, 3615, 3614, 3598, 3590, 3586, 3585, 3582,
  3575, 3562, 3561, 3558, 3557, 3556, 3550, 3548, 3545, 3514, 3505, 3490, 3485,
  3479, 3460, 3450, 3448, 3447, 3443, 3440, 3429, 3427, 3421, 3420, 3410, 3407
];

const CURL_HEADERS = [
  '-H', 'accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8',
  '-H', 'accept-language: en,ru;q=0.9',
  '-H', 'referer: https://app.workback.ai/issues/',
  '-H', 'sec-ch-ua: "Not(A:Brand";v="8", "Chromium";v="144", "Google Chrome";v="144"',
  '-H', 'sec-ch-ua-mobile: ?0',
  '-H', 'sec-ch-ua-platform: "macOS"',
  '-H', 'sec-fetch-dest: document',
  '-H', 'sec-fetch-mode: navigate',
  '-H', 'sec-fetch-site: same-origin',
  '-H', 'sec-fetch-user: ?1',
  '-H', 'upgrade-insecure-requests: 1',
  '-H', 'user-agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36',
];

// --- CLI args ---

function getCookieArg() {
  const idx = process.argv.indexOf('--cookie');
  if (idx === -1 || idx + 1 >= process.argv.length) return null;
  return process.argv[idx + 1];
}

function getIdsFileArg() {
  const idx = process.argv.indexOf('--ids-file');
  if (idx === -1 || idx + 1 >= process.argv.length) return null;
  return process.argv[idx + 1];
}

function getIntArg(flagName, defaultValue) {
  const idx = process.argv.indexOf(flagName);
  if (idx === -1 || idx + 1 >= process.argv.length) return defaultValue;
  const parsed = Number.parseInt(process.argv[idx + 1], 10);
  if (!Number.isInteger(parsed) || parsed <= 0) {
    throw new Error(`${flagName} must be a positive integer`);
  }
  return parsed;
}

// --- ID helpers ---

function getIdsFromCsv() {
  const csv = readFileSync(EXPORT_CSV, 'utf8');
  const lines = csv.split('\n').slice(2);
  return lines
    .map((line) => line.match(/^"(\d+)"/)?.[1])
    .filter(Boolean)
    .map(Number);
}

function getIdsFromFile(filePath) {
  const content = readFileSync(filePath, 'utf8');
  return [...new Set(
    content
      .split(/\r?\n/)
      .map((line) => line.trim())
      .filter(Boolean)
      .map((line) => line.match(/\d+/)?.[0])
      .filter(Boolean)
      .map(Number)
  )];
}

function getExistingIds() {
  if (!existsSync(ISSUES_DIR)) return [];
  return readdirSync(ISSUES_DIR)
    .filter((d) => /^\d+$/.test(d))
    .filter((d) => existsSync(join(ISSUES_DIR, d, `issue-${d}.md`)))
    .map(Number);
}

function getIdsToCrawl(fromCsv, idsFile, missingOnly) {
  let ids = ISSUE_IDS;
  if (idsFile) {
    ids = getIdsFromFile(idsFile);
  } else if (fromCsv) {
    ids = getIdsFromCsv();
  }
  if (missingOnly) {
    const existing = getExistingIds();
    ids = ids.filter((id) => !existing.includes(id));
  }
  return ids;
}

function getIssueIdsFromApi(cookie) {
  const apiUrl = `${BASE_URL}/api/org/wcag-issues`;
  const responseText = curlFetch(apiUrl, cookie);

  let payload;
  try {
    payload = JSON.parse(responseText);
  } catch {
    throw new Error('Could not parse /api/org/wcag-issues response as JSON.');
  }

  if (!Array.isArray(payload?.data)) {
    throw new Error('Unexpected /api/org/wcag-issues payload shape (missing data array).');
  }

  const ids = payload.data
    .map((issue) => Number.parseInt(issue?.id, 10))
    .filter((id) => Number.isInteger(id));

  if (ids.length === 0) {
    throw new Error('No issue IDs returned by /api/org/wcag-issues.');
  }

  return [...new Set(ids)];
}

function getIssueIdsFromHtml(html) {
  const ids = [];
  const seen = new Set();
  const issueHrefRegex = /\/issues\/(\d+)\/?/g;
  let match;

  while ((match = issueHrefRegex.exec(html)) !== null) {
    const id = Number.parseInt(match[1], 10);
    if (!Number.isInteger(id) || seen.has(id)) continue;
    seen.add(id);
    ids.push(id);
  }

  return ids;
}

function getIssueListPageUrl(pageNumber) {
  return pageNumber === 1
    ? `${BASE_URL}/issues/`
    : `${BASE_URL}/issues/?page=${pageNumber}`;
}

function discoverNewIssueIds(cookie, maxPages, stopAfterStalePages) {
  const existingIds = new Set(getExistingIds());

  try {
    const apiIds = getIssueIdsFromApi(cookie);
    const newFromApi = apiIds
      .filter((id) => !existingIds.has(id))
      .sort((a, b) => b - a);

    console.log(
      `Discovered ${newFromApi.length} new issue(s) from API (${apiIds.length} total issue(s) returned).`
    );
    return newFromApi;
  } catch (error) {
    console.warn(`API discovery failed, falling back to HTML page scan: ${error.message}`);
  }

  const discoveredNewIds = [];
  const seenInDiscovery = new Set();
  let stalePages = 0;
  let lastPageFingerprint = null;

  console.log(
    `Discovering new issues from Workback (max pages: ${maxPages}, stop after ${stopAfterStalePages} stale page(s))...`
  );

  for (let page = 1; page <= maxPages; page++) {
    const pageUrl = getIssueListPageUrl(page);
    const html = curlFetch(pageUrl, cookie);
    const pageIds = getIssueIdsFromHtml(html);

    if (page === 1 && pageIds.length === 0) {
      throw new Error('No issue links found on /issues/. Session may have expired or access is denied.');
    }

    if (pageIds.length === 0) {
      console.log(`  Page ${page}: no issue links found, stopping.`);
      break;
    }

    const pageFingerprint = pageIds.slice(0, 30).join(',');
    if (lastPageFingerprint && pageFingerprint === lastPageFingerprint) {
      console.log(`  Page ${page}: same content as previous page, stopping.`);
      break;
    }
    lastPageFingerprint = pageFingerprint;

    const newOnPage = [];
    for (const id of pageIds) {
      if (seenInDiscovery.has(id)) continue;
      seenInDiscovery.add(id);

      if (!existingIds.has(id)) {
        newOnPage.push(id);
      }
    }

    discoveredNewIds.push(...newOnPage);
    console.log(`  Page ${page}: found ${pageIds.length} issue link(s), ${newOnPage.length} new.`);

    if (newOnPage.length === 0) {
      stalePages += 1;
    } else {
      stalePages = 0;
    }

    if (stalePages >= stopAfterStalePages) {
      console.log(`  Stopping after ${stalePages} stale page(s).`);
      break;
    }
  }

  const deduped = [...new Set(discoveredNewIds)].sort((a, b) => b - a);
  console.log(`Discovered ${deduped.length} new issue(s).`);
  return deduped;
}

// --- Logging & progress ---

function logError(message) {
  const timestamp = new Date().toISOString();
  writeFileSync(ERROR_LOG, `[${timestamp}] ${message}\n`, { flag: 'a' });
  console.error(`ERROR: ${message}`);
}

function loadProgress() {
  if (existsSync(PROGRESS_FILE)) {
    try {
      return JSON.parse(readFileSync(PROGRESS_FILE, 'utf8'));
    } catch {
      return { completed: [], failed: [], current: null };
    }
  }
  return { completed: [], failed: [], current: null };
}

function saveProgress(progress) {
  writeFileSync(PROGRESS_FILE, JSON.stringify(progress, null, 2));
}

// --- curl helpers ---

function curlFetch(url, cookie) {
  const args = [
    'curl', '-s', '-L',
    '-b', cookie,
    ...CURL_HEADERS,
    url,
  ];
  const result = execSync(args.map(a => `'${a.replace(/'/g, "'\\''")}'`).join(' '), {
    encoding: 'utf-8',
    maxBuffer: 50 * 1024 * 1024,
    shell: true,
  });
  return result;
}

function curlDownload(url, outputPath, cookie) {
  const args = [
    'curl', '-s', '-L',
    '-b', cookie,
    ...CURL_HEADERS,
    '-o', outputPath,
    url,
  ];
  execSync(args.map(a => `'${a.replace(/'/g, "'\\''")}'`).join(' '), {
    encoding: 'utf-8',
    maxBuffer: 50 * 1024 * 1024,
    shell: true,
  });
}

// --- HTML extraction with cheerio ---

function extractIssueContent(issueId, cookie) {
  console.log(`\nExtracting issue ${issueId}...`);

  const issueUrl = `${BASE_URL}/issues/${issueId}/`;
  const html = curlFetch(issueUrl, cookie);

  if (!html || html.length < 500) {
    throw new Error(`Empty or too-short response for issue ${issueId} (${html.length} bytes). Session may have expired.`);
  }

  const $ = cheerioLoad(html);

  const title = $('h1').first().text().trim();
  if (!title) {
    throw new Error(`No <h1> found for issue ${issueId}. Page may be a login redirect.`);
  }

  // Extract metadata from .meta-item elements
  const metadata = {
    wcag: '',
    severity: '',
    status: '',
    stage: '',
    flow: '',
    pageUrl: '',
    auditDate: '',
  };

  $('.meta-item').each((_, el) => {
    const label = $(el).find('.meta-label').text().replace(':', '').trim();
    // Get text value, stripping SVGs and extra whitespace
    const valueEl = $(el).children().not('.meta-label').first();
    let value;
    if (valueEl.is('a')) {
      value = valueEl.attr('href') || valueEl.text().trim();
    } else {
      value = valueEl.text().trim();
    }

    const key = label.toLowerCase().replace(/\s+/g, '');
    if (key === 'wcag') metadata.wcag = value;
    else if (key === 'severity') metadata.severity = value;
    else if (key === 'status') metadata.status = value;
    else if (key === 'stage') metadata.stage = value;
    else if (key === 'flow') metadata.flow = value;
    else if (key === 'pageurl') metadata.pageUrl = value;
    else if (key === 'auditdate') metadata.auditDate = value;
  });

  // Extract sections by h2 headings
  const sections = {};
  $('h2').each((_, h2El) => {
    const sectionName = $(h2El).text().trim();
    if (!sectionName || sectionName === 'Activity') return;

    if (sectionName === 'Audit Evidence') {
      sections[sectionName] = 'See images section below.';
      return;
    }

    let content = '';
    let next = $(h2El).next();

    while (next.length && !next.is('h2') && !next.is('h1')) {
      const tag = next.prop('tagName')?.toUpperCase();
      if (tag === 'P') {
        content += next.text().trim() + '\n\n';
      } else if (tag === 'UL' || tag === 'OL') {
        next.find('li').each((_, li) => {
          content += '- ' + $(li).text().trim() + '\n';
        });
        content += '\n';
      } else if (tag === 'PRE') {
        content += '\n```\n' + next.text() + '\n```\n\n';
      } else if (tag === 'CODE') {
        content += '`' + next.text() + '` ';
      } else if (next.text().trim()) {
        content += next.text().trim() + '\n\n';
      }
      next = next.next();
    }

    if (content.trim()) {
      sections[sectionName] = content.trim();
    }
  });

  // Extract images (links to /api/gcs or /api/org/gcs)
  const images = [];
  $('a[href*="/api/"]').each((_, linkEl) => {
    const href = $(linkEl).attr('href');
    if (!href || (!href.includes('/gcs') && !href.includes('/api/org/'))) return;

    const img = $(linkEl).find('img');
    let imageName = $(linkEl).text().trim();
    if (!imageName && img.length) {
      imageName = img.attr('alt') || '';
    }
    if (!imageName) {
      const urlMatch = href.match(/evidence%2F([^&]+)/);
      imageName = urlMatch ? decodeURIComponent(urlMatch[1]) : `image-${images.length + 1}.png`;
    }

    const extMatch = imageName.match(/\.(png|jpg|jpeg|gif|svg|webp)$/i);
    const extension = extMatch ? extMatch[0] : '.png';
    imageName = imageName.replace(/\.[^.]+$/, '') + extension;
    imageName = imageName.replace(/[^a-zA-Z0-9.-]/g, '_');

    let imageUrl = href;
    if (!imageUrl.startsWith('http')) {
      imageUrl = BASE_URL + imageUrl;
    }

    images.push({
      name: imageName,
      url: imageUrl,
      alt: img.length ? (img.attr('alt') || '') : '',
    });
  });

  return { title, metadata, sections, images };
}

// --- Image handling ---

function isBrokenImage(filePath) {
  if (!existsSync(filePath)) return false;
  const buf = readFileSync(filePath);
  const start = buf.toString('utf8', 0, 100).trimStart();
  return start.startsWith('<') || start.startsWith('<!');
}

function downloadImage(imageUrl, outputPath, cookie) {
  try {
    curlDownload(imageUrl, outputPath, cookie);

    const buf = readFileSync(outputPath);
    const start = buf.toString('utf8', 0, 100).trimStart();
    if (start.startsWith('<') || start.startsWith('<!')) {
      logError(`Got HTML instead of image for ${imageUrl} - session may have expired`);
      return false;
    }
    return true;
  } catch (error) {
    logError(`Failed to download image ${imageUrl}: ${error.message}`);
    return false;
  }
}

function downloadImagesForIssue(issueId, content, cookie) {
  const issueDir = join(ISSUES_DIR, issueId.toString());
  const results = [];
  content.images.forEach((image) => {
    const safeName = image.name.replace(/[^a-zA-Z0-9.-]/g, '_');
    const imagePath = join(issueDir, safeName);
    if (downloadImage(image.url, imagePath, cookie)) {
      results.push({ originalName: image.name, localPath: safeName, alt: image.alt });
    }
  });
  return results;
}

// --- Markdown generation ---

function createMarkdownFile(issueId, content, cookie) {
  const issueDir = join(ISSUES_DIR, issueId.toString());
  mkdirSync(issueDir, { recursive: true });

  const imagePaths = downloadImagesForIssue(issueId, content, cookie);

  const frontmatter = `---
id: ${issueId}
title: "${content.title.replace(/"/g, '\\"')}"
status: ${content.metadata.status}
stage: ${content.metadata.stage}
severity: ${content.metadata.severity}
wcag: ${content.metadata.wcag}
flow: "${content.metadata.flow.replace(/"/g, '\\"')}"
pageUrl: "${content.metadata.pageUrl.replace(/"/g, '\\"')}"
auditDate: "${content.metadata.auditDate}"
---

`;

  let markdown = frontmatter + `# ${content.title}\n\n`;

  markdown += `## Metadata\n\n`;
  markdown += `- **WCAG**: ${content.metadata.wcag}\n`;
  markdown += `- **Severity**: ${content.metadata.severity}\n`;
  markdown += `- **Status**: ${content.metadata.status}\n`;
  markdown += `- **Stage**: ${content.metadata.stage}\n`;
  markdown += `- **Flow**: ${content.metadata.flow}\n`;
  markdown += `- **Page URL**: ${content.metadata.pageUrl}\n`;
  markdown += `- **Audit Date**: ${content.metadata.auditDate}\n\n`;

  Object.entries(content.sections).forEach(([sectionName, sectionContent]) => {
    markdown += `## ${sectionName}\n\n`;
    markdown += sectionContent + '\n\n';
  });

  if (imagePaths.length > 0) {
    markdown += `## Images\n\n`;
    imagePaths.forEach(img => {
      markdown += `![${img.alt || img.originalName}](${img.localPath})\n\n`;
    });
  }

  const mdPath = join(issueDir, `issue-${issueId}.md`);
  writeFileSync(mdPath, markdown);

  console.log(`  Created ${mdPath}`);
  return mdPath;
}

// --- Main flows ---

async function crawlIssues(issueIds, cookie) {
  mkdirSync(ISSUES_DIR, { recursive: true });

  const progress = loadProgress();
  const remaining = issueIds.filter(id => !progress.completed.includes(id));

  console.log(`Total issues to crawl: ${issueIds.length}`);
  console.log(`Already completed (this run): ${issueIds.length - remaining.length}`);
  console.log(`Failed: ${progress.failed.length}`);
  console.log(`Remaining: ${remaining.length}\n`);

  for (const issueId of remaining) {
    try {
      progress.current = issueId;
      saveProgress(progress);

      const content = extractIssueContent(issueId, cookie);
      createMarkdownFile(issueId, content, cookie);

      progress.completed.push(issueId);
      progress.current = null;
      saveProgress(progress);

      await new Promise(resolve => setTimeout(resolve, 500));
    } catch (error) {
      progress.failed.push({ id: issueId, error: error.message });
      progress.current = null;
      saveProgress(progress);
      console.error(`Failed to process issue ${issueId}`);
    }
  }

  console.log('\n=== Crawl Complete ===');
  console.log(`Completed: ${progress.completed.length}/${issueIds.length}`);
  console.log(`Failed: ${progress.failed.length}`);
  if (progress.failed.length > 0) {
    console.log(`\nFailed issues: ${progress.failed.map(f => f.id).join(', ')}`);
  }
}

async function fixImages(cookie) {
  mkdirSync(ISSUES_DIR, { recursive: true });

  const dirs = readdirSync(ISSUES_DIR, { withFileTypes: true })
    .filter(d => d.isDirectory() && /^\d+$/.test(d.name))
    .map(d => ({ id: parseInt(d.name, 10), path: join(ISSUES_DIR, d.name) }))
    .filter(d => existsSync(join(d.path, `issue-${d.id}.md`)));

  let fixed = 0;
  for (const { id } of dirs) {
    const issueDir = join(ISSUES_DIR, id.toString());
    const imageFiles = readdirSync(issueDir)
      .filter(f => /\.(jpeg|jpg|png|gif|webp)$/i.test(f))
      .map(f => join(issueDir, f));
    const broken = imageFiles.filter(isBrokenImage);
    if (broken.length === 0) continue;

    console.log(`\nFixing ${broken.length} broken image(s) for issue ${id}...`);
    try {
      const content = extractIssueContent(id, cookie);
      downloadImagesForIssue(id, content, cookie);
      fixed++;
      await new Promise(r => setTimeout(r, 500));
    } catch (error) {
      logError(`Failed to fix images for issue ${id}: ${error.message}`);
    }
  }
  console.log(`\nFixed images for ${fixed} issue(s).`);
}

// --- Entry point ---

const fixMode = process.argv.includes('--fix-images');
const fromCsv = process.argv.includes('--from-csv');
const idsFile = getIdsFileArg();
const missingOnly = process.argv.includes('--missing-only');
const dryRun = process.argv.includes('--dry-run');
const discoverNew = process.argv.includes('--discover-new');
const cookie = getCookieArg();

function printCookieUsage() {
  console.error('Missing --cookie flag. Usage:');
  console.error('  node crawl-issues.js --cookie "csrftoken=...; sessionid=..."');
  console.error('  node crawl-issues.js --discover-new --cookie "csrftoken=...; sessionid=..."');
  console.error('');
  console.error('Get cookies from browser: open app.workback.ai, DevTools → Application → Cookies,');
  console.error('or copy from a curl command (Network tab → Copy as cURL).');
}

async function main() {
  if (discoverNew && (fromCsv || idsFile)) {
    console.warn('Warning: --discover-new ignores --from-csv and --ids-file.');
  }

  if (fixMode) {
    if (!cookie) {
      printCookieUsage();
      process.exit(1);
    }
    await fixImages(cookie);
    return;
  }

  let issueIds;
  if (discoverNew) {
    if (!cookie) {
      printCookieUsage();
      process.exit(1);
    }
    const discoverPages = getIntArg('--discover-pages', DEFAULT_DISCOVER_PAGES);
    const stopAfterStalePages = getIntArg('--discover-stop-after-stale-pages', DEFAULT_STOP_AFTER_STALE_PAGES);
    issueIds = discoverNewIssueIds(cookie, discoverPages, stopAfterStalePages);
  } else {
    issueIds = getIdsToCrawl(fromCsv, idsFile, missingOnly);
  }

  if (issueIds.length === 0) {
    console.log('No issues to crawl. All issues from the source already exist in issues/.');
    return;
  }

  if (dryRun) {
    console.log(`Would crawl ${issueIds.length} issue(s):`);
    console.log(issueIds.join(', '));
    return;
  }

  if (!cookie) {
    printCookieUsage();
    process.exit(1);
  }

  await crawlIssues(issueIds, cookie);
}

main().catch(error => {
  console.error('Fatal error:', error);
  process.exit(1);
});