test-crawl.js

#!/usr/bin/env node

import { execSync } from 'child_process';
import { writeFileSync, mkdirSync, existsSync, readFileSync } from 'fs';
import { join } from 'path';
import { fileURLToPath } from 'url';
import { dirname } from 'path';

const __filename = fileURLToPath(import.meta.url);
const __dirname = dirname(__filename);

const BASE_URL = 'https://app.workback.ai';
const ISSUES_DIR = join(__dirname, 'issues');
const extractScriptPath = join(__dirname, 'extract-issue.js');

function execAgentBrowser(command) {
  try {
    return execSync(`agent-browser ${command}`, { 
      encoding: 'utf-8',
      stdio: ['pipe', 'pipe', 'pipe']
    });
  } catch (error) {
    throw new Error(`agent-browser command failed: ${command}\n${error.message}`);
  }
}

function getCookieFile() {
  const out = execAgentBrowser('cookies --json');
  let json;
  try {
    const start = out.indexOf('{');
    json = JSON.parse(start >= 0 ? out.slice(start) : '{}');
  } catch {
    return null;
  }
  const cookies = json?.data?.cookies || json?.cookies || [];
  const workback = cookies.filter(
    c => c.domain === 'app.workback.ai' || (c.domain?.endsWith && c.domain.endsWith('.workback.ai'))
  );
  if (workback.length === 0) return null;
  const lines = [
    '# Netscape HTTP Cookie File',
    ...workback.map(c => {
      const domain = (c.domain?.startsWith('.') ? '' : '.') + (c.domain || 'app.workback.ai');
      const path = c.path || '/';
      const secure = c.secure ? 'TRUE' : 'FALSE';
      const expires = Math.floor((c.expires > 0 ? c.expires : Date.now() / 1000 + 86400));
      return `${domain}\tTRUE\t${path}\t${secure}\t${expires}\t${c.name}\t${c.value}`;
    })
  ];
  const cookieFile = join(ISSUES_DIR, '.workback-cookies.txt');
  writeFileSync(cookieFile, lines.join('\n'));
  return cookieFile;
}

function downloadImage(imageUrl, outputPath) {
  try {
    const cookieFile = getCookieFile();
    if (!cookieFile) {
      console.error('No Workback.ai cookies - ensure browser is connected and logged in');
      return false;
    }
    execSync(`curl -s -L -b "${cookieFile}" "${imageUrl}" -o "${outputPath}"`, { encoding: 'utf-8' });
    const buf = readFileSync(outputPath, { start: 0, end: 99 });
    if (buf.toString('utf8').trimStart().startsWith('<')) {
      console.error('Got HTML instead of image - session may have expired');
      return false;
    }
    return true;
  } catch (error) {
    console.error(`Failed to download image ${imageUrl}: ${error.message}`);
    return false;
  }
}

// Test with issue 4226
const issueId = 4226;
const issueUrl = `${BASE_URL}/issues/${issueId}/`;

console.log(`Testing crawl for issue ${issueId}...`);

// Connect to browser
execAgentBrowser('connect 9222');

// Navigate to issue page
console.log('Navigating to issue page...');
execAgentBrowser(`open ${issueUrl}`);

// Wait for page to load
execAgentBrowser('wait --load networkidle');
execAgentBrowser('wait 2000');

// Extract content
console.log('Extracting content...');
const contentJson = execAgentBrowser(`eval "$(cat "${extractScriptPath}")"`);
// Handle double-encoded JSON (sometimes agent-browser wraps in quotes)
let jsonStr = contentJson.trim();
if (jsonStr.startsWith('"') && jsonStr.endsWith('"')) {
  jsonStr = JSON.parse(jsonStr); // Unwrap outer quotes
}
const content = JSON.parse(jsonStr);

console.log('Extracted:', JSON.stringify(content, null, 2));

// Create directory
const issueDir = join(ISSUES_DIR, issueId.toString());
mkdirSync(issueDir, { recursive: true });

// Download images
const imagePaths = [];
content.images.forEach((image) => {
  const imagePath = join(issueDir, image.name);
  if (downloadImage(image.url, imagePath)) {
    imagePaths.push({
      originalName: image.name,
      localPath: image.name,
      alt: image.alt
    });
    console.log(`Downloaded: ${image.name}`);
  }
});

// Create markdown
const frontmatter = `---
id: ${issueId}
title: "${content.title.replace(/"/g, '\\"')}"
status: ${content.metadata.status}
stage: ${content.metadata.stage}
severity: ${content.metadata.severity}
wcag: ${content.metadata.wcag}
flow: "${content.metadata.flow.replace(/"/g, '\\"')}"
pageUrl: "${content.metadata.pageUrl.replace(/"/g, '\\"')}"
auditDate: "${content.metadata.auditDate}"
---

`;

let markdown = frontmatter + `# ${content.title}\n\n`;

markdown += `## Metadata\n\n`;
markdown += `- **WCAG**: ${content.metadata.wcag}\n`;
markdown += `- **Severity**: ${content.metadata.severity}\n`;
markdown += `- **Status**: ${content.metadata.status}\n`;
markdown += `- **Stage**: ${content.metadata.stage}\n`;
markdown += `- **Flow**: ${content.metadata.flow}\n`;
markdown += `- **Page URL**: ${content.metadata.pageUrl}\n`;
markdown += `- **Audit Date**: ${content.metadata.auditDate}\n\n`;

Object.entries(content.sections).forEach(([sectionName, sectionContent]) => {
  markdown += `## ${sectionName}\n\n`;
  markdown += sectionContent + '\n\n';
});

if (imagePaths.length > 0) {
  markdown += `## Images\n\n`;
  imagePaths.forEach(img => {
    markdown += `![${img.alt || img.originalName}](${img.localPath})\n\n`;
  });
}

const mdPath = join(issueDir, `issue-${issueId}.md`);
writeFileSync(mdPath, markdown);

console.log(`\n✓ Created ${mdPath}`);
console.log(`✓ Downloaded ${imagePaths.length} images`);