deduplicate.js

const fs = require('fs').promises;
const OpenAI = require('openai');

const openai = new OpenAI({
  apiKey: process.env.OPENAI_API_KEY
});

function cosineSimilarity(vecA, vecB) {
  let dotProduct = 0;
  let normA = 0;
  let normB = 0;

  for (let i = 0; i < vecA.length; i++) {
    dotProduct += vecA[i] * vecB[i];
    normA += vecA[i] * vecA[i];
    normB += vecB[i] * vecB[i];
  }

  return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
}

async function findDuplicatesWithEmbeddings() {
  // Load your questions
  const questions = JSON.parse(
    await fs.readFile('extracted_questions.json', 'utf8')
  );

  console.log(`Processing ${questions.length} questions...`);

  // Step 1: Generate embeddings in batches (API allows up to 2048 inputs per request)
  const embeddings = [];
  const batchSize = 100; // Conservative batch size

  for (let i = 0; i < questions.length; i += batchSize) {
    const batch = questions.slice(i, Math.min(i + batchSize, questions.length));
    console.log(`Getting embeddings for questions ${i}-${i + batch.length}...`);

    try {
      const response = await openai.embeddings.create({
        model: 'text-embedding-3-small', // Cheaper and faster, still very good
        input: batch.map(q => q.question),
        encoding_format: 'float' // Default, returns as array of numbers
      });

      embeddings.push(...response.data.map(d => d.embedding));
    } catch (error) {
      console.error(`Error in batch ${i}:`, error);
      // Add retry logic here if needed
    }

    // Rate limiting
    await new Promise(resolve => setTimeout(resolve, 200));
  }

  // Step 2: Calculate similarity matrix and group similar questions
  const duplicateGroups = [];
  const processed = new Set();
  const SIMILARITY_THRESHOLD = 0.85; // Adjust based on your needs

  for (let i = 0; i < questions.length; i++) {
    if (processed.has(i)) continue;

    const group = {
      canonical: questions[i],
      duplicates: [],
      similarities: []
    };

    // Find all similar questions
    for (let j = i + 1; j < questions.length; j++) {
      if (processed.has(j)) continue;

      const similarity = cosineSimilarity(embeddings[i], embeddings[j]);

      if (similarity >= SIMILARITY_THRESHOLD) {
        group.duplicates.push(questions[j]);
        group.similarities.push({
          question: questions[j],
          similarity: similarity
        });
        processed.add(j);
      }
    }

    // Only add groups that have duplicates
    if (group.duplicates.length > 0) {
      processed.add(i);
      duplicateGroups.push(group);
    }
  }

  // Step 3: Optional - Have GPT confirm edge cases (0.85-0.90 similarity)
  const edgeCases = [];

  for (const group of duplicateGroups) {
    const uncertainDuplicates = group.similarities.filter(
      s => s.similarity >= 0.85 && s.similarity < 0.90
    );

    if (uncertainDuplicates.length > 0) {
      edgeCases.push({
        canonical: group.canonical,
        uncertain: uncertainDuplicates
      });
    }
  }

  // Verify edge cases with GPT
  if (edgeCases.length > 0) {
    console.log(`\nVerifying ${edgeCases.length} edge cases with GPT...`);

    for (const edgeCase of edgeCases) {
      const response = await openai.chat.completions.parse({
        model: 'gpt-4o-mini',
        messages: [{
          role: 'user',
          content: `Are these questions asking about the same thing?
          
Main question: "${edgeCase.canonical.question}"

Potential duplicates:
${edgeCase.uncertain.map((u, i) => `${i+1}. "${u.question.question}" (similarity: ${u.similarity.toFixed(3)})`).join('\n')}`
        }],
        response_format: {
          type: 'json_schema',
          json_schema: {
            name: 'verification',
            strict: true,
            schema: {
              type: 'object',
              properties: {
                confirmed_duplicates: {
                  type: 'array',
                  items: { type: 'integer' }
                }
              },
              required: ['confirmed_duplicates']
            }
          }
        }
      });

      // Update groups based on GPT confirmation
      const confirmed = response.choices[0].message.parsed.confirmed_duplicates;
      // ... handle the confirmed duplicates
    }
  }

  // Step 4: Save results
  const output = {
    total_questions: questions.length,
    duplicate_groups: duplicateGroups.length,
    groups: duplicateGroups.map(g => ({
      canonical: g.canonical,
      duplicates: g.duplicates,
      all_ids: [g.canonical.message_id, ...g.duplicates.map(d => d.message_id)]
    }))
  };

  await fs.writeFile(
    'deduplicated_questions.json',
    JSON.stringify(output, null, 2)
  );

  console.log(`\nFound ${duplicateGroups.length} groups of duplicates`);
  console.log(`Reduced ${questions.length} questions to ${questions.length - processed.size} unique questions`);

  return output;
}

// Run it
findDuplicatesWithEmbeddings().catch(console.error);