deduplicate-questions.js

require('dotenv').config();
const fs = require('fs').promises;

function cosineSimilarity(vec1, vec2) {
  let dotProduct = 0;
  let norm1 = 0;
  let norm2 = 0;
  
  for (let i = 0; i < vec1.length; i++) {
    dotProduct += vec1[i] * vec2[i];
    norm1 += vec1[i] * vec1[i];
    norm2 += vec2[i] * vec2[i];
  }
  
  return dotProduct / (Math.sqrt(norm1) * Math.sqrt(norm2));
}

async function deduplicateQuestions() {
  const inputFile = 'questions-with-embeddings.json';
  const outputFile = 'question-groups-2.json';
  const similarityThreshold = 0.7;
  
  try {
    const fileContent = await fs.readFile(inputFile, 'utf8');
    const questions = JSON.parse(fileContent);
    
    console.log(`Total questions to process: ${questions.length}`);
    console.log(`Similarity threshold: ${(similarityThreshold * 100).toFixed(0)}%`);
    
    const questionGroups = [];
    const processedIds = new Set();
    
    for (let i = 0; i < questions.length; i++) {
      if (processedIds.has(questions[i].question_id)) {
        continue;
      }
      
      const group = {
        group_id: questions[i].question_id,
        questions: [{
          question_id: questions[i].question_id,
          question: questions[i].question,
          message_id: questions[i].message_id
        }]
      };
      
      processedIds.add(questions[i].question_id);
      
      for (let j = i + 1; j < questions.length; j++) {
        if (processedIds.has(questions[j].question_id)) {
          continue;
        }
        
        const similarity = cosineSimilarity(
          questions[i].embedding,
          questions[j].embedding
        );
        
        if (similarity >= similarityThreshold) {
          group.questions.push({
            question_id: questions[j].question_id,
            question: questions[j].question,
            message_id: questions[j].message_id
          });
          processedIds.add(questions[j].question_id);
        }
      }
      
      questionGroups.push(group);
      
      // Print group ID if group has more than 1 question
      if (group.questions.length > 1) {
        console.log(`Group ${group.group_id}: ${group.questions.length} similar questions`);
      }
      
      if ((i + 1) % 100 === 0) {
        console.log(`Processed ${i + 1} questions, found ${questionGroups.length} groups so far`);
        
        // Save progress
        await fs.writeFile(outputFile, JSON.stringify(questionGroups, null, 2));
      }
    }
    
    // Final save
    await fs.writeFile(outputFile, JSON.stringify(questionGroups, null, 2));
    
    // Calculate statistics
    const totalQuestions = questions.length;
    const totalGroups = questionGroups.length;
    const duplicatesFound = totalQuestions - totalGroups;
    const largestGroup = Math.max(...questionGroups.map(g => g.questions.length));
    const averageGroupSize = (totalQuestions / totalGroups).toFixed(2);
    
    console.log(`\n✓ Deduplication complete!`);
    console.log(`✓ Original questions: ${totalQuestions}`);
    console.log(`✓ Question groups: ${totalGroups}`);
    console.log(`✓ Duplicates found: ${duplicatesFound}`);
    console.log(`✓ Largest group: ${largestGroup} questions`);
    console.log(`✓ Average group size: ${averageGroupSize} questions`);
    console.log(`✓ Results saved to: ${outputFile}`);
    
  } catch (error) {
    console.error('Error:', error);
    process.exit(1);
  }
}

deduplicateQuestions();