matcher.py

"""
Keyword matching module for FAQ bot
Implements fuzzy matching with synonym support and scoring
"""
import json
import re
from typing import List, Dict, Optional, Tuple


class FAQMatcher:
    """FAQ matcher using keyword-based algorithm with scoring"""
    
    def __init__(self, faq_file_path: str = 'faq_data.json', threshold: float = 0.3):
        """
        Initialize the FAQ matcher
        
        Args:
            faq_file_path: Path to the FAQ JSON file
            threshold: Minimum confidence score for a match (0.0 to 1.0)
        """
        self.threshold = threshold
        self.faqs = self._load_faqs(faq_file_path)
        
        # Common word variations and synonyms
        self.synonyms = {
            'cost': ['price', 'fee', 'charge', 'expense', 'much'],
            'ship': ['deliver', 'send', 'dispatch', 'mail', 'delivery', 'shipping'],
            'buy': ['purchase', 'order', 'get', 'acquire'],
            'return': ['send back', 'refund', 'exchange'],
            'help': ['support', 'assist', 'service'],
            'broken': ['damaged', 'defective', 'faulty'],
            'pay': ['payment', 'checkout', 'billing', 'paid'],
            'fast': ['quick', 'express', 'rapid', 'speedy'],
            'slow': ['delayed', 'late', 'stuck'],
            'where': ['location', 'find', 'locate'],
            'when': ['time', 'date', 'schedule', 'arrive'],
            'how': ['way', 'method', 'process'],
            'authentic': ['genuine', 'real', 'legitimate', 'original'],
            'accept': ['take', 'use', 'support'],
            'method': ['option', 'way', 'type'],
        }
        
        # Common misspellings (simple version)
        self.typo_corrections = {
            'recieve': 'receive',
            'recieved': 'received',
            'warrenty': 'warranty',
            'garantee': 'guarantee',
            'shiping': 'shipping',
            'delivry': 'delivery',
        }
    
    def _load_faqs(self, faq_file_path: str) -> List[Dict]:
        """Load FAQ data from JSON file"""
        try:
            with open(faq_file_path, 'r', encoding='utf-8') as f:
                data = json.load(f)
                return data.get('faqs', [])
        except FileNotFoundError:
            print(f"Warning: FAQ file not found at {faq_file_path}")
            return []
        except json.JSONDecodeError as e:
            print(f"Error parsing FAQ JSON: {e}")
            return []
    
    def _normalize_text(self, text: str) -> str:
        """
        Normalize text for matching
        - Convert to lowercase
        - Correct common typos
        - Remove extra whitespace and punctuation
        """
        if not text:
            return ""
        
        # Convert to lowercase
        text = text.lower()
        
        # Correct common typos
        for typo, correction in self.typo_corrections.items():
            text = text.replace(typo, correction)
        
        # Remove punctuation and extra whitespace
        text = re.sub(r'[^\w\s]', ' ', text)
        text = re.sub(r'\s+', ' ', text).strip()
        
        return text
    
    def _get_word_variations(self, word: str) -> List[str]:
        """Get synonyms and variations of a word"""
        variations = [word]
        
        # Add synonyms
        for key, synonyms in self.synonyms.items():
            if word == key:
                variations.extend(synonyms)
            elif word in synonyms:
                variations.append(key)
                variations.extend([s for s in synonyms if s != word])
        
        return list(set(variations))
    
    def _calculate_match_score(self, user_message: str, faq: Dict) -> float:
        """
        Calculate match score between user message and FAQ
        Returns a score between 0.0 and 1.0
        """
        normalized_message = self._normalize_text(user_message)
        message_words = set(normalized_message.split())
        
        if not message_words:
            return 0.0
        
        # Get keywords from FAQ
        faq_keywords = set(k.lower() for k in faq.get('keywords', []))
        
        # Also consider words from the question itself
        normalized_question = self._normalize_text(faq.get('question', ''))
        question_words = set(normalized_question.split())
        
        # Remove common stop words from question words
        stop_words = {'the', 'a', 'an', 'is', 'are', 'was', 'were', 'be', 'been', 'being', 
                     'have', 'has', 'had', 'do', 'does', 'did', 'will', 'would', 'should',
                     'can', 'could', 'may', 'might', 'must', 'shall', 'your', 'you', 'i',
                     'we', 'they', 'them', 'their', 'our', 'my', 'what', 'when', 'where',
                     'who', 'which', 'how'}
        question_words = question_words - stop_words
        
        # Combine keywords and filtered question words
        all_faq_words = faq_keywords.union(question_words)
        
        if not all_faq_words:
            return 0.0
        
        # Count matches considering synonyms
        matches = 0
        keyword_matches = 0
        for message_word in message_words:
            # Skip stop words in user message
            if message_word in stop_words:
                continue
                
            # Check direct match
            if message_word in all_faq_words:
                matches += 1.0
                # Track if this is a keyword match (not just question word)
                if message_word in faq_keywords:
                    keyword_matches += 1.0
            else:
                # Check synonym matches (with slightly lower weight)
                word_variations = self._get_word_variations(message_word)
                for variation in word_variations:
                    if variation in all_faq_words:
                        matches += 0.8
                        if variation in faq_keywords:
                            keyword_matches += 0.8
                        break
        
        # Calculate score
        # We use a combination of precision (matches/message_words) 
        # and recall (matches/faq_words)
        # Filter out stop words from message_words for score calculation
        filtered_message_words = [w for w in message_words if w not in stop_words]
        if not filtered_message_words:
            return 0.0
            
        precision = matches / len(filtered_message_words)
        recall = matches / len(all_faq_words)
        
        # F1 score (harmonic mean of precision and recall)
        if precision + recall == 0:
            return 0.0
        
        score = 2 * (precision * recall) / (precision + recall)
        
        # Boost score if there are keyword matches (not just question matches)
        if faq_keywords and keyword_matches > 0:
            keyword_ratio = keyword_matches / matches if matches > 0 else 0
            score *= (1.0 + 0.2 * keyword_ratio)  # Up to 20% boost for keyword matches
        
        return min(score, 1.0)  # Cap at 1.0
    
    def find_best_match(self, user_message: str) -> Tuple[Optional[Dict], float]:
        """
        Find the best matching FAQ for a user message
        
        Args:
            user_message: The user's question/message
            
        Returns:
            Tuple of (best_faq, confidence_score) or (None, 0.0) if no match
        """
        if not user_message or not self.faqs:
            return None, 0.0
        
        best_faq = None
        best_score = 0.0
        
        # Calculate scores for all FAQs
        for faq in self.faqs:
            score = self._calculate_match_score(user_message, faq)
            if score > best_score:
                best_score = score
                best_faq = faq
        
        # Only return if score meets threshold
        if best_score >= self.threshold:
            return best_faq, best_score
        
        return None, best_score
    
    def get_answer(self, user_message: str) -> str:
        """
        Get answer for a user message
        
        Args:
            user_message: The user's question/message
            
        Returns:
            Answer string (or fallback message if no match)
        """
        faq, score = self.find_best_match(user_message)
        
        if faq:
            return faq.get('answer', 'I found information but cannot retrieve the answer.')
        
        # Fallback response
        return (
            "Амар байна уу? KoreaBox.mn сайтын Ai туслах байна. Та сонирхсон асуултынхаа өмнөх дугаарыг бичнэ үү.\n" 
"1.Ямар сайтаас захиалга хийх вэ?\n"
"2.Хэзээ ирэх вэ? (Ирэх хугацаа)\n"
"3.Каргоны төлбөр хэд вэ?\n"
"4.Үйлчилгээний болон нэмэлт төлбөр байна уу?\n"
"5.Монголд ачаагаа хаанаас авах вэ?\n"
"6.Хүргэлтээр авч болох уу?\n"
"7.Төлбөрөө яаж төлөх вэ?\n"
"8.Би админд хандаж мэдэгдэл үлдээмээр байна.\n"
        )
    
    def get_multiple_matches(self, user_message: str, top_n: int = 3) -> List[Tuple[Dict, float]]:
        """
        Get top N matching FAQs
        
        Args:
            user_message: The user's question/message
            top_n: Number of top matches to return
            
        Returns:
            List of (faq, score) tuples sorted by score
        """
        if not user_message or not self.faqs:
            return []
        
        # Calculate scores for all FAQs
        scored_faqs = []
        for faq in self.faqs:
            score = self._calculate_match_score(user_message, faq)
            if score >= self.threshold:
                scored_faqs.append((faq, score))
        
        # Sort by score (highest first) and return top N
        scored_faqs.sort(key=lambda x: x[1], reverse=True)
        return scored_faqs[:top_n]