app.py

# Smart Product Recommender - Backend Implementation
# File: app.py

from flask import Flask, request, jsonify
from flask_cors import CORS
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
from sentence_transformers import SentenceTransformer
import faiss
import pickle
import os
from typing import List, Dict, Any
import logging
from conversational_search import ConversationalSearchParser, SearchIntent
from gemini_conversational_search import GeminiConversationalSearch, GeminiSearchIntent

app = Flask(__name__)
CORS(app)

# Setup logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class ProductRecommender:
    def __init__(self):
        self.products_df = None
        self.reviews_df = None
        self.model = SentenceTransformer('all-MiniLM-L6-v2')  # Lightweight embedding model
        self.product_embeddings = None
        self.faiss_index = None
        self.tfidf_vectorizer = TfidfVectorizer(max_features=5000, stop_words='english')
        self.tfidf_matrix = None
        self.conversation_parser = ConversationalSearchParser()
        self.gemini_search = GeminiConversationalSearch()
        
    def load_data(self, products_path: str, reviews_path: str = None):
        """Load product and review datasets"""
        try:
            # Import data processor
            from data_processor import DataProcessor
            processor = DataProcessor()
            
            # Load products using the robust data processor
            if products_path.endswith(('.json', '.jsonl', '.gz')):
                # Use Amazon data processor for JSON files
                products_df, _ = processor.process_dataset(products_path, 'amazon_products')
                self.products_df = products_df
            else:
                # Use CSV processor for CSV files
                products_df, _ = processor.process_dataset(products_path, 'csv')
                self.products_df = products_df
            
            if self.products_df is None or self.products_df.empty:
                logger.error("No products loaded from the dataset")
                return False
            
            # Load reviews if provided
            if reviews_path and os.path.exists(reviews_path):
                if reviews_path.endswith(('.json', '.jsonl', '.gz')):
                    _, reviews_df = processor.process_dataset(reviews_path, 'amazon_reviews')
                    self.reviews_df = reviews_df
                else:
                    reviews_df, _ = processor.process_dataset(reviews_path, 'csv')
                    self.reviews_df = reviews_df
                    
            logger.info(f"Loaded {len(self.products_df)} products")
            if self.reviews_df is not None:
                logger.info(f"Loaded {len(self.reviews_df)} reviews")
                
            self._preprocess_data()
            return True
            
        except Exception as e:
            logger.error(f"Error loading data: {e}")
            return False
    
    def _preprocess_data(self):
        """Clean and preprocess the data"""
        # Handle missing values
        self.products_df = self.products_df.fillna('')
        
        # Create combined text for embeddings
        text_columns = []
        for col in ['title', 'description', 'category', 'brand']:
            if col in self.products_df.columns:
                text_columns.append(col)
        
        if text_columns:
            self.products_df['combined_text'] = self.products_df[text_columns].apply(
                lambda x: ' '.join(x.astype(str)), axis=1
            )
        else:
            # Fallback if columns don't match expected names
            text_cols = [col for col in self.products_df.columns if 'text' in col.lower() or 'title' in col.lower()]
            if text_cols:
                self.products_df['combined_text'] = self.products_df[text_cols[0]].astype(str)
            else:
                self.products_df['combined_text'] = self.products_df.iloc[:, 0].astype(str)
    
    def build_embeddings(self):
        """Create product embeddings using sentence transformers"""
        try:
            logger.info("Building product embeddings...")
            
            # Create embeddings
            texts = self.products_df['combined_text'].tolist()
            self.product_embeddings = self.model.encode(texts, show_progress_bar=True)
            
            # Build FAISS index for fast similarity search
            dimension = self.product_embeddings.shape[1]
            self.faiss_index = faiss.IndexFlatIP(dimension)  # Inner product for cosine similarity
            
            # Normalize embeddings for cosine similarity
            normalized_embeddings = self.product_embeddings / np.linalg.norm(self.product_embeddings, axis=1, keepdims=True)
            self.faiss_index.add(normalized_embeddings.astype('float32'))
            
            # Build TF-IDF matrix for text-based similarity
            self.tfidf_matrix = self.tfidf_vectorizer.fit_transform(texts)
            
            logger.info("Embeddings built successfully")
            return True
            
        except Exception as e:
            logger.error(f"Error building embeddings: {e}")
            return False
    
    def get_similar_products(self, product_id: int, n_recommendations: int = 5) -> List[Dict]:
        """Get similar products using vector similarity"""
        try:
            product_idx = self.products_df[self.products_df.index == product_id].index[0]
            
            # Get embedding for the product
            query_embedding = self.product_embeddings[product_idx:product_idx+1]
            query_embedding = query_embedding / np.linalg.norm(query_embedding)
            
            # Search for similar products
            scores, indices = self.faiss_index.search(query_embedding.astype('float32'), n_recommendations + 1)
            
            # Exclude the query product itself
            similar_products = []
            for i, (score, idx) in enumerate(zip(scores[0], indices[0])):
                if idx != product_idx:
                    product_data = self.products_df.iloc[idx].to_dict()
                    product_data['similarity_score'] = float(score)
                    similar_products.append(product_data)
                    
                if len(similar_products) >= n_recommendations:
                    break
            
            return similar_products
            
        except Exception as e:
            logger.error(f"Error getting similar products: {e}")
            return []
    
    def get_complementary_products(self, product_id: int, n_recommendations: int = 5) -> List[Dict]:
        """Get complementary products using category and brand diversity"""
        try:
            product = self.products_df.iloc[product_id]
            product_category = product.get('category', '')
            product_brand = product.get('brand', '')
            
            # Define complementary categories
            complementary_map = {
                'Electronics': ['Accessories', 'Cases', 'Cables'],
                'Phones': ['Cases', 'Screen Protectors', 'Chargers', 'Headphones'],
                'Laptops': ['Bags', 'Mouse', 'Keyboards', 'Monitors'],
                'Books': ['Bookmarks', 'Reading Lights', 'Book Stands'],
                'Clothing': ['Accessories', 'Shoes', 'Bags']
            }
            
            # Get products from complementary categories
            complementary_categories = complementary_map.get(product_category, [])
            
            if complementary_categories:
                mask = self.products_df['category'].isin(complementary_categories)
                candidates = self.products_df[mask]
            else:
                # Fallback: get products from different brands in same category
                mask = (self.products_df['category'] == product_category) & \
                       (self.products_df['brand'] != product_brand)
                candidates = self.products_df[mask]
            
            # Rank by rating and popularity if available
            if 'rating' in candidates.columns:
                candidates = candidates.sort_values('rating', ascending=False)
            
            return candidates.head(n_recommendations).to_dict('records')
            
        except Exception as e:
            logger.error(f"Error getting complementary products: {e}")
            return []
    
    def search_products(self, query: str, n_results: int = 10) -> List[Dict]:
        """Search products using text similarity"""
        try:
            # Encode search query
            query_embedding = self.model.encode([query])
            query_embedding = query_embedding / np.linalg.norm(query_embedding)
            
            # Search using FAISS
            scores, indices = self.faiss_index.search(query_embedding.astype('float32'), n_results)
            
            results = []
            for score, idx in zip(scores[0], indices[0]):
                product_data = self.products_df.iloc[idx].to_dict()
                product_data['search_score'] = float(score)
                results.append(product_data)
            
            return results
            
        except Exception as e:
            logger.error(f"Error searching products: {e}")
            return []
    
    def conversational_search(self, query: str, n_results: int = 10) -> Dict:
        """Search products using conversational natural language query"""
        try:
            # Parse the natural language query
            intent = self.conversation_parser.parse_query(query)
            
            # Start with all products
            filtered_df = self.products_df.copy()
            
            # Apply filters based on parsed intent
            if intent.product_type:
                # Filter by product type (search in title and category)
                product_mask = (
                    filtered_df['title'].str.contains(intent.product_type, case=False, na=False) |
                    filtered_df['category'].str.contains(intent.product_type, case=False, na=False)
                )
                filtered_df = filtered_df[product_mask]
            
            if intent.brand:
                # Filter by brand
                brand_mask = filtered_df['brand'].str.contains(intent.brand, case=False, na=False)
                filtered_df = filtered_df[brand_mask]
            
            if intent.price_min is not None:
                filtered_df = filtered_df[filtered_df['price'] >= intent.price_min]
            
            if intent.price_max is not None:
                filtered_df = filtered_df[filtered_df['price'] <= intent.price_max]
            
            if intent.rating_min is not None:
                filtered_df = filtered_df[filtered_df['rating'] >= intent.rating_min]
            
            if intent.use_case:
                # Filter by use case (search in title and description)
                use_case_mask = (
                    filtered_df['title'].str.contains(intent.use_case, case=False, na=False) |
                    filtered_df['description'].str.contains(intent.use_case, case=False, na=False)
                )
                filtered_df = filtered_df[use_case_mask]
            
            # If we have filtered results, perform semantic search within them
            if len(filtered_df) > 0:
                # Get indices of filtered products
                filtered_indices = filtered_df.index.tolist()
                
                # Perform semantic search
                query_embedding = self.model.encode([query])
                query_embedding = query_embedding / np.linalg.norm(query_embedding)
                
                # Search in the full index but filter results
                scores, indices = self.faiss_index.search(query_embedding.astype('float32'), min(len(self.products_df), n_results * 3))
                
                # Filter results to only include products that match our criteria
                results = []
                for score, idx in zip(scores[0], indices[0]):
                    if idx in filtered_indices and len(results) < n_results:
                        product_data = self.products_df.iloc[idx].to_dict()
                        product_data['search_score'] = float(score)
                        product_data['conversation_match'] = True
                        results.append(product_data)
                
                # If we don't have enough results, add more from filtered set
                if len(results) < n_results:
                    remaining_products = filtered_df[~filtered_df.index.isin([r['asin'] if 'asin' in r else i for i, r in enumerate(results)])]
                    for _, product in remaining_products.head(n_results - len(results)).iterrows():
                        product_data = product.to_dict()
                        product_data['search_score'] = 0.5  # Default score for filtered matches
                        product_data['conversation_match'] = True
                        results.append(product_data)
            else:
                # No products match the criteria, fall back to regular search
                results = self.search_products(query, n_results)
                for result in results:
                    result['conversation_match'] = False
            
            return {
                'results': results,
                'intent': {
                    'product_type': intent.product_type,
                    'price_min': intent.price_min,
                    'price_max': intent.price_max,
                    'brand': intent.brand,
                    'use_case': intent.use_case,
                    'rating_min': intent.rating_min,
                    'summary': self.conversation_parser.generate_search_summary(intent)
                },
                'total_matches': len(filtered_df) if len(filtered_df) > 0 else len(results)
            }
            
        except Exception as e:
            logger.error(f"Error in conversational search: {e}")
            # Fall back to regular search
            results = self.search_products(query, n_results)
            return {
                'results': results,
                'intent': {'summary': 'Regular search (parsing failed)'},
                'total_matches': len(results)
            }
    
    def gemini_conversational_search(self, query: str, n_results: int = 10) -> Dict:
        """Enhanced conversational search using Gemini LLM"""
        try:
            # Extract intent using Gemini
            intent = self.gemini_search.extract_intent_with_gemini(query)
            
            # Start with all products
            filtered_df = self.products_df.copy()
            
            # Apply filters based on Gemini-extracted intent
            if intent.product_type:
                product_mask = (
                    filtered_df['title'].str.contains(intent.product_type, case=False, na=False) |
                    filtered_df['category'].str.contains(intent.product_type, case=False, na=False)
                )
                filtered_df = filtered_df[product_mask]
            
            if intent.brand:
                brand_mask = filtered_df['brand'].str.contains(intent.brand, case=False, na=False)
                filtered_df = filtered_df[brand_mask]
            
            if intent.price_min is not None:
                filtered_df = filtered_df[filtered_df['price'] >= intent.price_min]
            
            if intent.price_max is not None:
                filtered_df = filtered_df[filtered_df['price'] <= intent.price_max]
            
            if intent.use_case:
                use_case_mask = (
                    filtered_df['title'].str.contains(intent.use_case, case=False, na=False) |
                    filtered_df['description'].str.contains(intent.use_case, case=False, na=False)
                )
                filtered_df = filtered_df[use_case_mask]
            
            # Apply feature filters
            for feature in intent.features:
                feature_mask = (
                    filtered_df['title'].str.contains(feature, case=False, na=False) |
                    filtered_df['description'].str.contains(feature, case=False, na=False)
                )
                filtered_df = filtered_df[feature_mask]
            
            # Perform semantic search within filtered results
            if len(filtered_df) > 0:
                filtered_indices = filtered_df.index.tolist()
                
                # Semantic search
                query_embedding = self.model.encode([query])
                query_embedding = query_embedding / np.linalg.norm(query_embedding)
                
                scores, indices = self.faiss_index.search(query_embedding.astype('float32'), min(len(self.products_df), n_results * 3))
                
                # Filter and analyze results
                results = []
                analyses = []
                
                for score, idx in zip(scores[0], indices[0]):
                    if idx in filtered_indices and len(results) < n_results:
                        product_data = self.products_df.iloc[idx].to_dict()
                        product_data['search_score'] = float(score)
                        
                        # Use Gemini to analyze product relevance
                        analysis = self.gemini_search.analyze_product_relevance(intent, product_data)
                        product_data['gemini_relevance'] = analysis.get('relevance_score', 0.5)
                        product_data['match_reasons'] = analysis.get('match_reasons', [])
                        product_data['concerns'] = analysis.get('concerns', [])
                        product_data['recommendation'] = analysis.get('recommendation', 'acceptable')
                        
                        results.append(product_data)
                        analyses.append(analysis)
                
                # Sort by combined score (semantic + Gemini relevance)
                for result in results:
                    combined_score = (result['search_score'] * 0.6) + (result['gemini_relevance'] * 0.4)
                    result['combined_score'] = combined_score
                
                results.sort(key=lambda x: x['combined_score'], reverse=True)
                
                # Generate conversational response
                conversational_response = self.gemini_search.generate_conversational_response(intent, results, analyses)
                
            else:
                # No filtered results, fall back to regular search
                results = self.search_products(query, n_results)
                for result in results:
                    result['gemini_relevance'] = 0.5
                    result['match_reasons'] = []
                    result['concerns'] = []
                    result['recommendation'] = 'acceptable'
                
                conversational_response = f"I found {len(results)} general results for '{query}'. Consider refining your search for better matches."
            
            return {
                'results': results,
                'intent': {
                    'product_type': intent.product_type,
                    'brand': intent.brand,
                    'price_min': intent.price_min,
                    'price_max': intent.price_max,
                    'use_case': intent.use_case,
                    'features': intent.features,
                    'quality_preference': intent.quality_preference,
                    'summary': intent.search_summary,
                    'confidence': intent.confidence_score,
                    'clarification_needed': intent.clarification_needed,
                    'clarification_questions': intent.clarification_questions
                },
                'total_matches': len(filtered_df) if len(filtered_df) > 0 else len(results),
                'conversational_response': conversational_response,
                'powered_by': 'gemini'
            }
            
        except Exception as e:
            logger.error(f"Error in Gemini conversational search: {e}")
            # Fall back to regular conversational search
            return self.conversational_search(query, n_results)

# Initialize recommender
recommender = ProductRecommender()

@app.route('/api/load_data', methods=['POST'])
def load_data():
    """Load dataset files"""
    try:
        data = request.get_json()
        products_path = data.get('products_path')
        reviews_path = data.get('reviews_path')
        
        if not products_path:
            return jsonify({'error': 'Products path is required'}), 400
            
        if not os.path.exists(products_path):
            return jsonify({'error': f'Products file not found: {products_path}'}), 400
        
        logger.info(f"Loading data from: {products_path}")
        success = recommender.load_data(products_path, reviews_path)
        
        if success:
            logger.info("Data loaded successfully, building embeddings...")
            # Build embeddings
            embedding_success = recommender.build_embeddings()
            if embedding_success:
                total_products = len(recommender.products_df) if recommender.products_df is not None else 0
                return jsonify({
                    'message': f'Data loaded and embeddings built successfully. {total_products} products loaded.',
                    'total_products': total_products
                })
            else:
                return jsonify({'error': 'Data loaded but failed to build embeddings'}), 500
        else:
            return jsonify({'error': 'Failed to load data. Please check the file format and try again.'}), 500
            
    except Exception as e:
        logger.error(f"Unexpected error in load_data: {e}")
        return jsonify({'error': f'Unexpected error: {str(e)}'}), 500

@app.route('/api/products', methods=['GET'])
def get_products():
    """Get all products with pagination"""
    if recommender.products_df is None:
        return jsonify({'error': 'No data loaded'}), 400
    
    page = request.args.get('page', 1, type=int)
    per_page = request.args.get('per_page', 20, type=int)
    category = request.args.get('category')
    
    df = recommender.products_df.copy()
    
    if category and category != 'all':
        df = df[df['category'] == category]
    
    # Pagination
    start = (page - 1) * per_page
    end = start + per_page
    products = df.iloc[start:end].to_dict('records')
    
    return jsonify({
        'products': products,
        'total': len(df),
        'page': page,
        'per_page': per_page
    })

@app.route('/api/recommendations/similar/<int:product_id>')
def get_similar(product_id):
    """Get similar product recommendations"""
    if recommender.faiss_index is None:
        return jsonify({'error': 'Embeddings not built'}), 400
    
    n_recs = request.args.get('n', 5, type=int)
    similar_products = recommender.get_similar_products(product_id, n_recs)
    
    return jsonify({'recommendations': similar_products})

@app.route('/api/recommendations/complementary/<int:product_id>')
def get_complementary(product_id):
    """Get complementary product recommendations"""
    if recommender.products_df is None:
        return jsonify({'error': 'No data loaded'}), 400
    
    n_recs = request.args.get('n', 5, type=int)
    complementary_products = recommender.get_complementary_products(product_id, n_recs)
    
    return jsonify({'recommendations': complementary_products})

@app.route('/api/search')
def search_products():
    """Search products"""
    if recommender.faiss_index is None:
        return jsonify({'error': 'Embeddings not built'}), 400
    
    query = request.args.get('q', '')
    n_results = request.args.get('n', 10, type=int)
    
    if not query:
        return jsonify({'error': 'Query parameter required'}), 400
    
    results = recommender.search_products(query, n_results)
    return jsonify({'results': results})

@app.route('/api/conversational-search')
def conversational_search():
    """Conversational natural language search"""
    if recommender.faiss_index is None:
        return jsonify({'error': 'Embeddings not built'}), 400
    
    query = request.args.get('q', '')
    n_results = request.args.get('n', 10, type=int)
    
    if not query:
        return jsonify({'error': 'Query parameter required'}), 400
    
    results = recommender.conversational_search(query, n_results)
    return jsonify(results)

@app.route('/api/gemini-search')
def gemini_conversational_search():
    """Gemini-powered conversational search"""
    if recommender.faiss_index is None:
        return jsonify({'error': 'Embeddings not built'}), 400
    
    query = request.args.get('q', '')
    n_results = request.args.get('n', 10, type=int)
    
    if not query:
        return jsonify({'error': 'Query parameter required'}), 400
    
    results = recommender.gemini_conversational_search(query, n_results)
    return jsonify(results)

@app.route('/api/product/<int:product_id>')
def get_product(product_id):
    """Get single product details"""
    if recommender.products_df is None:
        return jsonify({'error': 'No data loaded'}), 400
    
    try:
        product = recommender.products_df.iloc[product_id].to_dict()
        return jsonify(product)
    except IndexError:
        return jsonify({'error': 'Product not found'}), 404

@app.route('/api/categories')
def get_categories():
    """Get all product categories"""
    if recommender.products_df is None:
        return jsonify({'error': 'No data loaded'}), 400
    
    categories = recommender.products_df['category'].unique().tolist()
    return jsonify({'categories': categories})

@app.route('/api/status')
def get_status():
    """Get system status"""
    status = {
        'data_loaded': recommender.products_df is not None,
        'embeddings_built': recommender.faiss_index is not None,
        'total_products': len(recommender.products_df) if recommender.products_df is not None else 0
    }
    return jsonify(status)

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0', port=5000)