sentiment_analysis.py

import pandas as pd
import numpy as np
from nltk.sentiment.vader import SentimentIntensityAnalyzer
import matplotlib.pyplot as plt
import seaborn as sns
import plotly.express as px
import plotly.graph_objects as go
from plotly.subplots import make_subplots
from collections import Counter
import streamlit as st

# Initialize the sentiment analyzer
sia = SentimentIntensityAnalyzer()

def enhanced_sentiment_analysis(df, selected_user='Overall'):
    """
    Enhanced sentiment analysis with detailed breakdown and visualizations.
    Adds:
      - compound (–1 to 1)
      - scaled_sentiment_score (0–100)
      - sentiment_label (Very Negative … Very Positive)
      - sentiment_category (Positive/Neutral/Negative)
      - detailed_sentiment (Very Positive … Very Negative)
    """
    # 1. Filter by user if needed
    if selected_user != 'Overall':
        df = df[df['user'] == selected_user].copy()
    else:
        df = df.copy()
    
    # 2. Remove system messages and media placeholders
    df = df[
        (df['user'] != 'group_notification') &
        (df['message'].str.strip() != '<Media omitted>') &
        (df['message'].str.len() > 0)
    ]
    
    # 3. Compute VADER scores for each message
    sentiment_scores = [sia.polarity_scores(msg) for msg in df['message']]
    sentiment_df = pd.DataFrame(sentiment_scores)
    
    # 4. Merge scores back into df
    df = df.reset_index(drop=True)
    df[['negative', 'neutral', 'positive', 'compound']] = sentiment_df
    
    # 5. Scale compound from (–1,1) to (0,100)
    df['scaled_sentiment_score'] = ((df['compound'] + 1) / 2 * 100).round(2)
    
    # 6. Map scaled score to text label
    def map_score_to_caption(score):
        if score < 20:
            return 'Very Negative'
        elif score < 40:
            return 'Negative'
        elif score < 60:
            return 'Neutral'
        elif score < 80:
            return 'Positive'
        else:
            return 'Very Positive'
    
    df['sentiment_label'] = df['scaled_sentiment_score'].apply(map_score_to_caption)
    
    # 7. Simple category (Positive, Neutral, Negative) based on compound
    def categorize_sentiment(compound_score):
        if compound_score >= 0.05:
            return 'Positive'
        elif compound_score <= -0.05:
            return 'Negative'
        else:
            return 'Neutral'
    
    df['sentiment_category'] = df['compound'].apply(categorize_sentiment)
    
    # 8. Detailed categories (Very Positive … Very Negative)
    def detailed_sentiment(compound_score):
        if compound_score >= 0.5:
            return 'Very Positive'
        elif compound_score >= 0.05:
            return 'Positive'
        elif compound_score > -0.05:
            return 'Neutral'
        elif compound_score > -0.5:
            return 'Negative'
        else:
            return 'Very Negative'
    
    df['detailed_sentiment'] = df['compound'].apply(detailed_sentiment)
    
    return df

def sentiment_summary_stats(df):
    """Generate comprehensive sentiment summary statistics, including scaled scores."""
    summary = {
        'total_messages': len(df),
        'avg_scaled_score': df['scaled_sentiment_score'].mean(),
        'sentiment_distribution': df['sentiment_category'].value_counts().to_dict(),
        'detailed_sentiment_distribution': df['detailed_sentiment'].value_counts().to_dict(),
        'most_positive_scaled': df['scaled_sentiment_score'].max(),
        'most_negative_scaled': df['scaled_sentiment_score'].min(),
        'neutral_percentage': (df['sentiment_category'] == 'Neutral').sum() / len(df) * 100,
        'positive_percentage': (df['sentiment_category'] == 'Positive').sum() / len(df) * 100,
        'negative_percentage': (df['sentiment_category'] == 'Negative').sum() / len(df) * 100,
    }
    return summary

def sentiment_by_user(df):
    """Analyze sentiment breakdown by user."""
    if 'user' not in df.columns:
        return None
    
    # 1. Compute averages of scaled_sentiment_score
    user_sentiment = df.groupby('user').agg({
        'scaled_sentiment_score': ['mean', 'std', 'count'],
        # 2. Convert raw positive/neutral/negative proportions to percentages
        'positive': 'mean',
        'negative': 'mean',
        'neutral': 'mean'
    }).round(2)
    
    user_sentiment.columns = [
        'avg_scaled_score', 'std_scaled_score', 'message_count',
        'avg_positive_pct', 'avg_negative_pct', 'avg_neutral_pct'
    ]
    
    # Multiply avg_positive_pct etc. by 100 to put on 0–100 scale
    user_sentiment['avg_positive_pct'] = (user_sentiment['avg_positive_pct'] * 100).round(2)
    user_sentiment['avg_negative_pct'] = (user_sentiment['avg_negative_pct'] * 100).round(2)
    user_sentiment['avg_neutral_pct']  = (user_sentiment['avg_neutral_pct'] * 100).round(2)
    
    # Sentiment category counts per user
    sentiment_counts = df.groupby(['user', 'sentiment_category']).size().unstack(fill_value=0)
    return user_sentiment.reset_index(), sentiment_counts

def sentiment_over_time(df):
    """Analyze sentiment trends over time using scaled score."""
    if 'date' not in df.columns:
        return None
    
    # Daily sentiment (scaled)
    daily_sentiment = df.groupby(df['date'].dt.date).agg({
        'scaled_sentiment_score': 'mean'
    }).reset_index()
    
    # Monthly sentiment (scaled)
    df['year_month'] = df['date'].dt.to_period('M')
    monthly_sentiment = df.groupby('year_month').agg({
        'scaled_sentiment_score': 'mean'
    }).reset_index()
    
    return daily_sentiment, monthly_sentiment

def get_extreme_messages(df, n=5):
    """Get most positive and negative messages by scaled score."""
    most_positive = df.nlargest(n, 'scaled_sentiment_score')[['user', 'message', 'scaled_sentiment_score', 'date']]
    most_negative = df.nsmallest(n, 'scaled_sentiment_score')[['user', 'message', 'scaled_sentiment_score', 'date']]
    return most_positive, most_negative

def sentiment_by_hour(df):
    """Analyze scaled sentiment patterns by hour of day."""
    if 'hour' not in df.columns:
        return None
    hourly_sentiment = df.groupby('hour').agg({
        'scaled_sentiment_score': 'mean'
    }).reset_index()
    return hourly_sentiment

def sentiment_by_day_of_week(df):
    """Analyze scaled sentiment patterns by day of week."""
    if 'day_name' not in df.columns:
        return None
    day_order = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
    daily_sentiment = df.groupby('day_name').agg({
        'scaled_sentiment_score': 'mean'
    }).reindex(day_order).reset_index()
    return daily_sentiment

def create_sentiment_visualizations(df):
    """Create comprehensive sentiment visualizations using scaled scores."""
    # 1. Sentiment Distribution Pie Chart (categories)
    fig1 = px.pie(
        values=df['sentiment_category'].value_counts().values,
        names=df['sentiment_category'].value_counts().index,
        title="Overall Sentiment Distribution (Categories)",
        color_discrete_map={'Positive': '#2E8B57', 'Neutral': '#FFD700', 'Negative': '#DC143C'}
    )
    
    # 2. Detailed Sentiment Distribution (Very Positive … Very Negative)
    fig2 = px.bar(
        x=df['detailed_sentiment'].value_counts().index,
        y=df['detailed_sentiment'].value_counts().values,
        title="Detailed Sentiment Breakdown",
        color=df['detailed_sentiment'].value_counts().values,
        color_continuous_scale='RdYlGn'
    )
    
    # 3. Scaled Score Distribution (0–100)
    fig3 = px.histogram(
        df, x='scaled_sentiment_score', nbins=50,
        title="Distribution of Scaled Sentiment Scores (0–100)",
        labels={'scaled_sentiment_score': 'Scaled Score', 'count': 'Frequency'},
        color_discrete_sequence=['skyblue']
    )
    fig3.add_vline(x=50, line_dash="dash", line_color="red", annotation_text="Neutral (50)")
    
    # 4. Sentiment Over Time (if date available)
    fig4 = None
    if 'date' in df.columns:
        daily_avg = df.groupby(df['date'].dt.date)['scaled_sentiment_score'].mean().reset_index()
        fig4 = px.line(
            daily_avg, x='date', y='scaled_sentiment_score',
            title="Sentiment Trend Over Time (Scaled Score)",
            labels={'scaled_sentiment_score': 'Avg Scaled Score', 'date': 'Date'}
        )
        fig4.add_hline(y=50, line_dash="dash", line_color="red", annotation_text="Neutral (50)")
    
    # 5. Sentiment by Hour (if hour available)
    fig5 = None
    if 'hour' in df.columns:
        hourly_data = sentiment_by_hour(df)
        fig5 = px.line(
            hourly_data, x='hour', y='scaled_sentiment_score',
            title="Sentiment Pattern by Hour of Day (Scaled Score)",
            labels={'scaled_sentiment_score': 'Avg Scaled Score', 'hour': 'Hour'}
        )
        fig5.add_hline(y=50, line_dash="dash", line_color="red", annotation_text="Neutral (50)")
    
    return fig1, fig2, fig3, fig4, fig5

def render_sentiment_analysis_ui(df, selected_user='Overall'):
    """Streamlit UI for comprehensive sentiment analysis (scaled scores)."""
    st.title("🧠 Sentiment Analysis (0–100 Scale)")
    
    # Perform analysis
    with st.spinner("Analyzing sentiments..."):
        sentiment_df = enhanced_sentiment_analysis(df, selected_user)
        if len(sentiment_df) == 0:
            st.warning("No messages found for sentiment analysis!")
            return
    
    # Summary Statistics
    st.subheader("📊 Sentiment Summary (Scaled Scores)")
    summary = sentiment_summary_stats(sentiment_df)
    col1, col2, col3, col4 = st.columns(4)
    with col1:
        st.metric("Total Messages", summary['total_messages'])
    with col2:
        st.metric("Avg Scaled Score", f"{summary['avg_scaled_score']:.2f} / 100")
    with col3:
        st.metric("Highest Scaled Score", f"{summary['most_positive_scaled']:.2f}")
    with col4:
        st.metric("Lowest Scaled Score", f"{summary['most_negative_scaled']:.2f}")
    
    # Sentiment Category Percentages
    col1, col2, col3 = st.columns(3)
    with col1:
        st.metric("😊 Positive %", f"{summary['positive_percentage']:.1f}%")
    with col2:
        st.metric("😐 Neutral %", f"{summary['neutral_percentage']:.1f}%")
    with col3:
        st.metric("😞 Negative %", f"{summary['negative_percentage']:.1f}%")
    
    # Visualizations
    st.subheader("📈 Sentiment Visualizations")
    tab1, tab2, tab3, tab4, tab5 = st.tabs([
        "Category Breakdown", "Detailed Breakdown", "Score Histogram", 
        "Time Trends", "Hourly Patterns"
    ])
    fig1, fig2, fig3, fig4, fig5 = create_sentiment_visualizations(sentiment_df)
    
    with tab1:
        if fig1:
            st.plotly_chart(fig1, use_container_width=True)
    with tab2:
        if fig2:
            st.plotly_chart(fig2, use_container_width=True)
    with tab3:
        if fig3:
            st.plotly_chart(fig3, use_container_width=True)
    with tab4:
        if fig4:
            st.plotly_chart(fig4, use_container_width=True)
        else:
            st.info("Time trend analysis requires date information")
    with tab5:
        if fig5:
            st.plotly_chart(fig5, use_container_width=True)
        else:
            st.info("Hourly pattern analysis requires time information")
    
    # Extreme Messages by Scaled Score
    st.subheader("🔥 Most Extreme Messages (Scaled Score)")
    most_positive, most_negative = get_extreme_messages(sentiment_df)
    col1, col2 = st.columns(2)
    with col1:
        st.write("**Most Positive Messages:**")
        for _, row in most_positive.iterrows():
            # Show only first 2 lines of message
            snippet = "\n".join(row['message'].split("\n")[:2])
            st.write(f"**{row['user']}** (Score: {row['scaled_sentiment_score']:.2f})")
            st.write(f"_{snippet}..._")
            with st.expander("Show full message"):
                st.write(row['message'])
            st.write("---")
    with col2:
        st.write("**Most Negative Messages:**")
        for _, row in most_negative.iterrows():
            snippet = "\n".join(row['message'].split("\n")[:2])
            st.write(f"**{row['user']}** (Score: {row['scaled_sentiment_score']:.2f})")
            st.write(f"_{snippet}..._")
            with st.expander("Show full message"):
                st.write(row['message'])
            st.write("---")
    
    # Sentiment by User (Overall only)
    if selected_user == 'Overall':
        st.subheader("👥 Sentiment by User (Avg Scaled Score)")
        user_stats, user_sentiment_counts = sentiment_by_user(sentiment_df)
        if user_stats is not None:
            st.dataframe(user_stats.sort_values('avg_scaled_score', ascending=False))
            fig_users = px.bar(
                user_stats.head(10),
                x='user', y='avg_scaled_score',
                title="Top 10 Users by Avg Scaled Sentiment",
                color='avg_scaled_score',
                color_continuous_scale='RdYlGn'
            )
            st.plotly_chart(fig_users, use_container_width=True)
    
    # Raw Data View
    with st.expander("🔍 View Raw Sentiment Data"):
        st.dataframe(
            sentiment_df[['user', 'message', 'scaled_sentiment_score', 
                          'sentiment_label', 'sentiment_category', 'date']]
            .sort_values('scaled_sentiment_score', ascending=False)
        )

# Usage example for integration with an older helper interface
def integrate_with_existing_helper():
    def sentiment_analysis(df, selected_user='Overall'):
        sentiment_df = enhanced_sentiment_analysis(df, selected_user)
        summary = sentiment_summary_stats(sentiment_df)
        summary_series = pd.Series({
            'count': summary['total_messages'],
            'mean': summary['avg_scaled_score'],
            'std': sentiment_df['scaled_sentiment_score'].std(),
            'min': summary['most_negative_scaled'],
            '25%': sentiment_df['scaled_sentiment_score'].quantile(0.25),
            '50%': sentiment_df['scaled_sentiment_score'].median(),
            '75%': sentiment_df['scaled_sentiment_score'].quantile(0.75),
            'max': summary['most_positive_scaled']
        })
        return summary_series, sentiment_df[['user', 'message', 'scaled_sentiment_score']]