exploratory-analysis.py

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
from pathlib import Path

# plot settings
sns.set_style("whitegrid")
plt.rcParams['figure.figsize'] = (14, 8)
plt.rcParams['font.size'] = 10

# make output folder
OUTPUT_DIR = Path("eda_outputs")
OUTPUT_DIR.mkdir(exist_ok=True)


def load_data():
    """load data"""
    print("Loading cleaned dataset...")
    df = pd.read_csv("mideast_development_indicators_cleaned.csv")
    df['date'] = pd.to_datetime(df['date'])
    df['year'] = df['date'].dt.year
    print(f"Dataset loaded: {df.shape}")
    return df


def basic_statistics(df):
    """basic stats"""
    print("\n" + "="*80)
    print("BASIC STATISTICS")
    print("="*80)

    print(f"\nDataset Shape: {df.shape}")
    print(f"Countries: {df['country'].nunique()}")
    print(f"Time Period: {df['year'].min()} - {df['year'].max()}")
    print(f"Total Years: {df['year'].nunique()}")

    numeric_cols = df.select_dtypes(include=[np.number]).columns.tolist()
    if 'year' in numeric_cols:
        numeric_cols.remove('year')

    print(f"\nNumeric Indicators: {len(numeric_cols)}")

    return numeric_cols


def key_indicators_summary(df, numeric_cols):
    """key indicator stats"""
    print("\n" + "="*80)
    print("KEY INDICATORS SUMMARY (2000-2023)")
    print("="*80)

    # main indicators to look at
    key_indicators = {
        'GDP_current_US': 'GDP (Current US$)',
        'population': 'Population',
        'life_expectancy_at_birth': 'Life Expectancy',
        'birth_rate': 'Birth Rate',
        'death_rate': 'Death Rate',
        'inflation_annual%': 'Inflation (%)',
        'individuals_using_internet%': 'Internet Usage (%)',
        'CO2_emisions': 'CO2 Emissions',
        'political_stability_estimate': 'Political Stability',
        'control_of_corruption_estimate': 'Control of Corruption',
    }

    print("\nIndicator Statistics Across All Countries (2000-2023):")
    print("-" * 80)

    for col, label in key_indicators.items():
        if col in df.columns:
            stats = df[col].describe()
            missing_pct = (df[col].isna().sum() / len(df)) * 100
            print(f"\n{label} ({col}):")
            print(f"  Count: {stats['count']:.0f} ({100-missing_pct:.1f}% complete)")
            print(f"  Mean:  {stats['mean']:.2f}")
            print(f"  Std:   {stats['std']:.2f}")
            print(f"  Min:   {stats['min']:.2f}")
            print(f"  Max:   {stats['max']:.2f}")


def country_profiles(df):
    """show country profiles"""
    print("\n" + "="*80)
    print("COUNTRY PROFILES")
    print("="*80)

    countries = sorted(df['country'].unique())

    for country in countries:
        country_data = df[df['country'] == country]

        print(f"\n{country}:")
        print(f"  Records: {len(country_data)}")
        print(f"  Years: {country_data['year'].min()}-{country_data['year'].max()}")

        latest_year_data = country_data.sort_values('year', ascending=False).iloc[0]

        if pd.notna(latest_year_data.get('GDP_current_US')):
            gdp = latest_year_data['GDP_current_US'] / 1e9
            print(f"  Latest GDP: ${gdp:.2f}B ({int(latest_year_data['year'])})")

        if pd.notna(latest_year_data.get('population')):
            pop = latest_year_data['population'] / 1e6
            print(f"  Latest Population: {pop:.2f}M ({int(latest_year_data['year'])})")

        if pd.notna(latest_year_data.get('life_expectancy_at_birth')):
            life_exp = latest_year_data['life_expectancy_at_birth']
            print(f"  Latest Life Expectancy: {life_exp:.1f} years ({int(latest_year_data['year'])})")


def temporal_trends(df):
    """check trends over time"""
    print("\n" + "="*80)
    print("TEMPORAL TRENDS ANALYSIS")
    print("="*80)

    print("\nGDP Growth Patterns:")
    gdp_by_year = df.groupby('year')['GDP_current_US'].agg(['mean', 'median', 'std']).dropna()
    if len(gdp_by_year) > 0:
        print(f"  Average GDP (2000): ${gdp_by_year.iloc[0]['mean']/1e9:.2f}B")
        print(f"  Average GDP (2023): ${gdp_by_year.iloc[-1]['mean']/1e9:.2f}B")
        growth_rate = ((gdp_by_year.iloc[-1]['mean'] / gdp_by_year.iloc[0]['mean']) - 1) * 100
        print(f"  Average Growth: {growth_rate:.1f}%")

    # Life expectancy trends
    print("\nLife Expectancy Trends:")
    life_exp_by_year = df.groupby('year')['life_expectancy_at_birth'].agg(['mean', 'min', 'max']).dropna()
    if len(life_exp_by_year) > 0:
        print(f"  Average (2000): {life_exp_by_year.iloc[0]['mean']:.1f} years")
        print(f"  Average (2023): {life_exp_by_year.iloc[-1]['mean']:.1f} years")
        print(f"  Improvement: {life_exp_by_year.iloc[-1]['mean'] - life_exp_by_year.iloc[0]['mean']:.1f} years")

    # Internet usage trends
    print("\nInternet Usage Trends:")
    internet_by_year = df.groupby('year')['individuals_using_internet%'].agg(['mean', 'min', 'max']).dropna()
    if len(internet_by_year) > 0:
        first_year = internet_by_year.index[0]
        last_year = internet_by_year.index[-1]
        print(f"  Average ({first_year}): {internet_by_year.iloc[0]['mean']:.1f}%")
        print(f"  Average ({last_year}): {internet_by_year.iloc[-1]['mean']:.1f}%")
        print(f"  Growth: {internet_by_year.iloc[-1]['mean'] - internet_by_year.iloc[0]['mean']:.1f} percentage points")


def correlation_analysis(df, numeric_cols):
    """find correlations"""
    print("\n" + "="*80)
    print("CORRELATION ANALYSIS")
    print("="*80)

    # TODO: add more indicators if needed
    key_cols = [
        'GDP_current_US',
        'population',
        'life_expectancy_at_birth',
        'individuals_using_internet%',
        'CO2_emisions',
        'inflation_annual%',
        'political_stability_estimate',
        'control_of_corruption_estimate',
    ]

    available_cols = [col for col in key_cols if col in df.columns]

    if len(available_cols) > 1:
        corr_data = df[available_cols].corr()

        print("\nTop Positive Correlations:")
        # this took a while to figure out
        corr_pairs = []
        for i in range(len(corr_data.columns)):
            for j in range(i+1, len(corr_data.columns)):
                corr_pairs.append({
                    'var1': corr_data.columns[i],
                    'var2': corr_data.columns[j],
                    'corr': corr_data.iloc[i, j]
                })

        corr_df = pd.DataFrame(corr_pairs).dropna()
        top_positive = corr_df.nlargest(5, 'corr')

        for _, row in top_positive.iterrows():
            print(f"  {row['var1']} <-> {row['var2']}: {row['corr']:.3f}")

        print("\nTop Negative Correlations:")
        top_negative = corr_df.nsmallest(5, 'corr')
        for _, row in top_negative.iterrows():
            print(f"  {row['var1']} <-> {row['var2']}: {row['corr']:.3f}")


def country_comparisons(df):
    """compare countries"""
    print("\n" + "="*80)
    print("COUNTRY COMPARISONS (Latest Available Data)")
    print("="*80)

    # Get latest year data for each country
    latest_data = df.sort_values('year').groupby('country').last().reset_index()

    # GDP Rankings
    print("\nGDP Rankings (Current US$):")
    gdp_ranking = latest_data[['country', 'GDP_current_US', 'year']].dropna().sort_values('GDP_current_US', ascending=False)
    for idx, row in gdp_ranking.head(10).iterrows():
        print(f"  {row['country']:30} ${row['GDP_current_US']/1e9:>8.2f}B ({int(row['year'])})")

    # Life Expectancy Rankings
    print("\nLife Expectancy Rankings:")
    life_ranking = latest_data[['country', 'life_expectancy_at_birth', 'year']].dropna().sort_values('life_expectancy_at_birth', ascending=False)
    for idx, row in life_ranking.head(10).iterrows():
        print(f"  {row['country']:30} {row['life_expectancy_at_birth']:>6.1f} years ({int(row['year'])})")

    # Internet Usage Rankings
    print("\nInternet Usage Rankings:")
    internet_ranking = latest_data[['country', 'individuals_using_internet%', 'year']].dropna().sort_values('individuals_using_internet%', ascending=False)
    for idx, row in internet_ranking.head(10).iterrows():
        print(f"  {row['country']:30} {row['individuals_using_internet%']:>6.1f}% ({int(row['year'])})")

    # Political Stability Rankings
    print("\nPolitical Stability Rankings:")
    stability_ranking = latest_data[['country', 'political_stability_estimate', 'year']].dropna().sort_values('political_stability_estimate', ascending=False)
    for idx, row in stability_ranking.head(10).iterrows():
        print(f"  {row['country']:30} {row['political_stability_estimate']:>6.2f} ({int(row['year'])})")


def create_visualizations(df):
    """Create key visualizations."""
    print("\n" + "="*80)
    print("GENERATING VISUALIZATIONS")
    print("="*80)

    # 1. GDP Trends Over Time
    print("\n1. Creating GDP trends visualization...")
    plt.figure(figsize=(16, 10))
    countries = sorted(df['country'].unique())

    for country in countries:
        country_data = df[df['country'] == country]
        plt.plot(country_data['year'], country_data['GDP_current_US']/1e9,
                marker='o', label=country, linewidth=2, markersize=4)

    plt.title('GDP Trends Across Middle Eastern Countries (2000-2023)', fontsize=16, fontweight='bold')
    plt.xlabel('Year', fontsize=12)
    plt.ylabel('GDP (Billion US$)', fontsize=12)
    plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left', fontsize=10)
    plt.grid(True, alpha=0.3)
    plt.tight_layout()
    plt.savefig(OUTPUT_DIR / 'gdp_trends.png', dpi=300, bbox_inches='tight')
    plt.close()

    # 2. Life Expectancy Trends
    print("2. Creating life expectancy trends visualization...")
    plt.figure(figsize=(16, 10))

    for country in countries:
        country_data = df[df['country'] == country]
        plt.plot(country_data['year'], country_data['life_expectancy_at_birth'],
                marker='o', label=country, linewidth=2, markersize=4)

    plt.title('Life Expectancy Trends Across Middle Eastern Countries (2000-2023)', fontsize=16, fontweight='bold')
    plt.xlabel('Year', fontsize=12)
    plt.ylabel('Life Expectancy (Years)', fontsize=12)
    plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left', fontsize=10)
    plt.grid(True, alpha=0.3)
    plt.tight_layout()
    plt.savefig(OUTPUT_DIR / 'life_expectancy_trends.png', dpi=300, bbox_inches='tight')
    plt.close()

    # 3. Internet Usage Growth
    print("3. Creating internet usage trends visualization...")
    plt.figure(figsize=(16, 10))

    for country in countries:
        country_data = df[df['country'] == country]
        plt.plot(country_data['year'], country_data['individuals_using_internet%'],
                marker='o', label=country, linewidth=2, markersize=4)

    plt.title('Internet Usage Growth Across Middle Eastern Countries (2000-2023)', fontsize=16, fontweight='bold')
    plt.xlabel('Year', fontsize=12)
    plt.ylabel('Individuals Using Internet (%)', fontsize=12)
    plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left', fontsize=10)
    plt.grid(True, alpha=0.3)
    plt.tight_layout()
    plt.savefig(OUTPUT_DIR / 'internet_usage_trends.png', dpi=300, bbox_inches='tight')
    plt.close()

    # 4. Political Stability Over Time
    print("4. Creating political stability trends visualization...")
    plt.figure(figsize=(16, 10))

    for country in countries:
        country_data = df[df['country'] == country]
        plt.plot(country_data['year'], country_data['political_stability_estimate'],
                marker='o', label=country, linewidth=2, markersize=4)

    plt.title('Political Stability Trends Across Middle Eastern Countries (2000-2023)', fontsize=16, fontweight='bold')
    plt.xlabel('Year', fontsize=12)
    plt.ylabel('Political Stability Estimate', fontsize=12)
    plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left', fontsize=10)
    plt.grid(True, alpha=0.3)
    plt.axhline(y=0, color='black', linestyle='--', alpha=0.5)
    plt.tight_layout()
    plt.savefig(OUTPUT_DIR / 'political_stability_trends.png', dpi=300, bbox_inches='tight')
    plt.close()

    # 5. CO2 Emissions Trends
    print("5. Creating CO2 emissions trends visualization...")
    plt.figure(figsize=(16, 10))

    for country in countries:
        country_data = df[df['country'] == country]
        plt.plot(country_data['year'], country_data['CO2_emisions'],
                marker='o', label=country, linewidth=2, markersize=4)

    plt.title('CO2 Emissions Trends Across Middle Eastern Countries (2000-2023)', fontsize=16, fontweight='bold')
    plt.xlabel('Year', fontsize=12)
    plt.ylabel('CO2 Emissions (kt)', fontsize=12)
    plt.legend(bbox_to_anchor=(1.05, 1), loc='upper left', fontsize=10)
    plt.grid(True, alpha=0.3)
    plt.tight_layout()
    plt.savefig(OUTPUT_DIR / 'co2_emissions_trends.png', dpi=300, bbox_inches='tight')
    plt.close()

    # 6. Latest Year Comparison - Multiple Metrics
    print("6. Creating latest year comparison visualization...")
    latest_data = df.sort_values('year').groupby('country').last().reset_index()

    fig, axes = plt.subplots(2, 2, figsize=(18, 12))

    # GDP
    gdp_data = latest_data.dropna(subset=['GDP_current_US']).sort_values('GDP_current_US', ascending=True)
    axes[0, 0].barh(gdp_data['country'], gdp_data['GDP_current_US']/1e9)
    axes[0, 0].set_xlabel('GDP (Billion US$)')
    axes[0, 0].set_title('GDP Comparison (Latest Year)')
    axes[0, 0].grid(True, alpha=0.3)

    # Life Expectancy
    life_data = latest_data.dropna(subset=['life_expectancy_at_birth']).sort_values('life_expectancy_at_birth', ascending=True)
    axes[0, 1].barh(life_data['country'], life_data['life_expectancy_at_birth'])
    axes[0, 1].set_xlabel('Life Expectancy (Years)')
    axes[0, 1].set_title('Life Expectancy Comparison (Latest Year)')
    axes[0, 1].grid(True, alpha=0.3)

    # Internet Usage
    internet_data = latest_data.dropna(subset=['individuals_using_internet%']).sort_values('individuals_using_internet%', ascending=True)
    axes[1, 0].barh(internet_data['country'], internet_data['individuals_using_internet%'])
    axes[1, 0].set_xlabel('Internet Usage (%)')
    axes[1, 0].set_title('Internet Usage Comparison (Latest Year)')
    axes[1, 0].grid(True, alpha=0.3)

    # Political Stability
    stability_data = latest_data.dropna(subset=['political_stability_estimate']).sort_values('political_stability_estimate', ascending=True)
    colors = ['red' if x < 0 else 'green' for x in stability_data['political_stability_estimate']]
    axes[1, 1].barh(stability_data['country'], stability_data['political_stability_estimate'], color=colors)
    axes[1, 1].set_xlabel('Political Stability Estimate')
    axes[1, 1].set_title('Political Stability Comparison (Latest Year)')
    axes[1, 1].axvline(x=0, color='black', linestyle='--', alpha=0.5)
    axes[1, 1].grid(True, alpha=0.3)

    plt.tight_layout()
    plt.savefig(OUTPUT_DIR / 'latest_year_comparison.png', dpi=300, bbox_inches='tight')
    plt.close()

    print(f"\n✓ All visualizations saved to '{OUTPUT_DIR}/' directory")


def generate_summary_report(df, numeric_cols):
    """Generate a text summary report."""
    print("\n" + "="*80)
    print("GENERATING SUMMARY REPORT")
    print("="*80)

    report_lines = []
    report_lines.append("=" * 80)
    report_lines.append("EXPLORATORY DATA ANALYSIS SUMMARY REPORT")
    report_lines.append("Middle East Development Indicators (2000-2023)")
    report_lines.append("=" * 80)
    report_lines.append(f"\nGenerated: {pd.Timestamp.now().strftime('%Y-%m-%d %H:%M:%S')}")
    report_lines.append(f"\nDataset: mideast_development_indicators_cleaned.csv")
    report_lines.append(f"Records: {len(df)}")
    report_lines.append(f"Countries: {df['country'].nunique()}")
    report_lines.append(f"Time Period: {df['year'].min()}-{df['year'].max()}")
    report_lines.append(f"Indicators: {len(numeric_cols)}")

    report_lines.append("\n" + "-" * 80)
    report_lines.append("KEY FINDINGS")
    report_lines.append("-" * 80)

    # Economic findings
    report_lines.append("\n1. ECONOMIC DEVELOPMENT:")
    latest_data = df.sort_values('year').groupby('country').last().reset_index()
    top_gdp = latest_data.nlargest(3, 'GDP_current_US')[['country', 'GDP_current_US']]
    report_lines.append("   Top 3 Economies by GDP:")
    for idx, row in top_gdp.iterrows():
        report_lines.append(f"   - {row['country']}: ${row['GDP_current_US']/1e9:.2f}B")

    # Health findings
    report_lines.append("\n2. HEALTH & DEMOGRAPHICS:")
    top_life = latest_data.nlargest(3, 'life_expectancy_at_birth')[['country', 'life_expectancy_at_birth']]
    report_lines.append("   Top 3 Countries by Life Expectancy:")
    for idx, row in top_life.iterrows():
        report_lines.append(f"   - {row['country']}: {row['life_expectancy_at_birth']:.1f} years")

    # Technology findings
    report_lines.append("\n3. DIGITAL TRANSFORMATION:")
    top_internet = latest_data.nlargest(3, 'individuals_using_internet%')[['country', 'individuals_using_internet%']]
    report_lines.append("   Top 3 Countries by Internet Usage:")
    for idx, row in top_internet.iterrows():
        report_lines.append(f"   - {row['country']}: {row['individuals_using_internet%']:.1f}%")

    # Governance findings
    report_lines.append("\n4. GOVERNANCE & STABILITY:")
    top_stability = latest_data.nlargest(3, 'political_stability_estimate')[['country', 'political_stability_estimate']]
    report_lines.append("   Top 3 Countries by Political Stability:")
    for idx, row in top_stability.iterrows():
        report_lines.append(f"   - {row['country']}: {row['political_stability_estimate']:.2f}")

    report_lines.append("\n" + "=" * 80)
    report_lines.append("END OF REPORT")
    report_lines.append("=" * 80)

    # Save report
    report_path = OUTPUT_DIR / 'eda_summary_report.txt'
    with open(report_path, 'w') as f:
        f.write('\n'.join(report_lines))

    print(f"\n✓ Summary report saved to '{report_path}'")


def main():
    """Main EDA execution function."""
    print("\n" + "="*80)
    print("EXPLORATORY DATA ANALYSIS")
    print("Middle East Development Indicators (2000-2023)")
    print("="*80)

    # Load data
    df = load_data()

    # Basic statistics
    numeric_cols = basic_statistics(df)

    # Key indicators summary
    key_indicators_summary(df, numeric_cols)

    # Country profiles
    country_profiles(df)

    # Temporal trends
    temporal_trends(df)

    # Correlation analysis
    correlation_analysis(df, numeric_cols)

    # Country comparisons
    country_comparisons(df)

    # Create visualizations
    create_visualizations(df)

    # Generate summary report
    generate_summary_report(df, numeric_cols)

    print("\n" + "="*80)
    print("✓ EXPLORATORY DATA ANALYSIS COMPLETE")
    print("="*80)
    print(f"\nOutputs saved to: {OUTPUT_DIR}/")
    print("  - 6 visualization PNG files")
    print("  - 1 summary report (TXT)")
    print("\n")


if __name__ == "__main__":
    main()