|
| 1 | +#!/usr/bin/env python |
| 2 | +""" |
| 3 | +Debug script for Fair Value Gap indicator. |
| 4 | +
|
| 5 | +This script creates synthetic data with known FVG patterns and tests the indicator. |
| 6 | +It also analyzes real data to understand why FVGs aren't being detected. |
| 7 | +
|
| 8 | +Author: ProjectX SDK |
| 9 | +Date: 2025-01-12 |
| 10 | +""" |
| 11 | + |
| 12 | +import asyncio |
| 13 | +import os |
| 14 | +import sys |
| 15 | + |
| 16 | +import polars as pl |
| 17 | + |
| 18 | +# Add parent directory to path for imports |
| 19 | +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
| 20 | + |
| 21 | +from project_x_py import ProjectX |
| 22 | +from project_x_py.indicators import FVG |
| 23 | + |
| 24 | + |
| 25 | +def create_synthetic_fvg_data() -> pl.DataFrame: |
| 26 | + """Create synthetic OHLCV data with known Fair Value Gaps.""" |
| 27 | + from datetime import datetime, timedelta |
| 28 | + |
| 29 | + base_time = datetime.now() |
| 30 | + |
| 31 | + # Create data with intentional FVG patterns |
| 32 | + data = { |
| 33 | + "timestamp": [base_time + timedelta(minutes=i * 15) for i in range(10)], |
| 34 | + "open": [100.0, 102.0, 105.0, 108.0, 104.0, 106.0, 103.0, 105.0, 107.0, 106.0], |
| 35 | + "high": [101.0, 103.0, 106.0, 110.0, 105.0, 107.0, 104.0, 106.0, 108.0, 107.0], |
| 36 | + "low": [99.0, 101.0, 104.0, 107.0, 103.0, 105.0, 102.0, 104.0, 106.0, 105.0], |
| 37 | + "close": [100.5, 102.5, 105.5, 108.5, 104.5, 106.5, 103.5, 105.5, 107.5, 106.5], |
| 38 | + "volume": [1000, 1100, 1200, 1300, 1400, 1500, 1600, 1700, 1800, 1900], |
| 39 | + } |
| 40 | + |
| 41 | + # Add a clear bullish FVG at index 5-7 |
| 42 | + # Bar 5: high = 102 |
| 43 | + # Bar 6: low = 105, high = 108 (low > prev high) |
| 44 | + # Bar 7: low = 110 (low > bar 6 high AND bar 6 low > bar 5 high) |
| 45 | + data["high"][5] = 102.0 # Bar 5 high |
| 46 | + data["low"][5] = 100.0 # Bar 5 low |
| 47 | + |
| 48 | + data["high"][6] = 108.0 # Bar 6 high |
| 49 | + data["low"][6] = 105.0 # Bar 6 low (> bar 5 high) |
| 50 | + |
| 51 | + data["high"][7] = 115.0 # Bar 7 high |
| 52 | + data["low"][7] = 110.0 # Bar 7 low (> bar 6 high) |
| 53 | + |
| 54 | + # This should create a bullish FVG at bar 7 |
| 55 | + # Because: bar7.low (110) > bar6.high (108) AND bar6.low (105) > bar5.high (102) |
| 56 | + |
| 57 | + return pl.DataFrame(data) |
| 58 | + |
| 59 | + |
| 60 | +def analyze_real_data_for_gaps(df: pl.DataFrame) -> None: |
| 61 | + """Analyze real data to understand why FVGs aren't being detected.""" |
| 62 | + print("\n" + "=" * 80) |
| 63 | + print("ANALYZING DATA FOR POTENTIAL GAPS") |
| 64 | + print("=" * 80) |
| 65 | + |
| 66 | + # Add shifted columns for analysis |
| 67 | + analysis_df = df.with_columns( |
| 68 | + [ |
| 69 | + pl.col("high").shift(1).alias("prev_high"), |
| 70 | + pl.col("low").shift(1).alias("prev_low"), |
| 71 | + pl.col("high").shift(2).alias("prev2_high"), |
| 72 | + pl.col("low").shift(2).alias("prev2_low"), |
| 73 | + ] |
| 74 | + ) |
| 75 | + |
| 76 | + # Check bullish FVG conditions |
| 77 | + analysis_df = analysis_df.with_columns( |
| 78 | + [ |
| 79 | + # First condition: current low > prev high |
| 80 | + (pl.col("low") > pl.col("prev_high")).alias("bullish_cond1"), |
| 81 | + # Second condition: prev low > prev2 high |
| 82 | + (pl.col("prev_low") > pl.col("prev2_high")).alias("bullish_cond2"), |
| 83 | + # Both conditions |
| 84 | + ( |
| 85 | + (pl.col("low") > pl.col("prev_high")) |
| 86 | + & (pl.col("prev_low") > pl.col("prev2_high")) |
| 87 | + ).alias("bullish_fvg"), |
| 88 | + ] |
| 89 | + ) |
| 90 | + |
| 91 | + # Check bearish FVG conditions |
| 92 | + analysis_df = analysis_df.with_columns( |
| 93 | + [ |
| 94 | + # First condition: current high < prev low |
| 95 | + (pl.col("high") < pl.col("prev_low")).alias("bearish_cond1"), |
| 96 | + # Second condition: prev high < prev2 low |
| 97 | + (pl.col("prev_high") < pl.col("prev2_low")).alias("bearish_cond2"), |
| 98 | + # Both conditions |
| 99 | + ( |
| 100 | + (pl.col("high") < pl.col("prev_low")) |
| 101 | + & (pl.col("prev_high") < pl.col("prev2_low")) |
| 102 | + ).alias("bearish_fvg"), |
| 103 | + ] |
| 104 | + ) |
| 105 | + |
| 106 | + # Count how often each condition is met |
| 107 | + bullish_cond1_count = analysis_df.filter(pl.col("bullish_cond1")).height |
| 108 | + bullish_cond2_count = analysis_df.filter(pl.col("bullish_cond2")).height |
| 109 | + bullish_both_count = analysis_df.filter(pl.col("bullish_fvg")).height |
| 110 | + |
| 111 | + bearish_cond1_count = analysis_df.filter(pl.col("bearish_cond1")).height |
| 112 | + bearish_cond2_count = analysis_df.filter(pl.col("bearish_cond2")).height |
| 113 | + bearish_both_count = analysis_df.filter(pl.col("bearish_fvg")).height |
| 114 | + |
| 115 | + print(f"\n📊 BULLISH FVG CONDITIONS:") |
| 116 | + print( |
| 117 | + f" Condition 1 (low > prev_high): {bullish_cond1_count}/{df.height} bars ({bullish_cond1_count / df.height * 100:.1f}%)" |
| 118 | + ) |
| 119 | + print( |
| 120 | + f" Condition 2 (prev_low > prev2_high): {bullish_cond2_count}/{df.height} bars ({bullish_cond2_count / df.height * 100:.1f}%)" |
| 121 | + ) |
| 122 | + print(f" Both conditions met: {bullish_both_count} bars") |
| 123 | + |
| 124 | + print(f"\n📊 BEARISH FVG CONDITIONS:") |
| 125 | + print( |
| 126 | + f" Condition 1 (high < prev_low): {bearish_cond1_count}/{df.height} bars ({bearish_cond1_count / df.height * 100:.1f}%)" |
| 127 | + ) |
| 128 | + print( |
| 129 | + f" Condition 2 (prev_high < prev2_low): {bearish_cond2_count}/{df.height} bars ({bearish_cond2_count / df.height * 100:.1f}%)" |
| 130 | + ) |
| 131 | + print(f" Both conditions met: {bearish_both_count} bars") |
| 132 | + |
| 133 | + # Show examples where condition 1 is met but not condition 2 |
| 134 | + if bullish_cond1_count > 0 and bullish_both_count == 0: |
| 135 | + print("\n⚠️ Examples where bullish condition 1 is met but not condition 2:") |
| 136 | + examples = analysis_df.filter(pl.col("bullish_cond1")).head(3) |
| 137 | + for row in examples.iter_rows(named=True): |
| 138 | + print(f" Time: {row['timestamp']}") |
| 139 | + print(f" Current: L={row['low']:.2f}, H={row['high']:.2f}") |
| 140 | + print(f" Prev: L={row['prev_low']:.2f}, H={row['prev_high']:.2f}") |
| 141 | + print(f" Prev2: L={row['prev2_low']:.2f}, H={row['prev2_high']:.2f}") |
| 142 | + print( |
| 143 | + f" Cond1: {row['low']:.2f} > {row['prev_high']:.2f} = {row['bullish_cond1']}" |
| 144 | + ) |
| 145 | + print( |
| 146 | + f" Cond2: {row['prev_low']:.2f} > {row['prev2_high']:.2f} = {row['bullish_cond2']}" |
| 147 | + ) |
| 148 | + |
| 149 | + # Look for simpler gap patterns (just current low > prev high or current high < prev low) |
| 150 | + simple_bullish_gaps = analysis_df.filter(pl.col("low") > pl.col("prev_high")).height |
| 151 | + simple_bearish_gaps = analysis_df.filter(pl.col("high") < pl.col("prev_low")).height |
| 152 | + |
| 153 | + print(f"\n💡 SIMPLER GAP PATTERNS:") |
| 154 | + print(f" Simple bullish gaps (low > prev_high): {simple_bullish_gaps} bars") |
| 155 | + print(f" Simple bearish gaps (high < prev_low): {simple_bearish_gaps} bars") |
| 156 | + |
| 157 | + if simple_bullish_gaps > 0: |
| 158 | + print("\n Examples of simple bullish gaps:") |
| 159 | + examples = analysis_df.filter(pl.col("low") > pl.col("prev_high")).head(5) |
| 160 | + for row in examples.iter_rows(named=True): |
| 161 | + gap_size = row["low"] - row["prev_high"] |
| 162 | + print( |
| 163 | + f" {row['timestamp']}: Gap of ${gap_size:.2f} (Low ${row['low']:.2f} > Prev High ${row['prev_high']:.2f})" |
| 164 | + ) |
| 165 | + |
| 166 | + |
| 167 | +async def main(): |
| 168 | + """Main function to debug FVG indicator.""" |
| 169 | + print("🔍 Fair Value Gap Indicator Debug") |
| 170 | + print("=" * 80) |
| 171 | + |
| 172 | + # Test with synthetic data first |
| 173 | + print("\n1️⃣ TESTING WITH SYNTHETIC DATA") |
| 174 | + print("-" * 40) |
| 175 | + |
| 176 | + synthetic_df = create_synthetic_fvg_data() |
| 177 | + print(f"Created synthetic data with {synthetic_df.height} bars") |
| 178 | + print("\nSynthetic data:") |
| 179 | + print(synthetic_df.select(["timestamp", "open", "high", "low", "close"])) |
| 180 | + |
| 181 | + # Apply FVG indicator to synthetic data |
| 182 | + synthetic_with_fvg = synthetic_df.pipe(FVG, min_gap_size=0.0) |
| 183 | + |
| 184 | + # Check for detected FVGs |
| 185 | + bullish_fvgs = synthetic_with_fvg.filter(pl.col("fvg_bullish")) |
| 186 | + bearish_fvgs = synthetic_with_fvg.filter(pl.col("fvg_bearish")) |
| 187 | + |
| 188 | + print(f"\n✅ Detected {bullish_fvgs.height} bullish FVGs in synthetic data") |
| 189 | + print(f"✅ Detected {bearish_fvgs.height} bearish FVGs in synthetic data") |
| 190 | + |
| 191 | + if bullish_fvgs.height > 0: |
| 192 | + print("\nBullish FVGs found:") |
| 193 | + for row in bullish_fvgs.iter_rows(named=True): |
| 194 | + print(f" Time: {row['timestamp']}") |
| 195 | + print(f" Gap Top: ${row.get('fvg_gap_top', 'N/A')}") |
| 196 | + print(f" Gap Bottom: ${row.get('fvg_gap_bottom', 'N/A')}") |
| 197 | + print(f" Gap Size: ${row.get('fvg_gap_size', 'N/A')}") |
| 198 | + |
| 199 | + # Now test with real data |
| 200 | + print("\n2️⃣ TESTING WITH REAL MARKET DATA") |
| 201 | + print("-" * 40) |
| 202 | + |
| 203 | + try: |
| 204 | + async with ProjectX.from_env() as client: |
| 205 | + await client.authenticate() |
| 206 | + print(f"✅ Connected to account: {client.account_info.name}") |
| 207 | + |
| 208 | + # Get historical data - try 1-minute bars for more samples |
| 209 | + print("\n📥 Loading 1-minute bar data...") |
| 210 | + bars_df = await client.get_bars( |
| 211 | + "MNQ", |
| 212 | + days=10, |
| 213 | + interval=1, # 1-minute bars |
| 214 | + unit=2, # 2 = minutes |
| 215 | + ) |
| 216 | + |
| 217 | + if bars_df is None or bars_df.is_empty(): |
| 218 | + print("❌ No data retrieved!") |
| 219 | + return |
| 220 | + |
| 221 | + print(f"✅ Loaded {bars_df.height} bars") |
| 222 | + |
| 223 | + # Analyze why FVGs aren't being detected |
| 224 | + analyze_real_data_for_gaps(bars_df) |
| 225 | + |
| 226 | + # Apply FVG indicator with very low threshold |
| 227 | + print("\n3️⃣ APPLYING FVG INDICATOR TO REAL DATA") |
| 228 | + print("-" * 40) |
| 229 | + |
| 230 | + fvg_df = bars_df.pipe(FVG, min_gap_size=0.0) |
| 231 | + |
| 232 | + # Check results |
| 233 | + bullish_count = fvg_df.filter(pl.col("fvg_bullish")).height |
| 234 | + bearish_count = fvg_df.filter(pl.col("fvg_bearish")).height |
| 235 | + |
| 236 | + print(f"\n📈 Bullish FVGs detected: {bullish_count}") |
| 237 | + print(f"📉 Bearish FVGs detected: {bearish_count}") |
| 238 | + |
| 239 | + if bullish_count == 0 and bearish_count == 0: |
| 240 | + print("\n⚠️ No FVGs detected! The conditions may be too strict.") |
| 241 | + print("The current FVG definition requires:") |
| 242 | + print(" - Three consecutive bars with very specific gap patterns") |
| 243 | + print( |
| 244 | + " - These patterns are extremely rare in normal market conditions" |
| 245 | + ) |
| 246 | + print("\n💡 Suggestion: Consider using simpler gap detection logic") |
| 247 | + |
| 248 | + except Exception as e: |
| 249 | + print(f"\n❌ Error: {e}") |
| 250 | + import traceback |
| 251 | + |
| 252 | + traceback.print_exc() |
| 253 | + |
| 254 | + |
| 255 | +if __name__ == "__main__": |
| 256 | + asyncio.run(main()) |
0 commit comments