diff --git a/lookup_plugins/parse_mt_report.py b/lookup_plugins/parse_mt_report.py index f40b600..88f30e1 100644 --- a/lookup_plugins/parse_mt_report.py +++ b/lookup_plugins/parse_mt_report.py @@ -6,16 +6,58 @@ from bs4 import BeautifulSoup import csv import json +import argparse +import chardet import xml.etree.ElementTree as ET +def parse_spread(value): + # return e.g., "Current" and "30" from such pattern: "Current (30)". Current and 30 may change. + # Regular expression pattern to extract values + pattern = r'(\w+)\s*\((\d+)\)' + match = re.match(pattern, value) + if match: + return { + "type": match[1], + "value": convert_value(match[2]) + } + + pattern = r'(\d+)' + match = re.match(pattern, value) + if match: + return { + "type": 'Unset', + "value": convert_value(match[1]) + } + + raise AnsibleError("Could not parse spread value for parse_spread()! Passed: \"%s\"" % value) + +def parse_symbol(value): + # Return first letters from the value until space or end of line. + return re.match(r'([A-Z]+)', value)[1] + def parse_period(value): # Regular expression pattern to extract values pattern = r'(\w+)\s*\((\d{4}\.\d{2}\.\d{2})\s*-\s*(\d{4}\.\d{2}\.\d{2})\)' if match := re.match(pattern, value): - return {"period": match[1], "date_start": match[2], "date_end": match[3]} - print("Could not parse date for parse_period()! Passed: \"%s\"" % value, file=sys.stderr) - exit(1) + return { + "period": match[1], + "date_start": match[2], + "date_end": match[3] + } + + # If no match, try to to use this pattern: 1 Minute (M1) 2023.01.13 00:00 - 2023.01.23 23:59 (2023.01.13 - 2023.01.24) + # For period it would be M1, for date_start it would be 2023.01.13, for date end it would be 2023.01.23. + pattern = r'.*?\((\w+)\)\s+(\d{4}\.\d{2}\.\d{2}).*?\-\s+(\d{4}\.\d{2}\.\d{2})' + + if match := re.match(pattern, value): + return { + "period": match[1], + "date_start": match[2], + "date_end": match[3] + } + + raise AnsibleError("Could not parse period for parse_period()! Passed: \"%s\"" % value) def parse_val_prc(value): match = re.match(r'(-*\d+(\.\d+)*)\s+\((-*\d+(\.\d+)*)%\)', value) @@ -25,8 +67,7 @@ def parse_val_prc(value): "percentage": convert_value(match.group(3)) } else: - print("Could not parse value for parse_val_prc()! Passed: \"%s\"" % value, file=sys.stderr) - exit(1) + raise AnsibleError("Could not parse value for parse_val_prc()! Passed: \"%s\"" % value) def parse_prc_val(value): match = re.match(r'(-*\d+(\.\d+)*)%\s+\((-*\d+(\.\d+)*)\)', value) @@ -35,8 +76,7 @@ def parse_prc_val(value): "value": convert_value(match[3]), "percentage": convert_value(match[1]) } - print("Could not parse value for parse_prc_val()! Passed: \"%s\"" % value, file=sys.stderr) - exit(1) + raise AnsibleError("Could not parse value for parse_prc_val()! Passed: \"%s\"" % value) def parse_val_of(value): match = re.match(r'(-*\d+(\.\d+)*)\s+\((-*\d+(\.\d+)*)\)', value) @@ -46,14 +86,12 @@ def parse_val_of(value): "of": convert_value(match.group(3)) } else: - print("Could not parse value for parse_val_of()! Passed: \"%s\"" % value, file=sys.stderr) - exit(1) + raise AnsibleError("Could not parse value for parse_val_of()! Passed: \"%s\"" % value) def parse_val_diff(value): if match := re.match(r'(-*\d+(\.\d+)*)\s+\((-*\d+(\.\d+)*)\)', value): return {"value": convert_value(match[1]), "diff": convert_value(match[3])} - print("Could not parse value for parse_val_diff()! Passed: \"%s\"" % value, file=sys.stderr) - exit(1) + raise AnsibleError("Could not parse value for parse_val_diff()! Passed: \"%s\"" % value) def parse_time(value): match = re.match(r'(\d+):(\d+):(\d+)', value) @@ -64,8 +102,7 @@ def parse_time(value): "s": convert_value(match.group(3)) } else: - print("Could not parse value for parse_time()! Passed: \"%s\"" % value, file=sys.stderr) - exit(1) + raise AnsibleError("Could not parse time for parse_time()! Passed: \"%s\"" % value) def convert_value(value): if value.lower() == 'true': @@ -82,17 +119,24 @@ def convert_value(value): return value def extract_header_table(html_content): + # Detecting whether it's MT4 report. We need to check if html_content string contains "Modelling Quality" text. + is_mt4 = "Modelling quality" in html_content + soup = BeautifulSoup(html_content, "html.parser") # Find the table with the specified div content. - table = soup.find('b', string="Strategy Tester Report").find_parent('table') + table = soup.find_all('table')[0] - # Extract pairs of td values. rows = [] + + if is_mt4: + # Extract expert advisor name from the table. It's the first sibling of the div above Strategy Tester Report. + expert_name = soup.find('b', string="Strategy Tester Report").find_parent('div').find_next_sibling('div').text.strip() + rows.append(["Expert:", expert_name]) + + # Extract pairs of td values. for tr in table.find_all('tr'): tds = tr.find_all('td') - values = [] - for td in tds: - values.append(td.text.strip()) + values = [col.text.strip() for col in tds] rows.append(values) data = {} @@ -109,8 +153,8 @@ def extract_header_table(html_content): if i >= len(rows): break - if i < 4: - continue + # if i < 4: + # continue if len(rows[i]) == 0: continue @@ -118,16 +162,25 @@ def extract_header_table(html_content): key = rows[i][0] value = rows[i][1] if len(rows[i]) > 1 else '' - if key == "Expert:": + # Remove trailing spaces and colons from the key. + key = key.rstrip(':').strip().lower() + + # Expert + if key == "expert": data["expert"] = convert_value(value) - elif key == "Symbol:": - data["symbol"] = convert_value(value) - elif key == "Period:": + # Symbol + elif key == "symbol": + data["symbol"] = parse_symbol(value) + # Period + # Date Start + # Date End + elif key == "period": period = parse_period(value) data["period"] = period["period"] data["date_start"] = period["date_start"] data["date_end"] = period["date_end"] - elif key == "Inputs:": + # Inputs + elif key == "inputs": data["inputs"] = {} # Starting from the same row as "Inputs:". for input_i in range(i, len(rows)): @@ -141,83 +194,177 @@ def extract_header_table(html_content): data["inputs"][input_key] = convert_value(input_value) i = input_i - elif key == "Currency:": + # Inputs (MT4) + elif key == "parameters": + data["inputs"] = {} + values = rows[i][1].split(";") + for k in range(0, len(values)): + if values[k] == "": + continue + (input_key, input_value) = values[k].split("=") + data["inputs"][input_key.strip()] = convert_value(input_value) + # Currency ? + elif key == "currency": data["currency"] = value - elif key == "Initial Deposit:": + # Initial Deposit + # Spread (MT4 only) + elif key == "initial deposit": data["initial_deposit"] = convert_value(value) - elif key == "Leverage:": + if is_mt4: + data["spread"] = parse_spread(rows[i][5]) + # Leverage ? + elif key == "leverage": data["leverage"] = convert_value(value.split(":")[0]) / convert_value(value.split(":")[1]) - elif key == "History Quality:": + # History Quality ? + elif key == "history quality": data["history_quality"] = convert_value(value.split("%")[0]) - elif key == "Bars:": + # Bars + # Ticks ? + # Symbols ? + # Modelling Quality (MT4) + elif key == "bars": data["bars"] = convert_value(value) data["ticks"] = convert_value(rows[i][3]) data["symbols"] = convert_value(rows[i][5]) - elif key == "Total Net Profit:": + elif key == "bars in test": # MT4 + data["bars"] = convert_value(value) + data["ticks"] = convert_value(rows[i][3]) + data["modelling_quality"] = convert_value(rows[i][5]) + # @todo + data["symbols"] = 1 + # Mismached charts errors (MT4 only) + elif key == "mismatched charts errors": + data["mismatched_charts_errors"] = convert_value(value) + # Total Net Profit + # Balance Drawdown Absolute + # Equity Drawdown Absolute + elif key == "total net profit": data["total_net_profit"] = convert_value(value) - data["balance_drawdown_absolute"] = convert_value(rows[i][3]) - data["equity_drawdown_absolute"] = convert_value(rows[i][5]) - elif key == "Gross Profit:": + if is_mt4: + data["gross_profit"] = convert_value(rows[i][3]) + data["gross_loss"] = convert_value(rows[i][5]) + else: + data["balance_drawdown_absolute"] = convert_value(rows[i][3]) + data["equity_drawdown_absolute"] = convert_value(rows[i][5]) + + # Gross Profit (for MT4 is's a part of Total Net Profit) + # Balance Drawdown Maximal (MT5 here) + # Equity Drawdown Maximal (MT5 here) + elif key == "gross profit": data["gross_profit"] = convert_value(value) data["balance_drawdown_maximal"] = parse_val_prc(rows[i][3]) data["equity_drawdown_maximal"] = parse_val_prc(rows[i][5]) - elif key == "Gross Loss:": + # Balance Drawdown Absolute (MT4 here) ? Not sure if it's correct. + # Balance Drawdown Maximal (MT4 here) ? Not sure if it's correct. + # Balance Drawdown Relative (MT4 here) ? Not sure if it's correct. + elif key == "absolute drawdown": + data["balance_drawdown_absolute"] = convert_value(value) + data["balance_drawdown_maximal"] = parse_val_prc(rows[i][3]) + data["balance_drawdown_relative"] = parse_prc_val(rows[i][5]) + # Gross Loss (for MT4 is's a part of Total Net Profit) + # Balance Drawdown Relative ? + # Equity Drawdown Relative ? + elif key == "gross loss": data["gross_loss"] = convert_value(value) data["balance_drawdown_relative"] = parse_prc_val(rows[i][3]) data["equity_drawdown_relative"] = parse_prc_val(rows[i][5]) - elif key == "Profit Factor:": + # Profit Factor + # Expected Payoff + # Margin Level + elif key == "profit factor": data["profit_factor"] = convert_value(value) data["expected_payoff"] = convert_value(rows[i][3]) data["margin_level"] = convert_value(rows[i][5]) - elif key == "Recovery Factor:": + # Recovery Factor ? + # Sharpe Ratio ? + # Z-Score ? + elif key == "recovery factor": data["recovery_factor"] = convert_value(value) data["sharpe_ratio"] = convert_value(rows[i][3]) data["z_score"] = parse_val_prc(rows[i][5]) - elif key == "AHPR:": + # AHPR ? + # LR Correlation ? + # OnTester Result ? + elif key == "ahpr": data["ahpr"] = parse_val_prc(value) data["lr_correlation"] = convert_value(rows[i][3]) data["ontester_result"] = convert_value(rows[i][5]) - elif key == "GHPR:": + # GHPR ? + # LR Standard Error ? + elif key == "ghpr": data["ghpr"] = parse_val_prc(value) data["lr_standard_error"] = convert_value(rows[i][3]) - elif key == "Total Trades:": - data["total_trades"] = convert_value(value) - data["short_trades_won"] = parse_val_prc(rows[i][3]) - data["long_trades_won"] = parse_val_prc(rows[i][5]) - elif key == "Total Deals:": + # Total Deals / Total Trades (MT5) + # Profit Trades (MT5) + # Loss Trades (MT5) + elif key == "total deals": data["total_deals"] = convert_value(value) data["profit_trades"] = parse_val_prc(rows[i][3]) data["loss_trades"] = parse_val_prc(rows[i][5]) - elif value == "Largest profit trade:": + # Profit Trades (MT4) + # Loss Trades (MT4) + elif value == "Profit trades (% of total)": + data["profit_trades"] = parse_val_prc(rows[i][2]) + data["loss_trades"] = parse_val_prc(rows[i][4]) + # Total Deals / Total Trades (MT4) + # Short Trades / Short Positions (MT4) + # Long Trades / Long Positions (MT4) + elif key == "total trades": + data["total_trades"] = convert_value(value) + data["short_trades_won"] = parse_val_prc(rows[i][3]) + data["long_trades_won"] = parse_val_prc(rows[i][5]) + # Largest Profit trade + # Largest Loss trade + elif value == "largest profit trade" or (key == "largest" and value == "profit trade"): data["largest_profit_trade"] = convert_value(rows[i][2]) data["largest_loss_trade"] = convert_value(rows[i][4]) - elif value == "Average profit trade:": + # Average Frofit Trade + # Average Loss Trade + elif value == "average profit trade" or (key == "average" and value == "profit trade"): data["average_profit_trade"] = convert_value(rows[i][2]) data["average_loss_trade"] = convert_value(rows[i][4]) - elif value == "Maximum :": - data["maximum"] = parse_val_of(rows[i][2]) + # Maximum Consecutive Wins + # Maximum Consecutive Losses + elif value == "maximum" or (key == "maximum" and value == "consecutive wins (profit in money)"): + data["maximum_consecutive_wins"] = parse_val_of(rows[i][2]) data["maximum_consecutive_losses"] = parse_val_diff(rows[i][4]) - elif value == "Maximal :": - data["maximal"] = parse_val_of(rows[i][2]) + # Maximal Consecutive Wins + # Maximal Consecutive Losses + elif value == "maximal" or (key == "maximal" and value == "consecutive profit (count of wins)"): + data["maximal_consecutive_profit"] = parse_val_of(rows[i][2]) data["maximal_consecutive_losses_num"] = parse_val_diff(rows[i][4]) - elif value == "Average :": - data["average"] = convert_value(rows[i][2]) + # Average Consecutive Wins + # Average Consecutive Losses + elif value == "average" or (key == "average" and value == "consecutive wins"): + data["average_consecutive_wins"] = convert_value(rows[i][2]) data["average_consecutive_losses"] = convert_value(rows[i][4]) - elif key == "Correlation (Profits,MFE):": + # Correlation Profits MFE ? + # Correlation Profits MAE ? + # Correlation Profits MFE MAE ? + elif key == "correlation (profits,mfe)": data["correlation_profits_mfe"] = convert_value(value) data["correlation_profits_mae"] = convert_value(rows[i][3]) data["correlation_profits_mfe_mae"] = convert_value(rows[i][5]) - elif key == "Minimal position holding time:": + # Minimal Position Holding Time ? + # Maximal Position Holding Time ? + # Average Position Holding Time ? + elif key == "minimal position holding time": data["minimal_position_holding_time"] = parse_time(value) data["maximal_position_holding_time"] = parse_time(rows[i][3]) data["average_position_holding_time"] = parse_time(rows[i][5]) else: pass # Skip row. + return data def extract_orders_table(html_content): - html_content = "