log2json.py

import argparse
import json
import os
import re
from collections import defaultdict


def parse_logs_to_json(log_file: str, output_dir: str):
    """
    Parses a structured log file generated by the benchmark script and
    creates a separate JSON report for each model-compiler run.
    """
    try:
        with open(log_file, "r", encoding="utf-8") as f:
            lines = f.readlines()
    except FileNotFoundError:
        print(f"Error: Log file not found at '{log_file}'")
        return
    except Exception as e:
        print(f"Error reading log file: {e}")
        return

    # This dictionary will hold the parsed data for all runs found in the log file.
    # Key: The model path from the '[Processing]' line, which is unique per run.
    # Value: The dictionary that will be converted to JSON.
    all_runs_data = {}
    current_run_key = None

    # Define regex patterns for each type of log line we need to parse.
    patterns = {
        "processing": re.compile(r"\[Processing\] (.+)"),
        "config": re.compile(r"\[Config\] (\S+): (.+)"),
        "performance": re.compile(r"\[Performance\]\[(\w+)\]: (.+)"),
        "datatype": re.compile(r"\[Datatype\]\[(\w+)\]: (.+)"),
        "correctness": re.compile(r"\[Correctness\](\[.+\]): (.+)"),
        "result_status": re.compile(r"\[Result\] status: (.+)"),
        "failure": re.compile(r"\[Fail due to (.+)\.\]"),
        "speedup": re.compile(r"\[Speedup\]\[(\w+)\]: (.+)"),
    }

    for i, line in enumerate(lines):
        # Check for the start of a new model run
        processing_match = patterns["processing"].search(line)
        if processing_match:
            current_run_key = processing_match.group(1).strip()
            # Initialize a nested dictionary structure for this new run
            all_runs_data[current_run_key] = {
                "configuration": {},
                "correctness": {},
                "performance": {
                    "eager": {},
                    "compiled": {},
                    "datatype": {},
                    "speedup": {},
                },
                "result": {
                    "status": "unknown",
                },
            }
            continue

        # If we haven't identified a run yet, skip the line
        if not current_run_key:
            continue

        # Get the data dictionary for the current run
        data = all_runs_data[current_run_key]

        # Try to match other patterns
        config_match = patterns["config"].search(line)
        if config_match:
            key, value = config_match.groups()
            data["configuration"][key.strip()] = value.strip()
            continue

        performance_match = patterns["performance"].search(line)
        if performance_match:
            key, value_str = performance_match.groups()
            # The performance value is a JSON string, so we load it
            data["performance"][key.strip()] = json.loads(value_str)
            continue

        datatype_match = patterns["datatype"].search(line)
        if datatype_match:
            key, value_str = datatype_match.groups()
            # The datatype value is a space-separated string
            data["performance"]["datatype"][key.strip()] = value_str.strip().split()
            continue

        correctness_match = patterns["correctness"].search(line)
        if correctness_match:
            key, value_str = correctness_match.groups()
            values = []
            for v in value_str.strip().split():
                try:
                    # Try to convert to int if it's a whole number, else float
                    values.append(int(v) if "." not in v else float(v))
                except ValueError:
                    # Handle non-numeric values like 'nan'
                    values.append(float(v))
            data["correctness"][key.strip()] = values
            continue

        # Look for the status, and if it's "failed", look ahead to the next line.
        result_status_match = patterns["result_status"].search(line)
        if result_status_match:
            status = result_status_match.group(1).strip()
            data["result"]["status"] = status
            if status == "failed" and (i + 1) < len(lines):
                error_reason_match = patterns["failure"].search(lines[i + 1])
                if error_reason_match:
                    reason = error_reason_match.group(1).lower()
                    if "eager" in reason:
                        data["performance"]["failure"] = "eager"
                        data["result"]["status"] = "eager_fail"
                    elif "compiled" in reason:
                        data["performance"]["failure"] = "compiled"
                        data["result"]["status"] = "compile_fail"
                    else:
                        data["performance"]["failure"] = "other"
                        data["result"]["status"] = "runtime_fail"
            continue

        speedup_match = patterns["speedup"].search(line)
        if speedup_match:
            key, value_str = speedup_match.groups()
            data["performance"]["speedup"][key.strip()] = float(value_str)
            continue

    # After parsing all lines, write the results to JSON files
    if not all_runs_data:
        print("No processable log entries found in the file.")
        return

    os.makedirs(output_dir, exist_ok=True)

    for run_key, data in all_runs_data.items():
        try:
            path_parts = run_key.split(os.sep)
            # The last part is the subgraph name, e.g., 'subgraph_4'
            subgraph_name = path_parts[-1]
            # The model name is extracted from the configuration
            model_name = data["configuration"]["model"]
            compiler_name = data["configuration"]["compiler"]
            # for PyTorch
            filename = f"{model_name}_{compiler_name}.json"
            # for Paddle
            # filename = f"{model_name}_{subgraph_name}_{compiler_name}.json"
            filepath = os.path.join(output_dir, filename)

            # Build result field with status and speedup
            if data["result"]["status"] == "success":
                speedup_data = {}
                if "e2e" in data["performance"]["speedup"]:
                    speedup_data["e2e"] = {
                        "mean": data["performance"]["speedup"]["e2e"]
                    }
                if "gpu" in data["performance"]["speedup"]:
                    speedup_data["gpu"] = {
                        "mean": data["performance"]["speedup"]["gpu"]
                    }
                if speedup_data:
                    data["result"]["speedup"] = speedup_data

            with open(filepath, "w", encoding="utf-8") as f:
                json.dump(data, f, indent=4)

            print(f"Successfully generated report: {filepath}")

        except KeyError as e:
            print(
                f"Warning: Could not generate report for '{run_key}' due to missing key: {e}"
            )
        except Exception as e:
            print(
                f"Warning: An unexpected error occurred while writing report for '{run_key}': {e}"
            )


def main():
    parser = argparse.ArgumentParser(
        description="Convert benchmark logs to JSON reports.",
        formatter_class=argparse.RawTextHelpFormatter,
    )
    parser.add_argument(
        "--log-file",
        type=str,
        required=True,
        help="Path to the benchmark log file generated by test_compiler.py.",
    )
    parser.add_argument(
        "--output-dir",
        type=str,
        required=True,
        help="Directory to save the structured JSON result files.",
    )
    args = parser.parse_args()
    parse_logs_to_json(args.log_file, args.output_dir)


if __name__ == "__main__":
    main()