OlympusPipeliner/process-input-files.py at main · ShogunYash/OlympusPipeliner · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
#!/usr/bin/env python3
import os
import subprocess
import sys
import glob
import time

def process_input_files(input_dir, src_dir, output_dir):
    """
    Process all .csv input files using forward and noforward executables.

    Args:
    input_dir (str): Directory containing input .csv files
    src_dir (str): Directory containing executables
    output_dir (str): Directory to store output files
    """
    # Validate input directories exist
    if not os.path.exists(input_dir):
        print(f"Error: Input directory {input_dir} does not exist.")
        sys.exit(1)

    if not os.path.exists(src_dir):
        print(f"Error: Source directory {src_dir} does not exist.")
        sys.exit(1)

    # Create output directory if it doesn't exist
    os.makedirs(output_dir, exist_ok=True)

    # Path to executables
    forward_exe = os.path.join(src_dir, 'forward')
    noforward_exe = os.path.join(src_dir, 'noforward')

    # Validate executables exist
    if not os.path.exists(forward_exe):
        print(f"Error: forward executable not found in {src_dir}")
        sys.exit(1)

    if not os.path.exists(noforward_exe):
        print(f"Error: noforward executable not found in {src_dir}")
        sys.exit(1)

    # Process each .csv file
    for filename in os.listdir(input_dir):
        if filename.endswith('.txt'):
            input_file_path = os.path.join(input_dir, filename)
            base_filename = os.path.splitext(filename)[0]

            # Clear previous output files that might conflict
            for old_file in glob.glob(f"{output_dir}/{base_filename}_*_out.csv"):
                os.remove(old_file)

            # Run forward executable with full path to output directory
            try:
                print(f"Processing {filename} with forward...")
                env = os.environ.copy()
                env["OUTPUT_DIR"] = output_dir  # Pass output dir as environment variable

                forward_output = subprocess.run(
                    [forward_exe, input_file_path, '50'],
                    capture_output=True,
                    text=True,
                    env=env,
                    cwd=script_dir  # Run from script directory
                )

                # Check for output file in possible locations
                time.sleep(1)  # Give a moment for file system to update

                # Check multiple potential output locations
                possible_paths = [
                    os.path.join(output_dir, f"{base_filename}_forward_out.csv"),
                    os.path.join(script_dir, "outputfiles", f"{base_filename}_forward_out.csv"),
                    os.path.join(os.path.dirname(script_dir), "outputfiles", f"{base_filename}_forward_out.csv")
                ]

                found_path = None
                for path in possible_paths:
                    if os.path.exists(path):
                        found_path = path
                        break

                if found_path:
                    # If found but not in the desired location, copy it there
                    if found_path != os.path.join(output_dir, f"{base_filename}_forward_out.csv"):
                        import shutil
                        target_path = os.path.join(output_dir, f"{base_filename}_forward_out.csv")
                        shutil.copy2(found_path, target_path)
                        print(f"Found output at {found_path}, copied to {target_path}")
                    else:
                        print(f"Successfully processed {filename} with forward, output at {found_path}")
                else:
                    print(f"Warning: Could not find output file for {filename} with forward")
                    print(f"Command output: {forward_output.stdout}")
                    print(f"Command errors: {forward_output.stderr}")
            except subprocess.CalledProcessError as e:
                print(f"Error processing {filename} with forward: {e}")
                continue

            # Run noforward executable with similar logic
            try:
                print(f"Processing {filename} with noforward...")
                env = os.environ.copy()
                env["OUTPUT_DIR"] = output_dir

                noforward_output = subprocess.run(
                    [noforward_exe, input_file_path, '50'],
                    capture_output=True,
                    text=True,
                    env=env,
                    cwd=script_dir
                )

                time.sleep(1)

                possible_paths = [
                    os.path.join(output_dir, f"{base_filename}_no_forward_out.csv"),
                    os.path.join(script_dir, "outputfiles", f"{base_filename}_no_forward_out.csv"),
                    os.path.join(os.path.dirname(script_dir), "outputfiles", f"{base_filename}_no_forward_out.csv")
                ]

                found_path = None
                for path in possible_paths:
                    if os.path.exists(path):
                        found_path = path
                        break

                if found_path:
                    if found_path != os.path.join(output_dir, f"{base_filename}_no_forward_out.csv"):
                        import shutil
                        target_path = os.path.join(output_dir, f"{base_filename}_no_forward_out.csv")
                        shutil.copy2(found_path, target_path)
                        print(f"Found output at {found_path}, copied to {target_path}")
                    else:
                        print(f"Successfully processed {filename} with noforward, output at {found_path}")
                else:
                    print(f"Warning: Could not find output file for {filename} with noforward")
                    print(f"Command output: {noforward_output.stdout}")
                    print(f"Command errors: {noforward_output.stderr}")
            except subprocess.CalledProcessError as e:
                print(f"Error processing {filename} with noforward: {e}")
                continue

def main():
    # Set directories - adjust these paths as needed
    global script_dir
    script_dir = os.path.dirname(os.path.abspath(__file__))
    input_dir = os.path.join(script_dir, 'inputfiles')
    src_dir = os.path.join(script_dir, 'src')
    output_dir = os.path.join(script_dir, 'outputfiles')

    process_input_files(input_dir, src_dir, output_dir)

if __name__ == "__main__":
    main()