Skip to content

Commit 611718c

Browse files
iNeil77iNeil77
committed
Adding compile scripts
1 parent b3cb45d commit 611718c

File tree

9 files changed

+1165
-0
lines changed

9 files changed

+1165
-0
lines changed

‎ir/Compile_C++.py‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
import argparse
2+
import os
3+
import pandas
4+
import pickle
5+
import subprocess
6+
from multiprocessing import (
7+
Pool,
8+
current_process,
9+
)
10+
from tqdm import tqdm
11+
12+
root_path = "/home/ubuntu/Downloads/Collated/C++"
13+
out_path = "/home/ubuntu/Downloads/Scripts/C++"
14+
15+
16+
def compile_worker(dir_list_chunk):
17+
result = pandas.DataFrame(columns=["Source", "Perf_Compile_Status", "Size_Compile_Status"])
18+
result = result.set_index("Source")
19+
with (
20+
open(
21+
f"{out_path}/Logs/out_{current_process().pid}.txt",
22+
"a+"
23+
)
24+
if os.path.exists(f"{out_path}/Logs/out_{current_process().pid}.txt")
25+
else open(
26+
f"{out_path}/Logs/out_{current_process().pid}.txt",
27+
"x+"
28+
) as outfile,
29+
open(
30+
f"{out_path}/Logs/err_{current_process().pid}.txt",
31+
"a+"
32+
)
33+
if os.path.exists(f"{out_path}/Logs/err_{current_process().pid}.txt")
34+
else open(
35+
f"{out_path}/Logs/err_{current_process().pid}.txt",
36+
"x+"
37+
) as errfile
38+
):
39+
for folder in tqdm(
40+
dir_list_chunk,
41+
desc=f"Worker - {current_process().pid}: "
42+
):
43+
src_path = f"{root_path}/{folder}/source.cpp"
44+
perf_path = f"{root_path}/{folder}/llvm_O3.ll"
45+
size_path = f"{root_path}/{folder}/llvm_OZ.ll"
46+
47+
perf_returncode = subprocess.call(
48+
[
49+
"clang++-17", "-S",
50+
"-O3",
51+
"-fno-discard-value-names",
52+
"-fstandalone-debug",
53+
"-emit-llvm",
54+
f"{src_path}",
55+
"-o", f"{perf_path}"
56+
],
57+
shell=False,
58+
stdout=outfile,
59+
stderr=errfile
60+
)
61+
62+
size_returncode = subprocess.call(
63+
[
64+
"clang++-17", "-S",
65+
"-Oz",
66+
"-fno-discard-value-names",
67+
"-fstandalone-debug",
68+
"-emit-llvm",
69+
f"{src_path}",
70+
"-o", f"{size_path}"
71+
],
72+
shell=False,
73+
stdout=outfile,
74+
stderr=errfile
75+
)
76+
77+
result.loc[folder] = [perf_returncode, size_returncode]
78+
79+
return result
80+
81+
82+
83+
def main(args):
84+
dir_list = os.listdir(root_path)
85+
dir_list.sort()
86+
if args.subset:
87+
dir_list = dir_list[:args.subset]
88+
print(f"Obtained {len(dir_list)} source files for compilation")
89+
with (
90+
open(
91+
f"{out_path}/dir_list.pickle",
92+
"wb"
93+
)
94+
if os.path.exists(f"{out_path}/dir_list.pickle")
95+
else open(
96+
f"{out_path}/dir_list.pickle",
97+
"xb"
98+
) as sp
99+
):
100+
pickle.dump(dir_list, sp)
101+
102+
dir_list_chunked = [dir_list[i::args.num_workers] for i in range(args.num_workers)]
103+
with Pool(args.num_workers) as pool:
104+
results_list = pool.map(compile_worker, dir_list_chunked)
105+
results_df = pandas.concat(results_list, ignore_index=False)
106+
results_df.to_parquet(
107+
f"{out_path}/results.parquet"
108+
)
109+
110+
if __name__ == "__main__":
111+
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
112+
parser.add_argument(
113+
"--num_workers",
114+
type=int,
115+
required=False,
116+
default=8,
117+
help="Number of workers to spawn for compilation"
118+
)
119+
parser.add_argument(
120+
"--subset",
121+
type=int,
122+
required=False,
123+
default=None,
124+
help="Number of source files to compile"
125+
)
126+
args = parser.parse_args()
127+
main(args)

‎ir/Compile_C.py‎

Lines changed: 127 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
import argparse
2+
import os
3+
import pandas
4+
import pickle
5+
import subprocess
6+
from multiprocessing import (
7+
Pool,
8+
current_process,
9+
)
10+
from tqdm import tqdm
11+
12+
root_path = "/data/TheStack_IR/OJ_Samples/Collated/C"
13+
out_path = "/data/TheStack_IR/OJ_Samples/Scripts/C"
14+
15+
16+
def compile_worker(dir_list_chunk):
17+
result = pandas.DataFrame(columns=["Source", "Perf_Compile_Status", "Size_Compile_Status"])
18+
result = result.set_index("Source")
19+
with (
20+
open(
21+
f"{out_path}/Logs/out_{current_process().pid}.txt",
22+
"a+"
23+
)
24+
if os.path.exists(f"{out_path}/Logs/out_{current_process().pid}.txt")
25+
else open(
26+
f"{out_path}/Logs/out_{current_process().pid}.txt",
27+
"x+"
28+
) as outfile,
29+
open(
30+
f"{out_path}/Logs/err_{current_process().pid}.txt",
31+
"a+"
32+
)
33+
if os.path.exists(f"{out_path}/Logs/err_{current_process().pid}.txt")
34+
else open(
35+
f"{out_path}/Logs/err_{current_process().pid}.txt",
36+
"x+"
37+
) as errfile
38+
):
39+
for folder in tqdm(
40+
dir_list_chunk,
41+
desc=f"Worker - {current_process().pid}: "
42+
):
43+
src_path = f"{root_path}/{folder}/source.c"
44+
perf_path = f"{root_path}/{folder}/llvm_O3.ll"
45+
size_path = f"{root_path}/{folder}/llvm_OZ.ll"
46+
47+
perf_returncode = subprocess.call(
48+
[
49+
"clang", "-S",
50+
"-O3",
51+
"-fno-discard-value-names",
52+
"-fstandalone-debug",
53+
"-emit-llvm",
54+
f"{src_path}",
55+
"-o", f"{perf_path}"
56+
],
57+
shell=False,
58+
stdout=outfile,
59+
stderr=errfile
60+
)
61+
62+
size_returncode = subprocess.call(
63+
[
64+
"clang", "-S",
65+
"-Oz",
66+
"-fno-discard-value-names",
67+
"-fstandalone-debug",
68+
"-emit-llvm",
69+
f"{src_path}",
70+
"-o", f"{size_path}"
71+
],
72+
shell=False,
73+
stdout=outfile,
74+
stderr=errfile
75+
)
76+
77+
result.loc[folder] = [perf_returncode, size_returncode]
78+
79+
return result
80+
81+
82+
83+
def main(args):
84+
dir_list = os.listdir(root_path)
85+
dir_list.sort()
86+
if args.subset:
87+
dir_list = dir_list[:args.subset]
88+
print(f"Obtained {len(dir_list)} source files for compilation")
89+
with (
90+
open(
91+
f"{out_path}/dir_list.pickle",
92+
"wb"
93+
)
94+
if os.path.exists(f"{out_path}/dir_list.pickle")
95+
else open(
96+
f"{out_path}/dir_list.pickle",
97+
"xb"
98+
)
99+
) as sp:
100+
pickle.dump(dir_list, sp)
101+
102+
dir_list_chunked = [dir_list[i::args.num_workers] for i in range(args.num_workers)]
103+
with Pool(args.num_workers) as pool:
104+
results_list = pool.map(compile_worker, dir_list_chunked)
105+
results_df = pandas.concat(results_list, ignore_index=False)
106+
results_df.to_parquet(
107+
f"{out_path}/results.parquet"
108+
)
109+
110+
if __name__ == "__main__":
111+
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
112+
parser.add_argument(
113+
"--num_workers",
114+
type=int,
115+
required=False,
116+
default=8,
117+
help="Number of workers to spawn for compilation"
118+
)
119+
parser.add_argument(
120+
"--subset",
121+
type=int,
122+
required=False,
123+
default=None,
124+
help="Number of source files to compile"
125+
)
126+
args = parser.parse_args()
127+
main(args)

‎ir/Compile_D.py‎

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
import argparse
2+
import os
3+
import pandas
4+
import pickle
5+
import subprocess
6+
from multiprocessing import (
7+
Pool,
8+
current_process,
9+
)
10+
from tqdm import tqdm
11+
12+
root_path = "/data/TheStack_IR/OJ_Samples/Collated/D"
13+
out_path = "/data/TheStack_IR/OJ_Samples/Scripts/D"
14+
15+
16+
def compile_worker(dir_list_chunk):
17+
result = pandas.DataFrame(columns=["Source", "Perf_Compile_Status", "Size_Compile_Status"])
18+
result = result.set_index("Source")
19+
with (
20+
open(
21+
f"{out_path}/Logs/out_{current_process().pid}.txt",
22+
"a+"
23+
)
24+
if os.path.exists(f"{out_path}/Logs/out_{current_process().pid}.txt")
25+
else open(
26+
f"{out_path}/Logs/out_{current_process().pid}.txt",
27+
"x+"
28+
) as outfile,
29+
open(
30+
f"{out_path}/Logs/err_{current_process().pid}.txt",
31+
"a+"
32+
)
33+
if os.path.exists(f"{out_path}/Logs/err_{current_process().pid}.txt")
34+
else open(
35+
f"{out_path}/Logs/err_{current_process().pid}.txt",
36+
"x+"
37+
) as errfile
38+
):
39+
for folder in tqdm(
40+
dir_list_chunk,
41+
desc=f"Worker - {current_process().pid}: "
42+
):
43+
src_path = f"{root_path}/{folder}/source.d"
44+
perf_path = f"{root_path}/{folder}/llvm_O3.ll"
45+
size_path = f"{root_path}/{folder}/llvm_OZ.ll"
46+
47+
perf_returncode = subprocess.call(
48+
[
49+
"ldc2",
50+
"-O3",
51+
"--output-ll",
52+
"--noasm",
53+
f"{src_path}",
54+
"-of", f"{perf_path}"
55+
],
56+
shell=False,
57+
stdout=outfile,
58+
stderr=errfile
59+
)
60+
61+
size_returncode = subprocess.call(
62+
[
63+
"ldc2",
64+
"-Oz",
65+
"--output-ll",
66+
"--noasm",
67+
f"{src_path}",
68+
"-of", f"{size_path}"
69+
],
70+
shell=False,
71+
stdout=outfile,
72+
stderr=errfile
73+
)
74+
75+
result.loc[folder] = [perf_returncode, size_returncode]
76+
77+
return result
78+
79+
80+
81+
def main(args):
82+
dir_list = os.listdir(root_path)
83+
dir_list.sort()
84+
if args.subset:
85+
dir_list = dir_list[:args.subset]
86+
print(f"Obtained {len(dir_list)} source files for compilation")
87+
with (
88+
open(
89+
f"{out_path}/dir_list.pickle",
90+
"wb"
91+
)
92+
if os.path.exists(f"{out_path}/dir_list.pickle")
93+
else open(
94+
f"{out_path}/dir_list.pickle",
95+
"xb"
96+
)
97+
) as sp:
98+
pickle.dump(dir_list, sp)
99+
100+
dir_list_chunked = [dir_list[i::args.num_workers] for i in range(args.num_workers)]
101+
with Pool(args.num_workers) as pool:
102+
results_list = pool.map(compile_worker, dir_list_chunked)
103+
results_df = pandas.concat(results_list, ignore_index=False)
104+
results_df.to_parquet(
105+
f"{out_path}/results.parquet"
106+
)
107+
108+
if __name__ == "__main__":
109+
parser = argparse.ArgumentParser(formatter_class=argparse.ArgumentDefaultsHelpFormatter)
110+
parser.add_argument(
111+
"--num_workers",
112+
type=int,
113+
required=False,
114+
default=8,
115+
help="Number of workers to spawn for compilation"
116+
)
117+
parser.add_argument(
118+
"--subset",
119+
type=int,
120+
required=False,
121+
default=None,
122+
help="Number of source files to compile"
123+
)
124+
args = parser.parse_args()
125+
main(args)

0 commit comments

Comments
 (0)