Skip to content

Commit 66377df

Browse files
jungmairjungmair
committed
wip
1 parent 3a97f57 commit 66377df

File tree

4 files changed

+226
-335
lines changed

4 files changed

+226
-335
lines changed

‎backend/backend.py‎

Lines changed: 35 additions & 177 deletions
Original file line numberDiff line numberDiff line change
@@ -47,113 +47,6 @@ def table_to_json(raw_table):
4747
return result_dict
4848

4949

50-
def get_analyzed_query_plan(query_str, db):
51-
if db not in ["tpch-1", "tpcds-1", "uni", "job"]:
52-
raise RuntimeError("Unknown Database")
53-
# Write query to temporary file
54-
with tempfile.NamedTemporaryFile(mode="w", delete=True) as f:
55-
f.write(query_str)
56-
query_file = f.name
57-
f.flush()
58-
# Define the chained command as a string
59-
command = BINARY_DIR + "sql-to-mlir " + query_file + " " + DATA_ROOT + db + " | " + BINARY_DIR + "mlir-db-opt --use-db " + DATA_ROOT + db + " --relalg-query-opt --relalg-track-tuples"
60-
print(command)
61-
# Create a temporary file to store the output
62-
with tempfile.NamedTemporaryFile(mode='w', delete=True) as tmpfile:
63-
# Call the chained command using subprocess.run()
64-
result = subprocess.run(command, shell=True, stdout=tmpfile, text=True, timeout=5,
65-
env={"LINGODB_PARALLELISM": "4"})
66-
67-
# Check that the command exited successfully
68-
if result.returncode == 0:
69-
try:
70-
# Build command string
71-
cmd = [BINARY_DIR + "mlir-to-json", tmpfile.name,
72-
DATA_ROOT + db]
73-
74-
# Execute command and capture output
75-
output = subprocess.check_output(cmd, universal_newlines=True, stderr=subprocess.STDOUT, timeout=20)
76-
return json.loads(output.split("\n")[0]);
77-
78-
except subprocess.CalledProcessError as e:
79-
# Print error message to stderr
80-
print(e.output, file=sys.stderr)
81-
raise HTTPException(status_code=400, detail="Query could not be executed")
82-
except subprocess.TimeoutExpired as e:
83-
raise HTTPException(status_code=400, detail="Query took too long")
84-
else:
85-
# Print the error message
86-
print(result.stderr.strip())
87-
88-
def analyze(query_str, db):
89-
if db not in ["tpch-1", "tpcds-1", "uni", "job"]:
90-
raise RuntimeError("Unknown Database")
91-
with tempfile.NamedTemporaryFile(mode="w", delete=True) as f:
92-
f.write(query_str)
93-
query_file = f.name
94-
f.flush()
95-
with tempfile.TemporaryDirectory() as snapshotdir:
96-
print(BINARY_DIR+"run-sql "+query_file+" "+ DATA_ROOT + db)
97-
output = subprocess.check_output([BINARY_DIR+"run-sql",query_file, DATA_ROOT + db], universal_newlines=True, stderr=subprocess.STDOUT, timeout=20,
98-
env={"LINGODB_SNAPSHOT_DIR": snapshotdir, "LINGODB_SNAPSHOT_PASSES": "true", "LINGODB_SNAPSHOT_LEVEL":"important", "LINGODB_EXECUTION_MODE":"NONE"})
99-
result = subprocess.run(f"bash {SCRIPT_DIR}/clean-snapshot.sh {BINARY_DIR} {snapshotdir}/important-snapshot-qopt.mlir {snapshotdir}/important-snapshot-qopt.mlir.alt",
100-
universal_newlines=True, stderr=subprocess.STDOUT, shell=True)
101-
print(result)
102-
result = subprocess.run(f"bash {SCRIPT_DIR}/clean-snapshot.sh {BINARY_DIR} {snapshotdir}/important-snapshot-subop-opt.mlir {snapshotdir}/important-snapshot-subop-opt.mlir.alt",
103-
universal_newlines=True, stderr=subprocess.STDOUT, shell=True)
104-
print(os.listdir(snapshotdir))
105-
106-
relalg_plan = subprocess.check_output([BINARY_DIR + "mlir-to-json", snapshotdir+"/important-snapshot-qopt.mlir.alt"],
107-
universal_newlines=True, stderr=subprocess.STDOUT)
108-
subop_plan = subprocess.check_output([BINARY_DIR + "mlir-subop-to-json", snapshotdir+"/important-snapshot-subop-opt.mlir.alt"],
109-
universal_newlines=True)
110-
analyzed_snapshots = subprocess.check_output([BINARY_DIR + "mlir-analyze-snapshots", snapshotdir+"/important-snapshot-info.json"],
111-
universal_newlines=True)
112-
print(os.listdir(snapshotdir))
113-
return {"plan":json.loads(relalg_plan.split("\n")[0]), "subopplan":json.loads(subop_plan.split("\n")[0]), "mlir": json.loads(analyzed_snapshots)}
114-
115-
116-
117-
return {}
118-
119-
120-
def get_query_plan(query_str, db):
121-
if db not in ["tpch-1", "tpcds-1", "uni", "job"]:
122-
raise RuntimeError("Unknown Database")
123-
# Write query to temporary file
124-
with tempfile.NamedTemporaryFile(mode="w", delete=True) as f:
125-
f.write(query_str)
126-
query_file = f.name
127-
f.flush()
128-
# Define the chained command as a string
129-
command = BINARY_DIR + "sql-to-mlir " + query_file + " " + DATA_ROOT + db + " | " + BINARY_DIR + "/mlir-db-opt --use-db " + DATA_ROOT + db + " --relalg-query-opt"
130-
print(command)
131-
# Create a temporary file to store the output
132-
with tempfile.TemporaryDirectory(mode='w', delete=True) as tmpfile:
133-
# Call the chained command using subprocess.run()
134-
result = subprocess.run(command, shell=True, stdout=tmpfile, text=True, timeout=5)
135-
136-
# Check that the command exited successfully
137-
if result.returncode == 0:
138-
try:
139-
# Build command string
140-
cmd = [BINARY_DIR + "mlir-to-json", tmpfile.name]
141-
142-
# Execute command and capture output
143-
output = subprocess.check_output(cmd, universal_newlines=True, stderr=subprocess.STDOUT, timeout=5)
144-
return json.loads(output.split("\n")[0]);
145-
146-
except subprocess.CalledProcessError as e:
147-
# Print error message to stderr
148-
print(e.output, file=sys.stderr)
149-
raise HTTPException(status_code=400, detail="Query could not be executed")
150-
except json.decoder.JSONDecodeError as e:
151-
raise HTTPException(status_code=400, detail="JSON Query Plan could not be created")
152-
else:
153-
# Print the error message
154-
print(result.stderr.strip())
155-
156-
15750
def run_sql_query(query_str, db):
15851
if db not in ["tpch-1", "tpcds-1", "uni", "job"]:
15952
raise HTTPException(status_code=403, detail="Unknown Database")
@@ -202,88 +95,53 @@ def run_sql_query(query_str, db):
20295
os.remove(query_file)
20396

20497

205-
def sql_to_mlir(query_str, db):
206-
if db not in ["tpch-1", "tpcds-1", "uni", "job"]:
207-
raise HTTPException(status_code=403, detail="Unknown Database")
208-
# Write query to temporary file
98+
@api_app.post("/analyze")
99+
async def analyze(database: str = Body(...), query: str = Body(...), real_card: bool = Body(...)):
100+
if database not in ["tpch-1", "tpcds-1", "uni", "job"]:
101+
raise RuntimeError("Unknown Database")
209102
with tempfile.NamedTemporaryFile(mode="w", delete=True) as f:
210-
f.write(query_str)
103+
f.write(query)
211104
query_file = f.name
212105
f.flush()
213-
try:
214-
# Build command string
215-
cmd = [BINARY_DIR + "sql-to-mlir", query_file,
216-
DATA_ROOT + db]
217-
# Execute command and capture output
218-
output = subprocess.check_output(cmd, universal_newlines=True, timeout=5)
219-
print(cmd)
220-
return output
221-
222-
except subprocess.CalledProcessError as e:
223-
# Print error message to stderr
224-
print(e.output, file=sys.stderr)
225-
raise HTTPException(status_code=400, detail="Failed to generate MLIR")
226-
227-
228-
def mlir_opt(mlir_str, db, opts):
229-
if db and db not in ["tpch-1", "tpcds-1", "uni", "job"]:
230-
raise HTTPException(status_code=403, detail="Unknown Database")
231-
# Write query to temporary file
232-
with tempfile.NamedTemporaryFile(mode="w", delete=True) as f:
233-
f.write(mlir_str)
234-
mlir_file = f.name
235-
f.flush()
236-
try:
237-
# Build command string
238-
cmd = [BINARY_DIR + "mlir-db-opt"]
239-
if db:
240-
cmd.append("--use-db")
241-
cmd.append(DATA_ROOT + db)
242-
cmd.extend(opts)
243-
cmd.append(mlir_file)
244-
# Execute command and capture output
245-
output = subprocess.check_output(cmd, universal_newlines=True, timeout=5)
246-
print(cmd)
247-
return output
248-
249-
except subprocess.CalledProcessError as e:
250-
# Print error message to stderr
251-
print(e.output, file=sys.stderr)
252-
raise HTTPException(status_code=400, detail="Failed to generate MLIR")
253-
254-
255-
@api_app.post("/query_plan")
256-
async def query_plan(database: str = Body(...), query: str = Body(...)):
257-
return analyze(query, database)
106+
with tempfile.TemporaryDirectory() as snapshotdir:
107+
print(BINARY_DIR + "run-sql " + query_file + " " + DATA_ROOT + database)
108+
output = subprocess.check_output([BINARY_DIR + "run-sql", query_file, DATA_ROOT + database],
109+
universal_newlines=True, stderr=subprocess.STDOUT, timeout=20,
110+
env={"LINGODB_SNAPSHOT_DIR": snapshotdir,
111+
"LINGODB_SNAPSHOT_PASSES": "true",
112+
"LINGODB_SNAPSHOT_LEVEL": "important",
113+
"LINGODB_EXECUTION_MODE": "NONE"})
114+
result = subprocess.run(
115+
f"bash {SCRIPT_DIR}/clean-snapshot.sh {BINARY_DIR} {snapshotdir}/important-snapshot-qopt.mlir {snapshotdir}/important-snapshot-qopt.mlir.alt",
116+
universal_newlines=True, stderr=subprocess.STDOUT, shell=True)
117+
print(result)
118+
result = subprocess.run(
119+
f"bash {SCRIPT_DIR}/clean-snapshot.sh {BINARY_DIR} {snapshotdir}/important-snapshot-subop-opt.mlir {snapshotdir}/important-snapshot-subop-opt.mlir.alt",
120+
universal_newlines=True, stderr=subprocess.STDOUT, shell=True)
121+
print(os.listdir(snapshotdir))
258122

123+
relalg_plan = subprocess.check_output(
124+
[BINARY_DIR + "mlir-to-json", snapshotdir + "/important-snapshot-qopt.mlir.alt"] + (
125+
[DATA_ROOT + database] if real_card else []),
126+
universal_newlines=True, stderr=subprocess.STDOUT)
127+
subop_plan = subprocess.check_output(
128+
[BINARY_DIR + "mlir-subop-to-json", snapshotdir + "/important-snapshot-subop-opt.mlir.alt"],
129+
universal_newlines=True)
130+
analyzed_snapshots = subprocess.check_output(
131+
[BINARY_DIR + "mlir-analyze-snapshots", snapshotdir + "/important-snapshot-info.json"],
132+
universal_newlines=True)
133+
print(os.listdir(snapshotdir))
134+
return {"plan": json.loads(relalg_plan.split("\n")[0]), "subopplan": json.loads(subop_plan.split("\n")[0]),
135+
"mlir": json.loads(analyzed_snapshots)}
259136

260-
@api_app.post("/analyzed_query_plan")
261-
async def analyzed_query_plan(database: str = Body(...), query: str = Body(...)):
262-
return get_analyzed_query_plan(query, database)
137+
return {}
263138

264139

265140
@api_app.post("/execute")
266141
async def execute(database: str = Body(...), query: str = Body(...)):
267142
return run_sql_query(query, database)
268143

269144

270-
@api_app.post("/mlir_steps")
271-
async def mlir_steps(database: str = Body(...), query: str = Body(...)):
272-
canonical = sql_to_mlir(query, database)
273-
qopt = mlir_opt(canonical, database, ["--relalg-query-opt"])
274-
subop = mlir_opt(qopt, None, ["--lower-relalg-to-subop"])
275-
imperative = mlir_opt(subop, None, ["--lower-subop"])
276-
lowlevel = mlir_opt(imperative, None, ["--lower-db", "--lower-dsa"])
277-
278-
return {
279-
"canonical": canonical,
280-
"qopt": qopt,
281-
"subop": subop,
282-
"imperative": imperative,
283-
"lowlevel": lowlevel,
284-
}
285-
286-
287145
app.mount("/api", api_app)
288146
if "WEBINTERFACE_LOCAL" not in os.environ:
289147
app.mount("/", StaticFiles(directory="/webinterface/frontend", html=True), name="frontend")

‎backend/test.py‎

Lines changed: 0 additions & 64 deletions
This file was deleted.

0 commit comments

Comments
 (0)