wip

Author: jungmair

.github/workflows/build-interface.yml

@@ -0,0 +1,87 @@
+name: Deploy Web Application
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  trigger-build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Trigger build in Repo A
+      id: trigger-build
+      run: |
+        curl -L \
+          -X POST \
+          -H "Accept: application/vnd.github+json" \
+          -H "Authorization: Bearer ${{ secrets.LINGODB_TOKEN }}" \
+          -H "X-GitHub-Api-Version: 2022-11-28" \
+          https://api.github.com/repos/lingo-db/lingo-db/actions/workflows/release-for-webinterface.yml/dispatches \
+          -d '{"ref":"b0d8f7a"}'
+
+    - name: Wait for a few seconds
+      run: sleep 5
+
+    - name: Get workflow run ID
+      id: get-run-id
+      run: |
+        run_id=$(curl -L \
+          -H "Accept: application/vnd.github+json" \
+          -H "Authorization: Bearer ${{ secrets.LINGODB_TOKEN }}" \
+          -H "X-GitHub-Api-Version: 2022-11-28" \
+          https://api.github.com/repos/lingo-db/lingo-db/actions/workflows/release-for-webinterface.yml/runs \
+          | jq -r '.workflow_runs[0].id')
+        echo "::set-output name=run_id::$run_id"
+
+  wait-for-build:
+    runs-on: ubuntu-latest
+    needs: trigger-build
+
+    steps:
+    - name: Wait for build completion
+      id: wait-for-build
+      run: |
+        while true; do
+          status=$(curl -H "Authorization: Bearer ${{ secrets.LINGODB_TOKEN }}" \
+            https://api.github.com/repos/lingo-db/lingo-db/actions/runs/${{ needs.trigger-build.outputs.run_id }} \
+            | jq -r '.status')
+          if [ "$status" == "completed" ]; then
+            conclusion=$(curl -H "Authorization: Bearer ${{ secrets.LINGODB_TOKEN }}" \
+              https://api.github.com/repos/lingo-db/lingo-db/actions/runs/${{ needs.trigger-build.outputs.run_id }} \
+              | jq -r '.conclusion')
+            if [ "$conclusion" == "success" ]; then
+              echo "Build completed successfully."
+              break
+            else
+              echo "Build failed."
+              exit 1
+            fi
+          else
+            echo "Build in progress..."
+            sleep 30
+          fi
+        done
+
+    - name: List artifacts from specific run in Repo A
+      id: list-artifacts
+      run: |
+        artifacts=$(curl -L \
+          -H "Accept: application/vnd.github+json" \
+          -H "Authorization: Bearer ${{ secrets.LINGODB_TOKEN }}" \
+          -H "X-GitHub-Api-Version: 2022-11-28" \
+          https://api.github.com/repos/lingo-db/lingo-db/actions/runs/${{ needs.trigger-build.outputs.run_id }}/artifacts)
+        echo "$artifacts" > artifacts.json
+
+    - name: Extract artifact URL
+      id: extract-url
+      run: |
+        artifact_url=$(jq -r '.artifacts[0].archive_download_url' artifacts.json)
+        echo "::set-output name=url::$artifact_url"
+
+    - name: Download artifacts from Repo A
+      run: |
+        curl -L -o artifacts.zip -H "Authorization: Bearer ${{ secrets.LINGODB_TOKEN }}" \
+          ${{ steps.extract-url.outputs.url }}
+        unzip artifacts.zip -d ./artifacts

Dockerfile

@@ -1,10 +1,18 @@
-FROM gitlab.db.in.tum.de:5005/lingo-db/lingo-db/lingodb:latest
-RUN pip install fastapi uvicorn
+FROM ubuntu:latest
+RUN apt-get update && apt-get install -y python3-pip python3-venv
+ENV VIRTUAL_ENV=/root/venv
+RUN python3 -m venv $VIRTUAL_ENV
+RUN $VIRTUAL_ENV/bin/pip install fastapi uvicorn pyarrow===14.0.0 tbb-devel==2021.11.0
 ENV DATA_ROOT="/data/"
-ENV LINGODB_BINARY_DIR="/build/lingodb/"
+ENV LINGODB_BINARY_DIR="/lingodb/"
+ENV LINGODB_SCRIPT_DIR="/scripts/"
+ENV LD_LIBRARY_PATH=$VIRTUAL_ENV/lib:$VIRTUAL_ENV/lib/python3.12/site-packages/pyarrow:$LD_LIBRARY_PATH
 RUN mkdir /webinterface
 COPY backend/backend.py /webinterface/backend.py
-COPY frontend/build /webinterface/frontend
+COPY docker-build/frontend /webinterface/frontend
+COPY docker-build/lingodb-binaries /lingodb
+RUN mkdir /scripts
+COPY clean-snapshot.sh /scripts/clean-snapshot.sh
 RUN find /webinterface
 WORKDIR /webinterface
-ENTRYPOINT ["uvicorn", "backend:app","--host", "0.0.0.0","--port","80"]
+ENTRYPOINT ["/root/venv/bin/uvicorn" , "backend:app","--host", "0.0.0.0","--port","80"]

backend/backend.py

@@ -62,7 +62,7 @@ def run_sql_query(query_str, db):
 
         # Execute command and capture output
         output = subprocess.check_output(cmd, universal_newlines=True, stderr=subprocess.STDOUT, timeout=20,
-                                         env={"LINGODB_PARALLELISM": "4"})
+                                         env={**os.environ,"LINGODB_PARALLELISM": "4"})
         # Parse output and skip first and last 4 lines
         splitted = output.split("\n")
         header_list = splitted[-2].split()
@@ -107,7 +107,7 @@ async def analyze(database: str = Body(...), query: str = Body(...), real_card:
             print(BINARY_DIR + "run-sql " + query_file + " " + DATA_ROOT + database)
             output = subprocess.check_output([BINARY_DIR + "run-sql", query_file, DATA_ROOT + database],
                                              universal_newlines=True, stderr=subprocess.STDOUT, timeout=20,
-                                             env={"LINGODB_SNAPSHOT_DIR": snapshotdir,
+                                             env={**os.environ,"LINGODB_SNAPSHOT_DIR": snapshotdir,
                                                   "LINGODB_SNAPSHOT_PASSES": "true",
                                                   "LINGODB_SNAPSHOT_LEVEL": "important",
                                                   "LINGODB_EXECUTION_MODE": "NONE"})

clean-snapshot.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# Ensure the script is called with two arguments
+if [[ $# -ne 3 ]]; then
+  echo "Usage: $0 <binary_dir> <source_file> <target_file>"
+  exit 1
+fi
+
+binary_dir=$1
+source_file="$2"
+target_file="$3"
+${binary_dir}/mlir-db-opt --strip-debuginfo ${source_file} > ${target_file}
+
+
+
+# Count lines starting with "#loc" continuously from the beginning
+count=$(awk '!/^#loc/{exit} {count++} END {print count}' ${source_file})
+
+# Create a real temporary file for the empty lines
+temp_file=$(mktemp)
+
+# Generate the empty lines and write them to the temporary file
+for ((i=0; i<count; i++)); do echo ""; done > ${temp_file}
+
+# Prepend the empty lines to the target file
+cat ${target_file} >> ${temp_file} && mv ${temp_file} ${target_file}
+
+