Skip to content

Commit 3b124bd

Browse files
jungmairjungmair
committed
Cleanup & Build with github actions
1 parent 66377df commit 3b124bd

39 files changed

+230
-19884
lines changed

‎.github/workflows/build-interface.yml‎

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
name: Build webinterface docker
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
env:
8+
REGISTRY: ghcr.io
9+
IMAGE_NAME: ${{ github.repository }}
10+
LINGODB_REPOSITORY: lingo-db/lingodb-test-actions
11+
12+
jobs:
13+
build-docker:
14+
runs-on: ubuntu-latest
15+
permissions:
16+
contents: read
17+
packages: write
18+
attestations: write
19+
id-token: write
20+
steps:
21+
- name: Log in to the Container registry
22+
uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
23+
with:
24+
registry: ${{ env.REGISTRY }}
25+
username: ${{ github.actor }}
26+
password: ${{ secrets.GITHUB_TOKEN }}
27+
- name: Checkout repository
28+
uses: actions/checkout@v3
29+
- name: Set up Node.js
30+
uses: actions/setup-node@v3
31+
with:
32+
node-version: 18
33+
- name: Create docker-build path
34+
run: mkdir docker-uild
35+
- name: Install dependencies
36+
run: yarn install
37+
working-directory: frontend
38+
- name: Build WebInterface Frontend
39+
run: env REACT_APP_API_URL="" env BUILD_PATH="../../../docker-build/frontend" yarn workspace @lingodb/interface build
40+
working-directory: frontend
41+
- name: Authenticate GitHub CLI
42+
run: echo "${{ secrets.LINGODB_TOKEN }}" | gh auth login --with-token
43+
- name: Read LingoDB Commit
44+
id: get-lingodb-commit
45+
run: echo "lingodb_commit=$(cat lingodb-commit.txt)" >> $GITHUB_OUTPUT
46+
- name: Trigger build of lingodb binaries
47+
id: trigger-build
48+
run: |
49+
gh api repos/${{ env.LINGODB_REPOSITORY }}/actions/workflows/release-for-webinterface.yml/dispatches -F ref="main" -F inputs[ref]="b0d8f7a6f5be75a1b545ac54764473048f36b5d7"
50+
51+
- name: Wait for a few seconds
52+
run: sleep 5
53+
54+
- name: Get workflow run ID
55+
id: get-run-id
56+
run: |
57+
run_id=$(gh api repos/${{ env.LINGODB_REPOSITORY }}/actions/workflows/release-for-webinterface.yml/runs | jq -r '.workflow_runs[0].id')
58+
echo "build_run_id=$run_id" >> $GITHUB_OUTPUT
59+
- name: Get Workflow Run Status and Wait
60+
id: wait-for-run
61+
run: |
62+
build_run_id="${{ steps.get-run-id.outputs.build_run_id }}"
63+
64+
echo "Checking status of workflow run with ID $build_run_id"
65+
66+
while :; do
67+
status=$(gh api \
68+
repos/${{ env.LINGODB_REPOSITORY }}/actions/runs/$build_run_id \
69+
--jq '.status')
70+
71+
echo "Current status: $status"
72+
73+
if [[ "$status" == "completed" ]]; then
74+
conclusion=$(gh api \
75+
repos/${{ env.LINGODB_REPOSITORY }}/actions/runs/$build_run_id \
76+
--jq '.conclusion')
77+
78+
echo "Workflow completed with conclusion: $conclusion"
79+
80+
if [[ "$conclusion" != "success" ]]; then
81+
echo "Workflow did not succeed. Failing current workflow."
82+
exit 1
83+
fi
84+
85+
break
86+
fi
87+
88+
echo "Waiting for 10 seconds before checking again..."
89+
sleep 10
90+
done
91+
- name: Download and Extract LingoDB binaries
92+
run: |
93+
build_run_id="${{ steps.get-run-id.outputs.build_run_id }}"
94+
95+
echo "Fetching artifact for workflow run ID $build_run_id"
96+
97+
artifact_id=$(gh api \
98+
repos/${{ env.LINGODB_REPOSITORY }}/actions/runs/$build_run_id/artifacts \
99+
--jq '.artifacts[0].id')
100+
101+
echo "Artifact ID: $artifact_id"
102+
103+
gh api \
104+
repos/${{ env.LINGODB_REPOSITORY }}/actions/artifacts/$artifact_id/zip > artifact.zip
105+
106+
mkdir -p docker-build
107+
unzip artifact.zip -d docker-build/lingodb-binaries
108+
109+
echo "Artifact extracted to docker-build directory."
110+
- name: Build and push Docker image
111+
id: push
112+
uses: docker/build-push-action@f2a1d5e99d037542a71f64918e516c093c6f3fc4
113+
with:
114+
context: .
115+
push: true
116+
tags: ghcr.io/lingo-db/webinterface:latest, ghcr.io/lingo-db/webinterface:${{ github.sha }}
117+
- name: Generate artifact attestation
118+
uses: actions/attest-build-provenance@v2
119+
with:
120+
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
121+
subject-digest: ${{ steps.push.outputs.digest }}
122+
push-to-registry: true
123+

‎.github/workflows/publish-insights.yml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: Build and Deploy Insights App
2+
3+
on:
4+
push:
5+
branches:
6+
- main
7+
8+
# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
9+
permissions:
10+
contents: read
11+
pages: write
12+
id-token: write
13+
14+
# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
15+
# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
16+
concurrency:
17+
group: "pages"
18+
cancel-in-progress: false
19+
20+
jobs:
21+
build:
22+
runs-on: ubuntu-latest
23+
24+
steps:
25+
- name: Checkout repository
26+
uses: actions/checkout@v3
27+
28+
- name: Set up Node.js
29+
uses: actions/setup-node@v3
30+
with:
31+
node-version: 18
32+
33+
- name: Install dependencies
34+
run: yarn install
35+
working-directory: frontend
36+
37+
- name: Build Insights App
38+
run: yarn workspace @lingodb/insights build
39+
working-directory: frontend
40+
41+
42+
- name: Upload artifact
43+
uses: actions/upload-pages-artifact@v1
44+
with:
45+
path: ./frontend/packages/insights/build
46+
47+
# Deployment job
48+
deploy:
49+
environment:
50+
name: github-pages
51+
url: ${{ steps.deployment.outputs.page_url }}
52+
runs-on: ubuntu-latest
53+
needs: build
54+
steps:
55+
- name: Deploy to GitHub Pages
56+
id: deployment
57+
uses: actions/deploy-pages@v2

‎Dockerfile‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,18 @@
1-
FROM gitlab.db.in.tum.de:5005/lingo-db/lingo-db/lingodb:latest
2-
RUN pip install fastapi uvicorn
1+
FROM ubuntu:latest
2+
RUN apt-get update && apt-get install -y python3-pip python3-venv
3+
ENV VIRTUAL_ENV=/root/venv
4+
RUN python3 -m venv $VIRTUAL_ENV
5+
RUN $VIRTUAL_ENV/bin/pip install fastapi uvicorn pyarrow===14.0.0 tbb-devel==2021.11.0
36
ENV DATA_ROOT="/data/"
4-
ENV LINGODB_BINARY_DIR="/build/lingodb/"
7+
ENV LINGODB_BINARY_DIR="/lingodb/"
8+
ENV LINGODB_SCRIPT_DIR="/scripts/"
9+
ENV LD_LIBRARY_PATH=$VIRTUAL_ENV/lib:$VIRTUAL_ENV/lib/python3.12/site-packages/pyarrow:$LD_LIBRARY_PATH
510
RUN mkdir /webinterface
611
COPY backend/backend.py /webinterface/backend.py
7-
COPY frontend/build /webinterface/frontend
12+
COPY docker-build/frontend /webinterface/frontend
13+
COPY docker-build/lingodb-binaries /lingodb
14+
RUN mkdir /scripts
15+
COPY clean-snapshot.sh /scripts/clean-snapshot.sh
816
RUN find /webinterface
917
WORKDIR /webinterface
10-
ENTRYPOINT ["uvicorn", "backend:app","--host", "0.0.0.0","--port","80"]
18+
ENTRYPOINT ["/root/venv/bin/uvicorn" , "backend:app","--host", "0.0.0.0","--port","80"]

‎backend/backend.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,7 @@ def run_sql_query(query_str, db):
6262

6363
# Execute command and capture output
6464
output = subprocess.check_output(cmd, universal_newlines=True, stderr=subprocess.STDOUT, timeout=20,
65-
env={"LINGODB_PARALLELISM": "4"})
65+
env={**os.environ,"LINGODB_PARALLELISM": "4"})
6666
# Parse output and skip first and last 4 lines
6767
splitted = output.split("\n")
6868
header_list = splitted[-2].split()
@@ -107,7 +107,7 @@ async def analyze(database: str = Body(...), query: str = Body(...), real_card:
107107
print(BINARY_DIR + "run-sql " + query_file + " " + DATA_ROOT + database)
108108
output = subprocess.check_output([BINARY_DIR + "run-sql", query_file, DATA_ROOT + database],
109109
universal_newlines=True, stderr=subprocess.STDOUT, timeout=20,
110-
env={"LINGODB_SNAPSHOT_DIR": snapshotdir,
110+
env={**os.environ,"LINGODB_SNAPSHOT_DIR": snapshotdir,
111111
"LINGODB_SNAPSHOT_PASSES": "true",
112112
"LINGODB_SNAPSHOT_LEVEL": "important",
113113
"LINGODB_EXECUTION_MODE": "NONE"})

‎clean-snapshot.sh‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
3+
# Ensure the script is called with two arguments
4+
if [[ $# -ne 3 ]]; then
5+
echo "Usage: $0 <binary_dir> <source_file> <target_file>"
6+
exit 1
7+
fi
8+
9+
binary_dir=$1
10+
source_file="$2"
11+
target_file="$3"
12+
${binary_dir}/mlir-db-opt --strip-debuginfo ${source_file} > ${target_file}
13+
14+
15+
16+
# Count lines starting with "#loc" continuously from the beginning
17+
count=$(awk '!/^#loc/{exit} {count++} END {print count}' ${source_file})
18+
19+
# Create a real temporary file for the empty lines
20+
temp_file=$(mktemp)
21+
22+
# Generate the empty lines and write them to the temporary file
23+
for ((i=0; i<count; i++)); do echo ""; done > ${temp_file}
24+
25+
# Prepend the empty lines to the target file
26+
cat ${target_file} >> ${temp_file} && mv ${temp_file} ${target_file}
27+
28+

‎frontend-old/.gitignore‎

Lines changed: 0 additions & 24 deletions
This file was deleted.

‎frontend-old/README.md‎

Lines changed: 0 additions & 46 deletions
This file was deleted.

‎frontend-old/index.js‎

Lines changed: 0 additions & 1 deletion
This file was deleted.

0 commit comments

Comments
 (0)