Skip to content

Commit 6e332c8

Browse files
dinmukhamedmdinmukhamedm
authored
query engine for self-hosted (#878)
* query engine for self-hosted * fix ellipsis, add trace tags * fix docker compose
1 parent 29cd8c6 commit 6e332c8

27 files changed

+1676
-5
lines changed

‎.env‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,9 @@ RABBITMQ_DEFAULT_USER=admin
88
RABBITMQ_DEFAULT_PASS=adminpasswd
99

1010
CLICKHOUSE_USER=ch_user
11+
CLICKHOUSE_RO_USER=ch_user
1112
CLICKHOUSE_PASSWORD=ch_passwd
13+
CLICKHOUSE_RO_PASSWORD=ch_passwd
1214
POSTGRES_PORT=5433
1315
# must be exactly 32 bytes (64 hex characters)
1416
AEAD_SECRET_KEY=0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef
@@ -17,10 +19,10 @@ NEXTAUTH_SECRET=0123456789abcdef0123456789abcdef
1719

1820

1921
# Uncomment and set these to override the default exposed ports.
20-
# APP_SERVER_HOST_PORT=8080
21-
# APP_SERVER_GRPC_HOST_PORT=8081
22+
# APP_SERVER_HOST_PORT=8000
23+
# APP_SERVER_GRPC_HOST_PORT=8001
2224
# FRONTEND_HOST_PORT=5667
23-
# AGENT_MANAGER_HOST_PORT=8901
25+
# QUERY_ENGINE_HOST_PORT=8903
2426

2527
# Uncomment and set this to enable OpenAI support. Currently used in Next.js
2628
# to generate chat names.

‎.github/workflows/build-push.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,9 @@ jobs:
2020
- dockerfile: ./frontend/Dockerfile
2121
context: ./frontend
2222
image: ghcr.io/lmnr-ai/frontend
23+
- dockerfile: ./query-engine/Dockerfile
24+
context: ./query-engine
25+
image: ghcr.io/lmnr-ai/query-engine
2326
permissions:
2427
contents: read
2528
packages: write

‎.github/workflows/query-engine-tests-check.yml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
name: Check Query Engine Tests
2+
3+
on:
4+
pull_request:
5+
types:
6+
- synchronize
7+
- opened
8+
- reopened
9+
paths:
10+
- 'query-engine/**'
11+
12+
jobs:
13+
test:
14+
runs-on: ubuntu-latest
15+
16+
steps:
17+
- name: Checkout code
18+
uses: actions/checkout@v4
19+
- name: Set up Python
20+
uses: actions/setup-python@v5
21+
with:
22+
python-version: 3.13
23+
- name: Install uv
24+
uses: astral-sh/setup-uv@v6
25+
with:
26+
activate-environment: true
27+
- name: Install the project
28+
working-directory: ./query-engine
29+
run: uv sync --all-extras --dev
30+
- name: Run tests
31+
working-directory: ./query-engine
32+
run: uv run pytest

‎app-server/.env.example‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@ RABBITMQ_URL=amqp://admin:adminpasswd@localhost:5672/%2f
99
CLICKHOUSE_URL=http://localhost:8123
1010
CLICKHOUSE_USER=ch_user
1111
CLICKHOUSE_PASSWORD=ch_passwd
12+
CLICKHOUSE_RO_USER=ch_user
13+
CLICKHOUSE_RO_PASSWORD=ch_passwd
1214

1315
SHARED_SECRET_TOKEN=some_secret
1416
# must be exactly 32 bytes (64 hex characters)
@@ -18,5 +20,4 @@ ENVIRONMENT=FULL
1820
# Optional, if you want to use Redis for caching. Create a Redis/Valkey instance and set the URL here.
1921
# REDIS_URL=redis://localhost:6379
2022

21-
22-
QUERY_ENGINE_URL=
23+
QUERY_ENGINE_URL=http://localhost:8903

‎docker-compose-full.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,19 @@ services:
5959
timeout: 5s
6060
retries: 5
6161

62+
query-engine:
63+
image: ghcr.io/lmnr-ai/query-engine
64+
pull_policy: always
65+
ports:
66+
- "${QUERY_ENGINE_HOST_PORT:-8903}:8903"
67+
environment:
68+
PORT: 8903
69+
healthcheck:
70+
test: ["CMD", "sleep", "5"]
71+
interval: 10s
72+
timeout: 6s
73+
retries: 1
74+
6275
app-server:
6376
image: ghcr.io/lmnr-ai/app-server
6477
pull_policy: always
@@ -72,6 +85,8 @@ services:
7285
condition: service_healthy
7386
clickhouse:
7487
condition: service_started
88+
query-engine:
89+
condition: service_started
7590
environment:
7691
PORT: 8000
7792
GRPC_PORT: 8001
@@ -80,9 +95,12 @@ services:
8095
CLICKHOUSE_URL: http://clickhouse:8123
8196
CLICKHOUSE_USER: ${CLICKHOUSE_USER}
8297
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
98+
CLICKHOUSE_RO_USER: ${CLICKHOUSE_RO_USER}
99+
CLICKHOUSE_RO_PASSWORD: ${CLICKHOUSE_RO_PASSWORD}
83100
SHARED_SECRET_TOKEN: ${SHARED_SECRET_TOKEN}
84101
ENVIRONMENT: FULL
85102
AEAD_SECRET_KEY: ${AEAD_SECRET_KEY}
103+
QUERY_ENGINE_URL: http://query-engine:8903
86104

87105
frontend:
88106
image: ghcr.io/lmnr-ai/frontend

‎docker-compose-local-build.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,20 @@ services:
5454
timeout: 5s
5555
retries: 5
5656

57+
query-engine:
58+
build:
59+
context: ./query-engine
60+
container_name: query-engine
61+
ports:
62+
- "${QUERY_ENGINE_HOST_PORT:-8903}:8903"
63+
environment:
64+
PORT: 8903
65+
healthcheck:
66+
test: ["CMD", "sleep", "5"]
67+
interval: 10s
68+
timeout: 6s
69+
retries: 1
70+
5771
app-server:
5872
ports:
5973
- "${APP_SERVER_HOST_PORT:-8000}:8000"
@@ -68,6 +82,8 @@ services:
6882
condition: service_healthy
6983
clickhouse:
7084
condition: service_started
85+
query-engine:
86+
condition: service_healthy
7187
environment:
7288
PORT: 8000
7389
GRPC_PORT: 8001
@@ -77,8 +93,11 @@ services:
7793
CLICKHOUSE_URL: http://clickhouse:8123
7894
CLICKHOUSE_USER: ${CLICKHOUSE_USER}
7995
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
96+
CLICKHOUSE_RO_USER: ${CLICKHOUSE_RO_USER}
97+
CLICKHOUSE_RO_PASSWORD: ${CLICKHOUSE_RO_PASSWORD}
8098
ENVIRONMENT: FULL
8199
AEAD_SECRET_KEY: ${AEAD_SECRET_KEY}
100+
QUERY_ENGINE_URL: http://query-engine:8903
82101

83102
frontend:
84103
build:

‎docker-compose-local-dev-full.yml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,20 @@ services:
6363
timeout: 5s
6464
retries: 5
6565

66+
query-engine:
67+
build:
68+
context: ./query-engine
69+
container_name: query-engine
70+
ports:
71+
- "${QUERY_ENGINE_HOST_PORT:-8903}:8903"
72+
environment:
73+
PORT: 8903
74+
healthcheck:
75+
test: ["CMD", "sleep", "5"]
76+
interval: 10s
77+
timeout: 6s
78+
retries: 1
79+
6680
volumes:
6781
clickhouse-data:
6882
clickhouse-logs:

‎docker-compose-local-dev.yml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,19 @@ services:
5151
nofile:
5252
soft: 262144
5353
hard: 262144
54+
55+
query-engine:
56+
image: ghcr.io/lmnr-ai/query-engine
57+
pull_policy: always
58+
ports:
59+
- "${QUERY_ENGINE_HOST_PORT:-8903}:8903"
60+
environment:
61+
PORT: 8903
62+
healthcheck:
63+
test: ["CMD", "sleep", "5"]
64+
interval: 10s
65+
timeout: 6s
66+
retries: 1
5467

5568
app-server:
5669
image: ghcr.io/lmnr-ai/app-server
@@ -63,6 +76,8 @@ services:
6376
condition: service_healthy
6477
clickhouse:
6578
condition: service_started
79+
query-engine:
80+
condition: service_healthy
6681

6782
environment:
6883
PORT: 8000
@@ -72,8 +87,11 @@ services:
7287
CLICKHOUSE_URL: http://clickhouse:8123
7388
CLICKHOUSE_USER: ${CLICKHOUSE_USER}
7489
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
90+
CLICKHOUSE_RO_USER: ${CLICKHOUSE_RO_USER}
91+
CLICKHOUSE_RO_PASSWORD: ${CLICKHOUSE_RO_PASSWORD}
7592
ENVIRONMENT: LITE # this disables runtime dependency on rabbitmq
7693
AEAD_SECRET_KEY: ${AEAD_SECRET_KEY}
94+
QUERY_ENGINE_URL: http://query-engine:8903
7795

7896
volumes:
7997
postgres-data:

‎docker-compose.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,19 @@ services:
4343
nofile:
4444
soft: 262144
4545
hard: 262144
46+
47+
query-engine:
48+
image: ghcr.io/lmnr-ai/query-engine
49+
pull_policy: always
50+
ports:
51+
- "${QUERY_ENGINE_HOST_PORT:-8903}:8903"
52+
environment:
53+
PORT: 8903
54+
healthcheck:
55+
test: ["CMD", "sleep", "5"]
56+
interval: 10s
57+
timeout: 6s
58+
retries: 1
4659

4760
frontend:
4861
image: ghcr.io/lmnr-ai/frontend
@@ -64,6 +77,8 @@ services:
6477
CLICKHOUSE_URL: http://clickhouse:8123
6578
CLICKHOUSE_USER: ${CLICKHOUSE_USER}
6679
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
80+
CLICKHOUSE_RO_USER: ${CLICKHOUSE_RO_USER}
81+
CLICKHOUSE_RO_PASSWORD: ${CLICKHOUSE_RO_PASSWORD}
6782
OPENAI_API_KEY: ${OPENAI_API_KEY}
6883
AEAD_SECRET_KEY: ${AEAD_SECRET_KEY}
6984

@@ -88,6 +103,7 @@ services:
88103
CLICKHOUSE_PASSWORD: ${CLICKHOUSE_PASSWORD}
89104
ENVIRONMENT: LITE # this disables runtime dependency on rabbitmq
90105
AEAD_SECRET_KEY: ${AEAD_SECRET_KEY}
106+
QUERY_ENGINE_URL: http://query-engine:8903
91107

92108
volumes:
93109
postgres-data:

‎query-engine/.dockerignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
.venv
2+
__pycache__

0 commit comments

Comments
 (0)