build: use a single compose file with .env file

chore: add .env.debug configuration

chore: add .env.debug

feat: primitive parquet reader with page pruning

add poetry build for libviewer

add libviewer to rows

refactor: only extract metadata and don't try to calculate offset index

ci: update dockerfiles to include the rust toolchain and libviewer

chore: pin python to 3.12.11 in libviewer and update lockfile

feat: use PageIndexPolicy to optionally read offset index

feat: support querying RowsIndex with page pruning

build: add libviewer as a dependency to libcommon

style: ruff format libcommon changes

chore: use query_with_page_pruning from the rows endpoint

chore: fix mypy errors

style: import Sequence from collections.abc

build: don't use libviewer as an editable dependency

build: try to configure poetry to properly install libviewer

ci: temporarily disable poetry cache

style: fixx ruff check errors

build: relock projects depending on libcommon

build: add rust toolchain to more dockerfiles

build: copy the entire libviewer directory in dockerfiles because poetry install is called at the build phase

build: turn libviewer an optional dependency due to build difficulties

chore: missing api stage from dockerfile

ci: install libviewer extra in the libcommon build

style: fix ruff check error in parquet utils

ci: disable poetry cache

feat: raise TooBigRows exceptions if the scan size would exceed a limit

feat: implement binary truncation for page pruning reader

style: ignore variable shadowing ruff check

ci: install libviewer in the worker image

feat: pass hf_token to the opendal store

chore: remove files_to_index estimation

chore: poetry lock worker service

chore: remove reduntand gitignore entries from libviewer

ci: install libviewer in the worker build

style: fix mypy ignore

chore: cleanup the libviewer python code

style: try to please mypy due to missing import

style: make token optional

test: make the mocking compatible with the page pruning reader in test_first_rows

Author: kszucs

.dockerignore

@@ -14,6 +14,7 @@ assets
 **/.pytest_cache
 **/.coverage
 **/coverage.xml
-
-# rust build artifacts
-libs/libviewer/target
+**/target
+**/*.rs.bk
+**/.cargo/registry
+**/.cargo/git

.github/workflows/_unit-tests-python.yml

@@ -8,6 +8,10 @@ on:
       working-directory:
         required: true
         type: string
+      poetry-args:
+        required: false
+        type: string
+        default: ""
 env:
   # required to get access to use a cached poetry venv in "/home/runner/.cache/pypoetry/virtualenvs"
   POETRY_VIRTUALENVS_IN_PROJECT: false
@@ -31,16 +35,16 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.python-version }}
-          cache: "poetry"
-          cache-dependency-path: |
-            ${{ inputs.working-directory }}/poetry.lock
+          # cache: "poetry"
+          # cache-dependency-path: |
+          #   ${{ inputs.working-directory }}/poetry.lock
       - name: Install packages
         run: sudo apt update; sudo apt install -y ffmpeg libavcodec-extra libsndfile1
       - name: Install dependencies
         # "poetry env use" is required: https://github.com/actions/setup-python/issues/374#issuecomment-1088938718
         run: |
           poetry env use "${{ env.python-version }}"
-          poetry install
+          poetry install ${{ inputs.poetry-args }}
       - name: Launch mongo
         env:
           MONGO_PORT: ${{ env.mongo-port }}

.github/workflows/l-libcommon.yml

@@ -29,4 +29,5 @@ jobs:
     uses: ./.github/workflows/_unit-tests-python.yml
     with:
       working-directory: libs/libcommon
+      poetry-args: "--with libviewer"
     secrets: inherit

.github/workflows/s-worker.yml

@@ -31,4 +31,5 @@ jobs:
     uses: ./.github/workflows/_unit-tests-python.yml
     with:
       working-directory: services/worker
+      poetry-args: "--with libviewer"
     secrets: inherit

Dockerfile

@@ -1,120 +1,140 @@
 # Multi-stage Dockerfile for all dataset-viewer services and jobs
 # Build with: docker build --target <service_name> -t <tag> .
 
+ARG PYTHON_VERSION=3.12.11
+FROM python:${PYTHON_VERSION}-slim AS viewer
+
+# Install Rust and minimal build deps
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends curl build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Rust toolchain and maturin
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \
+    && . $HOME/.cargo/env \
+    && pip install maturin \
+    && rustc --version \
+    && cargo --version
+# Add cargo bin dir to PATH (so maturin + cargo available globally)
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Build libviewer
+COPY libs/libviewer /src/libs/libviewer
+WORKDIR /src/libs/libviewer
+RUN maturin build --release --strip --out /tmp/dist
+
 # Base stage with shared setup
-FROM python:3.12.11-slim AS common
+FROM python:${PYTHON_VERSION}-slim AS common
 
 # System dependencies
 RUN apt-get update \
     && apt-get install -y unzip wget procps htop ffmpeg libavcodec-extra libsndfile1 \
     && rm -rf /var/lib/apt/lists/*
 
 # Common environment variables
+ARG POETRY_VERSION=2.1.4
 ENV PYTHONFAULTHANDLER=1 \
     PYTHONUNBUFFERED=1 \
     PYTHONHASHSEED=random \
     PIP_NO_CACHE_DIR=1 \
     PIP_DISABLE_PIP_VERSION_CHECK=on \
     PIP_DEFAULT_TIMEOUT=100 \
     POETRY_NO_INTERACTION=1 \
-    POETRY_VERSION=2.1.4 \
     POETRY_VIRTUALENVS_CREATE=false \
     PATH="$PATH:/root/.local/bin"
 
 # Install pip and poetry
-RUN pip install -U pip && pip install "poetry==$POETRY_VERSION"
+RUN pip install -U pip && pip install "poetry==${POETRY_VERSION}"
 
 # Install libcommon's dependencies but not libcommon itself
 COPY libs/libcommon/poetry.lock \
      libs/libcommon/pyproject.toml \
      /src/libs/libcommon/
-RUN poetry install --no-cache --no-root --no-directory -P /src/libs/libcommon
-
-# Base image for services including libapi's dependencies
-FROM common AS service
-COPY libs/libapi/poetry.lock \
-     libs/libapi/pyproject.toml \
-     /src/libs/libapi/
-RUN poetry install --no-cache --no-root --no-directory -P /src/libs/libapi
+WORKDIR /src/libs/libcommon
+RUN poetry install --no-cache --no-root
 
 # Below are the actual API services which depend on libapi and libcommon.
-# Since the majority of the dependencies are already installed in the `api`
-# we let poetry to actually install the `libs` and the specific service.
+# Since the majority of the dependencies are already installed in the
+# `common` stage we let poetry to handle the rest.
 
 # API service
-FROM service AS api
+FROM common AS api
 COPY libs /src/libs
 COPY services/api /src/services/api
-RUN poetry install --no-cache -P /src/services/api
-WORKDIR /src/services/api/
+WORKDIR /src/services/api
+RUN poetry install --no-cache
 ENTRYPOINT ["poetry", "run", "python", "src/api/main.py"]
 
 # Admin service
-FROM service AS admin
+FROM common AS admin
 COPY libs /src/libs
 COPY services/admin /src/services/admin
-RUN poetry install --no-cache -P /src/services/admin
-WORKDIR /src/services/admin/
+WORKDIR /src/services/admin
+RUN poetry install --no-cache
 ENTRYPOINT ["poetry", "run", "python", "src/admin/main.py"]
 
 # Rows service
-FROM service AS rows
+FROM common AS rows
+COPY --from=viewer /tmp/dist /tmp/dist
+RUN pip install /tmp/dist/libviewer-*.whl
 COPY libs /src/libs
 COPY services/rows /src/services/rows
-RUN poetry install --no-cache -P /src/services/rows
-WORKDIR /src/services/rows/
+WORKDIR /src/services/rows
+RUN poetry install --no-cache
 ENTRYPOINT ["poetry", "run", "python", "src/rows/main.py"]
 
 # Search service
-FROM service AS search
+FROM common AS search
 COPY libs /src/libs
 COPY services/search /src/services/search
-RUN poetry install --no-cache -P /src/services/search
-WORKDIR /src/services/search/
+WORKDIR /src/services/search
+RUN poetry install --no-cache
 ENTRYPOINT ["poetry", "run", "python", "src/search/main.py"]
 
 # SSE API service
-FROM service AS sse-api
+FROM common AS sse-api
 COPY libs /src/libs
 COPY services/sse-api /src/services/sse-api
-RUN poetry install --no-cache -P /src/services/sse-api
-WORKDIR /src/services/sse-api/
+WORKDIR /src/services/sse-api
+RUN poetry install --no-cache
 ENTRYPOINT ["poetry", "run", "python", "src/sse_api/main.py"]
 
 # Webhook service
-FROM service AS webhook
+FROM common AS webhook
 COPY libs /src/libs
 COPY services/webhook /src/services/webhook
-RUN poetry install --no-cache -P /src/services/webhook
-WORKDIR /src/services/webhook/
+WORKDIR /src/services/webhook
+RUN poetry install --no-cache
 ENTRYPOINT ["poetry", "run", "python", "src/webhook/main.py"]
 
 # Worker service
 FROM common AS worker
+COPY --from=viewer /tmp/dist /tmp/dist
+RUN pip install /tmp/dist/libviewer-*.whl
 COPY libs /src/libs
 COPY services/worker /src/services/worker
+WORKDIR /src/services/worker
 # presidio-analyzer > spacy > thinc doesn't ship aarch64 wheels so need to compile
 RUN if [ "$(uname -m)" = "aarch64" ]; then \
       apt-get update && apt-get install -y build-essential && \
       rm -rf /var/lib/apt/lists/*; \
     fi
-RUN poetry install --no-cache -P /src/services/worker
+RUN poetry install --no-cache
 RUN python -m spacy download en_core_web_lg
-WORKDIR /src/services/worker/
 ENTRYPOINT ["poetry", "run", "python", "src/worker/main.py"]
 
 # Cache maintenance job
 FROM common AS cache_maintenance
 COPY libs /src/libs
 COPY jobs/cache_maintenance /src/jobs/cache_maintenance
-RUN poetry install --no-cache -P /src/jobs/cache_maintenance
-WORKDIR /src/jobs/cache_maintenance/
+WORKDIR /src/jobs/cache_maintenance
+RUN poetry install --no-cache
 ENTRYPOINT ["poetry", "run", "python", "src/cache_maintenance/main.py"]
 
 # MongoDB migration job
 FROM common AS mongodb_migration
 COPY libs /src/libs
 COPY jobs/mongodb_migration /src/jobs/mongodb_migration
-RUN poetry install --no-cache -P /src/jobs/mongodb_migration
-WORKDIR /src/jobs/mongodb_migration/
-ENTRYPOINT ["poetry", "run", "python", "src/mongodb_migration/main.py"]
+WORKDIR /src/jobs/mongodb_migration
+RUN poetry install --no-cache
+ENTRYPOINT ["poetry", "run", "python", "src/mongodb_migration/main.py"]

Makefile

@@ -49,6 +49,7 @@ e2e:
 
 .PHONY: install
 install:
+	$(MAKE) -C libs/libviewer install
 	$(MAKE) -C libs/libcommon install
 	$(MAKE) -C libs/libapi install
 	$(MAKE) -C jobs/cache_maintenance install