feat: primitive parquet reader with page pruning

.dockerignore

@@ -14,6 +14,7 @@ assets
 **/.pytest_cache
 **/.coverage
 **/coverage.xml
-
-# rust build artifacts
-libs/libviewer/target
+**/target
+**/*.rs.bk
+**/.cargo/registry
+**/.cargo/git

.github/workflows/_unit-tests-python.yml

@@ -8,6 +8,10 @@ on:
       working-directory:
         required: true
         type: string
+      poetry-args:
+        required: false
+        type: string
+        default: ""
 env:
   # required to get access to use a cached poetry venv in "/home/runner/.cache/pypoetry/virtualenvs"
   POETRY_VIRTUALENVS_IN_PROJECT: false
@@ -31,16 +35,16 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ env.python-version }}
-          cache: "poetry"
-          cache-dependency-path: |
-            ${{ inputs.working-directory }}/poetry.lock
+          # cache: "poetry"
+          # cache-dependency-path: |
+          #   ${{ inputs.working-directory }}/poetry.lock
       - name: Install packages
         run: sudo apt update; sudo apt install -y ffmpeg libavcodec-extra libsndfile1
       - name: Install dependencies
         # "poetry env use" is required: https://github.com/actions/setup-python/issues/374#issuecomment-1088938718
         run: |
           poetry env use "${{ env.python-version }}"
-          poetry install
+          poetry install ${{ inputs.poetry-args }}
       - name: Launch mongo
         env:
           MONGO_PORT: ${{ env.mongo-port }}

.github/workflows/l-libcommon.yml

@@ -9,24 +9,33 @@
       - main
     paths:
       - "libs/libcommon/**"
+      - "libs/libviewer/**"
       - ".github/workflows/l-libcommon.yml"
       - ".github/workflows/_quality-python.yml"
       - ".github/workflows/_unit-tests-python.yml"
       - "tools/docker-compose-mongo.yml"
   pull_request:
     paths:
       - "libs/libcommon/**"
+      - "libs/libviewer/**"
       - ".github/workflows/l-libcommon.yml"
       - ".github/workflows/_quality-python.yml"
       - ".github/workflows/_unit-tests-python.yml"
       - "tools/docker-compose-mongo.yml"
 jobs:
-  quality:
+  quality-libcommon:
     uses: ./.github/workflows/_quality-python.yml
     with:
       working-directory: libs/libcommon
-  unit-tests:
+  unit-tests-libcommon:
     uses: ./.github/workflows/_unit-tests-python.yml
     with:
       working-directory: libs/libcommon
+      poetry-args: "--with libviewer"
     secrets: inherit
+  unit-tests-libviewer:
+    uses: ./.github/workflows/_unit-tests-python.yml
+    with:
+      working-directory: libs/libviewer
+      poetry-args: "--with dev"
+    secrets: inherit

.github/workflows/s-worker.yml

@@ -31,4 +31,5 @@ jobs:
     uses: ./.github/workflows/_unit-tests-python.yml
     with:
       working-directory: services/worker
+      poetry-args: "--with libviewer"
     secrets: inherit

Dockerfile

@@ -1,120 +1,141 @@
 # Multi-stage Dockerfile for all dataset-viewer services and jobs
 # Build with: docker build --target <service_name> -t <tag> .
 
+ARG PYTHON_VERSION=3.12.11
+FROM python:${PYTHON_VERSION}-slim AS viewer
+
+# Install Rust and minimal build deps
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends curl build-essential \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install Rust toolchain and maturin
+RUN curl https://sh.rustup.rs -sSf | sh -s -- -y \
+    && . $HOME/.cargo/env \
+    && pip install maturin \
+    && rustc --version \
+    && cargo --version
+# Add cargo bin dir to PATH (so maturin + cargo available globally)
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Build libviewer
+COPY libs/libviewer /src/libs/libviewer
+WORKDIR /src/libs/libviewer
+# RUN maturin build --release --strip --out /tmp/dist
+RUN maturin build --profile dev --out /tmp/dist
+
 # Base stage with shared setup
-FROM python:3.12.11-slim AS common
+FROM python:${PYTHON_VERSION}-slim AS common
 
 # System dependencies
 RUN apt-get update \
     && apt-get install -y unzip wget procps htop ffmpeg libavcodec-extra libsndfile1 \
     && rm -rf /var/lib/apt/lists/*
 
 # Common environment variables
+ARG POETRY_VERSION=2.1.4
 ENV PYTHONFAULTHANDLER=1 \
     PYTHONUNBUFFERED=1 \
     PYTHONHASHSEED=random \
     PIP_NO_CACHE_DIR=1 \
     PIP_DISABLE_PIP_VERSION_CHECK=on \
     PIP_DEFAULT_TIMEOUT=100 \
     POETRY_NO_INTERACTION=1 \
-    POETRY_VERSION=2.1.4 \
     POETRY_VIRTUALENVS_CREATE=false \
     PATH="$PATH:/root/.local/bin"
 
 # Install pip and poetry
-RUN pip install -U pip && pip install "poetry==$POETRY_VERSION"
+RUN pip install -U pip && pip install "poetry==${POETRY_VERSION}"
 
 # Install libcommon's dependencies but not libcommon itself
 COPY libs/libcommon/poetry.lock \
      libs/libcommon/pyproject.toml \
      /src/libs/libcommon/
-RUN poetry install --no-cache --no-root --no-directory -P /src/libs/libcommon
-
-# Base image for services including libapi's dependencies
-FROM common AS service
-COPY libs/libapi/poetry.lock \
-     libs/libapi/pyproject.toml \
-     /src/libs/libapi/
-RUN poetry install --no-cache --no-root --no-directory -P /src/libs/libapi
+WORKDIR /src/libs/libcommon
+RUN poetry install --no-cache --no-root
 
 # Below are the actual API services which depend on libapi and libcommon.
-# Since the majority of the dependencies are already installed in the `api`
-# we let poetry to actually install the `libs` and the specific service.
+# Since the majority of the dependencies are already installed in the
+# `common` stage we let poetry to handle the rest.
 
 # API service
-FROM service AS api
+FROM common AS api
 COPY libs /src/libs
 COPY services/api /src/services/api
-RUN poetry install --no-cache -P /src/services/api
-WORKDIR /src/services/api/
+WORKDIR /src/services/api
+RUN poetry install --no-cache
 ENTRYPOINT ["poetry", "run", "python", "src/api/main.py"]
 
 # Admin service
-FROM service AS admin
+FROM common AS admin
 COPY libs /src/libs
 COPY services/admin /src/services/admin
-RUN poetry install --no-cache -P /src/services/admin
-WORKDIR /src/services/admin/
+WORKDIR /src/services/admin
+RUN poetry install --no-cache
 ENTRYPOINT ["poetry", "run", "python", "src/admin/main.py"]
 
 # Rows service
-FROM service AS rows
+FROM common AS rows
+COPY --from=viewer /tmp/dist /tmp/dist
+RUN pip install /tmp/dist/libviewer-*.whl
 COPY libs /src/libs
 COPY services/rows /src/services/rows
-RUN poetry install --no-cache -P /src/services/rows
-WORKDIR /src/services/rows/
+WORKDIR /src/services/rows
+RUN poetry install --no-cache
 ENTRYPOINT ["poetry", "run", "python", "src/rows/main.py"]
 
 # Search service
-FROM service AS search
+FROM common AS search
 COPY libs /src/libs
 COPY services/search /src/services/search
-RUN poetry install --no-cache -P /src/services/search
-WORKDIR /src/services/search/
+WORKDIR /src/services/search
+RUN poetry install --no-cache
 ENTRYPOINT ["poetry", "run", "python", "src/search/main.py"]
 
 # SSE API service
-FROM service AS sse-api
+FROM common AS sse-api
 COPY libs /src/libs
 COPY services/sse-api /src/services/sse-api
-RUN poetry install --no-cache -P /src/services/sse-api
-WORKDIR /src/services/sse-api/
+WORKDIR /src/services/sse-api
+RUN poetry install --no-cache
 ENTRYPOINT ["poetry", "run", "python", "src/sse_api/main.py"]
 
 # Webhook service
-FROM service AS webhook
+FROM common AS webhook
 COPY libs /src/libs
 COPY services/webhook /src/services/webhook
-RUN poetry install --no-cache -P /src/services/webhook
-WORKDIR /src/services/webhook/
+WORKDIR /src/services/webhook
+RUN poetry install --no-cache
 ENTRYPOINT ["poetry", "run", "python", "src/webhook/main.py"]
 
 # Worker service
 FROM common AS worker
+COPY --from=viewer /tmp/dist /tmp/dist
+RUN pip install /tmp/dist/libviewer-*.whl
 COPY libs /src/libs
 COPY services/worker /src/services/worker
+WORKDIR /src/services/worker
 # presidio-analyzer > spacy > thinc doesn't ship aarch64 wheels so need to compile
 RUN if [ "$(uname -m)" = "aarch64" ]; then \
       apt-get update && apt-get install -y build-essential && \
       rm -rf /var/lib/apt/lists/*; \
     fi
-RUN poetry install --no-cache -P /src/services/worker
+RUN poetry install --no-cache
 RUN python -m spacy download en_core_web_lg
-WORKDIR /src/services/worker/
 ENTRYPOINT ["poetry", "run", "python", "src/worker/main.py"]
 
 # Cache maintenance job
 FROM common AS cache_maintenance
 COPY libs /src/libs
 COPY jobs/cache_maintenance /src/jobs/cache_maintenance
-RUN poetry install --no-cache -P /src/jobs/cache_maintenance
-WORKDIR /src/jobs/cache_maintenance/
+WORKDIR /src/jobs/cache_maintenance
+RUN poetry install --no-cache
 ENTRYPOINT ["poetry", "run", "python", "src/cache_maintenance/main.py"]
 
 # MongoDB migration job
 FROM common AS mongodb_migration
 COPY libs /src/libs
 COPY jobs/mongodb_migration /src/jobs/mongodb_migration
-RUN poetry install --no-cache -P /src/jobs/mongodb_migration
-WORKDIR /src/jobs/mongodb_migration/
-ENTRYPOINT ["poetry", "run", "python", "src/mongodb_migration/main.py"]
+WORKDIR /src/jobs/mongodb_migration
+RUN poetry install --no-cache
+ENTRYPOINT ["poetry", "run", "python", "src/mongodb_migration/main.py"]

Makefile

@@ -49,6 +49,7 @@ e2e:
 
 .PHONY: install
 install:
+	$(MAKE) -C libs/libviewer install
 	$(MAKE) -C libs/libcommon install
 	$(MAKE) -C libs/libapi install
 	$(MAKE) -C jobs/cache_maintenance install

docker-compose.yml

@@ -44,6 +44,7 @@ x-service-env: &service-env
   API_HF_TIMEOUT_SECONDS: ${API_HF_TIMEOUT_SECONDS}
   API_MAX_AGE_LONG: ${API_MAX_AGE_LONG}
   API_MAX_AGE_SHORT: ${API_MAX_AGE_SHORT}
+  RUST_BACKTRACE: 1
 
 services:
   reverse-proxy: