Merge pull request #42 from SAFEHR-data/paul/restructure-docling

Restructure `docling`

Author: p-j-smith

README.md

@@ -13,14 +13,13 @@
 
 END COMMENT OUT-->
 
-**pyonb** is two things:
+`pyonb` is a Python library and suite of APIs that wrap open-source Optical Character Recognition (OCR) tools. It it designed for local deployment and can convert PDFs to structured text using the several
+OCR tools:
 
-- a Python SDK for document extraction via the Hyland OnBase REST API (_work in progress_)
-- a suite of APIs wrapped around open-source Optical Character Recognition (OCR) tools, designed for local deployment, for converting PDFs to structured text including:
-  - [Marker](https://github.com/VikParuchuri/marker)
-  - [PaddleOCR](https://github.com/PaddlePaddle/PaddleOCR)
-  - [Docling](https://github.com/docling-project/docling)
-  - [Kreuzberg](https://github.com/Goldziher/kreuzberg)
+- [Marker](https://github.com/VikParuchuri/marker)
+- [PaddleOCR](https://github.com/PaddlePaddle/PaddleOCR)
+- [Docling](https://github.com/docling-project/docling)
+- [Kreuzberg](https://github.com/Goldziher/kreuzberg)
 
 ## Getting Started
 
@@ -30,24 +29,35 @@ END COMMENT OUT-->
 
 ### Installation & Usage
 
-1. Rename `.env.sample` to `.env`.
+1. Clone `pyonb`
 
-2. Edit `.env` with the correct `HOST_DATA_FOLDER` location, e.g.:
+```sh
+git clone git@github.com:SAFEHR-data/pyonb.git
+cd pyonb
+```
+
+2. Rename `.env.sample` to `.env`.
 
 ```sh
-HOST_DATA_FOLDER="/absolute/path/to/documents/folder"
+mv .env.sample .env
+```
 
-# e.g. for unit tests on GAE:
-# HOST_DATA_FOLDER="/gae/pyonb/tests/data/single_synthetic_doc"
+3. Edit `.env` with the correct `DATA_FOLDER` location, e.g.:
+
+```sh
+DATA_FOLDER="path/to/documents/folder"
 ```
 
+where the path is relative to the `docker-compose.yml` file in the top-level `pyonb` directory.
+
 4. Set OCR service ports, e.g.:
 
 ```sh
 OCR_FORWARDING_API_PORT=8110
 MARKER_API_PORT=8112
 PADDLEOCR_API_PORT=8114
 DOCLING_API_PORT=8115
+KREUZBERG_API_PORT=8116
 ```
 
 > [!IMPORTANT]
@@ -60,7 +70,7 @@ DOCLING_API_PORT=8115
 > HTTP_PROXY=
 > ```
 
-5. Start the OCR API Server (e.g. using marker and docling):
+5. Start the OCR API Server (e.g. using `marker` and `docling`):
 
 ```sh
 docker compose --profile marker --profile docling up -d

docker-compose.yml

@@ -118,21 +118,21 @@ services:
   docling:
     profiles: [docling]
     build:
-      context: src/ocr/docling
+      context: packages/ocr/docling
       dockerfile: Dockerfile
       args:
         <<: *build-args-common
         DOCLING_API_PORT: ${DOCLING_API_PORT}
     environment:
       <<: [*proxy-common, *common-env]
-      CONTAINER_DATA_FOLDER: /data
+      DATA_FOLDER: /data
       DOCLING_API_PORT: ${DOCLING_API_PORT}
     env_file:
       - ./.env
     ports:
       - "${DOCLING_API_PORT}:${DOCLING_API_PORT}"
     volumes:
-      - ${HOST_DATA_FOLDER}:${CONTAINER_DATA_FOLDER:-/data}
+      - ${PWD}/${DATA_FOLDER}:/data
     networks:
       - pyonb_ocr_api
     healthcheck:
@@ -192,15 +192,15 @@ services:
         OCR_FORWARDING_API_PORT: ${OCR_FORWARDING_API_PORT}
     environment:
       <<: [*proxy-common, *common-env]
-      CONTAINER_DATA_FOLDER: /data
+      DATA_FOLDER: /data
       OCR_FORWARDING_API_PORT: ${OCR_FORWARDING_API_PORT}
     env_file:
       - ./.env
     ports:
       - "${OCR_FORWARDING_API_PORT}:${OCR_FORWARDING_API_PORT}"
     volumes:
       - ./src/api/app:/app
-      - ${HOST_DATA_FOLDER}:${CONTAINER_DATA_FOLDER:-/data}
+      - ${PWD}/${DATA_FOLDER}:/data
     networks:
       - pyonb_ocr_api
     healthcheck:

packages/ocr/docling/Dockerfile

@@ -0,0 +1,19 @@
+FROM ghcr.io/astral-sh/uv:python3.13-bookworm AS app
+
+SHELL ["/bin/bash", "-o", "pipefail", "-e", "-u", "-x", "-c"]
+
+WORKDIR /app
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+COPY ./pyproject.toml .
+COPY ./README.md .
+COPY ./src src/
+
+RUN uv venv
+RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked uv sync --no-editable --no-dev
+
+# make uvicorn etc available
+ENV PATH="/app/.venv/bin:$PATH"
+
+CMD uvicorn pyonb_docling.api:app --host 0.0.0.0 --port "$DOCLING_API_PORT" --workers 4 --use-colors

packages/ocr/docling/README.md

@@ -0,0 +1,30 @@
+# Instructions
+
+## Python
+
+First install `pyonb_docling`. From the top-level `pyonb` directory:
+
+```shell
+uv sync --extra docling
+```
+
+Then, to convert a PDF to markdown:
+
+```python
+import pyonb_docling
+
+result = pyonb_docling.convert_pdf_to_markdown(
+    file_path="path/to/data/input.pdf",
+)
+```
+
+## Docker Compose
+
+From the `pyonb/packages/ocr/docling` directory:
+
+```shell
+docker compose run docling data/input.pdf data/output.md
+```
+
+Note, you will need to set `DATA_FOLDER` in a `.env` file,
+e.g: `DATA_FOLDER=path/to/data/input.pdf`

src/ocr/docling/docker-compose.yml packages/ocr/docling/docker-compose.ymlsrc/ocr/docling/docker-compose.yml renamed to packages/ocr/docling/docker-compose.yml

@@ -0,0 +1,16 @@
+[build-system]
+build-backend = "hatchling.build"
+requires = ["hatchling"]
+
+[project]
+dependencies = [
+    "docling",
+    "fastapi[standard]",
+    "python-dotenv",
+    "uvicorn",
+]
+description = "pyonb wrapper around docling"
+name = "pyonb-docling"
+readme = "README.md"
+requires-python = ">=3.11"
+version = "0.1.0"

packages/ocr/docling/pyproject.toml

@@ -8,6 +8,8 @@
 from fastapi import FastAPI, File, HTTPException, UploadFile, status
 from fastapi.responses import JSONResponse, RedirectResponse
 
+from pyonb_docling.main import convert_pdf_to_markdown
+
 logging.basicConfig(
     filename="docling." + datetime.datetime.now(tz=datetime.UTC).strftime("%Y%m%d") + ".log",
     format="%(asctime)s %(message)s",
@@ -18,18 +20,6 @@
 logger = logging.getLogger()
 logger.setLevel(logging.DEBUG)
 
-# TODO(tom): improve imports - below try statements horrible
-try:
-    # local
-    from .main import convert_pdf_to_markdown
-except Exception:
-    logger.exception("Detected inside Docker container.")
-    # Docker container
-    try:
-        from main import convert_pdf_to_markdown  # type: ignore  # noqa: PGH003
-    except Exception:
-        logger.exception("Docling imports not possible.")
-
 app = FastAPI(swagger_ui_parameters={"tryItOutEnabled": True})
 
 

src/ocr/docling/__init__.py …cr/docling/src/pyonb_docling/__init__.pysrc/ocr/docling/__init__.py renamed to packages/ocr/docling/src/pyonb_docling/__init__.py src/ocr/docling/__init__.py renamed to packages/ocr/docling/src/pyonb_docling/__init__.py

@@ -43,6 +43,8 @@ optional-dependencies = {dev = [
     "ruff",
     "tox",
     "twine",
+], docling = [
+    "pyonb-docling",
 ], docs = [
     "mkdocs",
     "mkdocs-include-markdown-plugin",
@@ -153,6 +155,7 @@ gh.python."3.12" = ["py312"]
 gh.python."3.13" = ["py313"]
 
 [tool.uv.sources]
+pyonb-docling = {workspace = true}
 pyonb-kreuzberg = {workspace = true}
 
 [tool.uv.workspace]