fix: sdist and wheels content (#28)

* missing sdist in pypi index
* installing pybind11 as a build time python package dependency
* add missing windows pyd files

---------

Signed-off-by: Michele Dolfi <[email protected]>

Author: dolfim-ibm

.github/actions/setup-poetry/action.yml

@@ -13,7 +13,11 @@ runs:
     - uses: actions/setup-python@v4
       with:
         python-version: ${{ inputs.python-version }}
+        update-environment: false
         cache: 'poetry'
+    - name: Setup poetry env with correct python
+      run: poetry env use python3
+      shell: bash
     - name: Install only dependencies and not the package itself
       run: poetry install --all-extras --no-root
       shell: bash

.github/workflows/checks.yml

@@ -12,16 +12,16 @@ jobs:
         python-version: ['3.9', '3.10', '3.11', '3.12']
     steps:
       - uses: actions/checkout@v4
-      # - name: Install dependencies [linux]
-      #   run: sudo apt-get install -y libldap-common
-      #   shell: bash
       - uses: ./.github/actions/setup-poetry
         with:
           python-version: ${{ matrix.python-version }}
       - name: Run styling check
         run: poetry run pre-commit run --all-files
       - name: Install with poetry
-        run: poetry install --all-extras
+        run: |
+          poetry install --all-extras
+          ls -l
+          ls -l docling_parse
       - name: Testing
         run: |
           poetry run pytest -v tests

.github/workflows/wheels.yml

@@ -116,7 +116,7 @@ jobs:
             echo "Inspecting $file"
             poetry run python -m zipfile --list "$file"
           done
-          mkdir ./dist
+          mkdir -p ./dist
           cp wheelhouse/*.whl ./dist/
 
       # there is an error with the tagging of wheels for macosx-arm64
@@ -156,7 +156,7 @@ jobs:
             echo "Inspecting $file"
             poetry run python -m zipfile --list "$file"
           done
-          mkdir ./dist
+          mkdir -p ./dist
           cp wheelhouse/*.whl ./dist/
 
       - name: Set up QEMU [linux]
@@ -165,6 +165,13 @@ jobs:
         with:
             platforms: all
 
+      - name: Build sdist
+        # build only on Linux to avoid too many duplicates of the sdist
+        if: matrix.os.name == 'ubuntu-latest' 
+        run: |
+          echo "Building wheel ${CIBW_BUILD}"
+          poetry build -f sdist
+        
       - name: Build wheels [linux]
         if: matrix.os.name == 'ubuntu-latest'
         env:
@@ -186,7 +193,7 @@ jobs:
             echo "Inspecting $file"
             poetry run python -m zipfile --list "$file"
           done
-          mkdir ./dist
+          mkdir -p ./dist
           cp wheelhouse/*.whl ./dist/
 
       - name: Set up MSYS2 [windows]

CMakeLists.txt

@@ -102,7 +102,7 @@ include(cmake/extlib_utf8.git.cmake)
 include(cmake/extlib_jpeg.cmake)
 # include(cmake/extlib_qpdf_v10.cmake)
 include(cmake/extlib_qpdf_v11.cmake)
-include(cmake/extlib_pybind11.cmake)
+# include(cmake/extlib_pybind11.cmake)
 
 include(cmake/os_opts.cmake)
 
@@ -146,19 +146,9 @@ target_link_libraries(parse ${LIB_LINK})
 # ***  Python-binding     ***
 # ***************************
 
-# inspiration from
-# 1. https://github.com/pybind/cmake_example/blob/master/CMakeLists.txt`
-# 2. https://blog.devgenius.io/calling-python-and-c-code-using-pybind-99ab7fefa685
-# 3. https://github.com/pybind/cmake_example/issues/11
-
-include(FetchContent)
-FetchContent_Declare(pybind11
-  FETCHCONTENT_BASE_DIR extlib_pybind11 
-  GIT_REPOSITORY https://github.com/pybind/pybind11.git
-  GIT_TAG        v2.13.5
-  GIT_SHALLOW    TRUE
-)
-FetchContent_MakeAvailable(pybind11)
+# https://pybind11.readthedocs.io/en/stable/compiling.html
+find_package (Python COMPONENTS Interpreter Development)
+find_package(pybind11 CONFIG REQUIRED)
 
 pybind11_add_module(docling_parse "${TOPLEVEL_PREFIX_PATH}/app/pybind_parse.cpp")
 

build.py

@@ -5,17 +5,30 @@
 import sys
 
 import subprocess
+from typing import List
+import pybind11
 
 ROOT_DIR = os.path.abspath("./")
 BUILD_DIR = os.path.join(ROOT_DIR, "build")
 
+def get_pybind11_cmake_args():
+        pybind11_sys_path = os.getenv("PYBIND11_SYSPATH")
+        if pybind11_sys_path:
+            # pybind11_include_dir = os.path.join(pybind11_sys_path, "include")
+            pybind11_cmake_dir = os.path.join(pybind11_sys_path, "share", "cmake", "pybind11")
+        else:
+            # pybind11_include_dir = pybind11.get_include()
+            pybind11_cmake_dir = pybind11.get_cmake_dir()
+        # print(f"{pybind11_include_dir=}")
+        print(f"{pybind11_cmake_dir=}")
+        return [f"-Dpybind11_DIR={pybind11_cmake_dir}"]
 
-def run(cmd, cwd="./"):
+def run(cmd: List[str], cwd: str="./"):
 
-    print(f"\nlaunch: {cmd}")
+    print_cmd = " ".join(cmd)
+    print(f"\nlaunch: {print_cmd}")
 
-    parts = cmd.split(" ")
-    message = subprocess.run(parts, cwd=cwd)
+    message = subprocess.run(cmd, cwd=cwd)
 
     if "returncode=0" in str(message):
         print(f" -> SUCCESS")
@@ -27,20 +40,26 @@ def run(cmd, cwd="./"):
 
 def build_local(num_threads: int):
 
-    if not os.path.exists(BUILD_DIR):
-        print("python executable: ", sys.executable)
-
-        cmd = f"cmake -B {BUILD_DIR} -DPYTHON_EXECUTABLE={sys.executable}"
-        success = run(cmd, cwd=ROOT_DIR)
-        if not success:
-            raise RuntimeError("Error building.")
-    else:
-        print(f"build directory detected: {BUILD_DIR}")
+    print("python prefix: ", sys.exec_prefix)
+    config_cmd = [
+        "cmake",
+        "-B", f"{BUILD_DIR}",
+        f"-DPython_ROOT_DIR={sys.exec_prefix}",
+        f"-DPython3_ROOT_DIR={sys.exec_prefix}",
+    ]
+    config_cmd.extend(get_pybind11_cmake_args())
+    success = run(config_cmd, cwd=ROOT_DIR)
+    if not success:
+        raise RuntimeError("Error building.")
 
-    cmd = f"cmake --build {BUILD_DIR} --target install"
+    build_cmd = [
+        "cmake",
+        "--build", f"{BUILD_DIR}",
+        "--target=install",
+    ]
     if num_threads > 1:
-        cmd += f" -j {num_threads}"
-    success = run(cmd, cwd=ROOT_DIR)
+        build_cmd.extend(["-j", f"{num_threads}"])
+    success = run(build_cmd, cwd=ROOT_DIR)
     if not success:
         raise RuntimeError("Error building.")
 

pyproject.toml

@@ -29,7 +29,18 @@ readme = "README.md"
 packages = [{include = "docling_parse"}]
 include = [
     {path = "docling_parse/*.so", format = "wheel"},
-    {path = "docling_parse/pdf_resources", format = "wheel"}
+    {path = "docling_parse/*.pyd", format = "wheel"},
+    {path = "docling_parse/pdf_resources", format = ["sdist", "wheel"]},
+    {path = "docling_parse/pdf_resources", format = ["sdist", "wheel"]},
+    {path = "CMakeLists.txt", format = "sdist"},
+    {path = "build.py", format = "sdist"},
+    {path = "*.md", format = "sdist"},
+    {path = "poetry.lock", format = "sdist"},
+    {path = "app/*.cpp", format = "sdist"},
+    {path = "cmake/", format = "sdist"},
+    {path = "app/", format = "sdist"},
+    {path = "src/", format = "sdist"},
+    {path = "tests/", format = "sdist"},
 ]
 build = "build.py"
 
@@ -56,7 +67,7 @@ pytest = "^7.4.2"
 docling-parse = "docling_parse.run:main"
 
 [build-system]
-requires = ["poetry-core"]
+requires = ["poetry-core", "pybind11>=2.13.1"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.black]