Python bindings using scikit build core

.github/workflows/clang-tidy-review.yml

@@ -18,6 +18,7 @@ jobs:
       with:
         split_workflow: true
         build_dir: build
+        exclude: 'python-pybind/*'
         apt_packages: "libsnappy-dev, libzzip-dev, zlib1g-dev, libboost-all-dev, libzstd-dev"
         clang_tidy_checks: ''
         cmake_command: "cmake -Bbuild -DCMAKE_BUILD_TYPE=Debug -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ."

.github/workflows/python-cibuildwheel-pybind.yml

@@ -0,0 +1,139 @@
+name: Python Pybind cibuildwheel
+
+on:
+  push:
+    branches: [ master, release-* ]
+    paths-ignore:
+      - '.github/workflows/docs.yml'
+      - '.github/workflows/keyvi.yml'
+      - '.github/workflows/python-dockerimages-**.yml'
+      - '.github/workflows/rust**.yml'
+      - 'docker/**'
+      - 'doc/**'
+      - 'rust/**'
+  pull_request:
+    branches: [ master ]
+    paths-ignore:
+      - '.github/workflows/docs.yml'
+      - '.github/workflows/keyvi.yml'
+      - '.github/workflows/python-dockerimages-**.yml'
+      - '.github/workflows/rust**.yml'
+      - 'docker/**'
+      - 'doc/**'
+      - 'rust/**'
+  release:
+    types: [published]
+  workflow_dispatch:
+
+jobs:
+  build_wheels:
+    name: pybind-cibuildwheel ${{ matrix.os }}/${{ matrix.flavor }}/${{ matrix.target }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        # macos-15-intel: x86, macos-14: Arm64
+        os: [ubuntu-22.04, ubuntu-24.04-arm, macos-15-intel, macos-14]
+        # skip pypy, https://github.com/pypa/distutils/issues/283
+        flavor: ['cpython']
+        # separate musl and many on linux, for mac we just skip one of those
+        target: [ 'many', 'musl' ]
+        exclude:
+          - os: macos-15-intel
+            target: musl
+          - os: macos-14
+            target: musl
+
+    steps:
+      - uses: actions/checkout@v5
+      - name: ccache
+        uses: hendrikmuhs/[email protected]
+        with:
+          key: ${{ matrix.os }}-${{ matrix.target }}-${{ matrix.flavor }}-python-pybind
+
+      - name: Setup musllinux build
+        if: ${{ (runner.os == 'Linux') && (matrix.target == 'musl') }}
+        # workaround: set CXX to g++, so it does not pick clang++ for python 3.14 builds
+        run: |
+          echo "CIBW_SKIP=*manylinux*" >> $GITHUB_ENV
+          echo "CIBW_ENVIRONMENT_LINUX=CXX=g++" >> $GITHUB_ENV
+
+      - name: Setup manylinux build
+        if: ${{ (runner.os == 'Linux') && (matrix.target == 'many') }}
+        run: |
+          echo "CIBW_SKIP=*musllinux*" >> $GITHUB_ENV
+
+      - name: Skip pypy for cpython
+        if: ${{ matrix.flavor == 'cpython' }}
+        run: |
+          echo "CIBW_SKIP=${{ env.CIBW_SKIP }} pp*" >> $GITHUB_ENV
+
+      - name: Skip cpython for pypy
+        if: ${{ matrix.flavor == 'pypy' }}
+        run: |
+          echo "CIBW_SKIP=${{ env.CIBW_SKIP }} cp*" >> $GITHUB_ENV
+
+      - name: install mac dependencies
+        if: ${{ runner.os == 'macOS' }}
+        # 2nd command: workaround https://github.com/actions/setup-python/issues/577
+        run: |
+          brew update && \
+          brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done && \
+          brew install ccache zlib snappy boost
+
+      - name: set mac deployment target
+        if: runner.os == 'macOS' && runner.arch == 'X64'
+        run: |
+          echo "MACOSX_DEPLOYMENT_TARGET=15.0" >> $GITHUB_ENV
+
+      - name: set mac deployment target arm64
+        if: runner.os == 'macOS' && runner.arch == 'ARM64'
+        run: |
+          echo "MACOSX_DEPLOYMENT_TARGET=14.0" >> $GITHUB_ENV
+
+      - name: Build python wheels for ${{ matrix.os }}
+        uses: pypa/[email protected]
+        env:
+          # Skip CPython 3.8
+          CIBW_SKIP: ${{ env.CIBW_SKIP }} cp38-*
+
+          # only build native packages
+          CIBW_ARCHS: native
+
+          # skip tests on pypy, currently fails for indexer tests
+          CIBW_TEST_SKIP: "pp*"
+
+          # prefix ccache in path
+          CIBW_ENVIRONMENT_MACOS: ${{ env.CIBW_ENVIRONMENT_MACOS }} PATH=/usr/local/opt/ccache/libexec:/opt/homebrew/opt/ccache/libexec:$PATH
+          CIBW_ENVIRONMENT_LINUX: ${{ env.CIBW_ENVIRONMENT_LINUX }} PATH=/usr/local/bin:/usr/lib/ccache:$PATH CCACHE_DIR=/host${{ github.workspace }}/.ccache CCACHE_CONFIGPATH=/host/home/runner/.config/ccache/ccache.conf
+
+          # for debugging set this to 1,2 or 3
+          # CIBW_BUILD_VERBOSITY: 2
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: artifact-${{ matrix.os }}-${{ matrix.flavor }}-${{ matrix.target }}
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: sdist
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v5
+    - name: install Linux deps
+      run: |
+        sudo apt-get update && \
+        sudo apt-get install -y libsnappy-dev libzzip-dev zlib1g-dev libboost-all-dev ccache
+    - name: ccache
+      uses: hendrikmuhs/[email protected]
+      with:
+        key: ubuntu-sdist-python-pybind
+
+    - name: Build SDist
+      run: |
+        export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
+        pipx run build --sdist
+        python -m pip install dist/*.tar.gz -v && \
+        python -m pip install pytest && \
+        KEYVI_SKIP_TEST_DEPRECATIONS=1 KEYVI_MODULE_OVERWRITE=keyvi2 python -m pytest python/tests && \
+        python -m pip uninstall -y keyvi2

.github/workflows/python-cibuildwheel.yml

@@ -94,8 +94,8 @@ jobs:
       - name: Build python wheels for ${{ matrix.os }}
         uses: pypa/[email protected]
         env:
-          # Skip CPython 3.8
-          CIBW_SKIP: ${{ env.CIBW_SKIP }} cp38-*
+          # Skip CPython 3.8 and free threading
+          CIBW_SKIP: ${{ env.CIBW_SKIP }} cp38-* cp3??t-*
 
           # only build native packages
           CIBW_ARCHS: native
@@ -125,6 +125,7 @@ jobs:
           # for debugging set this to 1,2 or 3
           # CIBW_BUILD_VERBOSITY: 2
         with:
+          config-file: "{package}/pyproject.toml"
           package-dir: python
 
       - uses: actions/upload-artifact@v5

.gitignore

@@ -31,7 +31,7 @@
 *.orig
 
 # cmake build dir
-build/*
+/*build*
 */cmake-build-debug/*
 build_dir_debug/
 cmake-build-debug/
@@ -45,3 +45,12 @@ cmake-build-debug/
 
 # vim swap files
 *.swp
+
+# python
+*.egg-info
+.venv
+
+# pybind build folder
+python*/*build*
+python*/dist
+python*/.cache/

CMakeLists.txt

@@ -9,11 +9,13 @@ endif()
 #### Options
 
 option(KEYVI_C_BINDINGS "Keyvi: Build C binding" ${PROJECT_IS_TOP_LEVEL})
+option(KEYVI_PYTHON_BINDINGS "Keyvi: Build Python module" OFF)
 option(KEYVI_TESTS "Keyvi: Build unit tests" ${PROJECT_IS_TOP_LEVEL})
 option(KEYVI_BINARIES "Keyvi: Build keyvi binaries" ${PROJECT_IS_TOP_LEVEL})
 option(KEYVI_CLANG_TIDY "Keyvi: Build with clang tidy" OFF)
 option(KEYVI_DOCS "Keyvi: Build docs" ${PROJECT_IS_TOP_LEVEL})
 
+
 #### Linting
 if(KEYVI_CLANG_TIDY)
   find_program(CLANGTIDY clang-tidy)
@@ -275,6 +277,10 @@ target_include_directories(keyvi INTERFACE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>
 target_compile_definitions(keyvi INTERFACE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
 target_link_libraries(keyvi INTERFACE ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${ZSTD_LIBRARIES} ${_OS_LIBRARIES})
 
+if (KEYVI_PYTHON_BINDINGS)
+    add_subdirectory(python-pybind)
+endif ()
+
 ### docs
 
 # don't run it as part of a non-toplevel build, e.g. python

pyproject.toml

@@ -0,0 +1,96 @@
+[build-system]
+requires = ["scikit-build-core>=0.10", "pybind11"]
+build-backend = "scikit_build_core.build"
+
+[project]
+name = "keyvi2"
+version = "0.7.4dev0"
+dependencies = [
+    "msgpack>=1.0.0",
+]
+
+[dependency-groups]
+lint = [
+    'ruff>=0.5.0',
+]
+test = [
+    'pytest>=8.3.5',
+    'pytest-env>=1.1.5',
+    'python-snappy>=0.7.3',
+    'zstd>=1.5.7.2'
+]
+dev = [
+    { include-group = "test" },
+    { include-group = "lint" },
+    "tox",
+]
+
+
+[tool.scikit-build]
+wheel.expand-macos-universal-tags = true
+minimum-version = "build-system.requires"
+
+[tool.scikit-build.cmake.define]
+KEYVI_PYTHON_BINDINGS = "ON"
+KEYVI_C_BINDINGS = "OFF"
+KEYVI_TESTS = "OFF"
+KEYVI_BINARIES = "OFF"
+KEYVI_CLANG_TIDY = "OFF"
+KEYVI_DOCS = "OFF"
+
+[tool.pytest.ini_options]
+env = [
+    "KEYVI_SKIP_TEST_DEPRECATIONS = 1",
+    "KEYVI_MODULE_OVERWRITE=keyvi2"
+]
+minversion = "7.1.1"
+addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"]
+xfail_strict = true
+log_cli_level = "INFO"
+filterwarnings = [
+  "error",
+  "ignore::pytest.PytestCacheWarning",
+]
+testpaths = ["python/tests"]
+
+[tool.cibuildwheel]
+build-frontend = "default"
+test-groups = ["test"]
+test-command = "pytest {project}/python-pybind/tests"
+manylinux-x86_64-image = "keyvidev/manylinux-builder-x86_64"
+musllinux-x86_64-image = "keyvidev/musllinux-builder-x86_64"
+manylinux-aarch64-image = "keyvidev/manylinux-builder-aarch64"
+musllinux-aarch64-image = "keyvidev/musllinux-builder-aarch64"
+
+[tool.ruff.lint]
+extend-select = [
+  "B",           # flake8-bugbear
+  "I",           # isort
+  "ARG",         # flake8-unused-arguments
+  "C4",          # flake8-comprehensions
+  "EM",          # flake8-errmsg
+  "ICN",         # flake8-import-conventions
+  "G",           # flake8-logging-format
+  "PGH",         # pygrep-hooks
+  "PIE",         # flake8-pie
+  "PL",          # pylint
+  "PT",          # flake8-pytest-style
+  "PTH",         # flake8-use-pathlib
+  "RET",         # flake8-return
+  "RUF",         # Ruff-specific
+  "SIM",         # flake8-simplify
+  "T20",         # flake8-print
+  "UP",          # pyupgrade
+  "YTT",         # flake8-2020
+  "EXE",         # flake8-executable
+  "NPY",         # NumPy specific rules
+  "PD",          # pandas-vet
+]
+ignore = [
+  "PLR09",    # Too many X
+  "PLR2004",  # Magic comparison
+]
+isort.required-imports = ["from __future__ import annotations"]
+
+[tool.ruff.lint.per-file-ignores]
+"tests/**" = ["T20"]

python-pybind/CMakeLists.txt

@@ -0,0 +1,26 @@
+cmake_minimum_required(VERSION 3.15...3.27)
+
+# Scikit-build-core sets these values for you, or you can just hard-code the
+# name and version.
+project(
+  ${SKBUILD_PROJECT_NAME}
+  VERSION ${SKBUILD_PROJECT_VERSION}
+  LANGUAGES CXX)
+
+# Find the module development requirements (requires FindPython from 3.17 or
+# scikit-build-core's built-in backport)
+find_package(Python REQUIRED COMPONENTS Interpreter Development.Module)
+find_package(pybind11 CONFIG REQUIRED)
+
+# Add a library using FindPython's tooling (pybind11 also provides a helper like
+# this)
+FILE(GLOB_RECURSE KEYVI_PYBIND_SOURCES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} src/*.cpp)
+pybind11_add_module(keyvi2 ${KEYVI_PYBIND_SOURCES})
+
+target_link_libraries(keyvi2 PRIVATE keyvi)
+
+# This is passing in the version as a define just as an example
+target_compile_definitions(keyvi2 PRIVATE VERSION_INFO=${PROJECT_VERSION})
+
+# The install directory is the output (wheel) directory
+install(TARGETS keyvi2 DESTINATION .)

python-pybind/src/.clang-format

@@ -0,0 +1,12 @@
+---
+BasedOnStyle: Google
+ColumnLimit: '120'
+Language: Cpp
+Standard: c++17
+TabWidth: '2'
+UseTab: Never
+ConstructorInitializerIndentWidth: 4
+AllowShortFunctionsOnASingleLine: Inline
+IncludeBlocks: Preserve
+
+...

python-pybind/src/CPPLINT.cfg

@@ -0,0 +1,3 @@
+linelength=120
+root=.
+filter=-build/include_subdir,-whitespace/indent_namespace