create an alternative python binding using pybind11

.github/workflows/python-cibuildwheel-pybind.yml

@@ -0,0 +1,183 @@
+name: Python Pybind cibuildwheel
+
+on:
+  push:
+    branches: [master, release-*]
+  pull_request:
+    branches: [master]
+  workflow_dispatch:
+
+jobs:
+  build_wheels:
+    name: cibuildwheel ${{ matrix.os }}/${{ matrix.arch }}/${{ matrix.flavor }}/${{ matrix.target }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-22.04]
+        # separate archs, so they use individual caches
+        arch: ["x86_64", "arm64"]
+        flavor: ["cpython", "pypy"]
+        # separate musl and many on linux, for mac we just skip one of those
+        target: ["many", "musl"]
+        exclude:
+          - os: ubuntu-22.04
+            target: musl
+            flavor: pypy
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up QEMU
+        if: ${{ (runner.os == 'Linux') && (matrix.arch == 'arm64') }}
+        uses: docker/setup-qemu-action@v3
+        with:
+          platforms: all
+      - name: ccache
+        uses: hendrikmuhs/[email protected]
+        with:
+          key: ${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.target }}-${{ matrix.flavor }}-python
+
+      - name: Sets env for x86_64
+        run: |
+          echo "CIBW_ARCHS_LINUX=auto64" >> $GITHUB_ENV
+          echo "CIBW_ARCHS_MACOS=x86_64" >> $GITHUB_ENV
+        if: matrix.arch == 'x86_64'
+
+      - name: Sets env for arm64
+        run: |
+          echo "CIBW_ARCHS_LINUX=aarch64" >> $GITHUB_ENV
+          echo "CIBW_ARCHS_MACOS=arm64" >> $GITHUB_ENV
+        if: matrix.arch == 'arm64'
+
+      - name: Skip manylinux for musllinux target
+        if: ${{ (runner.os == 'Linux') && (matrix.target == 'musl') }}
+        run: |
+          echo "CIBW_SKIP=*manylinux*" >> $GITHUB_ENV
+
+      - name: Skip musllinux for manylinux target
+        if: ${{ (runner.os == 'Linux') && (matrix.target == 'many') }}
+        run: |
+          echo "CIBW_SKIP=*musllinux*" >> $GITHUB_ENV
+
+      - name: Skip pypy for cpython
+        if: ${{ matrix.flavor == 'cpython' }}
+        run: |
+          echo "CIBW_SKIP=${{ env.CIBW_SKIP }} pp*" >> $GITHUB_ENV
+
+      - name: Skip cpython for pypy
+        if: ${{ matrix.flavor == 'pypy' }}
+        run: |
+          echo "CIBW_SKIP=${{ env.CIBW_SKIP }} cp*" >> $GITHUB_ENV
+
+      - name: install mac dependencies
+        if: ${{ runner.os == 'macOS' }}
+        # 2nd command: workaround https://github.com/actions/setup-python/issues/577
+        run: |
+          brew update && \
+          brew list -1 | grep python | while read formula; do brew unlink $formula; brew link --overwrite $formula; done && \
+          brew install ccache
+
+      - name: install mac dependencies X86_64
+        if: ${{ (runner.os == 'macOS') && (matrix.arch == 'x86_64') }}
+        run: |
+          brew update && \
+          brew install zlib snappy boost
+
+      - name: install mac dependencies arm64
+        if: ${{ (runner.os == 'macOS') && (matrix.arch == 'arm64') }}
+        run: |
+          set -e
+          echo "MACOSX_DEPLOYMENT_TARGET=12.3.0" >> $GITHUB_ENV
+          echo "_CMAKE_PREFIX_PATH=${{ github.workspace }}/arm64-homebrew" >> $GITHUB_ENV
+          echo "CIBW_REPAIR_WHEEL_COMMAND_MACOS=DYLD_LIBRARY_PATH=${{ github.workspace }}/arm64-homebrew delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}" >> $GITHUB_ENV
+          mkdir arm64-homebrew && curl -L https://github.com/Homebrew/brew/tarball/master | tar xz --strip 1 -C arm64-homebrew
+          PACKAGES=( icu4c xz lz4 zstd zlib snappy boost )
+          for PACKAGE in "${PACKAGES[@]}"
+          do
+            response=$(arm64-homebrew/bin/brew fetch --force --bottle-tag=arm64_sonoma $PACKAGE | grep Downloaded )
+            download_path=$(echo $response | xargs -n 1 | tail -1)
+            arm64-homebrew/bin/brew reinstall -vd $download_path
+          done
+          arm64-homebrew/bin/brew config
+          ls /Users/runner/work/keyvi/keyvi/arm64-homebrew
+
+      - name: Build python wheels for ${{ matrix.os }} on ${{ matrix.arch }}
+        uses: pypa/[email protected]
+        env:
+          # Skip CPython 3.6 and CPython 3.7
+          CIBW_SKIP: ${{ env.CIBW_SKIP }} cp36-* cp37-* pp37-*
+
+          # skip testing all python versions on linux arm, only test 3.12
+          # skip tests on pypy, currently fails for indexer tests
+          CIBW_TEST_SKIP: "*p{38,39,310,311}-m*linux_aarch64 pp*"
+
+          # (many)linux custom docker images
+          CIBW_MANYLINUX_X86_64_IMAGE: "keyvidev/manylinux-builder-x86_64"
+          CIBW_MANYLINUX_AARCH64_IMAGE: "keyvidev/manylinux-builder-aarch64"
+          CIBW_MUSLLINUX_X86_64_IMAGE: "keyvidev/musllinux-builder-x86_64"
+          CIBW_MUSLLINUX_AARCH64_IMAGE: "keyvidev/musllinux-builder-aarch64"
+
+          # ccache using path
+          CIBW_ENVIRONMENT_MACOS: PATH=/usr/local/opt/ccache/libexec:$PATH
+          CIBW_ENVIRONMENT_LINUX: PATH=/usr/local/bin:/usr/lib/ccache:$PATH CCACHE_DIR=/host${{ github.workspace }}/.ccache CCACHE_CONFIGPATH=/host/home/runner/.config/ccache/ccache.conf
+
+          # python dependencies
+          CIBW_BEFORE_BUILD: pip install -r python/requirements.txt
+
+          # testing
+          CIBW_TEST_REQUIRES: pytest
+          CIBW_TEST_COMMAND: >
+            python -m pytest {package}/tests
+
+          # for debugging set this to 1,2 or 3
+          # CIBW_BUILD_VERBOSITY: 2
+        with:
+          package-dir: python-pybind
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: artifact-${{ matrix.os }}-${{ matrix.arch }}-${{ matrix.flavor }}-${{ matrix.target }}
+          path: ./wheelhouse/*.whl
+
+  build_sdist:
+    name: sdist
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: install Linux deps
+        run: |
+          sudo apt-get update && \
+          sudo apt-get install -y libsnappy-dev libzzip-dev zlib1g-dev libboost-all-dev ccache
+      - name: ccache
+        uses: hendrikmuhs/[email protected]
+        with:
+          key: ${{ matrix.os }}-sdist-python
+
+      - name: Build SDist
+        run: |
+          export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
+          cd python-pybind && \
+          python -m pip install . && \
+          python setup.py sdist -d wheelhouse && \
+          python -m pip install wheelhouse/*.tar.gz -v && \
+          python -m pip install pytest && \
+          python -m pytest tests && \
+          python -m pip uninstall -y keyvi_pybind11
+
+      - uses: actions/upload-artifact@v4
+        with:
+          name: artifact-sdist
+          path: python-pybind/wheelhouse/*.tar.gz
+
+  upload_all:
+    needs: [build_wheels, build_sdist]
+    runs-on: ubuntu-latest
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
+    steps:
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: artifact-*
+          merge-multiple: true
+          path: dist
+
+      - uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.pypi_password }}

.gitignore

@@ -31,7 +31,7 @@
 *.orig
 
 # cmake build dir
-build/*
+/*build*
 */cmake-build-debug/*
 build_dir_debug/
 cmake-build-debug/
@@ -45,3 +45,11 @@ cmake-build-debug/
 
 # vim swap files
 *.swp
+
+# python
+*.egg-info
+
+# pybind build folder
+python*/*build*
+python*/dist
+python*/.cache/

CMakeLists.txt

@@ -6,6 +6,15 @@ if (CMAKE_BUILD_TYPE)
   string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_UPPER)
 endif()
 
+#### check if build top-level or as sub-build
+if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR)
+  set(KEYVI_MASTER_PROJECT ON)
+  message(STATUS "CMake ${CMAKE_VERSION}")
+else()
+  set(KEYVI_MASTER_PROJECT OFF)
+endif()
+
+
 #### Linting
 find_program(CLANGTIDY clang-tidy)
 if(CLANGTIDY)
@@ -141,67 +150,69 @@ string(REPLACE " " ";" _KEYVI_COMPILE_DEFINITIONS_LIST "${_KEYVI_COMPILE_DEFINIT
 
 #### Targets ####
 
-# keyvicompiler
-add_executable(keyvicompiler keyvi/bin/keyvicompiler/keyvicompiler.cpp)
-target_link_libraries(keyvicompiler ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
-target_compile_options(keyvicompiler PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
-target_compile_definitions(keyvicompiler PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
-target_include_directories(keyvicompiler PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
-
-install (TARGETS keyvicompiler DESTINATION bin COMPONENT applications OPTIONAL)
-
-# keyviinspector
-add_executable(keyviinspector keyvi/bin/keyviinspector/keyviinspector.cpp)
-target_link_libraries(keyviinspector ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
-target_compile_options(keyviinspector PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
-target_compile_definitions(keyviinspector PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
-target_include_directories(keyviinspector PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
-
-install (TARGETS keyviinspector DESTINATION bin COMPONENT applications OPTIONAL)
-
-# keyvimerger
-add_executable(keyvimerger keyvi/bin/keyvimerger/keyvimerger.cpp)
-target_link_libraries(keyvimerger ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
-target_compile_options(keyvimerger PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
-target_compile_definitions(keyvimerger PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
-target_include_directories(keyvimerger PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
-
-install (TARGETS keyvimerger DESTINATION bin COMPONENT applications)
-
-# keyvi_c
-add_library(keyvi_c SHARED keyvi/bin/keyvi_c/c_api.cpp)
-target_link_libraries(keyvi_c ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
-target_compile_options(keyvi_c PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
-target_compile_definitions(keyvi_c PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
-target_include_directories(keyvi_c PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
-
-# unit tests
-FILE(GLOB_RECURSE UNIT_TEST_SOURCES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} keyvi/tests/keyvi/*.cpp)
-add_executable(unit_test_all ${UNIT_TEST_SOURCES})
-target_link_libraries(unit_test_all ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
-target_compile_options(unit_test_all PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
-target_compile_definitions(unit_test_all PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
-target_include_directories(unit_test_all PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
-add_dependencies(unit_test_all keyvimerger)
-
-if (WIN32)
-  message(STATUS "zlib: ${ZLIB_LIBRARY_RELEASE}")
-  # copies the dlls required to run to the build folder
-  foreach(LIB ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY_RELEASE} ${Boost_FILESYSTEM_LIBRARY_RELEASE} ${ZLIB_LIBRARY_RELEASE})
-    get_filename_component(UTF_BASE_NAME ${LIB} NAME_WE)
-    get_filename_component(UTF_PATH ${LIB} PATH)
-	if(EXISTS "${UTF_PATH}/${UTF_BASE_NAME}.dll")
-      add_custom_command(TARGET unit_test_all POST_BUILD
-        COMMAND ${CMAKE_COMMAND} -E copy "${UTF_PATH}/${UTF_BASE_NAME}.dll" ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}
-        )
-	# zlib might be stored in a different folder
-	elseif(EXISTS "${UTF_PATH}/../bin/${UTF_BASE_NAME}.dll")
-      add_custom_command(TARGET unit_test_all POST_BUILD
-        COMMAND ${CMAKE_COMMAND} -E copy "${UTF_PATH}/../bin/${UTF_BASE_NAME}.dll" ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}
-        )	
-	endif()
-  endforeach()
-endif (WIN32)
+if(KEYVI_MASTER_PROJECT)
+  # keyvicompiler
+  add_executable(keyvicompiler keyvi/bin/keyvicompiler/keyvicompiler.cpp)
+  target_link_libraries(keyvicompiler ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
+  target_compile_options(keyvicompiler PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
+  target_compile_definitions(keyvicompiler PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
+  target_include_directories(keyvicompiler PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
+
+  install (TARGETS keyvicompiler DESTINATION bin COMPONENT applications OPTIONAL)
+
+  # keyviinspector
+  add_executable(keyviinspector keyvi/bin/keyviinspector/keyviinspector.cpp)
+  target_link_libraries(keyviinspector ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
+  target_compile_options(keyviinspector PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
+  target_compile_definitions(keyviinspector PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
+  target_include_directories(keyviinspector PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
+
+  install (TARGETS keyviinspector DESTINATION bin COMPONENT applications OPTIONAL)
+
+  # keyvimerger
+  add_executable(keyvimerger keyvi/bin/keyvimerger/keyvimerger.cpp)
+  target_link_libraries(keyvimerger ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
+  target_compile_options(keyvimerger PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
+  target_compile_definitions(keyvimerger PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
+  target_include_directories(keyvimerger PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
+
+  install (TARGETS keyvimerger DESTINATION bin COMPONENT applications)
+
+  # keyvi_c
+  add_library(keyvi_c SHARED keyvi/bin/keyvi_c/c_api.cpp)
+  target_link_libraries(keyvi_c ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
+  target_compile_options(keyvi_c PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
+  target_compile_definitions(keyvi_c PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
+  target_include_directories(keyvi_c PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
+
+  # unit tests
+  FILE(GLOB_RECURSE UNIT_TEST_SOURCES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} keyvi/tests/keyvi/*.cpp)
+  add_executable(unit_test_all ${UNIT_TEST_SOURCES})
+  target_link_libraries(unit_test_all ${Boost_LIBRARIES} ${ZLIB_LIBRARIES} ${Snappy_LIBRARY} ${_OS_LIBRARIES})
+  target_compile_options(unit_test_all PRIVATE ${_KEYVI_CXX_FLAGS_LIST})
+  target_compile_definitions(unit_test_all PRIVATE ${_KEYVI_COMPILE_DEFINITIONS_LIST})
+  target_include_directories(unit_test_all PRIVATE "$<BUILD_INTERFACE:${KEYVI_INCLUDES}>")
+  add_dependencies(unit_test_all keyvimerger)
+
+  if (WIN32)
+    message(STATUS "zlib: ${ZLIB_LIBRARY_RELEASE}")
+    # copies the dlls required to run to the build folder
+    foreach(LIB ${Boost_UNIT_TEST_FRAMEWORK_LIBRARY_RELEASE} ${Boost_FILESYSTEM_LIBRARY_RELEASE} ${ZLIB_LIBRARY_RELEASE})
+      get_filename_component(UTF_BASE_NAME ${LIB} NAME_WE)
+      get_filename_component(UTF_PATH ${LIB} PATH)
+    if(EXISTS "${UTF_PATH}/${UTF_BASE_NAME}.dll")
+        add_custom_command(TARGET unit_test_all POST_BUILD
+          COMMAND ${CMAKE_COMMAND} -E copy "${UTF_PATH}/${UTF_BASE_NAME}.dll" ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}
+          )
+    # zlib might be stored in a different folder
+    elseif(EXISTS "${UTF_PATH}/../bin/${UTF_BASE_NAME}.dll")
+        add_custom_command(TARGET unit_test_all POST_BUILD
+          COMMAND ${CMAKE_COMMAND} -E copy "${UTF_PATH}/../bin/${UTF_BASE_NAME}.dll" ${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_BUILD_TYPE}
+          )	
+    endif()
+    endforeach()
+  endif (WIN32)
+endif(KEYVI_MASTER_PROJECT)
 
 # bindings
 add_custom_target(bindings

keyvi/check-style.sh

@@ -10,7 +10,7 @@ else
 commit_range="upstream/master...HEAD"
 fi
 
-infiles=`git diff --name-only --diff-filter=ACMRT $(echo ${commit_range} | sed 's/\.//') | grep -v "3rdparty" | grep -E "\.(cpp|h)$"`
+infiles=`git diff --name-only --diff-filter=ACMRT $(echo ${commit_range} | sed 's/\.//') | grep -v "3rdparty" | grep -v "pybind11" | grep -E "\.(cpp|h)$"`
 
 clang_format_files=()
 cpplint_files=()

keyvi/include/keyvi/dictionary/match.h

@@ -58,7 +58,7 @@ namespace dictionary {
 #ifdef Py_PYTHON_H
 class attributes_visitor : public boost::static_visitor<PyObject*> {
  public:
-  PyObject* operator()(int i) const { return PyInt_FromLong(i); }
+  PyObject* operator()(int i) const { return PyLong_FromLong(i); }
 
   PyObject* operator()(double i) const { return PyFloat_FromDouble(i); }
 

python-pybind/CMakeLists.txt

@@ -0,0 +1,14 @@
+cmake_minimum_required(VERSION 3.4...3.18)
+project(keyvi_pybind11)
+
+add_subdirectory(pybind11)
+add_subdirectory(keyvi_cpp)
+
+FILE(GLOB_RECURSE KEYVI_PYBIND_SOURCES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} src/*.cpp)
+pybind11_add_module(keyvi_pybind11 ${KEYVI_PYBIND_SOURCES})
+
+# KEYVI_VERSION_INFO is defined by setup.py and passed into the C++ code as a
+# define (VERSION_INFO) here.
+target_compile_definitions(keyvi_pybind11
+                           PRIVATE VERSION_INFO=${KEYVI_VERSION_INFO})
+target_link_libraries(keyvi_pybind11 PRIVATE keyvi)