Make the project buildable again & enable multiplatform tests (#202)

For context: I'm the maintainer of
[pyHanko](https://github.com/MatthiasValvekens/pyHanko), which has had
an optional dependency on python-pkcs11 for PKCS#11 support for 4+ years
at this point.

I was led down this rabbit hole by a user report complaining about the
fact that the current latest release (from 2020, prior to this project
moving to the pyauth org) isn't installable on Windows with Python 3.12
or greater (MatthiasValvekens/pyHanko#558, #197).

While I'm not much of a Windows dev, this issue seemed fairly obvious to
fix, so I set out to do so, only to discover that the isolation enforced
by the typical modern Python build flow (in particular, when running `uv
build`) is fundamentally at odds with the way this project was
historically structured/built. So besides fixing #197, I also addressed
a number of other issues:

- Restructured the Cython build to actually work when executed from a
build frontend (this included fixing the source distribution post-`uv`
migration)
- Refactored the Cython layer to remove deprecated constructs
(conditional compilation, Cython-level `include`s, ...).
- The only platform-dependent logic was in the code that loads the
PKCS#11 library, so I moved that to a `load_module.c` file that exposes
a common C interface with platform-dependent implementations.
- I had to consolidate the Cython files a bit to be compliant with the
"include-free" style while still building everything as a single C
extension.
- Eliminated the dependency on `oscrypto` in the tests, which in turn
gets rid of the hard dependency on OpenSSL 1.1.x, which in turn also
means that all the hacks in the GitHub workflow for tests could be
removed as well.

I'm upstreaming my changes in the hope that it'll get this project into
a state where new releases can be cut more easily again, since I rely on
this library quite heavily. I'd like to avoid having to fork it if
that's at all avoidable...


-----

PS: Unsolicited opinion about testing: For a project like this, IMO
there's also no way around setting up a multiplatform test matrix
(including at least one linux/windows/macOS variant each). Ideally that
would include testing against actual HSMs, but operating a hardware test
bench isn't exactly trivial or cheap for an OSS project, so the fact
that this isn't being done is certainly understandable. Funnily enough
the readme still claims that there is CI running against real hardware,
but that hasn't been the case for several years at this point ;). I have
several devices lying around at home that I can test with, but I'm
currently limited to manual testing on that front.


PPS: With these changes, the `cibuildwheel` action appears to work, by
the way.

---

EDIT: I added `windows-latest` and `macos-latest` to the CI test matrix
now :).

Author: MatthiasValvekens

.github/workflows/tests.yml

@@ -1,26 +1,29 @@
 name: Tests
 on:
-  push:
+  push: {}
+  workflow_dispatch: {}
 env:
   UV_PYTHON_PREFERENCE: only-system
   PKCS11_TOKEN_LABEL: TEST
   PKCS11_TOKEN_PIN: 1234
   PKCS11_TOKEN_SO_PIN: 5678
 jobs:
   run:
-    # Run in Ubuntu 22.04 right now, as oscrypto fails on OpenSSL versions with a
-    # double-digit patch number (such as provided by Ubuntu 24.04):
-    #   https://community.snowflake.com/s/article/Python-Connector-fails-to-connect-with-LibraryNotFoundError-Error-detecting-the-version-of-libcrypto
-    #   https://github.com/wbond/oscrypto/issues/78
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.os }}
     strategy:
       matrix:
+        os:
+          - ubuntu-latest
+          - windows-latest
+          - macos-latest
         python-version:
           - "3.9"
           - "3.10"
           - "3.11"
           - "3.12"
-          - "3.13"
+          # 3.13.4 has a regression in C extension builds,
+          # need to stick to 3.13.3 until 3.13.5 is available on runners
+          - "3.13.3"
 
     steps:
       - name: Acquire sources
@@ -30,28 +33,40 @@ jobs:
         uses: actions/setup-python@v5.0.0
         with:
           python-version: ${{ matrix.python-version }}
-          architecture: x64
-
+      - name: Install Softhsm
+        shell: bash
+        run: |
+          if [[ $OS_NAME == 'ubuntu-latest' ]]; then
+            sudo apt-get install softhsm2
+            mkdir softhsm_tokens
+            echo "directories.tokendir = $(pwd)/softhsm_tokens" > /tmp/softhsm2.conf
+            echo "SOFTHSM2_CONF=/tmp/softhsm2.conf" >> "$GITHUB_ENV"
+            echo "PKCS11_MODULE=/usr/lib/softhsm/libsofthsm2.so" >> "$GITHUB_ENV"
+          elif [[ $OS_NAME == 'macos-latest' ]]; then
+            brew install softhsm
+            echo "PKCS11_MODULE=/opt/homebrew/lib/softhsm/libsofthsm2.so" >> "$GITHUB_ENV"
+          elif [[ $OS_NAME == 'windows-latest' ]]; then
+            choco install softhsm.install
+            echo "SOFTHSM2_CONF=D:/SoftHSM2/etc/softhsm2.conf" >> "$GITHUB_ENV"
+            echo "PKCS11_MODULE=D:/SoftHSM2/lib/softhsm2-x64.dll" >> "$GITHUB_ENV"
+            echo "D:/SoftHSM2/bin" >> "$GITHUB_PATH"
+            echo "D:/SoftHSM2/lib" >> "$GITHUB_PATH"
+          else
+            echo "$OS_NAME is not a supported target system"
+            exit 1
+          fi
+        env:
+          OS_NAME: ${{ matrix.os }}
+      - name: Initialize SoftHSM token
+        shell: bash
+        run: |
+          softhsm2-util --init-token --free --label $PKCS11_TOKEN_LABEL --pin $PKCS11_TOKEN_PIN --so-pin $PKCS11_TOKEN_SO_PIN
       - name: Install uv
         uses: astral-sh/setup-uv@v4
         with:
           enable-cache: true
           python-version: ${{ matrix.python-version }}
-
       - name: Install dev dependencies
         run: uv sync --all-extras
-
-      # Locally compile softhsmv2. For unknown reasons, the version installed by Ubuntu fails on
-      # Github Actions (while working e.g. in Docker).
-      - name: Install Softhsm
-        run: |
-          curl https://dist.opendnssec.org/source/softhsm-2.6.1.tar.gz | tar -zxv
-          (cd softhsm-2.6.1 && ./configure --prefix=$HOME --disable-p11-kit --disable-gost && make all install CC="gcc" CXX="g++")
-          echo "$HOME/bin" >> "$GITHUB_PATH"
-          echo "PKCS11_MODULE=$HOME/lib/softhsm/libsofthsm2.so" >> "$GITHUB_ENV"
-
-      - name: Initialize token
-        run: softhsm2-util --init-token --free --label $PKCS11_TOKEN_LABEL --pin $PKCS11_TOKEN_PIN --so-pin $PKCS11_TOKEN_SO_PIN
-
       - name: Run tests
         run: uv run pytest -v

MANIFEST.in

@@ -0,0 +1,2 @@
+graft extern/
+include pkcs11/*.pxd

extern/.gitignore

@@ -0,0 +1 @@
+!*.c

extern/load_module.c

@@ -0,0 +1,99 @@
+#define PY_SSIZE_T_CLEAN
+#include "load_module.h"
+
+
+#ifdef _WIN32
+static PyObject* p11_error() {
+    DWORD dwMessageId = GetLastError();
+    LPWSTR msgbuffer = NULL;
+
+    if (dwMessageId != 0) {
+        DWORD l = FormatMessageW(FORMAT_MESSAGE_ALLOCATE_BUFFER|FORMAT_MESSAGE_FROM_SYSTEM|FORMAT_MESSAGE_IGNORE_INSERTS,
+                    NULL,
+                    dwMessageId,
+                    MAKELANGID(LANG_USER_DEFAULT, SUBLANG_DEFAULT),
+                    (LPWSTR) &msgbuffer,
+                    0,
+                    NULL);
+        PyObject* errmsg = PyUnicode_FromWideChar(msgbuffer, l);
+        LocalFree(msgbuffer);
+        return errmsg;
+    } else {
+        return NULL;
+    }
+}
+
+static P11_HANDLE* p11_open(PyObject *path_str) {
+    wchar_t *path = PyUnicode_AsWideCharString(path_str, NULL);
+    LIB_HANDLE handle = LoadLibraryW(path);
+    PyMem_Free(path);
+
+    P11_HANDLE* result = NULL;
+    if (handle != NULL) {
+        void * ptr = GetProcAddress(handle, "C_GetFunctionList");
+        if (ptr != NULL) {
+             result = (P11_HANDLE*) PyMem_Malloc(sizeof(P11_HANDLE));
+             result->lib_handle = handle;
+             result->get_function_list_ptr = ptr;
+        }
+    }
+    return result;
+}
+
+static int p11_close(P11_HANDLE* handle) {
+    if(handle != NULL) {
+        LIB_HANDLE lib_handle = handle->lib_handle;
+        PyMem_Free(handle);
+        if (lib_handle != NULL) {
+            return FreeLibrary(lib_handle);
+        }
+    }
+    return 0;
+}
+
+#else
+
+static PyObject* p11_error() {
+    char* error = dlerror();
+
+    if (error == NULL) {
+        return NULL;
+    }
+    int len = strlen(error);
+    PyObject* result = PyUnicode_DecodeUTF8(error, len, NULL);
+    PyMem_Free(error);
+    return result;
+}
+
+static P11_HANDLE* p11_open(PyObject *path_str) {
+    const char *path = PyUnicode_AsUTF8AndSize(path_str, NULL);
+    // Note: python manages this buffer, no need to deallocate it here
+
+
+    P11_HANDLE* result = NULL;
+
+    LIB_HANDLE handle = dlopen(path, RTLD_LAZY | RTLD_LOCAL);
+
+    if (handle != NULL) {
+        void * ptr = dlsym(handle, "C_GetFunctionList");
+        if (ptr != NULL) {
+             result = (P11_HANDLE*) PyMem_Malloc(sizeof(P11_HANDLE));
+             result->lib_handle = handle;
+             result->get_function_list_ptr = ptr;
+        }
+    }
+    return result;
+}
+
+static int p11_close(P11_HANDLE* handle) {
+    if(handle != NULL) {
+        LIB_HANDLE lib_handle = handle->lib_handle;
+        PyMem_Free(handle);
+        if (lib_handle != NULL) {
+            return dlclose(lib_handle);
+        }
+    }
+    return 0;
+}
+
+#endif

extern/load_module.h

@@ -0,0 +1,21 @@
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#ifdef _WIN32
+#include "Windows.h"
+typedef HINSTANCE LIB_HANDLE;
+#else
+#include <dlfcn.h>
+typedef void *LIB_HANDLE;
+#endif
+
+#ifndef P11_HANDLE
+typedef struct P11_HANDLE {
+    LIB_HANDLE lib_handle;
+    void * get_function_list_ptr;
+} P11_HANDLE;
+#endif
+
+static PyObject* p11_error();
+static P11_HANDLE* p11_open(PyObject *path_str);
+static int p11_close(P11_HANDLE* handle);