Merge pull request #9 from sqliteai/python-build

feat(python): workflow to build and upload the pypi package

Author: danielebriggi

.github/workflows/main.yml

@@ -1,4 +1,4 @@
-name: build, test and release sqlite-ai
+name: Build, Test and Release SQLite AI
 on:
   push:
   workflow_dispatch:

.github/workflows/python-package.yml

@@ -13,30 +13,32 @@ on:
 jobs:
   build-and-publish:
     runs-on: ${{ matrix.os }}
+    permissions:
+      id-token: write  # mandatory for Pypi trusted publishing
     strategy:
       matrix:
         include:
           - os: ubuntu-latest
             platform: linux
             python-version: "3.10"
             arch: x86_64
-            plat_name: linux_x86_64
+            plat_name: manylinux2014_x86_64
           - os: ubuntu-latest
             platform: linux
             python-version: "3.10"
             arch: arm64
-            plat_name: linux_aarch64
-          - os: windows-latest
+            plat_name: manylinux2014_aarch64
+          - os: ubuntu-latest
             platform: windows
             python-version: "3.10"
             arch: x86_64
             plat_name: win_amd64
-          - os: macos-latest
+          - os: ubuntu-latest
             platform: macos
             python-version: "3.10"
             arch: x86_64
             plat_name: macosx_10_9_x86_64
-          - os: macos-latest
+          - os: ubuntu-latest
             platform: macos
             python-version: "3.10"
             arch: arm64
@@ -56,9 +58,9 @@ jobs:
 
       - name: Install build dependencies
         run: |
-          cd python-package
+          cd packages/python
           python3 -m pip install --upgrade pip
-          pip install .[dev]
+          python3 -m pip install -r requirements-dev.txt
 
       - name: Get version
         id: get_version
@@ -73,17 +75,22 @@ jobs:
 
       - name: Download artifacts for current platform
         run: |
+          cd packages/python
           python3 download_artifacts.py "${{ matrix.plat_name }}" "${{ steps.get_version.outputs.version }}"
 
       - name: Build wheel
         env:
           PACKAGE_VERSION: ${{ steps.get_version.outputs.version }}
         run: |
-          cd python-package
-          python setup.py clean --all bdist_wheel --plat-name "${{ matrix.plat_name }}"
+          cd packages/python
+          python setup.py bdist_wheel --plat-name "${{ matrix.plat_name }}"
 
       - name: Publish to PyPI
-        if: github.event_name == 'release'
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          packages-dir: python-package/dist
+          packages-dir: packages/python/dist
+          verbose: true
+          # Avoid workflow to fail if the version has already been published
+          skip-existing: true
+          # Upload to Test Pypi for testing
+          # repository-url: https://test.pypi.org/legacy/

README.md

@@ -40,6 +40,17 @@ Download the appropriate pre-built binary for your platform from the official [R
 SELECT load_extension('./ai');
 ```
 
+### Python Package
+
+Python developers can quickly get started using the ready-to-use `sqlite-ai` package available on PyPI:
+
+```bash
+pip install sqlite-ai
+```
+
+For usage details and examples, see the [Python package documentation](./packages/python/README.md).
+
+
 ## Getting Started
 
 Here's a quick example to get started with SQLite Sync:

packages/python/.gitignore

@@ -0,0 +1,23 @@
+# Build directories
+build/
+dist/
+*.egg-info/
+
+# Python cache
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+
+# Virtual environments
+venv/
+env/
+ENV/
+
+# Package files
+*.whl
+*.tar.gz
+
+# Temporary files
+*.tmp
+*.log

packages/python/MANIFEST.in

@@ -0,0 +1,3 @@
+include README.md
+include LICENSE
+recursive-include src/sqliteai/binaries *

packages/python/README.md

@@ -0,0 +1,49 @@
+## SQLite AI Python package
+
+This package provides the sqlite-ai extension prebuilt binaries for multiple platforms and architectures.
+
+### SQLite-AI
+
+SQLite-AI is an extension for SQLite that brings artificial intelligence capabilities directly into the database. It enables developers to run, fine-tune, and serve AI models from within SQLite using simple SQL queries — ideal for on-device and edge applications where low-latency and offline inference are critical. The extension is actively developed by [SQLite AI](https://sqlite.ai), some API and features are still evolving.
+
+More details on the official repository [sqliteai/sqlite-ai](https://github.com/sqliteai/sqlite-ai).
+
+### Documentation
+
+For detailed information on all available functions, their parameters, and examples, refer to the [comprehensive API Reference](https://github.com/sqliteai/sqlite-ai/blob/main/API.md).
+
+### Supported Platforms and Architectures
+
+| Platform      | Arch         | Subpackage name         | Binary name |
+| ------------- | ------------ | ----------------------- | ----------- |
+| Linux (CPU)   | x86_64/arm64 | sqliteai.binaries.cpu   | ai.so       |
+| Linux (GPU)   | x86_64/arm64 | sqliteai.binaries.gpu   | ai.so       |
+| Windows (CPU) | x86_64       | sqliteai.binaries.cpu   | ai.dll      |
+| Windows (GPU) | x86_64       | sqliteai.binaries.gpu   | ai.dll      |
+| macOS (CPU)   | x86_64/arm64 | sqliteai.binaries.cpu   | ai.dylib    |
+
+## Usage
+
+> **Note:** Some SQLite installations on certain operating systems may have extension loading disabled by default.   
+If you encounter issues loading the extension, refer to the [sqlite-extensions-guide](https://github.com/sqliteai/sqlite-extensions-guide/) for platform-specific instructions on enabling and using SQLite extensions.
+
+```python
+import importlib.resources
+import sqlite3
+
+# Connect to your SQLite database
+conn = sqlite3.connect("example.db")
+
+# Load the sqlite-ai extension
+# pip will install the correct binary package for your platform and architecture
+# Choose between CPU or GPU variant
+ext_path = importlib.resources.files("sqliteai.binaries.cpu") / "ai"
+
+conn.enable_load_extension(True)
+conn.load_extension(str(ext_path))
+conn.enable_load_extension(False)
+
+
+# Now you can use sqlite-ai features in your SQL queries
+print(conn.execute("SELECT ai_version();").fetchone())
+```

packages/python/download_artifacts.py

@@ -0,0 +1,99 @@
+import sys
+import zipfile
+import requests
+from pathlib import Path
+import shutil
+
+
+# == USAGE ==
+# python3 download_artifacts.py PLATFORM VERSION
+#   eg: python3 download_artifacts.py linux_x86_64 "0.5.9"
+
+REPO = "sqliteai/sqlite-ai"
+RELEASE_URL = f"https://github.com/{REPO}/releases/download"
+
+# Map Python plat_name to artifact names
+ARTIFACTS = {
+    "manylinux2014_x86_64": ["ai-linux-cpu-x86_64", "ai-linux-gpu-x86_64"],
+    "manylinux2014_aarch64": [
+        "ai-linux-cpu-arm64",
+        "ai-linux-gpu-arm64",
+    ],
+    "win_amd64": ["ai-windows-cpu-x86_64", "ai-windows-gpu-x86_64"],
+    "macosx_10_9_x86_64": ["ai-macos"],
+    "macosx_11_0_arm64": ["ai-macos"],
+}
+
+BINARY_NAME = {
+    "manylinux2014_x86_64": "ai.so",
+    "manylinux2014_aarch64": "ai.so",
+    "win_amd64": "ai.dll",
+    "macosx_10_9_x86_64": "ai.dylib",
+    "macosx_11_0_arm64": "ai.dylib",
+}
+
+BINARIES_DIR = Path(__file__).parent / "src/sqliteai/binaries"
+
+
+def download_and_extract(artifact_name, bin_name, version):
+    artifact = f"{artifact_name}-{version}.zip"
+    url = f"{RELEASE_URL}/{version}/{artifact}"
+    print(f"Downloading {url}")
+
+    r = requests.get(url)
+    if r.status_code != 200:
+        print(f"Failed to download {artifact}: {r.status_code}")
+        sys.exit(1)
+
+    zip_path = BINARIES_DIR / artifact
+    with open(zip_path, "wb") as f:
+        f.write(r.content)
+
+    subdir = "gpu" if "gpu" in artifact_name else "cpu"
+    out_dir = BINARIES_DIR / subdir
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    with zipfile.ZipFile(zip_path, "r") as zip_ref:
+        for member in zip_ref.namelist():
+            if member.endswith(bin_name):
+                zip_ref.extract(member, out_dir)
+
+                # Move to expected name/location
+                src = out_dir / member
+                dst = out_dir / bin_name
+                src.rename(dst)
+
+                print(f"Extracted {dst}")
+
+    zip_path.unlink()
+
+
+def main():
+    version = None
+    platform = None
+    if len(sys.argv) == 3:
+        platform = sys.argv[1].lower()
+        version = sys.argv[2]
+
+    if not version or not platform:
+        print(
+            'Error: Version is not specified.\nUsage: \n   python3 download_artifacts.py linux_x86_64 "0.5.9"'
+        )
+        sys.exit(1)
+
+    print(BINARIES_DIR)
+    if BINARIES_DIR.exists():
+        shutil.rmtree(BINARIES_DIR)
+    BINARIES_DIR.mkdir(parents=True, exist_ok=True)
+
+    platform_artifacts = ARTIFACTS.get(platform, [])
+    if not platform_artifacts:
+        print(f"Error: Unknown platform '{platform}'")
+        sys.exit(1)
+
+    for artifact_name in platform_artifacts:
+        download_and_extract(artifact_name, BINARY_NAME[platform], version)
+
+
+if __name__ == "__main__":
+    main()

packages/python/pyproject.toml

@@ -0,0 +1,25 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel", "toml"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "sqlite-ai"
+dynamic = ["version"]
+description = "Python prebuilt binaries for SQLite AI extension for all supported platforms and architectures."
+authors = [
+    { name = "SQLite AI Team" }
+]
+readme = "README.md"
+requires-python = ">=3"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Operating System :: POSIX :: Linux",
+    "Operating System :: Microsoft :: Windows",
+    "Operating System :: MacOS :: MacOS X"
+]
+
+[project.urls]
+Homepage = "https://sqlite.ai"
+Documentation = "https://github.com/sqliteai/sqlite-ai/blob/main/API.md"
+Repository = "https://github.com/sqliteai/sqlite-ai"
+Issues = "https://github.com/sqliteai/sqlite-ai/issues"

packages/python/requirements-dev.txt

@@ -0,0 +1,3 @@
+requests
+toml
+wheel

packages/python/setup.py

@@ -0,0 +1,68 @@
+import setuptools
+import toml
+import os
+import sys
+
+usage = """
+Usage: python setup.py bdist_wheel --plat-name <platform>
+The PACKAGE_VERSION environment variable must be set to the desired version.
+
+Example:
+  PACKAGE_VERSION=0.5.9 python setup.py bdist_wheel --plat-name linux_x86_64
+"""
+
+with open("pyproject.toml", "r") as f:
+    pyproject = toml.load(f)
+
+project = pyproject["project"]
+
+# Get version from environment or default
+version = os.environ.get("PACKAGE_VERSION", "")
+if not version:
+    print("PACKAGE_VERSION environment variable is not set.")
+    print(usage)
+    sys.exit(1)
+
+# Get Python platform name from --plat-name argument
+plat_name = None
+for i, arg in enumerate(sys.argv):
+    if arg == "--plat-name" and i + 1 < len(sys.argv):
+        plat_name = sys.argv[i + 1]
+        break
+
+if not plat_name:
+    print("Error: --plat-name argument is required")
+    print(usage)
+    sys.exit(1)
+
+# Map plat_name to classifier
+classifier_map = {
+    "manylinux2014_x86_64": "Operating System :: POSIX :: Linux",
+    "manylinux2014_aarch64": "Operating System :: POSIX :: Linux",
+    "win_amd64": "Operating System :: Microsoft :: Windows",
+    "macosx_10_9_x86_64": "Operating System :: MacOS",
+    "macosx_11_0_arm64": "Operating System :: MacOS",
+}
+
+classifier = classifier_map.get(plat_name)
+if not classifier:
+    print(f"Unknown plat_name: {plat_name}")
+    sys.exit(1)
+
+with open("README.md", "r", encoding="utf-8") as f:
+    long_description = f.read()
+
+setuptools.setup(
+    name=project["name"],
+    version=version,
+    description=project.get("description", ""),
+    author=project["authors"][0]["name"] if project.get("authors") else "",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url=project["urls"]["Homepage"],
+    packages=setuptools.find_packages(where="src"),
+    package_dir={"": "src"},
+    include_package_data=True,
+    python_requires=project.get("requires-python", ">=3"),
+    classifiers=project.get("classifiers", []),
+)