feat(workflow): build pypi package

Author: Daniele Briggi

.github/workflows/pypi-package.yaml

@@ -0,0 +1,61 @@
+name: Build and Publish Python Package
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "Version to use for the Python package (e.g. 0.1.0)"
+        required: true
+        type: string
+  release:
+    types: [published]
+
+jobs:
+  build-and-publish:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write # mandatory for Pypi trusted publishing
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: false
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+
+      - name: Install build dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          python3 -m pip install .[dev]
+
+      - name: Get version
+        id: get_version
+        run: |
+          if [[ "${{ github.event_name }}" == "release" ]]; then
+            VERSION="${{ github.event.release.tag_name }}"
+          else
+            VERSION="${{ github.event.inputs.version }}"
+          fi
+          VERSION=${VERSION#v}
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+
+      - name: Build
+        env:
+          PACKAGE_VERSION: ${{ steps.get_version.outputs.version }}
+        run: |
+          # Update version in pyproject.toml
+          sed -i 's/^version = ".*"/version = "${{ steps.get_version.outputs.version }}"/' pyproject.toml
+          python setup.py bdist_wheel
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          packages-dir: dist
+          verbose: true
+          # Avoid workflow to fail if the version has already been published
+          skip-existing: true
+          # Upload to Test Pypi for testing
+          repository-url: https://test.pypi.org/legacy/

.pre-commit-config.yaml

@@ -11,6 +11,11 @@ repos:
     rev: 25.1.0
     hooks:
     -   id: black
+- repo: https://github.com/PyCQA/autoflake
+  rev: v2.3.1
+  hooks:
+    - id: autoflake
+      args: ["--remove-all-unused-imports", "--remove-unused-variables", "--ignore-init-module-imports", "--in-place", "--recursive", "."]
 - repo: https://github.com/pycqa/isort
   rev: 6.0.1
   hooks:

MANIFEST.in

@@ -0,0 +1 @@
+include README.md

README.md

@@ -1,7 +1,165 @@
-# sqlite-rag
+# SQLite RAG
+
+A hybrid search engine built on SQLite with AI and Vector extensions. SQLite-RAG combines vector similarity search with full-text search using Reciprocal Rank Fusion (RRF) for enhanced document retrieval.
+
+## Features
+
+- **Hybrid Search**: Combines vector embeddings with full-text search for optimal results
+- **SQLite-based**: Built on SQLite with AI and Vector extensions for reliability and performance
+- **Multi-format Support**: Process 25+ file formats including PDF, DOCX, Markdown, code files
+- **Intelligent Chunking**: Token-aware text chunking with configurable overlap
+- **Interactive CLI**: Command-line interface with interactive REPL mode
+- **Flexible Configuration**: Customizable embedding models, search weights, and chunking parameters
 
 ## Installation
 
 ```bash
-pip install .[dev]
+pip install sqlite-rag
+```
+
+## Quick Start
+
+```bash
+# Initialize and add documents
+sqlite-rag add /path/to/documents --recursive
+
+# Search your documents
+sqlite-rag search "your search query"
+
+# Interactive mode
+sqlite-rag
+> help
+> search "interactive search"
+> exit
+```
+
+## CLI Commands
+
+### Document Management
+
+**Add files or directories:**
+```bash
+sqlite-rag add <path> [--recursive] [--absolute-paths] [--metadata '{"key": "value"}']
+```
+
+**Add raw text:**
+```bash
+sqlite-rag add-text "your text content" [uri] [--metadata '{"key": "value"}']
+```
+
+**List all documents:**
+```bash
+sqlite-rag list
+```
+
+**Remove documents:**
+```bash
+sqlite-rag remove <path-or-uuid> [--yes]
+```
+
+### Search & Query
+
+**Hybrid search:**
+```bash
+sqlite-rag search "your query" [--limit 10] [--debug]
+```
+
+Use `--debug` to see detailed ranking information including vector ranks, FTS ranks, and combined scores.
+
+### Database Operations
+
+**Rebuild indexes and embeddings:**
+```bash
+sqlite-rag rebuild [--remove-missing]
 ```
+
+**Clear entire database:**
+```bash
+sqlite-rag reset [--yes]
+```
+
+### Configuration
+
+**View current settings:**
+```bash
+sqlite-rag settings
+```
+
+**Update configuration:**
+```bash
+sqlite-rag set [options]
+```
+
+Available settings:
+- `--model-path-or-name`: Embedding model (file path or HuggingFace model)
+- `--embedding-dim`: Vector dimensions
+- `--chunk-size`: Text chunk size (tokens)
+- `--chunk-overlap`: Token overlap between chunks
+- `--weight-fts`: Full-text search weight (0.0-1.0)
+- `--weight-vec`: Vector search weight (0.0-1.0)
+- `--quantize-scan`: Enable quantized vectors for faster search
+- `--quantize-preload`: Preload quantized vectors in memory
+
+## Python API
+
+```python
+from sqlite_rag import SQLiteRag
+
+# Create RAG instance
+rag = SQLiteRag.create("./database.sqlite")
+
+# Add documents
+rag.add("/path/to/documents", recursive=True)
+rag.add_text("Raw text content", uri="doc.txt")
+
+# Search
+results = rag.search("search query", top_k=5)
+for result in results:
+    print(f"Score: {result.score}")
+    print(f"Content: {result.content}")
+    print(f"URI: {result.uri}")
+
+# List documents
+documents = rag.list_documents()
+
+# Remove document
+rag.remove_document("document-id-or-path")
+
+# Database operations
+rag.rebuild(remove_missing=True)
+rag.reset()
+```
+
+## Supported File Formats
+
+SQLite-RAG supports 25+ file formats through the MarkItDown library:
+
+- **Text**: `.txt`, `.md`, `.csv`, `.json`, `.xml`
+- **Documents**: `.pdf`, `.docx`, `.pptx`, `.xlsx`
+- **Code**: `.py`, `.js`, `.html`, `.css`, `.sql`
+- **And many more**: `.rtf`, `.odt`, `.epub`, `.zip`, etc.
+
+## How It Works
+
+1. **Document Processing**: Files are processed and split into overlapping chunks
+2. **Embedding Generation**: Text chunks are converted to vector embeddings using AI models
+3. **Dual Indexing**: Content is indexed for both vector similarity and full-text search
+4. **Hybrid Search**: Queries are processed through both search methods
+5. **Result Fusion**: Results are combined using Reciprocal Rank Fusion for optimal relevance
+
+## Default Configuration
+
+- **Model**: Qwen3-Embedding-0.6B (Q8_0 quantized, 1024 dimensions)
+- **Chunking**: 12,000 tokens per chunk with 1,200 token overlap
+- **Vectors**: FLOAT16 storage with cosine similarity
+- **Search**: Equal weighting (1.0) for vector and full-text results
+- **Database**: `./sqliterag.sqlite`
+
+## Extensions Required
+
+SQLite-RAG requires these SQLite extensions:
+
+- **[sqlite-ai](https://github.com/sqliteai/sqlite-ai)**: LLM model loading and embedding generation
+- **[sqlite-vector](https://github.com/sqliteai/sqlite-vector)**: Vector storage and similarity search
+
+These are automatically installed as dependencies.

pyproject.toml

@@ -1,13 +1,20 @@
 [build-system]
-requires = ["setuptools>=61.0", "wheel"]
+requires = ["setuptools>=61.0", "wheel", "toml"]
 build-backend = "setuptools.build_meta"
 
 [project]
 name = "sqlite-rag"
-version = "0.1.0"
+version = "0.0.0"
 description = "Hybird search with SQLite AI and SQLite Vector"
-authors = [{name = "User"}]
+authors = [{name = "SQLite AI Team"}]
 requires-python = ">=3.10"
+readme = "README.md"
+classifiers = [
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Operating System :: OS Independent",
+]
 dependencies = [
     "attrs",
     "typer",
@@ -17,7 +24,6 @@ dependencies = [
     "sqliteai-vector"
 ]
 
-# .. or [dependency-groups] ?
 [project.optional-dependencies]
 dev = [
     "pytest",
@@ -29,6 +35,11 @@ dev = [
     "pre-commit"
 ]
 
+[project.urls]
+Homepage = "https://sqlite.ai"
+Repository = "https://github.com/sqliteai/sqlite-rag"
+Issues = "https://github.com/sqliteai/sqlite-rag/issues"
+
 [project.scripts]
 sqlite-rag = "sqlite_rag.cli:cli"
 

setup.py

@@ -0,0 +1,25 @@
+import setuptools
+import toml
+
+with open("pyproject.toml", "r") as f:
+    pyproject = toml.load(f)
+
+project = pyproject["project"]
+
+with open("README.md", "r", encoding="utf-8") as f:
+    long_description = f.read()
+
+setuptools.setup(
+    name=project["name"],
+    version=project["version"],
+    description=project.get("description", ""),
+    author=project["authors"][0]["name"] if project.get("authors") else "",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    url=project["urls"]["Homepage"],
+    packages=setuptools.find_packages(where="src"),
+    package_dir={"": "src"},
+    include_package_data=True,
+    python_requires=project.get("requires-python", ">=3.10"),
+    classifiers=project.get("classifiers", []),
+)

src/sqlite_rag/sqliterag.py

@@ -1,7 +1,7 @@
 import sqlite3
 from dataclasses import asdict
 from pathlib import Path
-from typing import Optional
+from typing import Any, Optional
 
 from sqlite_rag.logger import Logger
 from sqlite_rag.models.document_result import DocumentResult
@@ -43,7 +43,7 @@ def _ensure_initialized(self):
 
     @staticmethod
     def create(
-        db_path: str = "./sqliterag.sqlite", settings: Optional[Settings] = None
+        db_path: str = "./sqliterag.sqlite", settings: Optional[dict[str, Any]] = None
     ) -> "SQLiteRag":
         """Create a new SQLiteRag instance with the given settings.
 

tests/test_chunker.py

@@ -157,7 +157,7 @@ def test_character_splitting_token_limit(self, chunker_tiny):
         # Each chunk should respect the chunk size limit
         for chunk in chunks:
             token_count = chunker_tiny._get_token_count(chunk.content)
-            assert token_count <= chunker_tiny.settings.chunk_size
+            assert token_count <= chunker_tiny._settings.chunk_size
 
 
 class TestOverlapFunctionality:

tests/test_database.py

@@ -6,7 +6,7 @@
 
 class TestDatabase:
     def test_db_initialization(self):
-        conn = sqlite3.connect(":memory")
+        conn = sqlite3.connect(":memory:")
         Database.initialize(conn, Settings())
 
         # Check if the tables exist

tests/test_engine.py

@@ -1,3 +1,5 @@
+import pytest
+
 from sqlite_rag.chunker import Chunker
 from sqlite_rag.engine import Engine
 from sqlite_rag.models.chunk import Chunk
@@ -20,6 +22,7 @@ def test_search_with_empty_database(self, engine):
 
         assert len(results) == 0
 
+    @pytest.mark.skip(reason="Waiting for sqlite-ai context to be fixed")
     def test_search_with_semantic_and_fts(self, db_conn):
         # Arrange
         conn, settings = db_conn