Skip to content

Commit 1bbc5df

Browse files
cofincofin
authored
feat: driver refactor & cache integration (#44)
Another large refactor based on simplifying code structure & adding a caching layer that can be used at multiple layers in the library.
1 parent 785754c commit 1bbc5df

File tree

404 files changed

+51518
-68355
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

404 files changed

+51518
-68355
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ jobs:
106106
run: uv sync --all-extras --dev
107107

108108
- name: Test
109-
run: uv run pytest -m ""
109+
run: uv run pytest -n 2 --dist=loadgroup tests
110110

111111
# test-windows:
112112
# runs-on: windows-latest

‎.github/workflows/publish.yml‎

Lines changed: 234 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,9 @@ on:
66
workflow_dispatch:
77

88
jobs:
9-
publish-release:
9+
build-source:
10+
name: Build source distribution
1011
runs-on: ubuntu-latest
11-
permissions:
12-
id-token: write
1312
steps:
1413
- name: Check out repository
1514
uses: actions/checkout@v4
@@ -21,10 +20,239 @@ jobs:
2120
run: uv python install 3.12
2221

2322
- name: Install dependencies
24-
run: uv sync --all-extras
23+
run: uv sync --extra performance --group build
24+
25+
- name: Build source distribution
26+
run: uv build --sdist
27+
28+
- name: Upload source artifacts
29+
uses: actions/upload-artifact@v4
30+
with:
31+
name: source-dist
32+
path: dist/*.tar.gz
33+
34+
build-wheels-standard:
35+
name: Build standard wheels (${{ matrix.os }}, Python ${{ matrix.python-version }})
36+
strategy:
37+
fail-fast: false
38+
matrix:
39+
os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
40+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
41+
exclude:
42+
- os: macos-13
43+
python-version: "3.13"
44+
- os: windows-latest
45+
python-version: "3.9"
46+
47+
runs-on: ${{ matrix.os }}
48+
steps:
49+
- name: Check out repository
50+
uses: actions/checkout@v4
51+
52+
- name: Install uv
53+
uses: astral-sh/setup-uv@v4
54+
55+
- name: Set up Python ${{ matrix.python-version }}
56+
run: uv python install ${{ matrix.python-version }}
57+
58+
- name: Install dependencies
59+
run: uv sync --extra performance --group build
60+
61+
- name: Build standard wheel
62+
run: uv build --wheel
63+
64+
- name: Upload wheel artifacts
65+
uses: actions/upload-artifact@v4
66+
with:
67+
name: wheels-standard-${{ matrix.os }}-py${{ matrix.python-version }}
68+
path: dist/*.whl
69+
70+
build-wheels-mypyc:
71+
name: Build MyPyC wheels (${{ matrix.os }}, Python ${{ matrix.python-version }})
72+
strategy:
73+
fail-fast: false
74+
matrix:
75+
os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
76+
python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
77+
exclude:
78+
- os: windows-latest
79+
python-version: "3.9"
80+
- os: macos-13
81+
python-version: "3.13"
82+
83+
runs-on: ${{ matrix.os }}
84+
steps:
85+
- name: Check out repository
86+
uses: actions/checkout@v4
87+
88+
- name: Install uv
89+
uses: astral-sh/setup-uv@v4
90+
91+
- name: Set up Python ${{ matrix.python-version }}
92+
run: uv python install ${{ matrix.python-version }}
93+
94+
- name: Install dependencies with mypyc extras
95+
run: uv sync --extra mypyc --group build
96+
97+
- name: Set up MyPyC environment variables
98+
run: |
99+
echo "MYPYC_OPT_LEVEL=3" >> $GITHUB_ENV
100+
echo "MYPYC_DEBUG_LEVEL=0" >> $GITHUB_ENV
101+
echo "MYPYC_MULTI_FILE=1" >> $GITHUB_ENV
102+
shell: bash
103+
104+
- name: Build MyPyC wheel
105+
run: uv build --wheel
106+
env:
107+
HATCH_BUILD_HOOK_ENABLE_MYPYC: "1"
108+
109+
- name: Rename wheel to indicate mypyc
110+
run: |
111+
for wheel in dist/*.whl; do
112+
if [[ -f "$wheel" ]]; then
113+
base=$(basename "$wheel" .whl)
114+
dir=$(dirname "$wheel")
115+
mv "$wheel" "$dir/${base}+mypyc.whl"
116+
fi
117+
done
118+
shell: bash
119+
120+
- name: Upload mypyc wheel artifacts
121+
uses: actions/upload-artifact@v4
122+
with:
123+
name: wheels-mypyc-${{ matrix.os }}-py${{ matrix.python-version }}
124+
path: dist/*.whl
125+
126+
build-universal-wheels:
127+
name: Build universal wheels with cibuildwheel
128+
runs-on: ${{ matrix.os }}
129+
strategy:
130+
fail-fast: false
131+
matrix:
132+
os: [ubuntu-latest, windows-latest, macos-latest]
133+
cibw_python: ["cp39", "cp310", "cp311", "cp312", "cp313"]
134+
cibw_arch: ["x86_64", "aarch64", "arm64"]
135+
exclude:
136+
- os: ubuntu-latest
137+
cibw_arch: arm64
138+
- os: windows-latest
139+
cibw_arch: aarch64
140+
- os: windows-latest
141+
cibw_arch: arm64
142+
- os: macos-latest
143+
cibw_arch: aarch64
144+
145+
steps:
146+
- name: Check out repository
147+
uses: actions/checkout@v4
148+
149+
- name: Set up QEMU (for ARM builds on Linux)
150+
if: matrix.os == 'ubuntu-latest' && matrix.cibw_arch == 'aarch64'
151+
uses: docker/setup-qemu-action@v3
152+
with:
153+
platforms: arm64
154+
155+
- name: Build wheels with cibuildwheel
156+
uses: pypa/[email protected]
157+
env:
158+
CIBW_BUILD: ${{ matrix.cibw_python }}-*
159+
CIBW_ARCHS: ${{ matrix.cibw_arch }}
160+
CIBW_BUILD_VERBOSITY: 2
161+
CIBW_ENVIRONMENT: >
162+
MYPYC_OPT_LEVEL=3
163+
MYPYC_DEBUG_LEVEL=0
164+
MYPYC_MULTI_FILE=1
165+
HATCH_BUILD_HOOK_ENABLE_MYPYC=1
166+
CIBW_BEFORE_BUILD: |
167+
python -m pip install uv
168+
uv sync --extra mypyc --group build
169+
CIBW_BUILD_FRONTEND: "build[uv]"
170+
CIBW_TEST_SKIP: "*"
171+
172+
- name: Upload cibuildwheel artifacts
173+
uses: actions/upload-artifact@v4
174+
with:
175+
name: wheels-cibw-${{ matrix.os }}-${{ matrix.cibw_python }}-${{ matrix.cibw_arch }}
176+
path: wheelhouse/*.whl
177+
178+
test-wheels:
179+
name: Test wheels on ${{ matrix.os }}
180+
needs: [build-wheels-standard, build-wheels-mypyc]
181+
strategy:
182+
fail-fast: false
183+
matrix:
184+
os: [ubuntu-latest, windows-latest, macos-latest]
185+
python-version: ["3.10", "3.12"]
186+
187+
runs-on: ${{ matrix.os }}
188+
steps:
189+
- name: Check out repository
190+
uses: actions/checkout@v4
191+
192+
- name: Install uv
193+
uses: astral-sh/setup-uv@v4
194+
195+
- name: Set up Python ${{ matrix.python-version }}
196+
run: uv python install ${{ matrix.python-version }}
197+
198+
- name: Download standard wheel artifacts
199+
uses: actions/download-artifact@v4
200+
with:
201+
name: wheels-standard-${{ matrix.os }}-py${{ matrix.python-version }}
202+
path: dist-standard/
203+
204+
- name: Download mypyc wheel artifacts
205+
uses: actions/download-artifact@v4
206+
with:
207+
name: wheels-mypyc-${{ matrix.os }}-py${{ matrix.python-version }}
208+
path: dist-mypyc/
209+
210+
- name: Test standard wheel installation
211+
run: |
212+
uv venv test-standard
213+
uv pip install --find-links dist-standard/ sqlspec
214+
uv run --python test-standard python -c "import sqlspec; print('Standard wheel OK')"
215+
216+
- name: Test mypyc wheel installation
217+
run: |
218+
uv venv test-mypyc
219+
uv pip install --find-links dist-mypyc/ sqlspec
220+
uv run --python test-mypyc python -c "import sqlspec; print('MyPyC wheel OK')"
221+
222+
publish-release:
223+
name: Publish to PyPI
224+
needs: [build-source, build-wheels-standard, build-wheels-mypyc, build-universal-wheels, test-wheels]
225+
runs-on: ubuntu-latest
226+
permissions:
227+
id-token: write
228+
environment:
229+
name: pypi
230+
url: https://pypi.org/project/sqlspec/
231+
232+
steps:
233+
- name: Download all artifacts
234+
uses: actions/download-artifact@v4
235+
with:
236+
pattern: "*"
237+
merge-multiple: true
238+
path: dist/
239+
240+
- name: List all built packages
241+
run: |
242+
echo "=== All built packages ==="
243+
find dist/ -name "*.whl" -o -name "*.tar.gz" | sort
244+
echo "=== Package count ==="
245+
find dist/ -name "*.whl" | wc -l
246+
find dist/ -name "*.tar.gz" | wc -l
25247
26-
- name: Build package
27-
run: uv build
248+
- name: Verify package integrity
249+
run: |
250+
python -m pip install twine
251+
twine check dist/*
28252
29253
- name: Publish package distributions to PyPI
30254
uses: pypa/gh-action-pypi-publish@release/v1
255+
with:
256+
packages-dir: dist/
257+
verbose: true
258+
print-hash: true

‎.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,3 +54,10 @@ requirements/*
5454
tools/*.json
5555
benchmarks/
5656
.benchmark
57+
*.db
58+
*.db-journal
59+
*.db-wal
60+
*.db-shm
61+
*.duckdb
62+
.crush
63+
CRUSH.md

‎.pre-commit-config.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ repos:
77
- id: conventional-pre-commit
88
stages: [commit-msg]
99
- repo: https://github.com/pre-commit/pre-commit-hooks
10-
rev: v5.0.0
10+
rev: v6.0.0
1111
hooks:
1212
- id: check-ast
1313
- id: check-case-conflict
@@ -17,7 +17,7 @@ repos:
1717
- id: mixed-line-ending
1818
- id: trailing-whitespace
1919
- repo: https://github.com/charliermarsh/ruff-pre-commit
20-
rev: "v0.12.2"
20+
rev: "v0.12.8"
2121
hooks:
2222
- id: ruff
2323
args: ["--fix"]

‎Makefile‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,18 @@ install: destroy clean ## Install the project, depe
4848
@uv sync --all-extras --dev
4949
@echo "${OK} Installation complete! 🎉"
5050

51+
.PHONY: install-compiled
52+
install-compiled: destroy clean ## Install with mypyc compilation for performance
53+
@echo "${INFO} Starting fresh installation with mypyc compilation..."
54+
@uv python pin 3.12 >/dev/null 2>&1
55+
@uv venv >/dev/null 2>&1
56+
@echo "${INFO} Installing in editable mode with mypyc compilation..."
57+
@HATCH_BUILD_HOOKS_ENABLE=1 uv pip install -e .
58+
@uv sync --all-extras --dev
59+
@echo "${OK} Performance installation complete! 🚀"
60+
@echo "${INFO} Verifying compilation..."
61+
@find sqlspec -name "*.so" | wc -l | xargs -I {} echo "${OK} Compiled {} modules"
62+
5163
.PHONY: destroy
5264
destroy: ## Destroy the virtual environment
5365
@echo "${INFO} Destroying virtual environment... 🗑️"
@@ -83,6 +95,22 @@ build: ## Build the package
8395
@uv build >/dev/null 2>&1
8496
@echo "${OK} Package build complete"
8597

98+
.PHONY: build-performance
99+
build-performance: ## Build package with mypyc compilation
100+
@echo "${INFO} Building package with mypyc compilation... 📦"
101+
@HATCH_BUILD_HOOKS_ENABLE=1 uv build >/dev/null 2>&1
102+
@echo "${OK} Performance package build complete 🚀"
103+
104+
.PHONY: test-mypyc
105+
test-mypyc: ## Test mypyc compilation on individual modules
106+
@echo "${INFO} Testing mypyc compilation... 🔧"
107+
@uv run mypyc --check-untyped-defs sqlspec/utils/statement_hashing.py
108+
@uv run mypyc --check-untyped-defs sqlspec/utils/text.py
109+
@uv run mypyc --check-untyped-defs sqlspec/utils/sync_tools.py
110+
@uv run mypyc --check-untyped-defs sqlspec/statement/cache.py
111+
@echo "${OK} Mypyc compilation tests passed ✨"
112+
113+
86114
.PHONY: release
87115
release: ## Bump version and create release tag
88116
@echo "${INFO} Preparing for release... 📦"
@@ -108,6 +136,8 @@ clean: ## Cleanup temporary build a
108136
@find . -name '*~' -exec rm -f {} + >/dev/null 2>&1
109137
@find . -name '__pycache__' -exec rm -rf {} + >/dev/null 2>&1
110138
@find . -name '.ipynb_checkpoints' -exec rm -rf {} + >/dev/null 2>&1
139+
@find . -name '*.so' -exec rm -f {} + >/dev/null 2>&1
140+
@find . -name '*.c' -exec rm -f {} + >/dev/null 2>&1
111141
@echo "${OK} Working directory cleaned"
112142
$(MAKE) docs-clean
113143

@@ -215,6 +245,49 @@ docs-linkcheck-full: ## Run full documentation lin
215245
@uv run sphinx-build -b linkcheck ./docs ./docs/_build -D linkcheck_anchors=0
216246
@echo "${OK} Full link check complete"
217247

248+
# =============================================================================
249+
# Development Infrastructure
250+
# =============================================================================
251+
252+
.PHONY: infra-up
253+
infra-up: ## Start development infrastructure (databases, storage)
254+
@echo "${INFO} Starting development infrastructure..."
255+
@./tools/local-infra.sh up
256+
@echo "${OK} Development infrastructure ready ✨"
257+
258+
.PHONY: infra-down
259+
infra-down: ## Stop development infrastructure
260+
@echo "${INFO} Stopping development infrastructure..."
261+
@./tools/local-infra.sh down --quiet
262+
@echo "${OK} Development infrastructure stopped"
263+
264+
.PHONY: infra-status
265+
infra-status: ## Show development infrastructure status
266+
@./tools/local-infra.sh status
267+
268+
.PHONY: infra-cleanup
269+
infra-cleanup: ## Clean up development infrastructure
270+
@echo "${WARN} This will remove all development containers and volumes"
271+
@./tools/local-infra.sh cleanup
272+
273+
.PHONY: infra-postgres
274+
infra-postgres: ## Start only PostgreSQL
275+
@echo "${INFO} Starting PostgreSQL..."
276+
@./tools/local-infra.sh up postgres --quiet
277+
@echo "${OK} PostgreSQL ready on port 5433"
278+
279+
.PHONY: infra-oracle
280+
infra-oracle: ## Start only Oracle
281+
@echo "${INFO} Starting Oracle..."
282+
@./tools/local-infra.sh up oracle --quiet
283+
@echo "${OK} Oracle ready on port 1522"
284+
285+
.PHONY: infra-mysql
286+
infra-mysql: ## Start only MySQL
287+
@echo "${INFO} Starting MySQL..."
288+
@./tools/local-infra.sh up mysql --quiet
289+
@echo "${OK} MySQL ready on port 3307"
290+
218291
# =============================================================================
219292
# End of Makefile
220293
# =============================================================================

0 commit comments

Comments
 (0)