Merge pull request #160 from Datenschule/use-uv

Switch to uv for dependency locking

Author: k-nut

.github/dependabot.yml

@@ -0,0 +1,11 @@
+version: 2
+enable-beta-ecosystems: true
+updates:
+  - package-ecosystem: "uv" 
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/test.yml

@@ -5,14 +5,8 @@ on:
 
 jobs:
   build:
-    env:
-      UV_SYSTEM_PYTHON: 1
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [ '3.11', '3.12']
-      fail-fast: false
-    name: Tests with Python ${{ matrix.python-version }}
+    name: Run tests
     services:
       postgres:
         image: postgis/postgis:13-3.1-alpine
@@ -36,9 +30,8 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v1
         with:
-          python-version: ${{ matrix.python-version }}
-          architecture: x64
-      - run: uv pip install -r requirements.txt
+          python-version-file: ".python-version"
+      - run: uv sync --locked --all-extras
       - run: createdb jedeschule_test
         env:
           PGHOST: localhost
@@ -47,7 +40,7 @@ jobs:
           PGUSER: postgres
       - run: |
           set -e
-          alembic upgrade head
-          python test_models.py
+          uv run alembic upgrade head
+          uv run python test_models.py
         env:
          DATABASE_URL: postgresql://postgres:[email protected]:5432/jedeschule_test

.github/workflows/test_scraper_changes.yml

@@ -5,8 +5,6 @@ on:
 
 jobs:
   build:
-    env:
-      UV_SYSTEM_PYTHON: 1
     runs-on: ubuntu-latest
     name: Show changes in scraper results
     steps:
@@ -18,7 +16,6 @@ jobs:
       - name: Setup python
         uses: actions/setup-python@v1
         with:
-          python-version: 3.11
-          architecture: x64
-      - run: uv pip install -r requirements.txt
-      - run: ./test_changes.sh
+          python-version-file: ".python-version"
+      - run: uv sync --locked --all-extras
+      - run: uv run ./test_changes.sh

.python-version

@@ -0,0 +1 @@
+3.12

Dockerfile

@@ -2,7 +2,9 @@ FROM scrapinghub/scrapinghub-stack-scrapy:2.11
 
 COPY . .
 
-RUN pip install -r requirements.txt
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+RUN uv pip install -r requirements.txt
 
 # Install pg_isready to await db start
 RUN apt-get update && \

README.md

@@ -60,10 +60,11 @@ When available, we try to use the geolocations provided by the data publishers.
 | TH    | ❌ No                 | -                                            |
 
 ## Installation
-Note that at least Python 3.7 is required.
+Dependency management is done using [uv](https://docs.astral.sh/uv/). Make sure
+to have it installed and then run the following command to install the dependencies:
 
 ```bash
-pip install -r requirements.txt
+uv sync --locked --all-extras
 ```
 
 Make your you have a postgres database with postgis support 
@@ -74,7 +75,7 @@ export DATABASE_URL=postgres://[email protected]:5432/jedeschule
 ```
 Then, run the migrations to bring you database up to date
 ```sh
-alembic upgrade head
+uv run alembic upgrade head
 ```
 
 
@@ -84,8 +85,8 @@ database for test data. After creating it, run the following steps
 (assuming you called your database `jedeschule_scraper_test`):
 ```
 export DATABASE_URL=postgres://[email protected]:5432/jedeschule_scraper_test
-alembic upgrade head
-python test_models.py
+uv run alembic upgrade head
+uv run python test_models.py
 ```
 
 If you made changes to scrapers, you can also run a script to check the 
@@ -108,7 +109,7 @@ bash ./scrape_all.sh
 ### Updating data for a single state
 To get updates for a single state, run
 ```bash
-scrapy crawl <state_name>
+uv run scrapy crawl <state_name>
 # use `scrapy list` to get a list of all available states
 ```
 

pyproject.toml

@@ -0,0 +1,16 @@
+[project]
+name = "jedeschule-scraper"
+version = "0.1.0"
+description = "Web scrapers for the jedeschule.codefor.de project"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+    "alembic==1.3.3",
+    "geoalchemy2==0.8.4",
+    "psycopg2==2.9.10",
+    "pyproj==3.6.1",
+    "requests==2.32.0",
+    "scrapy==2.11.2",
+    "sqlalchemy==1.3.10",
+    "xlrd==1.1.0",
+]

requirements.txt

@@ -8,5 +8,5 @@ do
   # cannot even use in the end.
   # This is why we set the maximum amount of items that are
   # allowed to crash to 3 here.
-  scrapy crawl $spider -s "CLOSESPIDER_ERRORCOUNT=3" --loglevel=INFO
+  uv run scrapy crawl $spider -s "CLOSESPIDER_ERRORCOUNT=3" --loglevel=INFO
 done