Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ Make your you have a postgres database with postgis support
running if you want to use the database pipeline and expor
t an environment variable pointing to it like so:
```sh
export DATABASE_URL=postgres://[email protected]:5432/jedeschule
export DATABASE_URL=postgresql://[email protected]:5432/jedeschule
```
Then, run the migrations to bring you database up to date
```sh
Expand All @@ -84,7 +84,7 @@ To run some very basic integration tests, you will need another postgres
database for test data. After creating it, run the following steps
(assuming you called your database `jedeschule_scraper_test`):
```
export DATABASE_URL=postgres://[email protected]:5432/jedeschule_scraper_test
export DATABASE_URL=postgresql://[email protected]:5432/jedeschule_scraper_test
uv run alembic upgrade head
uv run python test_models.py
```
Expand Down
3 changes: 2 additions & 1 deletion alembic/versions/b3913e0b45ac_add_location_to_school.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
"""

import geoalchemy2
from sqlalchemy import text

from alembic import op
import sqlalchemy as sa
Expand All @@ -21,7 +22,7 @@

def upgrade():
conn = op.get_bind()
conn.execute("CREATE EXTENSION IF NOT EXISTS postgis;")
conn.execute(text("CREATE EXTENSION IF NOT EXISTS postgis;"))
op.add_column(
"schools",
sa.Column(
Expand Down
2 changes: 1 addition & 1 deletion jedeschule/pipelines/db_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def update_or_create(item: SchoolPipelineItem, session=None) -> School:
session = get_session()

school_data = {**item.info}
school = session.query(School).get(item.info["id"])
school = session.get(School, item.info["id"])
latitude = school_data.pop("latitude", None)
longitude = school_data.pop("longitude", None)
if latitude is not None and longitude is not None:
Expand Down
6 changes: 3 additions & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,13 @@ description = "Web scrapers for the jedeschule.codefor.de project"
readme = "README.md"
requires-python = ">=3.12"
dependencies = [
"alembic==1.3.3",
"geoalchemy2==0.8.4",
"alembic==1.16.4",
"geoalchemy2==0.18.0",
"openpyxl>=3.1.5",
"psycopg2==2.9.10",
"pyproj==3.7.1",
"requests==2.32.4",
"scrapy==2.13.3",
"sqlalchemy==1.3.10",
"sqlalchemy==2.0.43",
"xmltodict==0.14.2",
]
Loading