Merge pull request #4 from OpenDataServices/1-get-datasets

Implement GET /datasets endpoint

Author: tillywoodfield

.github/workflows/ci.yml

@@ -0,0 +1,25 @@
+name: CI
+on: [push]
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      - name: Install dev requirements
+        run: pip install -r requirements_dev.txt
+      - name: Install local package
+        run: pip install .
+      - name: Check black
+        run: black --check oc4ids_datastore_api/ tests/
+      - name: Check isort
+        run: isort --check-only oc4ids_datastore_api/ tests/
+      - name: Check flake8
+        run: flake8 oc4ids_datastore_api/ tests/
+      - name: Check mypy
+        run: mypy oc4ids_datastore_api/ tests/
+      - name: Run tests
+        run: pytest

.gitignore

@@ -0,0 +1,2 @@
+.venv
+__pycache__

.python-version

@@ -0,0 +1 @@
+3.12

README.md

@@ -1 +1,42 @@
-# oc4ids-datastore-api
+# OC4IDS Datastore API
+
+## Local Development
+
+### Prerequisites
+
+- Python 3.12
+
+### Install Python requirements
+
+```
+python -m venv .venv
+source .venv/bin/activate
+pip install -r requirements_dev.txt
+```
+
+### Set database enrivonment variable
+
+With a read-only user, set the path to the already existing database, which is created by [oc4ids-datastore-pipeline](https://github.com/OpenDataServices/oc4ids-datastore-pipeline).
+
+```
+export DATABASE_URL="postgresql://oc4ids_datastore_read_only@localhost/oc4ids_datastore"
+```
+
+### Run app
+
+```
+fastapi dev oc4ids_datastore_api/main.py
+```
+
+### View the OpenAPI schema
+
+While the app is running, go to `http://127.0.0.1:8000/docs/`
+
+### Run linting and type checking
+
+```
+black oc4ids_datastore_api/ tests/
+isort oc4ids_datastore_api/ tests/
+flake8 oc4ids_datastore_api/ tests/
+mypy oc4ids_datastore_api/ tests/
+```

oc4ids_datastore_api/__init__.py

@@ -0,0 +1,21 @@
+import os
+from typing import Sequence
+
+from sqlalchemy import Engine
+from sqlmodel import Session, create_engine, select
+
+from oc4ids_datastore_api.models import DatasetSQLModel
+
+_engine = None
+
+
+def get_engine() -> Engine:
+    global _engine
+    if _engine is None:
+        _engine = create_engine(os.environ["DATABASE_URL"])
+    return _engine
+
+
+def fetch_all_datasets() -> Sequence[DatasetSQLModel]:
+    with Session(get_engine()) as session:
+        return session.exec(select(DatasetSQLModel)).all()

oc4ids_datastore_api/database.py

@@ -0,0 +1,11 @@
+from fastapi import FastAPI
+
+from oc4ids_datastore_api.schemas import Dataset
+from oc4ids_datastore_api.services import get_all_datasets
+
+app = FastAPI()
+
+
+@app.get("/datasets")
+def get_datasets() -> list[Dataset]:
+    return get_all_datasets()

oc4ids_datastore_api/main.py

@@ -0,0 +1,17 @@
+import datetime
+
+from sqlmodel import Field, SQLModel
+
+
+class DatasetSQLModel(SQLModel, table=True):
+    __tablename__ = "dataset"
+
+    dataset_id: str = Field(primary_key=True)
+    source_url: str
+    publisher_name: str
+    license_url: str | None
+    license_name: str | None
+    json_url: str | None
+    csv_url: str | None
+    xlsx_url: str | None
+    updated_at: datetime.datetime

oc4ids_datastore_api/models.py

@@ -0,0 +1,25 @@
+import datetime
+
+from pydantic import BaseModel
+
+
+class Publisher(BaseModel):
+    name: str
+
+
+class License(BaseModel):
+    url: str | None
+    name: str | None
+
+
+class Download(BaseModel):
+    format: str
+    url: str
+
+
+class Dataset(BaseModel):
+    loaded_at: datetime.datetime
+    source_url: str
+    publisher: Publisher
+    license: License
+    downloads: list[Download]

oc4ids_datastore_api/schemas.py

@@ -0,0 +1,26 @@
+from oc4ids_datastore_api.database import fetch_all_datasets
+from oc4ids_datastore_api.models import DatasetSQLModel
+from oc4ids_datastore_api.schemas import Dataset, Download, License, Publisher
+
+
+def _transform_dataset(dataset: DatasetSQLModel) -> Dataset:
+    download_urls = {
+        "json": dataset.json_url,
+        "csv": dataset.csv_url,
+        "xlsx": dataset.xlsx_url,
+    }
+    downloads = [
+        Download(format=format, url=url) for format, url in download_urls.items() if url
+    ]
+    return Dataset(
+        loaded_at=dataset.updated_at,
+        source_url=dataset.source_url,
+        publisher=Publisher(name=dataset.publisher_name),
+        license=License(url=dataset.license_url, name=dataset.license_name),
+        downloads=downloads,
+    )
+
+
+def get_all_datasets() -> list[Dataset]:
+    datasets = fetch_all_datasets()
+    return [_transform_dataset(dataset) for dataset in datasets]