feat: migration for create dataset table

Author: tillywoodfield

README.md

@@ -23,6 +23,12 @@ pip install -r requirements_dev.txt
 export DATABASE_URL="postgresql://oc4ids_datastore@localhost/oc4ids_datastore"
 ```
 
+### Run database migrations
+
+```
+alembic upgrade head
+```
+
 ### Run app
 
 ```
@@ -44,3 +50,9 @@ mypy oc4ids_datastore_pipeline/ tests/
 ```
 pytest
 ```
+
+### Generating new database migrations
+
+```
+alembic revision --autogenerate -m "<MESSAGE HERE>"
+```

migrations/versions/aaabf849b37f_create_dataset_table.py

@@ -0,0 +1,39 @@
+"""create dataset table
+
+Revision ID: aaabf849b37f
+Revises:
+Create Date: 2025-02-04 17:14:46.411090
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "aaabf849b37f"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "dataset",
+        sa.Column("dataset_id", sa.String(), nullable=False),
+        sa.Column("source_url", sa.String(), nullable=False),
+        sa.Column("publisher_name", sa.String(), nullable=False),
+        sa.Column("json_url", sa.String(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.PrimaryKeyConstraint("dataset_id"),
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table("dataset")
+    # ### end Alembic commands ###

oc4ids_datastore_pipeline/database.py

@@ -36,12 +36,6 @@ def get_engine() -> Engine:
     return _engine
 
 
-def create_tables() -> None:
-    logger.info("Creating database tables")
-    Base.metadata.create_all(get_engine())
-    logger.info("Created tables successfully")
-
-
 def save_dataset(dataset: Dataset) -> None:
     with Session(get_engine()) as session:
         session.merge(dataset)

oc4ids_datastore_pipeline/pipeline.py

@@ -7,7 +7,7 @@
 import requests
 from libcoveoc4ids.api import oc4ids_json_output
 
-from oc4ids_datastore_pipeline.database import Dataset, create_tables, save_dataset
+from oc4ids_datastore_pipeline.database import Dataset, save_dataset
 
 logger = logging.getLogger(__name__)
 
@@ -105,5 +105,4 @@ def process_datasets() -> None:
 
 
 def run() -> None:
-    create_tables()
     process_datasets()