Upload service-definition.yml to the storage_client

cdsobs/api.py

@@ -8,6 +8,7 @@
 
 import h5netcdf
 import pandas
+import yaml
 from sqlalchemy.orm import Session
 
 from cdsobs import warning_tracker
@@ -151,6 +152,36 @@ def _run_for_batch(time_space_batch):
     _run_sanity_check(
         config, dataset_name, service_definition, session, source, version
     )
+    # upload the service-definition.yml to the bucket
+    s3_client = S3Client.from_config(config.s3config)
+    bucket = s3_client.get_bucket_name(dataset_name)
+    if s3_client.object_exists(bucket=bucket, name="service_definition.yml"):
+        # CHECK IF IT IS DIFERENT
+        temp_file = tempfile.NamedTemporaryFile()
+        s3_client.download_file(
+            bucket_name=bucket,
+            object_name="service_definition.yml",
+            ofile=temp_file.name,
+        )
+        with open(temp_file.name, "rb") as old, open(
+            service_definition.path, "rb"
+        ) as new:
+            new_service_definition = yaml.safe_load(new)
+            old_service_definition = yaml.safe_load(old)
+        if new_service_definition != old_service_definition:
+            logger.warning("service_definition.yml has changed, re-uploading")
+            s3_client.upload_file(
+                destination_bucket=bucket,
+                object_name="service_definition.yml",
+                file_to_upload=service_definition.path,
+            )
+    else:
+        s3_client.upload_file(
+            destination_bucket=bucket,
+            object_name="service_definition.yml",
+            file_to_upload=service_definition.path,
+        )
+
     # Log successful, taking the warnings into account
     final_message = _print_final_message(
         dataset_name, source, start_year, end_year, "ingestion pipeline"

cdsobs/api_rest/endpoints.py

@@ -14,7 +14,6 @@
 from cdsobs.retrieve.models import RetrieveArgs
 from cdsobs.retrieve.retrieve_services import get_catalogue_entries, get_urls
 from cdsobs.service_definition.api import get_service_definition
-from cdsobs.service_definition.service_definition_models import ServiceDefinition
 from cdsobs.storage import S3Client
 from cdsobs.utils.exceptions import ConfigNotFound, DataNotFoundException, SizeError
 from cdsobs.utils.utils import get_database_session
@@ -109,17 +108,28 @@ def get_sources(
     return list(service_definition.sources)
 
 
-@router.get("/{dataset}/service_definition")
-def get_dataset_service_definition(
+@router.get("/{dataset}/service_definition.yml")
+def get_dataset_service_definition_yaml(
     dataset: str,
     session: Annotated[HttpAPISession, Depends(session_gen)],
-) -> ServiceDefinition:
+) -> StreamingResponse:
     """Get the service definition for a dataset."""
     try:
-        return get_service_definition(session.cdsobs_config, dataset)
+        s3_client = S3Client.from_config(session.cdsobs_config.s3config)
+        bucket_name = s3_client.get_bucket_name(dataset_name=dataset)
+        s3_obj = s3_client.s3.Object(bucket_name, "service_definition.yml")
+        file_like = s3_obj.get()["Body"]
+        return StreamingResponse(
+            content=file_like,
+            media_type="application/x-yaml",
+            headers={
+                "Content-Disposition": "attachment; filename=service_definition.yml"
+            },
+        )
     except FileNotFoundError:
         raise make_http_exception(
-            status_code=404, message=f"Service definition not found for {dataset=}"
+            status_code=404,
+            message=f"Form service_definition.yml not found for {dataset=}.",
         )
 
 

cdsobs/service_definition/api.py

@@ -97,4 +97,4 @@ def get_service_definition(
     with open(path_to_yaml) as f:
         data = yaml.safe_load(f)
 
-    return ServiceDefinition(**data)
+    return ServiceDefinition(**data, path=path_to_yaml)

cdsobs/service_definition/service_definition_models.py

@@ -1,4 +1,5 @@
 """Servide definition models."""
+from pathlib import Path
 from typing import Dict
 
 import numpy
@@ -172,3 +173,4 @@ class ServiceDefinition(BaseModel, extra="forbid"):
     global_attributes: dict
     space_columns: SpaceColumns | None = None
     sources: dict[str, SourceDefinition]
+    path: Path | None = None

tests/test_http_api.py

@@ -1,3 +1,4 @@
+import yaml
 from starlette.testclient import TestClient
 
 from cdsobs.api_rest.app import app
@@ -45,9 +46,10 @@ def test_session() -> HttpAPISession:
 
 def test_service_definition(test_config):
     dataset = "insitu-observations-gnss"
-    actual = client.get(f"/{dataset}/service_definition").json()
-    expected = get_service_definition(test_config, dataset).model_dump()
-    assert actual == expected
+    actual = client.get(f"/{dataset}/service_definition.yml").text
+    expected = get_service_definition(test_config, dataset).model_dump(exclude_none=True, exclude={"path"})
+    expected.pop("path", None)
+    assert yaml.safe_load(actual) == expected
 
 
 def test_capabilities_datasets(test_config, test_repository):