diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index accde9b..6b407b6 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -12,11 +12,12 @@ jobs: - "3.6" - "3.7" - "3.8" - services: - mongodb: - image: mongo:latest - ports: - - 27017:27017 + - "3.9" +# services: +# mongodb: +# image: mongo:latest +# ports: +# - 27017:27017 steps: - uses: actions/checkout@master - name: Setup Python diff --git a/README.md b/README.md index c19b618..5df3364 100644 --- a/README.md +++ b/README.md @@ -97,6 +97,7 @@ _La API puede devolver información de Paradas y listados en tiempo real de los - Add endpoints for static buses info - Add integration tests - Add detailed install & configuration instructions +- Improve endpoint and services/controllers in-code organization --- @@ -107,6 +108,7 @@ _La API puede devolver información de Paradas y listados en tiempo real de los - _Añadir endpoints para consulta de información estática de buses_ - _Añadir tests de integración_ - _Añadir instrucciones detalladas de instalación y configuración_ +- _Mejorar organización en código de endpoints y servicios/controladores_ ## Disclaimer diff --git a/requirements.txt b/requirements.txt index 176e453..8da0c5d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ cachetools==4.2.2 # TTLCache pymongo==3.10.1 # MongoDB client motor==2.1.0 # MongoDB Async client loguru==0.5.3 # Logging +pytimeparse==1.1.8 # Parse human duration strings into seconds diff --git a/sample.env b/sample.env index e1c9392..6b3d073 100644 --- a/sample.env +++ b/sample.env @@ -35,3 +35,23 @@ buses_normal_limit=5 mongo_uri=mongodb://127.0.0.1 mongo_stops_db=vigobusapi mongo_stops_collection=stops + +# # # # # # # # # # # # # # # # # # # # + +### Google Maps API Settings ### + +# API key for using the Google Maps API. If not set, the features involving its usage will be unavailable (usage would raise exceptions) +google_maps_api_key= + +# Default values for GET Stop Map +google_maps_stop_map_default_size_x=1280 +google_maps_stop_map_default_size_y=720 +google_maps_stop_map_default_zoom=17 +google_maps_stop_map_default_type=roadmap + +# Default values for GET Photo/Streetview +stop_photo_default_size_x=2000 +stop_photo_default_size_y=2000 + +# Language in which print texts in static Maps pictures (2 characters country code) +google_maps_language=es diff --git a/tests/unit/test_checksumable_objects.py b/tests/unit/test_checksumable_objects.py new file mode 100644 index 0000000..45b6837 --- /dev/null +++ b/tests/unit/test_checksumable_objects.py @@ -0,0 +1,46 @@ +import pytest +from vigobusapi.services.google_maps import GoogleMapRequest +from vigobusapi.utils import new_hash_values + + +@pytest.mark.parametrize("obj", [ + GoogleMapRequest(location_x=1, location_y=2, size_x=1, size_y=2, zoom=1, + map_type=GoogleMapRequest.MapTypes.roadmap), + GoogleMapRequest(location_x=1, location_y=2, size_x=1, size_y=2, zoom=2, + map_type=GoogleMapRequest.MapTypes.terrain), + GoogleMapRequest(location_x=1, location_y=2, size_x=1, size_y=2, zoom=2, + map_type=GoogleMapRequest.MapTypes.hybrid), + GoogleMapRequest(location_x=1, location_y=2, size_x=1, size_y=2, zoom=2, + map_type=GoogleMapRequest.MapTypes.satellite, + tags=[GoogleMapRequest.Tag(label="A", location_x=10, location_y=20), + GoogleMapRequest.Tag(label="B", location_x=30, location_y=40)]) +]) +def test_google_map_request(obj: GoogleMapRequest): + tags_hash_value = "NoTags" + if obj.tags: + tags_checksums = list() + for tag in obj.tags: + tag_hash = new_hash_values( + tag.label, + tag.location_x, + tag.location_y, + algorithm="md5" + ) + tags_checksums.append(tag_hash.hexdigest()) + + tags_hash_value = sorted(tags_checksums) + + _hash = new_hash_values( + obj.location_x, + obj.location_y, + obj.size_x, + obj.size_y, + obj.zoom, + obj.map_type.value, + tags_hash_value, + algorithm="sha256" + ) + + expected_checksum = _hash.hexdigest() + obj_checksum = obj.checksum_value + assert obj_checksum == expected_checksum diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py new file mode 100644 index 0000000..19246a2 --- /dev/null +++ b/tests/unit/test_utils.py @@ -0,0 +1,45 @@ +import hashlib +import pytest +from vigobusapi.utils import new_hash_values, update_hash_values + + +@pytest.mark.parametrize("algorithm", ["md5", "sha256"]) +def test_new_hash_values(algorithm: str): + hashes = { + "md5": hashlib.md5, + "sha256": hashlib.sha256 + } + data = ["a string", 1, True, None, [1, "other string", False, None]] + + _hash = hashes[algorithm]() + for chunk in data: + _hash.update(str(chunk).encode()) + + expected_hexdigest = _hash.hexdigest() + result_hexdigest = new_hash_values(*data, algorithm=algorithm).hexdigest() + + assert result_hexdigest == expected_hexdigest + + +@pytest.mark.parametrize("algorithm", ["md5", "sha256"]) +def test_update_hash_values(algorithm: str): + hashes = { + "md5": hashlib.md5, + "sha256": hashlib.sha256 + } + full_hash = hashes[algorithm]() + original_hash = hashes[algorithm]() + + original_data = ["initial string", 0.0] + for chunk in original_data: + original_hash.update(str(chunk).encode()) + full_hash.update(str(chunk).encode()) + + new_data = ["a string", 1, True, None, [1, "other string", False, None]] + for chunk in new_data: + full_hash.update(str(chunk).encode()) + + expected_hexdigest = full_hash.hexdigest() + result_hexdigest = update_hash_values(*new_data, _hash=original_hash).hexdigest() + + assert result_hexdigest == expected_hexdigest diff --git a/vigobusapi/app.py b/vigobusapi/app.py index 5f2ee69..e137ad4 100644 --- a/vigobusapi/app.py +++ b/vigobusapi/app.py @@ -1,19 +1,15 @@ """APP -Module with all the available endpoints and the FastAPI initialization. +FastAPI initialization. """ -# # Native # # -from typing import Optional, Set - # # Installed # # import uvicorn -from fastapi import FastAPI, Response, Query, HTTPException +from fastapi import FastAPI # # Project # # -from vigobusapi.entities import Stop, Stops, BusesResponse +from vigobusapi.routes import setup_routes from vigobusapi.request_handler import request_handler from vigobusapi.settings import settings -from vigobusapi.vigobus_getters import get_stop, get_stops, get_buses, search_stops from vigobusapi.services import MongoDB from vigobusapi.logger import logger @@ -23,6 +19,7 @@ title=settings.api_name ) app.middleware("http")(request_handler) +setup_routes(app) @app.on_event("startup") @@ -32,58 +29,6 @@ async def app_setup(): await MongoDB.initialize() -@app.get("/status") -async def endpoint_status(): - return Response( - content="OK", - media_type="text/plain", - status_code=200 - ) - - -@app.get("/stops", response_model=Stops) -async def endpoint_get_stops( - stop_name: Optional[str] = Query(None), - limit: Optional[int] = Query(None), - stops_ids: Optional[Set[int]] = Query(None, alias="stop_id") -): - """Endpoint to search/list stops by different filters. Only one filter can be used. - Returns 400 if no filters given. - The filters available are: - - - stop_name: search by a single string in stop names. "limit" can be used for limiting results size. - - stop_id: repeatable param for getting multiple stops by id on a single request. Not found errors are ignored. - """ - with logger.contextualize(**locals()): - if stop_name is not None: - stops = await search_stops(stop_name=stop_name, limit=limit) - elif stops_ids: - stops = await get_stops(stops_ids) - else: - raise HTTPException(status_code=400, detail="No filters given") - return [stop.dict() for stop in stops] - - -@app.get("/stop/{stop_id}", response_model=Stop) -async def endpoint_get_stop(stop_id: int): - """Endpoint to get information of a Stop giving the Stop ID - """ - with logger.contextualize(**locals()): - stop = await get_stop(stop_id) - return stop.dict() - - -@app.get("/buses/{stop_id}", response_model=BusesResponse) -@app.get("/stop/{stop_id}/buses", response_model=BusesResponse) -async def endpoint_get_buses(stop_id: int, get_all_buses: bool = False): - """Endpoint to get a list of Buses coming to a Stop giving the Stop ID. - By default the shortest available list of buses is returned, unless 'get_all_buses' param is True - """ - with logger.contextualize(**locals()): - buses_result = await get_buses(stop_id, get_all_buses=get_all_buses) - return buses_result.dict() - - def run(): """Run the API using Uvicorn """ diff --git a/vigobusapi/entities.py b/vigobusapi/entities.py index a8947e7..84a9165 100644 --- a/vigobusapi/entities.py +++ b/vigobusapi/entities.py @@ -13,7 +13,7 @@ # # Package # # from vigobusapi.exceptions import StopNotExist -__all__ = ("Stop", "Stops", "OptionalStop", "StopOrNotExist", "Bus", "Buses", "BusesResponse") +__all__ = ("BaseMongoModel", "Stop", "Stops", "OptionalStop", "StopOrNotExist", "Bus", "Buses", "BusesResponse") class BaseModel(pydantic.BaseModel): @@ -24,6 +24,28 @@ def dict(self, *args, skip_none=True, **kwargs): return {k: v for k, v in d.items() if (not skip_none or v is not None)} +class BaseMongoModel(pydantic.BaseModel): + # TODO Use in Stop models + class Config(pydantic.BaseModel.Config): + id_field: Optional[str] = None + + def to_mongo(self, **kwargs) -> dict: + d = self.dict(**kwargs) + if self.Config.id_field is None: + return d + + d["_id"] = d.pop(self.Config.id_field) + return d + + @classmethod + def from_mongo(cls, d: dict): + if cls.Config.id_field is not None: + d[cls.Config.id_field] = d.pop("_id") + + # noinspection PyArgumentList + return cls(**d) + + class Bus(BaseModel): line: str route: str @@ -77,6 +99,10 @@ def get_mongo_dict(self): d.pop("source") return d + @property + def has_location(self): + return self.lat is not None and self.lon is not None + OptionalStop = Optional[Stop] StopOrNotExist = Union[Stop, StopNotExist] diff --git a/vigobusapi/routes/__init__.py b/vigobusapi/routes/__init__.py new file mode 100644 index 0000000..1253e3e --- /dev/null +++ b/vigobusapi/routes/__init__.py @@ -0,0 +1 @@ +from .bootstrap import * diff --git a/vigobusapi/routes/_api.py b/vigobusapi/routes/_api.py new file mode 100644 index 0000000..f5661c6 --- /dev/null +++ b/vigobusapi/routes/_api.py @@ -0,0 +1,18 @@ +"""API "General" Routes +""" + +# # Installed # # +from fastapi import APIRouter, Response + +__all__ = ("router",) + +router = APIRouter() + + +@router.get("/status") +async def endpoint_status(): + return Response( + content="OK", + media_type="text/plain", + status_code=200 + ) diff --git a/vigobusapi/routes/_maps.py b/vigobusapi/routes/_maps.py new file mode 100644 index 0000000..34ada5f --- /dev/null +++ b/vigobusapi/routes/_maps.py @@ -0,0 +1,206 @@ +"""MAPS Routes +""" + +# # Native # # +import io +import json +from dataclasses import dataclass +from typing import * + +# # Installed # # +from fastapi import APIRouter, Query, Depends, HTTPException, Response +from starlette.responses import StreamingResponse, PlainTextResponse + +# # Project # # +from vigobusapi.settings import google_maps_settings +from vigobusapi.vigobus_getters import get_stop, get_stops +from vigobusapi.services.google_maps import (GoogleMapRequest, GoogleStreetviewRequest, + get_map, get_photo, get_cached_metadata, update_cached_metadata, CachedMap) + +__all__ = ("router",) + +router = APIRouter() + + +@dataclass +class MapQueryParams: + size_x: int = google_maps_settings.stop_map_default_size_x + size_y: int = google_maps_settings.stop_map_default_size_y + zoom: int = google_maps_settings.stop_map_default_zoom + map_type: GoogleMapRequest.MapTypes = google_maps_settings.stop_map_default_type + + +@dataclass +class PhotoQueryParams: + size_x: int = google_maps_settings.stop_photo_default_size_x + size_y: int = google_maps_settings.stop_photo_default_size_y + + +@dataclass +class MapCacheSetParams: + id: str + telegram_file_id: str + + +StopsTagsRelation = Dict[int, GoogleMapRequest.Tag] + + +def _format_map_response( + image: Optional[bytes] = None, + telegram_file_id: Optional[str] = None, + cache_metadata: Optional[CachedMap] = None, + stops_tags_relation: Optional[StopsTagsRelation] = None +) -> Response: + """Generate a Response for a fetched image or cached item, based on the given arguments.""" + headers = dict() + if cache_metadata is not None: + headers["X-Maps-Cache-ID"] = cache_metadata.id + if stops_tags_relation is not None: + headers["X-Maps-Stops-Tags"] = json.dumps(stops_tags_relation) + + if image is not None: + return StreamingResponse(content=io.BytesIO(image), media_type="image/png", headers=headers) + if telegram_file_id is not None: + return PlainTextResponse(content=telegram_file_id, headers=headers) + raise Exception("No file data or cache given for generating Response") + + +@router.get("/stop/{stop_id}/map") +@router.get("/stops/{stop_id}/map") +async def endpoint_get_stop_map( + stop_id: int, + map_params: MapQueryParams = Depends(), + get_telegram_file_id: bool = False +): + """Get a picture of a map with the stop location marked on it. + + If get_telegram_file_id=True, fetch the cached Telegram File ID and return it as plaintext. + If not available, the picture (cached or live) is returned. + + A header "X-Maps-Cache-ID" is returned, with the Cache ID of the map with the queried parameters. + This can be used for updating the cache. + """ + stop = await get_stop(stop_id) + if not stop.has_location: + raise HTTPException(status_code=409, detail="The stop does not have information about the location") + + map_request = GoogleMapRequest( + location_x=stop.lat, + location_y=stop.lon, + size_x=map_params.size_x, + size_y=map_params.size_y, + zoom=map_params.zoom, + map_type=map_params.map_type, + tags=[GoogleMapRequest.Tag(location_x=stop.lat, location_y=stop.lon)] + ) + + if get_telegram_file_id: + cache_metadata = await get_cached_metadata(map_request, fetch_image=False) + if cache_metadata and cache_metadata.telegram_file_id: + return _format_map_response( + telegram_file_id=cache_metadata.telegram_file_id, cache_metadata=cache_metadata + ) + + image, cache_metadata = await get_map(map_request) + return _format_map_response(image=image, cache_metadata=cache_metadata) + + +@router.get("/stops/map") +async def endpoint_get_stops_map( + stops_ids: Set[int] = Query(None, alias="stop_id", + min_items=1, max_items=len(GoogleMapRequest.Tag.get_allowed_labels())), + map_params: MapQueryParams = Depends(), + get_telegram_file_id: bool = False +): + """Get a picture of a map with the locations of the given stops marked on it. + + If get_telegram_file_id=True, fetch the cached Telegram File ID and return it as plaintext. + If not available, the picture (cached or live) is returned. + + Non existing stops, or those without location available, are ignored, + but if none of the given stops are valid, returns 404. + + A header "X-Maps-Stops-Tags" is returned, being a JSON associating the Stops IDs with the tag label on the map, + with the format: {"" : ""} + + A header "X-Maps-Cache-ID" is returned, with the Cache ID of the map with the queried parameters. + This can be used for updating the cache. + """ + stops = await get_stops(stops_ids) + stops = [stop for stop in stops if stop.has_location] + if not stops: + raise HTTPException(status_code=404, detail="None of the stops exist or have location available") + + stops_tags: List[GoogleMapRequest.Tag] = list() + stops_tags_relation: StopsTagsRelation = dict() + for i, stop in enumerate(stops): + tag_label = GoogleMapRequest.Tag.get_allowed_labels()[i] + tag = GoogleMapRequest.Tag(label=tag_label, location_x=stop.lat, location_y=stop.lon) + stops_tags.append(tag) + stops_tags_relation[stop.stop_id] = tag_label + + map_request = GoogleMapRequest( + size_x=map_params.size_x, + size_y=map_params.size_y, + zoom=map_params.zoom, + map_type=map_params.map_type, + tags=stops_tags + ) + + if get_telegram_file_id: + cache_metadata = await get_cached_metadata(map_request, fetch_image=False) + if cache_metadata and cache_metadata.telegram_file_id: + return _format_map_response( + telegram_file_id=cache_metadata.telegram_file_id, + cache_metadata=cache_metadata, + stops_tags_relation=stops_tags_relation + ) + + image, cache_metadata = await get_map(map_request) + return _format_map_response(image=image, cache_metadata=cache_metadata, stops_tags_relation=stops_tags_relation) + + +@router.get("/stop/{stop_id}/photo") +@router.get("/stops/{stop_id}/photo") +async def endpoint_get_stop_photo( + stop_id: int, + photo_params: PhotoQueryParams = Depends(), + get_telegram_file_id: bool = False +): + """Get a real street photo of the Stop location. + + If get_telegram_file_id=True, fetch the cached Telegram File ID and return it as plaintext. + If not available, the picture (cached or live) is returned. + + A header "X-Maps-Cache-ID" is returned, with the Cache ID of the map with the queried parameters. + This can be used for updating the cache. + """ + stop = await get_stop(stop_id) + if not stop.has_location: + raise HTTPException(status_code=409, detail="The stop does not have information about the location") + + photo_request = GoogleStreetviewRequest( + location_x=stop.lat, + location_y=stop.lon, + size_x=photo_params.size_x, + size_y=photo_params.size_y + ) + + if get_telegram_file_id: + cache_metadata = await get_cached_metadata(photo_request, fetch_image=False) + if cache_metadata and cache_metadata.telegram_file_id: + return _format_map_response(telegram_file_id=cache_metadata.telegram_file_id, cache_metadata=cache_metadata) + + image, cache_metadata = await get_photo(photo_request) + if not image: + raise HTTPException(status_code=404, detail="No StreetView photo available for the stop location") + return _format_map_response(image=image, cache_metadata=cache_metadata) + + +@router.put("/cache/maps", status_code=204) +async def update_maps_cache(cache_params: MapCacheSetParams = Depends()): + """Update fields from a cached map or photo. Can be used for setting the Telegram File ID of a persisted photo.""" + updated = await update_cached_metadata(cache_id=cache_params.id, telegram_file_id=cache_params.telegram_file_id) + if not updated: + raise HTTPException(status_code=404, detail=f"No cache found with id {cache_params.id}") + return Response(status_code=204) diff --git a/vigobusapi/routes/_stops_buses.py b/vigobusapi/routes/_stops_buses.py new file mode 100644 index 0000000..d88dc0b --- /dev/null +++ b/vigobusapi/routes/_stops_buses.py @@ -0,0 +1,61 @@ +"""STOPS & BUSES Routes +""" + +# # Native # # +from typing import Optional, Set + +# # Installed # # +from fastapi import APIRouter, Query, HTTPException + +# # Project # # +from vigobusapi.entities import Stop, Stops, BusesResponse +from vigobusapi.vigobus_getters import get_stop, get_stops, get_buses, search_stops +from vigobusapi.logger import logger + +__all__ = ("router",) + +router = APIRouter() + + +@router.get("/stops", response_model=Stops) +async def endpoint_get_stops( + stop_name: Optional[str] = Query(None), + limit: Optional[int] = Query(None), + stops_ids: Optional[Set[int]] = Query(None, alias="stop_id") +): + """Endpoint to search/list stops by different filters. Only one filter can be used. + Returns 400 if no filters given. + The filters available are: + + - stop_name: search by a single string in stop names. "limit" can be used for limiting results size. + - stop_id: repeatable param for getting multiple stops by id on a single request. Not found errors are ignored. + """ + with logger.contextualize(**locals()): + if stop_name is not None: + stops = await search_stops(stop_name=stop_name, limit=limit) + elif stops_ids: + stops = await get_stops(stops_ids) + else: + raise HTTPException(status_code=400, detail="No filters given") + return [stop.dict() for stop in stops] + + +@router.get("/stop/{stop_id}", response_model=Stop) +@router.get("/stops/{stop_id}", response_model=Stop) +async def endpoint_get_stop(stop_id: int): + """Endpoint to get information of a Stop giving the Stop ID + """ + with logger.contextualize(**locals()): + stop = await get_stop(stop_id) + return stop.dict() + + +@router.get("/buses/{stop_id}", response_model=BusesResponse) +@router.get("/stops/{stop_id}/buses", response_model=BusesResponse) +async def endpoint_get_buses(stop_id: int, get_all_buses: bool = False): + """Endpoint to get a list of Buses coming to a Stop giving the Stop ID. + By default the shortest available list of buses is returned, unless 'get_all_buses' param is True + """ + with logger.contextualize(**locals()): + buses_result = await get_buses(stop_id, get_all_buses=get_all_buses) + return buses_result.dict() \ No newline at end of file diff --git a/vigobusapi/routes/bootstrap.py b/vigobusapi/routes/bootstrap.py new file mode 100644 index 0000000..d4d0ec0 --- /dev/null +++ b/vigobusapi/routes/bootstrap.py @@ -0,0 +1,18 @@ +"""ROUTES BOOTSTRAP +Entrypoint for API routes setup +""" + +from fastapi import FastAPI + +from ._api import router as general_router +from ._stops_buses import router as stops_buses_router +from ._maps import router as maps_router + + +__all__ = ("setup_routes",) + + +def setup_routes(app: FastAPI): + app.include_router(general_router) + app.include_router(maps_router) + app.include_router(stops_buses_router) diff --git a/vigobusapi/services/google_maps/__init__.py b/vigobusapi/services/google_maps/__init__.py new file mode 100644 index 0000000..61b1616 --- /dev/null +++ b/vigobusapi/services/google_maps/__init__.py @@ -0,0 +1,8 @@ +"""GOOGLE MAPS +Classes and functions for acquiring static maps and photos using Google Maps & Street View Static APIs +""" + +from ._entities import * +from ._getter_maps import * +from ._getter_streetview import * +from ._cache import * diff --git a/vigobusapi/services/google_maps/_cache.py b/vigobusapi/services/google_maps/_cache.py new file mode 100644 index 0000000..196dff8 --- /dev/null +++ b/vigobusapi/services/google_maps/_cache.py @@ -0,0 +1,192 @@ +"""GOOGLE MAPS - CACHE +Cache utils for reading/writing persistence of cached Maps & StreetView pictures +""" + +# # Native # # +import enum +import datetime +import asyncio +from typing import Union, Optional + +# # Installed # # +from pymongo.results import UpdateResult + +# # Project # # +from vigobusapi.services.mongo import MongoDB +from vigobusapi.entities import BaseMongoModel +from vigobusapi.utils import get_datetime, get_datetime_plus_seconds, new_hash_values, ChecksumableClass, without +from vigobusapi.settings import settings +from vigobusapi.logger import logger +from ._entities import * + +__all__ = ("get_cached_metadata", "save_cached_metadata", "update_cached_metadata", + "MapRequestModels", "MapVendors", "MapTypes", "CachedMap") + +MapRequestModels = Union[GoogleMapRequest, GoogleStreetviewRequest] + + +class MapVendors(str, enum.Enum): + # Currently only using Google Maps API, but this field provides future-proof when more vendors are featured + google_maps = "google" + + +class MapTypes(str, enum.Enum): + map = "map" + photo = "streetview" + + +MAP_REQUESTS_TYPES = { + GoogleMapRequest: MapTypes.map, + GoogleStreetviewRequest: MapTypes.photo +} + + +class CachedMap(BaseMongoModel, ChecksumableClass): + """Representation of the Mongo document used for caching different types of map pictures.""" + + id: str = "" + """Document id, generated as: "{vendor}:{key}.""" + key: str + """Checksum generated from the "data" object, used as cache key. + Not used as document _id because multiple cached images (from different vendors) would have the same checksum.""" + vendor: MapVendors + """Remote source API/vendor the map was fetched from.""" + type: MapTypes + """Type of map picture.""" + data: MapRequestModels + """Original Request object used for fetching the object.""" + saved: datetime.datetime + """When this document was saved.""" + expiration: datetime.datetime + """When the document shall expire.""" + image: Optional[bytes] + """Image saved as-is. Optional because metadata can be fetched without the image, but should always be persisted.""" + telegram_file_id: Optional[str] + """File ID in Telegram, set after sending the picture via Telegram.""" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.id = self.checksum_value + + def metadata_dict(self, **kwargs) -> dict: + """Fetch the dict(), without the "image" field.""" + excludes = kwargs.pop("exclude", set()) + excludes.add("image") + kwargs["exclude"] = excludes + return self.dict(**kwargs) + + def metadata_json(self, **kwargs) -> str: + """Fetch the json(), without the "image" field.""" + excludes = kwargs.pop("exclude", set()) + excludes.add("image") + kwargs["exclude"] = excludes + return self.json(**kwargs) + + @property + def checksum_hash(self): + return new_hash_values( + self.key, + self.vendor.value, + self.type.value, + algorithm="sha256" + ) + + class Config(BaseMongoModel.Config): + id_field = "id" + + +def _get_mongo_filter_from_request(request: MapRequestModels): + return dict( + key=request.checksum_value, + type=MAP_REQUESTS_TYPES[type(request)] + ) + + +async def get_cached_metadata(request: MapRequestModels, fetch_image: bool) -> Optional[CachedMap]: + request_checksum = request.checksum_value + with logger.contextualize(map_request_checksum=request_checksum): + logger.bind(fetch_image=fetch_image).debug("Searching map cached metadata...") + + query_filter = _get_mongo_filter_from_request(request) + query_fields = dict(_id=False) + if not fetch_image: + query_fields["image"] = False + + # NOTE Currently only the first found document is used. + # If more API sources are included in the future, should support choosing vendor or having priorities. + document = await MongoDB.get_mongo().get_cache_maps_collection().find_one( + filter=query_filter, + projection=query_fields + ) + + if not document: + logger.debug("Map metadata not found in Mongo cache") + return None + + parsed_metadata = CachedMap(**document) + logger.bind(cached_metadata_document=without(document, "image"))\ + .debug("Read map cached metadata document from Mongo") + return parsed_metadata + + +async def update_cached_metadata(cache_id: str, telegram_file_id: str) -> bool: + """Update certain fields from a cached map metadata document. Currently only supports updating the Telegram File ID. + The identifier used is the document id. Returns True/False depending on if the document was found.""" + with logger.contextualize(cache_id=cache_id): + query_filter = dict(_id=cache_id) + query_update = { + "$set": { + "telegram_file_id": telegram_file_id + } + } + + logger.bind(cache_document_update=query_update).debug("Updating map cached metadata...") + r: dict = await MongoDB.get_mongo().get_cache_maps_collection().find_one_and_update( + filter=query_filter, + update=query_update + ) + + if r is None: + logger.debug("No map cache document found for update") + return False + + logger.debug("Updated map cache document") + return True + + +async def save_cached_metadata(request: MapRequestModels, image: bytes, background: bool) -> CachedMap: + """Save the given request object and image in cache. + Return the CachedMap object. If background=True, return instantly and persist in background.""" + metadata = CachedMap( + key=request.checksum_value, + vendor=MapVendors.google_maps.value, + type=MAP_REQUESTS_TYPES[type(request)], + data=request, + saved=get_datetime(), + expiration=get_datetime_plus_seconds(settings.mongo_cache_maps_ttl), + image=image + ) + + with logger.contextualize(map_cache_id=metadata.id): + async def __save(): + logger.debug("Saving map cache in MongoDB...") + r: UpdateResult = await MongoDB.get_mongo().get_cache_maps_collection().replace_one( + filter=dict(_id=metadata.id), + replacement=metadata.to_mongo(), + upsert=True + ) + + if r.upserted_id: + logger.debug("Inserted new map cache document") + elif r.modified_count: + logger.debug("Replaced existing map cache document") + else: + logger.error("No modified/inserted documents inserted in MongoDB") + + if background: + logger.debug("Map cache will be saved as background task") + asyncio.create_task(__save()) + else: + await __save() + + return metadata diff --git a/vigobusapi/services/google_maps/_entities.py b/vigobusapi/services/google_maps/_entities.py new file mode 100644 index 0000000..166bdf8 --- /dev/null +++ b/vigobusapi/services/google_maps/_entities.py @@ -0,0 +1,121 @@ +"""GOOGLE MAPS - ENTITIES +Classes used to define Google Maps/StreetView API requests +""" + +# # Native # # +import string +from enum import Enum +from typing import * + +# # Installed # # +from pydantic import BaseModel + +# # Project # # +from vigobusapi.utils import ChecksumableClass, new_hash_values, update_hash_values + +__all__ = ("GoogleMapRequest", "GoogleStreetviewRequest") + + +class _GoogleMapsBaseRequest(BaseModel, ChecksumableClass): + location_x: float + location_y: float + size_x: int + size_y: int + + @property + def location_str(self) -> Optional[str]: + if self.location_x is None or self.location_y is None: + return None + return f"{self.location_x},{self.location_y}" + + @property + def size_str(self): + return f"{self.size_x}x{self.size_y}" + + @property + def checksum_hash(self): + """Returns a SHA256 checksum of all the fields in the request""" + return new_hash_values( + self.location_x, + self.location_y, + self.size_x, + self.size_y, + algorithm="sha256" + ) + + +class GoogleMapRequest(_GoogleMapsBaseRequest): + + # Embed classes # + + class Tag(BaseModel, ChecksumableClass): + __ALLOWED_LABELS = [*[str(i) for i in range(1, 10)], *[c for c in string.ascii_uppercase]] + + label: Optional[str] = None # TODO constrain values accepted (avoid enum?) + location_x: float + location_y: float + + @classmethod + def get_allowed_labels(cls): + """Get a list with the available labels for Tags, corresponding to numbers 0~9 and characters A~Z.""" + return cls.__ALLOWED_LABELS + + @property + def location_str(self): + return f"{self.location_x},{self.location_y}" + + @property + def checksum_hash(self): + return new_hash_values( + self.label, + self.location_x, + self.location_y, + algorithm="md5" + ) + + class MapTypes(str, Enum): + """Available map types. + + References: + https://developers.google.com/maps/documentation/maps-static/start#MapTypes + """ + roadmap = "roadmap" + """specifies a standard roadmap image, as is normally shown on the Google Maps website""" + satellite = "satellite" + """specifies a satellite image""" + terrain = "terrain" + """specifies a physical relief map image, showing terrain and vegetation""" + hybrid = "hybrid" + """specifies a hybrid of the satellite and roadmap image, + showing a transparent layer of major streets and place names on the satellite image""" + + # Properties # + + @property + def checksum_hash(self): + _hash = super().checksum_hash + + if self.tags: + sorted_tags_checksums = sorted(tag.checksum_value for tag in self.tags) + else: + sorted_tags_checksums = "NoTags" + + return update_hash_values( + self.zoom, + self.map_type.value, + sorted_tags_checksums, + _hash=_hash + ) + + # Class Attributes # + + location_x: Optional[float] = None + location_y: Optional[float] = None + tags: Optional[List[Tag]] = None + zoom: int + """https://developers.google.com/maps/documentation/maps-static/start#Zoomlevels""" + map_type: MapTypes + + +class GoogleStreetviewRequest(_GoogleMapsBaseRequest): + pass diff --git a/vigobusapi/services/google_maps/_getter_maps.py b/vigobusapi/services/google_maps/_getter_maps.py new file mode 100644 index 0000000..7e45ea6 --- /dev/null +++ b/vigobusapi/services/google_maps/_getter_maps.py @@ -0,0 +1,81 @@ +"""GOOGLE MAPS - GETTER MAPS +Functions for acquiring a picture of a Map +""" + +# # Native # # +import asyncio +from typing import * + +# # Project # # +from vigobusapi.settings import google_maps_settings as settings +from vigobusapi.logger import logger +from ._requester import google_maps_request, ListOfTuples +from ._entities import GoogleMapRequest +from ._cache import save_cached_metadata, get_cached_metadata, CachedMap + +__all__ = ("get_map", "get_map_from_api") + +GOOGLE_MAPS_STATIC_API_URL = "https://maps.googleapis.com/maps/api/staticmap" + + +def _get_map_tags_params(request_tags: List[GoogleMapRequest.Tag]) -> ListOfTuples: + params = list() + for tag in request_tags: + tag_param_values = [tag.location_str] # Location always at the end + + if tag.label: + tag_param_values.insert(0, "label:" + tag.label) + + tag_param = "|".join(tag_param_values) + params.append(("markers", tag_param)) + + return params + + +def _get_map_params(request: GoogleMapRequest) -> ListOfTuples: + params = [ + ("size", request.size_str), + ("maptype", request.map_type.value), + ("language", settings.language), + ("format", "png8"), + ] + + location_str = request.location_str + if location_str: + params.append(("center", location_str)) + params.append(("zoom", str(request.zoom))) + + if request.tags: + params.extend(_get_map_tags_params(request.tags)) + + return params + + +async def get_map_from_api(request: GoogleMapRequest) -> Tuple[bytes, CachedMap]: + """Get a static Map picture from the Google Maps Static API. + Return the acquired PNG picture as bytes, and the CachedMap object. + The fetched picture is persisted on cache, running a fire & forget background task. + + References: + https://developers.google.com/maps/documentation/maps-static/overview + https://developers.google.com/maps/documentation/maps-static/start + """ + logger.bind(map_request=request.dict()).debug("Requesting Google Static Map picture...") + params = _get_map_params(request) + + image = (await google_maps_request(url=GOOGLE_MAPS_STATIC_API_URL, params=params)).content + logger.debug("Map acquired from Google Static Maps API") + + cache_metadata = await save_cached_metadata(request=request, image=image, background=True) + return image, cache_metadata + + +async def get_map(request: GoogleMapRequest, read_cache_first: bool = True) -> Tuple[bytes, CachedMap]: + """Get a static Map picture from cache (if read_cache_first=True) or the Google Maps Static API. + Return the acquired PNG picture as bytes, and the CachedMap object.""" + if read_cache_first: + cached_metadata = await get_cached_metadata(request, fetch_image=True) + if cached_metadata: + return cached_metadata.image, cached_metadata + + return await get_map_from_api(request) diff --git a/vigobusapi/services/google_maps/_getter_streetview.py b/vigobusapi/services/google_maps/_getter_streetview.py new file mode 100644 index 0000000..1b77572 --- /dev/null +++ b/vigobusapi/services/google_maps/_getter_streetview.py @@ -0,0 +1,68 @@ +"""GOOGLE MAPS - GETTER STREETVIEW +Functions for acquiring a photo of a location +""" + +# # Native # # +import asyncio +from typing import * + +# # Project # # +from vigobusapi.logger import logger +from ._entities import GoogleStreetviewRequest +from ._requester import google_maps_request, ListOfTuples +from ._cache import save_cached_metadata, get_cached_metadata, CachedMap + +__all__ = ("get_photo",) + +GOOGLE_MAPS_STATIC_API_URL = "https://maps.googleapis.com/maps/api/staticmap" +GOOGLE_STREETVIEW_STATIC_API_URL = "https://maps.googleapis.com/maps/api/streetview" + + +def _get_photo_params(request: GoogleStreetviewRequest) -> ListOfTuples: + params = [ + ("location", request.location_str), + ("size", request.size_str), + ("return_error_code", "true"), + ("source", "outdoor") + ] + return params + + +async def get_photo_from_api(request: GoogleStreetviewRequest) -> Tuple[Optional[bytes], Optional[CachedMap]]: + """Get a static StreetView picture from the Google StreetView Static API. + Return the acquired PNG picture as bytes, and the CachedMap object. + If the requested location does not have an available picture, returns None. + The fetched picture is persisted on cache, running a fire & forget background task. + + References: + https://developers.google.com/maps/documentation/streetview/overview + """ + logger.bind(streetview_request=request.dict()).debug("Requesting Google Static StreetView picture...") + # TODO Support specific parameters for tuning camera, if required + + params = _get_photo_params(request) + response = await google_maps_request(GOOGLE_STREETVIEW_STATIC_API_URL, params=params, expect_http_error=True) + if response.status_code == 404: + logger.debug("No StreetView picture available for the request") + return None, None + + response.raise_for_status() + image = response.content + logger.debug("Photo acquired from Google StreetView Static API") + + cache_metadata = await save_cached_metadata(request=request, image=image, background=True) + return image, cache_metadata + + +async def get_photo( + request: GoogleStreetviewRequest, read_cache_first: bool = True +) -> Tuple[Optional[bytes], Optional[CachedMap]]: + """Get a static StreetView picture from cache (if read_cache_first=True) or the Google StreetView Static API. + Return the acquired PNG picture as bytes, and the CachedMap object. + If the requested location does not have an available picture, returns (None, None).""" + if read_cache_first: + cached_metadata = await get_cached_metadata(request, fetch_image=True) + if cached_metadata: + return cached_metadata.image, cached_metadata + + return await get_photo_from_api(request) diff --git a/vigobusapi/services/google_maps/_requester.py b/vigobusapi/services/google_maps/_requester.py new file mode 100644 index 0000000..7910d88 --- /dev/null +++ b/vigobusapi/services/google_maps/_requester.py @@ -0,0 +1,38 @@ +"""GOOGLE MAPS - REQUESTER +Functions for requesting the Google Maps/StreetView APIs +""" + +# # Native # # +from typing import * + +# # Project # # +from vigobusapi.settings import google_maps_settings as settings +from vigobusapi.services.http_requester import http_request, ListOfTuples + +__all__ = ("google_maps_request", "ListOfTuples") + + +async def google_maps_request(url: str, params: Union[dict, ListOfTuples], expect_http_error: bool = False): + """HTTP requester for Google Maps API calls, automatically including the configured API key. + Raises exception if the API Key is not configured. + + :param url: URL for the Google API, WITHOUT query parameters + :param params: query parameters + :param expect_http_error: if True, raise_for_status=False and not_retry_400_errors=True + """ + if not settings.enabled: + raise Exception("Google Maps API Key not set in settings") + + if isinstance(params, list): + params.append(("key", settings.api_key)) + else: + params = dict(**params, key=settings.api_key) + + return await http_request( + url=url, + method="GET", + params=params, + retries=1, + raise_for_status=not expect_http_error, + not_retry_400_errors=expect_http_error + ) diff --git a/vigobusapi/services/google_maps/test_cache_models.py b/vigobusapi/services/google_maps/test_cache_models.py new file mode 100644 index 0000000..474714b --- /dev/null +++ b/vigobusapi/services/google_maps/test_cache_models.py @@ -0,0 +1,25 @@ +from vigobusapi.utils import get_datetime, new_hash_values +from ._entities import GoogleStreetviewRequest +from ._cache import CachedMap, MapVendors, MapTypes + + +def test_cached_map_generates_id(): + request = GoogleStreetviewRequest(location_x=1, location_y=1, size_x=1, size_y=1) + cache = CachedMap( + key=request.checksum_value, + vendor=MapVendors.google_maps, + type=MapTypes.photo, + data=request, + saved=get_datetime() + ) + + expected_id = new_hash_values( + cache.key, + cache.vendor.value, + cache.type.value, + algorithm="sha256" + ).hexdigest() + + assert cache.id == expected_id + assert cache.dict()["id"] == expected_id + assert cache.to_mongo()["_id"] == expected_id diff --git a/vigobusapi/services/http_requester.py b/vigobusapi/services/http_requester.py index 3f32ce9..822a525 100644 --- a/vigobusapi/services/http_requester.py +++ b/vigobusapi/services/http_requester.py @@ -13,13 +13,15 @@ from vigobusapi.settings import settings from vigobusapi.logger import logger -__all__ = ("http_request",) +__all__ = ("http_request", "ListOfTuples") + +ListOfTuples = List[Tuple[str, str]] async def http_request( url: str, method: str = "GET", - params: Optional[dict] = None, + params: Optional[Union[dict, ListOfTuples]] = None, body: Optional[Union[dict, str]] = None, headers: Optional[dict] = None, timeout: float = settings.http_timeout, @@ -31,16 +33,17 @@ async def http_request( :param url: URL to request :param method: HTTP method (default=GET) - :param params: URL query params as dict (default=None) + :param params: URL query params as dict or list of tuples (default=None) :param body: request body, usually a dict or string (default=None) :param headers: request headers as dict (default=None) :param timeout: timeout for each request retry in seconds (default=from settings) - :param retries: how many times to retry the request if it fails (default=from settings) + :param retries: how many times to retry the request if it fails (at least 1) (default=from settings) :param raise_for_status: if True, raise HTTPError if response is not successful (default=True) - :param not_retry_400_errors: if True, do not retry requests failed with a ~400 status code (default=True) + :param not_retry_400_errors: if True, do not retry requests failed with a 4xx status code (default=True) :return: the Response object :raises: requests_async.RequestTimeout | requests_async.RequestException """ + # TODO refactor "retries" arg, it is actually working as "tries" (requires at least 1) last_error = None last_status_code = None @@ -70,10 +73,19 @@ async def http_request( response_time = round(time.time() - start_time, 4) last_status_code = response.status_code + + # Log response + response_body = response.content + response_body_size = len(response_body) + if response_body_size > 500000 or b"\x00" in response_body: # 500kB + response_body = "binary or too large" + else: + response_body = response.text logger.bind( response_elapsed_time=response_time, response_status_code=last_status_code, - response_body=response.text + response_body=response_body, + response_body_size=response_body_size ).debug("Response received") if raise_for_status: diff --git a/vigobusapi/services/mongo.py b/vigobusapi/services/mongo.py index 2e3a070..3772743 100644 --- a/vigobusapi/services/mongo.py +++ b/vigobusapi/services/mongo.py @@ -1,4 +1,12 @@ +"""MONGO +MongoDB client class +""" + +# # Native # # +import asyncio + # # Installed # # +import pymongo.database from motor import motor_asyncio from pymongo import TEXT @@ -8,6 +16,17 @@ class MongoDB: + """Class containing the async MongoDB "Motor" client, and methods to acquire the database and collections used. + The class and Motor client are initialized and accessed in a Singleton way. + + First, the current class, the Motor client and other procedures for db/collection setup are initialized using the + initialize() classmethod, which must be called when the API server starts. + + Then, whenever the MongoDB must be accessed, the get_mongo() method can be used to fetch the MongoDB class instance + previously initialized, which lives as a class method. + + Database settings are acquired from the "settings" module and initialized class there. + """ _mongodb_instance = None # Singleton instance of the class _client = motor_asyncio.AsyncIOMotorClient @@ -16,16 +35,20 @@ def __init__(self): @property def client(self): + """MongoDB async "Motor" client.""" if self._client is None: raise Exception("Mongo client not initialized") return self._client - def get_database(self): + def get_database(self) -> pymongo.database.Database: return self.client[settings.mongo_stops_db] - def get_stops_collection(self): + def get_stops_collection(self) -> pymongo.database.Collection: return self.get_database()[settings.mongo_stops_collection] + def get_cache_maps_collection(self) -> pymongo.database.Collection: + return self.get_database()[settings.mongo_cache_maps_collection] + @classmethod async def initialize(cls): """Singleton initialization of MongoDB. Must run before the API server starts. Performs the following: @@ -41,19 +64,32 @@ async def initialize(cls): cls._mongodb_instance = mongo mongo._client = motor_asyncio.AsyncIOMotorClient(settings.mongo_uri) - # Create a Text Index on stop name, for search - # https://docs.mongodb.com/manual/core/index-text/#create-text-index - await mongo.get_stops_collection().create_index( - [("name", TEXT)], - background=True, - default_language="spanish" + logger.debug("Setting up MongoDB indexes...") + await asyncio.gather( + # Create a Text Index on stop name, for search + # https://docs.mongodb.com/manual/core/index-text/#create-text-index + mongo.get_stops_collection().create_index( + [("name", TEXT)], + background=True, + default_language="spanish" + ), + + # Create Expiration Index on cache collections (TTL with field indicating concrete expiration time) + # https://docs.mongodb.com/manual/core/index-ttl/ + mongo.get_cache_maps_collection().create_index( + "expiration", + name="expiration", + expireAfterSeconds=1, + background=True + ), ) logger.info("MongoDB initialized!") @classmethod def get_mongo(cls) -> "MongoDB": - """Singleton acquisition of MongoDB. The class should be initialized by calling the initialize() class method""" + """Singleton acquisition of MongoDB. + The class should be previously initialized by calling the initialize() class method.""" if cls._mongodb_instance is None: raise Exception("Mongo class not initialized") return cls._mongodb_instance diff --git a/vigobusapi/settings.py b/vigobusapi/settings.py index 950566c..60b1b4e 100644 --- a/vigobusapi/settings.py +++ b/vigobusapi/settings.py @@ -2,15 +2,27 @@ Declaration of the Settings class and instance that can be used to get any setting required """ +# # Native # # +import os +from typing import Optional + # # Installed # # -from pydantic import BaseSettings +import pydantic +import pytimeparse -__all__ = ("settings",) +__all__ = ("settings", "google_maps_settings") # TODO Split Settings by sections/groups in multiple classes +ENV_FILE = os.getenv("ENV_FILE", ".env") + + +class _BaseSettings(pydantic.BaseSettings): + class Config: + env_file = ENV_FILE + -class Settings(BaseSettings): +class Settings(_BaseSettings): endpoint_timeout: float = 30 http_timeout: float = 5 http_retries: int = 2 @@ -23,14 +35,44 @@ class Settings(BaseSettings): mongo_uri = "mongodb://localhost:27017" mongo_stops_db = "vigobusapi" mongo_stops_collection = "stops" + mongo_cache_maps_collection = "cache_maps" + mongo_cache_maps_ttl: int = "60 days" # seconds (can give human-readable time length, that will be converted) api_host = "0.0.0.0" api_port: int = 5000 api_name = "VigoBusAPI" api_log_level = "info" log_level = "info" - class Config: - env_file = ".env" + @pydantic.validator("mongo_cache_maps_ttl", pre=True) + def _parse_duration(cls, v): + """If the field is a non-digit string, parse a time length into seconds. + The value must be defined with the values accepted by the parsing library: + https://pypi.org/project/pytimeparse/""" + if isinstance(v, str) and not v.isdigit(): + parsed_v = pytimeparse.parse(v) + if parsed_v is None: + raise ValueError(f"Invalid duration string \"{v}\"") + return parsed_v + return v + + +class GoogleMapsSettings(_BaseSettings): + api_key: Optional[str] = None + stop_map_default_size_x: int = 1280 + stop_map_default_size_y: int = 720 + stop_map_default_zoom: int = 17 + stop_map_default_type: str = "roadmap" # TODO use enum (after refactoring to avoid circular dependency issue) + stop_photo_default_size_x: int = 2000 + stop_photo_default_size_y: int = 2000 + language: str = "es" + + @property + def enabled(self): + return bool(self.api_key) + + class Config(_BaseSettings.Config): + env_prefix = "GOOGLE_MAPS_" settings = Settings() +google_maps_settings = GoogleMapsSettings() diff --git a/vigobusapi/utils.py b/vigobusapi/utils.py new file mode 100644 index 0000000..3bec591 --- /dev/null +++ b/vigobusapi/utils.py @@ -0,0 +1,94 @@ +"""UTILS +Misc functions +""" + +# # Native # # +import abc +import hashlib +import json +import datetime +import base64 as _base64 +from typing import List, Dict, Union, Any + +# # Installed # # +import pydantic +from pydantic.json import pydantic_encoder + +__all__ = ( + "ChecksumableClass", "new_hash_values", "update_hash_values", "base64_encode", "json_encode_object", + "get_datetime", "get_datetime_plus_seconds", "without" +) + + +class ChecksumableClass(abc.ABC): + """Abstract class where inheritors can use methods for acquiring a checksum/hash, based on fields from the class.""" + + @property + @abc.abstractmethod + def checksum_hash(self): + """Returns a SHA256 checksum of all the fields in the request. + As an abstract method, must be completed returning a new hash object, from the hashlib library. The hash + shall be updated with the values from the object that will be used for calculating checksum, on a given order. + The methods new_hash_values() and update_hash_values() from utils module may be used for this purpose.""" + pass + + @property + def checksum_value(self) -> str: + """Returns the str value (hexdigest) from the checksum_hash output.""" + return self.checksum_hash.hexdigest() + + +def update_hash_values(*args, _hash): + """Update the given hash_obj (_Hash object created with the hashlib library) with the values given as *args, + in the given order. The values must be strings, or objects parseable to string using str(). + Returns the given object.""" + for value in args: + _hash.update(str(value).encode()) + return _hash + + +def new_hash_values(*args, algorithm: str): + """Create a new hash, using the given algorithm, and update it with the values given as *args, in the given order. + The values must be strings, or objects parseable to string using str(). + Returns the hash object (the hash/checksum value can be acquired by using str(output)).""" + _hash = hashlib.new(algorithm) + return update_hash_values(*args, _hash=_hash) + + +def base64_encode(data: str) -> str: + """Encode the given string as base64""" + return _base64.urlsafe_b64encode(data.encode()).decode() + + +def json_encode_object( + obj: Union[pydantic.BaseModel, List[pydantic.BaseModel], Dict[Any, pydantic.BaseModel]], + base64: bool = False +) -> str: + """Given a Pydantic object, a List of Pydantic objects, or a Dict with Pydantic objects, convert to JSON string. + If base64=True, return the JSON result base64-encoded. + """ + encoded = json.dumps(obj, default=pydantic_encoder) + if base64: + return base64_encode(encoded) + return encoded + + +def get_datetime(): + """Get current datetime as a datetime object, in UTC timezone.""" + return datetime.datetime.now(tz=datetime.timezone.utc) + + +def get_datetime_plus_seconds(seconds: int): + """Get current datetime as datetime object, in UTC timezone, with the given amount of seconds added.""" + return get_datetime() + datetime.timedelta(seconds=seconds) + + +def without(d: dict, *exclude: str) -> dict: + """Given dictionary, return a copy of it, without the given "exclude" key/s.""" + dd = d.copy() + for key in exclude: + try: + dd.pop(key) + except KeyError: + continue + return dd diff --git a/vigobusapi/vigobus_getters/auto_getters.py b/vigobusapi/vigobus_getters/auto_getters.py index 45c365a..93187fc 100644 --- a/vigobusapi/vigobus_getters/auto_getters.py +++ b/vigobusapi/vigobus_getters/auto_getters.py @@ -20,6 +20,7 @@ STOP_GETTERS = ( cache.get_stop, mongo.get_stop, + http.get_stop, html.get_stop ) """List of Stop Getter functions. diff --git a/vigobusapi/vigobus_getters/http/http.py b/vigobusapi/vigobus_getters/http/http.py index e5fe6c4..fa94538 100644 --- a/vigobusapi/vigobus_getters/http/http.py +++ b/vigobusapi/vigobus_getters/http/http.py @@ -4,30 +4,54 @@ # # Project # # from vigobusapi.services import http_request -from vigobusapi.entities import BusesResponse +from vigobusapi.entities import Stop, BusesResponse from vigobusapi.logger import logger # # Package # # -from .http_parser import parse_http_response +from .http_parser import parse_http_response_for_stop, parse_http_response_for_buses -__all__ = ("get_buses",) +__all__ = ("get_stop", "get_buses") ENDPOINT_URL = "https://datos.vigo.org/vci_api_app/api2.jsp" -async def get_buses(stop_id: int, get_all_buses: bool = False) -> BusesResponse: - """Async function to get the buses incoming to a Stop from the HTML data source. - The remote data source always returns the whole list of buses, but the output is shortened if get_all_buses=False. - """ - logger.debug("Searching buses on external HTTP data source...") - +async def _request_stop(stop_id: int) -> dict: + """Perform a request against the HTTP API. The endpoint returns both data for buses passing by and stop info, + so it can be used for acquiring both information. + Returns the response body as parsed dict (since the remote API returns a JSON payload).""" + # TODO Cache HTTP API responses (in case a Stop and Buses are queried in short time) params = {"id": stop_id, "ttl": 5, "tipo": "TRANSPORTE-ESTIMACION-PARADA"} response = await http_request( url=ENDPOINT_URL, params=params ) + return response.json() - buses_response = parse_http_response(data=response.json(), get_all_buses=get_all_buses, verify_stop_exists=False) - logger.bind(buses_response_data=buses_response.dict()).debug("Generated BusesResponse") +async def get_stop(stop_id: int) -> Stop: + """Async function to get information of a Stop from the HTTP API data source. + :param stop_id: Stop ID + :raises: requests_async.Timeout | requests_async.RequestException | + exceptions.StopNotExist | exceptions.exceptions.ParseError + """ + logger.bind(**locals()).debug("Searching stop on external HTTP API data source...") + + response_json = await _request_stop(stop_id) + stop_response = parse_http_response_for_stop(response_json) + + logger.bind(stop_response_data=stop_response.dict()).debug("Parsed Stop") + return stop_response + + +async def get_buses(stop_id: int, get_all_buses: bool = False) -> BusesResponse: + """Async function to get the buses incoming to a Stop from the HTTP API data source. + The remote data source always returns the whole list of buses, but the output is shortened if get_all_buses=False. + """ + logger.bind(**locals()).debug("Searching buses on external HTTP API data source...") + + response_json = await _request_stop(stop_id) + buses_response = parse_http_response_for_buses(data=response_json, + get_all_buses=get_all_buses, verify_stop_exists=False) + + logger.bind(buses_response_data=buses_response.dict()).debug("Generated BusesResponse") return buses_response diff --git a/vigobusapi/vigobus_getters/http/http_parser.py b/vigobusapi/vigobus_getters/http/http_parser.py index 3ad39c4..c8a2d82 100644 --- a/vigobusapi/vigobus_getters/http/http_parser.py +++ b/vigobusapi/vigobus_getters/http/http_parser.py @@ -1,18 +1,73 @@ """HTTP_PARSER Parsers for the HTTP external data source + +Sample of JSON payload returned by HTTP API: + +{ + "estimaciones":[ + { + "minutos":25, + "metros":4126, + "linea":"18", + "ruta":"\"A\" SARDOMA por MANTELAS" + }, + { + "minutos":85, + "metros":15120, + "linea":"18", + "ruta":"\"A\" SARDOMA por MANTELAS" + } + ], + "parada":[ + { + "nombre":"Baixada á Laxe 44", + "latitud":42.216415126, + "longitud":-8.719355076, + "stop_vitrasa":2290 + } + ] +} """ # # Project # # -from vigobusapi.entities import Bus, Buses, BusesResponse -from vigobusapi.vigobus_getters.string_fixes import fix_bus +from vigobusapi.entities import Stop, Bus, Buses, BusesResponse +from vigobusapi.vigobus_getters.string_fixes import fix_stop_name, fix_bus from vigobusapi.vigobus_getters.helpers import sort_buses from vigobusapi.exceptions import StopNotExist from vigobusapi.settings import settings -__all__ = ("parse_http_response",) +__all__ = ("parse_http_response_for_stop", "parse_http_response_for_buses") + + +def parse_http_response_for_stop(data: dict) -> Stop: + """Parse a HTTP API response and extract the Stop data from it. + :param data: JSON payload returned by HTTP API + :raises: exceptions.StopNotExist + """ + stop_data = data["parada"][0] + stop_id = stop_data["stop_vitrasa"] + stop_original_name = stop_data["nombre"] + stop_fixed_name = fix_stop_name(stop_original_name) + stop_lat = stop_data.get("latitud") + stop_lon = stop_data.get("longitud") + + return Stop( + stop_id=stop_id, + name=stop_fixed_name, + original_name=stop_original_name, + lat=stop_lat, + lon=stop_lon + ) -def parse_http_response(data: dict, get_all_buses: bool, verify_stop_exists: bool = True) -> BusesResponse: +def parse_http_response_for_buses(data: dict, get_all_buses: bool, verify_stop_exists: bool = True) -> BusesResponse: + """Parse a HTTP API response and extract the Stop data from it. + :param data: JSON payload returned by HTTP API + :param get_all_buses: if True, limit parsed & returned list of buses to settings.buses_normal_limit + :param verify_stop_exists: if the payload does not contain stop info, it will be considered that the stop + does not exist. In this case, if this argument is True, raise the StopNotExist exception + :raises: exceptions.StopNotExist + """ if verify_stop_exists and not data["parada"]: raise StopNotExist()