From e14bb34b793fb66ee4ce45792bb0d0eb78c0007a Mon Sep 17 00:00:00 2001 From: David-Lor <17401854+david-lor@users.noreply.github.com> Date: Mon, 11 Oct 2021 16:47:44 +0200 Subject: [PATCH 1/6] Maps & Streetview cache read before querying Google Maps API --- vigobusapi/services/google_maps/_cache.py | 5 ++-- .../services/google_maps/_getter_maps.py | 27 +++++++++++++++---- .../google_maps/_getter_streetview.py | 24 ++++++++++++++--- vigobusapi/utils.py | 13 ++++++++- 4 files changed, 58 insertions(+), 11 deletions(-) diff --git a/vigobusapi/services/google_maps/_cache.py b/vigobusapi/services/google_maps/_cache.py index 96638e7..212576c 100644 --- a/vigobusapi/services/google_maps/_cache.py +++ b/vigobusapi/services/google_maps/_cache.py @@ -13,7 +13,7 @@ # # Project # # from vigobusapi.services.mongo import MongoDB from vigobusapi.entities import BaseMongoModel -from vigobusapi.utils import get_datetime, new_hash_values, ChecksumableClass +from vigobusapi.utils import get_datetime, new_hash_values, ChecksumableClass, without from vigobusapi.logger import logger from ._entities import * @@ -106,7 +106,8 @@ async def get_cached_metadata(request: MapRequestModels, fetch_image: bool) -> O return None parsed_metadata = CachedMap(**document) - logger.debug("Read map cached metadata document from Mongo") + logger.bind(cached_metadata_document=without(document, "image"))\ + .debug("Read map cached metadata document from Mongo") return parsed_metadata diff --git a/vigobusapi/services/google_maps/_getter_maps.py b/vigobusapi/services/google_maps/_getter_maps.py index 8a1d2bd..93be8f5 100644 --- a/vigobusapi/services/google_maps/_getter_maps.py +++ b/vigobusapi/services/google_maps/_getter_maps.py @@ -11,9 +11,9 @@ from vigobusapi.logger import logger from ._requester import google_maps_request, ListOfTuples from ._entities import GoogleMapRequest -from ._cache import save_cached_metadata +from ._cache import save_cached_metadata, get_cached_metadata -__all__ = ("get_map",) +__all__ = ("get_map", "get_map_from_api") GOOGLE_MAPS_STATIC_API_URL = "https://maps.googleapis.com/maps/api/staticmap" @@ -51,7 +51,7 @@ def _get_map_params(request: GoogleMapRequest) -> ListOfTuples: return params -async def get_map(request: GoogleMapRequest) -> bytes: +async def get_map_from_api(request: GoogleMapRequest) -> bytes: """Get a static Map picture from the Google Maps Static API. Return the acquired PNG picture as bytes. The fetched picture is persisted on cache, running a fire & forget background task. @@ -60,9 +60,26 @@ async def get_map(request: GoogleMapRequest) -> bytes: https://developers.google.com/maps/documentation/maps-static/start """ logger.bind(map_request=request.dict()).debug("Requesting Google Static Map picture...") - # TODO cache loaded pictures - params = _get_map_params(request) + image = (await google_maps_request(url=GOOGLE_MAPS_STATIC_API_URL, params=params)).content + logger.debug("Map acquired from Google Static Maps API") + asyncio.create_task(save_cached_metadata(request=request, image=image)) return image + + +async def get_map(request: GoogleMapRequest, read_cache_first: bool = True) -> bytes: + """Get a static Map picture from cache (if read_cache_first=True) or the Google Maps Static API. + Return the acquired PNG picture as bytes.""" + image = None + + if read_cache_first: + cached_metadata = await get_cached_metadata(request, fetch_image=True) + if cached_metadata: + image = cached_metadata.image + + if image is None: + image = await get_map_from_api(request) + + return image diff --git a/vigobusapi/services/google_maps/_getter_streetview.py b/vigobusapi/services/google_maps/_getter_streetview.py index 2efb64c..ad1d2d6 100644 --- a/vigobusapi/services/google_maps/_getter_streetview.py +++ b/vigobusapi/services/google_maps/_getter_streetview.py @@ -10,7 +10,7 @@ from vigobusapi.logger import logger from ._entities import GoogleStreetviewRequest from ._requester import google_maps_request, ListOfTuples -from ._cache import save_cached_metadata +from ._cache import save_cached_metadata, get_cached_metadata __all__ = ("get_photo",) @@ -28,7 +28,7 @@ def _get_photo_params(request: GoogleStreetviewRequest) -> ListOfTuples: return params -async def get_photo(request: GoogleStreetviewRequest) -> Optional[bytes]: +async def get_photo_from_api(request: GoogleStreetviewRequest) -> Optional[bytes]: """Get a static StreetView picture from the Google StreetView Static API. Return the acquired PNG picture as bytes. If the requested location does not have an available picture, returns None. The fetched picture is persisted on cache, running a fire & forget background task. @@ -37,7 +37,6 @@ async def get_photo(request: GoogleStreetviewRequest) -> Optional[bytes]: https://developers.google.com/maps/documentation/streetview/overview """ logger.bind(streetview_request=request.dict()).debug("Requesting Google Static StreetView picture...") - # TODO cache loaded pictures # TODO Support specific parameters for tuning camera, if required params = _get_photo_params(request) @@ -48,5 +47,24 @@ async def get_photo(request: GoogleStreetviewRequest) -> Optional[bytes]: response.raise_for_status() image = response.content + logger.debug("Photo acquired from Google StreetView Static API") + asyncio.create_task(save_cached_metadata(request=request, image=image)) return image + + +async def get_photo(request: GoogleStreetviewRequest, read_cache_first: bool = True) -> Optional[bytes]: + """Get a static StreetView picture from cache (if read_cache_first=True) or the Google StreetView Static API. + Return the acquired PNG picture as bytes. + If the requested location does not have an available picture, returns None.""" + image = None + + if read_cache_first: + cached_metadata = await get_cached_metadata(request, fetch_image=True) + if cached_metadata: + image = cached_metadata.image + + if image is None: + image = await get_photo_from_api(request) + + return image diff --git a/vigobusapi/utils.py b/vigobusapi/utils.py index a2a173c..7cef6c8 100644 --- a/vigobusapi/utils.py +++ b/vigobusapi/utils.py @@ -16,7 +16,7 @@ __all__ = ( "ChecksumableClass", "new_hash_values", "update_hash_values", "base64_encode", "json_encode_object", - "get_datetime" + "get_datetime", "without" ) @@ -76,3 +76,14 @@ def json_encode_object( def get_datetime(): """Get current datetime as a datetime object, in UTC timezone.""" return datetime.datetime.now(tz=datetime.timezone.utc) + + +def without(d: dict, *exclude: str) -> dict: + """Given dictionary, return a copy of it, without the given "exclude" key/s.""" + dd = d.copy() + for key in exclude: + try: + dd.pop(key) + except KeyError: + continue + return dd From c22e9d09b151dadcfe87fd85b16a671c372cd4dd Mon Sep 17 00:00:00 2001 From: David-Lor <17401854+david-lor@users.noreply.github.com> Date: Mon, 11 Oct 2021 17:06:03 +0200 Subject: [PATCH 2/6] refactor: routers in different package and modules by context Add aliases to routes, using plurals as standard ways (e.g. "/stops/{stop_id}" instead of "/stop/{stop_id}"), but supporting old form --- vigobusapi/app.py | 171 +----------------------------- vigobusapi/routes/__init__.py | 1 + vigobusapi/routes/_api.py | 18 ++++ vigobusapi/routes/_maps.py | 124 ++++++++++++++++++++++ vigobusapi/routes/_stops_buses.py | 61 +++++++++++ vigobusapi/routes/bootstrap.py | 18 ++++ 6 files changed, 227 insertions(+), 166 deletions(-) create mode 100644 vigobusapi/routes/__init__.py create mode 100644 vigobusapi/routes/_api.py create mode 100644 vigobusapi/routes/_maps.py create mode 100644 vigobusapi/routes/_stops_buses.py create mode 100644 vigobusapi/routes/bootstrap.py diff --git a/vigobusapi/app.py b/vigobusapi/app.py index 9c50280..e137ad4 100644 --- a/vigobusapi/app.py +++ b/vigobusapi/app.py @@ -1,23 +1,15 @@ """APP -Module with all the available endpoints and the FastAPI initialization. +FastAPI initialization. """ -# # Native # # -import io -import json -from typing import Optional, Set - # # Installed # # import uvicorn -from fastapi import FastAPI, Response, Query, Depends, HTTPException -from starlette.responses import StreamingResponse +from fastapi import FastAPI # # Project # # -from vigobusapi.entities import Stop, Stops, BusesResponse +from vigobusapi.routes import setup_routes from vigobusapi.request_handler import request_handler -from vigobusapi.settings import settings, google_maps_settings -from vigobusapi.vigobus_getters import get_stop, get_stops, get_buses, search_stops -from vigobusapi.services.google_maps import GoogleMapRequest, GoogleStreetviewRequest, get_map, get_photo +from vigobusapi.settings import settings from vigobusapi.services import MongoDB from vigobusapi.logger import logger @@ -27,6 +19,7 @@ title=settings.api_name ) app.middleware("http")(request_handler) +setup_routes(app) @app.on_event("startup") @@ -36,160 +29,6 @@ async def app_setup(): await MongoDB.initialize() -@app.get("/status") -async def endpoint_status(): - return Response( - content="OK", - media_type="text/plain", - status_code=200 - ) - - -@app.get("/stops", response_model=Stops) -async def endpoint_get_stops( - stop_name: Optional[str] = Query(None), - limit: Optional[int] = Query(None), - stops_ids: Optional[Set[int]] = Query(None, alias="stop_id") -): - """Endpoint to search/list stops by different filters. Only one filter can be used. - Returns 400 if no filters given. - The filters available are: - - - stop_name: search by a single string in stop names. "limit" can be used for limiting results size. - - stop_id: repeatable param for getting multiple stops by id on a single request. Not found errors are ignored. - """ - with logger.contextualize(**locals()): - if stop_name is not None: - stops = await search_stops(stop_name=stop_name, limit=limit) - elif stops_ids: - stops = await get_stops(stops_ids) - else: - raise HTTPException(status_code=400, detail="No filters given") - return [stop.dict() for stop in stops] - - -@app.get("/stop/{stop_id}", response_model=Stop) -async def endpoint_get_stop(stop_id: int): - """Endpoint to get information of a Stop giving the Stop ID - """ - with logger.contextualize(**locals()): - stop = await get_stop(stop_id) - return stop.dict() - - -@app.get("/buses/{stop_id}", response_model=BusesResponse) -@app.get("/stop/{stop_id}/buses", response_model=BusesResponse) -async def endpoint_get_buses(stop_id: int, get_all_buses: bool = False): - """Endpoint to get a list of Buses coming to a Stop giving the Stop ID. - By default the shortest available list of buses is returned, unless 'get_all_buses' param is True - """ - with logger.contextualize(**locals()): - buses_result = await get_buses(stop_id, get_all_buses=get_all_buses) - return buses_result.dict() - - -class MapQueryParams: - def __init__( - self, - size_x: int = google_maps_settings.stop_map_default_size_x, - size_y: int = google_maps_settings.stop_map_default_size_y, - zoom: int = google_maps_settings.stop_map_default_zoom, - map_type: GoogleMapRequest.MapTypes = google_maps_settings.stop_map_default_type - ): - self.size_x = size_x - self.size_y = size_y - self.zoom = zoom - self.map_type = map_type - - -@app.get("/stop/{stop_id}/map") -async def endpoint_get_stop_map( - stop_id: int, - map_params: MapQueryParams = Depends() -): - """Get a picture of a map with the stop location marked on it.""" - stop = await get_stop(stop_id) - if not stop.has_location: - raise HTTPException(status_code=409, detail="The stop does not have information about the location") - - map_request = GoogleMapRequest( - location_x=stop.lat, - location_y=stop.lon, - size_x=map_params.size_x, - size_y=map_params.size_y, - zoom=map_params.zoom, - map_type=map_params.map_type, - tags=[GoogleMapRequest.Tag(location_x=stop.lat, location_y=stop.lon)] - ) - map_data = await get_map(map_request) - return StreamingResponse(io.BytesIO(map_data), media_type="image/png") - - -@app.get("/stops/map") -async def endpoint_get_stops_map( - stops_ids: Set[int] = Query(None, alias="stop_id", - min_items=1, max_items=len(GoogleMapRequest.Tag.get_allowed_labels())), - map_params: MapQueryParams = Depends(), -): - """Get a picture of a map with the locations of the given stops marked on it. - - Non existing stops, or those without location available, are ignored, - but if none of the given stops are valid, returns 404. - - A header "X-Stops-Tags" is returned, being a JSON associating the Stops IDs with the tag label on the map, - with the format: {"" : ""} - """ - stops = await get_stops(stops_ids) - stops = [stop for stop in stops if stop.has_location] - if not stops: - raise HTTPException(status_code=404, detail="None of the stops exist or have location available") - - stops_tags = list() - stops_tags_relation = dict() - for i, stop in enumerate(stops): - tag_label = GoogleMapRequest.Tag.get_allowed_labels()[i] - tag = GoogleMapRequest.Tag(label=tag_label, location_x=stop.lat, location_y=stop.lon) - stops_tags.append(tag) - stops_tags_relation[stop.stop_id] = tag_label - - map_request = GoogleMapRequest( - size_x=map_params.size_x, - size_y=map_params.size_y, - zoom=map_params.zoom, - map_type=map_params.map_type, - tags=stops_tags - ) - map_data = await get_map(map_request) - - return StreamingResponse( - content=io.BytesIO(map_data), - media_type="image/png", - headers={"X-Stops-Tags": json.dumps(stops_tags_relation)} - ) - - -@app.get("/stop/{stop_id}/photo") -async def endpoint_get_stop_photo( - stop_id: int, - size_x: int = google_maps_settings.stop_photo_default_size_x, - size_y: int = google_maps_settings.stop_photo_default_size_y, -): - stop = await get_stop(stop_id) - if not stop.has_location: - raise HTTPException(status_code=409, detail="The stop does not have information about the location") - - photo_request = GoogleStreetviewRequest( - location_x=stop.lat, - location_y=stop.lon, - size_x=size_x, - size_y=size_y - ) - photo_data = await get_photo(photo_request) - if not photo_data: - raise HTTPException(status_code=404, detail="No StreetView photo available for the stop location") - return StreamingResponse(io.BytesIO(photo_data), media_type="image/png") - - def run(): """Run the API using Uvicorn """ diff --git a/vigobusapi/routes/__init__.py b/vigobusapi/routes/__init__.py new file mode 100644 index 0000000..1253e3e --- /dev/null +++ b/vigobusapi/routes/__init__.py @@ -0,0 +1 @@ +from .bootstrap import * diff --git a/vigobusapi/routes/_api.py b/vigobusapi/routes/_api.py new file mode 100644 index 0000000..f5661c6 --- /dev/null +++ b/vigobusapi/routes/_api.py @@ -0,0 +1,18 @@ +"""API "General" Routes +""" + +# # Installed # # +from fastapi import APIRouter, Response + +__all__ = ("router",) + +router = APIRouter() + + +@router.get("/status") +async def endpoint_status(): + return Response( + content="OK", + media_type="text/plain", + status_code=200 + ) diff --git a/vigobusapi/routes/_maps.py b/vigobusapi/routes/_maps.py new file mode 100644 index 0000000..c774821 --- /dev/null +++ b/vigobusapi/routes/_maps.py @@ -0,0 +1,124 @@ +"""MAPS Routes +""" + +# # Native # # +import io +import json +from typing import Set + +# # Installed # # +from fastapi import APIRouter, Query, Depends, HTTPException +from starlette.responses import StreamingResponse + +# # Project # # +from vigobusapi.settings import google_maps_settings +from vigobusapi.vigobus_getters import get_stop, get_stops +from vigobusapi.services.google_maps import GoogleMapRequest, GoogleStreetviewRequest, get_map, get_photo + +__all__ = ("router",) + +router = APIRouter() + + +class MapQueryParams: + def __init__( + self, + size_x: int = google_maps_settings.stop_map_default_size_x, + size_y: int = google_maps_settings.stop_map_default_size_y, + zoom: int = google_maps_settings.stop_map_default_zoom, + map_type: GoogleMapRequest.MapTypes = google_maps_settings.stop_map_default_type + ): + self.size_x = size_x + self.size_y = size_y + self.zoom = zoom + self.map_type = map_type + + +@router.get("/stop/{stop_id}/map") +@router.get("/stops/{stop_id}/map") +async def endpoint_get_stop_map( + stop_id: int, + map_params: MapQueryParams = Depends() +): + """Get a picture of a map with the stop location marked on it.""" + stop = await get_stop(stop_id) + if not stop.has_location: + raise HTTPException(status_code=409, detail="The stop does not have information about the location") + + map_request = GoogleMapRequest( + location_x=stop.lat, + location_y=stop.lon, + size_x=map_params.size_x, + size_y=map_params.size_y, + zoom=map_params.zoom, + map_type=map_params.map_type, + tags=[GoogleMapRequest.Tag(location_x=stop.lat, location_y=stop.lon)] + ) + map_data = await get_map(map_request) + return StreamingResponse(io.BytesIO(map_data), media_type="image/png") + + +@router.get("/stops/map") +async def endpoint_get_stops_map( + stops_ids: Set[int] = Query(None, alias="stop_id", + min_items=1, max_items=len(GoogleMapRequest.Tag.get_allowed_labels())), + map_params: MapQueryParams = Depends(), +): + """Get a picture of a map with the locations of the given stops marked on it. + + Non existing stops, or those without location available, are ignored, + but if none of the given stops are valid, returns 404. + + A header "X-Stops-Tags" is returned, being a JSON associating the Stops IDs with the tag label on the map, + with the format: {"" : ""} + """ + stops = await get_stops(stops_ids) + stops = [stop for stop in stops if stop.has_location] + if not stops: + raise HTTPException(status_code=404, detail="None of the stops exist or have location available") + + stops_tags = list() + stops_tags_relation = dict() + for i, stop in enumerate(stops): + tag_label = GoogleMapRequest.Tag.get_allowed_labels()[i] + tag = GoogleMapRequest.Tag(label=tag_label, location_x=stop.lat, location_y=stop.lon) + stops_tags.append(tag) + stops_tags_relation[stop.stop_id] = tag_label + + map_request = GoogleMapRequest( + size_x=map_params.size_x, + size_y=map_params.size_y, + zoom=map_params.zoom, + map_type=map_params.map_type, + tags=stops_tags + ) + map_data = await get_map(map_request) + + return StreamingResponse( + content=io.BytesIO(map_data), + media_type="image/png", + headers={"X-Stops-Tags": json.dumps(stops_tags_relation)} + ) + + +@router.get("/stop/{stop_id}/photo") +@router.get("/stops/{stop_id}/photo") +async def endpoint_get_stop_photo( + stop_id: int, + size_x: int = google_maps_settings.stop_photo_default_size_x, + size_y: int = google_maps_settings.stop_photo_default_size_y, +): + stop = await get_stop(stop_id) + if not stop.has_location: + raise HTTPException(status_code=409, detail="The stop does not have information about the location") + + photo_request = GoogleStreetviewRequest( + location_x=stop.lat, + location_y=stop.lon, + size_x=size_x, + size_y=size_y + ) + photo_data = await get_photo(photo_request) + if not photo_data: + raise HTTPException(status_code=404, detail="No StreetView photo available for the stop location") + return StreamingResponse(io.BytesIO(photo_data), media_type="image/png") diff --git a/vigobusapi/routes/_stops_buses.py b/vigobusapi/routes/_stops_buses.py new file mode 100644 index 0000000..d88dc0b --- /dev/null +++ b/vigobusapi/routes/_stops_buses.py @@ -0,0 +1,61 @@ +"""STOPS & BUSES Routes +""" + +# # Native # # +from typing import Optional, Set + +# # Installed # # +from fastapi import APIRouter, Query, HTTPException + +# # Project # # +from vigobusapi.entities import Stop, Stops, BusesResponse +from vigobusapi.vigobus_getters import get_stop, get_stops, get_buses, search_stops +from vigobusapi.logger import logger + +__all__ = ("router",) + +router = APIRouter() + + +@router.get("/stops", response_model=Stops) +async def endpoint_get_stops( + stop_name: Optional[str] = Query(None), + limit: Optional[int] = Query(None), + stops_ids: Optional[Set[int]] = Query(None, alias="stop_id") +): + """Endpoint to search/list stops by different filters. Only one filter can be used. + Returns 400 if no filters given. + The filters available are: + + - stop_name: search by a single string in stop names. "limit" can be used for limiting results size. + - stop_id: repeatable param for getting multiple stops by id on a single request. Not found errors are ignored. + """ + with logger.contextualize(**locals()): + if stop_name is not None: + stops = await search_stops(stop_name=stop_name, limit=limit) + elif stops_ids: + stops = await get_stops(stops_ids) + else: + raise HTTPException(status_code=400, detail="No filters given") + return [stop.dict() for stop in stops] + + +@router.get("/stop/{stop_id}", response_model=Stop) +@router.get("/stops/{stop_id}", response_model=Stop) +async def endpoint_get_stop(stop_id: int): + """Endpoint to get information of a Stop giving the Stop ID + """ + with logger.contextualize(**locals()): + stop = await get_stop(stop_id) + return stop.dict() + + +@router.get("/buses/{stop_id}", response_model=BusesResponse) +@router.get("/stops/{stop_id}/buses", response_model=BusesResponse) +async def endpoint_get_buses(stop_id: int, get_all_buses: bool = False): + """Endpoint to get a list of Buses coming to a Stop giving the Stop ID. + By default the shortest available list of buses is returned, unless 'get_all_buses' param is True + """ + with logger.contextualize(**locals()): + buses_result = await get_buses(stop_id, get_all_buses=get_all_buses) + return buses_result.dict() \ No newline at end of file diff --git a/vigobusapi/routes/bootstrap.py b/vigobusapi/routes/bootstrap.py new file mode 100644 index 0000000..1b568b6 --- /dev/null +++ b/vigobusapi/routes/bootstrap.py @@ -0,0 +1,18 @@ +"""ROUTES BOOTSTRAP +Entrypoint for API routes setup +""" + +from fastapi import FastAPI + +from ._api import router as general_router +from ._stops_buses import router as stops_buses_router +from ._maps import router as maps_router + + +__all__ = ("setup_routes",) + + +def setup_routes(app: FastAPI): + app.include_router(general_router) + app.include_router(stops_buses_router) + app.include_router(maps_router) From ba5f83ccac6d75b1cb56ab7b79150796f833b3e2 Mon Sep 17 00:00:00 2001 From: David-Lor <17401854+david-lor@users.noreply.github.com> Date: Mon, 11 Oct 2021 17:30:51 +0200 Subject: [PATCH 3/6] Add endpoint & logic for setting Telegram File ID on cached map pictures --- vigobusapi/routes/_maps.py | 37 +++++++++++++++-------- vigobusapi/services/google_maps/_cache.py | 27 ++++++++++++++++- 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/vigobusapi/routes/_maps.py b/vigobusapi/routes/_maps.py index c774821..cf6a419 100644 --- a/vigobusapi/routes/_maps.py +++ b/vigobusapi/routes/_maps.py @@ -4,34 +4,36 @@ # # Native # # import io import json +from dataclasses import dataclass from typing import Set # # Installed # # -from fastapi import APIRouter, Query, Depends, HTTPException +from fastapi import APIRouter, Query, Depends, HTTPException, Response from starlette.responses import StreamingResponse # # Project # # from vigobusapi.settings import google_maps_settings from vigobusapi.vigobus_getters import get_stop, get_stops -from vigobusapi.services.google_maps import GoogleMapRequest, GoogleStreetviewRequest, get_map, get_photo +from vigobusapi.services.google_maps import (GoogleMapRequest, GoogleStreetviewRequest, + get_map, get_photo, update_cached_metadata) __all__ = ("router",) router = APIRouter() +@dataclass class MapQueryParams: - def __init__( - self, - size_x: int = google_maps_settings.stop_map_default_size_x, - size_y: int = google_maps_settings.stop_map_default_size_y, - zoom: int = google_maps_settings.stop_map_default_zoom, - map_type: GoogleMapRequest.MapTypes = google_maps_settings.stop_map_default_type - ): - self.size_x = size_x - self.size_y = size_y - self.zoom = zoom - self.map_type = map_type + size_x: int = google_maps_settings.stop_map_default_size_x, + size_y: int = google_maps_settings.stop_map_default_size_y, + zoom: int = google_maps_settings.stop_map_default_zoom, + map_type: GoogleMapRequest.MapTypes = google_maps_settings.stop_map_default_type + + +@dataclass +class MapCacheSetParams: + id: str + telegram_file_id: str @router.get("/stop/{stop_id}/map") @@ -122,3 +124,12 @@ async def endpoint_get_stop_photo( if not photo_data: raise HTTPException(status_code=404, detail="No StreetView photo available for the stop location") return StreamingResponse(io.BytesIO(photo_data), media_type="image/png") + + +@router.put("/cache/maps", status_code=204) +async def update_maps_cache(cache_params: MapCacheSetParams = Depends()): + """Update fields from a cached map or photo. Can be used for setting the Telegram File ID of a persisted photo.""" + updated = await update_cached_metadata(cache_id=cache_params.id, telegram_file_id=cache_params.telegram_file_id) + if not updated: + raise HTTPException(status_code=404, detail=f"No cache found with id {cache_params.id}") + return Response(status_code=204) diff --git a/vigobusapi/services/google_maps/_cache.py b/vigobusapi/services/google_maps/_cache.py index 212576c..523c940 100644 --- a/vigobusapi/services/google_maps/_cache.py +++ b/vigobusapi/services/google_maps/_cache.py @@ -17,7 +17,7 @@ from vigobusapi.logger import logger from ._entities import * -__all__ = ("get_cached_metadata", "save_cached_metadata", +__all__ = ("get_cached_metadata", "save_cached_metadata", "update_cached_metadata", "MapRequestModels", "MapVendors", "MapTypes", "CachedMap") MapRequestModels = Union[GoogleMapRequest, GoogleStreetviewRequest] @@ -111,6 +111,31 @@ async def get_cached_metadata(request: MapRequestModels, fetch_image: bool) -> O return parsed_metadata +async def update_cached_metadata(cache_id: str, telegram_file_id: str) -> bool: + """Update certain fields from a cached map metadata document. Currently only supports updating the Telegram File ID. + The identifier used is the document id. Returns True/False depending on if the document was found.""" + with logger.contextualize(cache_id=cache_id): + query_filter = dict(_id=cache_id) + query_update = { + "$set": { + "telegram_file_id": telegram_file_id + } + } + + logger.bind(cache_document_update=query_update).debug("Updating map cached metadata...") + r: dict = await MongoDB.get_mongo().get_cache_maps_collection().find_one_and_update( + filter=query_filter, + update=query_update + ) + + if r is None: + logger.debug("No map cache document found for update") + return False + + logger.debug("Updated map cache document") + return True + + async def save_cached_metadata(request: MapRequestModels, image: Optional[bytes]): metadata = CachedMap( key=request.checksum_value, From 05bd055bb7cfdcde6f95fba7ff03425c5adcad58 Mon Sep 17 00:00:00 2001 From: David-Lor <17401854+david-lor@users.noreply.github.com> Date: Tue, 12 Oct 2021 15:36:33 +0200 Subject: [PATCH 4/6] Return Cache ID on Map endpoints responses Rename stops-tags relation header on multi-stop map endpoint --- vigobusapi/routes/_maps.py | 123 ++++++++++++++---- vigobusapi/services/google_maps/_cache.py | 56 +++++--- .../services/google_maps/_getter_maps.py | 24 ++-- .../google_maps/_getter_streetview.py | 30 ++--- 4 files changed, 162 insertions(+), 71 deletions(-) diff --git a/vigobusapi/routes/_maps.py b/vigobusapi/routes/_maps.py index cf6a419..34ada5f 100644 --- a/vigobusapi/routes/_maps.py +++ b/vigobusapi/routes/_maps.py @@ -5,17 +5,17 @@ import io import json from dataclasses import dataclass -from typing import Set +from typing import * # # Installed # # from fastapi import APIRouter, Query, Depends, HTTPException, Response -from starlette.responses import StreamingResponse +from starlette.responses import StreamingResponse, PlainTextResponse # # Project # # from vigobusapi.settings import google_maps_settings from vigobusapi.vigobus_getters import get_stop, get_stops from vigobusapi.services.google_maps import (GoogleMapRequest, GoogleStreetviewRequest, - get_map, get_photo, update_cached_metadata) + get_map, get_photo, get_cached_metadata, update_cached_metadata, CachedMap) __all__ = ("router",) @@ -24,25 +24,62 @@ @dataclass class MapQueryParams: - size_x: int = google_maps_settings.stop_map_default_size_x, - size_y: int = google_maps_settings.stop_map_default_size_y, - zoom: int = google_maps_settings.stop_map_default_zoom, + size_x: int = google_maps_settings.stop_map_default_size_x + size_y: int = google_maps_settings.stop_map_default_size_y + zoom: int = google_maps_settings.stop_map_default_zoom map_type: GoogleMapRequest.MapTypes = google_maps_settings.stop_map_default_type +@dataclass +class PhotoQueryParams: + size_x: int = google_maps_settings.stop_photo_default_size_x + size_y: int = google_maps_settings.stop_photo_default_size_y + + @dataclass class MapCacheSetParams: id: str telegram_file_id: str +StopsTagsRelation = Dict[int, GoogleMapRequest.Tag] + + +def _format_map_response( + image: Optional[bytes] = None, + telegram_file_id: Optional[str] = None, + cache_metadata: Optional[CachedMap] = None, + stops_tags_relation: Optional[StopsTagsRelation] = None +) -> Response: + """Generate a Response for a fetched image or cached item, based on the given arguments.""" + headers = dict() + if cache_metadata is not None: + headers["X-Maps-Cache-ID"] = cache_metadata.id + if stops_tags_relation is not None: + headers["X-Maps-Stops-Tags"] = json.dumps(stops_tags_relation) + + if image is not None: + return StreamingResponse(content=io.BytesIO(image), media_type="image/png", headers=headers) + if telegram_file_id is not None: + return PlainTextResponse(content=telegram_file_id, headers=headers) + raise Exception("No file data or cache given for generating Response") + + @router.get("/stop/{stop_id}/map") @router.get("/stops/{stop_id}/map") async def endpoint_get_stop_map( stop_id: int, - map_params: MapQueryParams = Depends() + map_params: MapQueryParams = Depends(), + get_telegram_file_id: bool = False ): - """Get a picture of a map with the stop location marked on it.""" + """Get a picture of a map with the stop location marked on it. + + If get_telegram_file_id=True, fetch the cached Telegram File ID and return it as plaintext. + If not available, the picture (cached or live) is returned. + + A header "X-Maps-Cache-ID" is returned, with the Cache ID of the map with the queried parameters. + This can be used for updating the cache. + """ stop = await get_stop(stop_id) if not stop.has_location: raise HTTPException(status_code=409, detail="The stop does not have information about the location") @@ -56,8 +93,16 @@ async def endpoint_get_stop_map( map_type=map_params.map_type, tags=[GoogleMapRequest.Tag(location_x=stop.lat, location_y=stop.lon)] ) - map_data = await get_map(map_request) - return StreamingResponse(io.BytesIO(map_data), media_type="image/png") + + if get_telegram_file_id: + cache_metadata = await get_cached_metadata(map_request, fetch_image=False) + if cache_metadata and cache_metadata.telegram_file_id: + return _format_map_response( + telegram_file_id=cache_metadata.telegram_file_id, cache_metadata=cache_metadata + ) + + image, cache_metadata = await get_map(map_request) + return _format_map_response(image=image, cache_metadata=cache_metadata) @router.get("/stops/map") @@ -65,22 +110,29 @@ async def endpoint_get_stops_map( stops_ids: Set[int] = Query(None, alias="stop_id", min_items=1, max_items=len(GoogleMapRequest.Tag.get_allowed_labels())), map_params: MapQueryParams = Depends(), + get_telegram_file_id: bool = False ): """Get a picture of a map with the locations of the given stops marked on it. + If get_telegram_file_id=True, fetch the cached Telegram File ID and return it as plaintext. + If not available, the picture (cached or live) is returned. + Non existing stops, or those without location available, are ignored, but if none of the given stops are valid, returns 404. - A header "X-Stops-Tags" is returned, being a JSON associating the Stops IDs with the tag label on the map, + A header "X-Maps-Stops-Tags" is returned, being a JSON associating the Stops IDs with the tag label on the map, with the format: {"" : ""} + + A header "X-Maps-Cache-ID" is returned, with the Cache ID of the map with the queried parameters. + This can be used for updating the cache. """ stops = await get_stops(stops_ids) stops = [stop for stop in stops if stop.has_location] if not stops: raise HTTPException(status_code=404, detail="None of the stops exist or have location available") - stops_tags = list() - stops_tags_relation = dict() + stops_tags: List[GoogleMapRequest.Tag] = list() + stops_tags_relation: StopsTagsRelation = dict() for i, stop in enumerate(stops): tag_label = GoogleMapRequest.Tag.get_allowed_labels()[i] tag = GoogleMapRequest.Tag(label=tag_label, location_x=stop.lat, location_y=stop.lon) @@ -94,22 +146,35 @@ async def endpoint_get_stops_map( map_type=map_params.map_type, tags=stops_tags ) - map_data = await get_map(map_request) - return StreamingResponse( - content=io.BytesIO(map_data), - media_type="image/png", - headers={"X-Stops-Tags": json.dumps(stops_tags_relation)} - ) + if get_telegram_file_id: + cache_metadata = await get_cached_metadata(map_request, fetch_image=False) + if cache_metadata and cache_metadata.telegram_file_id: + return _format_map_response( + telegram_file_id=cache_metadata.telegram_file_id, + cache_metadata=cache_metadata, + stops_tags_relation=stops_tags_relation + ) + + image, cache_metadata = await get_map(map_request) + return _format_map_response(image=image, cache_metadata=cache_metadata, stops_tags_relation=stops_tags_relation) @router.get("/stop/{stop_id}/photo") @router.get("/stops/{stop_id}/photo") async def endpoint_get_stop_photo( stop_id: int, - size_x: int = google_maps_settings.stop_photo_default_size_x, - size_y: int = google_maps_settings.stop_photo_default_size_y, + photo_params: PhotoQueryParams = Depends(), + get_telegram_file_id: bool = False ): + """Get a real street photo of the Stop location. + + If get_telegram_file_id=True, fetch the cached Telegram File ID and return it as plaintext. + If not available, the picture (cached or live) is returned. + + A header "X-Maps-Cache-ID" is returned, with the Cache ID of the map with the queried parameters. + This can be used for updating the cache. + """ stop = await get_stop(stop_id) if not stop.has_location: raise HTTPException(status_code=409, detail="The stop does not have information about the location") @@ -117,13 +182,19 @@ async def endpoint_get_stop_photo( photo_request = GoogleStreetviewRequest( location_x=stop.lat, location_y=stop.lon, - size_x=size_x, - size_y=size_y + size_x=photo_params.size_x, + size_y=photo_params.size_y ) - photo_data = await get_photo(photo_request) - if not photo_data: + + if get_telegram_file_id: + cache_metadata = await get_cached_metadata(photo_request, fetch_image=False) + if cache_metadata and cache_metadata.telegram_file_id: + return _format_map_response(telegram_file_id=cache_metadata.telegram_file_id, cache_metadata=cache_metadata) + + image, cache_metadata = await get_photo(photo_request) + if not image: raise HTTPException(status_code=404, detail="No StreetView photo available for the stop location") - return StreamingResponse(io.BytesIO(photo_data), media_type="image/png") + return _format_map_response(image=image, cache_metadata=cache_metadata) @router.put("/cache/maps", status_code=204) diff --git a/vigobusapi/services/google_maps/_cache.py b/vigobusapi/services/google_maps/_cache.py index 523c940..6c2b8a2 100644 --- a/vigobusapi/services/google_maps/_cache.py +++ b/vigobusapi/services/google_maps/_cache.py @@ -5,6 +5,7 @@ # # Native # # import enum import datetime +import asyncio from typing import Union, Optional # # Installed # # @@ -56,7 +57,7 @@ class CachedMap(BaseMongoModel, ChecksumableClass): saved: datetime.datetime """When this document was saved. Used for TTL purposes.""" image: Optional[bytes] - """Image saved as-is.""" + """Image saved as-is. Optional because metadata can be fetched without the image, but should always be persisted.""" telegram_file_id: Optional[str] """File ID in Telegram, set after sending the picture via Telegram.""" @@ -64,6 +65,20 @@ def __init__(self, **kwargs): super().__init__(**kwargs) self.id = self.checksum_value + def metadata_dict(self, **kwargs) -> dict: + """Fetch the dict(), without the "image" field.""" + excludes = kwargs.pop("exclude", set()) + excludes.add("image") + kwargs["exclude"] = excludes + return self.dict(**kwargs) + + def metadata_json(self, **kwargs) -> str: + """Fetch the json(), without the "image" field.""" + excludes = kwargs.pop("exclude", set()) + excludes.add("image") + kwargs["exclude"] = excludes + return self.json(**kwargs) + @property def checksum_hash(self): return new_hash_values( @@ -86,7 +101,7 @@ def _get_mongo_filter_from_request(request: MapRequestModels): async def get_cached_metadata(request: MapRequestModels, fetch_image: bool) -> Optional[CachedMap]: request_checksum = request.checksum_value - with logger.contextualize(request_checksum=request_checksum): + with logger.contextualize(map_request_checksum=request_checksum): logger.bind(fetch_image=fetch_image).debug("Searching map cached metadata...") query_filter = _get_mongo_filter_from_request(request) @@ -136,7 +151,9 @@ async def update_cached_metadata(cache_id: str, telegram_file_id: str) -> bool: return True -async def save_cached_metadata(request: MapRequestModels, image: Optional[bytes]): +async def save_cached_metadata(request: MapRequestModels, image: bytes, background: bool) -> CachedMap: + """Save the given request object and image in cache. + Return the CachedMap object. If background=True, return instantly and persist in background.""" metadata = CachedMap( key=request.checksum_value, vendor=MapVendors.google_maps.value, @@ -147,16 +164,25 @@ async def save_cached_metadata(request: MapRequestModels, image: Optional[bytes] ) with logger.contextualize(map_cache_id=metadata.id): - logger.debug("Saving map cache in MongoDB...") - r: UpdateResult = await MongoDB.get_mongo().get_cache_maps_collection().replace_one( - filter=dict(_id=metadata.id), - replacement=metadata.to_mongo(), - upsert=True - ) - - if r.upserted_id: - logger.debug("Inserted new map cache document") - elif r.modified_count: - logger.debug("Replaced existing map cache document") + async def __save(): + logger.debug("Saving map cache in MongoDB...") + r: UpdateResult = await MongoDB.get_mongo().get_cache_maps_collection().replace_one( + filter=dict(_id=metadata.id), + replacement=metadata.to_mongo(), + upsert=True + ) + + if r.upserted_id: + logger.debug("Inserted new map cache document") + elif r.modified_count: + logger.debug("Replaced existing map cache document") + else: + logger.error("No modified/inserted documents inserted in MongoDB") + + if background: + logger.debug("Map cache will be saved as background task") + asyncio.create_task(__save()) else: - logger.error("No modified/inserted documents inserted in MongoDB") + await __save() + + return metadata diff --git a/vigobusapi/services/google_maps/_getter_maps.py b/vigobusapi/services/google_maps/_getter_maps.py index 93be8f5..7e45ea6 100644 --- a/vigobusapi/services/google_maps/_getter_maps.py +++ b/vigobusapi/services/google_maps/_getter_maps.py @@ -11,7 +11,7 @@ from vigobusapi.logger import logger from ._requester import google_maps_request, ListOfTuples from ._entities import GoogleMapRequest -from ._cache import save_cached_metadata, get_cached_metadata +from ._cache import save_cached_metadata, get_cached_metadata, CachedMap __all__ = ("get_map", "get_map_from_api") @@ -51,8 +51,9 @@ def _get_map_params(request: GoogleMapRequest) -> ListOfTuples: return params -async def get_map_from_api(request: GoogleMapRequest) -> bytes: - """Get a static Map picture from the Google Maps Static API. Return the acquired PNG picture as bytes. +async def get_map_from_api(request: GoogleMapRequest) -> Tuple[bytes, CachedMap]: + """Get a static Map picture from the Google Maps Static API. + Return the acquired PNG picture as bytes, and the CachedMap object. The fetched picture is persisted on cache, running a fire & forget background task. References: @@ -65,21 +66,16 @@ async def get_map_from_api(request: GoogleMapRequest) -> bytes: image = (await google_maps_request(url=GOOGLE_MAPS_STATIC_API_URL, params=params)).content logger.debug("Map acquired from Google Static Maps API") - asyncio.create_task(save_cached_metadata(request=request, image=image)) - return image + cache_metadata = await save_cached_metadata(request=request, image=image, background=True) + return image, cache_metadata -async def get_map(request: GoogleMapRequest, read_cache_first: bool = True) -> bytes: +async def get_map(request: GoogleMapRequest, read_cache_first: bool = True) -> Tuple[bytes, CachedMap]: """Get a static Map picture from cache (if read_cache_first=True) or the Google Maps Static API. - Return the acquired PNG picture as bytes.""" - image = None - + Return the acquired PNG picture as bytes, and the CachedMap object.""" if read_cache_first: cached_metadata = await get_cached_metadata(request, fetch_image=True) if cached_metadata: - image = cached_metadata.image - - if image is None: - image = await get_map_from_api(request) + return cached_metadata.image, cached_metadata - return image + return await get_map_from_api(request) diff --git a/vigobusapi/services/google_maps/_getter_streetview.py b/vigobusapi/services/google_maps/_getter_streetview.py index ad1d2d6..1b77572 100644 --- a/vigobusapi/services/google_maps/_getter_streetview.py +++ b/vigobusapi/services/google_maps/_getter_streetview.py @@ -10,7 +10,7 @@ from vigobusapi.logger import logger from ._entities import GoogleStreetviewRequest from ._requester import google_maps_request, ListOfTuples -from ._cache import save_cached_metadata, get_cached_metadata +from ._cache import save_cached_metadata, get_cached_metadata, CachedMap __all__ = ("get_photo",) @@ -28,8 +28,9 @@ def _get_photo_params(request: GoogleStreetviewRequest) -> ListOfTuples: return params -async def get_photo_from_api(request: GoogleStreetviewRequest) -> Optional[bytes]: - """Get a static StreetView picture from the Google StreetView Static API. Return the acquired PNG picture as bytes. +async def get_photo_from_api(request: GoogleStreetviewRequest) -> Tuple[Optional[bytes], Optional[CachedMap]]: + """Get a static StreetView picture from the Google StreetView Static API. + Return the acquired PNG picture as bytes, and the CachedMap object. If the requested location does not have an available picture, returns None. The fetched picture is persisted on cache, running a fire & forget background task. @@ -43,28 +44,25 @@ async def get_photo_from_api(request: GoogleStreetviewRequest) -> Optional[bytes response = await google_maps_request(GOOGLE_STREETVIEW_STATIC_API_URL, params=params, expect_http_error=True) if response.status_code == 404: logger.debug("No StreetView picture available for the request") - return None + return None, None response.raise_for_status() image = response.content logger.debug("Photo acquired from Google StreetView Static API") - asyncio.create_task(save_cached_metadata(request=request, image=image)) - return image + cache_metadata = await save_cached_metadata(request=request, image=image, background=True) + return image, cache_metadata -async def get_photo(request: GoogleStreetviewRequest, read_cache_first: bool = True) -> Optional[bytes]: +async def get_photo( + request: GoogleStreetviewRequest, read_cache_first: bool = True +) -> Tuple[Optional[bytes], Optional[CachedMap]]: """Get a static StreetView picture from cache (if read_cache_first=True) or the Google StreetView Static API. - Return the acquired PNG picture as bytes. - If the requested location does not have an available picture, returns None.""" - image = None - + Return the acquired PNG picture as bytes, and the CachedMap object. + If the requested location does not have an available picture, returns (None, None).""" if read_cache_first: cached_metadata = await get_cached_metadata(request, fetch_image=True) if cached_metadata: - image = cached_metadata.image - - if image is None: - image = await get_photo_from_api(request) + return cached_metadata.image, cached_metadata - return image + return await get_photo_from_api(request) From 020ef240223715435c8775980c9fff5e4d7c986e Mon Sep 17 00:00:00 2001 From: David-Lor <17401854+david-lor@users.noreply.github.com> Date: Tue, 12 Oct 2021 15:38:27 +0200 Subject: [PATCH 5/6] Change routes order (swap maps and stops-buses routers) Avoid conflict with /stops/map endpoint being handled as a /stop/{stop_id} endpoint. Endpoints may end up being refactored in the future --- vigobusapi/routes/bootstrap.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vigobusapi/routes/bootstrap.py b/vigobusapi/routes/bootstrap.py index 1b568b6..d4d0ec0 100644 --- a/vigobusapi/routes/bootstrap.py +++ b/vigobusapi/routes/bootstrap.py @@ -14,5 +14,5 @@ def setup_routes(app: FastAPI): app.include_router(general_router) - app.include_router(stops_buses_router) app.include_router(maps_router) + app.include_router(stops_buses_router) From ea4a5231e76fbe466010586a893c08f3e9d5bf77 Mon Sep 17 00:00:00 2001 From: David-Lor <17401854+david-lor@users.noreply.github.com> Date: Tue, 12 Oct 2021 15:39:42 +0200 Subject: [PATCH 6/6] Add pytimeparse to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 176e453..8da0c5d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ cachetools==4.2.2 # TTLCache pymongo==3.10.1 # MongoDB client motor==2.1.0 # MongoDB Async client loguru==0.5.3 # Logging +pytimeparse==1.1.8 # Parse human duration strings into seconds