Normalize multipart form data, fix imports

Author: GeoSander

pygeoapi/api/__init__.py

@@ -50,6 +50,7 @@
 import re
 import sys
 from typing import Any, Tuple, Union, Self
+from io import BytesIO
 
 from babel import Locale
 from dateutil.parser import parse as dateparse
@@ -64,9 +65,8 @@
 from pygeoapi.provider.base import (
     ProviderConnectionError, ProviderGenericError, ProviderTypeError)
 
-from pygeoapi.join_util import initialize_joins
 from pygeoapi.util import (
-    TEMPLATESDIR, UrlPrefetcher, dategetter,
+    TEMPLATESDIR, UrlPrefetcher, dategetter, FileObject,
     filter_dict_by_key_value, filter_providers_by_type, get_api_rules,
     get_base_url, get_provider_by_type, get_provider_default, get_typed_value,
     render_j2_template, to_json, get_choice_from_headers, get_from_headers
@@ -214,12 +214,12 @@ def __init__(self, request, supported_locales):
         # Set default request data
         self._data = b''
 
-        # Form data
-        self._form = {}
-
         # Copy request query parameters
         self._args = self._get_params(request)
 
+        # Form data: populate in from_* factory methods
+        self._form = {}
+
         # Get path info
         if hasattr(request, 'scope'):
             self._path_info = request.scope['path'].strip('/')
@@ -243,13 +243,12 @@ def from_flask(cls, request, supported_locales) -> 'APIRequest':
         """Factory class similar to with_data, but only for flask requests"""
         api_req = cls(request, supported_locales)
         api_req._data = request.data
-        # TODO: quick hack to retrieve multipart form data
-        if hasattr(request, 'form'):
-            for key, value in request.form.items():
-                api_req._form[key] = value
-        if hasattr(request, 'files'):
-            for key, value in request.files.items():
-                api_req._form[key] = value
+        for key, value in cls._formdata_flask(request):
+            LOGGER.debug(f"Setting form field '{key}'")
+            if key in api_req._form:
+                LOGGER.debug(f"Skipping duplicate form field '{key}'")
+                continue
+            api_req._form[key] = value
         return api_req
 
     @classmethod
@@ -258,15 +257,66 @@ async def from_starlette(cls, request, supported_locales) -> 'APIRequest':
         """
         api_req = cls(request, supported_locales)
         api_req._data = await request.body()
+        async for key, value in cls._formdata_starlette(request):
+            LOGGER.debug(f"Setting form field '{key}'")
+            if key in api_req._form:
+                LOGGER.debug(f"Skipping duplicate form field '{key}'")
+                continue
+            api_req._form[key] = value
         return api_req
 
     @classmethod
     def from_django(cls, request, supported_locales) -> 'APIRequest':
         """Factory class similar to with_data, but only for django requests"""
         api_req = cls(request, supported_locales)
         api_req._data = request.body
+        for key, value in cls._formdata_django(request):
+            LOGGER.debug(f"Setting form field '{key}'")
+            if key in api_req._form:
+                LOGGER.debug(f"Skipping duplicate form field '{key}'")
+                continue
+            api_req._form[key] = value
         return api_req
 
+    @staticmethod
+    def _formdata_flask(request):
+        """ Normalize Flask/Werkzeug form data. """
+
+        for key, value in getattr(request, 'form', {}).items():
+            yield key, value
+
+        for key, file_obj in getattr(request, 'files', {}).items():
+            yield key, FileObject(file_obj.filename, file_obj.content_type,
+                                  BytesIO(file_obj.read()))
+
+    @staticmethod
+    def _formdata_django(request):
+        """ Normalize Django form data. """
+
+        for key, value in getattr(request, 'POST', {}).items():
+            yield key, value
+
+        for key, file_obj in getattr(request, 'FILES', {}).items():
+            yield key, FileObject(file_obj.name, file_obj.content_type,
+                                  BytesIO(file_obj.read()))
+
+    @staticmethod
+    async def _formdata_starlette(request):
+        """ Normalize Starlette/FastAPI form data (async). """
+
+        form = await request.form()
+
+        for key, value in form.items():
+            if hasattr(value, 'filename'):
+                # It's a file: for Starlette, we need to read async
+                content = await value.read()
+                file_obj = FileObject(value.filename, value.content_type,
+                                      BytesIO(content))
+                yield key, file_obj
+            else:
+                # Regular form field
+                yield key, value
+
     @staticmethod
     def _get_params(request):
         """
@@ -364,7 +414,7 @@ def data(self) -> bytes:
 
     @property
     def form(self) -> dict:
-        """Returns the Request form data dict"""
+        """Returns the Request form data dict (multipart/form-data)"""
         return self._form
 
     @property
@@ -556,8 +606,13 @@ def __init__(self, config: dict, openapi: dict) -> Self | None:
         self.base_url = get_base_url(self.config)
         self.prefetcher = UrlPrefetcher()
 
-        # Build reference cache of join tables already/still on the server
-        initialize_joins(config)
+        setup_logger(self.config['logging'])
+
+        joins_api = all_apis().get('joins')
+        if joins_api:
+            # Initialize OGC API - Joins:
+            # build reference cache of join tables already/still on the server
+            joins_api.init(config)
 
         CHARSET[0] = config['server'].get('encoding', 'utf-8')
         if config['server'].get('gzip'):
@@ -576,8 +631,6 @@ def __init__(self, config: dict, openapi: dict) -> Self | None:
 
         self.pretty_print = self.config['server']['pretty_print']
 
-        setup_logger(self.config['logging'])
-
         # Create config clone for HTML templating with modified base URL
         self.tpl_config = deepcopy(self.config)
         self.tpl_config['server']['url'] = self.base_url

pygeoapi/api/itemtypes.py

@@ -47,8 +47,7 @@
 from pygeofilter.parsers.cql2_json import parse as parse_cql2_json
 from pyproj.exceptions import CRSError
 
-import join_util
-from pygeoapi import l10n
+from pygeoapi import l10n, join_util
 from pygeoapi.api import evaluate_limit
 from pygeoapi.crs import (DEFAULT_CRS, DEFAULT_STORAGE_CRS,
                           create_crs_transform_spec, get_supported_crs_list,
@@ -178,7 +177,7 @@ def get_collection_queryables(api: API, request: Union[APIRequest, Any],
         try:
             domains, _ = p.get_domains(properties)
         except NotImplementedError:
-            LOGGER.debug('Domains are not suported by this provider')
+            LOGGER.debug('Domains are not supported by this provider')
             domains = {}
 
     for k, v in p.fields.items():

pygeoapi/api/joins.py

@@ -29,18 +29,17 @@
 import logging
 from typing import Any
 
-import util
 from pygeoapi import l10n, join_util
+from pygeoapi.api import (
+    APIRequest, API, SYSTEM_LOCALE, FORMAT_TYPES,
+    F_JSON, F_JSONLD, F_HTML, HTTPStatus
+)
+from pygeoapi.openapi import get_visible_collections
 from pygeoapi.plugin import load_plugin, PLUGINS
 from pygeoapi.provider.base import ProviderTypeError
 from pygeoapi.util import (
     get_provider_by_type, to_json, filter_providers_by_type,
-    filter_dict_by_key_value
-)
-from pygeoapi.openapi import get_visible_collections
-from pygeoapi.api import (
-    APIRequest, API, SYSTEM_LOCALE, FORMAT_TYPES,
-    F_JSON, F_JSONLD, F_HTML, HTTPStatus
+    filter_dict_by_key_value, get_current_datetime
 )
 
 LOGGER = logging.getLogger(__name__)
@@ -69,6 +68,17 @@
 ]
 
 
+def init(cfg: dict) -> bool:
+    """
+    Shortcut to initialize join utility with config.
+
+    :param cfg: pygeoapi configuration dict
+
+    :returns: True if the join utility was initialized successfully
+    """
+    return join_util.init(cfg)
+
+
 def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str, dict]]:  # noqa
     """
     Get OpenAPI fragments
@@ -81,7 +91,7 @@ def get_oas_30(cfg: dict, locale: str) -> tuple[list[dict[str, str]], dict[str,
 
     paths = {}
 
-    if not join_util.CONFIG.enabled:
+    if not join_util.init(cfg):
         LOGGER.info('OpenAPI: skipping OGC API - Joins endpoints setup')
         return [], {'paths': paths}
 
@@ -692,18 +702,19 @@ def list_joins(api: API, request: APIRequest, dataset: str) -> tuple[dict, int,
 
     try:
         sources = join_util.list_sources(dataset)
-    except KeyError:
-        return api.get_exception(
-            HTTPStatus.NOT_FOUND, headers, request.format,
-            'NotFound', 'Collection not found'
-        )
     except Exception as e:
         LOGGER.error(str(e), exc_info=True)
         return api.get_exception(
             HTTPStatus.INTERNAL_SERVER_ERROR, headers, request.format,
             'NoApplicableCode', str(e)
         )
 
+    if not sources:
+        return api.get_exception(
+            HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound',
+            f'No joins found for collection: {dataset}'
+        )
+
     # Build the joins list with proper structure
     joins_list = []
     for source_id, source_obj in sources:
@@ -741,7 +752,7 @@ def list_joins(api: API, request: APIRequest, dataset: str) -> tuple[dict, int,
         'joins': joins_list,
         'numberMatched': len(joins_list),
         'numberReturned': len(joins_list),
-        'timeStamp': util.get_current_datetime()
+        'timeStamp': get_current_datetime()
     }
 
     return headers, HTTPStatus.OK, to_json(response, api.pretty_print)
@@ -907,18 +918,16 @@ def delete_join(api: API, request: APIRequest,
         )
 
     try:
-        join_util.remove_source(dataset, join_id)
+        if not join_util.remove_source(dataset, join_id):
+            msg = f'Join source {join_id} not found for collection {dataset}'
+            return api.get_exception(
+                HTTPStatus.NOT_FOUND, headers, request.format,
+                'NotFound', msg)
     except ValueError as e:
         LOGGER.error(f'Invalid request parameter: {e}', exc_info=True)
         return api.get_exception(
             HTTPStatus.BAD_REQUEST, headers, request.format,
             'InvalidParameterValue', str(e))
-    except KeyError as e:
-        msg = 'Collection or join source not found'
-        LOGGER.error(f'Invalid parameter value: {e}', exc_info=True)
-        return api.get_exception(
-            HTTPStatus.NOT_FOUND, headers, request.format,
-            'NotFound', msg)
     except Exception as e:
         LOGGER.error(f'Failed to delete join source: {e}',
                      exc_info=True)