Merge pull request #164 from lokalise/feature/PERF-1401_add_too_big_request_warning

PERF-1401 Add too big response warning

Author: bodrovis

docs/additional_info/changelog.rst

@@ -3,6 +3,11 @@
 Changelog
 =========
 
+3.5.1 (14-May-2025)
+-------------------
+
+* Added `_response_too_big` param in the `download_files` method for `Current contributor <https://developers.lokalise.com/reference/download-files>`_ endpoint
+
 3.5.0 (08-May-2025)
 -------------------
 

lokalise/_version.py

@@ -4,4 +4,4 @@
 This module contains plugin metadata.
 """
 
-__version__: str = "3.5.0"
+__version__: str = "3.5.1"

lokalise/client_methods/glossary_terms.py

@@ -13,8 +13,10 @@ class GlossaryTermsMethods(EndpointProviderMixin):
     """Glossary term client methods.
     """
 
-    def glossary_terms(self, project_id: str,
-                       params: Optional[Dict] = None) -> GlossaryTermsCollection:
+    def glossary_terms(
+            self,
+            project_id: str,
+            params: Optional[Dict] = None) -> GlossaryTermsCollection:
         """Fetches all glossary terms for the given project.
 
         :param str project_id: ID of the project
@@ -56,7 +58,8 @@ def create_glossary_terms(self,
 
     def update_glossary_terms(self,
                               project_id: str,
-                              params: Dict[str, Any]) -> GlossaryTermsCollection:
+                              params: Dict[str,
+                                           Any]) -> GlossaryTermsCollection:
         """Updates one or more glossary terms.
 
         :param str project_id: ID of the project

lokalise/endpoints/files_endpoint.py

@@ -3,10 +3,13 @@
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 Module containing files endpoint.
 """
+import warnings
 from typing import Dict, Union, Any, Optional
 from .base_endpoint import BaseEndpoint
 from .. import request
 
+warnings.formatwarning = lambda msg, *args, **kwargs: f"{msg}\n"
+
 
 class FilesEndpoint(BaseEndpoint):
     """Describes files endpoint.
@@ -33,7 +36,15 @@ def download(self, params: Dict[str, Any],
         :rtype dict:
         """
         path = self.path_with_params(**ids)
-        return request.post(self.client, path + 'download', params)
+
+        response = request.post(self.client, path + 'download', params)
+
+        if '_response_too_big' in response:
+            warnings.warn(
+                'Warning: Project is too big for sync export. Please use our async export function '
+                'instead (download_files_async)', UserWarning)
+
+        return response
 
     def download_async(self, params: Dict[str, Any],
                        **ids: Optional[Union[str, int]]) -> Dict:

lokalise/request.py

@@ -8,7 +8,7 @@
     :attribute str BASE_URL: path to the Lokalise APIv2.
     :attribute list PAGINATION_HEADERS: list of response headers that contain pagination data.
 """
-from typing import Optional, Dict
+from typing import Optional, Dict, Any
 import requests
 from lokalise.request_utils import raise_on_error, path_to_endpoint, __format_params
 import lokalise.client
@@ -124,18 +124,29 @@ def respond_with(response: requests.models.Response) -> Dict:
     return {**data, **extract_headers_from(response)}
 
 
-def extract_headers_from(response: requests.models.Response) -> Dict:
-    """Fetches pagination-related data from the response headers
-
-    :param response: Response from the API
-    :rtype dict:
+def extract_headers_from(response: requests.models.Response) -> Dict[str, Any]:
     """
-    return {
-        "_pagination": {
-            k.lower(): v for k,
-            v in response.headers.items() if k.lower() in PAGINATION_HEADERS
-        }
+    Pull pagination metadata (and the oversized flag) out of an HTTP response.
+
+    Returns a dict like:
+    {
+        "_pagination": { "x-pagination-page": "3", ... },
+        "_response_too_big": True   # only when x-response-too-big header is present
     }
+    """
+
+    headers_lower = {k.lower(): v for k, v in response.headers.items()}
+
+    pagination = {
+        k: v for k,
+        v in headers_lower.items() if k in PAGINATION_HEADERS}
+
+    result: Dict[str, Any] = {"_pagination": pagination}
+
+    if "x-response-too-big" in headers_lower:
+        result["_response_too_big"] = True
+
+    return result
 
 
 def options(client: lokalise.client.Client) -> Dict:

poetry.lock

@@ -1,6 +1,6 @@
 [project]
 name = "python-lokalise-api"
-version = "3.5.0"
+version = "3.5.1"
 description = "Official Python interface for the Lokalise API v2"
 authors = [
     {name = "Ilya Krukowski", email = "golosizpru@gmail.com"}

pyproject.toml

@@ -0,0 +1,60 @@
+interactions:
+- request:
+    body: '{"format": "json", "original_filenames": true, "replace_breaks": false}'
+    headers:
+      Accept:
+      - application/json
+      Accept-Encoding:
+      - gzip, deflate
+      Connection:
+      - keep-alive
+      Content-Length:
+      - '71'
+      User-Agent:
+      - python-lokalise-api plugin/0.0.1
+      x-api-token:
+      - FILTERED
+    method: POST
+    uri: https://api.lokalise.com/api2/projects/454087345e09f3e7e7eae3.57891254/files/download
+  response:
+    body:
+      string: '{"project_id":"454087345e09f3e7e7eae3.57891254","branch":"master","bundle_url":"https:\/\/s3-eu-west-1.amazonaws.com\/lokalise-assets\/files\/export\/454087345e09f3e7e7eae3.57891254\/aa2ea5d3b48a15f2069305b5e91c3e6c\/OnBoarding-locale.zip"}'
+    headers:
+      Access-Control-Allow-Headers:
+      - Content-Type
+      Access-Control-Allow-Origin:
+      - '*'
+      Cache-Control:
+      - max-age=0, must-revalidate, no-cache, no-store, private
+      Connection:
+      - keep-alive
+      Content-Type:
+      - application/json
+      Date:
+      - Fri, 19 Jun 2020 12:32:33 GMT
+      Expires:
+      - Fri, 19 Jun 2020 12:32:33 GMT
+      Pragma:
+      - no-cache
+      Server:
+      - nginx
+      Strict-Transport-Security:
+      - max-age=31536000
+      Transfer-Encoding:
+      - chunked
+      Vary:
+      - Accept-Encoding
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - deny
+      X-XSS-Protection:
+      - 1; mode=block
+      X-Response-Too-Big:
+      - 'true'
+      content-length:
+      - '240'
+    status:
+      code: 200
+      message: OK
+version: 1

tests/cassettes/files_test/test_download_files_too_big.yaml

@@ -65,6 +65,23 @@ def test_download_files(client):
     })
     assert response['project_id'] == PROJECT_ID
     assert r"amazonaws.com" in response['bundle_url']
+    assert '_response_too_big' not in response
+
+
+@pytest.mark.vcr
+def test_download_files_too_big(client):
+    """Tests files downloading with X-Response-Too-Big header
+    """
+    with pytest.warns(UserWarning, match="Project is too big for sync export"):
+        response = client.download_files(PROJECT_ID, {
+            "format": "json",
+            "original_filenames": True,
+            "replace_breaks": False
+        })
+
+    assert response['project_id'] == PROJECT_ID
+    assert r"amazonaws.com" in response['bundle_url']
+    assert '_response_too_big' in response
 
 
 @pytest.mark.vcr