feat: 1311 further optimize the pmtiles creation algorithm (#1378)

Author: jcpitre

functions-python/helpers/tests/test_transform.py

@@ -6,9 +6,9 @@
     to_boolean,
     get_nested_value,
     to_float,
-    get_safe_value,
-    get_safe_float,
-    get_safe_int,
+    get_safe_value_from_csv,
+    get_safe_float_from_csv,
+    get_safe_int_from_csv,
 )
 
 
@@ -94,82 +94,88 @@ def test_default_value(self):
 class TestGetSafeValue(unittest.TestCase):
     def test_valid_value(self):
         row = {"name": " Alice "}
-        self.assertEqual(get_safe_value(row, "name"), "Alice")
+        self.assertEqual(get_safe_value_from_csv(row, "name"), "Alice")
 
     def test_missing_column(self):
         row = {"age": 30}
-        self.assertIsNone(get_safe_value(row, "name"))
+        self.assertIsNone(get_safe_value_from_csv(row, "name"))
 
     def test_empty_string(self):
         row = {"name": "   "}
-        self.assertIsNone(get_safe_value(row, "name"))
+        self.assertIsNone(get_safe_value_from_csv(row, "name"))
 
     def test_nan_value(self):
         row = {"name": pd.NA}
-        self.assertIsNone(get_safe_value(row, "name"))
+        self.assertIsNone(get_safe_value_from_csv(row, "name"))
         row = {"name": float("nan")}
-        self.assertIsNone(get_safe_value(row, "name"))
+        self.assertIsNone(get_safe_value_from_csv(row, "name"))
 
     def test_default_value(self):
         row = {"name": ""}
         self.assertEqual(
-            get_safe_value(row, "name", default_value="default"), "default"
+            get_safe_value_from_csv(row, "name", default_value="default"), "default"
         )
 
 
 class TestGetSafeFloat(unittest.TestCase):
     def test_valid_float(self):
         row = {"value": "3.14"}
-        self.assertEqual(get_safe_float(row, "value"), 3.14)
+        self.assertEqual(get_safe_float_from_csv(row, "value"), 3.14)
         row = {"value": 2.5}
-        self.assertEqual(get_safe_float(row, "value"), 2.5)
+        self.assertEqual(get_safe_float_from_csv(row, "value"), 2.5)
         row = {"value": "0"}
-        self.assertEqual(get_safe_float(row, "value"), 0.0)
+        self.assertEqual(get_safe_float_from_csv(row, "value"), 0.0)
         row = {"value": 0}
-        self.assertEqual(get_safe_float(row, "value"), 0.0)
+        self.assertEqual(get_safe_float_from_csv(row, "value"), 0.0)
 
     def test_missing_column(self):
         row = {"other": 1.23}
-        self.assertIsNone(get_safe_float(row, "value"))
+        self.assertIsNone(get_safe_float_from_csv(row, "value"))
 
     def test_empty_string(self):
         row = {"value": "   "}
-        self.assertIsNone(get_safe_float(row, "value"))
+        self.assertIsNone(get_safe_float_from_csv(row, "value"))
 
     def test_nan_value(self):
         row = {"value": pd.NA}
-        self.assertIsNone(get_safe_float(row, "value"))
+        self.assertIsNone(get_safe_float_from_csv(row, "value"))
         row = {"value": float("nan")}
-        self.assertIsNone(get_safe_float(row, "value"))
+        self.assertIsNone(get_safe_float_from_csv(row, "value"))
 
     def test_invalid_float(self):
         row = {"value": "abc"}
-        self.assertIsNone(get_safe_float(row, "value"))
+        self.assertIsNone(get_safe_float_from_csv(row, "value"))
         row = {"value": None}
-        self.assertIsNone(get_safe_float(row, "value"))
+        self.assertIsNone(get_safe_float_from_csv(row, "value"))
 
     def test_default_value(self):
         row = {"value": ""}
-        self.assertEqual(get_safe_float(row, "value", default_value=1.23), 1.23)
+        self.assertEqual(
+            get_safe_float_from_csv(row, "value", default_value=1.23), 1.23
+        )
         row = {"value": "abc"}
-        self.assertEqual(get_safe_float(row, "value", default_value=4.56), 4.56)
+        self.assertEqual(
+            get_safe_float_from_csv(row, "value", default_value=4.56), 4.56
+        )
         row = {"value": None}
-        self.assertEqual(get_safe_float(row, "value", default_value=7.89), 7.89)
+        self.assertEqual(
+            get_safe_float_from_csv(row, "value", default_value=7.89), 7.89
+        )
 
 
 class TestGetSafeInt(unittest.TestCase):
     def test_valid_int(self):
         row = {"value": "42"}
-        self.assertEqual(get_safe_int(row, "value"), 42)
+        self.assertEqual(get_safe_int_from_csv(row, "value"), 42)
 
     def test_invalid_int(self):
         row = {"value": "abc"}
-        self.assertIsNone(get_safe_int(row, "value"))
+        self.assertIsNone(get_safe_int_from_csv(row, "value"))
 
     def test_missing_key(self):
         row = {}
-        self.assertIsNone(get_safe_int(row, "value"))
+        self.assertIsNone(get_safe_int_from_csv(row, "value"))
 
     def test_empty_string(self):
         row = {"value": ""}
-        self.assertIsNone(get_safe_int(row, "value"))
+        self.assertIsNone(get_safe_int_from_csv(row, "value"))

functions-python/helpers/transform.py

@@ -89,38 +89,70 @@ def to_float(value, default_value: Optional[float] = None) -> Optional[float]:
         return default_value
 
 
-def get_safe_value(row, column_name, default_value: str = None) -> Optional[str]:
+def get_safe_value_from_csv(
+    row, column_name, default_value: str = None
+) -> Optional[str]:
     """
-    Get a safe value from the row. If the value is missing or empty, return the default value.
+    Get a safe value from a csv row. If the value is missing or empty, return the default value.
+    """
+
+    raw_value = row.get(column_name, None)
+
+    return get_safe_value(raw_value, default_value)
+
+
+def get_safe_value(raw_value, default_value: str = None) -> Optional[str]:
+    """
+    Get a safe value. If the value is missing or empty, return the default value.
     """
     import pandas
 
-    value = row.get(column_name, None)
     if (
-        value is None
-        or pandas.isna(value)
-        or (isinstance(value, str) and value.strip() == "")
+        raw_value is None
+        or pandas.isna(raw_value)
+        or (isinstance(raw_value, str) and raw_value.strip() == "")
     ):
         return default_value
-    return f"{value}".strip()
+    return f"{raw_value}".strip()
+
+
+def get_safe_float_from_csv(
+    row, column_name, default_value: float = None
+) -> Optional[float]:
+    """
+    Get a safe float value from a csv row.
+    Use the default value if the value is missing or cannot be converted to float.
+    """
+    raw_value = row.get(column_name, None)
 
+    return get_safe_float(raw_value, default_value)
 
-def get_safe_float(row, column_name, default_value: float = None) -> Optional[float]:
+
+def get_safe_float(raw_value, default_value: float = None) -> Optional[float]:
     """
-    Get a safe float value from the row. If the value is missing or cannot be converted to float.
+    Get a safe float value. Use the default value if the value is missing or cannot be converted to float.
     """
-    safe_value = get_safe_value(row, column_name)
+    safe_value = get_safe_value(raw_value, default_value)
     try:
         return float(safe_value)
     except (ValueError, TypeError):
         return default_value
 
 
-def get_safe_int(row, column_name, default_value: int = None) -> Optional[int]:
+def get_safe_int_from_csv(row, column_name, default_value: int = None) -> Optional[int]:
+    """
+    Get a safe int value from a csv row. Use the default value if the value is missing or cannot be converted to int.
+    """
+    raw_value = row.get(column_name, None)
+
+    return get_safe_int(raw_value, default_value)
+
+
+def get_safe_int(raw_value, default_value: int = None) -> Optional[int]:
     """
-    Get a safe int value from the row. If the value is missing or cannot be converted to int.
+    Get a safe int value. Use the default value if the value is missing or cannot be converted to int.
     """
-    safe_value = get_safe_value(row, column_name)
+    safe_value = get_safe_value(raw_value, default_value)
     try:
         return int(safe_value)
     except (ValueError, TypeError):

functions-python/helpers/utils.py

@@ -363,7 +363,9 @@ def detect_encoding(
     enc = result.encoding.lower()
 
     # If UTF-8 is detected, always use utf-8-sig to strip BOM if present
-    if enc in ("utf_8", "utf-8", "utf8", "utf8mb4"):
+    # Treat ascii as UTF-8, since it's a subset of UTF-8 and it will prevent errors where UTF-8 characters are present
+    # after the first 100K characters of the file.
+    if enc in ("ascii", "utf_8", "utf-8", "utf8", "utf8mb4"):
         return "utf-8-sig"
 
     return enc

functions-python/pmtiles_builder/.coveragerc

@@ -7,4 +7,5 @@ omit =
 
 [report]
 exclude_lines =
-    if __name__ == .__main__.:
+    pragma: no cover
+    if __name__ == .__main__.:

functions-python/pmtiles_builder/function_config.json

@@ -2,14 +2,17 @@
   "name": "pmtiles-builder",
   "description": "The PMTiles Builder function creates PMTiles from dataset files",
   "entry_point": "build_pmtiles_handler",
-  "timeout": 1000,
+  "timeout": 1680,
   "memory": "8Gi",
   "trigger_http": true,
   "include_folders": ["helpers"],
   "include_api_folders": ["database_gen", "database", "common"],
   "environment_variables": [
     {
       "key": "DATASETS_BUCKET_NAME"
+    },
+    {
+      "key": "LOGGING_LEVEL"
     }
   ],
   "secret_environment_variables": [

functions-python/pmtiles_builder/requirements.txt

@@ -27,4 +27,7 @@ python-dotenv==1.0.0
 tippecanoe
 psutil
 pandas
+numpy
+pympler
+
 charset_normalizer