fix: Notion connector missing database properties fields (#490)

* Add optional description field to notion database properties

* Handle unsupported button db property

* Test notion db properties mappings

* Update changelog and version; Fix Notion connector missing database properties fields

* add htmlBuilder dependency to test dependencies

* add htmlBuilder dependency to test dependencies

* Remove duplicated description property

* Overwrite notion integration test fixtures

Author: mpolomdeepsense

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.0.22
+
+* **Fix Notion connector missing database properties fields**
+
 ## 1.0.21
 
 * **Fix Jira connector cloud option not working issue**

pyproject.toml

@@ -136,14 +136,14 @@ test = [
     "deepdiff",
     "bs4",
     "pandas",
-
     # Connector specific deps
     "cryptography",
     "fsspec",
     "vertexai",
     "pyiceberg",
     "pyarrow",
-    "networkx"
+    "networkx",
+    "htmlbuilder",
 ]
 # Add constraints needed for CI
 ci = [

test/integration/connectors/expected_results/notion_database/file_data/1722c3765a0a8082b382ebc2c62d3f4c.json

@@ -13,8 +13,8 @@
       "database_id": "1722c3765a0a8082b382ebc2c62d3f4c"
     },
     "date_created": "2025-01-05T18:34:00.000Z",
-    "date_modified": "2025-01-07T19:15:00.000Z",
-    "date_processed": "1736277913.3980532",
+    "date_modified": "2025-04-25T13:45:00.000Z",
+    "date_processed": "1745588835.1260726",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -24,7 +24,7 @@
       "object": "user"
     },
     "last_edited_by": {
-      "id": "118d872b-594c-8171-b46f-00020d10d8b2",
+      "id": "34c6c783-c141-44c9-930d-6f74dd8769c9",
       "object": "user"
     },
     "parent": {
@@ -34,6 +34,6 @@
     "url": "https://www.notion.so/1722c3765a0a8082b382ebc2c62d3f4c"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/h7/n848df9s5yn7ml8rxb61vhyc0000gp/T/tmpxu906ary/1722c3765a0a8082b382ebc2c62d3f4c.html",
+  "local_download_path": "/tmp/tmpopmj74lc/1722c3765a0a8082b382ebc2c62d3f4c.html",
   "display_name": null
 }

test/integration/connectors/expected_results/notion_page/downloads/1572c3765a0a806299f0dd6999f9e4c7.html

@@ -16,7 +16,7 @@
       <div>
         testtext2 testtext2 testtext2 testtext2 testtext2 testtext2 testtext2 testtext2 testtext2 testtext2
       </div>
-      <img src='https://prod-files-secure.s3.us-west-2.amazonaws.com/9e97f74a-ce4a-43ae-b704-4b8501948642/902effc2-1280-4e9c-92cc-77d940b24ac0/image.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45FSPPWI6X%2F20250107%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20250107T192520Z&X-Amz-Expires=3600&X-Amz-Signature=b6f7eb5d579e9412619e94adee500e627aea915f8f5d8a9383ddd9b832fe2098&X-Amz-SignedHeaders=host&x-id=GetObject'/>
+      <img src='https://prod-files-secure.s3.us-west-2.amazonaws.com/9e97f74a-ce4a-43ae-b704-4b8501948642/902effc2-1280-4e9c-92cc-77d940b24ac0/image.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=ASIAZI2LB466WLUW7QMG%2F20250425%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20250425T134727Z&X-Amz-Expires=3600&X-Amz-Security-Token=IQoJb3JpZ2luX2VjEJb%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaCXVzLXdlc3QtMiJHMEUCIEzyS69nxDRljZ06YqW%2FOmZ5O%2BDqX7%2BGapP7d%2BmEtS4vAiEAx3hj3IU3tEyLzKSItb%2BSBg7YbmlUpXSj9mnmCPazuJgq%2FwMILhAAGgw2Mzc0MjMxODM4MDUiDOB%2F9anCPuqKjN9CAircA%2Bj3YpyDwDTbmHqFJcdXi6kqOQnEOV%2FWRXvA0MMGE2BDpekp8c35XJrR9%2F8HgURE98viLioNEVPE1tL%2FHjggGePd1FNod5cYzW0ZAQSR6g4jvc4JFypzZUGCOKQjzRB98JfOoa0OZRscco8vdL3NxHyix9M8ULG6vPu%2BUSRZohSUtp75LUlRV%2FQv3rWUDqCfHS2yNFiaApcwFlXYuq%2B%2B2YDRWT7blDRArEnZAVlPerUb07vDpkdqI16YyP4nM%2FMAT79YcViVWsOGHtsXjqyOWKeW4T6XDx558YCudPVLXiJgg5vLqPrvI0pqnL4hHZ8urop0yTdGF2NZwyp4NORx7CUNCXiD%2B4NXsJvBWvHlQnQtM%2FLVDCjfxGYgE2k%2BHK7Jy4n%2Bm3bmnZbzPHSw5LNlXUE1Og%2Ft0sNcBaH4xLqpOJZfbCx36f8mt60zWRQXoR0t641oEoWcS9SWNFbtDs3EUTNq38M5KmBdZ18J6IGFcKSqLIhVXo9H4FAZL0gMETFvujo9ssAukwuweO4k11Eu1%2BrZpjd03Qv%2FTcRlyhhwLLeWjgTe6XslYRcMxSYsKWqgVjpf5jEvEfItcVURZmP40%2FY9Cha8LJp2eXyHL11WMPipugWGcdDIu3VS2YZGMMiZrsAGOqUBG5Id9hpzMlmW0EAc7YvSEBGxb%2BCkaR%2FkTslquCslUDdT92mTB2ss%2BDawd3afX4AlV8RX4mOcjIx41JEHEe9pGkGWOlLhkl7dmgfmiSZ0iaMEGZtSeeEu%2BAU4RDYJo4kqg%2F7vVSZDAeysJV1U%2Fn6brT9XPXwvV3lzPHda2egaX0g4OKBGzNyaT%2BPxsMjg6n2LHIfRKz%2BiXcBeDOov4gtjBvZMC2B%2B&X-Amz-Signature=45d59807b42ea635184e3f0ad03a3d659e019f1406673d062a74b3c3c4d81f1b&X-Amz-SignedHeaders=host&x-id=GetObject'/>
       <div>
         <ol style='margin-left: 0px'>
           <li>
@@ -137,7 +137,7 @@
           </td>
         </tr>
       </table>
-      <img src='https://prod-files-secure.s3.us-west-2.amazonaws.com/9e97f74a-ce4a-43ae-b704-4b8501948642/bef64626-dfad-4bf2-9486-0fe380e90e4f/image.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45FSPPWI6X%2F20250107%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20250107T192520Z&X-Amz-Expires=3600&X-Amz-Signature=f1decc7c99b1fe97c02246260df176e49ec040f48d00e619166cc6a41e33ce4e&X-Amz-SignedHeaders=host&x-id=GetObject'/>
+      <img src='https://prod-files-secure.s3.us-west-2.amazonaws.com/9e97f74a-ce4a-43ae-b704-4b8501948642/bef64626-dfad-4bf2-9486-0fe380e90e4f/image.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=ASIAZI2LB466WLUW7QMG%2F20250425%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20250425T134727Z&X-Amz-Expires=3600&X-Amz-Security-Token=IQoJb3JpZ2luX2VjEJb%2F%2F%2F%2F%2F%2F%2F%2F%2F%2FwEaCXVzLXdlc3QtMiJHMEUCIEzyS69nxDRljZ06YqW%2FOmZ5O%2BDqX7%2BGapP7d%2BmEtS4vAiEAx3hj3IU3tEyLzKSItb%2BSBg7YbmlUpXSj9mnmCPazuJgq%2FwMILhAAGgw2Mzc0MjMxODM4MDUiDOB%2F9anCPuqKjN9CAircA%2Bj3YpyDwDTbmHqFJcdXi6kqOQnEOV%2FWRXvA0MMGE2BDpekp8c35XJrR9%2F8HgURE98viLioNEVPE1tL%2FHjggGePd1FNod5cYzW0ZAQSR6g4jvc4JFypzZUGCOKQjzRB98JfOoa0OZRscco8vdL3NxHyix9M8ULG6vPu%2BUSRZohSUtp75LUlRV%2FQv3rWUDqCfHS2yNFiaApcwFlXYuq%2B%2B2YDRWT7blDRArEnZAVlPerUb07vDpkdqI16YyP4nM%2FMAT79YcViVWsOGHtsXjqyOWKeW4T6XDx558YCudPVLXiJgg5vLqPrvI0pqnL4hHZ8urop0yTdGF2NZwyp4NORx7CUNCXiD%2B4NXsJvBWvHlQnQtM%2FLVDCjfxGYgE2k%2BHK7Jy4n%2Bm3bmnZbzPHSw5LNlXUE1Og%2Ft0sNcBaH4xLqpOJZfbCx36f8mt60zWRQXoR0t641oEoWcS9SWNFbtDs3EUTNq38M5KmBdZ18J6IGFcKSqLIhVXo9H4FAZL0gMETFvujo9ssAukwuweO4k11Eu1%2BrZpjd03Qv%2FTcRlyhhwLLeWjgTe6XslYRcMxSYsKWqgVjpf5jEvEfItcVURZmP40%2FY9Cha8LJp2eXyHL11WMPipugWGcdDIu3VS2YZGMMiZrsAGOqUBG5Id9hpzMlmW0EAc7YvSEBGxb%2BCkaR%2FkTslquCslUDdT92mTB2ss%2BDawd3afX4AlV8RX4mOcjIx41JEHEe9pGkGWOlLhkl7dmgfmiSZ0iaMEGZtSeeEu%2BAU4RDYJo4kqg%2F7vVSZDAeysJV1U%2Fn6brT9XPXwvV3lzPHda2egaX0g4OKBGzNyaT%2BPxsMjg6n2LHIfRKz%2BiXcBeDOov4gtjBvZMC2B%2B&X-Amz-Signature=0e8cac29d83c138953bed589f69e160359037057306e043ecd1a5662b2087045&X-Amz-SignedHeaders=host&x-id=GetObject'/>
       <div>
         2 Columns in ColumnList
       </div>

test/integration/connectors/expected_results/notion_page/file_data/1572c3765a0a806299f0dd6999f9e4c7.json

@@ -14,7 +14,7 @@
     },
     "date_created": "2024-12-09T18:13:00.000Z",
     "date_modified": "2025-01-07T19:24:00.000Z",
-    "date_processed": "1736277919.434568",
+    "date_processed": "1745588845.2252257",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -34,6 +34,6 @@
     "url": "https://www.notion.so/test-doc1-1572c3765a0a806299f0dd6999f9e4c7"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/h7/n848df9s5yn7ml8rxb61vhyc0000gp/T/tmpluf__jry/1572c3765a0a806299f0dd6999f9e4c7.html",
+  "local_download_path": "/tmp/tmpb4ooe42f/1572c3765a0a806299f0dd6999f9e4c7.html",
   "display_name": null
 }

test/unit/connectors/notion/__init__.py

@@ -0,0 +1,105 @@
+import pytest
+import pytest_mock
+
+from unstructured_ingest.processes.connectors.notion.interfaces import DBCellBase, DBPropertyBase
+from unstructured_ingest.processes.connectors.notion.types.database_properties import (
+    db_cell_type_mapping,
+    db_prop_type_mapping,
+    map_cells,
+    map_properties,
+    unsupported_db_prop_types,
+)
+
+
+def test_map_properties_success(mocker: pytest_mock.MockerFixture):
+    mock_property = mocker.MagicMock(spec=DBPropertyBase)
+    mock_property.from_dict = mocker.MagicMock(return_value=mock_property)
+    mocker.patch.dict(db_prop_type_mapping, {"mock_type": mock_property})
+
+    props = {
+        "property1": {"type": "mock_type", "data": "value1"},
+        "property2": {"type": "mock_type", "data": "value2"},
+    }
+    result = map_properties(props)
+
+    assert len(result) == 2
+    assert "property1" in result
+    assert "property2" in result
+    mock_property.from_dict.assert_any_call({"type": "mock_type", "data": "value1"})
+    mock_property.from_dict.assert_any_call({"type": "mock_type", "data": "value2"})
+
+
+def test_map_properties_unsupported_type(caplog: pytest.LogCaptureFixture):
+    props = {
+        "property1": {"type": unsupported_db_prop_types[0], "data": "value1"},
+    }
+    result = map_properties(props)
+
+    assert result == {}
+    assert "Unsupported property type 'button' for property 'property1'. Skipping." in caplog.text
+
+
+def test_map_properties_key_error():
+    props = {
+        "property1": {"data": "value1"},  # Missing "type" key
+    }
+
+    with pytest.raises(KeyError, match="failed to map to associated database property"):
+        map_properties(props)
+
+
+def test_map_properties_invalid_type():
+    props = {
+        "property1": {"type": "non_existent_type", "data": "value1"},
+    }
+
+    with pytest.raises(KeyError, match="failed to map to associated database property"):
+        map_properties(props)
+
+
+def test_map_cells_success(mocker: pytest_mock.MockerFixture):
+    mock_property = mocker.MagicMock(spec=DBCellBase)
+    mock_property.from_dict = mocker.MagicMock(return_value=mock_property)
+    mocker.patch.dict(db_cell_type_mapping, {"mock_type": mock_property})
+
+    props = {
+        "property1": {"type": "mock_type", "data": "value1"},
+        "property2": {"type": "mock_type", "data": "value2"},
+    }
+
+    result = map_cells(props)
+
+    assert len(result) == 2
+    assert "property1" in result
+    assert "property2" in result
+    mock_property.from_dict.assert_any_call({"type": "mock_type", "data": "value1"})
+    mock_property.from_dict.assert_any_call({"type": "mock_type", "data": "value2"})
+
+
+def test_map_cells_unsupported_property():
+    props = {
+        "property1": {"type": "non_existent_type", "data": "value1"},
+    }
+
+    with pytest.raises(KeyError):
+        map_cells(props)
+
+
+def test_map_cells_key_error():
+    props = {
+        "property1": {"value": "MissingType"},
+    }
+
+    with pytest.raises(KeyError, match="failed to map to associated database property"):
+        map_cells(props)
+
+
+def test_map_cells_logs_warning_for_unsupported_type(caplog: pytest.LogCaptureFixture):
+    props = {
+        "property1": {"type": "button", "value": "Unsupported"},
+    }
+
+    result = map_cells(props)
+
+    assert result == {}
+    assert "Unsupported property type 'button' for property 'property1'. Skipping." in caplog.text

test/unit/connectors/notion/test__init__.py

@@ -1 +1 @@
-__version__ = "1.0.21"  # pragma: no cover
+__version__ = "1.0.22"  # pragma: no cover

unstructured_ingest/__version__.py

@@ -1,5 +1,6 @@
 from typing import Dict
 
+from unstructured_ingest.logger import logger
 from unstructured_ingest.processes.connectors.notion.interfaces import DBCellBase, DBPropertyBase
 
 from .checkbox import Checkbox, CheckboxCell
@@ -25,6 +26,13 @@
 from .url import URL, URLCell
 from .verification import Verification, VerificationCell
 
+# It's possible to add 'button' property to Notion database.
+# However, current Notion API documentation doesn't mention it.
+# Buttons are only functional inside Notion UI. We can simply
+# ignore them so that the we don't throw an error when trying
+# to map 'button' properties.
+unsupported_db_prop_types = ["button"]
+
 db_prop_type_mapping = {
     "checkbox": Checkbox,
     "created_by": CreatedBy,
@@ -55,7 +63,13 @@ def map_properties(props: Dict[str, dict]) -> Dict[str, DBPropertyBase]:
     mapped_dict = {}
     for k, v in props.items():
         try:
-            mapped_dict[k] = db_prop_type_mapping[v["type"]].from_dict(v)  # type: ignore
+            property_type = v["type"]
+            if property_type in unsupported_db_prop_types:
+                logger.warning(
+                    f"Unsupported property type '{property_type}' for property '{k}'. Skipping."
+                )
+                continue
+            mapped_dict[k] = db_prop_type_mapping[property_type].from_dict(v)  # type: ignore
         except KeyError as ke:
             raise KeyError(f"failed to map to associated database property -> {k}: {v}") from ke
 
@@ -92,8 +106,13 @@ def map_cells(props: Dict[str, dict]) -> Dict[str, DBCellBase]:
     mapped_dict = {}
     for k, v in props.items():
         try:
-            t = v["type"]
-            mapped_dict[k] = db_cell_type_mapping[t].from_dict(v)  # type: ignore
+            property_type = v["type"]
+            if property_type in unsupported_db_prop_types:
+                logger.warning(
+                    f"Unsupported property type '{property_type}' for property '{k}'. Skipping."
+                )
+                continue
+            mapped_dict[k] = db_cell_type_mapping[property_type].from_dict(v)  # type: ignore
         except KeyError as ke:
             raise KeyError(f"failed to map to associated database property -> {k}: {v}") from ke
 

unstructured_ingest/processes/connectors/notion/types/database_properties/__init__.py

@@ -13,6 +13,7 @@ class Checkbox(DBPropertyBase):
     id: str
     name: str
     type: str = "checkbox"
+    description: Optional[str] = None
     checkbox: dict = field(default_factory=dict)
 
     @classmethod