Update string datapoint constraints to be in line with API changes

CHANGELOG.md

@@ -12,6 +12,12 @@ Changes are grouped as follows
 - `Fixed` for any bug fixes.
 - `Security` in case of vulnerabilities.
 
+## 7.9.0
+
+### Changed
+
+* Updated string time series datapoints maximum size to be constrained by UTF-8 encoded bytes count instead of string length when using upload queue(s), aligning with the API behavior changes.
+
 ## 7.8.1
 
 ### Fixed

cognite/extractorutils/__init__.py

@@ -16,7 +16,7 @@
 Cognite extractor utils is a Python package that simplifies the development of new extractors.
 """
 
-__version__ = "7.8.1"
+__version__ = "7.9.0"
 from .base import Extractor
 
 __all__ = ["Extractor"]

cognite/extractorutils/uploader/time_series.py

@@ -53,7 +53,7 @@
 from cognite.extractorutils.util import EitherId, cognite_exceptions, retry
 
 MIN_DATAPOINT_TIMESTAMP = -2208988800000
-MAX_DATAPOINT_STRING_LENGTH = 255
+MAX_DATAPOINT_STRING_BYTES = 1023
 MAX_DATAPOINT_VALUE = 1e100
 MIN_DATAPOINT_VALUE = -1e100
 
@@ -154,7 +154,7 @@ def _verify_datapoint_value(self, value: int | float | datetime | str) -> bool:
                 math.isnan(value) or math.isinf(value) or value > MAX_DATAPOINT_VALUE or value < MIN_DATAPOINT_VALUE
             )
         elif isinstance(value, str):
-            return len(value) <= MAX_DATAPOINT_STRING_LENGTH
+            return len(value.encode("utf-8")) <= MAX_DATAPOINT_STRING_BYTES
         return not isinstance(value, datetime)
 
     def _is_datapoint_valid(

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "cognite-extractor-utils"
-version = "7.8.1"
+version = "7.9.0"
 description = "Utilities for easier development of extractors for CDF"
 authors = [
     {name = "Mathias Lohne", email = "mathias.lohne@cognite.com"}

tests/tests_integration/test_cdm_time_series_integration.py

@@ -15,12 +15,13 @@
 from cognite.client.data_classes.data_modeling import NodeApply, NodeId
 from cognite.client.data_classes.data_modeling.extractor_extensions.v1 import CogniteExtractorTimeSeriesApply
 from cognite.client.exceptions import CogniteAPIError, CogniteNotFoundError
-from cognite.extractorutils.uploader.time_series import CDMTimeSeriesUploadQueue
-
-MIN_DATAPOINT_TIMESTAMP = -2208988800000
-MAX_DATAPOINT_STRING_LENGTH = 255
-MAX_DATAPOINT_VALUE = 1e100
-MIN_DATAPOINT_VALUE = -1e100
+from cognite.extractorutils.uploader.time_series import (
+    MAX_DATAPOINT_STRING_BYTES,
+    MAX_DATAPOINT_VALUE,
+    MIN_DATAPOINT_TIMESTAMP,
+    MIN_DATAPOINT_VALUE,
+    CDMTimeSeriesUploadQueue,
+)
 
 
 @pytest.fixture
@@ -197,7 +198,7 @@ def test_cdm_queue_discards_invalid_values(set_upload_test: tuple[CogniteClient,
     bad_val_inf = (now + 1_000, float("inf"))
     bad_val_max = (now + 1_000, MAX_DATAPOINT_VALUE * 10)
     bad_val_min = (now + 1_000, MIN_DATAPOINT_VALUE * 10)
-    too_long_str = (now + 2_000, "x" * (MAX_DATAPOINT_STRING_LENGTH + 1))
+    too_long_str = (now + 2_000, "x" * (MAX_DATAPOINT_STRING_BYTES + 1))
     valid_temp_str_dp = (now + 3_000, "valid_short_string")
 
     queue.add_to_upload_queue(

tests/tests_unit/test_cdf_upload_queues.py

@@ -29,6 +29,7 @@
     SequenceUploadQueue,
     TimeSeriesUploadQueue,
 )
+from cognite.extractorutils.uploader.time_series import MAX_DATAPOINT_STRING_BYTES
 
 
 @patch("cognite.client.CogniteClient")
@@ -151,7 +152,10 @@ def post(_x: Any) -> None:
     queue.add_to_upload_queue(
         id=1, datapoints=[(start + 5, 5), (start + 1, math.inf), (start + 2, -math.inf), (start + 6, 6)]
     )
-    queue.add_to_upload_queue(id=3, datapoints=[(start + 7, "str1"), (start + 9, ("t" * 300)), (start + 8, "str2")])
+    queue.add_to_upload_queue(
+        id=3,
+        datapoints=[(start + 7, "str1"), (start + 9, ("t" * (MAX_DATAPOINT_STRING_BYTES + 1))), (start + 8, "str2")],
+    )
 
     time.sleep(2.1)