Merge pull request #1753 from weaviate/date-ms

handle date with nanoseconds

Author: dirkkul

test/test_util.py

@@ -1,6 +1,7 @@
 import unittest
 import uuid as uuid_lib
 from copy import deepcopy
+from datetime import datetime, timedelta, timezone
 from unittest.mock import Mock, patch
 
 import pytest
@@ -9,6 +10,7 @@
 from weaviate.exceptions import SchemaValidationException
 from weaviate.util import (
     MINIMUM_NO_WARNING_VERSION,
+    _datetime_from_weaviate_str,
     _is_sub_schema,
     _sanitize_str,
     generate_uuid5,
@@ -431,6 +433,37 @@ def test_is_weaviate_too_old(version: str, too_old: bool):
     assert is_weaviate_too_old(version) is too_old
 
 
+@pytest.mark.parametrize(
+    "input_str,expected",
+    [
+        # Test parsing with microseconds and Z timezone
+        (
+            "2023-01-15T14:30:45.123456Z",
+            datetime(2023, 1, 15, 14, 30, 45, 123456, tzinfo=timezone.utc),
+        ),
+        # Test parsing without microseconds
+        (
+            "2023-01-15T14:30:45Z",
+            datetime(2023, 1, 15, 14, 30, 45, 0, tzinfo=timezone.utc),
+        ),
+        # Test parsing with offset timezone
+        (
+            "2023-01-15T14:30:45.123456+02:00",
+            datetime(2023, 1, 15, 14, 30, 45, 123456, tzinfo=timezone(timedelta(hours=2))),
+        ),
+        # Test truncating excess microseconds
+        (
+            "2023-01-15T14:30:45.123456789Z",
+            datetime(2023, 1, 15, 14, 30, 45, 123456, tzinfo=timezone.utc),
+        ),
+        # Test handling year 0 (should return datetime.min)
+        ("0000-01-15T14:30:45.123456Z", datetime.min),
+    ],
+)
+def test_datetime_from_weaviate_str(input_str: str, expected: datetime) -> None:
+    assert _datetime_from_weaviate_str(input_str) == expected
+
+
 @pytest.mark.parametrize(
     "current_version,latest_version,too_old",
     [

weaviate/collections/queries/base_executor.py

@@ -270,15 +270,7 @@ def __deserialize_non_ref_prop(self, value: properties_pb2.Value) -> Any:
         if value.HasField("uuid_value"):
             return uuid_lib.UUID(value.uuid_value)
         if value.HasField("date_value"):
-            try:
-                return _datetime_from_weaviate_str(value.date_value)
-            except ValueError as e:
-                # note that the year 9999 is valid and does not need to be handled. for 5 digit years only the first
-                # 4 digits are considered and it wrapps around
-                if "year 0 is out of range" in str(e):
-                    _Warnings.datetime_year_zero(value.date_value)
-                    return datetime.datetime.min
-
+            return _datetime_from_weaviate_str(value.date_value)
         if value.HasField("string_value"):
             return str(value.string_value)
         if value.HasField("text_value"):

weaviate/util.py

@@ -716,16 +716,23 @@ def _datetime_to_string(value: TIME) -> str:
 
 
 def _datetime_from_weaviate_str(string: str) -> datetime.datetime:
+    if string[-1] != "Z":
+        string = "".join(string.rsplit(":", 1))
+
+    # Weaviate can return up to 9 digits for milliseconds, but Python datetime only supports 6 digits.
+    string = re.sub(r"(?<=\.\d{6})\d+(?=[Z+-])", "", string)
+    # pick format with or without microseconds
+    date_format = "%Y-%m-%dT%H:%M:%S.%f%z" if "." in string else "%Y-%m-%dT%H:%M:%S%z"
+
     try:
-        return datetime.datetime.strptime(
-            "".join(string.rsplit(":", 1) if string[-1] != "Z" else string),
-            "%Y-%m-%dT%H:%M:%S.%f%z",
-        )
-    except ValueError:  # if the string does not have microseconds
-        return datetime.datetime.strptime(
-            "".join(string.rsplit(":", 1) if string[-1] != "Z" else string),
-            "%Y-%m-%dT%H:%M:%S%z",
-        )
+        return datetime.datetime.strptime(string, date_format)
+    except ValueError as e:
+        # note that the year 9999 is valid and does not need to be handled. for 5 digit years only the first
+        # 4 digits are considered and it wrapps around
+        if "year 0 is out of range" in str(e):
+            _Warnings.datetime_year_zero(string)
+            return datetime.datetime.min
+        raise e
 
 
 class _WeaviateUUIDInt(uuid_lib.UUID):