Fix datetime deprecation warnings (#815)

* Fix datetime deprecation warnings

* Remove ndjson to appease mypy

* Add more typing to kafka generator and time

* Appease black

* Update changelog

* Silence mypy for confluent_kafka import

* Add more typing for confluent_kafka

Author: ppcad

CHANGELOG.md

@@ -21,6 +21,7 @@
 - Fixed assign_callback error in confluentkafka input
 - Fixed error logging in ` _get_configuration`, which caused the github checks to fail
 - Fix domain resolver errors for invalid domains
+- Fixed deprecation warnings caused by datetime when using Python >= 3.12
 
 ## 16.1.0
 ### Deprecations

logprep/generator/kafka/document_loader.py

@@ -1,10 +1,9 @@
 """For loading documents from Kafka or from file and preparing them for sending"""
 
 import json
-from datetime import datetime
+from datetime import datetime, UTC
 from logging import Logger
-
-import ndjson
+from typing import Any
 
 from logprep.generator.kafka.configuration import Configuration
 from logprep.generator.kafka.kafka_connector import KafkaConsumer
@@ -20,13 +19,13 @@ def __init__(self, config: Configuration, logger: Logger):
         self._timeout = config.kafka.consumer.timeout
         self._kafka_consumer = KafkaConsumer(config.kafka)
 
-    def _get_from_file(self) -> list:
-        with self._source_file.open("r", encoding="utf-8") as input_docs:
-            input_docs = ndjson.load(input_docs)
+    def _get_from_file(self) -> list[dict[Any, Any]]:
+        with self._source_file.open("r", encoding="utf-8") as docs_file:
+            input_docs: list[dict[Any, Any]] = [json.loads(doc) for doc in docs_file.readlines()]
             self._logger.info(f"Loaded {len(input_docs)} documents")
             return input_docs
 
-    def _get_from_kafka(self) -> list:
+    def _get_from_kafka(self) -> list[dict[Any, Any]]:
         documents = []
         cnt_invalid = 0
         for _ in range(self._source_count):
@@ -47,32 +46,32 @@ def _get_from_kafka(self) -> list:
         self._logger.info(f"Fetched {len(documents)} documents")
         return documents
 
-    def get_documents(self) -> list:
+    def get_documents(self) -> list[str]:
         """Get documents from Kafka format them so a unique value can be added easily"""
-        docs = self._get_raw_documents()
+        docs_json = self._get_raw_documents()
 
-        self._prepare_json_docs(docs)
-        docs = ndjson.dumps(docs).splitlines()
+        self._prepare_json_docs(docs_json)
+        docs = [json.dumps(doc) for doc in docs_json]
         self._prepare_string_addition_of_additional_fields(docs)
         return docs
 
-    def _get_raw_documents(self):
+    def _get_raw_documents(self) -> list[dict[Any, Any]]:
         return self._get_from_file() if self._source_file else self._get_from_kafka()
 
     @staticmethod
-    def _prepare_json_docs(docs: list, index_name="load-tester"):
+    def _prepare_json_docs(docs: list[dict[Any, Any]], index_name: str = "load-tester") -> None:
         for doc in docs:
             doc["_index"] = index_name
             doc["tags"] = ["load-tester"]
-            doc["@timestamp"] = f"{datetime.utcnow().isoformat()}Z"
+            doc["@timestamp"] = datetime.now(UTC).isoformat()
 
     @staticmethod
-    def _prepare_string_addition_of_additional_fields(docs: list):
+    def _prepare_string_addition_of_additional_fields(docs: list[str]) -> None:
         """Prepare document for unique value that will be added on sending"""
         for idx, doc in enumerate(docs):
             if '"' in doc and doc.endswith("}"):
                 docs[idx] = f'{doc[:-1]}, "load-tester-unique": "'
 
-    def shut_down(self):
+    def shut_down(self) -> None:
         """Shut down the Kafka consumer gracefully"""
         self._kafka_consumer.shut_down()

logprep/generator/kafka/kafka_connector.py

@@ -1,9 +1,9 @@
 """For retrieval and insertion of data from and into Kafka."""
 
 import json
-from typing import Optional
+from typing import Any
 
-from confluent_kafka import Consumer, Producer
+from confluent_kafka import Consumer, Producer  # type: ignore[import-untyped]
 
 from logprep.generator.kafka.configuration import Kafka
 
@@ -35,15 +35,15 @@ def __init__(self, config: Kafka):
 
         self._producer = Producer(self._config)
 
-    def store(self, document: str):
+    def store(self, document: str) -> None:
         """Write document into Kafka"""
         try:
             self._producer.produce(self._producer_topic, value=document)
             self._producer.poll(0)
         except BufferError:
             self._producer.flush(timeout=self._flush_timeout)
 
-    def shut_down(self):
+    def shut_down(self) -> None:
         """Gracefully close Kafka producer"""
         if self._producer is not None:
             self._producer.flush(self._flush_timeout)
@@ -76,7 +76,7 @@ def __init__(self, config: Kafka):
         self._consumer = Consumer(self._config)
         self._consumer.subscribe([config.consumer.topic])
 
-    def get(self, timeout: float) -> Optional[str]:
+    def get(self, timeout: float) -> Any:
         """Get document from Kafka"""
         record = self._consumer.poll(timeout=timeout)
         if not record:
@@ -86,7 +86,7 @@ def get(self, timeout: float) -> Optional[str]:
             raise record.error()
         return json.loads(record.value().decode("utf-8"))
 
-    def shut_down(self):
+    def shut_down(self) -> None:
         """Gracefully close Kafka consumer"""
         if self._consumer is not None:
             self._consumer.close()

logprep/util/time.py

@@ -1,13 +1,10 @@
 """logprep time helpers module"""
 
-from datetime import datetime, tzinfo
+from datetime import datetime, tzinfo, UTC
 from typing import Union
-from zoneinfo import ZoneInfo
 
 from logprep.abc.exceptions import LogprepException
 
-UTC = ZoneInfo("UTC")
-
 
 class TimeParserException(LogprepException):
     """exception class for time parsing"""
@@ -54,7 +51,7 @@ def from_timestamp(cls, timestamp: Union[int, float]) -> datetime:
         datetime
             datetime object
         """
-        time_object = datetime.utcfromtimestamp(timestamp)
+        time_object = datetime.fromtimestamp(timestamp, tz=UTC)
         time_object = cls._set_utc_if_timezone_is_missing(time_object)
         return time_object
 
@@ -108,7 +105,7 @@ def from_format(cls, source: str, format_str: str, set_missing_utc: bool = True)
             raise TimeParserException(str(error)) from error
 
     @classmethod
-    def _set_utc_if_timezone_is_missing(cls, time_object):
+    def _set_utc_if_timezone_is_missing(cls, time_object: datetime) -> datetime:
         if time_object.tzinfo is None:
             time_object = time_object.replace(tzinfo=UTC)
         return time_object

pyproject.toml

@@ -84,7 +84,6 @@ dependencies = [
   "msgspec",
   "boto3",
   "pydantic",
-  "ndjson",
   "click",
   "pandas",
   "tabulate",

tests/unit/generator/kafka/test_document_loader.py

@@ -119,7 +119,7 @@ def test_get_documents_source_empty_dict(self):
         for document in documents:
             assert re.match(
                 r'{"_index": "load-tester", "tags": \["load-tester"\], '
-                r'"@timestamp": "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z", '
+                r'"@timestamp": "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+00:00", '
                 r'"load-tester-unique": "',
                 document,
             )
@@ -133,7 +133,7 @@ def test_get_documents_source_multiple_times(self):
         for document in documents:
             assert re.match(
                 r'{"bar": "2", "_index": "load-tester", "tags": \["load-tester"\], '
-                r'"@timestamp": "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z", '
+                r'"@timestamp": "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+00:00", '
                 r'"load-tester-unique": "',
                 document,
             )
@@ -143,7 +143,7 @@ def test_get_documents_source_multiple_times(self):
         for document in documents:
             assert re.match(
                 r'{"foo": "1", "_index": "load-tester", "tags": \["load-tester"\], '
-                r'"@timestamp": "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z", '
+                r'"@timestamp": "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+00:00", '
                 r'"load-tester-unique": "',
                 document,
             )
@@ -159,7 +159,7 @@ def test_get_documents_multiple_source(self):
         for document in documents:
             assert re.match(
                 r'{"foo": "bar", "_index": "load-tester", "tags": \["load-tester"\], '
-                r'"@timestamp": "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z", '
+                r'"@timestamp": "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+00:00", '
                 r'"load-tester-unique": "',
                 document,
             )
@@ -175,7 +175,7 @@ def test_get_documents_more_source_than_available(self):
         for document in documents:
             assert re.match(
                 r'{"foo": "bar", "_index": "load-tester", "tags": \["load-tester"\], '
-                r'"@timestamp": "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z", '
+                r'"@timestamp": "\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+\+00:00", '
                 r'"load-tester-unique": "',
                 document,
             )

tests/unit/util/test_time.py

@@ -1,7 +1,7 @@
 # pylint: disable=missing-docstring
 # pylint: disable=protected-access
 # pylint: disable=too-many-arguments
-from datetime import datetime
+from datetime import datetime, UTC
 from zoneinfo import ZoneInfo
 
 import pytest
@@ -60,21 +60,21 @@ def test_from_format_returns(self, source, format_str, expected):
         for attribute, value in expected.items():
             assert getattr(timestamp, attribute) == value
 
-    @pytest.mark.parametrize("timezone", [None, ZoneInfo("UTC"), ZoneInfo("Europe/Berlin")])
+    @pytest.mark.parametrize("timezone", [None, UTC, ZoneInfo("Europe/Berlin")])
     def test_has_utc_if_timezone_was_set(self, timezone):
         datetime_time = datetime.now(timezone)
         time_parser_time = TimeParser.now(timezone)
         assert time_parser_time.second == pytest.approx(datetime_time.second, abs=1)
         if timezone is None:
-            assert time_parser_time.tzinfo == ZoneInfo("UTC")
+            assert time_parser_time.tzinfo == UTC
         else:
             assert time_parser_time.tzinfo == timezone
 
     def test_set_utc_if_timezone_is_missing_sets_timezone(self):
         datetime_time = datetime.now()
         assert datetime_time.tzinfo is None
         time_parser_time = TimeParser._set_utc_if_timezone_is_missing(datetime_time)
-        assert time_parser_time.tzinfo is ZoneInfo("UTC")
+        assert time_parser_time.tzinfo is UTC
         assert time_parser_time.second == pytest.approx(datetime_time.second, abs=1)
 
     @pytest.mark.parametrize(
@@ -90,28 +90,28 @@ def test_set_utc_if_timezone_is_missing_sets_timezone(self):
             (
                 "2021-03-13T11:23:13Z",
                 "ISO8601",
-                ZoneInfo("UTC"),
+                UTC,
                 "UTC",
                 {"year": 2021, "month": 3, "day": 13, "hour": 11, "minute": 23, "second": 13},
             ),
             (
                 "2021-03-13T11:23:13+01",
                 "ISO8601",
-                ZoneInfo("UTC"),
+                UTC,
                 "UTC+01:00",
                 {"year": 2021, "month": 3, "day": 13, "hour": 11, "minute": 23, "second": 13},
             ),
             (
                 "2021-03-13T11:23:13",
                 "ISO8601",
-                ZoneInfo("UTC"),
+                UTC,
                 "UTC",
                 {"year": 2021, "month": 3, "day": 13, "hour": 11, "minute": 23, "second": 13},
             ),
             (
                 "2021 03 13 - 11:23:13",
                 "%Y %m %d - %H:%M:%S",
-                ZoneInfo("UTC"),
+                UTC,
                 "UTC",
                 {"year": 2021, "month": 3, "day": 13, "hour": 11, "minute": 23, "second": 13},
             ),
@@ -132,7 +132,7 @@ def test_set_utc_if_timezone_is_missing_sets_timezone(self):
             (
                 "03 13 - 11:23:13",
                 "%m %d - %H:%M:%S",
-                ZoneInfo("UTC"),
+                UTC,
                 "UTC",
                 {
                     "year": datetime.now().year,
@@ -159,7 +159,7 @@ def test_parse_datetime_replaces_timezone_if_it_does_not_exist_in_string(
             (
                 "1615634593",
                 "UNIX",
-                ZoneInfo("UTC"),
+                UTC,
                 "UTC",
                 {"year": 2021, "month": 3, "day": 13, "hour": 11, "minute": 23, "second": 13},
             ),