feat: Unify datetime handling, remove utcnow() (#52)

Author: jirimoravcik

src/apify/_utils.py

@@ -234,13 +234,6 @@ async def _force_remove(filename: str) -> None:
         await remove(filename)
 
 
-def _json_serializer(obj: Any) -> str:  # TODO: Decide how to parse/dump/handle datetimes!
-    if isinstance(obj, (datetime)):
-        return obj.isoformat(timespec='milliseconds') + 'Z'
-    else:
-        return str(obj)
-
-
 def _filter_out_none_values_recursively(dictionary: Dict) -> Dict:
     """Return copy of the dictionary, recursively omitting all keys for which values are None."""
     return cast(dict, _filter_out_none_values_recursively_internal(dictionary))
@@ -261,7 +254,7 @@ def _filter_out_none_values_recursively_internal(dictionary: Dict, remove_empty_
 
 def _json_dumps(obj: Any) -> str:
     """Dump JSON to a string with the correct settings and serializer."""
-    return json.dumps(obj, ensure_ascii=False, indent=2, default=_json_serializer)
+    return json.dumps(obj, ensure_ascii=False, indent=2, default=str)
 
 
 uuid_regex = re.compile('[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}', re.I)

src/apify/actor.py

@@ -3,7 +3,7 @@
 import os
 import sys
 import traceback
-from datetime import datetime
+from datetime import datetime, timezone
 from types import TracebackType
 from typing import Any, Awaitable, Callable, Dict, List, Optional, Type, TypeVar, Union, cast
 
@@ -247,7 +247,7 @@ def _get_system_info(self) -> Dict:
         memory_usage_bytes = _get_memory_usage_bytes()
         # This is in camel case to be compatible with the events from the platform
         result = {
-            'createdAt': datetime.utcnow().isoformat(timespec='milliseconds') + 'Z',
+            'createdAt': datetime.now(timezone.utc).isoformat(timespec='milliseconds'),
             'cpuCurrentUsage': cpu_usage_percent,
             'memCurrentBytes': memory_usage_bytes,
         }

src/apify/memory_storage/resource_clients/dataset.py

@@ -1,7 +1,7 @@
 import json
 import os
 import uuid
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, List, Optional, Tuple, Union
 
 import aioshutil
@@ -49,9 +49,9 @@ def __init__(self, *, base_storage_directory: str, client: 'MemoryStorage', id:
         self._client = client
         self._name = name
         self._dataset_entries = {}
-        self._created_at = datetime.utcnow()
-        self._accessed_at = datetime.utcnow()
-        self._modified_at = datetime.utcnow()
+        self._created_at = datetime.now(timezone.utc)
+        self._accessed_at = datetime.now(timezone.utc)
+        self._modified_at = datetime.now(timezone.utc)
 
     async def get(self) -> Optional[Dict]:
         """Retrieve the dataset.
@@ -315,10 +315,10 @@ def to_dataset_info(self) -> Dict:
 
     async def _update_timestamps(self, has_been_modified: bool) -> None:
         """Update the timestamps of the dataset."""
-        self._accessed_at = datetime.utcnow()
+        self._accessed_at = datetime.now(timezone.utc)
 
         if has_been_modified:
-            self._modified_at = datetime.utcnow()
+            self._modified_at = datetime.now(timezone.utc)
 
         dataset_info = self.to_dataset_info()
         await _update_metadata(data=dataset_info, entity_directory=self._dataset_directory, write_metadata=self._client._write_metadata)
@@ -370,9 +370,9 @@ def _find_or_cache_dataset_by_possible_id(client: 'MemoryStorage', entry_name_or
     id: Union[str, None] = None
     name: Union[str, None] = None
     item_count = 0
-    created_at = datetime.utcnow()
-    accessed_at = datetime.utcnow()
-    modified_at = datetime.utcnow()
+    created_at = datetime.now(timezone.utc)
+    accessed_at = datetime.now(timezone.utc)
+    modified_at = datetime.now(timezone.utc)
     entries: Dict[str, Dict] = {}
 
     has_seen_metadata_file = False
@@ -389,9 +389,9 @@ def _find_or_cache_dataset_by_possible_id(client: 'MemoryStorage', entry_name_or
                 id = metadata['id']
                 name = metadata['name']
                 item_count = metadata['itemCount']
-                created_at = datetime.strptime(metadata['createdAt'], '%Y-%m-%dT%H:%M:%S.%fZ')
-                accessed_at = datetime.strptime(metadata['accessedAt'], '%Y-%m-%dT%H:%M:%S.%fZ')
-                modified_at = datetime.strptime(metadata['modifiedAt'], '%Y-%m-%dT%H:%M:%S.%fZ')
+                created_at = datetime.fromisoformat(metadata['createdAt'])
+                accessed_at = datetime.fromisoformat(metadata['accessedAt'])
+                modified_at = datetime.fromisoformat(metadata['modifiedAt'])
 
                 continue
 

src/apify/memory_storage/resource_clients/key_value_store.py

@@ -5,7 +5,7 @@
 import pathlib
 import uuid
 import warnings
-from datetime import datetime
+from datetime import datetime, timezone
 from operator import itemgetter
 from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Optional, Union
 
@@ -49,9 +49,9 @@ def __init__(self, *, base_storage_directory: str, client: 'MemoryStorage', id:
         self._client = client
         self._name = name
         self._key_value_entries = {}
-        self._created_at = datetime.utcnow()
-        self._accessed_at = datetime.utcnow()
-        self._modified_at = datetime.utcnow()
+        self._created_at = datetime.now(timezone.utc)
+        self._accessed_at = datetime.now(timezone.utc)
+        self._modified_at = datetime.now(timezone.utc)
 
     async def get(self) -> Optional[Dict]:
         """Retrieve the key-value store.
@@ -315,10 +315,10 @@ def to_key_value_store_info(self) -> Dict:
         }
 
     async def _update_timestamps(self, has_been_modified: bool) -> None:
-        self._accessed_at = datetime.utcnow()
+        self._accessed_at = datetime.now(timezone.utc)
 
         if has_been_modified:
-            self._modified_at = datetime.utcnow()
+            self._modified_at = datetime.now(timezone.utc)
 
         kv_store_info = self.to_key_value_store_info()
         await _update_metadata(data=kv_store_info, entity_directory=self._key_value_store_directory, write_metadata=self._client._write_metadata)
@@ -339,9 +339,9 @@ def _find_or_cache_key_value_store_by_possible_id(client: 'MemoryStorage', entry
 
     id: Union[str, None] = None
     name: Union[str, None] = None
-    created_at = datetime.utcnow()
-    accessed_at = datetime.utcnow()
-    modified_at = datetime.utcnow()
+    created_at = datetime.now(timezone.utc)
+    accessed_at = datetime.now(timezone.utc)
+    modified_at = datetime.now(timezone.utc)
     internal_records: Dict[str, Dict] = {}
 
     # Access the key value store folder
@@ -353,9 +353,9 @@ def _find_or_cache_key_value_store_by_possible_id(client: 'MemoryStorage', entry
                     metadata = json.load(f)
                 id = metadata['id']
                 name = metadata['name']
-                created_at = datetime.strptime(metadata['createdAt'], '%Y-%m-%dT%H:%M:%S.%fZ')
-                accessed_at = datetime.strptime(metadata['accessedAt'], '%Y-%m-%dT%H:%M:%S.%fZ')
-                modified_at = datetime.strptime(metadata['modifiedAt'], '%Y-%m-%dT%H:%M:%S.%fZ')
+                created_at = datetime.fromisoformat(metadata['createdAt'])
+                accessed_at = datetime.fromisoformat(metadata['accessedAt'])
+                modified_at = datetime.fromisoformat(metadata['modifiedAt'])
 
                 continue
 

src/apify/memory_storage/resource_clients/request_queue.py

@@ -1,7 +1,7 @@
 import json
 import os
 import uuid
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Dict, List, Optional, Union
 
 import aioshutil
@@ -43,9 +43,9 @@ def __init__(self, *, base_storage_directory: str, client: 'MemoryStorage', id:
         self._client = client
         self._name = name
         self._requests = {}
-        self._created_at = datetime.utcnow()
-        self._accessed_at = datetime.utcnow()
-        self._modified_at = datetime.utcnow()
+        self._created_at = datetime.now(timezone.utc)
+        self._accessed_at = datetime.now(timezone.utc)
+        self._modified_at = datetime.now(timezone.utc)
 
     async def get(self) -> Optional[Dict]:
         """Retrieve the request queue.
@@ -300,10 +300,10 @@ def to_request_queue_info(self) -> Dict:
         }
 
     async def _update_timestamps(self, has_been_modified: bool) -> None:
-        self._accessed_at = datetime.utcnow()
+        self._accessed_at = datetime.now(timezone.utc)
 
         if has_been_modified:
-            self._modified_at = datetime.utcnow()
+            self._modified_at = datetime.now(timezone.utc)
 
         request_queue_info = self.to_request_queue_info()
         await _update_metadata(data=request_queue_info, entity_directory=self._request_queue_directory, write_metadata=self._client._write_metadata)
@@ -336,7 +336,7 @@ def _calculate_order_no(self, request: Dict, forefront: Optional[bool]) -> Optio
         if request.get('handledAt') is not None:
             return None
 
-        timestamp = int(round(datetime.utcnow().timestamp()))
+        timestamp = int(round(datetime.now(timezone.utc).timestamp()))
 
         return -timestamp if forefront else timestamp
 
@@ -356,9 +356,9 @@ def _find_or_cache_request_queue_by_possible_id(client: 'MemoryStorage', entry_n
 
     id: Union[str, None] = None
     name: Union[str, None] = None
-    created_at = datetime.utcnow()
-    accessed_at = datetime.utcnow()
-    modified_at = datetime.utcnow()
+    created_at = datetime.now(timezone.utc)
+    accessed_at = datetime.now(timezone.utc)
+    modified_at = datetime.now(timezone.utc)
     handled_request_count = 0
     pending_request_count = 0
     entries: List[Dict] = []
@@ -372,9 +372,9 @@ def _find_or_cache_request_queue_by_possible_id(client: 'MemoryStorage', entry_n
                     metadata = json.load(f)
                 id = metadata['id']
                 name = metadata['name']
-                created_at = datetime.strptime(metadata['createdAt'], '%Y-%m-%dT%H:%M:%S.%fZ')
-                accessed_at = datetime.strptime(metadata['accessedAt'], '%Y-%m-%dT%H:%M:%S.%fZ')
-                modified_at = datetime.strptime(metadata['modifiedAt'], '%Y-%m-%dT%H:%M:%S.%fZ')
+                created_at = datetime.fromisoformat(metadata['createdAt'])
+                accessed_at = datetime.fromisoformat(metadata['accessedAt'])
+                modified_at = datetime.fromisoformat(metadata['modifiedAt'])
                 handled_request_count = metadata['handledRequestCount']
                 pending_request_count = metadata['pendingRequestCount']
 

tests/integration/README.md

@@ -113,10 +113,10 @@ if you need something really complex:
 async def test_something(self, make_actor: ActorFactory) -> None:
     actor_source_files = {
         'src/utils.py': """
-            from datetime import datetime
+            from datetime import datetime, timezone
 
             def get_current_datetime():
-                return datetime.utcnow()
+                return datetime.now(timezone.utc)
         """,
         'src/main.py': """
             import asyncio

tests/integration/test_fixtures.py

@@ -52,10 +52,10 @@ async def test_source_files(self, make_actor: ActorFactory) -> None:
         test_started_at = datetime.now(timezone.utc)
         actor_source_files = {
             'src/utils.py': """
-                from datetime import datetime
+                from datetime import datetime, timezone
 
                 def get_current_datetime():
-                    return datetime.utcnow()
+                    return datetime.now(timezone.utc)
             """,
             'src/main.py': """
                 import asyncio
@@ -77,7 +77,7 @@ async def main():
         output_record = await actor.last_run().key_value_store().get_record('OUTPUT')
         assert output_record is not None
 
-        output_datetime = datetime.fromisoformat(output_record['value']).replace(tzinfo=timezone.utc)
+        output_datetime = datetime.fromisoformat(output_record['value'])
         assert output_datetime > test_started_at
         assert output_datetime < datetime.now(timezone.utc)
 

tests/unit/memory_storage/resource_clients/test_request_queue.py

@@ -1,5 +1,5 @@
 import os
-from datetime import datetime
+from datetime import datetime, timezone
 
 import pytest
 
@@ -132,7 +132,7 @@ async def test_update_record(request_queue_client: RequestQueueClient) -> None:
     rq_info_before_update = await request_queue_client.get()
     assert rq_info_before_update is not None
     assert rq_info_before_update['pendingRequestCount'] == 1
-    request_update_info = await request_queue_client.update_request({**request, 'handledAt': datetime.utcnow()})
+    request_update_info = await request_queue_client.update_request({**request, 'handledAt': datetime.now(timezone.utc)})
     assert request_update_info['wasAlreadyHandled'] is False
     rq_info_after_update = await request_queue_client.get()
     assert rq_info_after_update is not None

tests/unit/test_utils.py

@@ -1,9 +1,9 @@
 import asyncio
-import datetime
 import io
 import os
 import uuid
 from collections import OrderedDict
+from datetime import datetime, timezone
 from enum import Enum
 
 import pytest
@@ -51,7 +51,7 @@ def test__fetch_and_parse_env_var(monkeypatch: pytest.MonkeyPatch) -> None:
     assert _fetch_and_parse_env_var(ApifyEnvVars.MEMORY_MBYTES) == 1024
     assert _fetch_and_parse_env_var(ApifyEnvVars.META_ORIGIN) == 'API'
     assert _fetch_and_parse_env_var(ApifyEnvVars.STARTED_AT) == \
-        datetime.datetime(2022, 12, 2, 15, 19, 34, 907000, tzinfo=datetime.timezone.utc)
+        datetime(2022, 12, 2, 15, 19, 34, 907000, tzinfo=timezone.utc)
 
     assert _fetch_and_parse_env_var('DUMMY_BOOL') == '1'  # type: ignore
     assert _fetch_and_parse_env_var('DUMMY_DATETIME') == '2022-12-02T15:19:34.907Z'  # type: ignore
@@ -96,7 +96,7 @@ def test__maybe_parse_bool() -> None:
 
 def test__maybe_parse_datetime() -> None:
     assert _maybe_parse_datetime('2022-12-02T15:19:34.907Z') == \
-        datetime.datetime(2022, 12, 2, 15, 19, 34, 907000, tzinfo=datetime.timezone.utc)
+        datetime(2022, 12, 2, 15, 19, 34, 907000, tzinfo=timezone.utc)
     assert _maybe_parse_datetime('2022-12-02T15:19:34.907') == '2022-12-02T15:19:34.907'
     assert _maybe_parse_datetime('anything') == 'anything'
 
@@ -275,14 +275,16 @@ def test__json_dumps() -> None:
   "number": 456,
   "nested": {
     "abc": "def"
-  }
+  },
+  "datetime": "2022-01-01 00:00:00+00:00"
 }"""
-    actual = _json_dumps({  # TODO: add a date into the object after datetime serialization format is finalized
+    actual = _json_dumps({
         'string': '123',
         'number': 456,
         'nested': {
             'abc': 'def',
         },
+        'datetime': datetime(2022, 1, 1, tzinfo=timezone.utc),
     })
     assert actual == expected
 
@@ -322,11 +324,11 @@ def test__budget_ow() -> None:
     _budget_ow({
         'a': 123,
         'b': 'string',
-        'c': datetime.datetime.utcnow(),
+        'c': datetime.now(timezone.utc),
     }, {
         'a': (int, True),
         'b': (str, False),
-        'c': (datetime.datetime, True),
+        'c': (datetime, True),
     })
     with pytest.raises(ValueError, match='required'):
         _budget_ow({}, {'id': (str, True)})