Remove dependency on pytz

All we are using pytz for is to using aware datetime object with UTC
timezone, which is perfectly doable with the standard datetime and
timezone objects.

This commit also converts some instances of "native" datetimes to
"aware".

Signed-off-by: Leandro Lucarella <[email protected]>

Author: llucax

minimum-requirements-ci.txt

@@ -9,7 +9,6 @@ pandas==1.3.5
 protobuf==3.20.2
 pyarrow==6.0.0
 pydantic==1.9.0
-pytz==2021.3
 sympy==1.10.1
 toml==0.10
 tqdm==4.38.0

pyproject.toml

@@ -36,7 +36,6 @@ dependencies = [
     "protobuf >= 3.20, < 4",
     "pyarrow >= 6.0.0, < 6.1",
     "pydantic >= 1.9",
-    "pytz >= 2021.3",
     "sympy >= 1.10.1, < 2",
     "toml >= 0.10",
     "tqdm >= 4.38.0, < 5",

src/frequenz/sdk/data_handling/time_series.py

@@ -9,14 +9,12 @@
 """
 from __future__ import annotations
 
-import datetime
 import enum
 import logging
 from dataclasses import dataclass, field
+from datetime import datetime, timedelta, timezone
 from typing import Collection, Dict, Generic, Optional, Set, TypeVar
 
-import pytz
-
 from .formula import Formula
 
 Key = TypeVar("Key")
@@ -104,13 +102,13 @@ class Status(enum.Enum):
         UNKNOWN = "unknown"
         ERROR = "error"
 
-    timestamp: datetime.datetime
+    timestamp: datetime
     value: Optional[Value] = None
     status: Status = Status.VALID
     broken_component_ids: Set[int] = field(default_factory=set)
 
     @staticmethod
-    def create_error(timestamp: datetime.datetime) -> TimeSeriesEntry[Value]:
+    def create_error(timestamp: datetime) -> TimeSeriesEntry[Value]:
         """Create a `TimeSeriesEntry` that contains an error.
 
         This can happen when the value would be NaN, e.g.
@@ -128,7 +126,7 @@ def create_error(timestamp: datetime.datetime) -> TimeSeriesEntry[Value]:
 
     @staticmethod
     def create_unknown(
-        timestamp: datetime.datetime, broken_component_ids: Optional[Set[int]] = None
+        timestamp: datetime, broken_component_ids: Optional[Set[int]] = None
     ) -> TimeSeriesEntry[Value]:
         """Create a `TimeSeriesEntry` that contains an unknown value.
 
@@ -174,11 +172,11 @@ class LatestEntryCache(Generic[Key, Value]):
 
     def __init__(self) -> None:
         """Initialize the class."""
-        self._latest_timestamp = pytz.utc.localize(datetime.datetime.min)
+        self._latest_timestamp = datetime.min.replace(tzinfo=timezone.utc)
         self._entries: Dict[Key, TimeSeriesEntry[Value]] = {}
 
     @property
-    def latest_timestamp(self) -> datetime.datetime:
+    def latest_timestamp(self) -> datetime:
         """Get the most recently observed timestamp across all keys in the cache.
 
         Returns:
@@ -209,7 +207,7 @@ def __contains__(self, key: Key) -> bool:
     def get(
         self,
         key: Key,
-        timedelta_tolerance: datetime.timedelta = datetime.timedelta.max,
+        timedelta_tolerance: timedelta = timedelta.max,
         default: Optional[TimeSeriesEntry[Value]] = None,
     ) -> CacheEntryLookupResult[Value]:
         """Get the cached entry for the specified key, if any.
@@ -232,7 +230,7 @@ def get(
                 retrieved from the cache has a timestamp greater than the latest saved
                 timestamp across all cache keys.
         """
-        if timedelta_tolerance < datetime.timedelta(0):
+        if timedelta_tolerance < timedelta(0):
             raise ValueError(
                 f"timedelta_tolerance cannot be less than 0, but "
                 f"{timedelta_tolerance} was provided"
@@ -320,7 +318,7 @@ def reset(self) -> None:
         slightly more efficient.
         """
         self.clear()
-        self._latest_timestamp = pytz.utc.localize(datetime.datetime.min)
+        self._latest_timestamp = datetime.min.replace(tzinfo=timezone.utc)
 
     def reset_latest_timestamp(self) -> bool:
         """Reset the `latest_timestamp` property to the lowest possible value.
@@ -342,7 +340,7 @@ def reset_latest_timestamp(self) -> bool:
 
         self._latest_timestamp = max(
             map(lambda x: x.timestamp, self._entries.values()),
-            default=pytz.utc.localize(datetime.datetime.min),
+            default=datetime.min.replace(tzinfo=timezone.utc),
         )
 
         if self._latest_timestamp > previous:
@@ -423,7 +421,7 @@ def evaluate(
         cache: LatestEntryCache[str, Value],
         formula_name: str = "",
         symbol_to_symbol_mapping: Optional[Dict[str, SymbolMapping]] = None,
-        timedelta_tolerance: datetime.timedelta = datetime.timedelta.max,
+        timedelta_tolerance: timedelta = timedelta.max,
         default_entry: Optional[TimeSeriesEntry[Value]] = None,
     ) -> Optional[TimeSeriesEntry[Value]]:
         """Evaluate the formula using time-series values from the provided cache.
@@ -463,7 +461,7 @@ def evaluate(
                 `timedelta_tolerance`
         """
         kwargs: Dict[str, Optional[Value]] = {}
-        timestamp = pytz.utc.localize(datetime.datetime.min)
+        timestamp = datetime.min.replace(tzinfo=timezone.utc)
 
         symbol_to_symbol_mapping = symbol_to_symbol_mapping or {}
         formula_broken_component_ids: Set[int] = set()

src/frequenz/sdk/data_ingestion/formula_calculator.py

@@ -7,13 +7,12 @@
 License
 MIT
 """
-import datetime as dt
 import logging
 from dataclasses import dataclass
+from datetime import datetime, timedelta, timezone
 from itertools import chain
 from typing import Any, Dict, List, Optional, Set, Tuple
 
-import pytz
 import sympy
 
 from ..data_handling.time_series import (
@@ -834,11 +833,9 @@ def compute(
                 cache=self.symbol_values,
                 formula_name=formula_name,
                 symbol_to_symbol_mapping=self.symbol_mappings,
-                timedelta_tolerance=dt.timedelta(
-                    seconds=self.component_data_timeout_sec
-                ),
+                timedelta_tolerance=timedelta(seconds=self.component_data_timeout_sec),
                 default_entry=TimeSeriesEntry[Any](
-                    timestamp=pytz.utc.localize(dt.datetime.now()), value=0.0
+                    timestamp=datetime.now(timezone.utc), value=0.0
                 ),
             )
             if res is not None:

src/frequenz/sdk/data_ingestion/load_historic_data.py

@@ -13,17 +13,16 @@
 MIT
 """
 
-import datetime as dt
 import glob
 import itertools
 import logging
 import os
 from dataclasses import dataclass
+from datetime import datetime, timezone
 from typing import Any, Callable, List, Optional
 
 import pandas as pd
 import pyarrow.parquet as pq
-import pytz
 from tqdm import tqdm
 
 logger = logging.Logger(__name__)
@@ -151,8 +150,8 @@ def gen_date_dirs(data_dir: str, dates: pd.DatetimeIndex) -> List[str]:
 
 def crop_df_list_by_time(
     df_list: List[pd.DataFrame],
-    start_time: dt.datetime,
-    end_time: dt.datetime,
+    start_time: datetime,
+    end_time: datetime,
 ) -> pd.DataFrame:
     """Concat and crop read data by the specified start and end time.
 
@@ -169,7 +168,7 @@ def crop_df_list_by_time(
             specified start and end times.
     """
     df0 = pd.concat(df_list).reset_index(drop=True)
-    df0["ts"] = pd.to_datetime(df0["ts"]).dt.tz_localize(pytz.UTC)
+    df0["ts"] = pd.to_datetime(df0["ts"]).tz_localize(timezone.utc)
     df0 = df0.loc[((df0["ts"] >= start_time) & (df0["ts"] <= end_time))].reset_index(
         drop=True
     )
@@ -256,15 +255,15 @@ def get_file_timestamps(self, filenames: List[str]) -> pd.Series:
                 for file in filenames
             ],
             format=self.file_time_format,
-        ).tz_localize(pytz.UTC)
+        ).tz_localize(timezone.utc)
         return timestamps
 
     def gen_datafile_list(
         self,
         data_dir: str,
         dates: pd.DatetimeIndex,
-        start_time: dt.datetime,
-        end_time: dt.datetime,
+        start_time: datetime,
+        end_time: datetime,
     ) -> List[str]:
         """Generate the list of historic parquet files to read.
 
@@ -352,8 +351,8 @@ def load_parquet_files(
     def read(
         self,
         load_hd_settings: LoadHistoricDataSettings,
-        start_time: dt.datetime,
-        end_time: dt.datetime,
+        start_time: datetime,
+        end_time: datetime,
     ) -> pd.DataFrame:
         """Read historical data.
 
@@ -373,9 +372,9 @@ def read(
             "component_id=" + str(load_hd_settings.component_info.component_id),
         )
         if start_time.tzinfo is None:
-            start_time = start_time.replace(tzinfo=pytz.UTC)
+            start_time = start_time.replace(tzinfo=timezone.utc)
         if end_time.tzinfo is None:
-            end_time = end_time.replace(tzinfo=pytz.UTC)
+            end_time = end_time.replace(tzinfo=timezone.utc)
         logger.info(
             "reading historic data from component id=%s within the time interval: %s to %s",
             load_hd_settings.component_info.component_id,

src/frequenz/sdk/data_ingestion/microgrid_data.py

@@ -11,11 +11,10 @@
 MIT
 """
 import asyncio
-import datetime as dt
 import logging
+from datetime import datetime, timezone
 from typing import Any, Dict, List, Optional, Set
 
-import pytz
 from frequenz.channels import Broadcast, Merge, Receiver, Select, Sender
 
 from ..actor.decorator import actor
@@ -99,7 +98,7 @@ async def resend_formulas(self) -> None:
         await asyncio.sleep(self._wait_for_data_sec)
         tasks: List["asyncio.Task[bool]"] = []
         while True:
-            start_time = dt.datetime.now(tz=pytz.UTC)
+            start_time = datetime.now(timezone.utc)
             # For every formula that was updated at least once, send that formula.
             for name, formula_result in self.formula_calculator.results.items():
                 if name not in self._outputs:
@@ -114,7 +113,7 @@ async def resend_formulas(self) -> None:
                     tasks.append(task)
 
             await asyncio.gather(*tasks, return_exceptions=True)
-            diff: float = (dt.datetime.now(pytz.UTC) - start_time).total_seconds()
+            diff: float = (datetime.now(timezone.utc) - start_time).total_seconds()
             if diff >= self._formula_update_interval_sec:
                 logger.error(
                     "Sending results of formulas took too long: %f, "

src/frequenz/sdk/power_distribution/utils.py

@@ -7,7 +7,7 @@
 MIT
 """
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, timezone
 from enum import Enum
 from typing import Dict, NamedTuple, Optional, Set
 
@@ -100,7 +100,7 @@ def mark_as_broken(self, component_id: int) -> None:
         Args:
             component_id: component id
         """
-        self._broken[component_id] = datetime.now()
+        self._broken[component_id] = datetime.now(timezone.utc)
 
     def update_retry(self, timeout_sec: float) -> None:
         """Change how long the component should be marked as broken.
@@ -121,7 +121,9 @@ def is_broken(self, component_id: int) -> bool:
         """
         if component_id in self._broken:
             last_broken = self._broken[component_id]
-            if (datetime.now() - last_broken).total_seconds() < self._timeout_sec:
+            if (
+                datetime.now(timezone.utc) - last_broken
+            ).total_seconds() < self._timeout_sec:
                 return True
 
             del self._broken[component_id]

tests/power_distribution/test_distribution_algorithm.py

@@ -8,7 +8,7 @@
 MIT
 """
 from dataclasses import dataclass
-from datetime import datetime
+from datetime import datetime, timezone
 from typing import Dict, List, Optional
 
 import frequenz.api.microgrid.microgrid_pb2 as microgrid_pb
@@ -72,7 +72,7 @@ def create_battery_msg(  # pylint: disable=too-many-arguments
     capacity: Metric,
     soc: Metric,
     power: Bound,
-    timestamp: datetime = datetime.utcnow(),
+    timestamp: datetime = datetime.now(timezone.utc),
 ) -> microgrid_pb.ComponentData:
     """Create protobuf battery components with given arguments.
 
@@ -111,7 +111,7 @@ def create_battery_msg(  # pylint: disable=too-many-arguments
 def create_inverter_msg(
     component_id: int,
     power: Bound,
-    timestamp: datetime = datetime.utcnow(),
+    timestamp: datetime = datetime.now(timezone.utc),
 ) -> microgrid_pb.ComponentData:
     """Create protobuf inverter components with given arguments.
 

tests/power_distribution/test_power_distributor.py

@@ -9,7 +9,7 @@
 import asyncio
 import re
 from dataclasses import dataclass
-from datetime import datetime, timedelta
+from datetime import datetime, timedelta, timezone
 from functools import partial
 from typing import Dict, Set, Tuple, TypeVar, Union
 from unittest import IsolatedAsyncioTestCase, mock
@@ -605,7 +605,7 @@ async def test_power_distributor_stale_battery_message(
                     capacity=Metric(98000),
                     soc=Metric(40, Bound(20, 80)),
                     power=Bound(-1000, 1000),
-                    timestamp=datetime.utcnow() - timedelta(seconds=62),
+                    timestamp=datetime.now(timezone.utc) - timedelta(seconds=62),
                 )
             else:
                 bat = create_battery_msg(
@@ -672,7 +672,7 @@ async def test_power_distributor_stale_all_components_message(
                     capacity=Metric(98000),
                     soc=Metric(40, Bound(20, 80)),
                     power=Bound(-1000, 1000),
-                    timestamp=datetime.utcnow() - timedelta(seconds=62),
+                    timestamp=datetime.now(timezone.utc) - timedelta(seconds=62),
                 )
             else:
                 bat = create_battery_msg(
@@ -689,7 +689,7 @@ async def test_power_distributor_stale_all_components_message(
                 inv = create_inverter_msg(
                     key_id,
                     power=Bound(-500, 500),
-                    timestamp=datetime.utcnow() - timedelta(seconds=62),
+                    timestamp=datetime.now(timezone.utc) - timedelta(seconds=62),
                 )
             else:
                 inv = create_inverter_msg(