Fix time zone handling

Author: stephanritscher

src/rctclient/cli.py

@@ -8,6 +8,7 @@
 # SPDX-License-Identifier: GPL-3.0-only
 
 import logging
+import pytz
 import select
 import socket
 import sys
@@ -28,6 +29,7 @@
 from .utils import decode_value, encode_value
 
 log = logging.getLogger('rctclient.cli')
+gmt = pytz.timezone('GMT')
 
 
 @click.group()
@@ -174,7 +176,7 @@ def read_value(ctx, port: int, host: str, id: Optional[str], name: Optional[str]
     is_ev = oinfo.response_data_type == DataType.EVENT_TABLE
     if is_ts or is_ev:
         sock.send(make_frame(command=Command.WRITE, id=oinfo.object_id,
-                             payload=encode_value(DataType.INT32, int(datetime.now().timestamp()))))
+                             payload=encode_value(DataType.INT32, int(datetime.now().replace(tzinfo=gmt).timestamp()))))
     else:
         sock.send(make_frame(command=Command.READ, id=oinfo.object_id))
     try:

src/rctclient/utils.py

@@ -3,6 +3,7 @@
 # Copyright 2020-2021, Stefan Valouch (svalouch)
 # SPDX-License-Identifier: GPL-3.0-only
 
+import pytz
 import struct
 from datetime import datetime
 from typing import overload, Dict, Tuple, Union
@@ -16,6 +17,7 @@
 
 from .types import DataType, EventEntry
 
+timezone = pytz.timezone('Europe/Berlin')
 
 # pylint: disable=invalid-name
 def CRC16(data: Union[bytes, bytearray]) -> int:
@@ -187,12 +189,12 @@ def _decode_timeseries(data: bytes) -> Tuple[datetime, Dict[datetime, int]]:
     '''
     Helper function to decode the timeseries type.
     '''
-    timestamp = datetime.fromtimestamp(struct.unpack('>I', data[0:4])[0])
+    timestamp = timezone.localize(datetime.utcfromtimestamp(struct.unpack('>I', data[0:4])[0]))
     tsval: Dict[datetime, int] = dict()
     assert len(data) % 4 == 0, 'Data should be divisible by 4'
     assert int(len(data) / 4 % 2) == 1, 'Data should be an even number of 4-byte pairs plus the starting timestamp'
     for pair in range(0, int(len(data) / 4 - 1), 2):
-        pair_ts = datetime.fromtimestamp(struct.unpack('>I', data[4 + pair * 4:4 + pair * 4 + 4])[0])
+        pair_ts = timezone.localize(datetime.utcfromtimestamp(struct.unpack('>I', data[4 + pair * 4:4 + pair * 4 + 4])[0]))
         pair_val = struct.unpack('>f', data[4 + pair * 4 + 4:4 + pair * 4 + 4 + 4])[0]
         tsval[pair_ts] = pair_val
     return timestamp, tsval
@@ -202,15 +204,15 @@ def _decode_event_table(data: bytes) -> Tuple[datetime, Dict[datetime, EventEntr
     '''
     Helper function to decode the event table type.
     '''
-    timestamp = datetime.fromtimestamp(struct.unpack('>I', data[0:4])[0])
+    timestamp = timezone.localize(datetime.utcfromtimestamp(struct.unpack('>I', data[0:4])[0]))
     tabval: Dict[datetime, EventEntry] = dict()
     assert len(data) % 4 == 0
     assert (len(data) - 4) % 20 == 0
     for pair in range(0, int(len(data) / 4 - 1), 5):
         # this is most likely a single byte of information, but this is not sure yet
         # entry_type = bytes([struct.unpack('>I', data[4 + pair * 4:4 + pair * 4 + 4])[0]]).decode('ascii')
         entry_type = struct.unpack('>I', data[4 + pair * 4:4 + pair * 4 + 4])[0]
-        timestamp = datetime.fromtimestamp(struct.unpack('>I', data[4 + pair * 4 + 4:4 + pair * 4 + 8])[0])
+        timestamp = timezone.localize(datetime.utcfromtimestamp(struct.unpack('>I', data[4 + pair * 4 + 4:4 + pair * 4 + 8])[0]))
         element2 = struct.unpack('>I', data[4 + pair * 4 + 8:4 + pair * 4 + 12])[0]
         element3 = struct.unpack('>I', data[4 + pair * 4 + 12:4 + pair * 4 + 16])[0]
         element4 = struct.unpack('>I', data[4 + pair * 4 + 16:4 + pair * 4 + 20])[0]
@@ -225,8 +227,8 @@ def _decode_event_table(data: bytes) -> Tuple[datetime, Dict[datetime, EventEntr
         #                                    value_old=value_old, value_new=value_new)
         # the rest is assumed to be range-based events
         # else:
-        #     timestamp_end = datetime.fromtimestamp(
-        #         struct.unpack('>I', data[4 + pair * 4 + 12:4 + pair * 4 + 16])[0])
+        #     timestamp_end = timezone.localize(datetime.utcfromtimestamp(
+        #         struct.unpack('>I', data[4 + pair * 4 + 12:4 + pair * 4 + 16])[0]))
         #     object_id = struct.unpack('>I', data[4 + pair * 4 + 16:4 + pair * 4 + 20])[0]
         #     tabval[timestamp] = EventEntry(timestamp=timestamp, object_id=object_id, entry_type=entry_type,
         #                                    timestamp_end=timestamp_end)

tools/timeseries2csv.py

@@ -29,6 +29,7 @@
 
 # pylint: disable=too-many-arguments,too-many-locals
 
+gmt = pytz.timezone('GMT')
 
 def datetime_range(start: datetime, end: datetime, delta: relativedelta):
     '''
@@ -114,7 +115,7 @@ def timeseries2csv(host: str, port: int, output: Optional[str], header_format: b
         sys.exit(1)
 
     timezone = pytz.timezone(time_zone)
-    now = datetime.now()
+    now = datetime.now(timezone)
 
     if resolution == 'minutes':
         oid_names = ['logger.minutes_ubat_log_ts', 'logger.minutes_ul3_log_ts', 'logger.minutes_ub_log_ts',
@@ -209,7 +210,7 @@ def timeseries2csv(host: str, port: int, output: Optional[str], header_format: b
         while highest_ts > ts_start and not iter_end:
             cprint(f'\ttimestamp: {highest_ts}')
             sock.send(make_frame(command=Command.WRITE, id=oid.object_id,
-                                 payload=encode_value(DataType.INT32, int(highest_ts.timestamp()))))
+                                 payload=encode_value(DataType.INT32, int(highest_ts.replace(tzinfo=gmt).timestamp()))))
 
             rframe = ReceiveFrame()
             while True:
@@ -313,7 +314,7 @@ def timeseries2csv(host: str, port: int, output: Optional[str], header_format: b
 
     for bts, btval in datetable.items():
         if btval:  # there may be holes in the data
-            writer.writerow([timezone.localize(bts).isoformat('T')] + [str(btval[name]) for name in names])
+            writer.writerow([bts.isoformat('T')] + [str(btval[name]) for name in names])
 
     if output != '-':
         fd.flush()