diff --git a/setup.py b/setup.py index 9167280..0097a7a 100644 --- a/setup.py +++ b/setup.py @@ -55,6 +55,7 @@ 'rctclient=rctclient.cli:cli [cli]', ], }, + scripts=['tools/timeseries2csv.py'], classifiers=[ 'Development Status :: 2 - Pre-Alpha', diff --git a/src/rctclient/cli.py b/src/rctclient/cli.py index 2c15813..1eb4363 100644 --- a/src/rctclient/cli.py +++ b/src/rctclient/cli.py @@ -11,7 +11,7 @@ import select import socket import sys -from datetime import datetime +from datetime import UTC, datetime from typing import List, Optional try: @@ -174,7 +174,7 @@ def read_value(ctx, port: int, host: str, id: Optional[str], name: Optional[str] is_ev = oinfo.response_data_type == DataType.EVENT_TABLE if is_ts or is_ev: sock.send(make_frame(command=Command.WRITE, id=oinfo.object_id, - payload=encode_value(DataType.INT32, int(datetime.now().timestamp())))) + payload=encode_value(DataType.INT32, int(datetime.now(UTC).timestamp())))) else: sock.send(make_frame(command=Command.READ, id=oinfo.object_id)) try: diff --git a/src/rctclient/utils.py b/src/rctclient/utils.py index 19e4cf3..5a9b4ec 100644 --- a/src/rctclient/utils.py +++ b/src/rctclient/utils.py @@ -4,7 +4,7 @@ # SPDX-License-Identifier: GPL-3.0-only import struct -from datetime import datetime +from datetime import UTC, datetime from typing import overload, Dict, Tuple, Union try: @@ -16,7 +16,6 @@ from .types import DataType, EventEntry - # pylint: disable=invalid-name def CRC16(data: Union[bytes, bytearray]) -> int: ''' @@ -187,12 +186,12 @@ def _decode_timeseries(data: bytes) -> Tuple[datetime, Dict[datetime, int]]: ''' Helper function to decode the timeseries type. ''' - timestamp = datetime.fromtimestamp(struct.unpack('>I', data[0:4])[0]) + timestamp = datetime.fromtimestamp(struct.unpack('>I', data[0:4])[0], UTC) tsval: Dict[datetime, int] = dict() assert len(data) % 4 == 0, 'Data should be divisible by 4' assert int(len(data) / 4 % 2) == 1, 'Data should be an even number of 4-byte pairs plus the starting timestamp' for pair in range(0, int(len(data) / 4 - 1), 2): - pair_ts = datetime.fromtimestamp(struct.unpack('>I', data[4 + pair * 4:4 + pair * 4 + 4])[0]) + pair_ts = datetime.fromtimestamp(struct.unpack('>I', data[4 + pair * 4:4 + pair * 4 + 4])[0], UTC) pair_val = struct.unpack('>f', data[4 + pair * 4 + 4:4 + pair * 4 + 4 + 4])[0] tsval[pair_ts] = pair_val return timestamp, tsval @@ -202,7 +201,7 @@ def _decode_event_table(data: bytes) -> Tuple[datetime, Dict[datetime, EventEntr ''' Helper function to decode the event table type. ''' - timestamp = datetime.fromtimestamp(struct.unpack('>I', data[0:4])[0]) + timestamp = datetime.fromtimestamp(struct.unpack('>I', data[0:4])[0], UTC) tabval: Dict[datetime, EventEntry] = dict() assert len(data) % 4 == 0 assert (len(data) - 4) % 20 == 0 @@ -210,7 +209,7 @@ def _decode_event_table(data: bytes) -> Tuple[datetime, Dict[datetime, EventEntr # this is most likely a single byte of information, but this is not sure yet # entry_type = bytes([struct.unpack('>I', data[4 + pair * 4:4 + pair * 4 + 4])[0]]).decode('ascii') entry_type = struct.unpack('>I', data[4 + pair * 4:4 + pair * 4 + 4])[0] - timestamp = datetime.fromtimestamp(struct.unpack('>I', data[4 + pair * 4 + 4:4 + pair * 4 + 8])[0]) + timestamp = datetime.fromtimestamp(struct.unpack('>I', data[4 + pair * 4 + 4:4 + pair * 4 + 8])[0], UTC) element2 = struct.unpack('>I', data[4 + pair * 4 + 8:4 + pair * 4 + 12])[0] element3 = struct.unpack('>I', data[4 + pair * 4 + 12:4 + pair * 4 + 16])[0] element4 = struct.unpack('>I', data[4 + pair * 4 + 16:4 + pair * 4 + 20])[0] @@ -226,7 +225,7 @@ def _decode_event_table(data: bytes) -> Tuple[datetime, Dict[datetime, EventEntr # the rest is assumed to be range-based events # else: # timestamp_end = datetime.fromtimestamp( - # struct.unpack('>I', data[4 + pair * 4 + 12:4 + pair * 4 + 16])[0]) + # struct.unpack('>I', data[4 + pair * 4 + 12:4 + pair * 4 + 16])[0], UTC) # object_id = struct.unpack('>I', data[4 + pair * 4 + 16:4 + pair * 4 + 20])[0] # tabval[timestamp] = EventEntry(timestamp=timestamp, object_id=object_id, entry_type=entry_type, # timestamp_end=timestamp_end) diff --git a/tools/timeseries2csv.py b/tools/timeseries2csv.py index 248f931..244c731 100755 --- a/tools/timeseries2csv.py +++ b/tools/timeseries2csv.py @@ -13,15 +13,15 @@ import socket import struct import sys -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from tempfile import mkstemp from typing import Dict, Optional +from zoneinfo import ZoneInfo import click -import pytz from dateutil.relativedelta import relativedelta -from rctclient.exceptions import FrameCRCMismatch +from rctclient.exceptions import FrameCRCMismatch, FrameLengthExceeded, InvalidCommand from rctclient.frame import ReceiveFrame, make_frame from rctclient.registry import REGISTRY as R from rctclient.types import Command, DataType @@ -29,7 +29,6 @@ # pylint: disable=too-many-arguments,too-many-locals - def datetime_range(start: datetime, end: datetime, delta: relativedelta): ''' Generator yielding datetime objects between `start` and `end` with `delta` increments. @@ -113,8 +112,12 @@ def timeseries2csv(host: str, port: int, output: Optional[str], header_format: b cprint('Error: --count must be a positive integer') sys.exit(1) - timezone = pytz.timezone(time_zone) - now = datetime.now() + try: + timezone = ZoneInfo(time_zone) + except: # noqa: E722 + cprint('Error: --time-zone is not valid') + sys.exit(1) + now = datetime.now(timezone) if resolution == 'minutes': oid_names = ['logger.minutes_ubat_log_ts', 'logger.minutes_ul3_log_ts', 'logger.minutes_ub_log_ts', @@ -208,8 +211,9 @@ def timeseries2csv(host: str, port: int, output: Optional[str], header_format: b while highest_ts > ts_start and not iter_end: cprint(f'\ttimestamp: {highest_ts}') + # rct power device seems to treat local time at GMT when converting from/to timestamps sock.send(make_frame(command=Command.WRITE, id=oid.object_id, - payload=encode_value(DataType.INT32, int(highest_ts.timestamp())))) + payload=encode_value(DataType.INT32, int(highest_ts.replace(tzinfo=UTC).timestamp())))) rframe = ReceiveFrame() while True: @@ -227,6 +231,12 @@ def timeseries2csv(host: str, port: int, output: Optional[str], header_format: b except FrameCRCMismatch: cprint('\tCRC error') break + except FrameLengthExceeded: + cprint('\tFrame length exceeded') + break + except InvalidCommand: + cprint('\tInvalid command') + break if rframe.complete(): break else: @@ -236,7 +246,7 @@ def timeseries2csv(host: str, port: int, output: Optional[str], header_format: b cprint('\tTimeout, retrying') break - if not rframe.complete(): + if not rframe.complete() or not rframe.crc_ok: cprint('\tIncomplete frame, retrying') continue @@ -255,6 +265,9 @@ def timeseries2csv(host: str, port: int, output: Optional[str], header_format: b # work with the data for t_ts, t_val in table.items(): + # rct power device seems to treat local time at GMT when converting from/to timestamps + t_ts = t_ts.replace(tzinfo=timezone) + # set the "highest" point in time to know what to request next when the day is not complete if t_ts < highest_ts: highest_ts = t_ts @@ -270,7 +283,7 @@ def timeseries2csv(host: str, port: int, output: Optional[str], header_format: b # correct up to one full minute nt_ts = t_ts.replace(second=0) if nt_ts not in datetable: - nt_ts = t_ts.replace(second=0, minute=t_ts.minute + 1) + nt_ts = t_ts.replace(second=0, minute=(t_ts.minute + 1) % 60, hour=t_ts.hour + (t_ts.minute + 1) // 60) if nt_ts not in datetable: cprint(f'\t{t_ts} does not fit raster, skipped') continue @@ -313,7 +326,7 @@ def timeseries2csv(host: str, port: int, output: Optional[str], header_format: b for bts, btval in datetable.items(): if btval: # there may be holes in the data - writer.writerow([timezone.localize(bts).isoformat('T')] + [str(btval[name]) for name in names]) + writer.writerow([bts.isoformat('T')] + [str(btval[name]) for name in names]) if output != '-': fd.flush()