Move to timezone aware datetimes (#1895)

* Move to timezone aware datetimes

With the shift of the discord.py library to timezone aware datetimes, this commit changes datetimes throughout the bot to be in the UTC timezone accordingly.

This has several advantages:
- There's no need to discard the TZ every time the datetime of a Discord object is fetched.
- Using TZ aware datetimes reduces the likelihood of silently adding bugs into the codebase (can't compare an aware datetime with a naive one).
- Our DB already stores datetimes in UTC, but we've been discarding the TZ so far whenever we read from it.

Specific places in the codebase continue using naive datetimes, mainly for UI purposes (for examples embed footers use naive datetimes to display local time).

* Improve ISODateTime converter documentation

Co-authored-by: Kieran Siek <[email protected]>

Author: mbaruh

bot/converters.py

@@ -2,7 +2,7 @@
 
 import re
 import typing as t
-from datetime import datetime
+from datetime import datetime, timezone
 from ssl import CertificateError
 
 import dateutil.parser
@@ -11,7 +11,7 @@
 from aiohttp import ClientConnectorError
 from dateutil.relativedelta import relativedelta
 from discord.ext.commands import BadArgument, Bot, Context, Converter, IDConverter, MemberConverter, UserConverter
-from discord.utils import DISCORD_EPOCH, escape_markdown, snowflake_time
+from discord.utils import escape_markdown, snowflake_time
 
 from bot import exts
 from bot.api import ResponseCodeError
@@ -28,7 +28,7 @@
 
 log = get_logger(__name__)
 
-DISCORD_EPOCH_DT = datetime.utcfromtimestamp(DISCORD_EPOCH / 1000)
+DISCORD_EPOCH_DT = snowflake_time(0)
 RE_USER_MENTION = re.compile(r"<@!?([0-9]+)>$")
 
 
@@ -273,14 +273,14 @@ async def convert(self, ctx: Context, arg: str) -> int:
         snowflake = int(arg)
 
         try:
-            time = snowflake_time(snowflake).replace(tzinfo=None)
+            time = snowflake_time(snowflake)
         except (OverflowError, OSError) as e:
             # Not sure if this can ever even happen, but let's be safe.
             raise BadArgument(f"{error}: {e}")
 
         if time < DISCORD_EPOCH_DT:
             raise BadArgument(f"{error}: timestamp is before the Discord epoch.")
-        elif (datetime.utcnow() - time).days < -1:
+        elif (datetime.now(timezone.utc) - time).days < -1:
             raise BadArgument(f"{error}: timestamp is too far into the future.")
 
         return snowflake
@@ -387,7 +387,7 @@ async def convert(self, ctx: Context, duration: str) -> datetime:
         The converter supports the same symbols for each unit of time as its parent class.
         """
         delta = await super().convert(ctx, duration)
-        now = datetime.utcnow()
+        now = datetime.now(timezone.utc)
 
         try:
             return now + delta
@@ -443,8 +443,8 @@ async def convert(self, ctx: Context, datetime_string: str) -> datetime:
         The converter is flexible in the formats it accepts, as it uses the `isoparse` method of
         `dateutil.parser`. In general, it accepts datetime strings that start with a date,
         optionally followed by a time. Specifying a timezone offset in the datetime string is
-        supported, but the `datetime` object will be converted to UTC and will be returned without
-        `tzinfo` as a timezone-unaware `datetime` object.
+        supported, but the `datetime` object will be converted to UTC. If no timezone is specified, the datetime will
+        be assumed to be in UTC already. In all cases, the returned object will have the UTC timezone.
 
         See: https://dateutil.readthedocs.io/en/stable/parser.html#dateutil.parser.isoparse
 
@@ -470,7 +470,8 @@ async def convert(self, ctx: Context, datetime_string: str) -> datetime:
 
         if dt.tzinfo:
             dt = dt.astimezone(dateutil.tz.UTC)
-            dt = dt.replace(tzinfo=None)
+        else:  # Without a timezone, assume it represents UTC.
+            dt = dt.replace(tzinfo=dateutil.tz.UTC)
 
         return dt
 

bot/exts/filters/antispam.py

@@ -2,11 +2,12 @@
 from collections import defaultdict
 from collections.abc import Mapping
 from dataclasses import dataclass, field
-from datetime import datetime, timedelta
+from datetime import timedelta
 from itertools import takewhile
 from operator import attrgetter, itemgetter
 from typing import Dict, Iterable, List, Set
 
+import arrow
 from discord import Colour, Member, Message, NotFound, Object, TextChannel
 from discord.ext.commands import Cog
 
@@ -177,21 +178,17 @@ async def on_message(self, message: Message) -> None:
 
         self.cache.append(message)
 
-        earliest_relevant_at = datetime.utcnow() - timedelta(seconds=self.max_interval)
-        relevant_messages = list(
-            takewhile(lambda msg: msg.created_at.replace(tzinfo=None) > earliest_relevant_at, self.cache)
-        )
+        earliest_relevant_at = arrow.utcnow() - timedelta(seconds=self.max_interval)
+        relevant_messages = list(takewhile(lambda msg: msg.created_at > earliest_relevant_at, self.cache))
 
         for rule_name in AntiSpamConfig.rules:
             rule_config = AntiSpamConfig.rules[rule_name]
             rule_function = RULE_FUNCTION_MAPPING[rule_name]
 
             # Create a list of messages that were sent in the interval that the rule cares about.
-            latest_interesting_stamp = datetime.utcnow() - timedelta(seconds=rule_config['interval'])
+            latest_interesting_stamp = arrow.utcnow() - timedelta(seconds=rule_config['interval'])
             messages_for_rule = list(
-                takewhile(
-                    lambda msg: msg.created_at.replace(tzinfo=None) > latest_interesting_stamp, relevant_messages
-                )
+                takewhile(lambda msg: msg.created_at > latest_interesting_stamp, relevant_messages)
             )
 
             result = await rule_function(message, messages_for_rule, rule_config)

bot/exts/filters/filtering.py

@@ -1,9 +1,10 @@
 import asyncio
 import re
-from datetime import datetime, timedelta
+from datetime import timedelta
 from typing import Any, Dict, List, Mapping, NamedTuple, Optional, Tuple, Union
 
-import dateutil
+import arrow
+import dateutil.parser
 import discord.errors
 import regex
 from async_rediscache import RedisCache
@@ -192,8 +193,8 @@ def get_name_matches(self, name: str) -> List[re.Match]:
     async def check_send_alert(self, member: Member) -> bool:
         """When there is less than 3 days after last alert, return `False`, otherwise `True`."""
         if last_alert := await self.name_alerts.get(member.id):
-            last_alert = datetime.utcfromtimestamp(last_alert)
-            if datetime.utcnow() - timedelta(days=DAYS_BETWEEN_ALERTS) < last_alert:
+            last_alert = arrow.get(last_alert)
+            if arrow.utcnow() - timedelta(days=DAYS_BETWEEN_ALERTS) < last_alert:
                 log.trace(f"Last alert was too recent for {member}'s nickname.")
                 return False
 
@@ -227,7 +228,7 @@ async def check_bad_words_in_name(self, member: Member) -> None:
             )
 
             # Update time when alert sent
-            await self.name_alerts.set(member.id, datetime.utcnow().timestamp())
+            await self.name_alerts.set(member.id, arrow.utcnow().timestamp())
 
     async def filter_eval(self, result: str, msg: Message) -> bool:
         """
@@ -603,25 +604,25 @@ async def notify_member(self, filtered_member: Member, reason: str, channel: Tex
 
     def schedule_msg_delete(self, msg: dict) -> None:
         """Delete an offensive message once its deletion date is reached."""
-        delete_at = dateutil.parser.isoparse(msg['delete_date']).replace(tzinfo=None)
+        delete_at = dateutil.parser.isoparse(msg['delete_date'])
         self.scheduler.schedule_at(delete_at, msg['id'], self.delete_offensive_msg(msg))
 
     async def reschedule_offensive_msg_deletion(self) -> None:
         """Get all the pending message deletion from the API and reschedule them."""
         await self.bot.wait_until_ready()
         response = await self.bot.api_client.get('bot/offensive-messages',)
 
-        now = datetime.utcnow()
+        now = arrow.utcnow()
 
         for msg in response:
-            delete_at = dateutil.parser.isoparse(msg['delete_date']).replace(tzinfo=None)
+            delete_at = dateutil.parser.isoparse(msg['delete_date'])
 
             if delete_at < now:
                 await self.delete_offensive_msg(msg)
             else:
                 self.schedule_msg_delete(msg)
 
-    async def delete_offensive_msg(self, msg: Mapping[str, str]) -> None:
+    async def delete_offensive_msg(self, msg: Mapping[str, int]) -> None:
         """Delete an offensive message, and then delete it from the db."""
         try:
             channel = self.bot.get_channel(msg['channel_id'])

bot/exts/fun/off_topic_names.py

@@ -1,6 +1,7 @@
 import difflib
-from datetime import datetime, timedelta
+from datetime import timedelta
 
+import arrow
 from discord import Colour, Embed
 from discord.ext.commands import Cog, Context, group, has_any_role
 from discord.utils import sleep_until
@@ -22,9 +23,9 @@ async def update_names(bot: Bot) -> None:
     while True:
         # Since we truncate the compute timedelta to seconds, we add one second to ensure
         # we go past midnight in the `seconds_to_sleep` set below.
-        today_at_midnight = datetime.utcnow().replace(microsecond=0, second=0, minute=0, hour=0)
+        today_at_midnight = arrow.utcnow().replace(microsecond=0, second=0, minute=0, hour=0)
         next_midnight = today_at_midnight + timedelta(days=1)
-        await sleep_until(next_midnight)
+        await sleep_until(next_midnight.datetime)
 
         try:
             channel_0_name, channel_1_name, channel_2_name = await bot.api_client.get(

bot/exts/moderation/defcon.py

@@ -4,6 +4,7 @@
 from enum import Enum
 from typing import Optional, Union
 
+import arrow
 from aioredis import RedisError
 from async_rediscache import RedisCache
 from dateutil.relativedelta import relativedelta
@@ -109,9 +110,9 @@ async def _sync_settings(self) -> None:
     async def on_member_join(self, member: Member) -> None:
         """Check newly joining users to see if they meet the account age threshold."""
         if self.threshold:
-            now = datetime.utcnow()
+            now = arrow.utcnow()
 
-            if now - member.created_at.replace(tzinfo=None) < relativedelta_to_timedelta(self.threshold):
+            if now - member.created_at < relativedelta_to_timedelta(self.threshold):
                 log.info(f"Rejecting user {member}: Account is too new")
 
                 message_sent = False
@@ -254,7 +255,8 @@ async def _update_threshold(
 
         expiry_message = ""
         if expiry:
-            expiry_message = f" for the next {humanize_delta(relativedelta(expiry, datetime.utcnow()), max_units=2)}"
+            activity_duration = relativedelta(expiry, arrow.utcnow().datetime)
+            expiry_message = f" for the next {humanize_delta(activity_duration, max_units=2)}"
 
         if self.threshold:
             channel_message = (

bot/exts/moderation/infraction/_scheduler.py

@@ -1,9 +1,9 @@
 import textwrap
 import typing as t
 from abc import abstractmethod
-from datetime import datetime
 from gettext import ngettext
 
+import arrow
 import dateutil.parser
 import discord
 from discord.ext.commands import Context
@@ -67,7 +67,7 @@ async def reschedule_infractions(self, supported_infractions: t.Container[str])
         # We make sure to fire this
         if to_schedule:
             next_reschedule_point = max(
-                dateutil.parser.isoparse(infr["expires_at"]).replace(tzinfo=None) for infr in to_schedule
+                dateutil.parser.isoparse(infr["expires_at"]) for infr in to_schedule
             )
             log.trace("Will reschedule remaining infractions at %s", next_reschedule_point)
 
@@ -83,8 +83,8 @@ async def reapply_infraction(
         """Reapply an infraction if it's still active or deactivate it if less than 60 sec left."""
         if infraction["expires_at"] is not None:
             # Calculate the time remaining, in seconds, for the mute.
-            expiry = dateutil.parser.isoparse(infraction["expires_at"]).replace(tzinfo=None)
-            delta = (expiry - datetime.utcnow()).total_seconds()
+            expiry = dateutil.parser.isoparse(infraction["expires_at"])
+            delta = (expiry - arrow.utcnow()).total_seconds()
         else:
             # If the infraction is permanent, it is not possible to get the time remaining.
             delta = None
@@ -382,7 +382,7 @@ async def deactivate_infraction(
 
         log.info(f"Marking infraction #{id_} as inactive (expired).")
 
-        expiry = dateutil.parser.isoparse(expiry).replace(tzinfo=None) if expiry else None
+        expiry = dateutil.parser.isoparse(expiry) if expiry else None
         created = time.format_infraction_with_duration(inserted_at, expiry)
 
         log_content = None
@@ -503,5 +503,5 @@ def schedule_expiration(self, infraction: _utils.Infraction) -> None:
         At the time of expiration, the infraction is marked as inactive on the website and the
         expiration task is cancelled.
         """
-        expiry = dateutil.parser.isoparse(infraction["expires_at"]).replace(tzinfo=None)
+        expiry = dateutil.parser.isoparse(infraction["expires_at"])
         self.scheduler.schedule_at(expiry, infraction["id"], self.deactivate_infraction(infraction))

bot/exts/moderation/infraction/management.py

@@ -315,7 +315,7 @@ def infraction_to_string(self, infraction: t.Dict[str, t.Any]) -> str:
             duration = "*Permanent*"
         else:
             date_from = datetime.fromtimestamp(float(time.DISCORD_TIMESTAMP_REGEX.match(created).group(1)))
-            date_to = dateutil.parser.isoparse(expires_at).replace(tzinfo=None)
+            date_to = dateutil.parser.isoparse(expires_at)
             duration = humanize_delta(relativedelta(date_to, date_from))
 
         lines = textwrap.dedent(f"""

bot/exts/moderation/modlog.py

@@ -2,7 +2,7 @@
 import difflib
 import itertools
 import typing as t
-from datetime import datetime
+from datetime import datetime, timezone
 from itertools import zip_longest
 
 import discord
@@ -58,7 +58,7 @@ async def upload_log(
             'bot/deleted-messages',
             json={
                 'actor': actor_id,
-                'creation': datetime.utcnow().isoformat(),
+                'creation': datetime.now(timezone.utc).isoformat(),
                 'deletedmessage_set': [
                     {
                         'id': message.id,
@@ -404,8 +404,8 @@ async def on_member_join(self, member: discord.Member) -> None:
         if member.guild.id != GuildConstant.id:
             return
 
-        now = datetime.utcnow()
-        difference = abs(relativedelta(now, member.created_at.replace(tzinfo=None)))
+        now = datetime.now(timezone.utc)
+        difference = abs(relativedelta(now, member.created_at))
 
         message = format_user(member) + "\n\n**Account age:** " + humanize_delta(difference)
 

bot/exts/moderation/modpings.py

@@ -1,5 +1,6 @@
 import datetime
 
+import arrow
 from async_rediscache import RedisCache
 from dateutil.parser import isoparse
 from discord import Embed, Member
@@ -57,7 +58,7 @@ async def reschedule_roles(self) -> None:
             if mod.id not in pings_off:
                 await self.reapply_role(mod)
             else:
-                expiry = isoparse(pings_off[mod.id]).replace(tzinfo=None)
+                expiry = isoparse(pings_off[mod.id])
                 self._role_scheduler.schedule_at(expiry, mod.id, self.reapply_role(mod))
 
     async def reapply_role(self, mod: Member) -> None:
@@ -92,7 +93,7 @@ async def off_command(self, ctx: Context, duration: Expiry) -> None:
 
         The duration cannot be longer than 30 days.
         """
-        delta = duration - datetime.datetime.utcnow()
+        delta = duration - arrow.utcnow()
         if delta > datetime.timedelta(days=30):
             await ctx.send(":x: Cannot remove the role for longer than 30 days.")
             return

bot/exts/moderation/voice_gate.py

@@ -1,7 +1,8 @@
 import asyncio
 from contextlib import suppress
-from datetime import datetime, timedelta
+from datetime import timedelta
 
+import arrow
 import discord
 from async_rediscache import RedisCache
 from discord import Colour, Member, VoiceState
@@ -166,8 +167,7 @@ async def voice_verify(self, ctx: Context, *_) -> None:
 
         checks = {
             "joined_at": (
-                ctx.author.joined_at.replace(tzinfo=None) > datetime.utcnow()
-                - timedelta(days=GateConf.minimum_days_member)
+                ctx.author.joined_at > arrow.utcnow() - timedelta(days=GateConf.minimum_days_member)
             ),
             "total_messages": data["total_messages"] < GateConf.minimum_messages,
             "voice_banned": data["voice_banned"],