Merge pull request #1068 from closeio/migrate-to-ms-graph-event-ical-uid

Migrate MS Events to use `iCalUId` for events created after cutoff date

Author: mrhiggi-close

inbox/config.py

@@ -1,6 +1,7 @@
 import errno
 import os
 
+import ciso8601
 import urllib3
 import yaml
 
@@ -124,6 +125,17 @@ def _update_config_from_env_variables(  # type: ignore[no-untyped-def]
     )
     config["CALENDAR_POLL_FREQUENCY"] = calendar_poll_frequencey
 
+    # MS Calendar events created before this cutoff will continue to use the legacy
+    # ID for all event association/deduplication. After the cutoff, the primary ID
+    # will be taken from iCalUId, to allow deduplication of events shared between
+    # calendars.
+    ms_graph_ical_uid_cutoff_str = os.environ.get(
+        "MS_GRAPH_ICAL_UID_CUTOFF", ""
+    ) or config.get("MS_GRAPH_ICAL_UID_CUTOFF", "2025-07-30T00:00:00Z")
+    config["MS_GRAPH_ICAL_UID_CUTOFF"] = ciso8601.parse_datetime(
+        ms_graph_ical_uid_cutoff_str
+    )
+
 
 def _get_process_name(config) -> None:  # type: ignore[no-untyped-def]
     if os.environ.get("PROCESS_NAME") is not None:

inbox/events/microsoft/events_provider.py

@@ -23,6 +23,7 @@
     validate_event,
 )
 from inbox.events.util import CalendarSyncResponse
+from inbox.logging import get_logger
 from inbox.models.account import Account
 from inbox.models.backends.outlook import MICROSOFT_CALENDAR_SCOPES
 from inbox.models.calendar import Calendar
@@ -46,6 +47,8 @@
 
 EVENT_FIELDS = [
     "id",
+    "iCalUId",
+    "createdDateTime",
     "type",
     "subject",
     "start",
@@ -154,27 +157,32 @@ def sync_events(
 
             if isinstance(event, RecurringEvent):
                 (exceptions, cancellations) = self._get_event_overrides(
-                    raw_event, event, read_only=read_only
+                    raw_event,
+                    event,
+                    ms_graph_event_id=raw_event["id"],
+                    read_only=read_only,
                 )
                 updates.extend(exceptions)  # type: ignore[arg-type]
                 updates.extend(cancellations)  # type: ignore[arg-type]
 
         return updates
 
-    def _get_event_overrides(  # type: ignore[no-untyped-def]
+    def _get_event_overrides(
         self,
         raw_master_event: MsGraphEvent,
         master_event: RecurringEvent,
         *,
-        read_only,
+        ms_graph_event_id: str,
+        read_only: bool,
     ) -> tuple[list[MsGraphEvent], list[MsGraphEvent]]:
         """
         Fetch recurring event instances and determine exceptions and cancellations.
 
         Arguments:
-            raw_master_event: Recurring master event as retruend by the API
-            master_event: Parsed recurring master event as ORM object
-            read_only: Does master event come from read-only calendar
+            raw_master_event: Recurring master event as returned by the API
+            master_event: Parsed recurring master event as an ORM object
+            ms_graph_event_id: Microsoft Graph event ID for use with Graph API
+            read_only: Does master event come from a read-only calendar
 
         Returns:
             Tuple of exceptions and cancellations
@@ -184,15 +192,28 @@ def _get_event_overrides(  # type: ignore[no-untyped-def]
 
         start = master_event.start
         end = start + MAX_RECURRING_EVENT_WINDOW
-
+        log = get_logger()
+        log.info(
+            "Getting overrides for event",
+            event_uid=master_event.uid,
+            event_ms_graph_id=ms_graph_event_id,
+        )
         raw_occurrences = cast(
             list[MsGraphEvent],
             list(
                 self.client.iter_event_instances(
-                    master_event.uid, start=start, end=end, fields=EVENT_FIELDS
+                    ms_graph_event_id,
+                    start=start,
+                    end=end,
+                    fields=EVENT_FIELDS,
                 )
             ),
         )
+        log.info(
+            "got overrides for event",
+            event_id=master_event.uid,
+            occurrence_count=len(raw_occurrences),
+        )
         (raw_exceptions, raw_cancellations) = (
             calculate_exception_and_canceled_occurrences(
                 raw_master_event, raw_occurrences, end

inbox/events/microsoft/graph_types.py

@@ -200,7 +200,9 @@ class MsGraphEvent(TypedDict):
     """
 
     id: str
+    iCalUId: str
     type: MsGraphEventType
+    createdDateTime: str
     start: MsGraphDateTimeTimeZone
     originalStart: str
     end: MsGraphDateTimeTimeZone

inbox/events/microsoft/parse.py

@@ -10,6 +10,7 @@
 import pytz
 import pytz.tzinfo
 
+from inbox.config import config
 from inbox.events.microsoft.graph_types import (
     ICalDayOfWeek,
     ICalFreq,
@@ -25,6 +26,7 @@
     MsGraphWeekIndex,
 )
 from inbox.events.timezones import windows_timezones
+from inbox.logging import get_logger
 from inbox.models.calendar import Calendar
 from inbox.models.event import Event
 from inbox.util.html import strip_tags
@@ -415,6 +417,11 @@ def synthesize_canceled_occurrence(
         + "-synthesizedCancellation-"
         + start_datetime.date().isoformat()
     )
+    cancellation_ical_uid = (
+        master_event["iCalUId"]
+        + "-synthesizedCancellation-"
+        + start_datetime.date().isoformat()
+    )
     cancellation_start = dump_datetime_as_msgraph_datetime_tz(start_datetime)
     assert start_datetime.tzinfo == pytz.UTC
     original_start = start_datetime.replace(tzinfo=None).isoformat() + "Z"
@@ -425,9 +432,13 @@ def synthesize_canceled_occurrence(
         start_datetime + duration
     )
 
+    # As this is still an MSGraphEvent, we need to maintain separate id and
+    # iCalUId. The decision for which to use will be made when parsing this into
+    # a sync engine internal Event model.
     result = {
         **master_event,
         "id": cancellation_id,
+        "iCalUId": cancellation_ical_uid,
         "type": "synthesizedCancellation",
         "isCancelled": True,
         "recurrence": None,
@@ -692,8 +703,19 @@ def parse_event(
     ]:
         assert master_event_uid
         assert event["type"] in ["exception", "synthesizedCancellation"]
-
-    uid = event["id"]
+    log = get_logger()
+    log.info(
+        "Parsing event",
+        event_ical_id=event["iCalUId"],
+        event_uid=event["id"],
+        created_at=event["createdDateTime"],
+        title=event["subject"],
+    )
+    ical_uid = event["iCalUId"]
+    event_id = event["id"]
+    created_date_time = datetime.datetime.fromisoformat(
+        event["createdDateTime"]
+    )
     raw_data = json.dumps(event)
     title = event["subject"] or ""
     start = parse_msgraph_datetime_tz_as_utc(event["start"])
@@ -744,7 +766,11 @@ def parse_event(
         original_start = None
 
     return Event.create(
-        uid=uid,
+        uid=(
+            ical_uid
+            if created_date_time > config["MS_GRAPH_ICAL_UID_CUTOFF"]
+            else event_id
+        ),
         raw_data=raw_data,
         title=title,
         description=description,