Merge pull request #671 from praekeltfoundation/update-mcxp-migration-scripts

Update MCxP migration scripts

Author: erikh360

scripts/migrate_to_turn/README.md

@@ -17,7 +17,7 @@ The `FIELD_MAPPING` variable should be updated with all the fields we want to mo
 
 The script will write all the contacts to a file with the start and end date in the name.
 
-It will also output the latest modified on date in the batch, this can then be used as a start date to get the next batch.
+It will also output the oldest modified on date in the batch, this can then be used as an end date to get the next batch when syncing new to old.
 
 ### update_turn_contacts.py
 
@@ -82,9 +82,9 @@ The key is the field name in Rapidpro, the value determinues the rest:
 1. Make sure the `FIELD_MAPPING` is configured
 1. Set the required environment variables: `TURN_TOKEN` and `RAPIDPRO_TOKEN`.
 1. Update the start date, end date, limit and field mapping in `fetch_rapidpro_contacts.py` script
-1. Run `python scripts/migrate_to_turn/fetch_rapidpro_contacts.py` and take note of the last modified date and the filename.
+1. Run `python scripts/migrate_to_turn/fetch_rapidpro_contacts.py` and take note of the oldest modified date and the filename.
 1. Run `python scripts/migrate_to_turn/update_turn_contacts.py contacts-2025-01-01-2025-01-07.csv > update_turn_contacts.json`
 1. Use jq to check if there were any errors `jq .response.status update_turn_contacts.json | sort | uniq -c`
 1. To retry errors, run `cat update_turn_contacts.json | python scripts/migrate_to_rapidpro/retry_requests.py > update_turn_contacts2.json`
 1. Repeat previous two steps until all contacts successfully completed.
-1. Update the start and end date in `fetch_rapidpro_contacts.py` script. Repeat from step 3.
+1. Update the end date in `fetch_rapidpro_contacts.py` script to the oldest modified date from the previous batch. Repeat from step 3.

scripts/migrate_to_turn/fetch_rapidpro_contacts.py

@@ -3,20 +3,42 @@
 from datetime import datetime
 
 import pytz
-from process_fields import process_datetime
+from process_fields import (
+    get_user_babies,
+    get_user_tier,
+    get_user_type,
+    process_datetime,
+    to_lowercase,
+)
 from temba_client.v2 import TembaClient
 
 RAPIDPRO_URL = "https://rapidpro.qa.momconnect.co.za"
 
-START_DATE = "2025-01-20 01:13:06"
-END_DATE = "2025-01-20 19:13:06"
+START_DATE = "2025-11-01 01:13:06"
+END_DATE = "2026-01-12 19:13:06"
 LIMIT = 1000
 
 # TODO: add all the fields here: <rapidpro-field-name>: <details>
 FIELD_MAPPING = {
-    "edd": {"turn_name": "test", "process": process_datetime, "type": "custom"},
+    "edd": {
+        "turn_name": "pregnancy_expected_due_date",
+        "process": process_datetime,
+        "type": "custom",
+    },
     "name": {"turn_name": "name", "type": "default"},
     "language": {"turn_name": "language", "type": "default"},
+    "research_consent": {
+        "turn_name": "research_consent",
+        "process": to_lowercase,
+        "type": "custom",
+    },
+    "clinic_code": {"turn_name": "clinic_code", "type": "custom"},
+}
+
+NEW_TURN_FIELD_MAPPING = {
+    "user_tier": {"process": get_user_tier},
+    "user_type": {"process": get_user_type},
+    "babies": {"process": get_user_babies},
 }
 
 client = TembaClient(RAPIDPRO_URL, os.environ["RAPIDPRO_TOKEN"])
@@ -30,11 +52,14 @@ def get_field_data(contact):
         if turn_details["type"] == "default":
             data[turn_field] = getattr(contact, rapidpro_field)
         else:
-            data[turn_field] = contact.fields[rapidpro_field]
+            data[turn_field] = contact.fields.get(rapidpro_field)
 
         if "process" in turn_details:
             data[turn_field] = turn_details["process"](data[turn_field])
 
+    for new_field, details in NEW_TURN_FIELD_MAPPING.items():
+        data[new_field] = details["process"](contact)
+
     return data
 
 
@@ -52,9 +77,9 @@ def is_opted_out(contact):
 def get_rapidpro_contacts(start_date=None, end_date=None):
     print(f"> Getting rapidpro contacts from {start_date} to {end_date}")
     contacts = []
-    latest_date = start_date.replace(tzinfo=pytz.utc)
+    oldest_date = end_date.replace(tzinfo=pytz.utc)
     for contact_batch in client.get_contacts(
-        before=end_date, after=start_date, reverse=True
+        before=end_date, after=start_date
     ).iterfetches(retry_on_rate_exceed=True):
         for contact in contact_batch:
             wa_id = get_wa_id(contact)
@@ -68,23 +93,23 @@ def get_rapidpro_contacts(start_date=None, end_date=None):
                 contacts.append(data)
 
                 modified_on = contact.modified_on.astimezone(pytz.utc)
-                if modified_on > latest_date:
-                    latest_date = modified_on
+                if modified_on < oldest_date:
+                    oldest_date = modified_on
 
                 if len(contacts) >= LIMIT:
-                    return contacts, latest_date
+                    return contacts, oldest_date
 
-    return contacts, latest_date
+    return contacts, oldest_date
 
 
 if __name__ == "__main__":
     start_date = datetime.strptime(START_DATE, "%Y-%m-%d %H:%M:%S")
     end_date = datetime.strptime(END_DATE, "%Y-%m-%d %H:%M:%S")
 
-    contacts, latest_date = get_rapidpro_contacts(start_date, end_date)
+    contacts, oldest_date = get_rapidpro_contacts(start_date, end_date)
 
     print(f"Found: {len(contacts)}")
-    print(f"Latest modified on date: {latest_date}")
+    print(f"Oldest modified on date: {oldest_date}")
 
     if contacts:
         start = START_DATE.split(" ")[0]

scripts/migrate_to_turn/process_fields.py

@@ -1,3 +1,4 @@
+import json
 from datetime import datetime
 
 import pytz
@@ -11,6 +12,12 @@ def is_datetime(date):
     return True
 
 
+def to_lowercase(value):
+    if isinstance(value, str):
+        return value.lower()
+    return value
+
+
 def process_datetime(value):
     if not value:
         return
@@ -20,3 +27,46 @@ def process_datetime(value):
         value = datetime.fromisoformat(value)
         value = value.astimezone(pytz.utc).isoformat()
         return value
+
+
+def get_user_tier(contact):
+    # TODO: figure out user tier based on rapidpro fields
+    return None
+
+
+def get_user_type(contact):
+    # TODO: figure out user type based on rapidpro fields
+    return None
+
+
+def get_user_babies(contact):
+    babies = []
+    fields = getattr(contact, "fields", {}) or {}
+    for index in range(1, 4):
+        dob = fields.get(f"baby_dob{index}")
+        if not dob:
+            continue
+
+        value = dob.replace("Z", "").split("+")[0]
+        if not is_datetime(value):
+            continue
+
+        birth_date = datetime.fromisoformat(value)
+        babies.append(
+            {
+                "all_vaccines_received": "all",
+                "appointments_attended": 0,
+                "baby_birth_day": birth_date.day,
+                "baby_birth_month": birth_date.month,
+                "baby_birth_year": birth_date.year,
+                "name": "",
+                "next_vacc_day": 0,
+                "next_vacc_month": 0,
+                "next_vacc_year": 0,
+                "pregnancy_edd": 0,
+                "vaccination_status_at_reg": "all",
+            }
+        )
+
+    if babies:
+        return json.dumps(babies)

scripts/migrate_to_turn/tests/test_process_fields.py

@@ -0,0 +1,103 @@
+import json
+from datetime import datetime
+from unittest import TestCase
+
+import pytz
+
+from scripts.migrate_to_turn.process_fields import (
+    get_user_babies,
+    get_user_tier,
+    get_user_type,
+    is_datetime,
+    process_datetime,
+)
+
+
+class IsDatetimeTests(TestCase):
+    def test_valid_isoformat(self):
+        """
+        Returns True for valid ISO format strings
+        """
+        self.assertTrue(is_datetime("2024-01-01"))
+        self.assertTrue(is_datetime("2024-01-01T12:30:00"))
+
+    def test_invalid_isoformat(self):
+        """
+        Returns False for invalid inputs
+        """
+        self.assertFalse(is_datetime("not-a-date"))
+        self.assertFalse(is_datetime(None))
+
+
+class ProcessDatetimeTests(TestCase):
+    def test_none_returns_none(self):
+        """
+        Returns None for empty values
+        """
+        self.assertIsNone(process_datetime(None))
+        self.assertIsNone(process_datetime(""))
+
+    def test_invalid_returns_none(self):
+        """
+        Returns None for invalid values
+        """
+        self.assertIsNone(process_datetime("not-a-date"))
+
+    def test_strips_timezone_and_converts_to_utc(self):
+        """
+        Removes timezone suffixes and normalizes to UTC
+        """
+        for value in ("2024-01-01T12:30:00Z", "2024-01-01T12:30:00+02:00"):
+            with self.subTest(value=value):
+                expected = datetime.fromisoformat("2024-01-01T12:30:00").astimezone(
+                    pytz.utc
+                )
+                self.assertEqual(process_datetime(value), expected.isoformat())
+
+
+class GetUserTierTests(TestCase):
+    def test_default(self):
+        """
+        Default behavior returns None
+        """
+        self.assertIsNone(get_user_tier(object()))
+
+
+class GetUserTypeTests(TestCase):
+    def test_default(self):
+        """
+        Default behavior returns None
+        """
+        self.assertIsNone(get_user_type(object()))
+
+
+class GetUserBabiesTests(TestCase):
+    def test_default(self):
+        """
+        Default behavior returns None
+        """
+        self.assertIsNone(get_user_babies(object()))
+
+    def test_baby_dobs_convert_to_babies(self):
+        """
+        Converts baby dob fields into babies payload
+        """
+        contact = type("Contact", (), {"fields": {"baby_dob1": "2025-01-02"}})()
+
+        baby_list = [
+            {
+                "all_vaccines_received": "all",
+                "appointments_attended": 0,
+                "baby_birth_day": 2,
+                "baby_birth_month": 1,
+                "baby_birth_year": 2025,
+                "name": "",
+                "next_vacc_day": 0,
+                "next_vacc_month": 0,
+                "next_vacc_year": 0,
+                "pregnancy_edd": 0,
+                "vaccination_status_at_reg": "all",
+            }
+        ]
+
+        self.assertEqual(get_user_babies(contact), json.dumps(baby_list))