Merge pull request #675 from praekeltfoundation/update-turn-migration-scripts

Small bug fixes for fetch and migrate to Turn

Author: erikh360

poetry.lock

@@ -63,6 +63,7 @@ dev = [
     "responses>=0.23.1",
     "pre-commit>=3.2.0",
     "freezegun>=1.2.2",
+    "aiohttp (>=3.13.3,<4.0.0)",
 ]
 
 [tool.pytest.ini_options]

pyproject.toml

@@ -19,8 +19,16 @@
 RAPIDPRO_URL = "https://rapidpro.qa.momconnect.co.za"
 
 START_DATE = "2025-11-01 01:13:06"
-END_DATE = "2026-01-12 19:13:06"
+END_DATE = "2026-01-22 19:13:06"
 LIMIT = 1000
+INCLUDE_OPTED_OUT = False
+
+MSISDN_FILTER = (
+    os.environ.get("MSISDN_FILTER", "").split(",")
+    if os.environ.get("MSISDN_FILTER")
+    else []
+)
+
 
 FIELD_MAPPING = {
     "edd": {
@@ -118,7 +126,11 @@ def get_rapidpro_contacts(client, start_date=None, end_date=None):
         for contact in contact_batch:
             wa_id = get_wa_id(contact)
 
-            if is_opted_out(contact):
+            if is_opted_out(contact) and not INCLUDE_OPTED_OUT:
+                continue
+
+            # TODO: filtering for testing only, remove later()
+            if wa_id and MSISDN_FILTER and wa_id not in MSISDN_FILTER:
                 continue
 
             if wa_id:

scripts/migrate_to_turn/fetch_rapidpro_contacts.py

@@ -84,7 +84,7 @@ def get_truthy_field(field_name):
     if opted_out:
         user_type = "deregistered_user"
     elif prebirth_messaging or postbirth_messaging:
-        user_type = "push_comprehensive_user"
+        user_type = "comprehensive_user"
 
         if (
             not prebirth_messaging

scripts/migrate_to_turn/process_fields.py

@@ -132,3 +132,75 @@ def test_fetch_rapidpro_contacts_writes_csv(self):
         }
 
         self.assertEqual(rows[0], expected_row)
+
+    def test_fetch_rapidpro_contacts_skips_opted_out(self):
+        contact = type(
+            "Contact",
+            (),
+            {
+                "fields": {"opted_out": "TRUE"},
+                "urns": ["whatsapp:27820000000"],
+                "modified_on": datetime(2025, 1, 3, 12, 0, 0, tzinfo=pytz.utc),
+            },
+        )()
+
+        client = FakeClient([[contact]])
+        start_date = "2025-01-01 00:00:00"
+        end_date = "2025-01-31 00:00:00"
+
+        with tempfile.TemporaryDirectory() as tmp_dir, chdir(tmp_dir):
+            with (
+                patch.object(fetch_rapidpro_contacts, "START_DATE", start_date),
+                patch.object(fetch_rapidpro_contacts, "END_DATE", end_date),
+            ):
+                fetch_rapidpro_contacts.fetch_rapidpro_contacts(client)
+
+            output_path = Path(tmp_dir) / "contacts-2025-01-01-2025-01-31.csv"
+            self.assertFalse(output_path.exists())
+
+    def test_fetch_rapidpro_contacts_msisdn_filter(self):
+        contact1 = type(
+            "Contact",
+            (),
+            {
+                "fields": {"opted_out": "FALSE"},
+                "urns": ["whatsapp:27820000001"],
+                "modified_on": datetime(2025, 1, 3, 12, 0, 0, tzinfo=pytz.utc),
+            },
+        )()
+
+        contact2 = type(
+            "Contact",
+            (),
+            {
+                "fields": {"opted_out": "FALSE"},
+                "urns": ["whatsapp:27820000002"],
+                "modified_on": datetime(2025, 1, 4, 12, 0, 0, tzinfo=pytz.utc),
+                "name": "Alice",
+                "language": "eng",
+            },
+        )()
+
+        client = FakeClient([[contact1, contact2]])
+        start_date = "2025-01-01 00:00:00"
+        end_date = "2025-01-31 00:00:00"
+
+        with tempfile.TemporaryDirectory() as tmp_dir, chdir(tmp_dir):
+            with (
+                patch.object(fetch_rapidpro_contacts, "START_DATE", start_date),
+                patch.object(fetch_rapidpro_contacts, "END_DATE", end_date),
+                patch.object(
+                    fetch_rapidpro_contacts,
+                    "MSISDN_FILTER",
+                    ["27820000002"],
+                ),
+            ):
+                fetch_rapidpro_contacts.fetch_rapidpro_contacts(client)
+
+            output_path = Path(tmp_dir) / "contacts-2025-01-01-2025-01-31.csv"
+            with output_path.open(newline="") as csv_file:
+                reader = csv.DictReader(csv_file)
+                rows = list(reader)
+
+        self.assertEqual(len(rows), 1)
+        self.assertEqual(rows[0]["urn"], "27820000002")

scripts/migrate_to_turn/tests/test_fetch_rapidpro_contacts.py

@@ -195,22 +195,22 @@ def test_opted_out_is_deregistered(self):
         contact = type("Contact", (), {"fields": {"opted_out": "TRUE"}})()
         self.assertEqual(get_user_type(contact), "deregistered_user")
 
-    def test_prebirth_messaging_is_push_comprehensive(self):
+    def test_prebirth_messaging_is_comprehensive(self):
         """
-        Prebirth messaging users are push comprehensive
+        Prebirth messaging users are comprehensive
         """
         contact = type("Contact", (), {"fields": {"prebirth_messaging": "TRUE"}})()
-        self.assertEqual(get_user_type(contact), "push_comprehensive_user")
+        self.assertEqual(get_user_type(contact), "comprehensive_user")
 
-    def test_postbirth_with_active_baby_is_push_comprehensive(self):
+    def test_postbirth_with_active_baby_is_comprehensive(self):
         """
-        Postbirth users with active baby are push comprehensive
+        Postbirth users with active baby are comprehensive
         """
         contact = type("Contact", (), {"fields": {"postbirth_messaging": "TRUE"}})()
         with mock.patch(
             "scripts.migrate_to_turn.process_fields.has_active_baby", return_value=True
         ):
-            self.assertEqual(get_user_type(contact), "push_comprehensive_user")
+            self.assertEqual(get_user_type(contact), "comprehensive_user")
 
     def test_postbirth_without_active_baby_is_alumni(self):
         """