Add data migration to lowercase emails, and only ingest lowercase emails. (#915)

* Add data migration to lowercase emails, and only ingest lowercase emails.

* Lint test_waiting_list

* Remove redundant calls

Author: MelissaAutumn

backend/src/appointment/database/repo/subscriber.py

@@ -19,7 +19,7 @@ def get(db: Session, subscriber_id: int) -> models.Subscriber | None:
 
 def get_by_email(db: Session, email: str) -> models.Subscriber | None:
     """retrieve subscriber by email"""
-    return db.query(models.Subscriber).filter(models.Subscriber.email == email).first()
+    return db.query(models.Subscriber).filter(models.Subscriber.email == email.lower()).first()
 
 
 def get_by_username(db: Session, username: str):
@@ -51,6 +51,10 @@ def create(db: Session, subscriber: schemas.SubscriberBase):
     """create new subscriber"""
     data = subscriber.model_dump()
 
+    data['email'] = data.get('email').lower()
+    if data.get('secondary_email'):
+        data['secondary_email'] = data.secondary_email.lower()
+
     # Filter incoming data to just the available model columns
     columns = models.Subscriber().get_columns()
     data = {k: v for k, v in data.items() if k in columns}
@@ -68,6 +72,9 @@ def update(db: Session, data: schemas.SubscriberIn, subscriber_id: int):
     """update all subscriber attributes, they can edit themselves"""
     db_subscriber = get(db, subscriber_id)
     for key, value in data:
+        if (key == 'email' or key == 'secondary_email') and value is not None:
+            value = value.lower()
+
         if value is not None:
             setattr(db_subscriber, key, value)
     db.commit()

backend/src/appointment/migrations/versions/2025_03_07_2109-537672c3933d_data_migration_clean_up_email_.py

@@ -0,0 +1,45 @@
+"""data migration clean up email case
+
+Revision ID: 537672c3933d
+Revises: 330fdd8cd0f8
+Create Date: 2025-03-07 21:09:16.541016
+
+"""
+
+from alembic import op
+from sqlalchemy.orm import Session
+
+from appointment.database import models
+
+# revision identifiers, used by Alembic.
+revision = '537672c3933d'
+down_revision = '330fdd8cd0f8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    session = Session(op.get_bind())
+    subscribers: list[models.Subscriber] = session.query(models.Subscriber).all()
+    for subscriber in subscribers:
+        # Lowercase the emails
+        subscriber.email = subscriber.email.lower()
+        if subscriber.secondary_email:
+            subscriber.secondary_email = subscriber.secondary_email.lower()
+
+        # Add the subscriber to the database session and commit (update) it
+        session.add(subscriber)
+        session.commit()
+
+    waiting_list: list[models.WaitingList] = session.query(models.WaitingList).all()
+    for waiting_list_individual in waiting_list:
+        # Lowercase the emails
+        waiting_list_individual.email = waiting_list_individual.email.lower()
+
+        # Add the waiting list individual to the database session and commit (update) it
+        session.add(waiting_list_individual)
+        session.commit()
+
+
+def downgrade() -> None:
+    pass

backend/src/appointment/routes/api.py

@@ -66,7 +66,7 @@ def update_me(
     me = repo.subscriber.update(db=db, data=data, subscriber_id=subscriber.id)
     return schemas.SubscriberMeOut(
         username=me.username,
-        email=me.email,
+        email=me.email.lower(),
         preferred_email=me.preferred_email,
         name=me.name,
         level=me.level,

backend/src/appointment/routes/auth.py

@@ -70,7 +70,7 @@ def can_login(data: schemas.CheckEmail, db: Session = Depends(get_db), fxa_clien
     """Determines if a user can go through the login flow"""
     if os.getenv('AUTH_SCHEME') == 'fxa':
         # This checks if a subscriber exists, or is in allowed list
-        return fxa_client.is_in_allow_list(db, data.email)
+        return fxa_client.is_in_allow_list(db, data.email.lower())
 
     # There's no waiting list setting on password login
     return True
@@ -158,7 +158,7 @@ def fxa_callback(
             f"{os.getenv('FRONTEND_URL', 'http://localhost:8080')}/login/?error={errors['email-not-in-session']}"
         )
 
-    email = request.session['fxa_user_email']
+    email = request.session['fxa_user_email'].lower()
     # We only use timezone during subscriber creation, or if their timezone is None
     timezone = request.session['fxa_user_timezone']
     invite_code = request.session.get('fxa_user_invite_code')
@@ -176,7 +176,7 @@ def fxa_callback(
     creds = fxa_client.get_credentials(code)
     profile = fxa_client.get_profile()
 
-    if profile['email'] != email:
+    if email == '' or profile.get('email', '').lower() != email:
         fxa_client.logout()
         return RedirectResponse(
             f"{os.getenv('FRONTEND_URL', 'http://localhost:8080')}/login/?error={errors['email-mismatch']}"

backend/src/appointment/routes/google.py

@@ -47,6 +47,8 @@ def google_auth(
     subscriber: Subscriber = Depends(get_subscriber),
 ):
     """Starts the google oauth process"""
+    if email:
+        email = email.lower()
     url, state = google_client.get_redirect_url(email)
 
     request.session[SESSION_OAUTH_STATE] = state
@@ -168,7 +170,7 @@ def disconnect_account(
     repo.calendar.delete_by_subscriber_and_provider(db, subscriber.id, provider=models.CalendarProvider.google)
 
     # Unassociated any secondary emails if they're attached to their google connection
-    if subscriber.secondary_email == google_connection.name:
+    if subscriber.secondary_email == google_connection.name.lower():
         subscriber.secondary_email = None
         db.add(subscriber)
         db.commit()

backend/src/appointment/routes/waiting_list.py

@@ -35,18 +35,19 @@ def join_the_waiting_list(
     db: Session = Depends(get_db)
 ):
     """Join the waiting list!"""
-    added = repo.invite.add_to_waiting_list(db, data.email)
+    email = data.email.lower()
+    added = repo.invite.add_to_waiting_list(db, email)
 
     # TODO: Replace our signed functionality with this
     # Not timed since we don't have a mechanism to re-send these...
     serializer = URLSafeSerializer(os.getenv('SIGNED_SECRET'), 'waiting-list')
-    confirm_token = serializer.dumps({'email': data.email, 'action': WaitingListAction.CONFIRM_EMAIL.value})
-    decline_token = serializer.dumps({'email': data.email, 'action': WaitingListAction.LEAVE.value})
+    confirm_token = serializer.dumps({'email': email, 'action': WaitingListAction.CONFIRM_EMAIL.value})
+    decline_token = serializer.dumps({'email': email, 'action': WaitingListAction.LEAVE.value})
 
     # If they were added, send the email
     if added:
         background_tasks.add_task(
-            send_confirm_email, to=data.email, confirm_token=confirm_token, decline_token=decline_token
+            send_confirm_email, to=email, confirm_token=confirm_token, decline_token=decline_token
         )
 
     return added
@@ -68,6 +69,8 @@ def act_on_waiting_list(data: schemas.TokenForWaitingList, db: Session = Depends
     if action is None or email is None:
         raise validation.InvalidLinkException()
 
+    email = email.lower()
+
     if action == WaitingListAction.CONFIRM_EMAIL.value:
         success = repo.invite.confirm_waiting_list_email(db, email)
     elif action == WaitingListAction.LEAVE.value:

backend/src/appointment/routes/webhooks.py

@@ -53,7 +53,7 @@ def fxa_process(
             case 'https://schemas.accounts.firefox.com/event/profile-change':
                 if event_data.get('email') is not None:
                     # Update the subscriber's email, we do this first in case there's a problem with get_profile()
-                    subscriber.email = event_data.get('email')
+                    subscriber.email = event_data.get('email').lower()
                     db.add(subscriber)
                     db.commit()
 

backend/test/integration/test_auth.py

@@ -13,6 +13,27 @@
 
 
 class TestAuth:
+    def test_can_login(self, with_db, with_client, make_pro_subscriber, faker):
+        # Request can-login with a capitalized email
+        capital_email = faker.email().capitalize()
+        subscriber = make_pro_subscriber(email=capital_email)
+
+        assert subscriber.email != capital_email
+        assert subscriber.email == capital_email.lower()
+
+        with with_db() as db:
+            # Check we've saved the email as lowercase
+            subscriber_check = repo.subscriber.get_by_email(db, capital_email.lower())
+            assert subscriber_check is not None
+            assert subscriber_check.email != capital_email
+            assert subscriber_check.email == capital_email.lower()
+
+        response = with_client.post('/can-login', json={'email': subscriber.email})
+
+        assert response.status_code == 200, response.text
+        data = response.json()
+        assert data is True
+
     def test_me(self, with_db, with_client):
         response = with_client.get('/me', headers=auth_headers)
         assert response.status_code == 200, response.text
@@ -198,21 +219,14 @@ def test_fxa_with_allowlist_and_with_bad_invite_code(self, with_client, with_l10
         email = '[email protected]'
         response = with_client.get(
             '/fxa_login',
-            params={
-                'email': email,
-                'invite_code': 'absolute nonsense!'
-            },
+            params={'email': email, 'invite_code': 'absolute nonsense!'},
         )
         assert response.status_code == 404, response.text
         data = response.json()
         assert data.get('detail') == l10n('invite-code-not-valid')
 
     def test_fxa_with_allowlist_and_with_used_invite_code(
-        self,
-        with_client,
-        with_l10n,
-        make_invite,
-        make_pro_subscriber
+        self, with_client, with_l10n, make_invite, make_pro_subscriber
     ):
         os.environ['AUTH_SCHEME'] = 'fxa'
         os.environ['FXA_ALLOW_LIST'] = '@example.org'
@@ -223,10 +237,7 @@ def test_fxa_with_allowlist_and_with_used_invite_code(
         email = '[email protected]'
         response = with_client.get(
             '/fxa_login',
-            params={
-                'email': email,
-                'invite_code': invite.code
-            },
+            params={'email': email, 'invite_code': invite.code},
         )
         assert response.status_code == 403, response.text
         data = response.json()
@@ -433,17 +444,12 @@ def test_fxa_token_success(self, make_basic_subscriber, with_client):
 
         subscriber = make_basic_subscriber(email='[email protected]')
         access_token_expires = timedelta(minutes=float(10))
-        one_time_access_token = create_access_token(data={
-            'sub': f'uid-{subscriber.id}',
-            'jti': secrets.token_urlsafe(16)
-        }, expires_delta=access_token_expires)
+        one_time_access_token = create_access_token(
+            data={'sub': f'uid-{subscriber.id}', 'jti': secrets.token_urlsafe(16)}, expires_delta=access_token_expires
+        )
 
         # Exchange the one-time token with a long-living token
-        response = with_client.post(
-            '/fxa-token', headers={
-                'Authorization': f'Bearer {one_time_access_token}'
-            }
-        )
+        response = with_client.post('/fxa-token', headers={'Authorization': f'Bearer {one_time_access_token}'})
 
         assert response.status_code == 200, response.text
 
@@ -454,11 +460,7 @@ def test_fxa_token_success(self, make_basic_subscriber, with_client):
         assert data.get('token_type') == 'bearer'
 
         # Test it out!
-        response = with_client.get(
-            '/me', headers={
-                'Authorization': f'Bearer {access_token}'
-            }
-        )
+        response = with_client.get('/me', headers={'Authorization': f'Bearer {access_token}'})
 
         assert response.status_code == 200, response.text
         assert response.json().get('email') == subscriber.email
@@ -471,16 +473,15 @@ def test_fxa_token_failed_due_to_non_one_time_token(self, make_basic_subscriber,
 
         subscriber = make_basic_subscriber(email='[email protected]')
         access_token_expires = timedelta(minutes=float(10))
-        regular_access_token = create_access_token(data={
-            'sub': f'uid-{subscriber.id}',
-        }, expires_delta=access_token_expires)
-
-        response = with_client.post(
-            '/fxa-token', headers={
-                'Authorization': f'Bearer {regular_access_token}'
-            }
+        regular_access_token = create_access_token(
+            data={
+                'sub': f'uid-{subscriber.id}',
+            },
+            expires_delta=access_token_expires,
         )
 
+        response = with_client.post('/fxa-token', headers={'Authorization': f'Bearer {regular_access_token}'})
+
         assert response.status_code == 401, response.text
 
     def test_fxa_token_failed_due_to_empty_auth(self, make_basic_subscriber, with_client):
@@ -489,9 +490,7 @@ def test_fxa_token_failed_due_to_empty_auth(self, make_basic_subscriber, with_cl
 
         del with_client.app.dependency_overrides[auth.get_subscriber]
 
-        response = with_client.post(
-            '/fxa-token'
-        )
+        response = with_client.post('/fxa-token')
 
         assert response.status_code == 401, response.text
 
@@ -504,17 +503,12 @@ def test_fxa_token_failed_due_to_invalid_auth_scheme(self, with_client, make_bas
 
         subscriber = make_basic_subscriber(email='[email protected]')
         access_token_expires = timedelta(minutes=float(10))
-        one_time_access_token = create_access_token(data={
-            'sub': f'uid-{subscriber.id}',
-            'jti': secrets.token_urlsafe(16)
-        }, expires_delta=access_token_expires)
+        one_time_access_token = create_access_token(
+            data={'sub': f'uid-{subscriber.id}', 'jti': secrets.token_urlsafe(16)}, expires_delta=access_token_expires
+        )
 
         # Exchange the one-time token with a long-living token
-        response = with_client.post(
-            '/fxa-token', headers={
-                'Authorization': f'Bearer {one_time_access_token}'
-            }
-        )
+        response = with_client.post('/fxa-token', headers={'Authorization': f'Bearer {one_time_access_token}'})
         os.environ['AUTH_SCHEME'] = saved_scheme
         assert response.status_code == 405, response.text
 
@@ -530,17 +524,15 @@ def test_auth(self, with_db, with_client):
             assert len(ecs) == 0
 
         with patch('appointment.controller.calendar.Tools.dns_caldav_lookup') as mock:
-            mock.return_value = "https://example.com", 300
+            mock.return_value = 'https://example.com', 300
 
             with patch('appointment.controller.calendar.CalDavConnector.sync_calendars') as sync_mock:
                 sync_mock.return_value = None
 
                 response = with_client.post(
                     '/caldav/auth',
-                    json={'user': '[email protected]',
-                    'url': 'example.com',
-                    'password': 'test'},
-                    headers=auth_headers
+                    json={'user': '[email protected]', 'url': 'example.com', 'password': 'test'},
+                    headers=auth_headers,
                 )
 
                 mock.assert_called()
@@ -559,19 +551,13 @@ def test_disconnect(self, with_db, with_client, make_external_connections, make_
         ec = make_external_connections(TEST_USER_ID, type=models.ExternalConnectionType.caldav, type_id=type_id)
         calendar = make_caldav_calendar(subscriber_id=TEST_USER_ID, user=username)
 
-        response = with_client.post(
-            '/caldav/disconnect', json={'type_id': ec.type_id},
-            headers=auth_headers
-        )
+        response = with_client.post('/caldav/disconnect', json={'type_id': ec.type_id}, headers=auth_headers)
 
         assert response.status_code == 200, response.content
 
         with with_db() as db:
             ecs = repo.external_connection.get_by_type(
-                db,
-                TEST_USER_ID,
-                models.ExternalConnectionType.caldav,
-                type_id=type_id
+                db, TEST_USER_ID, models.ExternalConnectionType.caldav, type_id=type_id
             )
             assert len(ecs) == 0
 
@@ -586,19 +572,13 @@ def test_disconnect(self, with_db, with_client, make_external_connections, make_
         ec = make_external_connections(TEST_USER_ID, type=models.ExternalConnectionType.google, type_id=type_id)
         calendar = make_google_calendar(subscriber_id=TEST_USER_ID)
 
-        response = with_client.post(
-            '/google/disconnect', json={'type_id': ec.type_id},
-            headers=auth_headers
-        )
+        response = with_client.post('/google/disconnect', json={'type_id': ec.type_id}, headers=auth_headers)
 
         assert response.status_code == 200, response.content
 
         with with_db() as db:
             ecs = repo.external_connection.get_by_type(
-                db,
-                TEST_USER_ID,
-                models.ExternalConnectionType.google,
-                type_id=type_id
+                db, TEST_USER_ID, models.ExternalConnectionType.google, type_id=type_id
             )
             assert len(ecs) == 0