made system more robust

Author: Zohair-coder

docker-compose.yml

@@ -1,4 +1,3 @@
-version: "3"
 services:
   scraper:
     build: .

src/login.py

@@ -10,6 +10,7 @@
 
 def login_with_drexel_connect(session: Session) -> Session:
     response = send_request(session, config.drexel_connect_base_url, method="GET")
+    assert response.status_code == 200, "Failed to get Drexel Connect login page"
     soup = BeautifulSoup(response.text, "html.parser")
 
     csrf_token = extract_csrf_token(soup)
@@ -24,11 +25,13 @@ def login_with_drexel_connect(session: Session) -> Session:
 
     # this should send the credentials and send the MFA request
     response = send_request(session, config.drexel_connect_base_url + form_action_path, data=login_payload, method="POST")
+    assert response.status_code == 200, "Failed to send request to Drexel Connect with username and password"
 
     soup = BeautifulSoup(response.text, "html.parser")
     data = parse_initial_mfa_page(soup)
 
     response = send_request(session, config.drexel_connect_base_url + data["url"], data=data["form-data"], method="POST")
+    assert response.status_code == 200, "Failed to request MFA code page from Drexel Connect"
     json_response = response.json()
 
     data = {
@@ -37,6 +40,7 @@ def login_with_drexel_connect(session: Session) -> Session:
     }
 
     response = send_request(session, config.drexel_connect_base_url + json_response["flowExURL"], data=data, method="POST")
+    assert response.status_code == 200, "Failed to receive MFA code page from Drexel Connect"
     soup = BeautifulSoup(response.text, "html.parser")
 
     parsed_data = parse_final_mfa_page(soup)
@@ -49,10 +53,9 @@ def login_with_drexel_connect(session: Session) -> Session:
         "j_mfaToken": totp_code,
     }
 
-    # session.post(
-    #     config.drexel_connect_base_url + parsed_data["url"], data=data
-    # )
-    send_request(session, config.drexel_connect_base_url + parsed_data["url"], data=data, method="POST")
+    response = send_request(session, config.drexel_connect_base_url + parsed_data["url"], data=data, method="POST")
+    assert response.status_code == 200, "Failed to send MFA code to Drexel Connect (final step)"
+
     return session
 
 

src/scrape.py

@@ -3,6 +3,8 @@
 import json
 import os
 from typing import Any
+import traceback
+import time
 
 from helpers import send_request
 from parse import parse_subject_page, parse_crn_page
@@ -15,9 +17,24 @@ def scrape(
 ) -> dict[str, dict[str, Any]]:
     session = Session()
 
-    session = login.login_with_drexel_connect(session)
-    assert "shib_idp_session" in session.cookies, "Failed to log in to Drexel Connect"
-
+    is_logged_into_drexel_connect = False
+    failiure_count = 0
+    while not is_logged_into_drexel_connect:
+        try:
+            session = login.login_with_drexel_connect(session)
+            if "shib_idp_session" in session.cookies:
+                is_logged_into_drexel_connect = True
+                break
+        except Exception:
+            print("Error logging in to Drexel Connect: ")
+            print(traceback.format_exc())
+            print("Trying again...")
+        
+        failiure_count += 1
+        if failiure_count > 5:
+            raise Exception("Failed to log in to Drexel Connect after 5 attempts")
+        time.sleep(1)
+        
     data: dict[str, dict[str, Any]] = {}
 
     if not all_colleges: