diff --git a/.github/workflows/functional_test.yml b/.github/workflows/functional_test.yml index 7e30e6e..d99d83c 100644 --- a/.github/workflows/functional_test.yml +++ b/.github/workflows/functional_test.yml @@ -34,7 +34,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Start PostgreSQL - run: docker-compose up -d postgres + run: docker compose up -d postgres - name: Wait for PostgreSQL to become ready run: | @@ -48,7 +48,7 @@ jobs: - name: Run the scraper env: - DREXEL_USERNAME: ${{ secrets.DREXEL_USERNAME }} + DREXEL_EMAIL: ${{ secrets.DREXEL_EMAIL }} DREXEL_PASSWORD: ${{ secrets.DREXEL_PASSWORD }} DREXEL_MFA_SECRET_KEY: ${{ secrets.DREXEL_MFA_SECRET_KEY }} run: docker compose run scraper python3 src/main.py --db --all-colleges --ratings @@ -104,11 +104,11 @@ jobs: fi - name: Reset database - run: docker compose run scraper sh -c 'apk add postgresql-client && ./scripts/reset-db.sh;' + run: docker compose run scraper sh -c 'apt-get install -y postgresql-client && ./scripts/reset-db.sh;' - name: Run scraper again (to test cache) env: - DREXEL_USERNAME: ${{ secrets.DREXEL_USERNAME }} + DREXEL_EMAIL: ${{ secrets.DREXEL_EMAIL }} DREXEL_PASSWORD: ${{ secrets.DREXEL_PASSWORD }} DREXEL_MFA_SECRET_KEY: ${{ secrets.DREXEL_MFA_SECRET_KEY }} run: docker compose run scraper python3 src/main.py --db --all-colleges --ratings @@ -164,4 +164,4 @@ jobs: fi - name: Cleanup - run: docker-compose down -v + run: docker compose down -v diff --git a/Dockerfile b/Dockerfile index 83ea06d..1a3bc6e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-alpine +FROM python:3.12 # set the working directory in the container WORKDIR /app @@ -6,11 +6,10 @@ WORKDIR /app # copy the current directory contents into the container at /app COPY . /app -# upgrade pip -RUN pip install --upgrade pip - # install dependencies -RUN pip install -r requirements.txt +RUN pip install --upgrade pip && \ + pip install -r requirements.txt && \ + playwright install chromium --with-deps # Run the Python script CMD ["python3", "src/main.py", "--db", "--all-colleges", "--ratings", "--email"] \ No newline at end of file diff --git a/README.md b/README.md index f852f37..2170ee0 100644 --- a/README.md +++ b/README.md @@ -18,11 +18,13 @@ Make sure you have [Python 3](https://www.python.org/downloads/) installed. Then ###### Mac/Linux ```bash pip3 install -r requirements.txt +playwright install ``` ###### Windows ```bash pip install -r requirements.txt +playwright install ``` ## Usage @@ -44,17 +46,21 @@ The scraper will output a JSON file called `data.json` in the same directory as You can modify the scraper to scrape other terms by changing the `year`, `quarter`, and `college_code` variables in `src/config.py`. +To view all the options that the scraper supports, run `python3 src/main.py --help` on Mac/Linux, or `python src/main.py --help` on Windows. + #### Authentication Since the term master schedule is only accessible to logged-in Drexel students, to run the scraper, you will need to provide your Drexel credentials as well as provide multi-factor authentication (MFA). -To provide your Drexel credentials, set the environment variable `DREXEL_USERNAME` to your Drexel username (abc123) and `DREXEL_PASSWORD` to the password you use to login to Drexel One. You can follow [this](https://phoenixnap.com/kb/windows-set-environment-variable) guide for Windows, and [this](https://phoenixnap.com/kb/set-environment-variable-mac) guide for MacOS to set environment variables. +To provide your Drexel credentials, set the environment variable `DREXEL_EMAIL` to your Drexel email (abc123@drexel.edu) and `DREXEL_PASSWORD` to the password you use to login to Drexel One. You can follow [this](https://phoenixnap.com/kb/windows-set-environment-variable) guide for Windows, and [this](https://phoenixnap.com/kb/set-environment-variable-mac) guide for MacOS to set environment variables. + +You will also need to go to [this page](https://mysignins.microsoft.com/security-info) and make sure "Authenticator app or hardware token" is the preferred sign-in method. Unfortunately, if you use Microsoft Authenticator as your MFA app you will not be able to run the scraper. You will have to delete the Microsoft Authenticator sign in method and install a different MFA app. There are two ways to provide MFA for the script to authenticate with. The first is easier if you're looking to run the script manually and quickly. The second is better if you are going to be running the script frequently, or if it needs to be automated. ###### Authenticate manually -You will authenticate the scraper manually as if you were logging into Drexel One, using a one-time code either from an authenticator app or that is texted to you. After setting the `DREXEL_USERNAME` and `DREXEL_PASSWORD` environment variables, run the scraper as explained [above](#Usage), and you will be prompted for your verification code. +You will authenticate the scraper manually as if you were logging into Drexel One, using a one-time code either from an authenticator app or that is texted to you. After setting the `DREXEL_EMAIL` and `DREXEL_PASSWORD` environment variables, run the scraper as explained [above](#Usage), and you will be prompted for your verification code. ###### Authenticate using a secret key diff --git a/cache/extra_course_data_cache.json b/cache/extra_course_data_cache.json new file mode 100644 index 0000000..d6de20a --- /dev/null +++ b/cache/extra_course_data_cache.json @@ -0,0 +1,1070 @@ +{ + "12550": { + "credits": "0.00", + "prereqs": "" + }, + "13581": { + "credits": "0.00", + "prereqs": "" + }, + "11643": { + "credits": "3.00", + "prereqs": "" + }, + "12551": { + "credits": "0.00", + "prereqs": "" + }, + "12552": { + "credits": "0.00", + "prereqs": "" + }, + "13065": { + "credits": "0.00", + "prereqs": "" + }, + "12553": { + "credits": "0.00", + "prereqs": "" + }, + "14784": { + "credits": "0.00", + "prereqs": "" + }, + "12554": { + "credits": "0.00", + "prereqs": "" + }, + "10240": { + "credits": "3.00", + "prereqs": "" + }, + "10688": { + "credits": "3.00", + "prereqs": "" + }, + "10994": { + "credits": "0.00", + "prereqs": "" + }, + "10995": { + "credits": "0.00", + "prereqs": "" + }, + "10996": { + "credits": "3.00", + "prereqs": "" + }, + "11325": { + "credits": "0.00", + "prereqs": "CS 171 Minimum Grade: C or CS 175 Minimum Grade: C" + }, + "11326": { + "credits": "3.00", + "prereqs": "CS 171 Minimum Grade: C or CS 175 Minimum Grade: C" + }, + "14786": { + "credits": "0.00", + "prereqs": "CS 265 Minimum Grade: C" + }, + "14787": { + "credits": "0.00", + "prereqs": "CS 265 Minimum Grade: C" + }, + "14789": { + "credits": "4.00", + "prereqs": "CS 265 Minimum Grade: C" + }, + "10202": { + "credits": "3.00", + "prereqs": "CS 172 Minimum Grade: C or ECE 105 Minimum Grade: D or ECEC 201 Minimum Grade: D" + }, + "13583": { + "credits": "3.00", + "prereqs": "CS 172 Minimum Grade: C or ECE 105 Minimum Grade: D or ECEC 201 Minimum Grade: D" + }, + "12154": { + "credits": "3.00", + "prereqs": "CS 172 Minimum Grade: C or ECE 105 Minimum Grade: D or ECEC 201 Minimum Grade: D" + }, + "12558": { + "credits": "3.00", + "prereqs": "CS 172 Minimum Grade: C or ECE 105 Minimum Grade: D or ECEC 201 Minimum Grade: D" + }, + "10712": { + "credits": "3.00", + "prereqs": "CS 172 Minimum Grade: C or ECE 105 Minimum Grade: D or ECEC 201 Minimum Grade: D" + }, + "11835": { + "credits": "3.00", + "prereqs": "CS 172 Minimum Grade: C or ECE 105 Minimum Grade: D or ECEC 201 Minimum Grade: D" + }, + "12559": { + "credits": "3.00", + "prereqs": "CS 172 Minimum Grade: C or CS 265 Minimum Grade: C or ECE 105 Minimum Grade: D or ECEC 201 Minimum Grade: D" + }, + "13584": { + "credits": "3.00", + "prereqs": "CS 172 Minimum Grade: C or CS 265 Minimum Grade: C or ECE 105 Minimum Grade: D or ECEC 201 Minimum Grade: D" + }, + "11836": { + "credits": "3.00", + "prereqs": "CS 172 Minimum Grade: C or CS 265 Minimum Grade: C or ECE 105 Minimum Grade: D or ECEC 201 Minimum Grade: D" + }, + "14791": { + "credits": "3.00", + "prereqs": "CS 172 Minimum Grade: C or CS 265 Minimum Grade: C or ECE 105 Minimum Grade: D or ECEC 201 Minimum Grade: D" + }, + "12560": { + "credits": "3.00", + "prereqs": "CS 172 Minimum Grade: C or CS 265 Minimum Grade: C or ECE 105 Minimum Grade: D or ECEC 201 Minimum Grade: D" + }, + "12562": { + "credits": "3.00", + "prereqs": "CS 260 Minimum Grade: C and CS 270 Minimum Grade: C and MATH 221 Minimum Grade: C" + }, + "13585": { + "credits": "3.00", + "prereqs": "CS 260 Minimum Grade: C and CS 270 Minimum Grade: C and MATH 221 Minimum Grade: C" + }, + "14792": { + "credits": "3.00", + "prereqs": "CS 260 Minimum Grade: C and CS 270 Minimum Grade: C and MATH 221 Minimum Grade: C" + }, + "10959": { + "credits": "0.00", + "prereqs": "(CS 270 Minimum Grade: C or ECE 200 Minimum Grade: D ) and (CS 172 Minimum Grade: C or ECEC 201 Minimum Grade: D or ECE 105 Minimum Grade: D )" + }, + "10732": { + "credits": "0.00", + "prereqs": "(CS 270 Minimum Grade: C or ECE 200 Minimum Grade: D ) and (CS 172 Minimum Grade: C or ECEC 201 Minimum Grade: D or ECE 105 Minimum Grade: D )" + }, + "11501": { + "credits": "0.00", + "prereqs": "(CS 270 Minimum Grade: C or ECE 200 Minimum Grade: D ) and (CS 172 Minimum Grade: C or ECEC 201 Minimum Grade: D or ECE 105 Minimum Grade: D )" + }, + "10731": { + "credits": "4.00", + "prereqs": "(CS 270 Minimum Grade: C or ECE 200 Minimum Grade: D ) and (CS 172 Minimum Grade: C or ECEC 201 Minimum Grade: D or ECE 105 Minimum Grade: D )" + }, + "10902": { + "credits": "4.00", + "prereqs": "(CS 270 Minimum Grade: C or ECE 200 Minimum Grade: D ) and (CS 172 Minimum Grade: C or ECEC 201 Minimum Grade: D or ECE 105 Minimum Grade: D )" + }, + "13586": { + "credits": "4.00", + "prereqs": "(CS 270 Minimum Grade: C or ECE 200 Minimum Grade: D ) and (CS 172 Minimum Grade: C or ECEC 201 Minimum Grade: D or ECE 105 Minimum Grade: D )" + }, + "13587": { + "credits": "3.00", + "prereqs": "CS 265 Minimum Grade: C" + }, + "10567": { + "credits": "3.00", + "prereqs": "CS 260 Minimum Grade: C and (CS 281 Minimum Grade: C or ECEC 355 Minimum Grade: D )" + }, + "13366": { + "credits": "3.00", + "prereqs": "CS 260 Minimum Grade: C and (CS 281 Minimum Grade: C or ECEC 355 Minimum Grade: D )" + }, + "13066": { + "credits": "3.00", + "prereqs": "CS 265 Minimum Grade: C" + }, + "14793": { + "credits": "3.00", + "prereqs": "CS 265 Minimum Grade: C" + }, + "12098": { + "credits": "3.00", + "prereqs": "CS 283 Minimum Grade: C or ECEC 353 Minimum Grade: D" + }, + "10129": { + "credits": "3.00", + "prereqs": "CS 260 Minimum Grade: C and CS 270 Minimum Grade: C" + }, + "12099": { + "credits": "3.00", + "prereqs": "CS 260 Minimum Grade: C and CS 270 Minimum Grade: C" + }, + "13067": { + "credits": "3.00", + "prereqs": "CS 260 Minimum Grade: C and CS 380 Minimum Grade: C" + }, + "10961": { + "credits": "3.00", + "prereqs": "CS 260 Minimum Grade: C and (MATH 201 Minimum Grade: C or MATH 261 Minimum Grade: C or ENGR 231 Minimum Grade: D )" + }, + "12563": { + "credits": "0.00", + "prereqs": "CS 260 Minimum Grade: C and CS 270 Minimum Grade: C and CS 277 Minimum Grade: C and (MATH 221 Minimum Grade: C or MATH 222 Minimum Grade: C )" + }, + "12564": { + "credits": "4.00", + "prereqs": "CS 260 Minimum Grade: C and CS 270 Minimum Grade: C and CS 277 Minimum Grade: C and (MATH 221 Minimum Grade: C or MATH 222 Minimum Grade: C )" + }, + "12565": { + "credits": "3.00", + "prereqs": "CS 260 Minimum Grade: C" + }, + "13370": { + "credits": "3.00", + "prereqs": "CS 260 Minimum Grade: C" + }, + "10107": { + "credits": "3.00", + "prereqs": "CS 361 Minimum Grade: D or CS 283 Minimum Grade: C or ECEC 353 Minimum Grade: D" + }, + "12566": { + "credits": "3.00", + "prereqs": "" + }, + "12100": { + "credits": "3.00", + "prereqs": "" + }, + "12567": { + "credits": "3.00", + "prereqs": "CS 501 Minimum Grade: C (CS 501 may be taken concurrently with CS 503)" + }, + "12101": { + "credits": "3.00", + "prereqs": "CS 501 Minimum Grade: C (CS 501 may be taken concurrently with CS 503)" + }, + "11163": { + "credits": "3.00", + "prereqs": "CS 504 Minimum Grade: C" + }, + "11178": { + "credits": "3.00", + "prereqs": "CS 504 Minimum Grade: C" + }, + "12568": { + "credits": "3.00", + "prereqs": "CS 504 Minimum Grade: C" + }, + "10265": { + "credits": "3.00", + "prereqs": "CS 504 Minimum Grade: C" + }, + "12155": { + "credits": "3.00", + "prereqs": "" + }, + "12156": { + "credits": "3.00", + "prereqs": "" + }, + "13588": { + "credits": "3.00", + "prereqs": "CS 504 Minimum Grade: C" + }, + "13589": { + "credits": "3.00", + "prereqs": "CS 504 Minimum Grade: C" + }, + "13068": { + "credits": "3.00", + "prereqs": "CS 504 Minimum Grade: C" + }, + "13071": { + "credits": "3.00", + "prereqs": "CS 504 Minimum Grade: C" + }, + "14797": { + "credits": "3.00", + "prereqs": "(CS 510 Minimum Grade: C and CS 613 Minimum Grade: C and CS 615 Minimum Grade: C ) or (CS 614 Minimum Grade: C and INFO 629 Minimum Grade: C )" + }, + "12779": { + "credits": "3.00", + "prereqs": "(CS 510 Minimum Grade: C and CS 613 Minimum Grade: C and CS 615 Minimum Grade: C ) or (CS 614 Minimum Grade: C and INFO 629 Minimum Grade: C )" + }, + "14798": { + "credits": "3.00", + "prereqs": "CS 510 Minimum Grade: C" + }, + "14799": { + "credits": "3.00", + "prereqs": "CS 510 Minimum Grade: C" + }, + "12102": { + "credits": "3.00", + "prereqs": "CS 504 Minimum Grade: C" + }, + "12103": { + "credits": "3.00", + "prereqs": "CS 504 Minimum Grade: C" + }, + "11186": { + "credits": "3.00", + "prereqs": "INFO 101 Minimum Grade: D" + }, + "14179": { + "credits": "3.00", + "prereqs": "INFO 101 Minimum Grade: D" + }, + "11528": { + "credits": "3.00", + "prereqs": "" + }, + "11529": { + "credits": "3.00", + "prereqs": "" + }, + "13592": { + "credits": "3.00", + "prereqs": "" + }, + "13593": { + "credits": "3.00", + "prereqs": "" + }, + "13594": { + "credits": "3.00", + "prereqs": "CT 420 Minimum Grade: D or CT 301 Minimum Grade: D" + }, + "13595": { + "credits": "3.00", + "prereqs": "CT 420 Minimum Grade: D or CT 301 Minimum Grade: D" + }, + "13596": { + "credits": "3.00", + "prereqs": "CT 330 Minimum Grade: D" + }, + "13597": { + "credits": "3.00", + "prereqs": "" + }, + "13072": { + "credits": "3.00", + "prereqs": "" + }, + "12579": { + "credits": "3.00", + "prereqs": "" + }, + "12571": { + "credits": "0.00", + "prereqs": "" + }, + "12572": { + "credits": "0.00", + "prereqs": "" + }, + "12573": { + "credits": "0.00", + "prereqs": "" + }, + "12574": { + "credits": "0.00", + "prereqs": "" + }, + "12575": { + "credits": "0.00", + "prereqs": "" + }, + "12576": { + "credits": "0.00", + "prereqs": "" + }, + "12721": { + "credits": "2.00", + "prereqs": "" + }, + "10876": { + "credits": "2.00", + "prereqs": "" + }, + "10877": { + "credits": "2.00", + "prereqs": "" + }, + "10878": { + "credits": "2.00", + "prereqs": "" + }, + "10879": { + "credits": "2.00", + "prereqs": "" + }, + "12224": { + "credits": "2.00", + "prereqs": "" + }, + "11837": { + "credits": "2.00", + "prereqs": "" + }, + "12577": { + "credits": "0.00", + "prereqs": "" + }, + "11162": { + "credits": "2.00", + "prereqs": "" + }, + "10909": { + "credits": "2.00", + "prereqs": "" + }, + "13950": { + "credits": "2.00", + "prereqs": "" + }, + "10934": { + "credits": "2.00", + "prereqs": "" + }, + "14810": { + "credits": "3.00", + "prereqs": "INFO 442 Minimum Grade: D or SE 310 Minimum Grade: D or (INFO 324 Minimum Grade: D and INFO 355 Minimum Grade: D )" + }, + "12578": { + "credits": "3.00", + "prereqs": "CS 570 Minimum Grade: C (CS 570 may be taken concurrently with DSCI 511)" + }, + "13591": { + "credits": "3.00", + "prereqs": "CS 570 Minimum Grade: C (CS 570 may be taken concurrently with DSCI 511)" + }, + "11504": { + "credits": "3.00", + "prereqs": "CS 570 Minimum Grade: C (CS 570 may be taken concurrently with DSCI 511)" + }, + "12135": { + "credits": "3.00", + "prereqs": "DSCI 521 Minimum Grade: C (DSCI 521 may be taken concurrently with DSCI 631)" + }, + "12176": { + "credits": "3.00", + "prereqs": "DSCI 521 Minimum Grade: C (DSCI 521 may be taken concurrently with DSCI 631)" + }, + "11716": { + "credits": "3.00", + "prereqs": "" + }, + "10881": { + "credits": "3.00", + "prereqs": "" + }, + "10658": { + "credits": "3.00", + "prereqs": "" + }, + "11711": { + "credits": "3.00", + "prereqs": "" + }, + "11717": { + "credits": "3.00", + "prereqs": "" + }, + "10882": { + "credits": "3.00", + "prereqs": "" + }, + "10709": { + "credits": "3.00", + "prereqs": "" + }, + "13958": { + "credits": "3.00", + "prereqs": "" + }, + "10710": { + "credits": "3.00", + "prereqs": "" + }, + "12580": { + "credits": "3.00", + "prereqs": "" + }, + "12581": { + "credits": "3.00", + "prereqs": "" + }, + "12582": { + "credits": "3.00", + "prereqs": "" + }, + "11468": { + "credits": "3.00", + "prereqs": "" + }, + "11530": { + "credits": "3.00", + "prereqs": "INFO 101 Minimum Grade: D" + }, + "12583": { + "credits": "3.00", + "prereqs": "INFO 101 Minimum Grade: D or SE 210 Minimum Grade: D" + }, + "13073": { + "credits": "3.00", + "prereqs": "INFO 153 Minimum Grade: D or CS 172 Minimum Grade: D" + }, + "12107": { + "credits": "3.00", + "prereqs": "" + }, + "12584": { + "credits": "3.00", + "prereqs": "INFO 110 Minimum Grade: D or INFO 151 Minimum Grade: D or CS 171 Minimum Grade: D or ECE 105 Minimum Grade: D or ECE 203 Minimum Grade: D" + }, + "13074": { + "credits": "3.00", + "prereqs": "INFO 110 Minimum Grade: D or INFO 151 Minimum Grade: D or CS 171 Minimum Grade: D or ECE 105 Minimum Grade: D or ECE 203 Minimum Grade: D" + }, + "12585": { + "credits": "3.00", + "prereqs": "(INFO 153 Minimum Grade: D or CS 172 Minimum Grade: D ) and INFO 200 Minimum Grade: D" + }, + "13303": { + "credits": "3.00", + "prereqs": "INFO 210 Minimum Grade: D and (CS 171 Minimum Grade: D or CS 175 Minimum Grade: D or INFO 152 Minimum Grade: D or SE 102 Minimum Grade: D )" + }, + "11426": { + "credits": "3.00", + "prereqs": "INFO 210 Minimum Grade: D and (CS 171 Minimum Grade: D or CS 175 Minimum Grade: D or INFO 152 Minimum Grade: D or SE 102 Minimum Grade: D )" + }, + "13598": { + "credits": "3.00", + "prereqs": "STAT 201 Minimum Grade: D or PBHL 211 Minimum Grade: D or MATH 311 Minimum Grade: D" + }, + "12586": { + "credits": "3.00", + "prereqs": "INFO 310 Minimum Grade: D or INFO 110 Minimum Grade: D" + }, + "13599": { + "credits": "3.00", + "prereqs": "(INFO 200 Minimum Grade: D or SE 210 Minimum Grade: D ) and (CS 172 Minimum Grade: D or CS 265 Minimum Grade: D or INFO 152 Minimum Grade: D )" + }, + "11222": { + "credits": "3.00", + "prereqs": "(INFO 200 Minimum Grade: D or SE 210 Minimum Grade: D ) and (CS 172 Minimum Grade: D or CS 265 Minimum Grade: D or INFO 152 Minimum Grade: D )" + }, + "14812": { + "credits": "3.00", + "prereqs": "" + }, + "14813": { + "credits": "3.00", + "prereqs": "" + }, + "14814": { + "credits": "3.00", + "prereqs": "" + }, + "11847": { + "credits": "3.00", + "prereqs": "" + }, + "11848": { + "credits": "3.00", + "prereqs": "" + }, + "10754": { + "credits": "3.00", + "prereqs": "" + }, + "12587": { + "credits": "3.00", + "prereqs": "" + }, + "10704": { + "credits": "3.00", + "prereqs": "" + }, + "11187": { + "credits": "3.00", + "prereqs": "" + }, + "11188": { + "credits": "3.00", + "prereqs": "" + }, + "13600": { + "credits": "3.00", + "prereqs": "" + }, + "12588": { + "credits": "3.00", + "prereqs": "" + }, + "12589": { + "credits": "3.00", + "prereqs": "" + }, + "14815": { + "credits": "3.00", + "prereqs": "INFO 590 Minimum Grade: C or INFO 522 Minimum Grade: C" + }, + "11532": { + "credits": "3.00", + "prereqs": "INFO 590 Minimum Grade: C or INFO 522 Minimum Grade: C" + }, + "10520": { + "credits": "3.00", + "prereqs": "" + }, + "14816": { + "credits": "3.00", + "prereqs": "" + }, + "10115": { + "credits": "3.00", + "prereqs": "" + }, + "14817": { + "credits": "3.00", + "prereqs": "INFO 508 Minimum Grade: C (INFO 508 may be taken concurrently with INFO 615) or INFO 608 Minimum Grade: C (INFO 608 may be taken concurrently with INFO 615) or DSRE 620 Minimum Grade: C (DSRE 620 may be taken concurrently with INFO 615)" + }, + "13601": { + "credits": "3.00", + "prereqs": "INFO 508 Minimum Grade: C (INFO 508 may be taken concurrently with INFO 615) or INFO 608 Minimum Grade: C (INFO 608 may be taken concurrently with INFO 615) or DSRE 620 Minimum Grade: C (DSRE 620 may be taken concurrently with INFO 615)" + }, + "14818": { + "credits": "3.00", + "prereqs": "" + }, + "13076": { + "credits": "3.00", + "prereqs": "" + }, + "13602": { + "credits": "3.00", + "prereqs": "INFO 540 Minimum Grade: C (INFO 540 may be taken concurrently with INFO 624) or INFO 590 Minimum Grade: C or CS 502 Minimum Grade: C or DSCI 511 Minimum Grade: C (DSCI 511 may be taken concurrently with INFO 624) or DSCI 521 Minimum Grade: C (DSCI 521 may be taken concurrently with INFO 624)" + }, + "13603": { + "credits": "3.00", + "prereqs": "INFO 540 Minimum Grade: C (INFO 540 may be taken concurrently with INFO 624) or INFO 590 Minimum Grade: C or CS 502 Minimum Grade: C or DSCI 511 Minimum Grade: C (DSCI 511 may be taken concurrently with INFO 624) or DSCI 521 Minimum Grade: C (DSCI 521 may be taken concurrently with INFO 624)" + }, + "11849": { + "credits": "3.00", + "prereqs": "" + }, + "11850": { + "credits": "3.00", + "prereqs": "" + }, + "10477": { + "credits": "3.00", + "prereqs": "" + }, + "10695": { + "credits": "3.00", + "prereqs": "" + }, + "11642": { + "credits": "3.00", + "prereqs": "INFO 540 Minimum Grade: C or INFO 590 Minimum Grade: C or INFO 648 Minimum Grade: C or DSCI 511 Minimum Grade: C (DSCI 511 may be taken concurrently with INFO 659) or DSCI 521 Minimum Grade: C (DSCI 521 may be taken concurrently with INFO 659)" + }, + "11533": { + "credits": "3.00", + "prereqs": "INFO 540 Minimum Grade: C or INFO 590 Minimum Grade: C or INFO 648 Minimum Grade: C or DSCI 511 Minimum Grade: C (DSCI 511 may be taken concurrently with INFO 659) or DSCI 521 Minimum Grade: C (DSCI 521 may be taken concurrently with INFO 659)" + }, + "12108": { + "credits": "3.00", + "prereqs": "" + }, + "11851": { + "credits": "3.00", + "prereqs": "" + }, + "13604": { + "credits": "3.00", + "prereqs": "" + }, + "13605": { + "credits": "3.00", + "prereqs": "INFO 508 Minimum Grade: C (INFO 508 may be taken concurrently with INFO 691) or INFO 608 Minimum Grade: C (INFO 608 may be taken concurrently with INFO 691) or DSRE 620 Minimum Grade: C (DSRE 620 may be taken concurrently with INFO 691)" + }, + "12209": { + "credits": "3.00", + "prereqs": "INFO 508 Minimum Grade: C (INFO 508 may be taken concurrently with INFO 691) or INFO 608 Minimum Grade: C (INFO 608 may be taken concurrently with INFO 691) or DSRE 620 Minimum Grade: C (DSRE 620 may be taken concurrently with INFO 691)" + }, + "13077": { + "credits": "3.00", + "prereqs": "" + }, + "13079": { + "credits": "3.00", + "prereqs": "" + }, + "14820": { + "credits": "3.00", + "prereqs": "" + }, + "12590": { + "credits": "3.00", + "prereqs": "" + }, + "14821": { + "credits": "3.00", + "prereqs": "" + }, + "11900": { + "credits": "1.00", + "prereqs": "" + }, + "15013": { + "credits": "1.00", + "prereqs": "" + }, + "14822": { + "credits": "3.00", + "prereqs": "" + }, + "14801": { + "credits": "3.00", + "prereqs": "CS 172 Minimum Grade: C" + }, + "14802": { + "credits": "3.00", + "prereqs": "CS 172 Minimum Grade: C" + }, + "14803": { + "credits": "3.00", + "prereqs": "CS 172 Minimum Grade: C" + }, + "14804": { + "credits": "3.00", + "prereqs": "CS 172 Minimum Grade: C" + }, + "14805": { + "credits": "3.00", + "prereqs": "CS 172 Minimum Grade: C" + }, + "10182": { + "credits": "3.00", + "prereqs": "CS 172 Minimum Grade: C" + }, + "10233": { + "credits": "3.00", + "prereqs": "CS 265 Minimum Grade: C and CS 260 Minimum Grade: C and (SE 181 Minimum Grade: C or SE 201 Minimum Grade: C )" + }, + "12106": { + "credits": "3.00", + "prereqs": "CS 265 Minimum Grade: C and CS 260 Minimum Grade: C and (SE 181 Minimum Grade: C or SE 201 Minimum Grade: C )" + }, + "11502": { + "credits": "3.00", + "prereqs": "CS 265 Minimum Grade: C and CS 260 Minimum Grade: C and (SE 181 Minimum Grade: C or SE 201 Minimum Grade: C )" + }, + "10175": { + "credits": "3.00", + "prereqs": "CS 260 Minimum Grade: C and (SE 181 Minimum Grade: C or SE 201 Minimum Grade: C )" + }, + "12219": { + "credits": "3.00", + "prereqs": "CS 260 Minimum Grade: C and (SE 181 Minimum Grade: C or SE 201 Minimum Grade: C )" + }, + "11896": { + "credits": "3.00", + "prereqs": "CS 260 Minimum Grade: C" + }, + "13957": { + "credits": "3.00", + "prereqs": "CS 260 Minimum Grade: C" + }, + "12569": { + "credits": "3.00", + "prereqs": "CS 504 Minimum Grade: C" + }, + "12570": { + "credits": "3.00", + "prereqs": "CS 504 Minimum Grade: C" + }, + "11327": { + "credits": "3.00", + "prereqs": "CS 504 Minimum Grade: C" + }, + "11328": { + "credits": "3.00", + "prereqs": "CS 504 Minimum Grade: C" + }, + "11329": { + "credits": "3.00", + "prereqs": "INFO 620 Minimum Grade: C or CS 502 Minimum Grade: C or INFO 600 Minimum Grade: C" + }, + "11330": { + "credits": "3.00", + "prereqs": "INFO 620 Minimum Grade: C or CS 502 Minimum Grade: C or INFO 600 Minimum Grade: C" + }, + "12782": { + "credits": "3.00", + "prereqs": "INFO 620 Minimum Grade: C or CS 502 Minimum Grade: C or SE 630 Minimum Grade: C or INFO 532 Minimum Grade: C" + }, + "12783": { + "credits": "3.00", + "prereqs": "INFO 620 Minimum Grade: C or CS 502 Minimum Grade: C or SE 630 Minimum Grade: C or INFO 532 Minimum Grade: C" + }, + "14808": { + "credits": "3.00", + "prereqs": "SE 570 Minimum Grade: C" + }, + "11503": { + "credits": "3.00", + "prereqs": "SE 570 Minimum Grade: C" + }, + "10903": { + "credits": "1.00", + "prereqs": "" + }, + "11701": { + "credits": "1.00", + "prereqs": "" + }, + "10904": { + "credits": "1.00", + "prereqs": "" + }, + "10905": { + "credits": "1.00", + "prereqs": "" + }, + "11332": { + "credits": "1.00", + "prereqs": "" + }, + "12677": { + "credits": "1.00", + "prereqs": "" + }, + "12678": { + "credits": "1.00", + "prereqs": "" + }, + "12679": { + "credits": "1.00", + "prereqs": "" + }, + "12680": { + "credits": "1.00", + "prereqs": "" + }, + "13239": { + "credits": "1.00", + "prereqs": "" + }, + "13240": { + "credits": "1.00", + "prereqs": "" + }, + "13241": { + "credits": "1.00", + "prereqs": "" + }, + "12555": { + "credits": "0.00", + "prereqs": "" + }, + "14785": { + "credits": "3.00", + "prereqs": "" + }, + "15543": { + "credits": "0.00", + "prereqs": "CS 265 Minimum Grade: C" + }, + "15545": { + "credits": "3.00", + "prereqs": "CS 265 Minimum Grade: C" + }, + "15546": { + "credits": "3.00", + "prereqs": "CS 265 Minimum Grade: C" + }, + "14795": { + "credits": "3.00", + "prereqs": "CS 504 Minimum Grade: C" + }, + "14796": { + "credits": "3.00", + "prereqs": "CS 504 Minimum Grade: C" + }, + "15367": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15368": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15369": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15370": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15371": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15372": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15373": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15374": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15375": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15376": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15377": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15378": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15379": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15380": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15381": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15382": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15383": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15384": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15385": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15386": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15387": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15388": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15389": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15529": { + "credits": "3.00", + "prereqs": "" + }, + "14819": { + "credits": "3.00", + "prereqs": "" + }, + "15333": { + "credits": "3.00", + "prereqs": "INFO 881 Minimum Grade: C" + }, + "15334": { + "credits": "3.00", + "prereqs": "INFO 881 Minimum Grade: C" + }, + "15305": { + "credits": "3.00 TO 6.00", + "prereqs": "" + }, + "15390": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15391": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15392": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15393": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15394": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15395": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15396": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15397": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15398": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15399": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15400": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15401": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15402": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15403": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15404": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15405": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15517": { + "credits": "1.00 TO 12.00", + "prereqs": "" + }, + "15544": { + "credits": "3.00", + "prereqs": "CS 265 Minimum Grade: C and CS 260 Minimum Grade: C and (SE 181 Minimum Grade: C or SE 201 Minimum Grade: C )" + } +} \ No newline at end of file diff --git a/cache/ratings_cache.json b/cache/ratings_cache.json index 3add892..0a3b906 100644 --- a/cache/ratings_cache.json +++ b/cache/ratings_cache.json @@ -371,5 +371,203 @@ "avgRating": 1.6, "legacyId": 2939644, "numRatings": 8 + }, + "Daniel W Moix": { + "avgDifficulty": 3.4, + "avgRating": 2.4, + "legacyId": 2840871, + "numRatings": 10 + }, + "Adelaida A Medlock": { + "avgDifficulty": 3.5, + "avgRating": 2.1, + "legacyId": 813841, + "numRatings": 48 + }, + "Kurt Schmidt": { + "avgDifficulty": 4.3, + "avgRating": 2.5, + "legacyId": 435636, + "numRatings": 71 + }, + "Brian S Mitchell": { + "avgDifficulty": 3.8, + "avgRating": 3.9, + "legacyId": 2836246, + "numRatings": 4 + }, + "Dario D Salvucci": { + "avgDifficulty": 3.4, + "avgRating": 5, + "legacyId": 2411977, + "numRatings": 7 + }, + "Widchard Faustin": null, + "Sam M Bever": { + "avgDifficulty": 2, + "avgRating": 5, + "legacyId": 2672839, + "numRatings": 3 + }, + "Feng Liu": { + "avgDifficulty": 3.8, + "avgRating": 3.7, + "legacyId": 2409027, + "numRatings": 4 + }, + "Eric Sun": { + "avgDifficulty": 3.6, + "avgRating": 3, + "legacyId": 1802507, + "numRatings": 17 + }, + "Geoffrey B Mainland": { + "avgDifficulty": 3.9, + "avgRating": 4.3, + "legacyId": 2446149, + "numRatings": 9 + }, + "Santiago Ontanon": null, + "Michelle L Rogers": { + "avgDifficulty": 2.5, + "avgRating": 2.5, + "legacyId": 1364532, + "numRatings": 8 + }, + "Sherri Hackett": { + "avgDifficulty": 4, + "avgRating": 1, + "legacyId": 2793029, + "numRatings": 2 + }, + "Jeff K Salvage": { + "avgDifficulty": 3, + "avgRating": 3.6, + "legacyId": 756839, + "numRatings": 15 + }, + "Filippos I Vokolos": { + "avgDifficulty": 3.3, + "avgRating": 2.9, + "legacyId": 1940069, + "numRatings": 15 + }, + "Mike J Galloway": { + "avgDifficulty": 2, + "avgRating": 3.5, + "legacyId": 1627698, + "numRatings": 1 + }, + "Sumaiya Tabassum": null, + "Yuan An": { + "avgDifficulty": 2.4, + "avgRating": 5, + "legacyId": 2797629, + "numRatings": 5 + }, + "David A Appelbaum": { + "avgDifficulty": 2, + "avgRating": 4, + "legacyId": 1703030, + "numRatings": 1 + }, + "Jung-Ran Park": { + "avgDifficulty": 3.4, + "avgRating": 4.1, + "legacyId": 1296385, + "numRatings": 14 + }, + "Il-Yeol Song": { + "avgDifficulty": 3.6, + "avgRating": 4, + "legacyId": 957870, + "numRatings": 11 + }, + "Alexander H Poole": { + "avgDifficulty": 5, + "avgRating": 4, + "legacyId": 2833092, + "numRatings": 1 + }, + "Sonia M Pascua": { + "avgDifficulty": 2.5, + "avgRating": 4, + "legacyId": 2567272, + "numRatings": 8 + }, + "Shruti V Phadke": { + "avgDifficulty": 1.7, + "avgRating": 5, + "legacyId": 1771085, + "numRatings": 3 + }, + "Chaomei Chen": null, + "Elizabeth A Campbell": { + "avgDifficulty": 1.5, + "avgRating": 5, + "legacyId": 2752748, + "numRatings": 2 + }, + "Jeff E Bullard": { + "avgDifficulty": 3.6, + "avgRating": 2.5, + "legacyId": 565937, + "numRatings": 28 + }, + "Michael D Ekstrand": { + "avgDifficulty": 3.3, + "avgRating": 4.4, + "legacyId": 2399980, + "numRatings": 4 + }, + "Karthik Seetharama Bhat": { + "avgDifficulty": 2, + "avgRating": 4.5, + "legacyId": 2399987, + "numRatings": 2 + }, + "Helena Mentis": null, + "Gregory W Hislop": { + "avgDifficulty": 3, + "avgRating": 3, + "legacyId": 2805770, + "numRatings": 1 + }, + "Thomas M Shortell": { + "avgDifficulty": 3.5, + "avgRating": 1, + "legacyId": 2698578, + "numRatings": 2 + }, + "Briana Green": { + "avgDifficulty": 1, + "avgRating": 3, + "legacyId": 2947960, + "numRatings": 1 + }, + "William T Ahern": { + "avgDifficulty": 2, + "avgRating": 3.8, + "legacyId": 2284507, + "numRatings": 28 + }, + "Kristy L Eells": { + "avgDifficulty": 4, + "avgRating": 4, + "legacyId": 2764267, + "numRatings": 1 + }, + "Alexis M Pinto": { + "avgDifficulty": 2, + "avgRating": 3.8, + "legacyId": 2604989, + "numRatings": 3 + }, + "Alla Fedosenko": null, + "Antonia M Hannon": { + "avgDifficulty": 3, + "avgRating": 4.5, + "legacyId": 2842539, + "numRatings": 2 } } \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index e3d343f..8a6f48a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ services: AWS_DEFAULT_REGION: us-east-1 AWS_ACCESS_KEY_ID: "my-access-key" AWS_SECRET_ACCESS_KEY: "my-secret-access-key" - DREXEL_USERNAME: ${DREXEL_USERNAME} + DREXEL_EMAIL: ${DREXEL_EMAIL} DREXEL_PASSWORD: ${DREXEL_PASSWORD} DREXEL_MFA_SECRET_KEY: ${DREXEL_MFA_SECRET_KEY} volumes: diff --git a/requirements.txt b/requirements.txt index 1ddcfe6..bdfd90a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,6 +10,7 @@ charset-normalizer==3.3.2 click==8.1.7 gitdb==4.0.11 GitPython==3.1.42 +greenlet==3.0.3 idna==3.6 jmespath==1.0.1 mypy==1.9.0 @@ -17,13 +18,14 @@ mypy-extensions==1.0.0 packaging==24.0 pathspec==0.12.1 platformdirs==4.2.0 +playwright==1.46.0 psycopg2-binary==2.9.9 +pyee==11.1.0 python-dateutil==2.9.0.post0 pytz==2024.1 requests==2.31.0 ruff==0.3.4 s3transfer==0.10.1 -setuptools==69.2.0 six==1.16.0 smmap==5.0.1 soupsieve==2.5 @@ -36,4 +38,3 @@ types-requests==2.31.0.20240311 types-s3transfer==0.10.0 typing_extensions==4.10.0 urllib3==2.2.1 -wheel==0.43.0 diff --git a/src/config.py b/src/config.py index f10f607..c1abb63 100644 --- a/src/config.py +++ b/src/config.py @@ -35,7 +35,7 @@ def get_environ(key: str, required: bool = True) -> str: # Drexel Connect Credentials -drexel_username = get_environ("DREXEL_USERNAME") +drexel_email = get_environ("DREXEL_EMAIL") drexel_password = get_environ("DREXEL_PASSWORD") # This is not required if the user is using a separate authenticator app and will manually approve the login attempt drexel_mfa_secret_key = get_environ("DREXEL_MFA_SECRET_KEY", False) or None diff --git a/src/login.py b/src/login.py index b62289c..4dade21 100644 --- a/src/login.py +++ b/src/login.py @@ -1,215 +1,88 @@ -from requests import Session -from requests.exceptions import JSONDecodeError -from bs4 import BeautifulSoup, Tag -import re -from typing import Any - +from playwright.sync_api import sync_playwright, ElementHandle import config import totp -from helpers import send_request +from requests import Session def login_with_drexel_connect(session: Session) -> Session: - response = send_request(session, config.drexel_connect_base_url, method="GET") - assert response.status_code == 200, "Failed to get Drexel Connect login page" - soup = BeautifulSoup(response.text, "html.parser") - - csrf_token = extract_csrf_token(soup) - form_action_path = extract_form_action_path(soup) - - login_payload = { - "j_username": config.drexel_username, - "j_password": config.drexel_password, - "csrf_token": csrf_token, - "_eventId_proceed": "", - } - - # this should send the credentials and send the MFA request - response = send_request( - session, - config.drexel_connect_base_url + form_action_path, - data=login_payload, - method="POST", - ) - assert ( - response.status_code == 200 - ), "Failed to send request to Drexel Connect with username and password" - - soup = BeautifulSoup(response.text, "html.parser") - data = parse_initial_mfa_page(soup) - - # the intial MFA page does not have the 'verification code' form field - # the following two requests are sent to fetch the html page with the 'verification code' form field - response = send_request( - session, - config.drexel_connect_base_url + data["url"], - data=data["form-data"], - method="POST", - ) - assert ( - response.status_code == 200 - ), "Failed to request MFA code page from Drexel Connect" - - try: - json_response = response.json() - except JSONDecodeError: - raise Exception( - "Failed to decode JSON response from Drexel Connect. Response: {}".format( - response.text - ) - ) - - data = { - json_response["csrfN"]: json_response["csrfV"], - "_eventId": json_response["actValue"], - } - - response = send_request( - session, - config.drexel_connect_base_url + json_response["flowExURL"], - data=data, - method="POST", - ) - # the response should be in HTML format that contains the 'verification code' form field - assert ( - response.status_code == 200 - ), "Failed to receive MFA code page from Drexel Connect" - soup = BeautifulSoup(response.text, "html.parser") - - parsed_data = parse_final_mfa_page(soup) - - if config.drexel_mfa_secret_key is not None: - mfa_token = totp.get_token(config.drexel_mfa_secret_key) - else: - mfa_token = input("Please input your MFA verification code: ") - - data = { - "csrf_token": parsed_data["csrf_token"], - "_eventId": "proceed", - "j_mfaToken": mfa_token, - } - - # this request sends the MFA code to Drexel Connect - response = send_request( - session, - config.drexel_connect_base_url + parsed_data["url"], - data=data, - method="POST", - ) - assert ( - response.status_code == 200 - ), "Failed to send MFA code to Drexel Connect (final step)" - - # the session should now have the required cookies to access the TMS website - return session - - -def extract_csrf_token(soup: BeautifulSoup) -> str: - csrf_token_input_tag = soup.find("input", {"name": "csrf_token"}) - - if not isinstance(csrf_token_input_tag, Tag): - raise Exception("Could not find CSRF token.") - - csrf_token = csrf_token_input_tag["value"] - - if not isinstance(csrf_token, str): - raise Exception( - f"CSRF token was not a string. Found: {csrf_token} of type: {type(csrf_token)}" - ) - - return csrf_token - - -def extract_form_action_path(soup: BeautifulSoup) -> str: - # the form is a child of a div with id "login-box" - login_box_div = soup.find("div", {"id": "login-box"}) - - if not isinstance(login_box_div, Tag): - raise Exception("Could not find login box div.") - - login_form = login_box_div.find("form") - - if not isinstance(login_form, Tag): - raise Exception("Could not find login form.") - - form_action_path = login_form["action"] - - if not isinstance(form_action_path, str): - raise Exception( - f"Form action path was not a string. Found: {form_action_path} of type: {type(form_action_path)}" - ) - - return form_action_path - - -def parse_initial_mfa_page(soup: BeautifulSoup) -> dict[str, Any]: - data = {} - - # get the first script tag that isn't empty - script_tag = soup.find("script", string=lambda text: text and len(text) > 0) - - if not isinstance(script_tag, Tag): - raise Exception("Could not find non-empty script tag.") - - script_content = script_tag.string - - if not isinstance(script_content, str): - raise Exception( - f"Script content was not a string. Found: {script_content} of type: {type(script_content)}" - ) - - url_match = re.search( - r"url:\s*['\"](/idp/profile/cas/login\?execution=[^'\"]+)['\"]", script_content - ) - if not url_match: - raise Exception("Could not find MFA URL.") - - event_id_match = event_id_match = re.search( - r"data:\s*'_eventId=([^'&]+)&csrf_token", script_content - ) - if not event_id_match: - raise Exception("Could not find MFA event ID.") - - csrf_token_match = re.search(r"csrf_token=([^'&]+)", script_content) - if not csrf_token_match: - raise Exception("Could not find MFA CSRF token.") - - data["url"] = url_match.group(1) - data["form-data"] = { - "_eventId": event_id_match.group(1), - "csrf_token": csrf_token_match.group(1), - } - - return data - - -def parse_final_mfa_page(soup: BeautifulSoup) -> dict[str, str]: - data: dict[str, str] = {} - - # get form by id "otp" - form = soup.find("form", {"id": "otp"}) - - if not isinstance(form, Tag): - raise Exception("Could not find OTP form.") - - url = form["action"] - - if not isinstance(url, str): - raise Exception(f"Action was not a string. Found: {url} of type: {type(url)}") - - csrf_token_input = form.find("input", {"name": "csrf_token"}) - - if not isinstance(csrf_token_input, Tag): - raise Exception("Could not find CSRF token input.") - - csrf_token = csrf_token_input["value"] - - if not isinstance(csrf_token, str): - raise Exception( - f"CSRF token was not a string. Found: {csrf_token} of type: {type(csrf_token)}" - ) - - data["url"] = url - data["csrf_token"] = csrf_token - - return data + # extra timeout waits added because sometimes + # page would load without the selected element + # being rendered + # a better approach would be welcome + extra_timeout = 2000 + + # ideally we would also want all our query + # selectors in config.py so that they can be + # changed easily if the site changes + + with sync_playwright() as p: + browser = p.chromium.launch() + context = browser.new_context() + page = context.new_page() + page.goto("https://connect.drexel.edu") + + page.wait_for_timeout(extra_timeout) + + sign_in = page.query_selector("button[name='_eventId_proceed']") + assert isinstance( + sign_in, ElementHandle + ), "Sign in button on Drexel Connect not found" + sign_in.click() + + page.wait_for_selector("input[name='loginfmt']") + page.wait_for_timeout(extra_timeout) + + email_input = page.query_selector("input[name='loginfmt']") + assert isinstance( + email_input, ElementHandle + ), "Email field on Microsoft Online not found" + email_input.fill(config.drexel_email) + + page.get_by_text("Next").click() + + page.wait_for_selector("input[name='passwd']") + page.wait_for_timeout(extra_timeout) + + password_input = page.query_selector("input[name='passwd']") + assert isinstance( + password_input, ElementHandle + ), "Password field on Microsoft Online not found" + password_input.fill(config.drexel_password) + + page.wait_for_selector("input[type='submit']") + page.wait_for_timeout(extra_timeout) + page.get_by_text("Sign in").click() + + if config.drexel_mfa_secret_key is not None: + mfa_token = totp.get_token(config.drexel_mfa_secret_key) + else: + mfa_token = input("Please input your MFA verification code: ") + + page.wait_for_selector("input[name='otc']") + page.wait_for_timeout(extra_timeout) + + mfa_input = page.query_selector("input[name='otc']") + assert isinstance( + mfa_input, ElementHandle + ), "MFA input field on Microsoft Online not found" + mfa_input.fill(mfa_token) + + page.wait_for_selector("input[type='submit']") + page.wait_for_timeout(extra_timeout) + + submit_button = page.query_selector("input[type='submit']") + assert isinstance( + submit_button, ElementHandle + ), "Submit button on Microsoft Online for MFA not found" + submit_button.click() + + page.wait_for_url("https://connect.drexel.edu/**") + page.wait_for_timeout(extra_timeout) + + for cookie in context.cookies(): + session.cookies.set( + cookie["name"], cookie["value"], domain=cookie["domain"] + ) # type: ignore + + browser.close() + return session diff --git a/src/main.py b/src/main.py index cde9b4b..851af89 100644 --- a/src/main.py +++ b/src/main.py @@ -17,27 +17,36 @@ def main(args: argparse.Namespace) -> None: data = scrape(include_ratings=args.ratings, all_colleges=args.all_colleges) assert len(data) > 0, "No data found" + print("Found {} items".format(len(data))) - with open("data.json", "w") as f: - json.dump(data, f, indent=4) + if not args.no_file: + with open(args.output_file, "w") as f: + json.dump(data, f, indent=4) - print("Found {} items".format(len(data))) - print("Data written to data.json") + print(f"Data written to {args.output_file}") if args.db: print("Time taken to scrape data: {} seconds".format(time.time() - start_time)) print() print("Updating database...") db.populate_db(data) - - print("Done!") + print("Done!") print("--- {} seconds ---".format(time.time() - start_time)) if __name__ == "__main__": parser = argparse.ArgumentParser( - description="Scrape data from Term Master Schedule and save it to a data.json file." + prog="python3 src/main.py", + description="Scrape data from Drexel Term Master Schedule and save it to a JSON file.", + ) + parser.add_argument( + "-o", + "--output-file", + metavar="FILE", + action="store", + default="data.json", + help="File to write the data to (include the .json extension in the file name)", ) parser.add_argument( "--ratings", @@ -49,6 +58,11 @@ def main(args: argparse.Namespace) -> None: action="store_true", help="Include all colleges in the data, not just the one in the config.py file", ) + parser.add_argument( + "--no-file", + action="store_true", + help="Do not write the data to a file", + ) parser.add_argument( "--db", action="store_true", diff --git a/src/scrape.py b/src/scrape.py index bbcfbc9..018052b 100644 --- a/src/scrape.py +++ b/src/scrape.py @@ -3,7 +3,6 @@ import json import os from typing import Any -import traceback import time from helpers import send_request @@ -17,6 +16,7 @@ def scrape( ) -> dict[str, dict[str, Any]]: session = Session() + print("Signing in...") is_logged_into_drexel_connect = False failiure_count = 0 reset_period = 1 # seconds @@ -33,7 +33,7 @@ def scrape( ) except Exception: print("Error logging in to Drexel Connect: ") - print(traceback.format_exc()) + # not printing stack trace in case password gets accidentally logged print(f"Trying again in {reset_period} seconds...") failiure_count += 1