Skip to content
18 changes: 17 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,23 @@ python src/main.py

The scraper will output a JSON file called `data.json` in the same directory as the scraper.

You can modify the scraper to scrape other terms by changing the `year`, `quarter`, and `college_code` variables in `src/config.py`.
#### Automatic Quarter Detection

The scraper now automatically detects the current Drexel quarter based on the current date. You no longer need to manually update the `year` and `quarter` values in `src/config.py`. The system uses the following schedule:

- **Fall Quarter (15)**: July 1 - September 27
- **Winter Quarter (25)**: September 28 - January 15
- **Spring Quarter (35)**: January 16 - April 14
- **Summer Quarter (45)**: April 15 - June 30

If you need to override the automatic detection (e.g., for testing or scraping a specific past/future quarter), you can set the `DREXEL_YEAR` and `DREXEL_QUARTER` environment variables:

```bash
export DREXEL_YEAR=2024
export DREXEL_QUARTER=35 # Spring quarter
```

You can still modify the `college_code` variable in `src/config.py` to scrape a specific college.

To view all the options that the scraper supports, run `python3 src/main.py --help` on Mac/Linux, or `python src/main.py --help` on Windows.

Expand Down
39 changes: 30 additions & 9 deletions src/config.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,31 @@
import os
import sys

# in format YYYY (e.g. 2022)
# example value: 2022
year = "2024"
# 15 for Fall, 25 for Winter, 35 for Spring, 45 for Summer
# example value: 45
quarter = "45"
# check college code by going to the tms website and selecting your college from the left sidebar
from quarter_utils import get_current_quarter_and_year, get_quarter_name

# Automatically determine the current quarter and year
# Can be overridden with DREXEL_YEAR and DREXEL_QUARTER environment variables
if "DREXEL_YEAR" in os.environ and "DREXEL_QUARTER" in os.environ:
year = os.environ["DREXEL_YEAR"]
quarter = os.environ["DREXEL_QUARTER"]
print(
f"Using manually configured {get_quarter_name(quarter)} {year} "
f"quarter (code: {quarter})"
)
else:
year, quarter = get_current_quarter_and_year()
print(
f"Using auto-detected {get_quarter_name(quarter)} {year} "
f"quarter (code: {quarter})"
)

# Note: These values are now automatically determined based on the current date
# Fall (15): July 1 - September 27
# Winter (25): September 28 - January 15
# Spring (35): January 16 - April 14
# Summer (45): April 15 - June 30
# check college code by going to the tms website and selecting your college
# from the left sidebar
# the URL bar should update and it should end with something like collCode=CI
# the characters after the = sign is your college code
# e.g. in this URL the college code is CI
Expand All @@ -27,7 +45,9 @@ def get_environ(key: str, required: bool = True) -> str:
return os.environ[key]
elif required:
print(
f"{key} is missing from your environment variables and is required to run this script. See {environ_help_url} for more information and help."
f"{key} is missing from your environment variables and is required "
f"to run this script. See {environ_help_url} for more information "
f"and help."
)
sys.exit(1)
else:
Expand All @@ -37,7 +57,8 @@ def get_environ(key: str, required: bool = True) -> str:
# Drexel Connect Credentials
drexel_email = get_environ("DREXEL_EMAIL")
drexel_password = get_environ("DREXEL_PASSWORD")
# This is not required if the user is using a separate authenticator app and will manually approve the login attempt
# This is not required if the user is using a separate authenticator app
# and will manually approve the login attempt
drexel_mfa_secret_key = get_environ("DREXEL_MFA_SECRET_KEY", False) or None

# URL's
Expand Down
29 changes: 22 additions & 7 deletions src/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
from datetime import datetime
from pytz import timezone
from typing import Any
import config
from quarter_utils import get_quarter_name


def populate_db(data: dict[str, dict[str, Any]]) -> None:
Expand Down Expand Up @@ -212,14 +214,27 @@ def create_tables(cur: cursor) -> None:
def update_metadata(cur: cursor) -> None:
tz = timezone("US/Eastern")
current_datetime = datetime.now(tz).strftime("%m/%d/%y %I:%M %p")
cur.execute(

# Get current quarter information
quarter_name = get_quarter_name(config.quarter)

# Update multiple metadata values
metadata_updates = [
("last_updated", current_datetime),
("current_year", config.year),
("current_quarter", config.quarter),
("current_quarter_name", quarter_name),
("current_term", f"{quarter_name} {config.year}"),
]

cur.executemany(
"""
INSERT INTO metadata (key, value)
VALUES ('last_updated', %s)
ON CONFLICT (key)
DO UPDATE SET value = EXCLUDED.value;
""",
(current_datetime,),
INSERT INTO metadata (key, value)
VALUES (%s, %s)
ON CONFLICT (key)
DO UPDATE SET value = EXCLUDED.value;
""",
metadata_updates,
)


Expand Down
72 changes: 72 additions & 0 deletions src/quarter_utils.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
"""Utilities for determining the current Drexel quarter based on date."""

from datetime import datetime


def get_current_quarter_and_year() -> tuple[str, str]:
"""
Determine the current Drexel quarter and year based on the current date.

Drexel quarters:
- Fall (15): July 1 - September 27
- Winter (25): September 28 - January 15
- Spring (35): January 16 - April 14
- Summer (45): April 15 - June 30

Returns:
Tuple of (year, quarter_code) as strings
"""
now = datetime.now()
month = now.month
day = now.day
year = now.year

# Determine quarter based on month and day
if month == 7 or month == 8 or (month == 9 and day <= 27):
# Fall quarter
quarter = "15"
elif (
(month == 9 and day >= 28)
or month == 10
or month == 11
or month == 12
or (month == 1 and day <= 15)
):
# Winter quarter
quarter = "25"
elif (
(month == 1 and day >= 16)
or month == 2
or month == 3
or (month == 4 and day <= 14)
):
# Spring quarter
quarter = "35"
else:
# Summer quarter (April 15 - June 30)
quarter = "45"

# For Winter quarter spanning two calendar years (Sept 28 - Jan 15),
# if we're in January, it belongs to the previous year's academic year
if quarter == "25" and month == 1:
year = year - 1

return str(year), quarter


def get_quarter_name(quarter_code: str) -> str:
"""Get the human-readable name for a quarter code."""
quarter_names = {
"15": "Fall",
"25": "Winter",
"35": "Spring",
"45": "Summer",
}
return quarter_names.get(quarter_code, "Unknown")


if __name__ == "__main__":
# Test the function
year, quarter = get_current_quarter_and_year()
quarter_name = get_quarter_name(quarter)
print(f"Current quarter: {quarter_name} {year} (code: {quarter})")