Merge pull request #84 from Zohair-coder/dev

Dev

Author: Zohair-coder

.devcontainer/devcontainer.json

@@ -12,8 +12,12 @@
 		"vscode": {
 			"extensions": [
 				"ms-python.python",
-				"ms-python.black-formatter"
-			]
+				"charliermarsh.ruff"
+			],
+			"settings": {
+				"editor.defaultFormatter": "charliermarsh.ruff",
+				"editor.formatOnSave": true
+			}
 		}
 	}
 }

.github/workflows/functional_test.yml

@@ -11,11 +11,21 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Check code formatting with Black
-      uses: psf/black@stable
+    - name: Install Python
+      uses: actions/setup-python@v5
       with:
-        options: "--check --verbose"
-        src: "./src"
+        python-version: "3.11"
+    
+    - name: Install linter/formatter dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install ruff
+
+    - name: Run Linter
+      run: ruff check --output-format=github src
+    
+    - name: Run Formatter
+      run: ruff format --check src
 
     - name: Set up Docker Compose
       uses: docker/setup-buildx-action@v3
@@ -87,7 +97,7 @@ jobs:
         fi
 
     - name: Reset database
-      run: docker compose run scraper sh -c 'apk add postgresql-client && ./reset_db.bash;'
+      run: docker compose run scraper sh -c 'apk add postgresql-client && ./scripts/reset_db.sh;'
 
     - name: Run scraper again (to test cache)
       run: docker compose run scraper python3 src/main.py --db --all-colleges --ratings

README.md

@@ -92,7 +92,7 @@ python src/main.py --db
 This will create a new database `schedulerdb` and the necessary tables if they aren't already created, and then insert the data into the database. If the data is already populated, it will update the existing data. To delete all the data, make sure the environment variables specified in `src/db_config.py` are set and then run the following command (make sure you're using the Git Bash terminal if you're using Windows):
 
 ```bash
-./reset_db.bash
+./scripts/reset_db.sh
 ```
 
 To view the schema for the tables, you can look at the `src/create_tables.sql` file.

docker-compose.yml

@@ -66,4 +66,4 @@ services:
       - EDGE_PORT=4566
     volumes:
       - "/var/run/docker.sock:/var/run/docker.sock"
-      - "./sns-init-scripts/create-sns-topic.sh:/etc/localstack/init/ready.d/init-aws.sh"
+      - "./scripts/create-sns-topic.sh:/etc/localstack/init/ready.d/init-aws.sh"

sns-init-scripts/create-sns-topic.sh scripts/create-sns-topic.shsns-init-scripts/create-sns-topic.sh renamed to scripts/create-sns-topic.sh

@@ -36,7 +36,6 @@ def main(args: argparse.Namespace):
 
 
 if __name__ == "__main__":
-
     parser = argparse.ArgumentParser(
         description="Scrape data from Term Master Schedule and save it to a data.json file."
     )
@@ -67,7 +66,7 @@ def main(args: argparse.Namespace):
         if not os.path.exists("performance"):
             os.makedirs("performance")
         cProfile.run("main(args)", "performance/profile_output.pstat")
-    except Exception as e:
+    except Exception:
         trace = traceback.format_exc()
         print(trace)
 

reset_db.bash scripts/reset_db.shreset_db.bash renamed to scripts/reset_db.sh

@@ -7,7 +7,6 @@
 def parse_subject_page(
     html, data: dict, include_ratings: bool = False, ratings_cache: dict = {}
 ):
-
     soup = BeautifulSoup(html, "html.parser")
     table_rows = soup.find_all("tr", class_=["odd", "even"])
 
@@ -82,7 +81,6 @@ def get_instructors(
         name = instructor.strip()
 
         if include_ratings:
-
             # Search for first word and last word in name first
             # If that doesn't work, try first two words
             # If that still doesn't work, don't include rating
@@ -142,7 +140,6 @@ def fix_encoding_issue(text: str) -> str:
 
 
 def parse_days(d: str):
-
     if d == "TBD":
         return None
 

src/main.py

@@ -62,7 +62,6 @@ def scrape_all_subjects(session: Session, data: dict, html: str, include_ratings
     for subject_page_link in college_page_soup.find_all(
         "a", href=lambda href: href and href.startswith("/webtms_du/courseList")
     ):
-
         try:
             response = send_request(
                 session, config.tms_base_url + subject_page_link["href"]