Add APM (Application Performance Monitoring) software

Dockerfile

@@ -12,4 +12,4 @@ RUN pip install --upgrade pip && \
     playwright install chromium --with-deps
 
 # Run the Python script
-CMD ["python3", "src/main.py", "--db", "--all-colleges", "--ratings", "--email"]
+CMD ["python3", "src/main.py", "--db", "--all-colleges", "--ratings", "--email"]

README.md

@@ -203,6 +203,15 @@ If you want to reset the Grafana settings, delete the grafana_data directory ins
 
 NOTE: Docker Compose is only used for local development. The production version of the scraper uses Kubernetes. The Kubernetes configuration files live in the `k8s` directory.
 
+## Logfire APM
+
+To set up Logfire APM for monitoring and error tracking, follow these steps:
+
+1. Sign up for a Logfire account at [Logfire](https://pydantic.dev/logfire).
+2. Obtain your Logfire API key from the Logfire dashboard.
+3. Set the environment variables `LOGFIRE_API_KEY` and `LOGFIRE_ENVIRONMENT` with your Logfire API key and the environment name (e.g., "development" or "production").
+4. Run the scraper as usual, and Logfire will automatically track performance and errors.
+
 ## Contributing
 
 If you wish to contribute, please take a look at the "Issues" tab and see if there are any issues you can help with. If you wish to add/request a new feature, you can create a new issue, and we can talk about it there before you start working on it. If you wish to work on an existing issue, please comment on the issue so that I can assign it to you. Once you have completed the issue, you can create a pull request to the dev branch, and I will review it. After merging the changes to the dev branch, the job will be deployed on the dev server (https://schedulerdev.zohair.dev). If there are no issues, I will merge the changes to the main branch, and the job will be deployed on the main server (https://scheduler.zohair.dev).

docker-compose.yml

@@ -14,6 +14,8 @@ services:
       DREXEL_EMAIL: ${DREXEL_EMAIL}
       DREXEL_PASSWORD: ${DREXEL_PASSWORD}
       DREXEL_MFA_SECRET_KEY: ${DREXEL_MFA_SECRET_KEY}
+      LOGFIRE_API_KEY: ${LOGFIRE_API_KEY}
+      LOGFIRE_ENVIRONMENT: ${LOGFIRE_ENVIRONMENT}
     volumes:
       - ./:/app
     depends_on:

requirements.txt

@@ -13,6 +13,7 @@ GitPython==3.1.42
 greenlet==3.0.3
 idna==3.6
 jmespath==1.0.1
+logfire==1.0.0
 mypy==1.9.0
 mypy-extensions==1.0.0
 packaging==24.0

src/config.py

@@ -51,6 +51,9 @@ def get_environ(key: str, required: bool = True) -> str:
 topic_arn = os.getenv("DREXEL_SCHEDULER_TOPIC_ARN")
 sns_endpoint = os.getenv("SNS_ENDPOINT_URL", None)
 
+# Logfire Configuration
+logfire_api_key = get_environ("LOGFIRE_API_KEY")
+logfire_environment = get_environ("LOGFIRE_ENVIRONMENT")
 
 # element attribute dictionaries the soup will look for
 class attributes:

src/main.py

@@ -9,7 +9,7 @@
 import argparse
 import emailer
 import db
-
+import logfire
 
 def main(args: argparse.Namespace) -> None:
     start_time = time.time()
@@ -77,11 +77,13 @@ def main(args: argparse.Namespace) -> None:
     args = parser.parse_args()
 
     try:
+        logfire.init(api_key=os.environ["LOGFIRE_API_KEY"], environment=os.environ["LOGFIRE_ENVIRONMENT"])
         if not os.path.exists("performance"):
             os.makedirs("performance")
         cProfile.run("main(args)", "performance/profile_output.pstat")
     except Exception:
         trace = traceback.format_exc()
+        logfire.log_error(trace)
         print(trace)
 
         if args.email: