diff --git a/Dockerfile b/Dockerfile index 1a3bc6e..cdd34a5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,4 +12,4 @@ RUN pip install --upgrade pip && \ playwright install chromium --with-deps # Run the Python script -CMD ["python3", "src/main.py", "--db", "--all-colleges", "--ratings", "--email"] \ No newline at end of file +CMD ["python3", "src/main.py", "--db", "--all-colleges", "--ratings", "--email"] diff --git a/README.md b/README.md index 2170ee0..ca774ea 100644 --- a/README.md +++ b/README.md @@ -203,6 +203,15 @@ If you want to reset the Grafana settings, delete the grafana_data directory ins NOTE: Docker Compose is only used for local development. The production version of the scraper uses Kubernetes. The Kubernetes configuration files live in the `k8s` directory. +## Logfire APM + +To set up Logfire APM for monitoring and error tracking, follow these steps: + +1. Sign up for a Logfire account at [Logfire](https://pydantic.dev/logfire). +2. Obtain your Logfire API key from the Logfire dashboard. +3. Set the environment variables `LOGFIRE_API_KEY` and `LOGFIRE_ENVIRONMENT` with your Logfire API key and the environment name (e.g., "development" or "production"). +4. Run the scraper as usual, and Logfire will automatically track performance and errors. + ## Contributing If you wish to contribute, please take a look at the "Issues" tab and see if there are any issues you can help with. If you wish to add/request a new feature, you can create a new issue, and we can talk about it there before you start working on it. If you wish to work on an existing issue, please comment on the issue so that I can assign it to you. Once you have completed the issue, you can create a pull request to the dev branch, and I will review it. After merging the changes to the dev branch, the job will be deployed on the dev server (https://schedulerdev.zohair.dev). If there are no issues, I will merge the changes to the main branch, and the job will be deployed on the main server (https://scheduler.zohair.dev). diff --git a/docker-compose.yml b/docker-compose.yml index 8a6f48a..d0094f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,8 @@ services: DREXEL_EMAIL: ${DREXEL_EMAIL} DREXEL_PASSWORD: ${DREXEL_PASSWORD} DREXEL_MFA_SECRET_KEY: ${DREXEL_MFA_SECRET_KEY} + LOGFIRE_API_KEY: ${LOGFIRE_API_KEY} + LOGFIRE_ENVIRONMENT: ${LOGFIRE_ENVIRONMENT} volumes: - ./:/app depends_on: diff --git a/requirements.txt b/requirements.txt index bdfd90a..cf4ea40 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,6 +13,7 @@ GitPython==3.1.42 greenlet==3.0.3 idna==3.6 jmespath==1.0.1 +logfire==1.0.0 mypy==1.9.0 mypy-extensions==1.0.0 packaging==24.0 diff --git a/src/config.py b/src/config.py index c1abb63..3024012 100644 --- a/src/config.py +++ b/src/config.py @@ -51,6 +51,9 @@ def get_environ(key: str, required: bool = True) -> str: topic_arn = os.getenv("DREXEL_SCHEDULER_TOPIC_ARN") sns_endpoint = os.getenv("SNS_ENDPOINT_URL", None) +# Logfire Configuration +logfire_api_key = get_environ("LOGFIRE_API_KEY") +logfire_environment = get_environ("LOGFIRE_ENVIRONMENT") # element attribute dictionaries the soup will look for class attributes: diff --git a/src/main.py b/src/main.py index 851af89..f3478de 100644 --- a/src/main.py +++ b/src/main.py @@ -9,7 +9,7 @@ import argparse import emailer import db - +import logfire def main(args: argparse.Namespace) -> None: start_time = time.time() @@ -77,11 +77,13 @@ def main(args: argparse.Namespace) -> None: args = parser.parse_args() try: + logfire.init(api_key=os.environ["LOGFIRE_API_KEY"], environment=os.environ["LOGFIRE_ENVIRONMENT"]) if not os.path.exists("performance"): os.makedirs("performance") cProfile.run("main(args)", "performance/profile_output.pstat") except Exception: trace = traceback.format_exc() + logfire.log_error(trace) print(trace) if args.email: