Merge pull request #83 from Zohair-coder/dev

Dev

Author: Zohair-coder

.devcontainer/Dockerfile

@@ -0,0 +1,19 @@
+FROM mcr.microsoft.com/devcontainers/python
+
+# set the working directory in the container
+WORKDIR /app
+
+# copy the current directory contents into the container at /app
+COPY . /app
+
+# upgrade pip
+RUN pip install --upgrade pip
+
+# install dependencies
+RUN pip install -r requirements.txt
+
+# only for dev container
+RUN apt-get update && apt-get install -y postgresql-client
+
+# Run the Python script
+CMD ["python3", "src/main.py", "--db", "--all-colleges", "--ratings", "--email"]

.devcontainer/devcontainer.json

@@ -0,0 +1,19 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-docker-compose
+{
+	"name": "Drexel Scraper Dev Container",
+	"dockerComposeFile": [
+		"../docker-compose.yml",
+		"docker-compose.yml"
+	],
+	"service": "scraper",
+	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-python.python",
+				"ms-python.black-formatter"
+			]
+		}
+	}
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,12 @@
+version: '3'
+services:
+  scraper:
+    build:
+      context: .
+      dockerfile: .devcontainer/Dockerfile
+
+    volumes:
+      - ..:/workspaces:cached
+
+    command: /bin/sh -c "while sleep 1000; do :; done"
+ 

.github/dependabot.yml

@@ -0,0 +1,25 @@
+version: 2
+updates:
+ - package-ecosystem: "devcontainers"
+   directory: "/"
+   schedule:
+     interval: weekly
+   open-pull-requests-limit: 0
+
+ - package-ecosystem: "docker"
+   directory: "/"
+   schedule:
+     interval: weekly
+   open-pull-requests-limit: 0
+
+ - package-ecosystem: "pip"
+   directory: "/"
+   schedule:
+     interval: weekly
+   open-pull-requests-limit: 0
+
+ - package-ecosystem: "github-actions"
+   directory: "/"
+   schedule:
+     interval: weekly
+   open-pull-requests-limit: 0

.github/workflows/functional_test.yml

@@ -9,7 +9,13 @@ jobs:
     name: Functional Test
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
+    
+    - name: Check code formatting with Black
+      uses: psf/black@stable
+      with:
+        options: "--check --verbose"
+        src: "./src"
 
     - name: Set up Docker Compose
       uses: docker/setup-buildx-action@v3

.gitignore

@@ -4,7 +4,6 @@ data.json
 .env
 postgres-data
 grafana_data
-.vscode
 trash.py
 profile_output.pstat
 callgrind.out.profile

.vscode/launch.json

@@ -0,0 +1,23 @@
+{
+  // Use IntelliSense to learn about possible attributes.
+  // Hover to view descriptions of existing attributes.
+  // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Python: main.py",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${workspaceFolder}/src/main.py",
+      "console": "integratedTerminal",
+      "justMyCode": true,
+      "cwd": "${workspaceFolder}",
+      "args": ["--ratings"]
+      // "args": ["--all-colleges"]
+      // "args": ["--db"]
+      // "args": ["--ratings", "--db", "--all-colleges"]
+      // "args": ["--ratings", "--db"]
+      // "args": ["--email"]
+    }
+  ]
+}

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10-alpine
+FROM python:3.12-alpine
 
 # set the working directory in the container
 WORKDIR /app

README.md

@@ -126,7 +126,7 @@ python src/main.py --db --all-colleges --ratings
 
 ## Docker
 
-You can also run the scraper in a Docker container. Make sure [Docker](https://docs.docker.com/get-docker/) is installed. Then run the following command to run it:
+You can also run the scraper in a Docker container. Make sure [Docker](https://docs.docker.com/get-docker/) is installed. You can then either open up the folder in VS Code as a Dev Container (preferred), or run the following command to run it:
 
 ```bash
 docker compose up -d --build

docker-compose.yml

@@ -8,10 +8,15 @@ services:
       SCHEDULER_DB_HOSTNAME: postgres
       SCHEDULER_DB_PORT: 5432
       SCHEDULER_DB_PG_PASSWORD: super-secret-password
+      SNS_ENDPOINT_URL: http://localstack:4566
+      AWS_DEFAULT_REGION: us-east-1
+      AWS_ACCESS_KEY_ID: "my-access-key"
+      AWS_SECRET_ACCESS_KEY: "my-secret-access-key"
     volumes:
       - ./:/app
     depends_on:
       - postgres
+      - localstack
 
   grafana:
     image: grafana/grafana-oss
@@ -49,3 +54,16 @@ services:
       - DATABASE_URL=postgres://postgres:super-secret-password@postgres:5432/postgres?sslmode=disable
     depends_on:
       - postgres
+  
+  # Used to imitate AWS services locally
+  # We are using SNS in this case for sending error emails
+  localstack:
+    image: localstack/localstack
+    ports:
+      - "4566:4566"
+    environment:
+      - SERVICES=sns
+      - EDGE_PORT=4566
+    volumes:
+      - "/var/run/docker.sock:/var/run/docker.sock"
+      - "./sns-init-scripts/create-sns-topic.sh:/etc/localstack/init/ready.d/init-aws.sh"