Skip to content

Commit db7b106

Browse files
Zohair-coderZohair-coder
committed
added cookie
1 parent 6cef2a0 commit db7b106

File tree

6 files changed

+294
-135
lines changed

6 files changed

+294
-135
lines changed

‎.github/workflows/functional_test.yml‎

Lines changed: 137 additions & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -1,159 +1,162 @@
1-
# name: Functional test
1+
name: Functional test
22

3-
# on:
4-
# pull_request:
5-
# branches: [ main, dev ]
3+
on:
4+
pull_request:
5+
branches: [ main, dev ]
66

7-
# jobs:
8-
# functional-test:
9-
# name: Functional Test
10-
# runs-on: ubuntu-latest
11-
# steps:
12-
# - uses: actions/checkout@v4
7+
env:
8+
DREXEL_SCHEDULER_SHIB_IDP_SESSION_COOKIE: ${{ secrets.DREXEL_SCHEDULER_SHIB_IDP_SESSION_COOKIE }}
9+
10+
jobs:
11+
functional-test:
12+
name: Functional Test
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: actions/checkout@v4
1316

14-
# - name: Install Python
15-
# uses: actions/setup-python@v5
16-
# with:
17-
# python-version: "3.11"
17+
- name: Install Python
18+
uses: actions/setup-python@v5
19+
with:
20+
python-version: "3.11"
1821

19-
# - name: Install dependencies
20-
# run: |
21-
# python -m pip install --upgrade pip
22-
# pip install -r requirements.txt
22+
- name: Install dependencies
23+
run: |
24+
python -m pip install --upgrade pip
25+
pip install -r requirements.txt
2326
24-
# - name: Run mypy
25-
# run: mypy src --strict
27+
- name: Run mypy
28+
run: mypy src --strict
2629

27-
# - name: Run Linter
28-
# run: ruff check --output-format=github src
30+
- name: Run Linter
31+
run: ruff check --output-format=github src
2932

30-
# - name: Run Formatter
31-
# run: ruff format --check src
33+
- name: Run Formatter
34+
run: ruff format --check src
3235

33-
# - name: Set up Docker Compose
34-
# uses: docker/setup-buildx-action@v3
36+
- name: Set up Docker Compose
37+
uses: docker/setup-buildx-action@v3
3538

36-
# - name: Start PostgreSQL
37-
# run: docker-compose up -d postgres
39+
- name: Start PostgreSQL
40+
run: docker-compose up -d postgres
3841

39-
# - name: Wait for PostgreSQL to become ready
40-
# run: |
41-
# until docker compose exec postgres pg_isready ; do
42-
# echo "Waiting for PostgreSQL to become ready..."
43-
# sleep 1
44-
# done
42+
- name: Wait for PostgreSQL to become ready
43+
run: |
44+
until docker compose exec postgres pg_isready ; do
45+
echo "Waiting for PostgreSQL to become ready..."
46+
sleep 1
47+
done
4548
46-
# - name: Delete cache
47-
# run: rm cache/*
49+
- name: Delete cache
50+
run: rm cache/*
4851

49-
# - name: Run the scraper
50-
# run: docker compose run scraper python3 src/main.py --db --all-colleges --ratings
52+
- name: Run the scraper
53+
run: docker compose run scraper python3 src/main.py --db --all-colleges --ratings
5154

52-
# - name: Verify courses data exists in database
53-
# run: |
54-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM courses;" | awk 'FNR == 3 {print $1}')
55-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
56-
# echo "No data found in the database. Failing the pipeline."
57-
# exit 1
58-
# else
59-
# echo "Data successfully stored in the database. Found $RESULT records."
60-
# fi
55+
- name: Verify courses data exists in database
56+
run: |
57+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM courses;" | awk 'FNR == 3 {print $1}')
58+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
59+
echo "No data found in the database. Failing the pipeline."
60+
exit 1
61+
else
62+
echo "Data successfully stored in the database. Found $RESULT records."
63+
fi
6164
62-
# - name: Verify instructors data exists in database
63-
# run: |
64-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM instructors;" | awk 'FNR == 3 {print $1}')
65-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
66-
# echo "No data found in the database. Failing the pipeline."
67-
# exit 1
68-
# else
69-
# echo "Data successfully stored in the database. Found $RESULT records."
70-
# fi
65+
- name: Verify instructors data exists in database
66+
run: |
67+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM instructors;" | awk 'FNR == 3 {print $1}')
68+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
69+
echo "No data found in the database. Failing the pipeline."
70+
exit 1
71+
else
72+
echo "Data successfully stored in the database. Found $RESULT records."
73+
fi
7174
72-
# - name: Verify course_instructor data exists in database
73-
# run: |
74-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM course_instructor;" | awk 'FNR == 3 {print $1}')
75-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
76-
# echo "No data found in the database. Failing the pipeline."
77-
# exit 1
78-
# else
79-
# echo "Data successfully stored in the database. Found $RESULT records."
80-
# fi
75+
- name: Verify course_instructor data exists in database
76+
run: |
77+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM course_instructor;" | awk 'FNR == 3 {print $1}')
78+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
79+
echo "No data found in the database. Failing the pipeline."
80+
exit 1
81+
else
82+
echo "Data successfully stored in the database. Found $RESULT records."
83+
fi
8184
82-
# - name: Verify all_course_instructor_data exists in database
83-
# run: |
84-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM all_course_instructor_data;" | awk 'FNR == 3 {print $1}')
85-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
86-
# echo "No data found in the database. Failing the pipeline."
87-
# exit 1
88-
# else
89-
# echo "Data successfully stored in the database. Found $RESULT records."
90-
# fi
85+
- name: Verify all_course_instructor_data exists in database
86+
run: |
87+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM all_course_instructor_data;" | awk 'FNR == 3 {print $1}')
88+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
89+
echo "No data found in the database. Failing the pipeline."
90+
exit 1
91+
else
92+
echo "Data successfully stored in the database. Found $RESULT records."
93+
fi
9194
92-
# - name: Verify metadata exists in database
93-
# run: |
94-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM metadata;" | awk 'FNR == 3 {print $1}')
95-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
96-
# echo "No data found in the database. Failing the pipeline."
97-
# exit 1
98-
# else
99-
# echo "Data successfully stored in the database. Found $RESULT records."
100-
# fi
95+
- name: Verify metadata exists in database
96+
run: |
97+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM metadata;" | awk 'FNR == 3 {print $1}')
98+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
99+
echo "No data found in the database. Failing the pipeline."
100+
exit 1
101+
else
102+
echo "Data successfully stored in the database. Found $RESULT records."
103+
fi
101104
102-
# - name: Reset database
103-
# run: docker compose run scraper sh -c 'apk add postgresql-client && ./scripts/reset-db.sh;'
105+
- name: Reset database
106+
run: docker compose run scraper sh -c 'apk add postgresql-client && ./scripts/reset-db.sh;'
104107

105-
# - name: Run scraper again (to test cache)
106-
# run: docker compose run scraper python3 src/main.py --db --all-colleges --ratings
108+
- name: Run scraper again (to test cache)
109+
run: docker compose run scraper python3 src/main.py --db --all-colleges --ratings
107110

108-
# - name: Verify courses data exists in database
109-
# run: |
110-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM courses;" | awk 'FNR == 3 {print $1}')
111-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
112-
# echo "No data found in the database. Failing the pipeline."
113-
# exit 1
114-
# else
115-
# echo "Data successfully stored in the database. Found $RESULT records."
116-
# fi
111+
- name: Verify courses data exists in database
112+
run: |
113+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM courses;" | awk 'FNR == 3 {print $1}')
114+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
115+
echo "No data found in the database. Failing the pipeline."
116+
exit 1
117+
else
118+
echo "Data successfully stored in the database. Found $RESULT records."
119+
fi
117120
118-
# - name: Verify instructors data exists in database
119-
# run: |
120-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM instructors;" | awk 'FNR == 3 {print $1}')
121-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
122-
# echo "No data found in the database. Failing the pipeline."
123-
# exit 1
124-
# else
125-
# echo "Data successfully stored in the database. Found $RESULT records."
126-
# fi
121+
- name: Verify instructors data exists in database
122+
run: |
123+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM instructors;" | awk 'FNR == 3 {print $1}')
124+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
125+
echo "No data found in the database. Failing the pipeline."
126+
exit 1
127+
else
128+
echo "Data successfully stored in the database. Found $RESULT records."
129+
fi
127130
128-
# - name: Verify course_instructor data exists in database
129-
# run: |
130-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM course_instructor;" | awk 'FNR == 3 {print $1}')
131-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
132-
# echo "No data found in the database. Failing the pipeline."
133-
# exit 1
134-
# else
135-
# echo "Data successfully stored in the database. Found $RESULT records."
136-
# fi
131+
- name: Verify course_instructor data exists in database
132+
run: |
133+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM course_instructor;" | awk 'FNR == 3 {print $1}')
134+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
135+
echo "No data found in the database. Failing the pipeline."
136+
exit 1
137+
else
138+
echo "Data successfully stored in the database. Found $RESULT records."
139+
fi
137140
138-
# - name: Verify all_course_instructor_data exists in database
139-
# run: |
140-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM all_course_instructor_data;" | awk 'FNR == 3 {print $1}')
141-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
142-
# echo "No data found in the database. Failing the pipeline."
143-
# exit 1
144-
# else
145-
# echo "Data successfully stored in the database. Found $RESULT records."
146-
# fi
141+
- name: Verify all_course_instructor_data exists in database
142+
run: |
143+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM all_course_instructor_data;" | awk 'FNR == 3 {print $1}')
144+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
145+
echo "No data found in the database. Failing the pipeline."
146+
exit 1
147+
else
148+
echo "Data successfully stored in the database. Found $RESULT records."
149+
fi
147150
148-
# - name: Verify metadata exists in database
149-
# run: |
150-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM metadata;" | awk 'FNR == 3 {print $1}')
151-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
152-
# echo "No data found in the database. Failing the pipeline."
153-
# exit 1
154-
# else
155-
# echo "Data successfully stored in the database. Found $RESULT records."
156-
# fi
151+
- name: Verify metadata exists in database
152+
run: |
153+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM metadata;" | awk 'FNR == 3 {print $1}')
154+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
155+
echo "No data found in the database. Failing the pipeline."
156+
exit 1
157+
else
158+
echo "Data successfully stored in the database. Found $RESULT records."
159+
fi
157160
158-
# - name: Cleanup
159-
# run: docker-compose down -v
161+
- name: Cleanup
162+
run: docker-compose down -v

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@ grafana_data
77
trash.py
88
profile_output.pstat
99
callgrind.out.profile
10-
scraperenv
10+
.venv

0 commit comments

Comments
 (0)