Skip to content

Commit 379410f

Browse files
Zohair-coderZohair-coder
authored
Merge pull request #92 from Zohair-coder/sso
added cookie
2 parents 34b3c37 + b4c142b commit 379410f

File tree

7 files changed

+295
-135
lines changed

7 files changed

+295
-135
lines changed

‎.github/workflows/functional_test.yml‎

Lines changed: 138 additions & 134 deletions
Original file line numberDiff line numberDiff line change
@@ -1,159 +1,163 @@
1-
# name: Functional test
1+
name: Functional test
22

3-
# on:
4-
# pull_request:
5-
# branches: [ main, dev ]
3+
on:
4+
pull_request:
5+
branches: [ main, dev ]
66

7-
# jobs:
8-
# functional-test:
9-
# name: Functional Test
10-
# runs-on: ubuntu-latest
11-
# steps:
12-
# - uses: actions/checkout@v4
7+
jobs:
8+
functional-test:
9+
name: Functional Test
10+
runs-on: ubuntu-latest
11+
steps:
12+
- uses: actions/checkout@v4
1313

14-
# - name: Install Python
15-
# uses: actions/setup-python@v5
16-
# with:
17-
# python-version: "3.11"
14+
- name: Install Python
15+
uses: actions/setup-python@v5
16+
with:
17+
python-version: "3.11"
1818

19-
# - name: Install dependencies
20-
# run: |
21-
# python -m pip install --upgrade pip
22-
# pip install -r requirements.txt
19+
- name: Install dependencies
20+
run: |
21+
python -m pip install --upgrade pip
22+
pip install -r requirements.txt
2323
24-
# - name: Run mypy
25-
# run: mypy src --strict
24+
- name: Run mypy
25+
run: mypy src --strict
2626

27-
# - name: Run Linter
28-
# run: ruff check --output-format=github src
27+
- name: Run Linter
28+
run: ruff check --output-format=github src
2929

30-
# - name: Run Formatter
31-
# run: ruff format --check src
30+
- name: Run Formatter
31+
run: ruff format --check src
3232

33-
# - name: Set up Docker Compose
34-
# uses: docker/setup-buildx-action@v3
33+
- name: Set up Docker Compose
34+
uses: docker/setup-buildx-action@v3
3535

36-
# - name: Start PostgreSQL
37-
# run: docker-compose up -d postgres
36+
- name: Start PostgreSQL
37+
run: docker-compose up -d postgres
3838

39-
# - name: Wait for PostgreSQL to become ready
40-
# run: |
41-
# until docker compose exec postgres pg_isready ; do
42-
# echo "Waiting for PostgreSQL to become ready..."
43-
# sleep 1
44-
# done
39+
- name: Wait for PostgreSQL to become ready
40+
run: |
41+
until docker compose exec postgres pg_isready ; do
42+
echo "Waiting for PostgreSQL to become ready..."
43+
sleep 1
44+
done
4545
46-
# - name: Delete cache
47-
# run: rm cache/*
46+
- name: Delete cache
47+
run: rm cache/*
4848

49-
# - name: Run the scraper
50-
# run: docker compose run scraper python3 src/main.py --db --all-colleges --ratings
49+
- name: Run the scraper
50+
env:
51+
DREXEL_SCHEDULER_SHIB_IDP_SESSION_COOKIE: ${{ secrets.DREXEL_SCHEDULER_SHIB_IDP_SESSION_COOKIE }}
52+
run: docker compose run scraper python3 src/main.py --db --all-colleges --ratings
5153

52-
# - name: Verify courses data exists in database
53-
# run: |
54-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM courses;" | awk 'FNR == 3 {print $1}')
55-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
56-
# echo "No data found in the database. Failing the pipeline."
57-
# exit 1
58-
# else
59-
# echo "Data successfully stored in the database. Found $RESULT records."
60-
# fi
54+
- name: Verify courses data exists in database
55+
run: |
56+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM courses;" | awk 'FNR == 3 {print $1}')
57+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
58+
echo "No data found in the database. Failing the pipeline."
59+
exit 1
60+
else
61+
echo "Data successfully stored in the database. Found $RESULT records."
62+
fi
6163
62-
# - name: Verify instructors data exists in database
63-
# run: |
64-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM instructors;" | awk 'FNR == 3 {print $1}')
65-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
66-
# echo "No data found in the database. Failing the pipeline."
67-
# exit 1
68-
# else
69-
# echo "Data successfully stored in the database. Found $RESULT records."
70-
# fi
64+
- name: Verify instructors data exists in database
65+
run: |
66+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM instructors;" | awk 'FNR == 3 {print $1}')
67+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
68+
echo "No data found in the database. Failing the pipeline."
69+
exit 1
70+
else
71+
echo "Data successfully stored in the database. Found $RESULT records."
72+
fi
7173
72-
# - name: Verify course_instructor data exists in database
73-
# run: |
74-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM course_instructor;" | awk 'FNR == 3 {print $1}')
75-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
76-
# echo "No data found in the database. Failing the pipeline."
77-
# exit 1
78-
# else
79-
# echo "Data successfully stored in the database. Found $RESULT records."
80-
# fi
74+
- name: Verify course_instructor data exists in database
75+
run: |
76+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM course_instructor;" | awk 'FNR == 3 {print $1}')
77+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
78+
echo "No data found in the database. Failing the pipeline."
79+
exit 1
80+
else
81+
echo "Data successfully stored in the database. Found $RESULT records."
82+
fi
8183
82-
# - name: Verify all_course_instructor_data exists in database
83-
# run: |
84-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM all_course_instructor_data;" | awk 'FNR == 3 {print $1}')
85-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
86-
# echo "No data found in the database. Failing the pipeline."
87-
# exit 1
88-
# else
89-
# echo "Data successfully stored in the database. Found $RESULT records."
90-
# fi
84+
- name: Verify all_course_instructor_data exists in database
85+
run: |
86+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM all_course_instructor_data;" | awk 'FNR == 3 {print $1}')
87+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
88+
echo "No data found in the database. Failing the pipeline."
89+
exit 1
90+
else
91+
echo "Data successfully stored in the database. Found $RESULT records."
92+
fi
9193
92-
# - name: Verify metadata exists in database
93-
# run: |
94-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM metadata;" | awk 'FNR == 3 {print $1}')
95-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
96-
# echo "No data found in the database. Failing the pipeline."
97-
# exit 1
98-
# else
99-
# echo "Data successfully stored in the database. Found $RESULT records."
100-
# fi
94+
- name: Verify metadata exists in database
95+
run: |
96+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM metadata;" | awk 'FNR == 3 {print $1}')
97+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
98+
echo "No data found in the database. Failing the pipeline."
99+
exit 1
100+
else
101+
echo "Data successfully stored in the database. Found $RESULT records."
102+
fi
101103
102-
# - name: Reset database
103-
# run: docker compose run scraper sh -c 'apk add postgresql-client && ./scripts/reset-db.sh;'
104+
- name: Reset database
105+
run: docker compose run scraper sh -c 'apk add postgresql-client && ./scripts/reset-db.sh;'
104106

105-
# - name: Run scraper again (to test cache)
106-
# run: docker compose run scraper python3 src/main.py --db --all-colleges --ratings
107+
- name: Run scraper again (to test cache)
108+
env:
109+
DREXEL_SCHEDULER_SHIB_IDP_SESSION_COOKIE: ${{ secrets.DREXEL_SCHEDULER_SHIB_IDP_SESSION_COOKIE }}
110+
run: docker compose run scraper python3 src/main.py --db --all-colleges --ratings
107111

108-
# - name: Verify courses data exists in database
109-
# run: |
110-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM courses;" | awk 'FNR == 3 {print $1}')
111-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
112-
# echo "No data found in the database. Failing the pipeline."
113-
# exit 1
114-
# else
115-
# echo "Data successfully stored in the database. Found $RESULT records."
116-
# fi
112+
- name: Verify courses data exists in database
113+
run: |
114+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM courses;" | awk 'FNR == 3 {print $1}')
115+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
116+
echo "No data found in the database. Failing the pipeline."
117+
exit 1
118+
else
119+
echo "Data successfully stored in the database. Found $RESULT records."
120+
fi
117121
118-
# - name: Verify instructors data exists in database
119-
# run: |
120-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM instructors;" | awk 'FNR == 3 {print $1}')
121-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
122-
# echo "No data found in the database. Failing the pipeline."
123-
# exit 1
124-
# else
125-
# echo "Data successfully stored in the database. Found $RESULT records."
126-
# fi
122+
- name: Verify instructors data exists in database
123+
run: |
124+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM instructors;" | awk 'FNR == 3 {print $1}')
125+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
126+
echo "No data found in the database. Failing the pipeline."
127+
exit 1
128+
else
129+
echo "Data successfully stored in the database. Found $RESULT records."
130+
fi
127131
128-
# - name: Verify course_instructor data exists in database
129-
# run: |
130-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM course_instructor;" | awk 'FNR == 3 {print $1}')
131-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
132-
# echo "No data found in the database. Failing the pipeline."
133-
# exit 1
134-
# else
135-
# echo "Data successfully stored in the database. Found $RESULT records."
136-
# fi
132+
- name: Verify course_instructor data exists in database
133+
run: |
134+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM course_instructor;" | awk 'FNR == 3 {print $1}')
135+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
136+
echo "No data found in the database. Failing the pipeline."
137+
exit 1
138+
else
139+
echo "Data successfully stored in the database. Found $RESULT records."
140+
fi
137141
138-
# - name: Verify all_course_instructor_data exists in database
139-
# run: |
140-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM all_course_instructor_data;" | awk 'FNR == 3 {print $1}')
141-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
142-
# echo "No data found in the database. Failing the pipeline."
143-
# exit 1
144-
# else
145-
# echo "Data successfully stored in the database. Found $RESULT records."
146-
# fi
142+
- name: Verify all_course_instructor_data exists in database
143+
run: |
144+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM all_course_instructor_data;" | awk 'FNR == 3 {print $1}')
145+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
146+
echo "No data found in the database. Failing the pipeline."
147+
exit 1
148+
else
149+
echo "Data successfully stored in the database. Found $RESULT records."
150+
fi
147151
148-
# - name: Verify metadata exists in database
149-
# run: |
150-
# RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM metadata;" | awk 'FNR == 3 {print $1}')
151-
# if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
152-
# echo "No data found in the database. Failing the pipeline."
153-
# exit 1
154-
# else
155-
# echo "Data successfully stored in the database. Found $RESULT records."
156-
# fi
152+
- name: Verify metadata exists in database
153+
run: |
154+
RESULT=$(docker compose exec postgres psql -U postgres -d postgres -c "SELECT COUNT(*) FROM metadata;" | awk 'FNR == 3 {print $1}')
155+
if [ -z "$RESULT" ] || [ "$RESULT" = "0" ]; then
156+
echo "No data found in the database. Failing the pipeline."
157+
exit 1
158+
else
159+
echo "Data successfully stored in the database. Found $RESULT records."
160+
fi
157161
158-
# - name: Cleanup
159-
# run: docker-compose down -v
162+
- name: Cleanup
163+
run: docker-compose down -v

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@ grafana_data
77
trash.py
88
profile_output.pstat
99
callgrind.out.profile
10-
scraperenv
10+
.venv

0 commit comments

Comments
 (0)