Skip to content

Commit be70240

Browse files
klaasnicolaasklaasnicolaas
authored
Revamp the project (#18)
* Update the documentation * Update docker files * Refactor project and seperate the cities from main
1 parent b71a0ba commit be70240

File tree

6 files changed

+122
-101
lines changed

6 files changed

+122
-101
lines changed

‎README.md‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,20 +18,22 @@
1818

1919
## About
2020

21-
This project makes it possible to read garages data from the API of
22-
[garages_amsterdam](https://github.com/klaasnicolaas/garages_amsterdam)
23-
and write it to a Laravel database.
21+
This project makes it possible to read garages data from their API and write it to a Laravel database to use within NIPKaart.
22+
23+
## Supported
24+
25+
- Municipality of [Amsterdam](https://github.com/klaasnicolaas/garages_amsterdam)
2426

2527
## Set-up
2628

27-
Build docker image:
29+
1. Change the `city` and `wait_time` in the **.env** file.
30+
2. Build docker image
2831
```bash
29-
docker build -t garages-amsterdam .
32+
docker build -t nipkaart-garages-[CITY] .
3033
```
31-
32-
Deploy stack:
33-
```bash'
34-
docker stack deploy -c cities/amsterdam.yml garages
34+
3. Deploy the stack
35+
```bash
36+
docker stack deploy -c cities/[CITY].yml garages
3537
```
3638

3739
## Contributing

‎__init__.py‎

Lines changed: 19 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -1,100 +1,32 @@
1-
import datetime, aiohttp, asyncio, garages_amsterdam, os, pymysql, time
1+
"""Scrape tool for parking garage into NIPKaart system."""
2+
import cities.amsterdam as amsterdam
23

4+
import asyncio, datetime, time, os
35
from dotenv import load_dotenv
46
from pathlib import Path
57

68
load_dotenv()
79
env_path = Path('.')/'.env'
810
load_dotenv(dotenv_path=env_path)
911

10-
# MYSQL credentials
11-
DB_SERVER = os.getenv("DB_SERVER")
12-
DB_PORT = int(os.getenv("DB_PORT"))
13-
DATABASE = os.getenv("DATABASE")
14-
DB_USERNAME = os.getenv("DB_USERNAME")
15-
DB_PASSWORD = os.getenv("DB_PASSWORD")
16-
1712
CITY = os.getenv("CITY")
1813
WAIT_TIME = int(os.getenv("WAIT_TIME"))
1914

20-
SLEEP_TIME = (60 * WAIT_TIME) # gelijk aan 10 minuten
21-
22-
# Connect to MySQL
23-
connection = pymysql.connect(host=DB_SERVER, port=DB_PORT, user=DB_USERNAME, password=DB_PASSWORD, database=DATABASE)
24-
cursor = connection.cursor()
25-
26-
async def async_get_garages():
27-
"""Get garage data from API."""
28-
async with aiohttp.ClientSession() as client:
29-
return await garages_amsterdam.get_garages(client)
30-
31-
def check_value(value):
32-
"""Check on null values."""
33-
if value == "":
34-
return 0
35-
else:
36-
return value
37-
38-
def purge_database(city):
39-
"""Purge the database tabel."""
40-
print(f'{TIME} - START met leeggooien van de database')
41-
try:
42-
sql = "DELETE FROM `garages_amsterdam` WHERE `city`=%s"
43-
cursor.execute(sql, city)
44-
connection.commit()
45-
except Exception as e:
46-
print(f'MySQL error: {e}')
47-
finally:
48-
print(f'{TIME} - Klaar met leegmaken van de database')
49-
50-
def update_database(data_set):
51-
"""Update the database with new data."""
52-
# purge_database(CITY)
53-
print(f'{TIME} - START bijwerken van database met nieuwe data')
54-
count=1
55-
try:
56-
for item in data_set:
57-
sql = """INSERT INTO `garages_amsterdam` (id, name, city, state, free_space_short, free_space_long, short_capacity, long_capacity, longitude, latitude, visibility, created_at, updated_at)
58-
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON DUPLICATE KEY
59-
UPDATE id=values(id),
60-
name=values(name),
61-
state=values(state),
62-
free_space_short=values(free_space_short),
63-
free_space_long=values(free_space_long),
64-
short_capacity=values(short_capacity),
65-
long_capacity=values(long_capacity),
66-
longitude=values(longitude),
67-
latitude=values(latitude),
68-
updated_at=values(updated_at)"""
69-
val = (count, str(item.garage_name), str(CITY) ,str(item.state), check_value(item.free_space_short), check_value(item.free_space_long), check_value(item.short_capacity), check_value(item.long_capacity), float(item.longitude), float(item.latitude), bool(True), (datetime.datetime.now()), (datetime.datetime.now()))
70-
cursor.execute(sql, val)
71-
count+=1
72-
connection.commit()
73-
74-
except Exception as e:
75-
print(f'MySQL error: {e}')
76-
finally:
77-
print(f'{TIME} - KLAAR met updaten van database')
78-
79-
def test_connection():
80-
"""Test the connection with MySQL Database."""
81-
try:
82-
cursor.execute('SELECT VERSION()')
83-
version = cursor.fetchone()
84-
print(f'Database version: {version[0]}')
85-
except Exception as e:
86-
print(f'MySQL error: {e}')
87-
finally:
88-
cursor.close()
89-
connection.close()
15+
testing = False
9016

9117
if __name__ == '__main__':
92-
print("start scrape program")
93-
# data_set = asyncio.run(async_get_garages())
94-
# test_connection()
95-
while True:
96-
TIME = datetime.datetime.now().strftime('%H:%M:%S')
97-
print(f'----------START----------')
98-
update_database(asyncio.run(async_get_garages()))
99-
print(f'----------DONE----------')
100-
time.sleep(SLEEP_TIME)
18+
print("--- Start scraping program ---")
19+
if testing:
20+
if CITY == "Amsterdam":
21+
data_set = asyncio.run(amsterdam.async_get_garages())
22+
print(data_set)
23+
amsterdam.test_connection()
24+
else:
25+
while True:
26+
TIME = datetime.datetime.now().strftime('%H:%M:%S')
27+
print(f'-------- START-{CITY} ---------')
28+
if CITY == "Amsterdam":
29+
data_set = asyncio.run(amsterdam.async_get_garages())
30+
amsterdam.update_database(data_set, CITY, TIME)
31+
print(f'--------- DONE-{CITY} ---------')
32+
time.sleep(60 * WAIT_TIME)

‎cities/amsterdam.py‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import datetime, aiohttp, garages_amsterdam
2+
3+
from database import connection, cursor
4+
5+
async def async_get_garages():
6+
"""Get garage data from API."""
7+
async with aiohttp.ClientSession() as client:
8+
return await garages_amsterdam.get_garages(client)
9+
10+
def check_value(value):
11+
"""Check on null values."""
12+
if value == "":
13+
return 0
14+
else:
15+
return value
16+
17+
def purge_database(city, time):
18+
"""Purge the database tabel."""
19+
print(f'{time} - START met leeggooien van de database')
20+
try:
21+
sql = "DELETE FROM `parking_garages` WHERE `city`=%s"
22+
cursor.execute(sql, city)
23+
connection.commit()
24+
except Exception as e:
25+
print(f'MySQL error: {e}')
26+
finally:
27+
print(f'{time} - Klaar met leegmaken van de database')
28+
29+
def update_database(data_set, city, time):
30+
"""Update the database with new data."""
31+
# purge_database(city)
32+
print(f'{time} - START bijwerken van database met nieuwe data')
33+
count=1
34+
try:
35+
for item in data_set:
36+
sql = """INSERT INTO `parking_garages` (id, name, city, state, free_space_short, free_space_long, short_capacity, long_capacity, longitude, latitude, visibility, created_at, updated_at)
37+
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s) ON DUPLICATE KEY
38+
UPDATE id=values(id),
39+
name=values(name),
40+
state=values(state),
41+
free_space_short=values(free_space_short),
42+
free_space_long=values(free_space_long),
43+
short_capacity=values(short_capacity),
44+
long_capacity=values(long_capacity),
45+
longitude=values(longitude),
46+
latitude=values(latitude),
47+
updated_at=values(updated_at)"""
48+
val = (count, str(item.garage_name), str(city) ,str(item.state), check_value(item.free_space_short), check_value(item.free_space_long), check_value(item.short_capacity), check_value(item.long_capacity), float(item.longitude), float(item.latitude), bool(True), (datetime.datetime.now()), (datetime.datetime.now()))
49+
cursor.execute(sql, val)
50+
count+=1
51+
connection.commit()
52+
53+
except Exception as e:
54+
print(f'MySQL error: {e}')
55+
finally:
56+
print(f'{time} - KLAAR met updaten van database')
57+
58+
def test_connection():
59+
"""Test the connection with MySQL Database."""
60+
try:
61+
cursor.execute('SELECT VERSION()')
62+
version = cursor.fetchone()
63+
print(f'Database version: {version[0]}')
64+
except Exception as e:
65+
print(f'MySQL error: {e}')
66+
finally:
67+
cursor.close()
68+
connection.close()

‎cities/amsterdam.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ version: "3"
33
services:
44
# Scrape Amsterdam Service
55
nipkaart_amsterdam:
6-
image: garages-amsterdam:latest
6+
image: nipkaart-garages-amsterdam:latest
77
networks:
88
- backend
99
deploy:

‎database/__init__.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
"""Setup database connection."""
2+
import pymysql, os
3+
4+
from dotenv import load_dotenv
5+
from pathlib import Path
6+
7+
load_dotenv()
8+
env_path = Path('.')/'.env'
9+
load_dotenv(dotenv_path=env_path)
10+
11+
# MYSQL credentials
12+
DB_SERVER = os.getenv("DB_SERVER")
13+
DB_PORT = int(os.getenv("DB_PORT"))
14+
DATABASE = os.getenv("DATABASE")
15+
DB_USERNAME = os.getenv("DB_USERNAME")
16+
DB_PASSWORD = os.getenv("DB_PASSWORD")
17+
18+
# Connect to MySQL
19+
connection = pymysql.connect(host=DB_SERVER, port=DB_PORT, user=DB_USERNAME, password=DB_PASSWORD, database=DATABASE)
20+
cursor = connection.cursor()

‎docker-compose.yml‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,12 @@
11
# Docker compose STACK file
22
version: "3"
33
services:
4-
# Scrape Amsterdam Service
5-
scrape_garages_amsterdam:
4+
# Scrape Service
5+
test_garage:
66
build:
77
context: .
88
dockerfile: Dockerfile
9-
container_name: "NIPKaart_garages_${CITY}"
10-
image: garages-amsterdam:latest
9+
container_name: "NIPKaart_${CITY}"
1110
networks:
1211
- backend
1312
restart: on-failure

0 commit comments

Comments
 (0)