Skip to content

Commit cc171a8

Browse files
MaxymVlasovMaxymVlasov
committed
Move configs to separate file
1 parent 67b6a44 commit cc171a8

File tree

3 files changed

+131
-124
lines changed

3 files changed

+131
-124
lines changed

‎data-transformer-app/Dockerfile‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,14 @@ LABEL maintainer="MaxymVlasov"
44

55
WORKDIR /app
66

7-
COPY main.py requirements-dev.txt ./
8-
7+
COPY requirements-dev.txt .
98
ARG ENV="${ENV:-prod}"
109
RUN if [ "$ENV" != 'prod' ]; then \
1110
pip3 --no-cache-dir install -r requirements-dev.txt; \
1211
fi
1312

13+
COPY main.py configs.py ./
14+
1415
VOLUME [ "/app/data/" ]
1516

1617
CMD ["python3", "-u", "main.py"]

‎data-transformer-app/configs.py‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
"""data-transformer-app configs."""
2+
from types import MappingProxyType
3+
4+
AQI: 'MappingProxyType[str, dict]' = MappingProxyType({
5+
# Key == `pm25_high`
6+
'pm25': {
7+
'12.0': {
8+
'aqi_high': 50,
9+
'aqi_low': 0,
10+
'pollutant_high': 12.0,
11+
'pollutant_low': 0.0,
12+
},
13+
'35.4': {
14+
'aqi_high': 100,
15+
'aqi_low': 51,
16+
'pollutant_high': 35.4,
17+
'pollutant_low': 12.1,
18+
},
19+
'55.4': {
20+
'aqi_high': 150,
21+
'aqi_low': 101,
22+
'pollutant_high': 55.4,
23+
'pollutant_low': 35.5,
24+
},
25+
'150.4': {
26+
'aqi_high': 200,
27+
'aqi_low': 151,
28+
'pollutant_high': 150.4,
29+
'pollutant_low': 55.5,
30+
},
31+
'250.4': {
32+
'aqi_high': 300,
33+
'aqi_low': 201,
34+
'pollutant_high': 250.4,
35+
'pollutant_low': 150.5,
36+
},
37+
'350.4': {
38+
'aqi_high': 400,
39+
'aqi_low': 301,
40+
'pollutant_high': 350.4,
41+
'pollutant_low': 250.5,
42+
},
43+
'500.4': {
44+
'aqi_high': 500,
45+
'aqi_low': 401,
46+
'pollutant_high': 500.4,
47+
'pollutant_low': 350.5,
48+
},
49+
},
50+
'pm10': {
51+
'54': {
52+
'aqi_high': 50,
53+
'aqi_low': 0,
54+
'pollutant_high': 54,
55+
'pollutant_low': 0,
56+
},
57+
'154': {
58+
'aqi_high': 100,
59+
'aqi_low': 51,
60+
'pollutant_high': 154,
61+
'pollutant_low': 55,
62+
},
63+
'254': {
64+
'aqi_high': 150,
65+
'aqi_low': 101,
66+
'pollutant_high': 254,
67+
'pollutant_low': 155,
68+
},
69+
'354': {
70+
'aqi_high': 200,
71+
'aqi_low': 151,
72+
'pollutant_high': 354,
73+
'pollutant_low': 255,
74+
},
75+
'424': {
76+
'aqi_high': 300,
77+
'aqi_low': 201,
78+
'pollutant_high': 424,
79+
'pollutant_low': 355,
80+
},
81+
'504': {
82+
'aqi_high': 301,
83+
'aqi_low': 400,
84+
'pollutant_high': 504,
85+
'pollutant_low': 425,
86+
},
87+
'604': {
88+
'aqi_high': 500,
89+
'aqi_low': 401,
90+
'pollutant_high': 604,
91+
'pollutant_low': 505,
92+
},
93+
},
94+
})
95+
96+
# DataFrame size, number of rows proceeded by one iteration.
97+
# More - high memore usage, low - too long.
98+
CHUNKSIZE = 10 ** 8
99+
100+
# sensors name from 'phenomenon' colum
101+
# sensors value - for user readability
102+
# !!! WARNING !!!: Don't use spaces and commas
103+
SENSORS = MappingProxyType({
104+
'pm10': 'PM10_mcg/m³',
105+
'pm25': 'PM2.5_mcg/m³',
106+
'heca_humidity': 'HECA_relative_humidity_%',
107+
'humidity': 'Relative_humidity_%',
108+
'min_micro': 'min_micro',
109+
'max_micro': 'max_micro',
110+
'pressure': 'Atmospheric_pressure_mmHg',
111+
'signal': 'Wi-Fi_signal_dBm',
112+
'temperature': 'Temperature_C°',
113+
'heca_temperature': 'HECA_temperature_C°',
114+
})
115+
116+
# Path to CSV-file(s) dir from SaveEcoBot
117+
PATH = 'data/original_data'

‎data-transformer-app/main.py‎

Lines changed: 11 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,12 @@
2020
import time
2121
from datetime import datetime
2222
from os import listdir
23-
from typing import Dict
2423

2524
import pandas as pd
25+
from configs import AQI
26+
from configs import CHUNKSIZE
27+
from configs import PATH
28+
from configs import SENSORS
2629

2730
try:
2831
from typeguard import typechecked # noqa: WPS433
@@ -121,121 +124,7 @@ def find_csv_filenames(path_to_dir: str, suffix: str = '.csv'):
121124
@typechecked
122125
def main() -> None:
123126
"""Logic."""
124-
aqi: Dict[str, dict] = {
125-
# Key == `pm25_high`
126-
'pm25': {
127-
'12.0': {
128-
'aqi_high': 50,
129-
'aqi_low': 0,
130-
'pollutant_high': 12.0,
131-
'pollutant_low': 0.0,
132-
},
133-
'35.4': {
134-
'aqi_high': 100,
135-
'aqi_low': 51,
136-
'pollutant_high': 35.4,
137-
'pollutant_low': 12.1,
138-
},
139-
'55.4': {
140-
'aqi_high': 150,
141-
'aqi_low': 101,
142-
'pollutant_high': 55.4,
143-
'pollutant_low': 35.5,
144-
},
145-
'150.4': {
146-
'aqi_high': 200,
147-
'aqi_low': 151,
148-
'pollutant_high': 150.4,
149-
'pollutant_low': 55.5,
150-
},
151-
'250.4': {
152-
'aqi_high': 300,
153-
'aqi_low': 201,
154-
'pollutant_high': 250.4,
155-
'pollutant_low': 150.5,
156-
},
157-
'350.4': {
158-
'aqi_high': 400,
159-
'aqi_low': 301,
160-
'pollutant_high': 350.4,
161-
'pollutant_low': 250.5,
162-
},
163-
'500.4': {
164-
'aqi_high': 500,
165-
'aqi_low': 401,
166-
'pollutant_high': 500.4,
167-
'pollutant_low': 350.5,
168-
},
169-
},
170-
'pm10': {
171-
'54': {
172-
'aqi_high': 50,
173-
'aqi_low': 0,
174-
'pollutant_high': 54,
175-
'pollutant_low': 0,
176-
},
177-
'154': {
178-
'aqi_high': 100,
179-
'aqi_low': 51,
180-
'pollutant_high': 154,
181-
'pollutant_low': 55,
182-
},
183-
'254': {
184-
'aqi_high': 150,
185-
'aqi_low': 101,
186-
'pollutant_high': 254,
187-
'pollutant_low': 155,
188-
},
189-
'354': {
190-
'aqi_high': 200,
191-
'aqi_low': 151,
192-
'pollutant_high': 354,
193-
'pollutant_low': 255,
194-
},
195-
'424': {
196-
'aqi_high': 300,
197-
'aqi_low': 201,
198-
'pollutant_high': 424,
199-
'pollutant_low': 355,
200-
},
201-
'504': {
202-
'aqi_high': 301,
203-
'aqi_low': 400,
204-
'pollutant_high': 504,
205-
'pollutant_low': 425,
206-
},
207-
'604': {
208-
'aqi_high': 500,
209-
'aqi_low': 401,
210-
'pollutant_high': 604,
211-
'pollutant_low': 505,
212-
},
213-
},
214-
}
215-
216-
# DataFrame size, number of rows proceeded by one iteration.
217-
# More - high memore usage, low - too long.
218-
chunk_size = 10 ** 8
219-
220-
# sensors name from 'phenomenon' colum
221-
# sensors value - for user readability
222-
# !!! WARNING !!!: Don't use spaces and commas
223-
sensors = {
224-
'pm10': 'PM10_mcg/m³',
225-
'pm25': 'PM2.5_mcg/m³',
226-
'heca_humidity': 'HECA_relative_humidity_%',
227-
'humidity': 'Relative_humidity_%',
228-
'min_micro': 'min_micro',
229-
'max_micro': 'max_micro',
230-
'pressure': 'Atmospheric_pressure_mmHg',
231-
'signal': 'Wi-Fi_signal_dBm',
232-
'temperature': 'Temperature_C°',
233-
'heca_temperature': 'HECA_temperature_C°',
234-
}
235-
236-
# SaveEcoBot CSV file
237-
path = 'data/original_data'
238-
files = find_csv_filenames(path)
127+
files = find_csv_filenames(PATH)
239128

240129
if not files:
241130
logger.error( # pylint: disable=logging-not-lazy
@@ -248,7 +137,7 @@ def main() -> None:
248137

249138
for filename in files:
250139

251-
for sensor, human_readable_sensor_name in sensors.items():
140+
for sensor, human_readable_sensor_name in SENSORS.items():
252141
sensor_file = f'{filename}-{sensor}'
253142
logger.info(
254143
f'\n{time.strftime("%H:%M:%S")} - ' +
@@ -263,13 +152,13 @@ def main() -> None:
263152
open(f'data/csv/{sensor_file}.csv', 'w').close() # noqa: WPS515
264153

265154
pandas_csv = pd.read_csv(
266-
f'{path}/{filename}',
267-
chunksize=chunk_size,
155+
f'{PATH}/{filename}',
156+
chunksize=CHUNKSIZE,
268157
delimiter=',',
269158
dtype=str,
270159
)
271160
for chunk in pandas_csv:
272-
logger.info(f'{time.strftime("%H:%M:%S")} ----- Proccess chunk rows: {chunk_size}')
161+
logger.info(f'{time.strftime("%H:%M:%S")} ----- Proccess chunk rows: {CHUNKSIZE}')
273162
process(chunk, sensor_file, sensor)
274163

275164
# Save uniq rows
@@ -303,7 +192,7 @@ def main() -> None:
303192
date = row[1]
304193
concentration = round(float(row[2]), 1)
305194

306-
if sensor not in aqi:
195+
if sensor not in AQI:
307196
write_influx_data(
308197
sensor_file,
309198
human_readable_sensor_name,
@@ -317,7 +206,7 @@ def main() -> None:
317206
# CALCULATING THE AQI
318207
#
319208

320-
for upper_bound, _ in aqi[sensor].items():
209+
for upper_bound, _ in AQI[sensor].items():
321210
if concentration < float(upper_bound):
322211
aqi_value = (
323212
(_['aqi_high'] - _['aqi_low'])

0 commit comments

Comments
 (0)