Skip to content

Commit 8a21396

Browse files
author
Germey
committed
add env prefix
1 parent 08385f6 commit 8a21396

File tree

4 files changed

+58
-38
lines changed

4 files changed

+58
-38
lines changed

‎proxypool/scheduler.py

Lines changed: 28 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ class Scheduler():
1818
"""
1919
scheduler
2020
"""
21-
21+
2222
def run_tester(self, cycle=CYCLE_TESTER):
2323
"""
2424
run tester
@@ -33,7 +33,7 @@ def run_tester(self, cycle=CYCLE_TESTER):
3333
tester.run()
3434
loop += 1
3535
time.sleep(cycle)
36-
36+
3737
def run_getter(self, cycle=CYCLE_GETTER):
3838
"""
3939
run getter
@@ -48,7 +48,7 @@ def run_getter(self, cycle=CYCLE_GETTER):
4848
getter.run()
4949
loop += 1
5050
time.sleep(cycle)
51-
51+
5252
def run_server(self):
5353
"""
5454
run server for api
@@ -57,42 +57,48 @@ def run_server(self):
5757
logger.info('server not enabled, exit')
5858
return
5959
app.run(host=API_HOST, port=API_PORT, threaded=API_THREADED)
60-
60+
6161
def run(self):
6262
global tester_process, getter_process, server_process
6363
try:
6464
logger.info('starting proxypool...')
6565
if ENABLE_TESTER:
66-
tester_process = multiprocessing.Process(target=self.run_tester)
66+
tester_process = multiprocessing.Process(
67+
target=self.run_tester)
6768
logger.info(f'starting tester, pid {tester_process.pid}...')
6869
tester_process.start()
69-
70+
7071
if ENABLE_GETTER:
71-
getter_process = multiprocessing.Process(target=self.run_getter)
72+
getter_process = multiprocessing.Process(
73+
target=self.run_getter)
7274
logger.info(f'starting getter, pid{getter_process.pid}...')
7375
getter_process.start()
74-
76+
7577
if ENABLE_SERVER:
76-
server_process = multiprocessing.Process(target=self.run_server)
78+
server_process = multiprocessing.Process(
79+
target=self.run_server)
7780
logger.info(f'starting server, pid{server_process.pid}...')
7881
server_process.start()
79-
80-
tester_process.join()
81-
getter_process.join()
82-
server_process.join()
82+
83+
tester_process and tester_process.join()
84+
getter_process and getter_process.join()
85+
server_process and server_process.join()
8386
except KeyboardInterrupt:
8487
logger.info('received keyboard interrupt signal')
85-
tester_process.terminate()
86-
getter_process.terminate()
87-
server_process.terminate()
88+
tester_process and tester_process.terminate()
89+
getter_process and getter_process.terminate()
90+
server_process and server_process.terminate()
8891
finally:
8992
# must call join method before calling is_alive
90-
tester_process.join()
91-
getter_process.join()
92-
server_process.join()
93-
logger.info(f'tester is {"alive" if tester_process.is_alive() else "dead"}')
94-
logger.info(f'getter is {"alive" if getter_process.is_alive() else "dead"}')
95-
logger.info(f'server is {"alive" if server_process.is_alive() else "dead"}')
93+
tester_process and tester_process.join()
94+
getter_process and getter_process.join()
95+
server_process and server_process.join()
96+
logger.info(
97+
f'tester is {"alive" if tester_process.is_alive() else "dead"}')
98+
logger.info(
99+
f'getter is {"alive" if getter_process.is_alive() else "dead"}')
100+
logger.info(
101+
f'server is {"alive" if server_process.is_alive() else "dead"}')
96102
logger.info('proxy terminated')
97103

98104

‎proxypool/setting.py

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,18 +24,19 @@
2424
APP_TEST = IS_TEST = APP_ENV == TEST_MODE
2525

2626
# redis host
27-
REDIS_HOST = env.str('REDIS_HOST', '127.0.0.1')
27+
REDIS_HOST = env.str('PROXYPOOL_REDIS_HOST',
28+
env.str('REDIS_HOST', '127.0.0.1'))
2829
# redis port
29-
REDIS_PORT = env.int('REDIS_PORT', 6379)
30+
REDIS_PORT = env.int('PROXYPOOL_REDIS_PORT', env.int('REDIS_PORT', 6379))
3031
# redis password, if no password, set it to None
31-
REDIS_PASSWORD = env.str('REDIS_PASSWORD', None)
32+
REDIS_PASSWORD = env.str('PROXYPOOL_REDIS_PASSWORD',
33+
env.str('REDIS_PASSWORD', None))
3234
# redis db, if no choice, set it to 0
33-
REDIS_DB = env.int('REDIS_DB', 0)
34-
# redis connection string, like redis://[password]@host:port or rediss://[password]@host:port/0
35-
REDIS_CONNECTION_STRING = env.str('REDIS_CONNECTION_STRING', None)
36-
37-
if REDIS_CONNECTION_STRING:
38-
REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB = parse_redis_connection_string(REDIS_CONNECTION_STRING)
35+
REDIS_DB = env.int('PROXYPOOL_REDIS_DB', env.int('REDIS_DB', 0))
36+
# redis connection string, like redis://[password]@host:port or rediss://[password]@host:port/0,
37+
# please refer to https://redis-py.readthedocs.io/en/stable/connections.html#redis.client.Redis.from_url
38+
REDIS_CONNECTION_STRING = env.str(
39+
'PROXYPOOL_REDIS_CONNECTION_STRING', env.str('REDIS_CONNECTION_STRING', None))
3940

4041
# redis hash table key name
4142
REDIS_KEY = env.str('REDIS_KEY', 'proxies:universal')
@@ -78,4 +79,3 @@
7879

7980
# logger.add(env.str('LOG_RUNTIME_FILE', join(LOG_DIR, 'runtime.log')), level='DEBUG', rotation='1 week', retention='20 days')
8081
# logger.add(env.str('LOG_ERROR_FILE', join(LOG_DIR, 'error.log')), level='ERROR', rotation='1 week')
81-

‎proxypool/storages/redis.py

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import redis
22
from proxypool.exceptions import PoolEmptyException
33
from proxypool.schemas.proxy import Proxy
4-
from proxypool.setting import REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \
4+
from proxypool.setting import REDIS_CONNECTION_STRING, REDIS_HOST, REDIS_PORT, REDIS_PASSWORD, REDIS_DB, REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MIN, \
55
PROXY_SCORE_INIT
66
from random import choice
77
from typing import List
@@ -18,14 +18,21 @@ class RedisClient(object):
1818
redis connection client of proxypool
1919
"""
2020

21-
def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=REDIS_DB, **kwargs):
21+
def __init__(self, host=REDIS_HOST, port=REDIS_PORT, password=REDIS_PASSWORD, db=REDIS_DB,
22+
connection_string=REDIS_CONNECTION_STRING, **kwargs):
2223
"""
2324
init redis client
2425
:param host: redis host
2526
:param port: redis port
2627
:param password: redis password
28+
:param connection_string: redis connection_string
2729
"""
28-
self.db = redis.StrictRedis(host=host, port=port, password=password, db=db, decode_responses=True, **kwargs)
30+
# if set connection_string, just use it
31+
if connection_string:
32+
self.db = redis.StrictRedis.from_url(connection_string)
33+
else:
34+
self.db = redis.StrictRedis(
35+
host=host, port=port, password=password, db=db, decode_responses=True, **kwargs)
2936

3037
def add(self, proxy: Proxy, score=PROXY_SCORE_INIT) -> int:
3138
"""
@@ -51,11 +58,13 @@ def random(self) -> Proxy:
5158
:return: proxy, like 8.8.8.8:8
5259
"""
5360
# try to get proxy with max score
54-
proxies = self.db.zrangebyscore(REDIS_KEY, PROXY_SCORE_MAX , PROXY_SCORE_MAX)
61+
proxies = self.db.zrangebyscore(
62+
REDIS_KEY, PROXY_SCORE_MAX, PROXY_SCORE_MAX)
5563
if len(proxies):
5664
return convert_proxy_or_proxies(choice(proxies))
5765
# else get proxy by rank
58-
proxies = self.db.zrevrange(REDIS_KEY, PROXY_SCORE_MIN , PROXY_SCORE_MAX)
66+
proxies = self.db.zrevrange(
67+
REDIS_KEY, PROXY_SCORE_MIN, PROXY_SCORE_MAX)
5968
if len(proxies):
6069
return convert_proxy_or_proxies(choice(proxies))
6170
# else raise error
@@ -125,4 +134,3 @@ def batch(self, cursor, count) -> List[Proxy]:
125134
conn = RedisClient()
126135
result = conn.random()
127136
print(result)
128-

‎proxypool/utils/proxy.py

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22

33

44
def is_valid_proxy(data):
5+
"""
6+
check this string is within proxy format
7+
"""
58
if data.__contains__(':'):
69
ip = data.split(':')[0]
710
port = data.split(':')[1]
@@ -11,6 +14,9 @@ def is_valid_proxy(data):
1114

1215

1316
def is_ip_valid(ip):
17+
"""
18+
check this string is within ip format
19+
"""
1420
a = ip.split('.')
1521
if len(a) != 4:
1622
return False

0 commit comments

Comments
 (0)