Refactor code structure

Also updated README.md

Author: 2knal

README.md

@@ -1,2 +1,46 @@
+<div align="center">
+  <img src="https://cdn.auth0.com/blog/wpad/logo.png" width=100px height=100px>
+
+ <br>
+ <br>
+
+ [![](https://img.shields.io/badge/Made_with-requests-blue?style=for-the-badge)](https://requests.readthedocs.io/en/master/ "Requests")
+</div>
+
 # random_proxies
-Python package to generate random proxy on the fly!
+
+### Python package to generate a random proxy on the fly!
+
+## Features
+
+- Supports `HTTP`, `HTTPS` or `SOCKS` proxy.
+> Currently support SOCKS over HTTP only with version 4.
+- Fetch specific country proxy by using country name or country code.
+- Fetch elite / transparent / anonymous proxies respectively.
+- Fetch directly from [free-proxy-list](https://free-proxy-list.net).
+- For better response time, fetch from an elasticsearch `cache_server`.
+- `cache_server` is updated via routines described [here](./random_proxies/cache_server/README.md)
+
+
+## Example usage
+
+```bash
+  git clone https://github.com/2knal/random_proxies.git`
+  cd random_proxies/ 
+  pip install -r requirements.txt 
+```
+Open python interpreter. (Supports version 3.4+)
+
+```python
+>>> from random_proxies import random_proxy
+>>> random_proxy()
+'23.101.2.247:81'
+```
+
+Refer more examples [here](./examples/)
+
+## TODO
+- [ ] Publish package version 0.0.1 
+- [ ] Scrape proxies from other sources
+- [ ] Add support for SOCKS version 5
+- [ ] Add unit tests

examples/example.py

@@ -0,0 +1,46 @@
+'''
+(Once the package is published)
+pip install random_proxies 
+    or 
+Follow example usage to import the package
+'''
+
+from random_proxies import random_proxy
+
+'''
+Fetch directly from free-proxy-list.net
+
+:param use_cache: Boolean value to fetch from cache_server or not (default: True)
+:return: returns `ip:port`
+:rtype: str
+'''
+random_proxy(use_cache=False)
+
+'''
+Fetch by protocol name.
+
+:param protocol: Protocol name, either http, https or socks (default: http)
+:return: returns `ip:port`
+:rtype: str
+'''
+random_proxy(protocol='https')
+
+'''
+Fetch by country name or code.
+
+:param country: Country name (default: None)
+:param code: Country code, should match with country name if added (default: None)
+:return: returns `ip:port`
+:rtype: str
+'''
+random_proxy(country='india')
+random_proxy(code='in')
+
+'''
+Fetch by proxy standard.
+
+:param standard: Proxy standard, either anonymous, elite proxy, transparent (default: anonymous)
+:return: returns `ip:port`
+:rtype: str
+'''
+random_proxy(standard='elite proxy')

proxies/random_proxies/db.py

@@ -1,4 +1,4 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, unicode_literals
 
-from proxies.random_proxies import random_proxy
+from random_proxies.proxies import random_proxy

proxies/__init__.py random_proxies/__init__.pyproxies/__init__.py renamed to random_proxies/__init__.py

@@ -5,10 +5,10 @@
 
 from time import time
 
-from proxies.cache_server.config import es
-from proxies.cache_server.config import is_good_proxy
-from proxies.cache_server.config import logger
-from proxies.cache_server.utils import add
+from random_proxies.cache_server.config import es
+from random_proxies.cache_server.config import is_good_proxy
+from random_proxies.cache_server.config import logger
+from random_proxies.cache_server.utils import add
 
 def _clean():
     # Get all the proxies from proxies index
@@ -18,13 +18,14 @@ def _clean():
     # Delete those which arent good
     for proxy in proxies:
         ip = proxy['ip address'] + ':' + proxy['port']
+        protocol = ('http', 'https')[proxy['https'] == 'yes']
+        
         # Implies SOCKS proxy
         if 'version' in proxy:
             ip = proxy['version'] + '://' + ip
-        protocol = ('http', 'https')[proxy['https'] == 'yes']
-
+            protocol = 'http'
         try:
-            # Only if it works
+            # If it doesn't work
             if not is_good_proxy(ip, protocol=protocol):
                 # Delete from proxies index
                 es.delete(index='proxies', doc_type='proxy', id=ip)

proxies/cache_server/README.md random_proxies/cache_server/README.mdproxies/cache_server/README.md renamed to random_proxies/cache_server/README.md

@@ -7,7 +7,6 @@
 from elasticsearch import Elasticsearch
 
 env_path = join(dirname(__file__), '.env')
-# print('ENV PATH:', env_path)
 load_dotenv(env_path)
 
 # GLOBALS
@@ -27,7 +26,8 @@
 if not es.indices.exists(index='recents'):
     es.indices.create(index='recents', ignore=400)
 
-from proxies.random_proxies.settings import BASE_URL, SSL_URL, SOCKS_URL
-from proxies.random_proxies.log import logger
-from proxies.random_proxies.utils import fetch, parse_response
-from proxies.random_proxies.proxy_health import is_good_proxy
+# Removing circular import
+from random_proxies.proxies.settings import BASE_URL, SSL_URL, SOCKS_URL
+from random_proxies.proxies.log import logger
+from random_proxies.proxies.utils import fetch, parse_response
+from random_proxies.proxies.proxy_health import is_good_proxy

proxies/cache_server/__init__.py random_proxies/cache_server/__init__.pyproxies/cache_server/__init__.py renamed to random_proxies/cache_server/__init__.py

@@ -5,30 +5,26 @@
 
 from time import time
 
-from proxies.cache_server.config import es
-from proxies.cache_server.config import fetch, parse_response
-from proxies.cache_server.config import is_good_proxy
-from proxies.cache_server.config import logger
-from proxies.cache_server.utils import add
-from proxies.cache_server.config import BASE_URL, SSL_URL, SOCKS_URL    
+from random_proxies.cache_server.config import es
+from random_proxies.cache_server.config import fetch, parse_response
+from random_proxies.cache_server.config import is_good_proxy
+from random_proxies.cache_server.config import logger
+from random_proxies.cache_server.utils import add
+from random_proxies.cache_server.config import BASE_URL, SSL_URL, SOCKS_URL    
 
 def _check():
     urls = [BASE_URL, SSL_URL, SOCKS_URL]
     proxies = []
+
     # Fetch all the proxies from these urls
     for url in urls:
-        print('URL:', url)
         res = fetch(url)
         # Passing empty conditions so that
         proxies.extend(parse_response(res, {}))
 
-    print('Total proxies: ', len(proxies))
     # Check if they work
     working_proxies = []
-    count = 0
     for proxy in proxies:
-        count += 1
-        print('Proxy count:', count)
         ip = proxy['ip address'] + ':' + proxy['port']
         protocol = ('http', 'https')[proxy['https'] == 'yes']