|
| 1 | +import os |
| 2 | +import threading |
| 3 | +from queue import Queue |
| 4 | +import time |
| 5 | +import requests |
| 6 | +from lib.configure import getProxyList as PROXYLIST |
| 7 | +from lib.configure import numThreads as THREADCOUNT |
| 8 | +from lib.configure import config |
| 9 | + |
| 10 | +class ProxyHelper(): |
| 11 | + |
| 12 | + def __init__(self): |
| 13 | + self.session = requests.Session() |
| 14 | + self.proxies = PROXYLIST() |
| 15 | + self.numProxies = len(PROXYLIST()) |
| 16 | + self.print_lock = threading.Lock() |
| 17 | + self.queue = Queue() |
| 18 | + self.good = [] |
| 19 | + self.bad = [] |
| 20 | + |
| 21 | + def checkJob(self, proxy): |
| 22 | + #sess = self.setProxy(self.session, proxy) |
| 23 | + proxyDict = { |
| 24 | + 'http:' : proxy, |
| 25 | + 'https:' : proxy, |
| 26 | + 'socks' : proxy |
| 27 | + } |
| 28 | + try: |
| 29 | + r = self.session.get('https://google.com', timeout=4, proxies=proxyDict) |
| 30 | + if r.status_code is 200: |
| 31 | + self.good.append(proxy) |
| 32 | + with self.print_lock: |
| 33 | + print("%s is working..." % proxy) |
| 34 | + else: |
| 35 | + raise Exception("Bad Proxy!") |
| 36 | + except Exception as error: |
| 37 | + self.bad.append(proxy) |
| 38 | + print(error) |
| 39 | + |
| 40 | + |
| 41 | + def threader(self): |
| 42 | + while True: |
| 43 | + item = self.queue.get() |
| 44 | + self.checkJob(item) |
| 45 | + self.queue.task_done() |
| 46 | + |
| 47 | + def setProxy(self, session, proxy): |
| 48 | + if proxy is not None: |
| 49 | + session.proxies.update({ |
| 50 | + 'http:' : proxy, |
| 51 | + 'https:' : proxy, |
| 52 | + 'socks' : proxy |
| 53 | + }) |
| 54 | + return session |
| 55 | + |
| 56 | + def checkProxies(self): |
| 57 | + |
| 58 | + print("Checking and filtering out bad proxies...") |
| 59 | + start = time.time() |
| 60 | + |
| 61 | + print("Starting up threads...") |
| 62 | + for x in range(THREADCOUNT()): |
| 63 | + t = threading.Thread(target = self.threader) |
| 64 | + t.daemon = True |
| 65 | + t.start() |
| 66 | + print("[Thread-%d] has started." % x) |
| 67 | + |
| 68 | + for item in self.proxies: |
| 69 | + self.queue.put(item) |
| 70 | + |
| 71 | + self.queue.join() |
| 72 | + print("Done.") |
| 73 | + |
| 74 | + gp = open('proxy_lists/good_proxies.txt', 'a') |
| 75 | + for p in self.good: |
| 76 | + gp.write("%s\n" % str(p)) |
| 77 | + gp.close() |
| 78 | + |
| 79 | + bp = open('proxy_lists/bad_proxies.txt', 'a') |
| 80 | + for p in self.bad: |
| 81 | + bp.write("%s\n" % str(p)) |
| 82 | + bp.close() |
| 83 | + |
| 84 | + total = str(time.time()-start) |
| 85 | + numBad = len(self.bad) |
| 86 | + print("\nSearched %s proxies and filtered out %s bad proxies in %s seconds" % (self.numProxies, numBad, total)) |
| 87 | + |
| 88 | + path = "proxy_lists/%s" % config["proxy"]["proxyList"] |
| 89 | + os.remove(path) |
| 90 | + os.rename('proxy_lists/good_proxies.txt', path) |
0 commit comments