final_version

Author: Sharatcg7

Procfile

@@ -0,0 +1 @@
+web: sh run.sh

README.md

@@ -0,0 +1,21 @@
+# Proxy Aggregator API
+
+
+## install
+
+```
+pip install -r requirements.txt
+```
+
+## run
+```
+sh run.sh  
+python app.py
+
+```
+
+then go to http://localhost:5000/ for the home page of the application
+
+## Authors
+
+* **Sharat Gujamagadi** - *552567* - 

Server.py

@@ -0,0 +1,195 @@
+import datetime
+
+import requests
+from bs4 import BeautifulSoup
+from sqlalchemy import create_engine
+import urllib.request
+import socket
+import urllib.error
+import time
+import json
+
+
+e = create_engine('sqlite:///database.db')
+
+#function to retrive proxy lists
+def get_proxy_list(url):
+    proxieslist = []
+    if 'rapidapi' in url:
+
+        querystring = {"limit": "150", "type": "HTTPS"}
+
+        headers = {
+            'x-rapidapi-host': "proxypage1.p.rapidapi.com",
+            'x-rapidapi-key': "6c57d35416msh577a78de53cc96ap18190ajsnaaa53be816e6",
+            'content-type': "application/x-www-form-urlencoded"
+        }
+
+        response = requests.get(url, headers=headers, params=querystring)
+        json_data = json.loads(response.text)
+        for data in json_data:
+            dict_line = {'IP': str(data['ip']), 'Port': str(data['port'])}
+            proxieslist.append(dict_line)
+
+    else:
+        r = requests.get(url)
+        proxy_response = BeautifulSoup(r.content, 'lxml')
+        table = proxy_response.find('table')
+        rows = table.find_all('tr')
+        for row in rows:
+            ip = row.contents[0].text
+            port = row.contents[1].text
+            secureconn = row.contents[6].text
+
+            if (secureconn == 'yes'):
+                dict_line = {'IP': ip, 'Port': port}
+                proxieslist.append(dict_line)
+
+    return proxieslist
+
+def is_bad_proxy(pip, testurl):
+    try:
+        proxy_handler = urllib.request.ProxyHandler({'http': pip})
+        opener = urllib.request.build_opener(proxy_handler)
+        opener.addheaders = [('User-agent', 'Mozilla/5.0')]
+        urllib.request.install_opener(opener)
+        req = urllib.request.Request(testurl)
+        sock = urllib.request.urlopen(req)
+    except urllib.error.HTTPError as e:
+        print('Error code: ', e.code)
+        return e.code
+    except Exception as detail:
+        print("ERROR:", detail)
+        return True
+    return False
+
+
+def basicproxytest(proxies, date):
+    conn = e.connect()
+    for proxy in proxies:
+        line = 'https://'+ proxy['IP'] + ':' + proxy['Port']
+        proxies = {'http': line, 'https': line}
+        print(proxies)
+        try:
+            testIP = requests.get('https://httpbin.org/ip', proxies = proxies, timeout = 0.5)
+            resposeIP = testIP.json()['origin']
+            origin = resposeIP.split(',')
+            if (origin[0] == proxy['IP']):
+                print(proxy['IP'])
+                conn.execute("UPDATE proxylists SET basictest = ? WHERE IP = ?",
+                             (date, proxy['IP']))
+        except:
+            print('Bad Proxies')
+
+    if (conn):
+        conn.close()
+        print("The SQLite connection is closed")
+
+def proxiestodb(proxies, date):
+    conn = e.connect()
+    for proxy in proxies:
+        try:
+            conn.execute("insert into proxylists (IP, Port, insertdate, lastupdate) values (?, ?, ?, ?)",
+                                 (proxy['IP'], proxy['Port'], date, date))
+            print('Data has been stored successfully')
+        except:
+            conn.execute("UPDATE proxylists SET lastupdate = ? WHERE IP = ?",
+                                 (date, proxy['IP']))
+            print('Lastupdate has been stored sucessfully')
+
+    if (conn):
+        conn.close()
+        print("The SQLite connection is closed")
+
+def updateprovidertable(url, date, records):
+    records = len(records)
+    conn = e.connect()
+    try:
+        conn.execute("UPDATE proxyprovider SET  lastupdate = ?, recordsfound = ? WHERE baseurl = ?",
+                     (date, records, url))
+        msg = 'Data has been updated successfully'
+        print(msg)
+    except:
+        msg = 'Error while updating provider table !'
+        print(msg)
+
+    if (conn):
+        conn.close()
+        print("The SQLite connection is closed")
+
+
+def functesturl(proxies, url):
+    testurl = 'https://'+ url
+    conn = e.connect()
+    goodproxy = []
+    badproxy = []
+    date = str(datetime.datetime.now())
+    dataformat = date[0:10] + ',' + date[11:19]
+
+    socket.setdefaulttimeout(120)
+    for currentproxy in proxies:
+        proxy_IP = currentproxy.split(':')
+
+        if is_bad_proxy(currentproxy, testurl):
+            print('not working. . . .')
+            badproxy.append(currentproxy)
+        else:
+            print('working . . .')
+            conn.execute("UPDATE proxylists SET urltest = ? WHERE IP = ?",
+                         (dataformat, proxy_IP[0]))
+            goodproxy.append(currentproxy)
+
+    recordsfound = len(goodproxy)
+    try:
+        conn.execute("insert into proxytest (testurl, successrecords, testdate) values (?, ?, ?)",
+                     (testurl, recordsfound, dataformat ))
+    except:
+        conn.execute("UPDATE proxytest SET successrecords = ?, testdate = ? WHERE testurl = ?",
+                     (recordsfound, dataformat, testurl))
+
+    if (conn):
+        conn.close()
+        print('completed . . ')
+        print("The SQLite connection is closed")
+
+
+def updatedb(jsondata):
+    proxylist = []
+    date = str(datetime.datetime.now())
+    dataformat = date[0:10] + ',' + date[11:19]
+    proxylist.append(jsondata)
+
+    proxiestodb(proxylist, dataformat)
+    print('Proxies --> Database completed ')
+
+    basicproxytest(proxylist, dataformat)
+    print('proxies --> basicproxytest completed')
+
+def backendserverupdate():
+
+    urls = ["https://www.sslproxies.org/", "https://free-proxy-list.net/", "https://proxypage1.p.rapidapi.com/v1/tier1"]
+
+    for url in urls:
+        print(url)
+
+        proxies = get_proxy_list(url)
+        print(proxies)
+
+        date = str(datetime.datetime.now())
+        dataformat = date[0:10] + ',' + date[11:19]
+
+        updateprovidertable(url, dataformat, proxies)
+
+        proxiestodb(proxies, dataformat)
+        print('Proxies --> Database completed ')
+
+        basicproxytest(proxies, dataformat)
+        print('proxies --> basicproxytest completed')
+
+if __name__ == '__main__':
+    while True:
+        backendserverupdate()
+        time.sleep(600)
+        print('Updating the server...')
+
+

app.py

@@ -0,0 +1,126 @@
+import datetime
+
+from flask import Flask, render_template,jsonify, request
+from flask_restful import Resource, Api
+from sqlalchemy import create_engine
+from Server import functesturl, updatedb, basicproxytest
+
+e = create_engine('sqlite:///database.db')  # loads db into memory
+
+
+# Resource
+app = Flask(__name__, template_folder="templates")
+api = Api(app)  # api is a collection of objects, where each object contains a specific functionality (GET, POST, etc)
+
+
+@app.route('/')
+def home():
+    """
+    This function just responds to the browser ULR
+    localhost:5000/
+
+    :return:        the rendered template 'home.html'
+    """
+    return render_template('home.html')
+
+@app.route('/proxies')
+def proxies():
+    return render_template('proxylistview.html')
+
+class index(Resource):
+    def get(self):
+        conn = e.connect()
+        try:
+            query = conn.execute("select * from proxyprovider")
+            rows = query.cursor.fetchall()
+            query_testurl = conn.execute("select * from proxytest")
+            rows_testurl = query_testurl.cursor.fetchall()
+            return jsonify({'tasks': rows, 'testtasks': rows_testurl})
+        except:
+             return {'msg': 'Error occurred ! '}, 400
+        conn.close()
+
+    def post(self):
+        json = request.get_json()
+        urlTest = json['url']
+        conn = e.connect()
+
+        query = conn.execute("select * from proxylists")
+        rows = query.cursor.fetchall()
+        if not rows:
+            return {'message': 'No Proxies Are Available in the Database !'}, 400
+        proxyList = []
+        for i in rows:
+            line = i[0]+':'+ str(i[1])
+            proxyList.append(line)
+        conn.close()
+        functesturl(proxyList, urlTest)
+        return { 'message': 'Proxy test with certain url is completed, find the details in Test URLS Division!' }, 201
+
+class proxiesretrive(Resource):
+    def get(self):
+        conn = e.connect()
+        try:
+            query = conn.execute("select * from proxylists")
+            rows = query.cursor.fetchall()
+            return jsonify({'tasks': rows})
+        except:
+            return {'msg': 'Error occurred ! '}, 400
+
+        conn.close()
+
+    def post(self):
+        json_data = request.get_json()
+        if not json_data:
+            return {'message': 'No input data provided'}, 400
+        conn = e.connect()
+        query = conn.execute("select * from proxylists where IP = ? ", (json_data['IP']))
+        row = query.cursor.fetchall()
+        if not row:
+            updatedb(json_data)
+            return { 'message': 'IP Entry Has Been Added Successfully ' }, 201
+        else:
+            return { 'message': 'IP is already exists, please enter new IP address !' }, 400
+
+        conn.close()
+
+    def put(self):
+        json_data = request.get_json()
+        proxy_list = []
+        if not json_data:
+            return {'message': 'No input data provided'}, 400
+        conn = e.connect()
+        query = conn.execute("select * from proxylists where IP = ? ", (json_data['oldIP']))
+        row = query.cursor.fetchall()
+        if not row:
+            return {'message': 'IP does not exist'}, 400
+        else:
+            date = str(datetime.datetime.now())
+            dataformat = date[0:10] + ',' + date[11:19]
+            conn.execute("UPDATE proxylists SET IP = ?, Port = ?, insertdate = ?, lastupdate = ? WHERE IP = ?",
+                         (json_data['newIP'],json_data['newPort'], dataformat, dataformat, json_data['oldIP']))
+            dict_line = {'IP': json_data['newIP'], 'Port': json_data['newPort']}
+            proxy_list.append(dict_line)
+            basicproxytest(proxy_list, dataformat)
+            return {'message': 'IP Entry Has Been Updated Successfully '}, 201
+
+        conn.close()
+
+    def delete(self):
+        json_data = request.get_json(force=True)
+        if json_data == 1:
+            print('deleted')
+            conn = e.connect()
+            conn.execute("DELETE from proxylists")
+            conn.close()
+            return {"status": 'success'}, 201
+        else:
+            return {'message': 'No input data provided'}, 400
+
+
+#resource route
+api.add_resource(proxiesretrive, '/proxiestest')
+api.add_resource(index, '/indexpage')
+
+if __name__ == '__main__':
+    app.run(debug=True)