Storing website dataBase, Start getting results from website with a basic structure of regex data

Author: Sanix-Darker

api/app.py

@@ -4,9 +4,58 @@
 #
 # Ressources / WebSites used to fetch data:
 # See WEBSITE_DATABASE 
-
+# on pending:
+# https://angel.co/europe/python/jobs~a
+# https://remoteml.com/tags/Python/~a
+# https://nodesk.co/remote-jobs/python/~a
+# https://www.codementor.io/freelance-jobs/python~a
+# https://www.indeed.com.ph/Remote-Python-jobs~a
+# https://remote4me.com/remote-python-jobs~a
+# https://www.irishjobs.ie/Remote-Python-Jobs~a
+# https://www.reed.co.uk/jobs/python-developer-remote/36144618~a
+# https://www.simplyhired.com/search?q=remote+python+developer&l=austin%2C+tx&job=ViO7geREaxO513t_xvNXtyVy2t9_kmX9_BjPGMevSnLgkOwH73PMww~a
+# https://remotees.com/remote-python-jobs~a
+# https://www.indeed.co.uk/Python-jobs-in-Remote~a
+# https://jobs.trovit.co.uk/remote-python-jobs~a
+# https://www.glassdoor.co.uk/Job/uk-remote-python-jobs-SRCH_IL.0,2_IN2_KO3,16.htm~a
+# https://www.cybercoders.com/jobs/remote-python-developer-jobs/~a
+# https://www.pearsonfrank.com/32655/uk-remote-python-developer~a
+# https://www.flexjobs.com/jobs/python~a
+# https://twitter.com/remotepython?lang=en~a
+# https://www.reddit.com/r/remotepython/~a
+# https://www.remotelyawesomejobs.com/remote-python-jobs/~a
+# https://europeremotely.com/remote-jobs/Python~a
+# https://www.glassdoor.com/Job/remote-python-jobs-SRCH_IL.0,6_IS11047_KO7,13.htm~a
+# https://www.glassdoor.com/Job/remote-python-jobs-SRCH_KO0,13.htm~a
+# https://www.linkedin.com/jobs/python-remote-jobs~a
+# https://stackoverflow.com/jobs/remote-developer-jobs-using-python~a
+# https://weworkremotely.com/remote-jobs/search?utf8=%E2%9C%93&term=python~a
+# https://www.workingnomads.co/remote-python-jobs~a
+# https://www.python.org/jobs/location/telecommute/~a
+# https://jobs.github.com/positions?description=python~a
+# https://pythonjobs.github.io/~a
+# https://www.python.org/jobs/~a
+# https://www.indeed.com/q-Remote-Python-Developer-jobs.html~a
+# https://www.indeed.com/q-Remote-Python-jobs.html~a
+# https://www.indeed.com/q-Python-jobs.html~a
+# https://www.indeed.com/q-Python-Developer-jobs.html~a
+# https://remoteok.io/remote-python-jobs~a
+# https://www.linkedin.com/jobs/python-jobs~a
+# https://www.linkedin.com/jobs/python-developer-jobs~a
+# https://www.remotepython.com/~a
+# https://www.pythonjobs.com/~a
+# https://www.glassdoor.com/Job/python-developer-jobs-SRCH_KO0,16.htm~a
+# https://www.naukri.com/python-developer-jobs~a
+# https://www.upwork.com/o/jobs/browse/skill/python/~a
+# https://www.seek.co.nz/python-jobs~a
+# https://www.nijobs.com/Python-Jobs-in-Belfast~a
+# https://www.irishjobs.ie/Python-Developer-Jobs-in-Dublin~a
+# https://stackoverflow.com/jobs/developer-jobs-using-python~a
+# https://www.technojobs.co.uk/python-jobs~a
+# https://www.pythonjobshq.com/~a
+import os
 import requests
-import BeautifulSoup
+from bs4 import BeautifulSoup
 import schedule
 import time
 
@@ -24,23 +73,29 @@ def Presentation():
 # -the example of schema to fetch and 
 # -the total of result to be get
 def fetch_fromIT(url, schema, total):
-    soup = BeautifulSoup('\n'.join(requests.get(url).text.splitlines()[1:total]), 'html.parser')
-    return [i.get('href') for i in soup.find_all(schema)]
+    data = requests.get(url).text
+    soup = BeautifulSoup(data, 'html.parser')
+    result = ""
+    for link in soup.find_all(schema.split(',')[0]):
+        result += link.get(schema.split(',')[1])
+        print(link.get(schema.split(',')[1]))
+    return result
 
 def getWebSite_and_schema_and_save_data():
-    result = ""
+    result = []
     # Getting list of links and schemas 
     # and remove whitespace characters like `\n` at the end of each line
     lines = [line.rstrip('\n') for line in open(WEBSITE_DATABASE)]
     # Loop in lines, fetch and get result
     for x in lines:
-        result += fetch_fromIT(x.plit('~')[0], x.plit('~')[1], 15)
+        print "Fetching on "+x.split('~')[0]
+        result += fetch_fromIT(x.split('~')[0], x.split('~')[1], 15)
     return result;
 
 # The method JOB
 def PYTOME_job():
     print "Starting the Job............."    
-    getWebSite_and_schema_and_save_data()
+    print getWebSite_and_schema_and_save_data()
 
 # Other Schedules Options
 # schedule.every(5).to(10).minutes.do(job)
@@ -51,7 +106,6 @@ def PYTOME_job():
 # Let say, automation every day at 00:01
 #schedule.every().day.at("00:01").do(PYTOME_job)
 
-
 Presentation()
 PYTOME_job()
 # while True:

api/data/website.ptm

@@ -1,50 +1,2 @@
-https://github.com/pyjobs/annonces~a
-https://djangogigs.com/gigs/remote/~a
-https://angel.co/europe/python/jobs~a
-https://remoteml.com/tags/Python/~a
-https://nodesk.co/remote-jobs/python/~a
-https://www.codementor.io/freelance-jobs/python~a
-https://www.indeed.com.ph/Remote-Python-jobs~a
-https://remote4me.com/remote-python-jobs~a
-https://www.irishjobs.ie/Remote-Python-Jobs~a
-https://www.reed.co.uk/jobs/python-developer-remote/36144618~a
-https://www.simplyhired.com/search?q=remote+python+developer&l=austin%2C+tx&job=ViO7geREaxO513t_xvNXtyVy2t9_kmX9_BjPGMevSnLgkOwH73PMww~a
-https://remotees.com/remote-python-jobs~a
-https://www.indeed.co.uk/Python-jobs-in-Remote~a
-https://jobs.trovit.co.uk/remote-python-jobs~a
-https://www.glassdoor.co.uk/Job/uk-remote-python-jobs-SRCH_IL.0,2_IN2_KO3,16.htm~a
-https://www.cybercoders.com/jobs/remote-python-developer-jobs/~a
-https://www.pearsonfrank.com/32655/uk-remote-python-developer~a
-https://www.flexjobs.com/jobs/python~a
-https://twitter.com/remotepython?lang=en~a
-https://www.reddit.com/r/remotepython/~a
-https://www.remotelyawesomejobs.com/remote-python-jobs/~a
-https://europeremotely.com/remote-jobs/Python~a
-https://www.glassdoor.com/Job/remote-python-jobs-SRCH_IL.0,6_IS11047_KO7,13.htm~a
-https://www.glassdoor.com/Job/remote-python-jobs-SRCH_KO0,13.htm~a
-https://www.linkedin.com/jobs/python-remote-jobs~a
-https://stackoverflow.com/jobs/remote-developer-jobs-using-python~a
-https://weworkremotely.com/remote-jobs/search?utf8=%E2%9C%93&term=python~a
-https://www.workingnomads.co/remote-python-jobs~a
-https://www.python.org/jobs/location/telecommute/~a
-https://jobs.github.com/positions?description=python~a
-https://pythonjobs.github.io/~a
-https://www.python.org/jobs/~a
-https://www.indeed.com/q-Remote-Python-Developer-jobs.html~a
-https://www.indeed.com/q-Remote-Python-jobs.html~a
-https://www.indeed.com/q-Python-jobs.html~a
-https://www.indeed.com/q-Python-Developer-jobs.html~a
-https://remoteok.io/remote-python-jobs~a
-https://www.linkedin.com/jobs/python-jobs~a
-https://www.linkedin.com/jobs/python-developer-jobs~a
-https://www.remotepython.com/~a
-https://www.pythonjobs.com/~a
-https://www.glassdoor.com/Job/python-developer-jobs-SRCH_KO0,16.htm~a
-https://www.naukri.com/python-developer-jobs~a
-https://www.upwork.com/o/jobs/browse/skill/python/~a
-https://www.seek.co.nz/python-jobs~a
-https://www.nijobs.com/Python-Jobs-in-Belfast~a
-https://www.irishjobs.ie/Python-Developer-Jobs-in-Dublin~a
-https://stackoverflow.com/jobs/developer-jobs-using-python~a
-https://www.technojobs.co.uk/python-jobs~a
-https://www.pythonjobshq.com/~a
+https://github.com/pyjobs/annonces~a,href
+https://djangogigs.com/gigs/remote/~a,href

api/requirements.txt

@@ -1,3 +1,3 @@
 requests==2.13.0
-BeautifulSoup
+BeautifulSoup4
 schedule