diff --git a/docs/requirements.txt b/docs/requirements.txt index 0b62f6e..5d2b061 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,3 +2,5 @@ BeautifulSoup4 requests urllib3 urwid +search_engine_parser +fake_useragent diff --git a/rebound/rebound.py b/rebound/rebound.py index b39a347..ba58a5c 100644 --- a/rebound/rebound.py +++ b/rebound/rebound.py @@ -13,9 +13,10 @@ from subprocess import PIPE, Popen from threading import Thread import webbrowser -import time from urwid.widget import (BOX, FLOW, FIXED) -import random +from search_engine_parser.core.engines.google import Search as GoogleSearch +from fake_useragent import UserAgent + SO_URL = "https://stackoverflow.com" @@ -41,39 +42,39 @@ SCROLLBAR_LEFT = "left" SCROLLBAR_RIGHT = "right" -USER_AGENTS = [ - "Mozilla/5.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", - "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)", - "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)", - "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6", - "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1", - "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0", - "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Firefox/59", - "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20", - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36', - 'Mozilla/5.0 (Windows NT 5.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36', - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36', - 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36', - 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36', - 'Mozilla/4.0 (compatible; MSIE 9.0; Windows NT 6.1)', - 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)', - 'Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko', - 'Mozilla/5.0 (Windows NT 6.2; WOW64; Trident/7.0; rv:11.0) like Gecko', - 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0)', - 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko', - 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)', - 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; Trident/7.0; rv:11.0) like Gecko', - 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)', -] +# USER_AGENTS = [ +# "Mozilla/5.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; AcooBrowser; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", +# "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)", +# "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)", +# "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6", +# "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1", +# "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0", +# "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Firefox/59", +# "Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.9.0.8) Gecko Fedora/1.9.0.8-1.fc10 Kazehakase/0.5.6", +# "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11", +# "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/535.20 (KHTML, like Gecko) Chrome/19.0.1036.7 Safari/535.20", +# 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36', +# 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36', +# 'Mozilla/5.0 (Windows NT 5.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36', +# 'Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.90 Safari/537.36', +# 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36', +# 'Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36', +# 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36', +# 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36', +# 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36', +# 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36', +# 'Mozilla/4.0 (compatible; MSIE 9.0; Windows NT 6.1)', +# 'Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko', +# 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)', +# 'Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko', +# 'Mozilla/5.0 (Windows NT 6.2; WOW64; Trident/7.0; rv:11.0) like Gecko', +# 'Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko', +# 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0)', +# 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko', +# 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0)', +# 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; Trident/7.0; rv:11.0) like Gecko', +# 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)', +# ] ################## @@ -240,36 +241,36 @@ def stylize_code(soup): return urwid.Text(stylized_text) -def get_search_results(soup): - """Returns a list of dictionaries containing each search result.""" - search_results = [] +# def get_search_results(soup): +# """Returns a list of dictionaries containing each search result.""" +# search_results = [] - for result in soup.find_all("div", class_="question-summary search-result"): - title_container = result.find_all("div", class_="result-link")[0].find_all("a")[0] +# for result in soup.find_all("div", class_="question-summary search-result"): +# title_container = result.find_all("div", class_="result-link")[0].find_all("a")[0] - if result.find_all("div", class_="status answered") != []: # Has answers - answer_count = int(result.find_all("div", class_="status answered")[0].find_all("strong")[0].text) - elif result.find_all("div", class_="status answered-accepted") != []: # Has an accepted answer (closed) - answer_count = int(result.find_all("div", class_="status answered-accepted")[0].find_all("strong")[0].text) - else: # No answers - answer_count = 0 +# if result.find_all("div", class_="status answered") != []: # Has answers +# answer_count = int(result.find_all("div", class_="status answered")[0].find_all("strong")[0].text) +# elif result.find_all("div", class_="status answered-accepted") != []: # Has an accepted answer (closed) +# answer_count = int(result.find_all("div", class_="status answered-accepted")[0].find_all("strong")[0].text) +# else: # No answers +# answer_count = 0 - search_results.append({ - "Title": title_container["title"], - #"Body": result.find_all("div", class_="excerpt")[0].text, - #"Votes": int(result.find_all("span", class_="vote-count-post ")[0].find_all("strong")[0].text), - "Answers": answer_count, - "URL": SO_URL + title_container["href"] - }) +# search_results.append({ +# "Title": title_container["title"], +# #"Body": result.find_all("div", class_="excerpt")[0].text, +# #"Votes": int(result.find_all("span", class_="vote-count-post ")[0].find_all("strong")[0].text), +# "Answers": answer_count, +# "URL": SO_URL + title_container["href"] +# }) - return search_results +# return search_results def souper(url): """Turns a given URL into a BeautifulSoup object.""" - + UAgent = UserAgent() try: - html = requests.get(url, headers={"User-Agent": random.choice(USER_AGENTS)}) + html = requests.get(url, headers={"User-Agent": UAgent.random}) except requests.exceptions.RequestException: sys.stdout.write("\n%s%s%s" % (RED, "Rebound was unable to fetch Stack Overflow results. " "Please check that you are connected to the internet.\n", END)) @@ -284,16 +285,30 @@ def souper(url): ## Main ## -def search_stackoverflow(query): - """Wrapper function for get_search_results.""" - soup = souper(SO_URL + "/search?pagesize=50&q=%s" % query.replace(' ', '+')) +# def search_stackoverflow(query): +# """Wrapper function for get_search_results.""" +# soup = souper(SO_URL + "/search?pagesize=50&q=%s" % query.replace(' ', '+')) - # TODO: Randomize the user agent +# # TODO: Randomize the user agent - if soup == None: - return (None, True) - else: - return (get_search_results(soup), False) +# if soup == None: +# return (None, True) +# else: +# return (get_search_results(soup), False) + +def search_google(query): + try: + query = query+" :"+SO_URL[8:] + google_search = GoogleSearch() + SearchArgs=(query,1) + google_search.clear_cache() + SearchDict=google_search.search(*SearchArgs) + except Exception as e: + sys.stdout.write("\n%s%s%s" % (RED,"Rebound was unable to fetch results. " + +str(e)+"\n Try again Later.", END)) + sys.exit(1) + + return [result for result in SearchDict] def get_question_and_answers(url): @@ -776,14 +791,16 @@ def _get_selected_link(self): for result in self.search_results: if title == self._stylize_title(result): # Found selected title's search_result dict - return result["URL"] + return result["link"] + #return result["URL"] def _stylize_title(self, search_result): - if search_result["Answers"] == 1: - return "%s (1 Answer)" % search_result["Title"] - else: - return "%s (%s Answers)" % (search_result["Title"], search_result["Answers"]) + return search_result["title"] + # if search_result["Answers"] == 1: + # return "%s (1 Answer)" % search_result["title"] + # else: + # return "%s (%s Answers)" % (search_result["title"], search_result["Answers"]) def _stylize_question(self, title, desc, stats): @@ -830,20 +847,24 @@ def print_help(): def main(): + if len(sys.argv) == 1 or sys.argv[1].lower() == "-h" or sys.argv[1].lower() == "--help": print_help() elif sys.argv[1].lower() == "-q" or sys.argv[1].lower() == "--query": query = ' '.join(sys.argv[2:]) - search_results, captcha = search_stackoverflow(query) + + #search_results, captcha = search_stackoverflow(query) + search_results = search_google(query) if search_results != []: - if captcha: - print("\n%s%s%s" % (RED, "Sorry, Stack Overflow blocked our request. Try again in a minute.\n", END)) - return - else: - App(search_results) # Opens interface + + # if captcha: + # print("\n%s%s%s" % (RED, "Sorry, Stack Overflow blocked our request. Try again in a minute.\n", END)) + # return + # else: + App(search_results) # Opens interface else: - print("\n%s%s%s" % (RED, "No Stack Overflow results found.\n", END)) + print("\n%s%s%s" % (RED, "No Google results found.\n", END)) else: language = get_language(sys.argv[1].lower()) # Gets the language name if language == '': # Unknown language @@ -860,17 +881,19 @@ def main(): error_msg = get_error_message(error, language) # Prepares error message for search if error_msg != None: language = 'java' if language == 'javac' else language # Fix language compiler command - query = "%s %s" % (language, error_msg) - search_results, captcha = search_stackoverflow(query) + query = "%s %s %s" % (language, error_msg) + #search_results, captcha = search_stackoverflow(query) + search_results = search_google(query) if search_results != []: - if captcha: - print("\n%s%s%s" % (RED, "Sorry, Stack Overflow blocked our request. Try again in a minute.\n", END)) - return - elif confirm("\nDisplay Stack Overflow results?"): + # if captcha: + # print("\n%s%s%s" % (RED, "Sorry, Stack Overflow blocked our request. Try again in a minute.\n", END)) + # return + #elif confirm("\nDisplay Stack Overflow results?"): + if confirm("\nDisplay Stack Overflow results?"): App(search_results) # Opens interface else: - print("\n%s%s%s" % (RED, "No Stack Overflow results found.\n", END)) + print("\n%s%s%s" % (RED, "No Google results found.\n", END)) else: print("\n%s%s%s" % (CYAN, "No error detected :)\n", END))