big routing refactoring

Author: chubin

lib/adapter/__init__.py

@@ -0,0 +1,71 @@
+import sys
+import os
+import glob
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+from globals import PATH_CHEAT_SHEETS
+
+def _remove_initial_underscore(filename):
+    if filename.startswith('_'):
+        filename = filename[1:]
+    return filename
+
+def _sanitize_dirname(dirname):
+    dirname = os.path.basename(dirname)
+    dirname = _remove_initial_underscore(dirname)
+    return dirname
+
+def _format_answer(dirname, filename):
+    return "%s/%s" % (_sanitize_dirname(dirname), filename)
+
+def _get_answer_files_from_folder():
+    topics = map(os.path.split, glob.glob(PATH_CHEAT_SHEETS + "*/*"))
+    return [_format_answer(dirname, filename)
+            for dirname, filename in topics if filename not in ['_info.yaml']]
+def _isdir(topic):
+    return os.path.isdir(topic)
+def _get_answers_and_dirs():
+    topics = glob.glob(PATH_CHEAT_SHEETS + "*")
+    answer_dirs = [_remove_initial_underscore(os.path.split(topic)[1]).rstrip('/')+'/'
+                   for topic in topics if _isdir(topic)]
+    answers = [os.path.split(topic)[1] for topic in topics if not _isdir(topic)]
+    return answers, answer_dirs
+
+def _update_cheat_sheets_topics():
+    answers = _get_answer_files_from_folder()
+    cheatsheet_answers, cheatsheet_dirs = _get_answers_and_dirs()
+    return answers+cheatsheet_answers, cheatsheet_dirs
+
+def get_list():
+    return _update_cheat_sheets_topics()[0]
+
+def get_dirs_list():
+    return _update_cheat_sheets_topics()[1]
+
+_CHEAT_SHEETS_LIST = get_list()
+def is_found(topic):
+    return topic in _CHEAT_SHEETS_LIST
+
+_CHEAT_SHEETS_DIRS = get_dirs_list()
+def is_dir_found(topic):
+    return topic in _CHEAT_SHEETS_DIRS
+
+def get_page(topic, request_options=None):
+    """
+    Get the cheat sheet topic from the own repository (cheat.sheets).
+    It's possible that topic directory starts with omitted underscore
+    """
+    filename = PATH_CHEAT_SHEETS + "%s" % topic
+    if not os.path.exists(filename):
+        filename = PATH_CHEAT_SHEETS + "_%s" % topic
+    if os.path.isdir(filename):
+        return ""
+    else:
+        return open(filename, "r").read().decode('utf-8')
+
+def get_dir(topic, request_options=None):
+    answer = []
+    for f_name in glob.glob(PATH_CHEAT_SHEETS + "%s/*" % topic.rstrip('/')):
+        answer.append(os.path.basename(f_name))
+    topics = sorted(answer)
+    return "\n".join(topics) + "\n"

lib/adapter/cheat_sheets.py

@@ -0,0 +1,67 @@
+from gevent.monkey import patch_all
+from gevent.subprocess import Popen, PIPE
+patch_all()
+
+import sys
+import os
+import glob
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from globals import PATH_TLDR_PAGES, PATH_CHEAT_PAGES
+
+def _get_filenames(path):
+    return [os.path.split(topic)[1] for topic in glob.glob(path)]
+
+def get_tldr_list():
+    return [filename[:-3]
+            for filename in _get_filenames(PATH_TLDR_PAGES) if filename.endswith('.md')]
+
+_TLDR_LIST = get_tldr_list()
+def tldr_is_found(topic):
+    return topic in _TLDR_LIST
+
+def get_tldr(topic, request_options=None):
+    cmd = ["tldr", topic]
+    proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
+    answer = proc.communicate()[0]
+
+    fixed_answer = []
+    for line in answer.splitlines():
+        line = line[2:]
+        if line.startswith('-'):
+            line = '# '+line[2:]
+        elif not line.startswith(' '):
+            line = "# "+line
+        else:
+            pass
+
+        fixed_answer.append(line)
+
+    answer = "\n".join(fixed_answer) + "\n"
+    return answer.decode('utf-8')
+
+def get_cheat_list():
+    return _get_filenames(PATH_CHEAT_PAGES)
+
+_CHEAT_LIST = get_cheat_list()
+def cheat_is_found(topic):
+    return topic in _CHEAT_LIST
+
+def get_cheat(topic, request_options=None):
+    cmd = ["cheat", topic]
+    proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
+    answer = proc.communicate()[0].decode('utf-8')
+    return answer
+
+def get_translation(topic, request_options=None):
+    from_, topic = topic.split('/', 1)
+    to_ = request_options.get('lang', 'en')
+    if '-' in from_:
+        from_, to_ = from_.split('-', 1)
+
+    cmd = ["/home/igor/cheat.sh/bin/get_translation",
+        from_, to_, topic.replace('+', ' ')]
+    print("calling:", cmd)
+    proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
+    answer = proc.communicate()[0].decode('utf-8')
+    return answer

lib/adapter/cmd.py

@@ -0,0 +1,6 @@
+class Adapter(object):
+    pass
+
+class cheatAdapter(Adapter):
+    pass
+

lib/adapter/common.py

@@ -0,0 +1,116 @@
+import sys
+import os
+import glob
+import collections
+
+from fuzzywuzzy import process, fuzz
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+from globals import MYDIR, COLOR_STYLES
+from colorize_internal import colorize_internal
+
+_INTERNAL_TOPICS = [
+    ":cht.sh",
+    ":bash_completion",
+    ":emacs",
+    ":emacs-ivy",
+    ":firstpage",
+    ":firstpage-v1",
+    ":firstpage-v2",
+    ":fish",
+    ":help",
+    ":intro",
+    ":list",
+    ":post",
+    ":styles",
+    ":styles-demo",
+    ":vim",
+    ":zsh",
+    ":share",
+    ]
+
+_COLORIZED_INTERNAL_TOPICS = [
+    ':intro',
+]
+
+class InternalPages(object):
+
+    def __init__(self, get_topic_type=None, get_topics_list=None):
+        self.get_topic_type = get_topic_type
+        self.get_topics_list = get_topics_list
+
+    def _get_stat(self):
+        stat = collections.Counter([
+            self.get_topic_type(topic)
+            for topic in self.get_topics_list()
+        ])
+
+        answer = ""
+        for key, val in stat.items():
+            answer += "%s %s\n" % (key, val)
+        return answer
+
+
+    @staticmethod
+    def get_list():
+        return _INTERNAL_TOPICS
+
+    def _get_list_answer(self, topic, request_options=None):
+        if '/' in topic:
+            topic_type, topic_name = topic.split('/', 1)
+            if topic_name == ":list":
+                topic_list = [x[len(topic_type)+1:]
+                              for x in self.get_topics_list()
+                              if x.startswith(topic_type + "/")]
+                return "\n".join(topic_list)+"\n"
+
+        answer = ""
+        if topic == ":list":
+            answer = "\n".join(x for x in self.get_topics_list()) + "\n"
+
+        return answer
+
+    def get_page(self, topic, request_options=None):
+        if topic.endswith('/:list') or topic.lstrip('/') == ':list':
+            return self._get_list_answer(topic)
+
+        answer = ""
+        if topic == ':styles':
+            answer = "\n".join(COLOR_STYLES) + "\n"
+        elif topic == ":stat":
+            answer = self._get_stat()+"\n"
+        elif topic in _INTERNAL_TOPICS:
+            answer = open(os.path.join(MYDIR, "share", topic[1:]+".txt"), "r").read()
+            if topic in _COLORIZED_INTERNAL_TOPICS:
+                answer = colorize_internal(answer)
+
+        return answer
+
+    def is_found(self, topic):
+        return topic in self.get_list()
+
+class UnknownPages(InternalPages):
+
+    @staticmethod
+    def get_list():
+        return []
+
+    @staticmethod
+    def is_found(topic):
+        return False
+
+    def get_page(self, topic, request_options=None):
+        topics_list = self.get_topics_list()
+        if topic.startswith(':'):
+            topics_list = [x for x in topics_list if x.startswith(':')]
+        else:
+            topics_list = [x for x in topics_list if not x.startswith(':')]
+
+        possible_topics = process.extract(topic, topics_list, scorer=fuzz.ratio)[:3]
+        possible_topics_text = "\n".join([("    * %s %s" % x) for x in possible_topics])
+        return """
+Unknown topic.
+Do you mean one of these topics maybe?
+
+%s
+    """ % possible_topics_text

lib/adapter/internal.py

@@ -0,0 +1,16 @@
+import sys
+import os
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+
+from globals import PATH_LATENZ
+
+def get_answer(topic, request_options=None):
+    sys.path.append(PATH_LATENZ)
+    import latencies
+    return latencies.render()
+
+def get_list():
+    return ['latencies']
+
+def is_found(topic):
+    return topic.lower() in ['latencies', 'late.nz', 'latency']

lib/adapter/latenz.py

@@ -0,0 +1,63 @@
+from gevent.monkey import patch_all
+from gevent.subprocess import Popen, PIPE
+patch_all()
+
+import sys
+import os
+import re
+
+from polyglot.detect import Detector
+from polyglot.detect.base import UnknownLanguage
+
+sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..')))
+from globals import MYDIR
+
+def get_page(topic, request_options=None):
+    """
+    Find answer for the `topic` question.
+    """
+
+    # if there is a language name in the section name,
+    # cut it off (de:python => python)
+    if '/' in topic:
+        section_name, topic = topic.split('/', 1)
+        if ':' in section_name:
+            _, section_name = section_name.split(':', 1)
+        topic = "%s/%s" % (section_name, topic)
+
+    # some clients send queries with - instead of + so we have to rewrite them to
+    topic = re.sub(r"(?<!-)-", ' ', topic)
+
+    topic_words = topic.split()
+
+    topic = " ".join(topic_words)
+
+    lang = 'en'
+    try:
+        query_text = topic # " ".join(topic)
+        query_text = re.sub('^[^/]*/+', '', query_text.rstrip('/'))
+        query_text = re.sub('/[0-9]+$', '', query_text)
+        query_text = re.sub('/[0-9]+$', '', query_text)
+        detector = Detector(query_text)
+        supposed_lang = detector.languages[0].code
+        if len(topic_words) > 2 or supposed_lang in ['az', 'ru', 'uk', 'de', 'fr', 'es', 'it', 'nl']:
+            lang = supposed_lang
+        if supposed_lang.startswith('zh_') or supposed_lang == 'zh':
+            lang = 'zh'
+        elif supposed_lang.startswith('pt_'):
+            lang = 'pt'
+        if supposed_lang in ['ja', 'ko']:
+            lang = supposed_lang
+
+    except UnknownLanguage:
+        print("Unknown language (%s)" % query_text)
+
+    if lang != 'en':
+        topic = ['--human-language', lang, topic]
+    else:
+        topic = [topic]
+
+    cmd = [os.path.join(MYDIR, "bin/get-answer-for-question")] + topic
+    proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
+    answer = proc.communicate()[0].decode('utf-8')
+    return answer