Merge pull request #4 from Linusp/optimize

optimize command `filter` and `fetch-unread`

Author: Linusp

CHANGELOG.md

@@ -1,5 +1,12 @@
 # CHANGELOG
 
+## v0.2.1
+
+Changed
+
+- Supported new output formats in command `fetch-unread`: `markdown` and `org-mode`
+- Changed command `filter`, see `example/rules.example.yaml` for details
+
 ## v0.2.0
 
 Added

examples/rules.example.yaml

@@ -1,13 +1,24 @@
-- field: title_or_content                  # title/content/title_or_content
-  type: include_any                        # include_any/include_all/exclude
-  rules:
-    - 权威认证
-    - 值得拥有
-    - 潮牌
-    - 新款
-    - 求助
-    - 招聘
-    - 免费版
-    - 包邮
-    - 天猫
-    - 淘宝
+- name: test
+  folders:
+    - inbox
+  fields: ["title", "content"]
+  filter:
+    type: include_any           # include_any/include_all/exclude
+    rules:
+      - 权威认证
+      - 值得拥有
+      - 潮牌
+      - 新款
+      - 求助
+      - 招聘
+      - 免费版
+      - 包邮
+      - 天猫
+      - 淘宝
+  actions:
+    - type: mark_as_read
+    - type: tag
+      tags: tag1,tag2,tag3
+    - type: like
+    - type: star
+    - type: broadcast

inoreader/filter.py

@@ -1,37 +1,25 @@
 import re
 
 
-class Filter(object):
-    def __init__(self, *args, **kwargs):
-        pass
-
-    @classmethod
-    def from_config(cls, config):
-        """
-        config: {
-            'type': 'include_any',
-            'rules': [],
-        }
-        """
-        cls_map = {
-            sub_cls.name: sub_cls
-            for sub_cls in cls.__subclasses__()
-        }
-
-        sub_cls = config['type']
-        rules = config['rules']
-        return cls_map[sub_cls](rules)
+_FILTERS = {}
 
-    def validate(self, text):
-        raise NotImplementedError
 
+def register_filter(name, override=False):
+    def wrap(cls):
+        global _FILTERS
+        if name not in _FILTERS or override:
+            _FILTERS[name] = cls
+
+        return cls
 
-class IncludeAnyFilter(Filter):
+    return wrap
 
-    name = 'include_any'
+
+@register_filter('include_any')
+class IncludeAnyFilter(object):
 
     def __init__(self, rules):
-        self.rules = [re.compile(regexp) for regexp in rules]
+        self.rules = [re.compile(regexp, re.IGNORECASE) for regexp in rules]
 
     def validate(self, text):
         for regexp in self.rules:
@@ -41,12 +29,11 @@ def validate(self, text):
         return False
 
 
-class IncludeAllFilter(Filter):
-
-    name = 'include_all'
+@register_filter('include_all')
+class IncludeAllFilter(object):
 
     def __init__(self, rules):
-        self.rules = [re.compile(regexp) for regexp in rules]
+        self.rules = [re.compile(regexp, re.IGNORECASE) for regexp in rules]
 
     def validate(self, text):
         for regexp in self.rules:
@@ -56,16 +43,25 @@ def validate(self, text):
         return True
 
 
-class ExcludeFilter(Filter):
-
-    name = 'exclude'
+@register_filter('exclude')
+class ExcludeFilter(object):
 
     def __init__(self, rules):
-        self.rules = [re.compile(regexp) for regexp in rules]
+        self.rules = [re.compile(regexp, re.IGNORECASE) for regexp in rules]
 
     def validate(self, text):
         for regexp in self.rules:
             if regexp.findall(text):
                 return False
 
         return True
+
+
+def get_filter(config):
+    filter_type = config['type']
+    if filter_type not in _FILTERS:
+        raise ValueError("unsupported filter type: {}".format(filter_type))
+
+    filter_cls = _FILTERS[filter_type]
+    params = {k: v for k, v in config.items() if k != 'type'}
+    return filter_cls(**params)

inoreader/main.py

@@ -8,11 +8,12 @@
 import codecs
 import argparse
 from datetime import datetime
+from collections import defaultdict
 from configparser import ConfigParser
 
 import yaml
 from inoreader import InoreaderClient
-from inoreader.filter import Filter
+from inoreader.filter import get_filter
 
 
 APPID_ENV_NAME = 'INOREADER_APP_ID'
@@ -141,7 +142,7 @@ def add_unread_fetch_parser(subparsers):
     parser.add_argument("-o", "--outfile", required=True, help="Filename to save articles")
     parser.add_argument(
         "--out-format",
-        choices=['json', 'csv', 'plain'],
+        choices=['json', 'csv', 'plain', 'markdown', 'org-mode'],
         default='json',
         help='Format of output file, default: json'
     )
@@ -166,6 +167,12 @@ def fetch_unread(folder, tags, outfile, out_format):
             print('TITLE: {}'.format(title), file=fout)
             print("CONTENT: {}".format(text), file=fout)
             print(file=fout)
+        elif out_format == 'markdown':
+            print('# {}\n'.format(title), file=fout)
+            print(text + '\n', file=fout)
+        elif out_format == 'org-mode':
+            print('* {}\n'.format(title), file=fout)
+            print(text + '\n', file=fout)
 
     print("[{}] fetched {} articles and saved them in {}".format(datetime.now(), idx + 1, outfile))
 
@@ -174,13 +181,7 @@ def fetch_unread(folder, tags, outfile, out_format):
 
 def add_filter_parser(subparsers):
     parser = subparsers.add_parser('filter', help='Select articles and do something')
-    parser.add_argument("-f", "--folder", required=True, help='Folder which articles belong to')
     parser.add_argument("-r", "--rules", required=True, help='YAML file with your rules')
-    parser.add_argument("-a", "--action", default='read',
-                        choices=['read', 'like', 'tag', 'broadcast', 'star'],
-                        help='Action you want to perform, default: read')
-    parser.add_argument("-t", "--tags",
-                        help="Tag(s) to be used when action is 'tag', seprate with comma")
 
 
 def apply_action(articles, client, action, tags):
@@ -190,7 +191,7 @@ def apply_action(articles, client, action, tags):
 
         for article in articles:
             print("Add tags [{}] on article: {}".format(tags, article.title))
-    elif action == 'read':
+    elif action == 'mark_as_read':
         client.mark_as_read(articles)
         for article in articles:
             print("Mark article as read: {}".format(article.title))
@@ -208,35 +209,70 @@ def apply_action(articles, client, action, tags):
             print("Starred article: {}".format(article.title))
 
 
-def filter_articles(folder, rules_file, action, tags):
+def filter_articles(rules_file):
     client = get_client()
     filters = []
     for rule in yaml.load(open(rules_file)):
+        name = rule.get('name')
+        folders = rule['folders']
+
+        fields = []
+        # only 'title' or 'content' is supported now
+        for field in rule.get('fields', ['title', 'content']):
+            if field not in ('title', 'content'):
+                continue
+            fields.append(field)
+        cur_filter = get_filter(rule['filter'])
+
+        actions = []
+        # only 'mark_as_read', 'like', 'star', 'broadcast', 'tag' is supported now
+        for action in rule.get('actions', [{'type': 'mark_as_read'}]):
+            if action['type'] not in ('mark_as_read', 'like', 'star', 'broadcast', 'tag'):
+                continue
+            actions.append(action)
+
         filters.append({
-            'field': rule.get('field', 'title'),
-            'filter': Filter.from_config(rule),
+            'name': name,
+            'folders': folders,
+            'fields': fields,
+            'filter': cur_filter,
+            'actions': actions
         })
 
-    matched_articles = []
-    for idx, article in enumerate(client.fetch_unread(folder=folder)):
-        matched = False
-        for article_filter in filters:
-            if article_filter['field'] in ('title', 'title_or_content') and \
-               article_filter['filter'].validate(article.title):
+    articles_by_foler = {}      # folder -> articles
+    matched_articles = defaultdict(list)       # action -> articles
+    for rule in filters:
+        articles = []
+        for folder in rule['folders']:
+            if folder not in articles_by_foler:
+                articles_by_foler[folder] = list(client.fetch_unread(folder=folder))
+
+            articles.extend(articles_by_foler[folder])
+
+        # FIXME: deduplicate
+        count = 0
+        for article in articles:
+            matched = False
+            if 'title' in rule['fields'] and rule['filter'].validate(article.title):
                 matched = True
-                break
-            if article_filter['field'] in ('content', 'title_or_content') and \
-               article_filter['filter'].validate(article.text):
+            if 'content' in rule['fields'] and rule['filter'].validate(article.text):
                 matched = True
-                break
-        if matched:
-            matched_articles.append(article)
-            if len(matched_articles) == 10:
-                apply_action(matched_articles, client, action, tags)
-                matched_articles = []
 
-    if matched_articles:
-        apply_action(matched_articles, client, action, tags)
+            if matched:
+                for action in rule['actions']:
+                    matched_articles[action['type']].append((article, action))
+
+                count += 1
+        print("[{}] matched {} articles with filter: {}".format(
+            datetime.now(), count, rule['name']))
+
+    for action_name in matched_articles:
+        articles, actions = zip(*matched_articles[action_name])
+        if action_name != 'tag':
+            apply_action(articles, client, action_name, None)
+        else:
+            for article, action in zip(articles, actions):
+                apply_action([article], client, 'tag', action['tags'])
 
 
 def main():
@@ -263,11 +299,7 @@ def main():
     elif args.command == 'fetch-unread':
         fetch_unread(args.folder, args.tags, args.outfile, args.out_format)
     elif args.command == 'filter':
-        if args.action == 'tag' and not args.tags:
-            print("Need at least one tag when action is 'tag'!")
-            sys.exit(1)
-
-        filter_articles(args.folder, args.rules, args.action, args.tags)
+        filter_articles(args.rules)
 
 
 if __name__ == '__main__':

setup.py

@@ -4,7 +4,7 @@
 from setuptools import setup, find_packages
 
 
-VERSION = '0.2.0'
+VERSION = '0.2.1'
 REQS = [
     'lxml',
     'requests',