Merge pull request #14 from georgedorn/master

Mastodon Favorites feed, mastodon client script, refactor

Author: SamR1

README.md

@@ -21,47 +21,33 @@ The RSS feed displays only the original tweets (not the retweets) and :
 - [Feedgenerator](https://pypi.python.org/pypi/feedgenerator)
 - [Tweepy](https://github.com/tweepy/tweepy)
 - [pytz](https://pypi.python.org/pypi/pytz/)
+- [PyYAML](https://github.com/yaml/pyyaml)
 - [BeautifulSoup](https://pypi.python.org/pypi/beautifulsoup4)
 - [Mastodon.py](https://github.com/halcy/Mastodon.py)
 - API keys Twitter and/or Mastodon 
 
 
 ## **Steps :**
-- install Python packages : flask, BeautifulSoup, Mastodon.py, feedgenerator, tweepy and pytz
+- install Python packages : flask, BeautifulSoup, Mastodon.py, feedgenerator, tweepy, PyYAML and pytz
 ```bash
-$ pip3 install flask bs4 feedgenerator tweepy pytz Mastodon.py
+$ pip3 install -r requirements.txt
 ```
 
 - clone this repo :
 ```bash
 $ git clone https://github.com/SamR1/python-twootfeed.git
 ```
 
+- Copy the included **config.example.yml** to **config.yml** and fill in fields for the client(s) you will use.
+
 - API Keys
     - for **Twitter** : see https://dev.twitter.com  
     copy/paste the Twitter API key values in **config.yml.example** ('_consumerKey_' and '_consumerSecret_')
-    - for **Mastodon** : see [Python wrapper for the Mastodon API](https://github.com/halcy/Mastodon.py)  
-       - generate client and user credentials files :  
-        ```python
-        from mastodon import Mastodon
-        
-        # Register app - only once!        
-        Mastodon.create_app(
-             'pytooterapp',
-              to_file = 'tootrss_clientcred.txt'
-        )        
-        
-        # Log in - either every time, or use persisted        
-        mastodon = Mastodon(client_id = 'tootrss_clientcred.txt')
-        mastodon.log_in(
-            'my_login_email@example.com',
-            'incrediblygoodpassword',
-            to_file = 'tootrss_usercred.txt'
-        )
-        ```
-        - copy/paste file names in **config.yml.example** ('_client_id_file_' and '_access_token_file_')
-        
-Rename the config file **config.yml**.
+    - for **Mastodon** : see [Python wrapper for the Mastodon API](https://mastodonpy.readthedocs.io/)
+      - Generate the client and user credentials manually via the [Mastodon client](https://mastodonpy.readthedocs.io/en/latest/#app-registration-and-user-authentication)
+        - note that using an instance other than https://mastodon.social requires adding `api_base_url` to most method calls.
+        - the file names for **client_id** and **access_token_file** go in the mastodon section of **config.yml**
+      - Or use the included script (`python3 create_mastodon_client.py`) which will register your app and prompt you to log in, creating the credential files for you.
 
 - Start the server
 ```bash
@@ -71,7 +57,7 @@ $ python3 -m flask run --host=0.0.0.0
 
 - the RSS feeds are available on these urls :  
    - for Twitter : http://localhost:5000/_keywords_ or http://localhost:5000/tweets/_keywords_
-   - for Mastodon : http://localhost:5000/toots/_keywords_
+   - for Mastodon : http://localhost:5000/toots/_keywords_ and http://localhost:5000/toot_favorites
 
 ## Examples :  
 ### Search on Twitter :  

app.py

@@ -11,18 +11,15 @@
 import tweepy
 import yaml
 import sys
-
+import os
+from config import get_config
 
 app = Flask(__name__)
 app.debug = True
 
+param = get_config()
 
-with open('config.yml', 'r') as stream:
-    try:
-        param = yaml.safe_load(stream)
-    except yaml.YAMLError as e:
-        print(e)
-        sys.exit()
+text_length_limit = int(param['feed'].get('text_length_limit', 100))
 
 # Twitter
 try:
@@ -40,9 +37,19 @@
 
 # Mastodon
 try:
+    client_file = param['mastodon']['client_id_file']
+    if not os.path.exists(client_file):
+        raise Exception("File not found: " + client_file)
+    access_token_file = param['mastodon']['access_token_file']
+    if not os.path.exists(access_token_file):
+        raise Exception("File not found: " + client_file)
+
+    mastodon_url = param['mastodon'].get('url', 'https://mastodon.social')
+
     mastodon = Mastodon(
-        client_id=param['mastodon']['client_id_file'],
-        access_token=param['mastodon']['access_token_file']
+        client_id=client_file,
+        access_token=access_token_file,
+        api_base_url=mastodon_url
     )
 except Exception as e:
     print('Error Mastodon instance creation: ' + str(e))
@@ -189,7 +196,8 @@ def tootfeed(query_feed):
                                '♻ : ' + str(toot['reblogs_count']) + ', ' + \
                                '✰ : ' + str(toot['favourites_count']) + '</div></blockquote>'
 
-            toot['created_at'] = datetime.datetime.strptime(toot['created_at'], '%Y-%m-%dT%H:%M:%S.%fZ')
+            if isinstance(toot['created_at'], str):
+                toot['created_at'] = datetime.datetime.strptime(toot['created_at'], '%Y-%m-%dT%H:%M:%S.%fZ')
 
             buffered.append(toot.copy())
 
@@ -204,12 +212,66 @@ def tootfeed(query_feed):
         for toot in buffered:
 
             text = BeautifulSoup(toot['content'], "html.parser").text
-            if len(text) > 100:
-                text = text[:100] + '... '
+            pubdate = toot['created_at']
+            if not pubdate.tzinfo:
+                pubdate = utc.localize(pubdate).astimezone(pytz.timezone(param['feed']['timezone']))
+
+            if len(text) > text_length_limit:
+                text = text[:text_length_limit] + '... '
+            f.add_item(title=toot['account']['display_name'] + ' (' + toot['account']['username'] + '): '
+                             + text,
+                       link=toot['url'],
+                       pubdate=pubdate,
+                       description=toot['htmltext'])
+
+        xml = f.writeString('UTF-8')
+    else:
+        xml = 'error - Mastodon parameters not defined'
+
+    return xml
+
+@app.route('/toot_favorites')
+def toot_favorites_feed():
+    """ generate an rss feed authenticated user's favorites """
+
+    if mastodonOK:
+        buffered = []
+        favorite_toots = mastodon.favourites()
+        for toot in favorite_toots:
+
+            toot['htmltext'] = '<blockquote><div><img src="' + toot['account']['avatar_static'] + \
+                                '" alt="' + toot['account']['display_name'] + \
+                                '" />   <strong>' + toot['account']['username'] + \
+                                ': </strong>' + toot['content'] + '<br>' + \
+                               '♻ : ' + str(toot['reblogs_count']) + ', ' + \
+                               '✰ : ' + str(toot['favourites_count']) + '</div></blockquote>'
+
+            if isinstance(toot['created_at'], str):
+                toot['created_at'] = datetime.datetime.strptime(toot['created_at'], '%Y-%m-%dT%H:%M:%S.%fZ')
+
+            buffered.append(toot.copy())
+
+        utc = pytz.utc
+        f = feedgenerator.Rss201rev2Feed(title=param['mastodon']['title'] + ' Favourites ',
+                                         link=param['mastodon']['url'] + '/web/favourites',
+                                         description=param['mastodon']['description'],
+                                         language=param['feed']['language'],
+                                         author_name=param['feed']['author_name'],
+                                         feed_url=param['feed']['feed_url'])
+
+        for toot in buffered:
+
+            text = BeautifulSoup(toot['content'], "html.parser").text
+            pubdate = toot['created_at']
+            if not pubdate.tzinfo:
+                pubdate = utc.localize(pubdate).astimezone(pytz.timezone(param['feed']['timezone']))
+
+            if len(text) > text_length_limit:
+                text = text[:text_length_limit] + '... '
             f.add_item(title=toot['account']['display_name'] + ' (' + toot['account']['username'] + '): '
                              + text,
                        link=toot['url'],
-                       pubdate=utc.localize(toot['created_at']).astimezone(pytz.timezone(param['feed']['timezone'])),
+                       pubdate=pubdate,
                        description=toot['htmltext'])
 
         xml = f.writeString('UTF-8')
@@ -220,4 +282,5 @@ def tootfeed(query_feed):
 
 
 if __name__ == "__main__":
-    app.run()
+    app.run(use_reloader=True)
+

config.example.yml

@@ -8,10 +8,12 @@ mastodon:
   url: 'https://mastodon.social'
   client_id_file: 'tootrss_clientcred.txt'
   access_token_file: 'tootrss_usercred.txt'
+  app_name: 'tootrss'  # Used to identify authenticated apps
   title: 'Recherche Mastodon : '
   description: "Résultat d'une recherche Mastodon retournée dans un flux RSS."
 feed:
   language: 'fr'
   author_name: ''
   feed_url: 'http://localhost:5000/'
   timezone: 'Europe/Paris'
+  text_length_limit: 100

config.py

@@ -0,0 +1,12 @@
+"""
+Loads and parses the configuration file.
+"""
+import yaml
+
+def get_config():
+    with open('config.yml', 'r', encoding='utf-8') as stream:
+        try:
+            return yaml.safe_load(stream)
+        except yaml.YAMLError as e:
+            print(e)
+            sys.exit()

create_mastodon_client.py

@@ -0,0 +1,36 @@
+from config import get_config
+from mastodon import Mastodon
+from getpass import getpass
+
+if __name__ == '__main__':
+    print("This script helps you create a new mastodon client and log in.")
+    print("Before we start, make sure config.yml exists.")
+    config = get_config()
+
+    mast_cfg = config['mastodon']
+    print("Configuration found.")
+    print("Looks like you want to use this instance: ", mast_cfg['url'])
+    print("If that's wrong, now is a good time to cancel (^C) and fix it.")
+    input("<enter> to continue")
+
+    print("Registering a new app with {url} called {app_name} and saving credentials in {client_id_file}".format(**mast_cfg))
+
+    Mastodon.create_app(mast_cfg['app_name'], api_base_url=mast_cfg['url'], to_file=mast_cfg['client_id_file'], scopes=['read'])
+    mastodon = Mastodon(client_id=mast_cfg['client_id_file'], api_base_url=mast_cfg['url'])
+    print("Registration successful.  Now to log in.")
+    user_email = input("User email: ")
+    password = getpass("Password (not shown and not saved):")
+
+    # Log in - either every time, or use persisted        
+    mastodon.log_in(user_email, password, to_file=mast_cfg['access_token_file'], scopes=['read'])
+
+    print("Verifying credentials...")
+    try:
+        res = mastodon.account_verify_credentials()
+        print("Credentials look good; client reports user's account name is: " + res['acct'])
+        print("Configuration complete; app should appear at: " + mast_cfg['url'] + "/oauth/authorized_applications")
+        print("You should not need to log in again unless this app is removed or credentials expire.")
+    except Exception as ex:
+        print("Something went wrong; mastodon client reported an error:")
+        print(ex)
+    

requirements.txt

@@ -0,0 +1,7 @@
+Flask==0.12.2
+bs4==0.0.1
+feedgenerator==1.9
+tweepy==3.5.0
+pytz==2017.3
+Mastodon.py==1.1.2
+PyYAML==3.12