feat: Save feed URL on redirect

Also adds test for saving feed URL on redirect.

Author: mcepl

rss2email/command.py

@@ -94,7 +94,7 @@ def run(feeds, args):
                             interval = interval
                         ))
                         _time.sleep(interval)
-                    feed.run(send=args.send, clean=args.clean)
+                    feed.run(send=args.send, clean=args.clean, save_config=args.save_config)
                 except _error.RSS2EmailError as e:
                     e.log()
                 last_server = current_server

rss2email/feed.py

@@ -386,9 +386,9 @@ def _fetch(self):
         f = _util.TimeLimitedFunction('feed {}'.format(self.name), timeout, _feedparser.parse)
         return f(self.url, self.etag, modified=self.modified, agent=self.user_agent, **kwargs)
 
-    def _process(self, parsed):
+    def _process(self, parsed, save_config=False):
         _LOG.info('process {}'.format(self))
-        self._check_for_errors(parsed)
+        self._check_for_errors(parsed, save_config=save_config)
         for entry in reversed(parsed.entries):
             _LOG.debug('processing {}'.format(entry.get('id', 'no-id')))
             processed = self._process_entry(parsed=parsed, entry=entry)
@@ -402,15 +402,16 @@ def _process(self, parsed):
                         continue
                 yield processed
 
-    def _check_for_errors(self, parsed):
+    def _check_for_errors(self, parsed, save_config=False):
         warned = False
         status = getattr(parsed, 'status', 200)
         _LOG.debug('HTTP status {}'.format(status))
         if status in [301, 308]:
             _LOG.info('redirect {} from {} to {}'.format(
                     self.name, self.url, parsed['url']))
             self.url = parsed['url']
-            # TODO: `url` is not saved -- add config option to call feeds.save_config() in run command
+            if save_config:
+                self.save_to_config()
         elif status == 304:
             _LOG.info('skipping {}: feed was not modified since last update'.format(
                     self.name, self.url))
@@ -917,7 +918,7 @@ def _send(self, sender, message):
         _email.send(recipient=self.to, message=message,
                     config=self.config, section=section)
 
-    def run(self, send=True, clean=False):
+    def run(self, send=True, clean=False, save_config=False):
         """Fetch and process the feed, mailing entry emails.
 
         >>> feed = Feed(
@@ -947,7 +948,7 @@ def run(self, send=True, clean=False):
                 raise _error.InvalidDigestType(type)
             digest = self._new_digest()
             seen = []
-            for (guid, state, sender, message) in self._process(parsed):
+            for (guid, state, sender, message) in self._process(parsed, save_config=save_config):
                 _LOG.debug('new message: {}'.format(message['Subject']))
                 seen.append((guid, state))
                 self._append_to_digest(digest=digest, message=message)
@@ -962,7 +963,7 @@ def run(self, send=True, clean=False):
                 for (guid, state) in seen:
                     self.seen[guid] = state
         else:
-            for (guid, state, sender, message) in self._process(parsed):
+            for (guid, state, sender, message) in self._process(parsed, save_config=save_config):
                 _LOG.debug('new message: {}'.format(message['Subject']))
                 if send:
                     self._send(sender=sender, message=message)

rss2email/main.py

@@ -112,8 +112,8 @@ def run(*args, **kwargs):
         default=True, action='store_const', const=False,
         help="fetch feeds, but don't send email")
     run_parser.add_argument(
-        '--clean', action='store_true',
-        help='clean old feed entries')
+        '--save-config', action='store_true',
+        help='save feed configuration changes (e.g. new URL after a redirect)')
     run_parser.add_argument(
         'index', nargs='*',
         help='feeds to fetch (defaults to fetching all feeds)')

test/test_main.py

@@ -249,6 +249,22 @@ def webserver_for_test_if_fetch(queue, timeout):
     finally:
         httpd.server_close()
 
+def webserver_for_test_redirect(queue, status_code):
+    class RedirectHandler(NoLogHandler):
+        def do_GET(self):
+            self.send_response(status_code)
+            self.send_header('Location', '/disqus/feed.rss')
+            self.end_headers()
+
+    httpd = http.server.HTTPServer(('', 0), RedirectHandler)
+    try:
+        port = httpd.server_address[1]
+        queue.put(port)
+        while queue.get() != "stop":
+            httpd.handle_request()
+    finally:
+        httpd.server_close()
+
 class TestFetch(unittest.TestCase):
     "Retrieving feeds from servers"
     def test_delay(self):
@@ -379,6 +395,27 @@ def test_only_new(self):
                 self.assertIn("seen", content["feeds"][0])
         self.assertEqual(queue.get(), "done")
 
+    def test_redirect(self):
+        "Saves feed URL on redirect"
+        cfg = """[DEFAULT]
+        to = [email protected]"""
+
+        queue = multiprocessing.Queue()
+        webserver_proc = multiprocessing.Process(target=webserver_for_test_redirect, args=(queue, 301))
+        webserver_proc.start()
+        port = queue.get()
+
+        with ExecContext(cfg) as ctx:
+            ctx.call("add", 'test', 'http://127.0.0.1:{port}/redirect'.format(port = port))
+
+            queue.put("next")
+            ctx.call("run", "--no-send", "--save-config")
+
+            with open(ctx.cfg_path, 'r') as f:
+                config = f.read()
+                self.assertIn("url = http://127.0.0.1:{port}/disqus/feed.rss".format(port=port), config)
+
+        queue.put("stop")
 
 def webserver_for_test_send(queue):
     httpd = http.server.HTTPServer(('', 0), NoLogHandler)