io.UrlReader: support Dropbox URLs

Author: kernc

Orange/data/io.py

@@ -16,8 +16,8 @@
 from itertools import chain, repeat
 from functools import lru_cache
 from collections import OrderedDict
-from urllib.parse import urlparse, unquote as urlunquote
-from urllib.request import urlopen
+from urllib.parse import urlparse, urlsplit, urlunsplit, unquote as urlunquote
+from urllib.request import urlopen, Request
 
 import bottleneck as bn
 import numpy as np
@@ -809,10 +809,17 @@ def write(cls, filename, tree):
 
 
 class UrlReader(FileFormat):
+    @staticmethod
+    def urlopen(url):
+        req = Request(
+            url,
+            # Avoid 403 error with servers that dislike scrapers
+            headers={'User-Agent': 'Mozilla/5.0 (X11; Linux) Gecko/20100101 Firefox/'})
+        return urlopen(req, timeout=10)
+
     def read(self):
         self.filename = self._trim(self._resolve_redirects(self.filename))
-
-        with contextlib.closing(urlopen(self.filename, timeout=10)) as response:
+        with contextlib.closing(self.urlopen(self.filename)) as response:
             name = self._suggest_filename(response.headers['content-disposition'])
             with NamedTemporaryFile(suffix=name, delete=False) as f:
                 f.write(response.read())
@@ -828,12 +835,14 @@ def read(self):
 
     def _resolve_redirects(self, url):
         # Resolve (potential) redirects to a final URL
-        with contextlib.closing(urlopen(url, timeout=10)) as response:
+        with contextlib.closing(self.urlopen(url)) as response:
             return response.url
 
-    def _trim(self, url):
+    @classmethod
+    def _trim(cls, url):
         URL_TRIMMERS = (
-            self._trim_googlesheet_url,
+            cls._trim_googlesheet,
+            cls._trim_dropbox,
         )
         for trim in URL_TRIMMERS:
             try:
@@ -844,7 +853,8 @@ def _trim(self, url):
                 break
         return url
 
-    def _trim_googlesheet_url(self, url):
+    @staticmethod
+    def _trim_googlesheet(url):
         match = re.match(r'(?:https?://)?(?:www\.)?'
                          'docs\.google\.com/spreadsheets/d/'
                          '(?P<workbook_id>[-\w_]+)'
@@ -861,6 +871,13 @@ def _trim_googlesheet_url(self, url):
             url += '&gid=' + sheet
         return url
 
+    @staticmethod
+    def _trim_dropbox(url):
+        parts = urlsplit(url)
+        if not parts.netloc.endswith('dropbox.com'):
+            raise ValueError
+        return urlunsplit(parts._replace(query='dl=1'))
+
     def _suggest_filename(self, content_disposition):
         default_name = re.sub(r'[\\:/]', '_', urlparse(self.filename).path)
 

Orange/tests/test_table.py

@@ -1313,8 +1313,6 @@ def test_load_from_url(self):
     class _MockUrlOpen(MagicMock):
         headers = {'content-disposition': 'attachment; filename="Something-FormResponses.tsv"; '
                                           'filename*=UTF-8''Something%20%28Responses%29.tsv'}
-        url = 'https://docs.google.com/spreadsheets/d/ABCD/edit'
-
         def __enter__(self): return self
 
         def __exit__(self, *args, **kwargs): pass
@@ -1327,12 +1325,16 @@ def read(self): return b'''\
     urlopen = _MockUrlOpen()
 
     @patch('Orange.data.io.urlopen', urlopen)
-    def test_google_sheets(self):
-        d = data.Table(self.urlopen.url)
-        self.urlopen.assert_called_with('https://docs.google.com/spreadsheets/d/ABCD/export?format=tsv',
-                                        timeout=10)
-        self.assertEqual(len(d), 2)
-        self.assertEqual(d.name, 'Something-FormResponses')
+    def test_trimmed_urls(self):
+        for url in ('https://docs.google.com/spreadsheets/d/ABCD/edit',
+                    'https://www.dropbox.com/s/ABCD/filename.csv'):
+            self._MockUrlOpen.url = url
+            d = data.Table(url)
+            request = self.urlopen.call_args[0][0]
+            self.assertNotEqual(url, request.full_url)
+            self.assertIn('Mozilla/5.0', request.headers.get('User-agent', ''))
+            self.assertEqual(len(d), 2)
+            self.assertEqual(d.name, 'Something-FormResponses')
 
 
 class CreateTableWithDomain(TableTests):