strip out 'data-yanked' from HTML page with package source URLs served by PyPI (fixes #3301)

Author: boegel

easybuild/tools/filetools.py

@@ -495,7 +495,13 @@ def pypi_source_urls(pkg_name):
         _log.debug("Failed to download %s to determine available PyPI URLs for %s", simple_url, pkg_name)
         res = []
     else:
-        parsed_html = ElementTree.parse(urls_html)
+        urls_txt = read_file(urls_html)
+
+        # strip out data-yanked attributes before parsing HTML
+        # see https://github.com/easybuilders/easybuild-framework/issues/3301
+        urls_txt = re.sub('\s*data-yanked', '', urls_txt)
+
+        parsed_html = ElementTree.ElementTree(ElementTree.fromstring(urls_txt))
         if hasattr(parsed_html, 'iter'):
             res = [a.attrib['href'] for a in parsed_html.iter('a')]
         else:

test/framework/filetools.py

@@ -1303,6 +1303,15 @@ def test_pypi_source_urls(self):
         # more than 50 releases at time of writing test, which always stay there
         self.assertTrue(len(res) > 50)
 
+        # check for Python package that has yanked releases,
+        # see https://github.com/easybuilders/easybuild-framework/issues/3301
+        res = ft.pypi_source_urls('ipython')
+        self.assertTrue(isinstance(res, list) and res)
+        prefix = 'https://pypi.python.org/packages'
+        for entry in res:
+            self.assertTrue(entry.startswith(prefix), "'%s' should start with '%s'" % (entry, prefix))
+            self.assertTrue('ipython' in entry, "Pattern 'ipython' should be found in '%s'" % entry)
+
     def test_derive_alt_pypi_url(self):
         """Test derive_alt_pypi_url() function."""
         url = 'https://pypi.python.org/packages/source/e/easybuild/easybuild-2.7.0.tar.gz'