Merge pull request #3303 from boegel/pypi_yanked_releases

strip out 'data-yanked' from HTML page with package source URLs served by PyPI

Author: lexming

easybuild/tools/filetools.py

@@ -495,7 +495,13 @@ def pypi_source_urls(pkg_name):
         _log.debug("Failed to download %s to determine available PyPI URLs for %s", simple_url, pkg_name)
         res = []
     else:
-        parsed_html = ElementTree.parse(urls_html)
+        urls_txt = read_file(urls_html)
+
+        # ignore yanked releases (see https://pypi.org/help/#yanked)
+        # see https://github.com/easybuilders/easybuild-framework/issues/3301
+        urls_txt = re.sub(r'<a.*?data-yanked.*?</a>', '', urls_txt)
+
+        parsed_html = ElementTree.ElementTree(ElementTree.fromstring(urls_txt))
         if hasattr(parsed_html, 'iter'):
             res = [a.attrib['href'] for a in parsed_html.iter('a')]
         else:

test/framework/filetools.py

@@ -1327,6 +1327,15 @@ def test_pypi_source_urls(self):
         # more than 50 releases at time of writing test, which always stay there
         self.assertTrue(len(res) > 50)
 
+        # check for Python package that has yanked releases,
+        # see https://github.com/easybuilders/easybuild-framework/issues/3301
+        res = ft.pypi_source_urls('ipython')
+        self.assertTrue(isinstance(res, list) and res)
+        prefix = 'https://pypi.python.org/packages'
+        for entry in res:
+            self.assertTrue(entry.startswith(prefix), "'%s' should start with '%s'" % (entry, prefix))
+            self.assertTrue('ipython' in entry, "Pattern 'ipython' should be found in '%s'" % entry)
+
     def test_derive_alt_pypi_url(self):
         """Test derive_alt_pypi_url() function."""
         url = 'https://pypi.python.org/packages/source/e/easybuild/easybuild-2.7.0.tar.gz'