Improve Pypi description cleanup #2465

Description lines are sometimes padded with 8 spaces. When this happen
these need to be trimmed and this was not taking place correctly

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

src/packagedcode/pypi.py

@@ -365,15 +365,28 @@ def get_description(metainfo, location=None):
 
 def clean_description(description):
     """
-    Return a cleaned description, removing extra leading whitespaces.
+    Return a cleaned description text, removing extra leading whitespaces if
+    needed. Some metadata formats padd each description line with 8 spaces. Some
+    do not. We check first and cleanup if needed.
     """
-    desc_lines = []
-    for line in (description or '').strip().splitlines(False):
-        if line.startswith(' ' * 8):
-            line = line[8:]
-        desc_lines.append(line)
+    # TODO: verify what is the impact of Description-Content-Type: if any
+    description = description or ''
+    description = description.strip()
+    lines = description.splitlines(False)
 
-    return '\n'.join(desc_lines)
+    space_padding = ' ' * 8
+
+    # we need cleaning if any of the first two lines starts with 8 spaces
+    need_cleaning = any(l.startswith(space_padding) for l in lines[:2])
+    if not need_cleaning:
+        return description
+
+    cleaned_lines = [
+        line[8:] if line.startswith(space_padding) else line
+        for line in lines
+    ]
+
+    return '\n'.join(cleaned_lines)
 
 
 def get_legacy_description(location):

tests/packagedcode/data/pypi/unpacked_wheel/metadata-2.0/Jinja2-2.10.dist-info-expected.json

@@ -6,7 +6,7 @@
   "qualifiers": {},
   "subpath": null,
   "primary_language": "Python",
-  "description": "A small but fast and easy to use stand-alone template engine written in pure python.\nJinja2\n~~~~~~\n\nJinja2 is a template engine written in pure Python.  It provides a\n`Django`_ inspired non-XML syntax but supports inline expressions and\nan optional `sandboxed`_ environment.\n\nNutshell\n--------\n\nHere a small example of a Jinja template::\n\n    {% extends 'base.html' %}\n    {% block title %}Memberlist{% endblock %}\n    {% block content %}\n      <ul>\n      {% for user in users %}\n<li><a href=\"{{ user.url }}\">{{ user.username }}</a></li>\n      {% endfor %}\n      </ul>\n    {% endblock %}\n\nPhilosophy\n----------\n\nApplication logic is for the controller but don't try to make the life\nfor the template designer too hard by giving him too few functionality.\n\nFor more informations visit the new `Jinja2 webpage`_ and `documentation`_.\n\n.. _sandboxed: https://en.wikipedia.org/wiki/Sandbox_(computer_security)\n.. _Django: https://www.djangoproject.com/\n.. _Jinja2 webpage: http://jinja.pocoo.org/\n.. _documentation: http://jinja.pocoo.org/2/documentation/",
+  "description": "A small but fast and easy to use stand-alone template engine written in pure python.\nJinja2\n~~~~~~\n\nJinja2 is a template engine written in pure Python.  It provides a\n`Django`_ inspired non-XML syntax but supports inline expressions and\nan optional `sandboxed`_ environment.\n\nNutshell\n--------\n\nHere a small example of a Jinja template::\n\n    {% extends 'base.html' %}\n    {% block title %}Memberlist{% endblock %}\n    {% block content %}\n      <ul>\n      {% for user in users %}\n        <li><a href=\"{{ user.url }}\">{{ user.username }}</a></li>\n      {% endfor %}\n      </ul>\n    {% endblock %}\n\nPhilosophy\n----------\n\nApplication logic is for the controller but don't try to make the life\nfor the template designer too hard by giving him too few functionality.\n\nFor more informations visit the new `Jinja2 webpage`_ and `documentation`_.\n\n.. _sandboxed: https://en.wikipedia.org/wiki/Sandbox_(computer_security)\n.. _Django: https://www.djangoproject.com/\n.. _Jinja2 webpage: http://jinja.pocoo.org/\n.. _documentation: http://jinja.pocoo.org/2/documentation/",
   "release_date": null,
   "parties": [
     {