Skip to content

Commit 3b813f0

Browse files
pombredannepombredanne
committed
Support Pypi nameless packages #2465
Improve tests and ensure we can handle nameless PyPI packages. Signed-off-by: Philippe Ombredanne <[email protected]>
1 parent 42e367f commit 3b813f0

File tree

9 files changed

+157
-77
lines changed

9 files changed

+157
-77
lines changed

‎src/packagedcode/pypi.py‎

Lines changed: 27 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
12
#
23
# Copyright (c) nexB Inc. and others. All rights reserved.
34
# ScanCode is a trademark of nexB Inc.
@@ -129,6 +130,7 @@ def parse(location):
129130
parse_sdist,
130131
)
131132

133+
# FIXME: this does not make sense
132134
# try all available parser in a well defined order
133135
for parser in parsers:
134136
package = parser(location)
@@ -192,22 +194,13 @@ def parse_archive(location):
192194
if not location or not location.endswith(bdist_file_suffixes):
193195
return
194196

195-
metafile = find_archive_metafile(location)
196-
if metafile:
197-
return parse_metadata(metafile)
198-
199-
200-
def find_archive_metafile(location):
201-
"""
202-
Return a Path-like object to a Python metafile found in a Python package egg
203-
or wheel archive at ``location`` or None.
204-
"""
205-
zf = zipfile.ZipFile(location)
206-
for path in ZipPath(zf).iterdir():
207-
if path.name.endswith(meta_dir_suffixes):
197+
with zipfile.ZipFile(location) as zf:
198+
for path in ZipPath(zf).iterdir():
199+
if not path.name.endswith(meta_dir_suffixes):
200+
continue
208201
for metapath in path.iterdir():
209202
if metapath.name.endswith(meta_file_names):
210-
return metapath
203+
return parse_metadata(metapath)
211204

212205

213206
sdist_file_suffixes = '.tar.gz', '.tar.bz2', '.zip',
@@ -247,11 +240,9 @@ def parse_setup_py(location):
247240

248241
setup_args = get_setup_py_args(location)
249242

250-
# FIXME: it may be legit to have a name-less package?
243+
# it may be legit to have a name-less package?
244+
# in anycase we do not want to fail because of that
251245
package_name = setup_args.get('name')
252-
if not package_name:
253-
return
254-
255246
urls, other_urls = get_urls(setup_args)
256247

257248
detected_version = setup_args.get('version')
@@ -359,18 +350,32 @@ def get_description(metainfo, location=None):
359350
# newer metadata versions use the payload for the description
360351
if hasattr(metainfo, 'get_payload'):
361352
description = metainfo.get_payload()
353+
description = description and description.strip() or None
362354
if not description:
363355
# legacymetadata versions use the Description for the description
364356
description = get_attribute(metainfo, 'Description')
365357
if not description and location:
366358
# older metadata versions can use a DESCRIPTION.rst file
367-
description = get_legacy_description(
368-
fileutils.parent_directory(location))
359+
description = get_legacy_description(location=fileutils.parent_directory(location))
369360

370361
summary = get_attribute(metainfo, 'Summary')
362+
description = clean_description(description)
371363
return build_description(summary, description)
372364

373365

366+
def clean_description(description):
367+
"""
368+
Return a cleaned description, removing extra leading whitespaces.
369+
"""
370+
desc_lines = []
371+
for line in (description or '').strip().splitlines(False):
372+
if line.startswith(' ' * 8):
373+
line = line[8:]
374+
desc_lines.append(line)
375+
376+
return '\n'.join(desc_lines)
377+
378+
374379
def get_legacy_description(location):
375380
"""
376381
Return the text of a legacy DESCRIPTION.rst.
@@ -732,7 +737,8 @@ def get_setup_py_args(location):
732737
for statement in tree.body:
733738
# We only care about function calls or assignments to functions named
734739
# `setup` or `main`
735-
if not (isinstance(statement, (ast.Expr, ast.Call, ast.Assign))
740+
if not (
741+
isinstance(statement, (ast.Expr, ast.Call, ast.Assign))
736742
and isinstance(statement.value, ast.Call)
737743
and isinstance(statement.value.func, ast.Name)
738744
# we also look for main as sometimes this is used instead of setup()

tests/packagedcode/data/pypi/setup.py/with_name.py renamed to tests/packagedcode/data/pypi/setup.py/with_name-setup.py

File renamed without changes.

‎tests/packagedcode/data/pypi/setup.py/with_name-setup.py.args.expected.json‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
{
2+
"name": "arpy",
3+
"version": "0.1.1",
4+
"description": "Library for accessing \"ar\" files",
5+
"author": "Stanis\u0142aw Pitucha",
6+
"author_email": "[email protected]",
7+
"url": "http://bitbucket.org/viraptor/arpy",
8+
"py_modules": [
9+
"arpy"
10+
],
11+
"license": "Simplified BSD"
12+
}

tests/packagedcode/data/pypi/setup.py/with_name-expected.json renamed to tests/packagedcode/data/pypi/setup.py/with_name-setup.py.expected.json

Lines changed: 46 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -1,46 +1,46 @@
1-
{
2-
"type": "pypi",
3-
"namespace": null,
4-
"name": "arpy",
5-
"version": "0.1.1",
6-
"qualifiers": {},
7-
"subpath": null,
8-
"primary_language": "Python",
9-
"description": "Library for accessing \"ar\" files",
10-
"release_date": null,
11-
"parties": [
12-
{
13-
"type": "person",
14-
"role": "author",
15-
"name": "Stanis\u0142aw Pitucha",
16-
"email": "[email protected]",
17-
"url": "http://bitbucket.org/viraptor/arpy"
18-
}
19-
],
20-
"keywords": [],
21-
"homepage_url": "http://bitbucket.org/viraptor/arpy",
22-
"download_url": null,
23-
"size": null,
24-
"sha1": null,
25-
"md5": null,
26-
"sha256": null,
27-
"sha512": null,
28-
"bug_tracking_url": null,
29-
"code_view_url": null,
30-
"vcs_url": null,
31-
"copyright": null,
32-
"license_expression": "bsd-simplified",
33-
"declared_license": {
34-
"license": "Simplified BSD",
35-
"classifiers": []
36-
},
37-
"notice_text": null,
38-
"root_path": null,
39-
"dependencies": [],
40-
"contains_source_code": null,
41-
"source_packages": [],
42-
"purl": "pkg:pypi/[email protected]",
43-
"repository_homepage_url": "https://pypi.org/project/arpy",
44-
"repository_download_url": "https://pypi.io/packages/source/a/arpy/arpy-0.1.1.tar.gz",
45-
"api_data_url": "http://pypi.python.org/pypi/arpy/0.1.1/json"
46-
}
1+
{
2+
"type": "pypi",
3+
"namespace": null,
4+
"name": "arpy",
5+
"version": "0.1.1",
6+
"qualifiers": {},
7+
"subpath": null,
8+
"primary_language": "Python",
9+
"description": "Library for accessing \"ar\" files",
10+
"release_date": null,
11+
"parties": [
12+
{
13+
"type": "person",
14+
"role": "author",
15+
"name": "Stanis\u0142aw Pitucha",
16+
"email": null,
17+
"url": null
18+
}
19+
],
20+
"keywords": [],
21+
"homepage_url": "http://bitbucket.org/viraptor/arpy",
22+
"download_url": null,
23+
"size": null,
24+
"sha1": null,
25+
"md5": null,
26+
"sha256": null,
27+
"sha512": null,
28+
"bug_tracking_url": null,
29+
"code_view_url": null,
30+
"vcs_url": null,
31+
"copyright": null,
32+
"license_expression": "bsd-simplified",
33+
"declared_license": {
34+
"license": "Simplified BSD"
35+
},
36+
"notice_text": null,
37+
"root_path": null,
38+
"dependencies": [],
39+
"contains_source_code": null,
40+
"source_packages": [],
41+
"extra_data": {},
42+
"purl": "pkg:pypi/[email protected]",
43+
"repository_homepage_url": "https://pypi.org/project/https://pypi.org",
44+
"repository_download_url": "https://pypi.org/packages/source/a/arpy/arpy-0.1.1.tar.gz",
45+
"api_data_url": "https://pypi.org/pypi/arpy/0.1.1/json"
46+
}

tests/packagedcode/data/pypi/setup.py/without_name.py renamed to tests/packagedcode/data/pypi/setup.py/without_name-setup.py

File renamed without changes.

‎tests/packagedcode/data/pypi/setup.py/without_name-setup.py.args.expected.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"version": "1.0.0",
3+
"description": "A sample package",
4+
"author": "Test",
5+
"author_email": "[email protected]",
6+
"package_requires": []
7+
}

‎tests/packagedcode/data/pypi/setup.py/without_name-setup.py.expected.json‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
{
2+
"type": "pypi",
3+
"namespace": null,
4+
"name": null,
5+
"version": "1.0.0",
6+
"qualifiers": {},
7+
"subpath": null,
8+
"primary_language": "Python",
9+
"description": "A sample package",
10+
"release_date": null,
11+
"parties": [
12+
{
13+
"type": "person",
14+
"role": "author",
15+
"name": "Test",
16+
"email": null,
17+
"url": null
18+
}
19+
],
20+
"keywords": [],
21+
"homepage_url": null,
22+
"download_url": null,
23+
"size": null,
24+
"sha1": null,
25+
"md5": null,
26+
"sha256": null,
27+
"sha512": null,
28+
"bug_tracking_url": null,
29+
"code_view_url": null,
30+
"vcs_url": null,
31+
"copyright": null,
32+
"license_expression": null,
33+
"declared_license": {},
34+
"notice_text": null,
35+
"root_path": null,
36+
"dependencies": [],
37+
"contains_source_code": null,
38+
"source_packages": [],
39+
"extra_data": {},
40+
"purl": null,
41+
"repository_homepage_url": null,
42+
"repository_download_url": null,
43+
"api_data_url": null
44+
}

‎tests/packagedcode/data/pypi/unpacked_wheel/metadata-2.0/Jinja2-2.10.dist-info-expected.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
"qualifiers": {},
77
"subpath": null,
88
"primary_language": "Python",
9-
"description": "A small but fast and easy to use stand-alone template engine written in pure python.\nJinja2\n~~~~~~\n\nJinja2 is a template engine written in pure Python. It provides a\n`Django`_ inspired non-XML syntax but supports inline expressions and\nan optional `sandboxed`_ environment.\n\nNutshell\n--------\n\nHere a small example of a Jinja template::\n\n {% extends 'base.html' %}\n {% block title %}Memberlist{% endblock %}\n {% block content %}\n <ul>\n {% for user in users %}\n <li><a href=\"{{ user.url }}\">{{ user.username }}</a></li>\n {% endfor %}\n </ul>\n {% endblock %}\n\nPhilosophy\n----------\n\nApplication logic is for the controller but don't try to make the life\nfor the template designer too hard by giving him too few functionality.\n\nFor more informations visit the new `Jinja2 webpage`_ and `documentation`_.\n\n.. _sandboxed: https://en.wikipedia.org/wiki/Sandbox_(computer_security)\n.. _Django: https://www.djangoproject.com/\n.. _Jinja2 webpage: http://jinja.pocoo.org/\n.. _documentation: http://jinja.pocoo.org/2/documentation/",
9+
"description": "A small but fast and easy to use stand-alone template engine written in pure python.\nJinja2\n~~~~~~\n\nJinja2 is a template engine written in pure Python. It provides a\n`Django`_ inspired non-XML syntax but supports inline expressions and\nan optional `sandboxed`_ environment.\n\nNutshell\n--------\n\nHere a small example of a Jinja template::\n\n {% extends 'base.html' %}\n {% block title %}Memberlist{% endblock %}\n {% block content %}\n <ul>\n {% for user in users %}\n<li><a href=\"{{ user.url }}\">{{ user.username }}</a></li>\n {% endfor %}\n </ul>\n {% endblock %}\n\nPhilosophy\n----------\n\nApplication logic is for the controller but don't try to make the life\nfor the template designer too hard by giving him too few functionality.\n\nFor more informations visit the new `Jinja2 webpage`_ and `documentation`_.\n\n.. _sandboxed: https://en.wikipedia.org/wiki/Sandbox_(computer_security)\n.. _Django: https://www.djangoproject.com/\n.. _Jinja2 webpage: http://jinja.pocoo.org/\n.. _documentation: http://jinja.pocoo.org/2/documentation/",
1010
"release_date": null,
1111
"parties": [
1212
{

‎tests/packagedcode/test_pypi.py‎

Lines changed: 20 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -173,17 +173,28 @@ def test_parse_metadata_unpacked_sdist_metadata_v10_subdir(self):
173173
self.check_package(package, expected_loc, regen=False)
174174

175175
def test_parse_setup_py_with_name(self):
176-
test_file = self.get_test_loc('pypi/setup.py/with_name.py')
177-
packages = pypi.parse_setup_py(test_file)
178-
expected_loc = self.get_test_loc('pypi/setup.py/with_name.py.expected')
179-
self.check_packages(packages, expected_loc, regen=False)
176+
test_file = self.get_test_loc('pypi/setup.py/with_name-setup.py')
177+
package = pypi.parse_setup_py(test_file)
178+
expected_loc = self.get_test_loc('pypi/setup.py/with_name-setup.py.expected.json', must_exist=False)
179+
self.check_package(package, expected_loc, regen=False)
180180

181181
def test_parse_setup_py_without_name(self):
182-
test_file = self.get_test_loc('pypi/setup.py/without_name.py')
183-
try:
184-
pypi.parse_setup_py(test_file)
185-
except AttributeError as e:
186-
assert "'NoneType' object has no attribute 'to_dict'" in str(e)
182+
test_file = self.get_test_loc('pypi/setup.py/without_name-setup.py')
183+
package = pypi.parse_setup_py(test_file)
184+
expected_loc = self.get_test_loc('pypi/setup.py/without_name-setup.py.expected.json', must_exist=False)
185+
self.check_package(package, expected_loc, regen=False)
186+
187+
def test_get_setup_py_args_with_name(self):
188+
test_file = self.get_test_loc('pypi/setup.py/with_name-setup.py')
189+
kwargs = pypi.get_setup_py_args(test_file)
190+
expected_loc = self.get_test_loc('pypi/setup.py/with_name-setup.py.args.expected.json', must_exist=False)
191+
check_result_equals_expected_json(kwargs, expected_loc, regen=False)
192+
193+
def test_get_setup_py_args_without_name(self):
194+
test_file = self.get_test_loc('pypi/setup.py/without_name-setup.py')
195+
kwargs = pypi.get_setup_py_args(test_file)
196+
expected_loc = self.get_test_loc('pypi/setup.py/without_name-setup.py.args.expected.json', must_exist=False)
197+
check_result_equals_expected_json(kwargs, expected_loc, regen=False)
187198

188199
def test_parse_metadata_unpacked_sdist_metadata_v11_1(self):
189200
test_file = self.get_test_loc('pypi/unpacked_sdist/metadata-1.1/python-mimeparse-1.6.0/PKG-INFO')

0 commit comments

Comments
 (0)