diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index 53ada87ab0..778d043492 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -73,8 +73,8 @@ cocoapods.PodfileLockHandler, cocoapods.PodfileHandler, - conda.CondaYamlHandler, conda.CondaMetaYamlHandler, + conda.CondaYamlHandler, conan.ConanFileHandler, conan.ConanDataHandler, diff --git a/src/packagedcode/conda.py b/src/packagedcode/conda.py index 225940d69f..80b71cbb12 100644 --- a/src/packagedcode/conda.py +++ b/src/packagedcode/conda.py @@ -14,6 +14,7 @@ from packagedcode import models from packagedcode.pypi import BaseDependencyFileHandler +from dparse2.parser import parse_requirement_line """ Handle Conda manifests and metadata, see https://docs.conda.io/en/latest/ @@ -23,18 +24,36 @@ """ # TODO: there are likely other package data files for Conda -# TODO: report platform - class CondaYamlHandler(BaseDependencyFileHandler): - # TODO: there are several other manifests worth adding datasource_id = 'conda_yaml' - path_patterns = ('*conda.yaml', '*conda.yml',) - default_package_type = 'pypi' + path_patterns = ('*conda*.yaml', '*env*.yaml', '*environment*.yaml') + default_package_type = 'conda' default_primary_language = 'Python' description = 'Conda yaml manifest' documentation_url = 'https://docs.conda.io/' + @classmethod + def parse(cls, location, package_only=False): + with open(location) as fi: + conda_data = saneyaml.load(fi.read()) + dependencies = get_conda_yaml_dependencies(conda_data=conda_data) + name = conda_data.get('name') + extra_data = {} + channels = conda_data.get('channels') + if channels: + extra_data['channels'] = channels + if name or dependencies: + package_data = dict( + datasource_id=cls.datasource_id, + type=cls.default_package_type, + name=name, + primary_language=cls.default_primary_language, + dependencies=dependencies, + extra_data=extra_data, + ) + yield models.PackageData.from_data(package_data, package_only) + class CondaMetaYamlHandler(models.DatafileHandler): datasource_id = 'conda_meta_yaml' @@ -83,9 +102,7 @@ def parse(cls, location, package_only=False): metayaml = get_meta_yaml_data(location) package_element = metayaml.get('package') or {} package_name = package_element.get('name') - if not package_name: - return - version = package_element.get('version') + package_version = package_element.get('version') # FIXME: source is source, not download source = metayaml.get('source') or {} @@ -99,6 +116,7 @@ def parse(cls, location, package_only=False): vcs_url = about.get('dev_url') dependencies = [] + extra_data = {} requirements = metayaml.get('requirements') or {} for scope, reqs in requirements.items(): # requirements format is like: @@ -107,14 +125,51 @@ def parse(cls, location, package_only=False): # u'progressbar2', u'python >=3.6'])]) for req in reqs: name, _, requirement = req.partition(" ") - purl = PackageURL(type=cls.default_package_type, name=name) + version = None + if requirement.startswith("=="): + _, version = requirement.split("==") + + # requirements may have namespace, version too + # - conda-forge::numpy=1.15.4 + namespace = None + if "::" in name: + namespace, name = name.split("::") + + is_pinned = False + if "=" in name: + name, version = name.split("=") + is_pinned = True + requirement = f"={version}" + + if name in ('pip', 'python'): + if not scope in extra_data: + extra_data[scope] = [req] + else: + extra_data[scope].append(req) + continue + + purl = PackageURL( + type=cls.default_package_type, + name=name, + namespace=namespace, + version=version, + ) + if "run" in scope: + is_runtime = True + is_optional = False + else: + is_runtime = False + is_optional = True + dependencies.append( models.DependentPackage( purl=purl.to_string(), extracted_requirement=requirement, scope=scope, - is_runtime=True, - is_optional=False, + is_runtime=is_runtime, + is_optional=is_optional, + is_pinned=is_pinned, + is_direct=True, ) ) @@ -122,7 +177,7 @@ def parse(cls, location, package_only=False): datasource_id=cls.datasource_id, type=cls.default_package_type, name=package_name, - version=version, + version=package_version, download_url=download_url, homepage_url=homepage_url, vcs_url=vcs_url, @@ -130,10 +185,92 @@ def parse(cls, location, package_only=False): sha256=sha256, extracted_license_statement=extracted_license_statement, dependencies=dependencies, + extra_data=extra_data, ) yield models.PackageData.from_data(package_data, package_only) +def get_conda_yaml_dependencies(conda_data): + """ + Return a list of DependentPackage mappins from conda and pypi + dependencies present in a `conda_data` mapping. + """ + dependencies = conda_data.get('dependencies') or [] + deps = [] + for dep in dependencies: + if isinstance(dep, str): + namespace = None + specs = None + is_pinned = False + + if "::" in dep: + namespace, dep = dep.split("::") + + req = parse_requirement_line(dep) + if req: + name = req.name + version = None + + specs = str(req.specs) + if '==' in specs: + version = specs.replace('==','') + is_pinned = True + purl = PackageURL(type='pypi', name=name, version=version) + else: + if "=" in dep: + dep, version = dep.split("=") + is_pinned = True + specs = f"={version}" + + purl = PackageURL( + type='conda', + namespace=namespace, + name=dep, + version=version, + ) + + if purl.name in ('pip', 'python'): + continue + + deps.append( + models.DependentPackage( + purl=purl.to_string(), + extracted_requirement=specs, + scope='dependencies', + is_runtime=True, + is_optional=False, + is_pinned=is_pinned, + is_direct=True, + ).to_dict() + ) + + elif isinstance(dep, dict): + for line in dep.get('pip', []): + req = parse_requirement_line(line) + if req: + name = req.name + version = None + is_pinned = False + specs = str(req.specs) + if '==' in specs: + version = specs.replace('==','') + is_pinned = True + purl = PackageURL(type='pypi', name=name, version=version) + deps.append( + models.DependentPackage( + purl=purl.to_string(), + extracted_requirement=specs, + scope='dependencies', + is_runtime=True, + is_optional=False, + is_pinned=is_pinned, + is_direct=True, + ).to_dict() + ) + + return deps + + def get_meta_yaml_data(location): """ Return a mapping of conda metadata loaded from a meta.yaml files. The format @@ -158,10 +295,21 @@ def get_meta_yaml_data(location): # Replace the variable with the value if '{{' in line and '}}' in line: for variable, value in variables.items(): - line = line.replace('{{ ' + variable + ' }}', value) + if "|lower" in line: + line = line.replace('{{ ' + variable + '|lower' + ' }}', value.lower()) + else: + line = line.replace('{{ ' + variable + ' }}', value) yaml_lines.append(line) - return saneyaml.load('\n'.join(yaml_lines)) + # Cleanup any remaining complex jinja template lines + # as the yaml load fails otherwise for unresolved jinja + cleaned_yaml_lines = [ + line + for line in yaml_lines + if not "{{" in line + ] + + return saneyaml.load(''.join(cleaned_yaml_lines)) def get_variables(location): diff --git a/src/packagedcode/models.py b/src/packagedcode/models.py index 40fa8bb3e9..d518c73330 100644 --- a/src/packagedcode/models.py +++ b/src/packagedcode/models.py @@ -1100,7 +1100,16 @@ def is_datafile(cls, location, filetypes=tuple(), _bare_filename=False): """ if filetype.is_file(location) or _bare_filename: loc = as_posixpath(location) - if any(fnmatchcase(loc, pat) for pat in cls.path_patterns): + + # Some extension strings are used interchangebly + extension_aliases = {"yaml": "yml"} + path_patterns = list(cls.path_patterns) + for pattern in cls.path_patterns: + for extension, extension_alias in extension_aliases.items(): + new_pattern = pattern.replace(extension, extension_alias) + path_patterns.append(new_pattern) + + if any(fnmatchcase(loc, pat) for pat in path_patterns): filetypes = filetypes or cls.filetypes if not filetypes: return True diff --git a/src/packagedcode/npm.py b/src/packagedcode/npm.py index f2a8aff6bc..6081f489d1 100644 --- a/src/packagedcode/npm.py +++ b/src/packagedcode/npm.py @@ -114,6 +114,10 @@ def assemble(cls, package_data, resource, codebase, package_adder): yield from models.DatafileHandler.assemble(package_data, resource, codebase, package_adder) return + # We do not have any package data detected here + if not package_resource.package_data: + return + assert len(package_resource.package_data) == 1, f'Invalid package.json for {package_resource.path}' pkg_data = package_resource.package_data[0] pkg_data = models.PackageData.from_dict(pkg_data) diff --git a/tests/packagedcode/data/conda/conda-yaml/phc-gnn/environment_gpu.yml b/tests/packagedcode/data/conda/conda-yaml/phc-gnn/environment_gpu.yml new file mode 100644 index 0000000000..312ca4b8fb --- /dev/null +++ b/tests/packagedcode/data/conda/conda-yaml/phc-gnn/environment_gpu.yml @@ -0,0 +1,40 @@ +# from https://raw.githubusercontent.com/bayer-science-for-a-better-life/phc-gnn/refs/heads/master/environment_gpu.yml + +name: phc-gnn + +channels: + - anaconda + - pytorch + - conda-forge + - defaults + +dependencies: + - pip=20.2.4 + - anaconda::python=3.8.5 + - anaconda::python-dateutil=2.8.1 + - cudatoolkit=10.1 + - magma-cuda101 + - cudnn=7.6.5 + - pytorch=1.7.1 + - torchvision=0.8.2 + - torchaudio=0.7.2 + - conda-forge::numpy=1.19.2 + - anaconda::scipy=1.5.2 + - conda-forge::matplotlib=3.3.2 + - anaconda::networkx=2.5 + - anaconda::scikit-learn=0.23.2 + - anaconda::notebook=6.1.4 + - anaconda::jupyter_client=6.1.7 + - anaconda::jupyter_core=4.6.3 + - anaconda::h5py=2.10.0 + - conda-forge::tqdm=4.50.0 + - conda-forge::tensorboard=2.4.0 + - pip: + - ogb==1.2.4 + - pytest==6.2.1 + - bottleneck==1.3.2 + # - torch-cluster==1.5.8 + # - torch-scatter==2.0.5 + # - torch-sparse==0.6.8 + # - torch-spline-conv==1.2.0 + # - torch-geometric==1.6.1 diff --git a/tests/packagedcode/data/conda/conda-yaml/phc-gnn/environment_gpu.yml-expected.json b/tests/packagedcode/data/conda/conda-yaml/phc-gnn/environment_gpu.yml-expected.json new file mode 100644 index 0000000000..5ccbfdd8c0 --- /dev/null +++ b/tests/packagedcode/data/conda/conda-yaml/phc-gnn/environment_gpu.yml-expected.json @@ -0,0 +1,285 @@ +[ + { + "type": "conda", + "namespace": null, + "name": "phc-gnn", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "channels": [ + "anaconda", + "pytorch", + "conda-forge", + "defaults" + ] + }, + "dependencies": [ + { + "purl": "pkg:conda/anaconda/python-dateutil@2.8.1", + "extracted_requirement": "=2.8.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/cudatoolkit@10.1", + "extracted_requirement": "=10.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/magma-cuda101", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/cudnn@7.6.5", + "extracted_requirement": "=7.6.5", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/pytorch@1.7.1", + "extracted_requirement": "=1.7.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/torchvision@0.8.2", + "extracted_requirement": "=0.8.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/torchaudio@0.7.2", + "extracted_requirement": "=0.7.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/numpy@1.19.2", + "extracted_requirement": "=1.19.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/anaconda/scipy@1.5.2", + "extracted_requirement": "=1.5.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/matplotlib@3.3.2", + "extracted_requirement": "=3.3.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/anaconda/networkx@2.5", + "extracted_requirement": "=2.5", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/anaconda/scikit-learn@0.23.2", + "extracted_requirement": "=0.23.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/anaconda/notebook@6.1.4", + "extracted_requirement": "=6.1.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/anaconda/jupyter_client@6.1.7", + "extracted_requirement": "=6.1.7", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/anaconda/jupyter_core@4.6.3", + "extracted_requirement": "=4.6.3", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/anaconda/h5py@2.10.0", + "extracted_requirement": "=2.10.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/tqdm@4.50.0", + "extracted_requirement": "=4.50.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/tensorboard@2.4.0", + "extracted_requirement": "=2.4.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/ogb@1.2.4", + "extracted_requirement": "==1.2.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pytest@6.2.1", + "extracted_requirement": "==6.2.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/bottleneck@1.3.2", + "extracted_requirement": "==1.3.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "conda_yaml", + "purl": "pkg:conda/phc-gnn" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/conda/conda-yaml/ringer/environment.yaml b/tests/packagedcode/data/conda/conda-yaml/ringer/environment.yaml new file mode 100644 index 0000000000..aacd67479e --- /dev/null +++ b/tests/packagedcode/data/conda/conda-yaml/ringer/environment.yaml @@ -0,0 +1,27 @@ +# https://raw.githubusercontent.com/Genentech/ringer/refs/heads/main/environment.yaml +name: ringer +channels: + - pytorch + - conda-forge + - huggingface +dependencies: + - python=3.8 + - pip + - pytorch::pytorch=1.12 + - huggingface::transformers=4.11.3 + - conda-forge::pytorch-lightning=1.6.4 + - conda-forge::numpy + - conda-forge::scipy + - conda-forge::pandas + - conda-forge::matplotlib + - conda-forge::seaborn + - conda-forge::astropy + - conda-forge::rdkit>=2022.09.5 + - conda-forge::pre-commit + - conda-forge::pytest + - conda-forge::rich + - conda-forge::shellingham + - conda-forge::typer + - conda-forge::wandb + - pip: + - ray diff --git a/tests/packagedcode/data/conda/conda-yaml/ringer/environment.yaml-expected.json b/tests/packagedcode/data/conda/conda-yaml/ringer/environment.yaml-expected.json new file mode 100644 index 0000000000..bcba7bb05c --- /dev/null +++ b/tests/packagedcode/data/conda/conda-yaml/ringer/environment.yaml-expected.json @@ -0,0 +1,240 @@ +[ + { + "type": "conda", + "namespace": null, + "name": "ringer", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "channels": [ + "pytorch", + "conda-forge", + "huggingface" + ] + }, + "dependencies": [ + { + "purl": "pkg:conda/pytorch/pytorch@1.12", + "extracted_requirement": "=1.12", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/huggingface/transformers@4.11.3", + "extracted_requirement": "=4.11.3", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/pytorch-lightning@1.6.4", + "extracted_requirement": "=1.6.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/numpy", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/scipy", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pandas", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/matplotlib", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/seaborn", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/astropy", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/rdkit", + "extracted_requirement": ">=2022.09.5", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pre-commit", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pytest", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/rich", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/shellingham", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/typer", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/wandb", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/ray", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "conda_yaml", + "purl": "pkg:conda/ringer" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/conda/meta.yaml b/tests/packagedcode/data/conda/meta-yaml/abeona/meta.yaml similarity index 100% rename from tests/packagedcode/data/conda/meta.yaml rename to tests/packagedcode/data/conda/meta-yaml/abeona/meta.yaml diff --git a/tests/packagedcode/data/conda/meta.yaml.expected.json b/tests/packagedcode/data/conda/meta-yaml/abeona/meta.yaml-expected.json similarity index 80% rename from tests/packagedcode/data/conda/meta.yaml.expected.json rename to tests/packagedcode/data/conda/meta-yaml/abeona/meta.yaml-expected.json index 3a85a0e3f4..8604df717b 100644 --- a/tests/packagedcode/data/conda/meta.yaml.expected.json +++ b/tests/packagedcode/data/conda/meta-yaml/abeona/meta.yaml-expected.json @@ -58,32 +58,18 @@ "file_references": [], "is_private": false, "is_virtual": false, - "extra_data": {}, + "extra_data": { + "host": [ + "pip", + "python" + ], + "run": [ + "python >=3.6" + ] + }, "dependencies": [ { - "purl": "pkg:conda/pip", - "extracted_requirement": "", - "scope": "host", - "is_runtime": true, - "is_optional": false, - "is_pinned": false, - "is_direct": true, - "resolved_package": {}, - "extra_data": {} - }, - { - "purl": "pkg:conda/python", - "extracted_requirement": "", - "scope": "host", - "is_runtime": true, - "is_optional": false, - "is_pinned": false, - "is_direct": true, - "resolved_package": {}, - "extra_data": {} - }, - { - "purl": "pkg:conda/mccortex", + "purl": "pkg:conda/mccortex@1.0", "extracted_requirement": "==1.0", "scope": "run", "is_runtime": true, @@ -94,7 +80,7 @@ "extra_data": {} }, { - "purl": "pkg:conda/nextflow", + "purl": "pkg:conda/nextflow@19.01.0", "extracted_requirement": "==19.01.0", "scope": "run", "is_runtime": true, @@ -105,7 +91,7 @@ "extra_data": {} }, { - "purl": "pkg:conda/cortexpy", + "purl": "pkg:conda/cortexpy@0.45.7", "extracted_requirement": "==0.45.7", "scope": "run", "is_runtime": true, @@ -116,7 +102,7 @@ "extra_data": {} }, { - "purl": "pkg:conda/kallisto", + "purl": "pkg:conda/kallisto@0.44.0", "extracted_requirement": "==0.44.0", "scope": "run", "is_runtime": true, @@ -158,17 +144,6 @@ "is_direct": true, "resolved_package": {}, "extra_data": {} - }, - { - "purl": "pkg:conda/python", - "extracted_requirement": ">=3.6", - "scope": "run", - "is_runtime": true, - "is_optional": false, - "is_pinned": false, - "is_direct": true, - "resolved_package": {}, - "extra_data": {} } ], "repository_homepage_url": null, diff --git a/tests/packagedcode/data/conda/meta-yaml/gcnvkernel/meta.yaml b/tests/packagedcode/data/conda/meta-yaml/gcnvkernel/meta.yaml new file mode 100644 index 0000000000..a0f60e8c74 --- /dev/null +++ b/tests/packagedcode/data/conda/meta-yaml/gcnvkernel/meta.yaml @@ -0,0 +1,68 @@ +# https://raw.githubusercontent.com/bioconda/bioconda-recipes/refs/heads/master/recipes/gcnvkernel/meta.yaml + +{% set name = "gcnvkernel" %} +{% set version = "0.9" %} +{% set gatk_version = "4.6.1.0" %} +{% set gatk_sha256 = "ac7015c3f0ef1852745ca0ef647adbf8ddef5db63ab485b00bc1ffe654814155" %} + +package: + name: {{ name|lower }} + version: {{ version }} + +source: + url: https://github.com/broadinstitute/gatk/releases/download/{{ gatk_version }}/gatk-{{ gatk_version }}.zip + sha256: {{ gatk_sha256 }} + +build: + noarch: python + number: 0 + preserve_egg_dir: true + run_exports: + - {{ pin_subpackage('gcnvkernel', max_pin="x") }} + +requirements: + build: + - conda-forge::python =3.10.13 + - unzip + run: + # The following are taken from the GATK python environment: + # + # https://github.com/broadinstitute/gatk/blob/master/scripts/gatkcondaenv.yml.template + - conda-forge::python =3.10.13 + - conda-forge::pip =23.3.1 + - conda-forge:blas =1.0=mkl + - conda-forge::numpy =1.26.2 + - conda-forge::pymc =5.10.1 + - conda-forge::pytensor =2.18.3 + - conda-forge::scipy =1.11.4 + - conda-forge::h5py =3.10.0 + - conda-forge::pytorch =2.1.0=*mkl*100 + - conda-forge::pytorch-lightning =2.4.0 + - conda-forge::scikit-learn =1.3.2 + - conda-forge::matplotlib =3.8.2 + - conda-forge::pandas =2.1.3 + - conda-forge::tqdm =4.66.1 + - conda-forge::dill =0.3.7 + - conda-forge::biopython =1.84 + - r-base =4.3.1 + - r-data.table =1.14.8 + - r-dplyr =1.1.3 + - r-getopt =1.20.4 + - r-ggplot2 =3.4.4 + - r-gplots =3.1.3 + - r-gsalib =2.2.1 + - r-optparse =1.7.3 + - r-backports =1.4.1 + - bioconda::pysam =0.22.0 + - conda-forge::pyvcf =0.6.8 + +test: + imports: + - gcnvkernel + +about: + home: https://www.broadinstitute.org/gatk/ + license: BSD-3-Clause + license_family: BSD + summary: Python package to support GATK gCNV calling. + dev_url: https://github.com/broadinstitute/gatk diff --git a/tests/packagedcode/data/conda/meta-yaml/gcnvkernel/meta.yaml-expected.json b/tests/packagedcode/data/conda/meta-yaml/gcnvkernel/meta.yaml-expected.json new file mode 100644 index 0000000000..e5b7b4a76e --- /dev/null +++ b/tests/packagedcode/data/conda/meta-yaml/gcnvkernel/meta.yaml-expected.json @@ -0,0 +1,364 @@ +[ + { + "type": "conda", + "namespace": null, + "name": "gcnvkernel", + "version": "0.9", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": "Python package to support GATK gCNV calling.", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "https://www.broadinstitute.org/gatk/", + "download_url": "https://github.com/broadinstitute/gatk/releases/download/4.6.1.0/gatk-4.6.1.0.zip", + "size": null, + "sha1": null, + "md5": null, + "sha256": "ac7015c3f0ef1852745ca0ef647adbf8ddef5db63ab485b00bc1ffe654814155", + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": "https://github.com/broadinstitute/gatk", + "copyright": null, + "holder": null, + "declared_license_expression": "bsd-new", + "declared_license_expression_spdx": "BSD-3-Clause", + "license_detections": [ + { + "license_expression": "bsd-new", + "license_expression_spdx": "BSD-3-Clause", + "matches": [ + { + "license_expression": "bsd-new", + "license_expression_spdx": "BSD-3-Clause", + "from_file": null, + "start_line": 1, + "end_line": 1, + "matcher": "1-hash", + "score": 100.0, + "matched_length": 3, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "bsd-new_10.RULE", + "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/bsd-new_10.RULE", + "matched_text": "BSD-3-Clause" + } + ], + "identifier": "bsd_new-50fa5753-f24d-ec04-33a1-36bb8ac0492c" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "BSD-3-Clause", + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "build": [ + "conda-forge::python =3.10.13" + ], + "run": [ + "conda-forge::python =3.10.13", + "conda-forge::pip =23.3.1" + ] + }, + "dependencies": [ + { + "purl": "pkg:conda/unzip", + "extracted_requirement": "", + "scope": "build", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge:blas", + "extracted_requirement": "=1.0=mkl", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/numpy", + "extracted_requirement": "=1.26.2", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/pymc", + "extracted_requirement": "=5.10.1", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/pytensor", + "extracted_requirement": "=2.18.3", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/scipy", + "extracted_requirement": "=1.11.4", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/h5py", + "extracted_requirement": "=3.10.0", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/pytorch", + "extracted_requirement": "=2.1.0=*mkl*100", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/pytorch-lightning", + "extracted_requirement": "=2.4.0", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/scikit-learn", + "extracted_requirement": "=1.3.2", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/matplotlib", + "extracted_requirement": "=3.8.2", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/pandas", + "extracted_requirement": "=2.1.3", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/tqdm", + "extracted_requirement": "=4.66.1", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/dill", + "extracted_requirement": "=0.3.7", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/biopython", + "extracted_requirement": "=1.84", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/r-base", + "extracted_requirement": "=4.3.1", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/r-data.table", + "extracted_requirement": "=1.14.8", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/r-dplyr", + "extracted_requirement": "=1.1.3", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/r-getopt", + "extracted_requirement": "=1.20.4", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/r-ggplot2", + "extracted_requirement": "=3.4.4", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/r-gplots", + "extracted_requirement": "=3.1.3", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/r-gsalib", + "extracted_requirement": "=2.2.1", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/r-optparse", + "extracted_requirement": "=1.7.3", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/r-backports", + "extracted_requirement": "=1.4.1", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/bioconda/pysam", + "extracted_requirement": "=0.22.0", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/pyvcf", + "extracted_requirement": "=0.6.8", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "conda_meta_yaml", + "purl": "pkg:conda/gcnvkernel@0.9" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/conda/meta-yaml/pippy/meta.yaml b/tests/packagedcode/data/conda/meta-yaml/pippy/meta.yaml new file mode 100644 index 0000000000..26365ce04a --- /dev/null +++ b/tests/packagedcode/data/conda/meta-yaml/pippy/meta.yaml @@ -0,0 +1,26 @@ +# cmd: conda build . -c auto-mech -c conda-forge +package: + name: pippy + version: "0.1.0" + +source: + path: . + +requirements: + host: + - conda-forge::python=3.7 + run: + - auto-mech::autoio-base + # - conda-forge::python=3.9 + # - conda-forge::numpy=1.21.2 + # - conda-forge::scipy=1.7.1 + - conda-forge::python=3.7 + - conda-forge::numpy=1.15.4 + - conda-forge::scipy=1.5.0 + - conda-forge::pyyaml + - conda-forge::mako + - conda-forge::pytest + - conda-forge::pytest-cov + - conda-forge::future + - conda-forge::flake8 + - conda-forge::pylint diff --git a/tests/packagedcode/data/conda/meta-yaml/pippy/meta.yaml-expected.json b/tests/packagedcode/data/conda/meta-yaml/pippy/meta.yaml-expected.json new file mode 100644 index 0000000000..ab6c9d5351 --- /dev/null +++ b/tests/packagedcode/data/conda/meta-yaml/pippy/meta.yaml-expected.json @@ -0,0 +1,164 @@ +[ + { + "type": "conda", + "namespace": null, + "name": "pippy", + "version": "0.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "host": [ + "conda-forge::python=3.7" + ], + "run": [ + "conda-forge::python=3.7" + ] + }, + "dependencies": [ + { + "purl": "pkg:conda/auto-mech/autoio-base", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/numpy@1.15.4", + "extracted_requirement": "=1.15.4", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/scipy@1.5.0", + "extracted_requirement": "=1.5.0", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/pyyaml", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/mako", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/pytest", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/pytest-cov", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/future", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/flake8", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/pylint", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "conda_meta_yaml", + "purl": "pkg:conda/pippy@0.1.0" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/conda/meta-yaml/pippy/meta.yaml-scancode.json b/tests/packagedcode/data/conda/meta-yaml/pippy/meta.yaml-scancode.json new file mode 100644 index 0000000000..d41b17c441 --- /dev/null +++ b/tests/packagedcode/data/conda/meta-yaml/pippy/meta.yaml-scancode.json @@ -0,0 +1,383 @@ +{ + "packages": [ + { + "type": "conda", + "namespace": null, + "name": "pippy", + "version": "0.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "host": [ + "conda-forge::python=3.7" + ], + "run": [ + "conda-forge::python=3.7" + ] + }, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "package_uid": "pkg:conda/pippy@0.1.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_paths": [ + "meta.yaml" + ], + "datasource_ids": [ + "conda_meta_yaml" + ], + "purl": "pkg:conda/pippy@0.1.0" + } + ], + "dependencies": [ + { + "purl": "pkg:conda/auto-mech/autoio-base", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:conda/auto-mech/autoio-base?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:conda/pippy@0.1.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "meta.yaml", + "datasource_id": "conda_meta_yaml" + }, + { + "purl": "pkg:conda/conda-forge/numpy@1.15.4", + "extracted_requirement": "=1.15.4", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:conda/conda-forge/numpy@1.15.4?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:conda/pippy@0.1.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "meta.yaml", + "datasource_id": "conda_meta_yaml" + }, + { + "purl": "pkg:conda/conda-forge/scipy@1.5.0", + "extracted_requirement": "=1.5.0", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:conda/conda-forge/scipy@1.5.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:conda/pippy@0.1.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "meta.yaml", + "datasource_id": "conda_meta_yaml" + }, + { + "purl": "pkg:conda/conda-forge/pyyaml", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:conda/conda-forge/pyyaml?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:conda/pippy@0.1.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "meta.yaml", + "datasource_id": "conda_meta_yaml" + }, + { + "purl": "pkg:conda/conda-forge/mako", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:conda/conda-forge/mako?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:conda/pippy@0.1.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "meta.yaml", + "datasource_id": "conda_meta_yaml" + }, + { + "purl": "pkg:conda/conda-forge/pytest", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:conda/conda-forge/pytest?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:conda/pippy@0.1.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "meta.yaml", + "datasource_id": "conda_meta_yaml" + }, + { + "purl": "pkg:conda/conda-forge/pytest-cov", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:conda/conda-forge/pytest-cov?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:conda/pippy@0.1.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "meta.yaml", + "datasource_id": "conda_meta_yaml" + }, + { + "purl": "pkg:conda/conda-forge/future", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:conda/conda-forge/future?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:conda/pippy@0.1.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "meta.yaml", + "datasource_id": "conda_meta_yaml" + }, + { + "purl": "pkg:conda/conda-forge/flake8", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:conda/conda-forge/flake8?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:conda/pippy@0.1.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "meta.yaml", + "datasource_id": "conda_meta_yaml" + }, + { + "purl": "pkg:conda/conda-forge/pylint", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {}, + "dependency_uid": "pkg:conda/conda-forge/pylint?uuid=fixed-uid-done-for-testing-5642512d1758", + "for_package_uid": "pkg:conda/pippy@0.1.0?uuid=fixed-uid-done-for-testing-5642512d1758", + "datafile_path": "meta.yaml", + "datasource_id": "conda_meta_yaml" + } + ], + "files": [ + { + "path": "meta.yaml", + "type": "file", + "package_data": [ + { + "type": "conda", + "namespace": null, + "name": "pippy", + "version": "0.1.0", + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "host": [ + "conda-forge::python=3.7" + ], + "run": [ + "conda-forge::python=3.7" + ] + }, + "dependencies": [ + { + "purl": "pkg:conda/auto-mech/autoio-base", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/numpy@1.15.4", + "extracted_requirement": "=1.15.4", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/scipy@1.5.0", + "extracted_requirement": "=1.5.0", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/pyyaml", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/mako", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/pytest", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/pytest-cov", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/future", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/flake8", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:conda/conda-forge/pylint", + "extracted_requirement": "", + "scope": "run", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "conda_meta_yaml", + "purl": "pkg:conda/pippy@0.1.0" + } + ], + "for_packages": [], + "scan_errors": [] + } + ] +} \ No newline at end of file diff --git a/tests/packagedcode/data/misc/conda/scenicplus.yaml b/tests/packagedcode/data/misc/conda/scenicplus.yaml new file mode 100644 index 0000000000..6c4e874946 --- /dev/null +++ b/tests/packagedcode/data/misc/conda/scenicplus.yaml @@ -0,0 +1,101 @@ +# https://raw.githubusercontent.com/KellisLab/benj/refs/heads/main/conda/scenicplus.yml + +name: scenicplus + +channels: + - conda-forge + - bioconda + +dependencies: + - conda-forge::python==3.8 + - conda-forge::pandas>=1.3.5 + - conda-forge::numpy + - conda-forge::tqdm + - conda-forge::ray-core + - conda-forge::attr + - conda-forge::typing + - conda-forge::matplotlib + - conda-forge::scikit-learn>=0.22.2 + # - conda-forge::statistics + - bioconda::pyranges + - conda-forge::scipy + - bioconda::pybiomart + - conda-forge::requests + - bioconda::arboreto>=0.1.6 + - bioconda::gseapy==0.10.8 + - conda-forge::networkx + - conda-forge::seaborn + - bioconda::pyBigWig + - conda-forge::plotly + - conda-forge::adjustText + - bioconda::macs2 + - conda-forge::lxml + - bioconda::tspex + - conda-forge::kaleido-core + - conda-forge::pyvis + - conda-forge::pygam + - conda-forge::scanpy~=1.9 + - conda-forge::cython + - conda-forge::plotnine + - conda-forge::mudata + - conda-forge::sphinx_rtd_theme + - conda-forge::nbsphinx + - conda-forge::nbsphinx-link + - conda-forge::numpydoc + - conda-forge::sphinx-book-theme + - conda-forge::pip + ### pycisTopic: + - conda-forge::python-annoy + - bioconda::bbknn + - conda-forge::ipython + - conda-forge::ipykernel + - conda-forge::gensim + - bioconda::harmonypy + - conda-forge::ipympl + - conda-forge::leidenalg + - conda-forge::pbr ## req for lda, which conda-forge doesn't have correctly + # - conda-forge::lda + # - conda-forge::loompy==3.0.7 ### conda-forge has not updated recently + - conda-forge::pyopenssl + - conda-forge::polars>=0.11.0 + - bioconda::pybedtools + - bioconda::pyfasta + - conda-forge::python-levenshtein + - bioconda::scanorama + - bioconda::scrublet + - conda-forge::statsmodels + - conda-forge::pytables + - conda-forge::umap-learn + - conda-forge::xlrd + - conda-forge::papermill + - conda-forge::bs4 + - conda-forge::pyarrow + ### pyscenic + - conda-forge::cytoolz + - conda-forge::aiohttp + - conda-forge::llvmlite + - conda-forge::numba>=0.51.2 + - conda-forge::attrs + - conda-forge::frozendict + - conda-forge::numexpr + - conda-forge::cloudpickle + - conda-forge::dask + - conda-forge::distributed + - conda-forge::boltons + - conda-forge::pyyaml + - conda-forge::interlap + - conda-forge::fsspec + ## other + - conda-forge::ply + - conda-forge::pandoc + - conda-forge::globre + - conda-forge::numpy_groupies + - pip: + - git+https://github.com/aertslab/pycisTopic@master#egg=pycisTopic + - git+https://github.com/aertslab/pycistarget@master#egg=pycistarget + - git+https://github.com/aertslab/pySCENIC@master#egg=pyscenic + - git+https://github.com/aertslab/LoomXpy@main#egg=loomxpy + - ctxcore>=0.2.0 + - tmtoolkit + - multiprocessing_on_dill + - lda diff --git a/tests/packagedcode/data/misc/conda/scenicplus.yaml-expected.json b/tests/packagedcode/data/misc/conda/scenicplus.yaml-expected.json new file mode 100644 index 0000000000..499a73bf6a --- /dev/null +++ b/tests/packagedcode/data/misc/conda/scenicplus.yaml-expected.json @@ -0,0 +1,921 @@ +[ + { + "type": "conda", + "namespace": null, + "name": "scenicplus", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "Python", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "channels": [ + "conda-forge", + "bioconda" + ] + }, + "dependencies": [ + { + "purl": "pkg:pypi/pandas", + "extracted_requirement": ">=1.3.5", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/numpy", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/tqdm", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/ray-core", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/attr", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/typing", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/matplotlib", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/scikit-learn", + "extracted_requirement": ">=0.22.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pyranges", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/scipy", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pybiomart", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/requests", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/arboreto", + "extracted_requirement": ">=0.1.6", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/gseapy@0.10.8", + "extracted_requirement": "==0.10.8", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": true, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/networkx", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/seaborn", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pybigwig", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/plotly", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/adjusttext", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/macs2", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/lxml", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/tspex", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/kaleido-core", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pyvis", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pygam", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/scanpy", + "extracted_requirement": "~=1.9", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/cython", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/plotnine", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/mudata", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/sphinx-rtd-theme", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/nbsphinx", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/nbsphinx-link", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/numpydoc", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/sphinx-book-theme", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/python-annoy", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/bbknn", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/ipython", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/ipykernel", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/gensim", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/harmonypy", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/ipympl", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/leidenalg", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pbr", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pyopenssl", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/polars", + "extracted_requirement": ">=0.11.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pybedtools", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pyfasta", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/python-levenshtein", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/scanorama", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/scrublet", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/statsmodels", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pytables", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/umap-learn", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/xlrd", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/papermill", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/bs4", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pyarrow", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/cytoolz", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/aiohttp", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/llvmlite", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/numba", + "extracted_requirement": ">=0.51.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/attrs", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/frozendict", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/numexpr", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/cloudpickle", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/dask", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/distributed", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/boltons", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pyyaml", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/interlap", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/fsspec", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/ply", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/pandoc", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/globre", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/numpy-groupies", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/ctxcore", + "extracted_requirement": ">=0.2.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/tmtoolkit", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/multiprocessing-on-dill", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pypi/lda", + "extracted_requirement": "", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "conda_yaml", + "purl": "pkg:conda/scenicplus" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/plugin/conda-package-expected.json b/tests/packagedcode/data/plugin/conda-package-expected.json index 3958d5888d..b0455b31e9 100644 --- a/tests/packagedcode/data/plugin/conda-package-expected.json +++ b/tests/packagedcode/data/plugin/conda-package-expected.json @@ -58,7 +58,14 @@ "source_packages": [], "is_private": false, "is_virtual": false, - "extra_data": {}, + "extra_data": { + "build": [ + "python 3.5.3 1" + ], + "run": [ + "python" + ] + }, "repository_homepage_url": null, "repository_download_url": null, "api_data_url": null, @@ -77,8 +84,8 @@ "purl": "pkg:conda/zlib", "extracted_requirement": "1.2.8 3", "scope": "build", - "is_runtime": true, - "is_optional": false, + "is_runtime": false, + "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, @@ -92,8 +99,8 @@ "purl": "pkg:conda/setuptools", "extracted_requirement": "27.2.0 py35_0", "scope": "build", - "is_runtime": true, - "is_optional": false, + "is_runtime": false, + "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, @@ -107,8 +114,8 @@ "purl": "pkg:conda/sqlite", "extracted_requirement": "3.13.0 0", "scope": "build", - "is_runtime": true, - "is_optional": false, + "is_runtime": false, + "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, @@ -122,8 +129,8 @@ "purl": "pkg:conda/openssl", "extracted_requirement": "1.0.2k 1", "scope": "build", - "is_runtime": true, - "is_optional": false, + "is_runtime": false, + "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, @@ -137,8 +144,8 @@ "purl": "pkg:conda/xz", "extracted_requirement": "5.2.2 1", "scope": "build", - "is_runtime": true, - "is_optional": false, + "is_runtime": false, + "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, @@ -152,8 +159,8 @@ "purl": "pkg:conda/krb5", "extracted_requirement": "1.13.2 0", "scope": "build", - "is_runtime": true, - "is_optional": false, + "is_runtime": false, + "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, @@ -167,8 +174,8 @@ "purl": "pkg:conda/pykerberos", "extracted_requirement": "1.1.10 py35_0", "scope": "build", - "is_runtime": true, - "is_optional": false, + "is_runtime": false, + "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, @@ -178,27 +185,12 @@ "datafile_path": "info/recipe.tar-extract/recipe/meta.yaml", "datasource_id": "conda_meta_yaml" }, - { - "purl": "pkg:conda/python", - "extracted_requirement": "3.5.3 1", - "scope": "build", - "is_runtime": true, - "is_optional": false, - "is_pinned": false, - "is_direct": true, - "resolved_package": {}, - "extra_data": {}, - "dependency_uid": "pkg:conda/python?uuid=fixed-uid-done-for-testing-5642512d1758", - "for_package_uid": "pkg:conda/requests-kerberos@0.8.0?uuid=fixed-uid-done-for-testing-5642512d1758", - "datafile_path": "info/recipe.tar-extract/recipe/meta.yaml", - "datasource_id": "conda_meta_yaml" - }, { "purl": "pkg:conda/requests", "extracted_requirement": "2.13.0 py35_0", "scope": "build", - "is_runtime": true, - "is_optional": false, + "is_runtime": false, + "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, @@ -208,21 +200,6 @@ "datafile_path": "info/recipe.tar-extract/recipe/meta.yaml", "datasource_id": "conda_meta_yaml" }, - { - "purl": "pkg:conda/python", - "extracted_requirement": "", - "scope": "run", - "is_runtime": true, - "is_optional": false, - "is_pinned": false, - "is_direct": true, - "resolved_package": {}, - "extra_data": {}, - "dependency_uid": "pkg:conda/python?uuid=fixed-uid-done-for-testing-5642512d1758", - "for_package_uid": "pkg:conda/requests-kerberos@0.8.0?uuid=fixed-uid-done-for-testing-5642512d1758", - "datafile_path": "info/recipe.tar-extract/recipe/meta.yaml", - "datasource_id": "conda_meta_yaml" - }, { "purl": "pkg:conda/requests", "extracted_requirement": ">=1.1.0", @@ -345,14 +322,21 @@ "file_references": [], "is_private": false, "is_virtual": false, - "extra_data": {}, + "extra_data": { + "build": [ + "python 3.5.3 1" + ], + "run": [ + "python" + ] + }, "dependencies": [ { "purl": "pkg:conda/zlib", "extracted_requirement": "1.2.8 3", "scope": "build", - "is_runtime": true, - "is_optional": false, + "is_runtime": false, + "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, @@ -362,8 +346,8 @@ "purl": "pkg:conda/setuptools", "extracted_requirement": "27.2.0 py35_0", "scope": "build", - "is_runtime": true, - "is_optional": false, + "is_runtime": false, + "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, @@ -373,8 +357,8 @@ "purl": "pkg:conda/sqlite", "extracted_requirement": "3.13.0 0", "scope": "build", - "is_runtime": true, - "is_optional": false, + "is_runtime": false, + "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, @@ -384,8 +368,8 @@ "purl": "pkg:conda/openssl", "extracted_requirement": "1.0.2k 1", "scope": "build", - "is_runtime": true, - "is_optional": false, + "is_runtime": false, + "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, @@ -395,8 +379,8 @@ "purl": "pkg:conda/xz", "extracted_requirement": "5.2.2 1", "scope": "build", - "is_runtime": true, - "is_optional": false, + "is_runtime": false, + "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, @@ -406,8 +390,8 @@ "purl": "pkg:conda/krb5", "extracted_requirement": "1.13.2 0", "scope": "build", - "is_runtime": true, - "is_optional": false, + "is_runtime": false, + "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, @@ -417,19 +401,8 @@ "purl": "pkg:conda/pykerberos", "extracted_requirement": "1.1.10 py35_0", "scope": "build", - "is_runtime": true, - "is_optional": false, - "is_pinned": false, - "is_direct": true, - "resolved_package": {}, - "extra_data": {} - }, - { - "purl": "pkg:conda/python", - "extracted_requirement": "3.5.3 1", - "scope": "build", - "is_runtime": true, - "is_optional": false, + "is_runtime": false, + "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, @@ -439,19 +412,8 @@ "purl": "pkg:conda/requests", "extracted_requirement": "2.13.0 py35_0", "scope": "build", - "is_runtime": true, - "is_optional": false, - "is_pinned": false, - "is_direct": true, - "resolved_package": {}, - "extra_data": {} - }, - { - "purl": "pkg:conda/python", - "extracted_requirement": "", - "scope": "run", - "is_runtime": true, - "is_optional": false, + "is_runtime": false, + "is_optional": true, "is_pinned": false, "is_direct": true, "resolved_package": {}, diff --git a/tests/packagedcode/data/plugin/plugins_list_linux.txt b/tests/packagedcode/data/plugin/plugins_list_linux.txt index 9468fab2cc..33886e8722 100755 --- a/tests/packagedcode/data/plugin/plugins_list_linux.txt +++ b/tests/packagedcode/data/plugin/plugins_list_linux.txt @@ -202,6 +202,13 @@ Package type: conda description: Conda meta.yml manifest path_patterns: '*/meta.yaml' -------------------------------------------- +Package type: conda + datasource_id: conda_yaml + documentation URL: https://docs.conda.io/ + primary language: Python + description: Conda yaml manifest + path_patterns: '*conda*.yaml', '*env*.yaml', '*environment*.yaml' +-------------------------------------------- Package type: cpan datasource_id: cpan_dist_ini documentation URL: https://metacpan.org/pod/Dist::Zilla::Tutorial @@ -685,13 +692,6 @@ Package type: pubspec description: Dart pubspec manifest path_patterns: '*pubspec.yaml' -------------------------------------------- -Package type: pypi - datasource_id: conda_yaml - documentation URL: https://docs.conda.io/ - primary language: Python - description: Conda yaml manifest - path_patterns: '*conda.yaml', '*conda.yml' --------------------------------------------- Package type: pypi datasource_id: pip_requirements documentation URL: https://pip.pypa.io/en/latest/reference/requirements-file-format/ diff --git a/tests/packagedcode/test_conda.py b/tests/packagedcode/test_conda.py index 1b50e49679..05df09149a 100644 --- a/tests/packagedcode/test_conda.py +++ b/tests/packagedcode/test_conda.py @@ -14,14 +14,16 @@ from packages_test_utils import PackageTester from packagedcode import conda from scancode_config import REGEN_TEST_FIXTURES +from scancode.cli_test_utils import run_scan_click +from scancode.cli_test_utils import check_json_scan class TestConda(PackageTester): test_data_dir = os.path.join(os.path.dirname(__file__), 'data') - def test_parse_get_varialble(self): - test_file = self.get_test_loc('conda/meta.yaml') + def test_parse_get_variables(self): + test_file = self.get_test_loc('conda/meta-yaml/abeona/meta.yaml') results = conda.get_variables(test_file) expected = { 'version': '0.45.0', @@ -29,25 +31,85 @@ def test_parse_get_varialble(self): } assert results == expected + def test_parse_get_variables_2(self): + test_file = self.get_test_loc('conda/meta-yaml/gcnvkernel/meta.yaml') + results = conda.get_variables(test_file) + expected = { + 'version': '0.9', + 'name': 'gcnvkernel', + 'gatk_version': '4.6.1.0', + 'gatk_sha256': 'ac7015c3f0ef1852745ca0ef647adbf8ddef5db63ab485b00bc1ffe654814155', + } + assert results == expected + def test_get_meta_yaml_data(self): - test_file = self.get_test_loc('conda/meta.yaml') + test_file = self.get_test_loc('conda/meta-yaml/abeona/meta.yaml') results = conda.get_meta_yaml_data(test_file) assert list(results.items())[0] == (u'package', dict([(u'name', u'abeona'), (u'version', u'0.45.0')])) - def test_condayml_is_package_data_file(self): - test_file = self.get_test_loc('conda/meta.yaml') + def test_conda_meta_yml_is_package_data_file(self): + test_file = self.get_test_loc('conda/meta-yaml/abeona/meta.yaml') assert conda.CondaMetaYamlHandler.is_datafile(test_file) - def test_parse(self): - test_file = self.get_test_loc('conda/meta.yaml') + def test_parse_conda_meta_yaml_with_templates_and_about(self): + test_file = self.get_test_loc('conda/meta-yaml/abeona/meta.yaml') package = conda.CondaMetaYamlHandler.parse(test_file) - expected_loc = self.get_test_loc('conda/meta.yaml.expected.json') + expected_loc = self.get_test_loc('conda/meta-yaml/abeona/meta.yaml-expected.json') self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) - def test_root_dir(self): + def test_conda_meta_yaml_root_dir(self): test_path = 'requests-kerberos-0.8.0-py35_0.tar.bz2-extract/info/recipe.tar-extract/recipe/meta.yaml' test_dir = self.get_test_loc('conda/requests-kerberos-0.8.0-py35_0.tar.bz2-extract') codebase = Codebase(test_dir) resource = codebase.get_resource(path=test_path) proot = conda.CondaMetaYamlHandler.get_conda_root(resource, codebase) assert proot.location == test_dir + + def test_conda_yml_is_package_data_file_with_conda_dir(self): + test_file = self.get_test_loc('misc/conda/scenicplus.yaml') + assert conda.CondaYamlHandler.is_datafile(test_file) + + def test_conda_yml_is_package_data_file_environment(self): + test_file = self.get_test_loc('conda/conda-yaml/phc-gnn/environment_gpu.yml') + assert conda.CondaYamlHandler.is_datafile(test_file) + + def test_parse_conda_meta_yaml_package_from_non_default_channels(self): + test_file = self.get_test_loc('conda/meta-yaml/pippy/meta.yaml') + package = conda.CondaMetaYamlHandler.parse(test_file) + expected_loc = self.get_test_loc('conda/meta-yaml/pippy/meta.yaml-expected.json') + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + + def test_parse_conda_meta_yaml_package_and_assemble(self): + test_file = self.get_test_loc('conda/meta-yaml/pippy/meta.yaml') + result_file = self.get_temp_file('results.json') + run_scan_click(['--package', test_file, '--json', result_file]) + expected_file = self.get_test_loc('conda/meta-yaml/pippy/meta.yaml-scancode.json') + check_json_scan( + expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES + ) + + def test_parse_simple_conda_yaml(self): + test_file = self.get_test_loc('conda/conda-yaml/phc-gnn/environment_gpu.yml') + assert conda.CondaYamlHandler.is_datafile(test_file) + + package = conda.CondaYamlHandler.parse(test_file) + expected_loc = self.get_test_loc('conda/conda-yaml/phc-gnn/environment_gpu.yml-expected.json') + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + + def test_parse_conda_yaml_with_pip_source_repos(self): + test_file = self.get_test_loc('misc/conda/scenicplus.yaml') + package = conda.CondaYamlHandler.parse(test_file) + expected_loc = self.get_test_loc('misc/conda/scenicplus.yaml-expected.json') + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + + def test_parse_conda_meta_yaml_with_multiple_templates(self): + test_file = self.get_test_loc('conda/meta-yaml/gcnvkernel/meta.yaml') + package = conda.CondaMetaYamlHandler.parse(test_file) + expected_loc = self.get_test_loc('conda/meta-yaml/gcnvkernel/meta.yaml-expected.json') + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES) + + def test_parse_conda_yaml_simple_dependencies(self): + test_file = self.get_test_loc('conda/conda-yaml/ringer/environment.yaml') + package = conda.CondaYamlHandler.parse(test_file) + expected_loc = self.get_test_loc('conda/conda-yaml/ringer/environment.yaml-expected.json') + self.check_packages_data(package, expected_loc, regen=REGEN_TEST_FIXTURES)