diff --git a/setup.cfg b/setup.cfg index 5f88b4a1ff..d6dae21100 100644 --- a/setup.cfg +++ b/setup.cfg @@ -79,6 +79,7 @@ install_requires = license_expression >= 21.6.14 lxml >= 4.6.5 MarkupSafe >= 1.0 + oelint_parser >= 1.1.0 packageurl_python >= 0.9.0 packaging >= 21.0.0 # parameter_expansion >= 0.2.0 diff --git a/src/packagedcode/__init__.py b/src/packagedcode/__init__.py index 3eace113a7..d9174ab5f3 100644 --- a/src/packagedcode/__init__.py +++ b/src/packagedcode/__init__.py @@ -10,6 +10,7 @@ import attr from packagedcode import about +from packagedcode import bitbake from packagedcode import bower from packagedcode import build from packagedcode import build_gradle @@ -55,6 +56,7 @@ models.Axis2Mar, about.Aboutfile, + bitbake.BitbakeBbClassManifest, npm.PackageJson, npm.PackageLockJson, npm.YarnLockJson, diff --git a/src/packagedcode/bitbake.py b/src/packagedcode/bitbake.py new file mode 100644 index 0000000000..ac524a4906 --- /dev/null +++ b/src/packagedcode/bitbake.py @@ -0,0 +1,128 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import logging + +import attr +from oelint_parser.cls_stash import Stash +from oelint_parser.cls_item import Variable + +from commoncode import filetype +from packagedcode import models + +TRACE = False + +logger = logging.getLogger(__name__) + +if TRACE: + import sys + logging.basicConfig(stream=sys.stdout) + logger.setLevel(logging.DEBUG) + + +@attr.s() +class BitbakePacakge(models.Package): + default_type = 'bitbake' + + @classmethod + def get_package_root(cls, manifest_resource, codebase): + return manifest_resource.parent(codebase) + +@attr.s() +class BitbakeBbManifest(BitbakePacakge, models.PackageManifest): + file_patterns = ('*.bb',) + extensions = ('.bb', '.bbclass',) + + @classmethod + def is_manifest(cls, location): + return ( + filetype.is_file(location) + and location.lower().endswith(('.bb',)) + ) + + @classmethod + def recognize(cls, location): + yield parse(location) + + +def parse(location): + """ + Return a Package object from an BitBake file or None. + """ + _stash = Stash() + # add any bitbake like file + _stash.AddFile(location) + + # Resolves proper cross file dependencies + _stash.Finalize() + + # get all variables from all prsed files + data = {} + for item in _stash.GetItemsFor(): + try: + # Create a package dictionary with VarName as the key and + # VarValueStripped as the value + name = item.VarName + value = item.VarValueStripped + try: + if data[name]: + data[name] += '\n' + value + except: + data[name] = value + except: + # Continue to see if there is any VarName value until the end of + # the file + continue + + return build_package(data) + + +def build_package(data): + """ + Return a Package built from Bake `data` mapping. + """ + name = data.get('PN') + version = data.get('PV') + description = data.get('DESCRIPTION') + homepage_url = data.get('HOMEPAGE') + download_url = data.get('PREMIRRORS') + declared_license = data.get('LICENSE') + dependencies = data.get('DEPENDS', []) + + # The item.VarName for SRC_URI[*] from the parser are SRC_URI + # Therefore, I cannot differentiate md5, sha1, or src file location reference + # See: https://github.com/priv-kweihmann/oelint-parser/issues/3 + sha1 = data.get('SRC_URI[sha1sum]') + md5 = data.get('SRC_URI[md5sum]') + sha256 = data.get('SRC_URI[sha256sum]') + sha512 = data.get('SRC_URI[sha512sum]') + + # There are some RDEPENDS_* fields such as "RDEPENDS_${PN}" which I need to + # check with the substring + # FIXME: we should create a DependentPackage + for d in data: + if d.startswith('RDEPENDS'): + if dependencies: + dependencies += '\n' + data[d] + else: + dependencies = data[d] + + return BitbakeBbManifest( + name=name, + version=version, + description=description, + homepage_url=homepage_url, + download_url=download_url, + sha1=sha1, + md5=md5, + sha256=sha256, + sha512=sha512, + declared_license=declared_license, + dependencies=dependencies + ) diff --git a/tests/packagedcode/data/bitbake/initramfs-live-install-testfs_1.0.bb b/tests/packagedcode/data/bitbake/initramfs-live-install-testfs_1.0.bb new file mode 100644 index 0000000000..13f9945ca1 --- /dev/null +++ b/tests/packagedcode/data/bitbake/initramfs-live-install-testfs_1.0.bb @@ -0,0 +1,16 @@ +SUMMARY = "Live image install script with a second rootfs/kernel" +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420" +SRC_URI = "file://init-install-testfs.sh" + +RDEPENDS_${PN} = "grub parted e2fsprogs-mke2fs" + +S = "${WORKDIR}" + +do_install() { + install -m 0755 ${WORKDIR}/init-install-testfs.sh ${D}/install.sh +} + +INHIBIT_DEFAULT_DEPS = "1" +FILES_${PN} = " /install.sh " +COMPATIBLE_HOST = "(i.86|x86_64).*-linux" \ No newline at end of file diff --git a/tests/packagedcode/data/bitbake/initramfs-live-install-testfs_1.0.bb-expected b/tests/packagedcode/data/bitbake/initramfs-live-install-testfs_1.0.bb-expected new file mode 100644 index 0000000000..049c5cc3da --- /dev/null +++ b/tests/packagedcode/data/bitbake/initramfs-live-install-testfs_1.0.bb-expected @@ -0,0 +1,35 @@ +{ + "type": "bitbake", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "mit", + "declared_license": "MIT", + "notice_text": null, + "root_path": null, + "dependencies": "grub parted e2fsprogs-mke2fs", + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/data/bitbake/netbase_6.1.bb b/tests/packagedcode/data/bitbake/netbase_6.1.bb new file mode 100644 index 0000000000..71a97e75fc --- /dev/null +++ b/tests/packagedcode/data/bitbake/netbase_6.1.bb @@ -0,0 +1,20 @@ +SUMMARY = "Basic TCP/IP networking support" +DESCRIPTION = "This package provides the necessary infrastructure for basic TCP/IP based networking" +HOMEPAGE = "http://packages.debian.org/netbase" +SECTION = "base" +LICENSE = "GPLv2" +LIC_FILES_CHKSUM = "file://debian/copyright;md5=3dd6192d306f582dee7687da3d8748ab" +PE = "1" + +SRC_URI = "${DEBIAN_MIRROR}/main/n/${BPN}/${BPN}_${PV}.tar.xz" + +SRC_URI[md5sum] = "e5871a3a5c8390557b8033cf19316a55" +SRC_URI[sha256sum] = "084d743bd84d4d9380bac4c71c51e57406dce44f5a69289bb823c903e9b035d8" + +UPSTREAM_CHECK_URI = "${DEBIAN_MIRROR}/main/n/netbase/" +do_install () { + install -d ${D}/${mandir}/man8 ${D}${sysconfdir} + install -m 0644 ${S}/etc/rpc ${D}${sysconfdir}/rpc + install -m 0644 ${S}/etc/protocols ${D}${sysconfdir}/protocols + install -m 0644 ${S}/etc/services ${D}${sysconfdir}/services +} \ No newline at end of file diff --git a/tests/packagedcode/data/bitbake/netbase_6.1.bb-expected b/tests/packagedcode/data/bitbake/netbase_6.1.bb-expected new file mode 100644 index 0000000000..4160099045 --- /dev/null +++ b/tests/packagedcode/data/bitbake/netbase_6.1.bb-expected @@ -0,0 +1,35 @@ +{ + "type": "bitbake", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": null, + "description": "This package provides the necessary infrastructure for basic TCP/IP based networking", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": "http://packages.debian.org/netbase", + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "license_expression": "gpl-2.0", + "declared_license": "GPLv2", + "notice_text": null, + "root_path": null, + "dependencies": [], + "contains_source_code": null, + "source_packages": [], + "purl": null, + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null +} \ No newline at end of file diff --git a/tests/packagedcode/test_bitbake.py b/tests/packagedcode/test_bitbake.py new file mode 100644 index 0000000000..edfa78df4a --- /dev/null +++ b/tests/packagedcode/test_bitbake.py @@ -0,0 +1,30 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# ScanCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/nexB/scancode-toolkit for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +import os.path + +from packagedcode import bitbake + +from packages_test_utils import PackageTester + + +class TestBitbakeBbManifest(PackageTester): + test_data_dir = os.path.join(os.path.dirname(__file__), 'data') + + def test_parse_bitbake(self): + test_file = self.get_test_loc('bitbake/netbase_6.1.bb') + package = bitbake.BitbakeBbManifest.recognize(test_file) + expected_loc = self.get_test_loc('bitbake/netbase_6.1.bb-expected') + self.check_package(package, expected_loc, regen=False) + + def test_parse_bitbake_dependencies(self): + test_file = self.get_test_loc('bitbake/initramfs-live-install-testfs_1.0.bb') + package = bitbake.BitbakeBbManifest.recognize(test_file) + expected_loc = self.get_test_loc('bitbake/initramfs-live-install-testfs_1.0.bb-expected') + self.check_package(package, expected_loc, regen=False)