Collect data from yocto/bitbake .bb #1243

setup.cfg

@@ -79,6 +79,7 @@ install_requires =
     license_expression >= 21.6.14
     lxml >= 4.6.5
     MarkupSafe >= 1.0
+    oelint_parser >= 1.1.0
     packageurl_python >= 0.9.0
     packaging >= 21.0.0
     # parameter_expansion >= 0.2.0

src/packagedcode/__init__.py

@@ -10,6 +10,7 @@
 import attr
 
 from packagedcode import about
+from packagedcode import bitbake
 from packagedcode import bower
 from packagedcode import build
 from packagedcode import build_gradle
@@ -55,6 +56,7 @@
     models.Axis2Mar,
 
     about.Aboutfile,
+    bitbake.BitbakeBbClassManifest,
     npm.PackageJson,
     npm.PackageLockJson,
     npm.YarnLockJson,

src/packagedcode/bitbake.py

@@ -0,0 +1,128 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import logging
+
+import attr
+from oelint_parser.cls_stash import Stash
+from oelint_parser.cls_item import Variable
+
+from commoncode import filetype
+from packagedcode import models
+
+TRACE = False
+
+logger = logging.getLogger(__name__)
+
+if TRACE:
+    import sys
+    logging.basicConfig(stream=sys.stdout)
+    logger.setLevel(logging.DEBUG)
+
+
+@attr.s()
+class BitbakePacakge(models.Package):
+    default_type = 'bitbake'
+
+    @classmethod
+    def get_package_root(cls, manifest_resource, codebase):
+        return manifest_resource.parent(codebase)
+
+@attr.s()
+class BitbakeBbManifest(BitbakePacakge, models.PackageManifest):
+    file_patterns = ('*.bb',)
+    extensions = ('.bb', '.bbclass',)
+
+    @classmethod
+    def is_manifest(cls, location):
+        return (
+            filetype.is_file(location)
+            and location.lower().endswith(('.bb',))
+        )
+
+    @classmethod
+    def recognize(cls, location):
+        yield parse(location)
+
+
+def parse(location):
+    """
+    Return a Package object from an BitBake file or None.
+    """
+    _stash = Stash()
+    # add any bitbake like file
+    _stash.AddFile(location)
+
+    # Resolves proper cross file dependencies
+    _stash.Finalize()
+
+    # get all variables from all prsed files
+    data = {}
+    for item in _stash.GetItemsFor():
+        try:
+            # Create a package dictionary with VarName as the key and
+            # VarValueStripped as the value
+            name = item.VarName
+            value = item.VarValueStripped
+            try:
+                if data[name]:
+                    data[name] += '\n' + value
+            except:
+                data[name] = value
+        except:
+            # Continue to see if there is any VarName value until the end of
+            # the file
+            continue
+
+    return build_package(data)
+
+
+def build_package(data):
+    """
+    Return a Package built from Bake `data` mapping.
+    """
+    name = data.get('PN')
+    version = data.get('PV')
+    description = data.get('DESCRIPTION')
+    homepage_url = data.get('HOMEPAGE')
+    download_url = data.get('PREMIRRORS')
+    declared_license = data.get('LICENSE')
+    dependencies = data.get('DEPENDS', [])
+
+    # The item.VarName for SRC_URI[*] from the parser are SRC_URI
+    # Therefore, I cannot differentiate md5, sha1, or src file location reference
+    # See: https://github.com/priv-kweihmann/oelint-parser/issues/3
+    sha1 = data.get('SRC_URI[sha1sum]')
+    md5 = data.get('SRC_URI[md5sum]')
+    sha256 = data.get('SRC_URI[sha256sum]')
+    sha512 = data.get('SRC_URI[sha512sum]')
+
+    # There are some RDEPENDS_* fields such as "RDEPENDS_${PN}" which I need to
+    # check with the substring
+    # FIXME: we should create a DependentPackage
+    for d in data:
+        if d.startswith('RDEPENDS'):
+            if dependencies:
+                dependencies += '\n' + data[d]
+            else:
+                dependencies = data[d]
+
+    return BitbakeBbManifest(
+        name=name,
+        version=version,
+        description=description,
+        homepage_url=homepage_url,
+        download_url=download_url,
+        sha1=sha1,
+        md5=md5,
+        sha256=sha256,
+        sha512=sha512,
+        declared_license=declared_license,
+        dependencies=dependencies
+    )

tests/packagedcode/data/bitbake/initramfs-live-install-testfs_1.0.bb

@@ -0,0 +1,16 @@
+SUMMARY = "Live image install script with a second rootfs/kernel"
+LICENSE = "MIT"
+LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420"
+SRC_URI = "file://init-install-testfs.sh"
+
+RDEPENDS_${PN} = "grub parted e2fsprogs-mke2fs"
+
+S = "${WORKDIR}"
+
+do_install() {
+        install -m 0755 ${WORKDIR}/init-install-testfs.sh ${D}/install.sh
+}
+
+INHIBIT_DEFAULT_DEPS = "1"
+FILES_${PN} = " /install.sh "
+COMPATIBLE_HOST = "(i.86|x86_64).*-linux"

tests/packagedcode/data/bitbake/initramfs-live-install-testfs_1.0.bb-expected

@@ -0,0 +1,35 @@
+{
+  "type": "bitbake",
+  "namespace": null,
+  "name": null,
+  "version": null,
+  "qualifiers": {},
+  "subpath": null,
+  "primary_language": null,
+  "description": null,
+  "release_date": null,
+  "parties": [],
+  "keywords": [],
+  "homepage_url": null,
+  "download_url": null,
+  "size": null,
+  "sha1": null,
+  "md5": null,
+  "sha256": null,
+  "sha512": null,
+  "bug_tracking_url": null,
+  "code_view_url": null,
+  "vcs_url": null,
+  "copyright": null,
+  "license_expression": "mit",
+  "declared_license": "MIT",
+  "notice_text": null,
+  "root_path": null,
+  "dependencies": "grub parted e2fsprogs-mke2fs",
+  "contains_source_code": null,
+  "source_packages": [],
+  "purl": null,
+  "repository_homepage_url": null,
+  "repository_download_url": null,
+  "api_data_url": null
+}

tests/packagedcode/data/bitbake/netbase_6.1.bb

@@ -0,0 +1,20 @@
+SUMMARY = "Basic TCP/IP networking support"
+DESCRIPTION = "This package provides the necessary infrastructure for basic TCP/IP based networking"
+HOMEPAGE = "http://packages.debian.org/netbase"
+SECTION = "base"
+LICENSE = "GPLv2"
+LIC_FILES_CHKSUM = "file://debian/copyright;md5=3dd6192d306f582dee7687da3d8748ab"
+PE = "1"
+
+SRC_URI = "${DEBIAN_MIRROR}/main/n/${BPN}/${BPN}_${PV}.tar.xz"
+
+SRC_URI[md5sum] = "e5871a3a5c8390557b8033cf19316a55"
+SRC_URI[sha256sum] = "084d743bd84d4d9380bac4c71c51e57406dce44f5a69289bb823c903e9b035d8"
+
+UPSTREAM_CHECK_URI = "${DEBIAN_MIRROR}/main/n/netbase/"
+do_install () {
+	install -d ${D}/${mandir}/man8 ${D}${sysconfdir}
+	install -m 0644 ${S}/etc/rpc ${D}${sysconfdir}/rpc
+	install -m 0644 ${S}/etc/protocols ${D}${sysconfdir}/protocols
+	install -m 0644 ${S}/etc/services ${D}${sysconfdir}/services
+}

tests/packagedcode/data/bitbake/netbase_6.1.bb-expected

@@ -0,0 +1,35 @@
+{
+  "type": "bitbake",
+  "namespace": null,
+  "name": null,
+  "version": null,
+  "qualifiers": {},
+  "subpath": null,
+  "primary_language": null,
+  "description": "This package provides the necessary infrastructure for basic TCP/IP based networking",
+  "release_date": null,
+  "parties": [],
+  "keywords": [],
+  "homepage_url": "http://packages.debian.org/netbase",
+  "download_url": null,
+  "size": null,
+  "sha1": null,
+  "md5": null,
+  "sha256": null,
+  "sha512": null,
+  "bug_tracking_url": null,
+  "code_view_url": null,
+  "vcs_url": null,
+  "copyright": null,
+  "license_expression": "gpl-2.0",
+  "declared_license": "GPLv2",
+  "notice_text": null,
+  "root_path": null,
+  "dependencies": [],
+  "contains_source_code": null,
+  "source_packages": [],
+  "purl": null,
+  "repository_homepage_url": null,
+  "repository_download_url": null,
+  "api_data_url": null
+}

tests/packagedcode/test_bitbake.py

@@ -0,0 +1,30 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+
+import os.path
+
+from packagedcode import bitbake
+
+from packages_test_utils import PackageTester
+
+
+class TestBitbakeBbManifest(PackageTester):
+    test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
+
+    def test_parse_bitbake(self):
+        test_file = self.get_test_loc('bitbake/netbase_6.1.bb')
+        package = bitbake.BitbakeBbManifest.recognize(test_file)
+        expected_loc = self.get_test_loc('bitbake/netbase_6.1.bb-expected')
+        self.check_package(package, expected_loc, regen=False)
+
+    def test_parse_bitbake_dependencies(self):
+        test_file = self.get_test_loc('bitbake/initramfs-live-install-testfs_1.0.bb')
+        package = bitbake.BitbakeBbManifest.recognize(test_file)
+        expected_loc = self.get_test_loc('bitbake/initramfs-live-install-testfs_1.0.bb-expected')
+        self.check_package(package, expected_loc, regen=False)