Use latest API for bitbake parsing #1243

Signed-off-by: Philippe Ombredanne <[email protected]>

Author: pombredanne

src/packagedcode/__init__.py

@@ -57,7 +57,7 @@
 
     bower.BowerJsonHandler,
 
-    bitbake.BitbakePackage,
+    bitbake.BitbakeBbManifestHandler,
 
     build_gradle.BuildGradleHandler,
 

src/packagedcode/bitbake.py

@@ -1,42 +1,21 @@
 #
-# Copyright (c) 2020 nexB Inc. and others. All rights reserved.
-# http://nexb.com and https://github.com/nexB/scancode-toolkit/
-# The ScanCode software is licensed under the Apache License version 2.0.
-# Data generated with ScanCode require an acknowledgment.
+# Copyright (c) nexB Inc. and others. All rights reserved.
 # ScanCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/scancode-toolkit for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
 #
-# You may not use this software except in compliance with the License.
-# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
-# Unless required by applicable law or agreed to in writing, software distributed
-# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
-# CONDITIONS OF ANY KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations under the License.
-#
-# When you publish or redistribute any data created with ScanCode or any ScanCode
-# derivative work, you must accompany this data with the following acknowledgment:
-#
-#  Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
-#  OR CONDITIONS OF ANY KIND, either express or implied. No content created from
-#  ScanCode should be considered or used as legal advice. Consult an Attorney
-#  for any legal advice.
-#  ScanCode is a free software code scanning tool from nexB Inc. and others.
-#  Visit https://github.com/nexB/scancode-toolkit/ for support and download.
-
-from __future__ import absolute_import
-from __future__ import print_function
-from __future__ import unicode_literals
 
 import logging
+import re
 
-import attr
+from oelint_parser.cls_stash import Stash
+from packageurl import PackageURL
 
-from commoncode import filetype
 from packagedcode import models
 
-from oelint_parser.cls_stash import Stash
-from oelint_parser.cls_item import Variable
-
-TRACE = False
+TRACE = True
 
 logger = logging.getLogger(__name__)
 
@@ -45,125 +24,169 @@
     logging.basicConfig(stream=sys.stdout)
     logger.setLevel(logging.DEBUG)
 
-@attr.s()
-class BitbakePackage(models.Package):
-    metafiles = ('*.bb',)
-    default_type = 'bitbake'
 
-    @classmethod
-    def recognize(cls, location):
-        yield parse(location)
+class BitbakeBbManifestHandler(models.DatafileHandler):
+    datasource_id = 'bitbake_bb_recipe'
+    # note that there are .bbappend, .bbclass and bitbake.conf files.
+    path_patterns = ('*.bb',)
+    default_package_type = 'bitbake'
+    description = 'BitBake bb recipe manifest'
+    documentation_url = 'https://docs.yoctoproject.org/bitbake/bitbake-user-manual/bitbake-user-manual-metadata.html'
 
     @classmethod
-    def get_package_root(cls, manifest_resource, codebase):
-        return manifest_resource.parent(codebase)
+    def parse(cls, location):
+
+        oestash = Stash(quiet=True)
+
+        # add any bitbake like-file
+        # TODO: may be we should handle the bbclass and bbappend here?
+        oestash.AddFile(location)
+
+        # Resolves proper cross file dependencies
+        oestash.Finalize()
+
+        # collect all variables of interest.
+        # TODO: we should not get list values. Instead plain strings
+        data = {
+            k: ' '.join(v) if isinstance(v, (list, tuple)) else v
+            for k, v in oestash.ExpandVar(filename=location).items()
+            if v
+        }
+        name = data.get('PN')
+        version = data.get('PV')
+        description = data.get('DESCRIPTION')
+        homepage_url = data.get('HOMEPAGE')
+        download_url = data.get('PREMIRRORS')
+        declared_license = data.get('LICENSE')
+
+        # The item.VarName for SRC_URI[*] from the parser are SRC_URI
+        # Therefore, I cannot differentiate md5, sha1, or src file location reference
+        # See: https://github.com/priv-kweihmann/oelint-parser/issues/3
+        sha1 = data.get('SRC_URI[sha1sum]')
+        md5 = data.get('SRC_URI[md5sum]')
+        sha256 = data.get('SRC_URI[sha256sum]')
+        sha512 = data.get('SRC_URI[sha512sum]')
+
+        dependencies = []
+
+        # Build deps: this is a list of plain BB recipes names
+        # https://docs.yoctoproject.org/bitbake/bitbake-user-manual/bitbake-user-manual-ref-variables.html#term-DEPENDS
+        build_deps = data.get('DEPENDS', '').split()
+        for build_dep in build_deps:
+            dep_purl = PackageURL(
+                type=cls.default_package_type,
+                name=build_dep,
+            ).to_string()
+
+            dependency = models.DependentPackage(
+                purl=dep_purl,
+                extracted_requirement=build_dep,
+                scope='build',
+                is_runtime=False,
+                is_optional=True,
+                is_resolved=False,
+            )
+            dependencies.append(dependency)
+
+        # Runtime deps:this is a list of Package names with an optional (=> 12) version constraint
+        # https://docs.yoctoproject.org/bitbake/bitbake-user-manual/bitbake-user-manual-ref-variables.html#term-RDEPENDS
+        # FIXME: There are some fields such as "RDEPENDS_${PN}" so these may not be correct in all cases
+        for key, value in data.items():
+            if not key.startswith('RDEPENDS'):
+                continue
+            if not value:
+                continue
+            for name, constraint in get_bitbake_deps(dependencies=value):
+                if TRACE:
+                    logger.debug(f'RDEPENDS: name={name}, constraint={constraint}')
+                dep_purl = PackageURL(
+                    type=cls.default_package_type,
+                    name=name,
+                ).to_string()
+
+                extracted_requirement = name
+                if constraint:
+                    extracted_requirement += f' ({constraint})'
+
+                dependency = models.DependentPackage(
+                    purl=dep_purl,
+                    extracted_requirement=extracted_requirement,
+                    scope='install',
+                    is_runtime=True,
+                    is_optional=False,
+                    is_resolved=False,
+                )
+
+                dependencies.append(dependency)
+
+        yield models.PackageData(
+            datasource_id=cls.datasource_id,
+            type=cls.default_package_type,
+            name=name,
+            version=version,
+            description=description,
+            homepage_url=homepage_url,
+            download_url=download_url,
+            sha1=sha1,
+            md5=md5,
+            sha256=sha256,
+            sha512=sha512,
+            declared_license=declared_license,
+            dependencies=dependencies,
+        )
 
+    @classmethod
+    def assign_package_to_resources(cls, package, resource, codebase, package_adder):
+        return models.DatafileHandler.assign_package_to_parent_tree(package, resource, codebase, package_adder)
 
-def is_bb_file(location):
-    return (filetype.is_file(location)
-            and location.lower().endswith(('.bb',)))
 
-def parse(location):
+def get_bitbake_deps(dependencies):
     """
-    Return a Package object from an ABOUT file or None.
+    Return a list of tuple of (name, version constraint) given a BitBake
+    dependencies string. "version constraint" can be None.
+
+    See https://docs.yoctoproject.org/ref-manual/variables.html?#term-RDEPENDS
+    For example:
+    >>> expected = [('ABC', None), ('abcd', '=>12312'), ('defg', None)]
+    >>> result = get_bitbake_deps(" ABC abcd (= >  12312) defg ")
+    >>> assert result == expected, result
+    >>> expected = [('grub', '==12.23'), ('parted', None), ('e2fsprogs-mke2fs', None)]
+    >>> result = get_bitbake_deps("grub (== 12.23) parted e2fsprogs-mke2fs")
+    >>> assert result == expected, result
     """
-    if not is_bb_file(location):
-        return
-
-    _stash = Stash()
-    # add any bitbake like file
-    _stash.AddFile(location)
-    
-    # Resolves proper cross file dependencies
-    _stash.Finalize()
-
-    # get all variables of the name PV from all files
-    package_dict = {}
-    for item in _stash.GetItemsFor():
-        try:
-            # Create a package dictionary with VarName as the key and
-            # VarValueStripped as the value
-            name = item.VarName
-            value = item.VarValueStripped
-            try:
-                if package_dict[name]:
-                    package_dict[name] += '\n' + value
-            except:
-                package_dict[name] = value
-        except:
-            # Continue to see if there is any VarName value until the end of
-            # the file
-            continue
-
-    return build_package(package_dict)
-
-
-def build_package(package_dict):
+    return [split_name_constraint(nc) for nc in split_deps(dependencies)]
+
+
+def split_name_constraint(dependency):
     """
-    Return a Package built from `package_dict` obtained from the .bb files.
+    Return a tuple (name, version constraint) strings given a name (version
+    constraint) BitBake dependency string.
+    See https://docs.yoctoproject.org/ref-manual/variables.html?#term-RDEPENDS
+    For example:
+    >>> assert split_name_constraint(" abcd  ( = 12312 ) ") == ("abcd", "=12312",)
+    >>> assert split_name_constraint("abcd ") == ("abcd", None)
     """
-    # Initialization
-    name = None
-    version = None
-    description = None
-    homepage_url = None
-    download_url = None
-    sha1 = None
-    md5 = None
-    sha256 = None
-    sha512 = None
-    declared_license = None
-    dependencies = None
-    if 'PN' in package_dict:
-        name = package_dict['PN']
-    if 'PV' in package_dict:
-        version = package_dict['PV']
-    if 'DESCRIPTION' in package_dict:
-        description = package_dict['DESCRIPTION']
-    if 'HOMEPAGE' in package_dict:
-        homepage_url = package_dict['HOMEPAGE']
-    if 'PREMIRRORS' in package_dict:
-        download_url = package_dict['PREMIRRORS']
-    #The item.VarName for SRC_URI[*] from the parser are SRC_URI
-    #Therefore, I cannot differentiate md5,sha1, or src file location reference
-    # Entered an issue ticket: https://github.com/priv-kweihmann/oelint-parser/issues/3
+    no_spaces = dependency.replace(' ', '')
+    if '(' in no_spaces:
+        name, _, constraint = no_spaces.partition('(')
+        constraint = constraint.rstrip(')')
+        return name, constraint
+    return no_spaces, None
+
+
+def split_deps(dependencies):
     """
-    if 'SRC_URI[sha1sum]' in package_dict:
-        sha1 = package_dict['SRC_URI[sha1sum]']
-    if 'SRC_URI[md5sum]' in package_dict:
-        md5 = package_dict['SRC_URI[md5sum]']
-    if 'SRC_URI[sha256sum]' in package_dict:
-        sha256 = package_dict['SRC_URI[sha256sum]']
-    if 'SRC_URI[sha512sum]' in package_dict:
-        sha512 = package_dict['SRC_URI[sha512sum]']
+    Return a list of name (version constraint) strings given a BitBake
+    dependencies string.
+
+    See https://docs.yoctoproject.org/ref-manual/variables.html?#term-RDEPENDS
+    For example:
+    >>> expected = ['ABC', 'abcd (= > 12312)', 'defg', 'foo', 'bar']
+    >>> result = split_deps(" ABC abcd (= >  12312) defg   foo bar ")
+    >>> assert result == expected, result
     """
-    if 'LICENSE' in package_dict:
-        declared_license = package_dict['LICENSE']
-    if 'DEPENDS' in package_dict:
-        if dependencies:
-            dependencies += '\n' + package_dict['DEPENDS']
-        else:
-            dependencies = package_dict['DEPENDS']
-    # There are some RDEPENDS_* fields such as "RDEPENDS_${PN}" which I need to
-    # check with the substring
-    for d in package_dict:
-        if 'RDEPENDS' in d:
-            if dependencies:
-                dependencies += '\n' + package_dict[d]
-            else:
-                dependencies = package_dict[d]
-
-    return BitbakePackage(
-        type='bitbake',
-        name=name,
-        version=version,
-        description=description,
-        homepage_url=homepage_url,
-        download_url=download_url,
-        sha1=sha1,
-        md5=md5,
-        sha256=sha256,
-        sha512=sha512,
-        declared_license=declared_license,
-        dependencies=dependencies
-    )
+    normalized_spaces = ' '.join(dependencies.split())
+    name = r'\w[\w\d_-]+'
+    version_constraint = r'\([<>= ]+[^<>= ]+\s?\)'
+    splitter = re.compile(fr'({name}\s?(?:{version_constraint})?)').findall
+    return [s.strip() for s in splitter(normalized_spaces)]

tests/packagedcode/data/bitbake/initramfs-live-install-testfs_1.0.bb

@@ -2,8 +2,9 @@ SUMMARY = "Live image install script with a second rootfs/kernel"
 LICENSE = "MIT"
 LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420"
 SRC_URI = "file://init-install-testfs.sh"
+DEPENDS = "do_this and that"
 
-RDEPENDS_${PN} = "grub parted e2fsprogs-mke2fs"
+RDEPENDS_${PN} = "grub (== 12.23) parted e2fsprogs-mke2fs"
 
 S = "${WORKDIR}"