Skip to content

Commit 9050a1c

Browse files
chinyeunglipombredanne
authored andcommitted
Integrate bitbake parser in package scan #1243
Signed-off-by: Chin Yeung Li <[email protected]> Signed-off-by: Philippe Ombredanne <[email protected]>
1 parent 5d33b05 commit 9050a1c

File tree

8 files changed

+327
-0
lines changed

8 files changed

+327
-0
lines changed

setup.cfg

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -77,6 +77,7 @@ install_requires =
7777
lxml >= 4.6.3, < 5.0.0
7878
MarkupSafe >= 1.0
7979
nltk >= 3.2, !=3.6, < 4.0
80+
oelint_parser
8081
packageurl_python >= 0.9.0
8182
packaging > 20
8283
pdfminer.six >= 20200101

src/packagedcode/__init__.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
#
99

1010
from packagedcode import about
11+
from packagedcode import bitbake
1112
from packagedcode import bower
1213
from packagedcode import build
1314
from packagedcode import cargo
@@ -45,6 +46,7 @@
4546
models.Axis2Mar,
4647

4748
about.AboutPackage,
49+
bitbake.BitbakePackage,
4850
npm.NpmPackage,
4951
phpcomposer.PHPComposerPackage,
5052
haxe.HaxePackage,

src/packagedcode/bitbake.py

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
#
2+
# Copyright (c) 2020 nexB Inc. and others. All rights reserved.
3+
# http://nexb.com and https://github.com/nexB/scancode-toolkit/
4+
# The ScanCode software is licensed under the Apache License version 2.0.
5+
# Data generated with ScanCode require an acknowledgment.
6+
# ScanCode is a trademark of nexB Inc.
7+
#
8+
# You may not use this software except in compliance with the License.
9+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software distributed
11+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
13+
# specific language governing permissions and limitations under the License.
14+
#
15+
# When you publish or redistribute any data created with ScanCode or any ScanCode
16+
# derivative work, you must accompany this data with the following acknowledgment:
17+
#
18+
# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
19+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
20+
# ScanCode should be considered or used as legal advice. Consult an Attorney
21+
# for any legal advice.
22+
# ScanCode is a free software code scanning tool from nexB Inc. and others.
23+
# Visit https://github.com/nexB/scancode-toolkit/ for support and download.
24+
25+
from __future__ import absolute_import
26+
from __future__ import print_function
27+
from __future__ import unicode_literals
28+
29+
import logging
30+
31+
import attr
32+
33+
from commoncode import filetype
34+
from packagedcode import models
35+
36+
from oelint_parser.cls_stash import Stash
37+
from oelint_parser.cls_item import Variable
38+
39+
TRACE = False
40+
41+
logger = logging.getLogger(__name__)
42+
43+
if TRACE:
44+
import sys
45+
logging.basicConfig(stream=sys.stdout)
46+
logger.setLevel(logging.DEBUG)
47+
48+
@attr.s()
49+
class BitbakePackage(models.Package):
50+
metafiles = ('*.bb',)
51+
default_type = 'bitbake'
52+
53+
@classmethod
54+
def recognize(cls, location):
55+
yield parse(location)
56+
57+
@classmethod
58+
def get_package_root(cls, manifest_resource, codebase):
59+
return manifest_resource.parent(codebase)
60+
61+
62+
def is_bb_file(location):
63+
return (filetype.is_file(location)
64+
and location.lower().endswith(('.bb',)))
65+
66+
def parse(location):
67+
"""
68+
Return a Package object from an ABOUT file or None.
69+
"""
70+
if not is_bb_file(location):
71+
return
72+
73+
_stash = Stash()
74+
# add any bitbake like file
75+
_stash.AddFile(location)
76+
77+
# Resolves proper cross file dependencies
78+
_stash.Finalize()
79+
80+
# get all variables of the name PV from all files
81+
package_dict = {}
82+
for item in _stash.GetItemsFor():
83+
try:
84+
# Create a package dictionary with VarName as the key and
85+
# VarValueStripped as the value
86+
name = item.VarName
87+
value = item.VarValueStripped
88+
try:
89+
if package_dict[name]:
90+
package_dict[name] += '\n' + value
91+
except:
92+
package_dict[name] = value
93+
except:
94+
# Continue to see if there is any VarName value until the end of
95+
# the file
96+
continue
97+
98+
return build_package(package_dict)
99+
100+
101+
def build_package(package_dict):
102+
"""
103+
Return a Package built from `package_dict` obtained from the .bb files.
104+
"""
105+
# Initialization
106+
name = None
107+
version = None
108+
description = None
109+
homepage_url = None
110+
download_url = None
111+
sha1 = None
112+
md5 = None
113+
sha256 = None
114+
sha512 = None
115+
declared_license = None
116+
dependencies = None
117+
if 'PN' in package_dict:
118+
name = package_dict['PN']
119+
if 'PV' in package_dict:
120+
version = package_dict['PV']
121+
if 'DESCRIPTION' in package_dict:
122+
description = package_dict['DESCRIPTION']
123+
if 'HOMEPAGE' in package_dict:
124+
homepage_url = package_dict['HOMEPAGE']
125+
if 'PREMIRRORS' in package_dict:
126+
download_url = package_dict['PREMIRRORS']
127+
#The item.VarName for SRC_URI[*] from the parser are SRC_URI
128+
#Therefore, I cannot differentiate md5,sha1, or src file location reference
129+
# Entered an issue ticket: https://github.com/priv-kweihmann/oelint-parser/issues/3
130+
"""
131+
if 'SRC_URI[sha1sum]' in package_dict:
132+
sha1 = package_dict['SRC_URI[sha1sum]']
133+
if 'SRC_URI[md5sum]' in package_dict:
134+
md5 = package_dict['SRC_URI[md5sum]']
135+
if 'SRC_URI[sha256sum]' in package_dict:
136+
sha256 = package_dict['SRC_URI[sha256sum]']
137+
if 'SRC_URI[sha512sum]' in package_dict:
138+
sha512 = package_dict['SRC_URI[sha512sum]']
139+
"""
140+
if 'LICENSE' in package_dict:
141+
declared_license = package_dict['LICENSE']
142+
if 'DEPENDS' in package_dict:
143+
if dependencies:
144+
dependencies += '\n' + package_dict['DEPENDS']
145+
else:
146+
dependencies = package_dict['DEPENDS']
147+
# There are some RDEPENDS_* fields such as "RDEPENDS_${PN}" which I need to
148+
# check with the substring
149+
for d in package_dict:
150+
if 'RDEPENDS' in d:
151+
if dependencies:
152+
dependencies += '\n' + package_dict[d]
153+
else:
154+
dependencies = package_dict[d]
155+
156+
return BitbakePackage(
157+
type='bitbake',
158+
name=name,
159+
version=version,
160+
description=description,
161+
homepage_url=homepage_url,
162+
download_url=download_url,
163+
sha1=sha1,
164+
md5=md5,
165+
sha256=sha256,
166+
sha512=sha512,
167+
declared_license=declared_license,
168+
dependencies=dependencies
169+
)
Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
SUMMARY = "Live image install script with a second rootfs/kernel"
2+
LICENSE = "MIT"
3+
LIC_FILES_CHKSUM = "file://${COREBASE}/meta/COPYING.MIT;md5=3da9cfbcb788c80a0384361b4de20420"
4+
SRC_URI = "file://init-install-testfs.sh"
5+
6+
RDEPENDS_${PN} = "grub parted e2fsprogs-mke2fs"
7+
8+
S = "${WORKDIR}"
9+
10+
do_install() {
11+
install -m 0755 ${WORKDIR}/init-install-testfs.sh ${D}/install.sh
12+
}
13+
14+
INHIBIT_DEFAULT_DEPS = "1"
15+
FILES_${PN} = " /install.sh "
16+
COMPATIBLE_HOST = "(i.86|x86_64).*-linux"
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
{
2+
"type": "bitbake",
3+
"namespace": null,
4+
"name": null,
5+
"version": null,
6+
"qualifiers": {},
7+
"subpath": null,
8+
"primary_language": null,
9+
"description": null,
10+
"release_date": null,
11+
"parties": [],
12+
"keywords": [],
13+
"homepage_url": null,
14+
"download_url": null,
15+
"size": null,
16+
"sha1": null,
17+
"md5": null,
18+
"sha256": null,
19+
"sha512": null,
20+
"bug_tracking_url": null,
21+
"code_view_url": null,
22+
"vcs_url": null,
23+
"copyright": null,
24+
"license_expression": "mit",
25+
"declared_license": "MIT",
26+
"notice_text": null,
27+
"root_path": null,
28+
"dependencies": "grub parted e2fsprogs-mke2fs",
29+
"contains_source_code": null,
30+
"source_packages": [],
31+
"purl": null,
32+
"repository_homepage_url": null,
33+
"repository_download_url": null,
34+
"api_data_url": null
35+
}
Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
SUMMARY = "Basic TCP/IP networking support"
2+
DESCRIPTION = "This package provides the necessary infrastructure for basic TCP/IP based networking"
3+
HOMEPAGE = "http://packages.debian.org/netbase"
4+
SECTION = "base"
5+
LICENSE = "GPLv2"
6+
LIC_FILES_CHKSUM = "file://debian/copyright;md5=3dd6192d306f582dee7687da3d8748ab"
7+
PE = "1"
8+
9+
SRC_URI = "${DEBIAN_MIRROR}/main/n/${BPN}/${BPN}_${PV}.tar.xz"
10+
11+
SRC_URI[md5sum] = "e5871a3a5c8390557b8033cf19316a55"
12+
SRC_URI[sha256sum] = "084d743bd84d4d9380bac4c71c51e57406dce44f5a69289bb823c903e9b035d8"
13+
14+
UPSTREAM_CHECK_URI = "${DEBIAN_MIRROR}/main/n/netbase/"
15+
do_install () {
16+
install -d ${D}/${mandir}/man8 ${D}${sysconfdir}
17+
install -m 0644 ${S}/etc/rpc ${D}${sysconfdir}/rpc
18+
install -m 0644 ${S}/etc/protocols ${D}${sysconfdir}/protocols
19+
install -m 0644 ${S}/etc/services ${D}${sysconfdir}/services
20+
}
Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
{
2+
"type": "bitbake",
3+
"namespace": null,
4+
"name": null,
5+
"version": null,
6+
"qualifiers": {},
7+
"subpath": null,
8+
"primary_language": null,
9+
"description": "This package provides the necessary infrastructure for basic TCP/IP based networking",
10+
"release_date": null,
11+
"parties": [],
12+
"keywords": [],
13+
"homepage_url": "http://packages.debian.org/netbase",
14+
"download_url": null,
15+
"size": null,
16+
"sha1": null,
17+
"md5": null,
18+
"sha256": null,
19+
"sha512": null,
20+
"bug_tracking_url": null,
21+
"code_view_url": null,
22+
"vcs_url": null,
23+
"copyright": null,
24+
"license_expression": "gpl-2.0",
25+
"declared_license": "GPLv2",
26+
"notice_text": null,
27+
"root_path": null,
28+
"dependencies": null,
29+
"contains_source_code": null,
30+
"source_packages": [],
31+
"purl": null,
32+
"repository_homepage_url": null,
33+
"repository_download_url": null,
34+
"api_data_url": null
35+
}

tests/packagedcode/test_bitbake.py

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#
2+
# Copyright (c) 2020 nexB Inc. and others. All rights reserved.
3+
# http://nexb.com and https://github.com/nexB/scancode-toolkit/
4+
# The ScanCode software is licensed under the Apache License version 2.0.
5+
# Data generated with ScanCode require an acknowledgment.
6+
# ScanCode is a trademark of nexB Inc.
7+
#
8+
# You may not use this software except in compliance with the License.
9+
# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
10+
# Unless required by applicable law or agreed to in writing, software distributed
11+
# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
12+
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
13+
# specific language governing permissions and limitations under the License.
14+
#
15+
# When you publish or redistribute any data created with ScanCode or any ScanCode
16+
# derivative work, you must accompany this data with the following acknowledgment:
17+
#
18+
# Generated with ScanCode and provided on an "AS IS" BASIS, WITHOUT WARRANTIES
19+
# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
20+
# ScanCode should be considered or used as legal advice. Consult an Attorney
21+
# for any legal advice.
22+
# ScanCode is a free software code scanning tool from nexB Inc. and others.
23+
# Visit https://github.com/nexB/scancode-toolkit/ for support and download.
24+
25+
from __future__ import absolute_import
26+
from __future__ import print_function
27+
from __future__ import unicode_literals
28+
29+
import os.path
30+
31+
from packagedcode import bitbake
32+
33+
from packages_test_utils import PackageTester
34+
35+
36+
class TestBitbake(PackageTester):
37+
test_data_dir = os.path.join(os.path.dirname(__file__), 'data')
38+
39+
def test_parse_bitbake(self):
40+
test_file = self.get_test_loc('bitbake/netbase_6.1.bb')
41+
package = bitbake.parse(test_file)
42+
expected_loc = self.get_test_loc('bitbake/netbase_6.1.bb-expected')
43+
self.check_package(package, expected_loc, regen=False)
44+
45+
def test_parse_bitbake_dependencies(self):
46+
test_file = self.get_test_loc('bitbake/initramfs-live-install-testfs_1.0.bb')
47+
package = bitbake.parse(test_file)
48+
expected_loc = self.get_test_loc('bitbake/initramfs-live-install-testfs_1.0.bb-expected')
49+
self.check_package(package, expected_loc, regen=False)

0 commit comments

Comments
 (0)