Skip to content

Commit cee6d68

Browse files
pombredannepombredanne
authored
Merge pull request #2616 from akugarg/follow_reference
Follow license reference to another file
2 parents 5db9d05 + 6aa5d9d commit cee6d68

File tree

13 files changed

+3723
-24
lines changed

13 files changed

+3723
-24
lines changed

‎src/licensedcode/models.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -953,6 +953,10 @@ def validate(self, licensing=None):
953953
except InvalidRule as e:
954954
yield f'Failed to parse and validate license_expression: {e}'
955955

956+
if self.referenced_filenames:
957+
if len(set(self.referenced_filenames)) != len(self.referenced_filenames):
958+
yield 'referenced_filenames cannot contain duplicates.'
959+
956960
def license_keys(self, unique=True):
957961
"""
958962
Return a list of license keys for this rule.

‎src/licensedcode/plugin_license.py‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
import attr
1313

14+
from commoncode import fileutils
1415
from commoncode.cliutils import PluggableCommandLineOption
1516
from plugincode.scan import ScanPlugin
1617
from plugincode.scan import scan_impl
@@ -116,3 +117,73 @@ def get_scanner(
116117
license_text_diagnostics=license_text_diagnostics,
117118
license_url_template=license_url_template
118119
)
120+
121+
def process_codebase(self, codebase, **kwargs):
122+
123+
if codebase.has_single_resource:
124+
return
125+
126+
for resource in codebase.walk(topdown=False):
127+
match_reference_license(resource,codebase)
128+
129+
130+
def match_reference_license(resource, codebase):
131+
"""
132+
Return the ``resource`` Resource updating and saving it in place, after adding new
133+
license matches (licenses and license_expressions) following their Rule
134+
``referenced_filenames`` if any. Return None if this is not a file Resource.
135+
"""
136+
if not resource.is_file:
137+
return
138+
139+
licenses = resource.licenses
140+
license_expressions = resource.license_expressions
141+
if not licenses:
142+
return
143+
144+
referenced_licenses = []
145+
referenced_license_expressions = []
146+
referenced_filenames = get_referenced_filenames(licenses)
147+
modified = False
148+
149+
for referenced_filename in referenced_filenames:
150+
new_resource = find_referenced_resource(referenced_filename=referenced_filename, resource=resource, codebase=codebase)
151+
if new_resource:
152+
modified = True
153+
referenced_licenses.extend(new_resource.licenses)
154+
referenced_license_expressions.extend(new_resource.license_expressions)
155+
156+
licenses.extend(referenced_licenses)
157+
license_expressions.extend(referenced_license_expressions)
158+
159+
if modified:
160+
codebase.save_resource(resource)
161+
return resource
162+
163+
164+
def get_referenced_filenames(license_matches):
165+
"""
166+
Return a list of unique referenced filenames found in the rules of a list of ``license_matches``
167+
"""
168+
referenced_filenames = []
169+
for license_match in license_matches:
170+
referenced_files = license_match['matched_rule']['referenced_filenames']
171+
for referenced_filename in referenced_files:
172+
if not referenced_filename in referenced_filenames:
173+
referenced_filenames.append(referenced_filename)
174+
175+
return referenced_filenames
176+
177+
178+
def find_referenced_resource(referenced_filename, resource, codebase, **kwargs):
179+
"""
180+
Return a Resource matching the ``referenced_filename`` path or filename given a ``resource`` in ``codebase``.
181+
Return None if the ``referenced_filename`` cannot be found in the same directory as the base ``resource``.
182+
``referenced_filename`` is the path or filename referenced in a LicenseMatch of ``resource``,
183+
"""
184+
parent = resource.parent(codebase)
185+
186+
for child in parent.children(codebase):
187+
path = child.path
188+
if path.endswith(referenced_filename) or fileutils.file_base_name(child.path) == referenced_filename:
189+
return child

‎tests/licensedcode/data/plugin_license/license_reference/scan-ref.expected.json‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
{
2+
"headers": [
3+
{
4+
"tool_name": "scancode-toolkit",
5+
"options": {
6+
"input": "<path>",
7+
"--json": "<file>",
8+
"--license": true,
9+
"--license-text": true,
10+
"--license-text-diagnostics": true,
11+
"--strip-root": true
12+
},
13+
"notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
14+
"message": null,
15+
"errors": [],
16+
"extra_data": {
17+
"files_count": 2
18+
}
19+
}
20+
],
21+
"files": [
22+
{
23+
"path": "LICENSE",
24+
"type": "file",
25+
"licenses": [
26+
{
27+
"key": "mit",
28+
"score": 100.0,
29+
"name": "MIT License",
30+
"short_name": "MIT License",
31+
"category": "Permissive",
32+
"is_exception": false,
33+
"is_unknown": false,
34+
"owner": "MIT",
35+
"homepage_url": "http://opensource.org/licenses/mit-license.php",
36+
"text_url": "http://opensource.org/licenses/mit-license.php",
37+
"reference_url": "https://scancode-licensedb.aboutcode.org/mit",
38+
"scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/mit.LICENSE",
39+
"scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/mit.yml",
40+
"spdx_license_key": "MIT",
41+
"spdx_url": "https://spdx.org/licenses/MIT",
42+
"start_line": 1,
43+
"end_line": 1,
44+
"matched_rule": {
45+
"identifier": "mit_66.RULE",
46+
"license_expression": "mit",
47+
"licenses": [
48+
"mit"
49+
],
50+
"referenced_filenames": [],
51+
"is_license_text": false,
52+
"is_license_notice": true,
53+
"is_license_reference": false,
54+
"is_license_tag": false,
55+
"is_license_intro": false,
56+
"has_unknown": false,
57+
"matcher": "1-hash",
58+
"rule_length": 10,
59+
"matched_length": 10,
60+
"match_coverage": 100.0,
61+
"rule_relevance": 100
62+
},
63+
"matched_text": "that is licensed under [MIT](http://opensource.org/licenses/MIT)."
64+
}
65+
],
66+
"license_expressions": [
67+
"mit"
68+
],
69+
"percentage_of_license_text": 100.0,
70+
"scan_errors": []
71+
},
72+
{
73+
"path": "license-notice.txt",
74+
"type": "file",
75+
"licenses": [
76+
{
77+
"key": "unknown-license-reference",
78+
"score": 27.0,
79+
"name": "Unknown License file reference",
80+
"short_name": "Unknown License reference",
81+
"category": "Unstated License",
82+
"is_exception": false,
83+
"is_unknown": true,
84+
"owner": "Unspecified",
85+
"homepage_url": null,
86+
"text_url": "",
87+
"reference_url": "https://scancode-licensedb.aboutcode.org/unknown-license-reference",
88+
"scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.LICENSE",
89+
"scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.yml",
90+
"spdx_license_key": "LicenseRef-scancode-unknown-license-reference",
91+
"spdx_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.LICENSE",
92+
"start_line": 34,
93+
"end_line": 34,
94+
"matched_rule": {
95+
"identifier": "unknown-license-reference_25.RULE",
96+
"license_expression": "unknown-license-reference",
97+
"licenses": [
98+
"unknown-license-reference"
99+
],
100+
"referenced_filenames": [
101+
"LICENSE"
102+
],
103+
"is_license_text": false,
104+
"is_license_notice": false,
105+
"is_license_reference": false,
106+
"is_license_tag": true,
107+
"is_license_intro": false,
108+
"has_unknown": true,
109+
"matcher": "2-aho",
110+
"rule_length": 5,
111+
"matched_length": 5,
112+
"match_coverage": 100.0,
113+
"rule_relevance": 27
114+
},
115+
"matched_text": "license\": \"SEE LICENSE IN LICENSE."
116+
},
117+
{
118+
"key": "mit",
119+
"score": 100.0,
120+
"name": "MIT License",
121+
"short_name": "MIT License",
122+
"category": "Permissive",
123+
"is_exception": false,
124+
"is_unknown": false,
125+
"owner": "MIT",
126+
"homepage_url": "http://opensource.org/licenses/mit-license.php",
127+
"text_url": "http://opensource.org/licenses/mit-license.php",
128+
"reference_url": "https://scancode-licensedb.aboutcode.org/mit",
129+
"scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/mit.LICENSE",
130+
"scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/mit.yml",
131+
"spdx_license_key": "MIT",
132+
"spdx_url": "https://spdx.org/licenses/MIT",
133+
"start_line": 1,
134+
"end_line": 1,
135+
"matched_rule": {
136+
"identifier": "mit_66.RULE",
137+
"license_expression": "mit",
138+
"licenses": [
139+
"mit"
140+
],
141+
"referenced_filenames": [],
142+
"is_license_text": false,
143+
"is_license_notice": true,
144+
"is_license_reference": false,
145+
"is_license_tag": false,
146+
"is_license_intro": false,
147+
"has_unknown": false,
148+
"matcher": "1-hash",
149+
"rule_length": 10,
150+
"matched_length": 10,
151+
"match_coverage": 100.0,
152+
"rule_relevance": 100
153+
},
154+
"matched_text": "that is licensed under [MIT](http://opensource.org/licenses/MIT)."
155+
}
156+
],
157+
"license_expressions": [
158+
"unknown-license-reference",
159+
"mit"
160+
],
161+
"percentage_of_license_text": 0.2,
162+
"scan_errors": []
163+
}
164+
]
165+
}

‎tests/licensedcode/data/plugin_license/license_reference/scan-wref.expected.json‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
{
2+
"headers": [
3+
{
4+
"tool_name": "scancode-toolkit",
5+
"options": {
6+
"input": "<path>",
7+
"--json": "<file>",
8+
"--license": true,
9+
"--license-text": true,
10+
"--license-text-diagnostics": true,
11+
"--strip-root": true
12+
},
13+
"notice": "Generated with ScanCode and provided on an \"AS IS\" BASIS, WITHOUT WARRANTIES\nOR CONDITIONS OF ANY KIND, either express or implied. No content created from\nScanCode should be considered or used as legal advice. Consult an Attorney\nfor any legal advice.\nScanCode is a free software code scanning tool from nexB Inc. and others.\nVisit https://github.com/nexB/scancode-toolkit/ for support and download.",
14+
"message": null,
15+
"errors": [],
16+
"extra_data": {
17+
"files_count": 1
18+
}
19+
}
20+
],
21+
"files": [
22+
{
23+
"path": "license-notice.txt",
24+
"type": "file",
25+
"licenses": [
26+
{
27+
"key": "unknown-license-reference",
28+
"score": 27.0,
29+
"name": "Unknown License file reference",
30+
"short_name": "Unknown License reference",
31+
"category": "Unstated License",
32+
"is_exception": false,
33+
"is_unknown": true,
34+
"owner": "Unspecified",
35+
"homepage_url": null,
36+
"text_url": "",
37+
"reference_url": "https://scancode-licensedb.aboutcode.org/unknown-license-reference",
38+
"scancode_text_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.LICENSE",
39+
"scancode_data_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.yml",
40+
"spdx_license_key": "LicenseRef-scancode-unknown-license-reference",
41+
"spdx_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/licenses/unknown-license-reference.LICENSE",
42+
"start_line": 34,
43+
"end_line": 34,
44+
"matched_rule": {
45+
"identifier": "unknown-license-reference_25.RULE",
46+
"license_expression": "unknown-license-reference",
47+
"licenses": [
48+
"unknown-license-reference"
49+
],
50+
"referenced_filenames": [
51+
"LICENSE"
52+
],
53+
"is_license_text": false,
54+
"is_license_notice": false,
55+
"is_license_reference": false,
56+
"is_license_tag": true,
57+
"is_license_intro": false,
58+
"has_unknown": true,
59+
"matcher": "2-aho",
60+
"rule_length": 5,
61+
"matched_length": 5,
62+
"match_coverage": 100.0,
63+
"rule_relevance": 27
64+
},
65+
"matched_text": "license\": \"SEE LICENSE IN LICENSE."
66+
}
67+
],
68+
"license_expressions": [
69+
"unknown-license-reference"
70+
],
71+
"percentage_of_license_text": 0.2,
72+
"scan_errors": []
73+
}
74+
]
75+
}

‎tests/licensedcode/data/plugin_license/license_reference/scan/scan-ref/LICENSE‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
that is licensed under [MIT](http://opensource.org/licenses/MIT).

0 commit comments

Comments
 (0)