Skip to content

Refining scancode result #207

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Refining scancode result #207

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
df2e604
Scancode result cleanup:SPDX-license-identifier
JustinWonjaePark Apr 21, 2025
5ec2fb0
Scancode result cleanup:unkown-license-reference
JustinWonjaePark Apr 24, 2025
8a5a2a9
Remove unnecessary changes and fix flake8
JustinWonjaePark Apr 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 27 additions & 5 deletions src/fosslight_source/_parsing_scancode_file_item.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,14 @@
os.path.sep for dir_name in _exclude_directory]
_exclude_directory.append("/.")
REMOVE_LICENSE = ["warranty-disclaimer"]
regex = re.compile(r'licenseref-(\S+)', re.IGNORECASE)
regex = re.compile(r'(?:licenseref-|SPDX-license-identifier-)([^",\s]+)', re.IGNORECASE)
find_word = re.compile(rb"SPDX-PackageDownloadLocation\s*:\s*(\S+)", re.IGNORECASE)
KEYWORD_SPDX_ID = r'SPDX-License-Identifier\s*[\S]+'
KEYWORD_DOWNLOAD_LOC = r'DownloadLocation\s*[\S]+'
KEYWORD_SCANCODE_UNKNOWN = "unknown-spdx"
KEYWORD_SCANCODE_PROPRIETARY_LICENSE = "proprietary-license"
KEYWORD_UNKNOWN_LICENSE_REFERENCE = "unknown-license-reference"
KEYWORD_LGE_PROPRIETARY = "lge-proprietary"
SPDX_REPLACE_WORDS = ["(", ")"]
KEY_AND = r"(?<=\s)and(?=\s)"
KEY_OR = r"(?<=\s)or(?=\s)"
Expand Down Expand Up @@ -132,12 +135,12 @@ def parsing_scancode_32_earlier(scancode_file_list: list, has_error: bool = Fals
license_value = spdx.lower()

if license_value != "":
if key == KEYWORD_SCANCODE_UNKNOWN:
if key == KEYWORD_SCANCODE_UNKNOWN or key == KEYWORD_SCANCODE_PROPRIETARY_LICENSE:
try:
matched_txt = lic_item.get("matched_text", "").lower()
matched = regex.search(matched_txt)
if matched:
license_value = str(matched.group())
license_value = str(matched.group(1))
except Exception:
pass

Expand All @@ -162,6 +165,13 @@ def parsing_scancode_32_earlier(scancode_file_list: list, has_error: bool = Fals
result_item.is_license_text = matched_rule.get("is_license_text", False)

if len(license_detected) > 0:
# Remove unknown-license-reference and leave only lge-proprietary
license_lower = [x.lower() for x in license_detected]
if (
KEYWORD_UNKNOWN_LICENSE_REFERENCE.lower() in license_lower
and KEYWORD_LGE_PROPRIETARY.lower() in license_lower
):
license_detected.remove(KEYWORD_UNKNOWN_LICENSE_REFERENCE)
Comment on lines +168 to +174
Copy link
Contributor

@dd-jy dd-jy Apr 24, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@JustinWonjaePark
다른 문구에서 검출되었다면 unknown 유지해야 하는 것이 아닐지 문의드립니다.

result_item.licenses = license_detected

if len(license_expression_list) > 0:
Expand Down Expand Up @@ -241,11 +251,14 @@ def parsing_scancode_32_later(
found_lic = found_lic.strip()
if found_lic in REMOVE_LICENSE:
continue
elif found_lic == KEYWORD_SCANCODE_UNKNOWN:
elif (
found_lic == KEYWORD_SCANCODE_UNKNOWN
or found_lic == KEYWORD_SCANCODE_PROPRIETARY_LICENSE
):
try:
matched = regex.search(matched_txt.lower())
if matched:
found_lic = str(matched.group())
found_lic = str(matched.group(1))
except Exception:
pass
for word in replace_word:
Expand All @@ -258,6 +271,15 @@ def parsing_scancode_32_later(
lic_info = MatchedLicense(found_lic, "", matched_txt, file_path)
license_list[lic_matched_key] = lic_info
license_detected.append(found_lic)

# Remove unknown-license-reference and leave only lge-proprietary
license_lower = [x.lower() for x in license_detected]
if (
KEYWORD_UNKNOWN_LICENSE_REFERENCE.lower() in license_lower
and KEYWORD_LGE_PROPRIETARY.lower() in license_lower
):
license_detected.remove(KEYWORD_UNKNOWN_LICENSE_REFERENCE)

result_item.licenses = license_detected
if len(license_detected) > 1:
license_expression_spdx = file.get("detected_license_expression_spdx", "")
Expand Down
6 changes: 4 additions & 2 deletions tox.ini
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ commands =
pytest -v --flake8

[testenv:flake8]
basepython = python3.11
deps = flake8
commands =
pytest tests/test_tox.py::test_flake8
changedir = {toxinidir}
commands =
flake8 src/fosslight_source tests
Loading