From 280e83fa5deb0c55a84e2bd4dd52c8c165d50165 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Wed, 25 Sep 2024 16:17:32 -0700 Subject: [PATCH 1/6] Handle yarn.lock dependencies with @ in constraint #3931 Signed-off-by: Jono Yang --- src/packagedcode/npm.py | 22 ++- .../yarn-lock/v1-other-constraint/yarn.lock | 14 ++ .../v1-other-constraint/yarn.lock-expected | 157 ++++++++++++++++++ tests/packagedcode/test_npm.py | 7 + 4 files changed, 193 insertions(+), 7 deletions(-) create mode 100644 tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock create mode 100644 tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock-expected diff --git a/src/packagedcode/npm.py b/src/packagedcode/npm.py index 4f7c1f3aa3..a4d1c72b2d 100644 --- a/src/packagedcode/npm.py +++ b/src/packagedcode/npm.py @@ -207,7 +207,7 @@ def assemble(cls, package_data, resource, codebase, package_adder): @classmethod def yield_npm_dependencies_and_resources(cls, package_resource, package_data, package_uid, codebase, package_adder): - + # in all cases yield possible dependencies yield from yield_dependencies_from_package_data(package_data, package_resource.path, package_uid) @@ -424,7 +424,7 @@ def get_workspace_members(cls, workspaces, codebase, workspace_root_path): workspace_members.append(resource) # Case 3: This is a complex glob pattern, we are doing a full codebase walk - # and glob matching each resource + # and glob matching each resource else: for resource in workspace_root_path: if NpmPackageJsonHandler.is_datafile(resource.location) and fnmatch.fnmatch( @@ -469,7 +469,7 @@ def update_workspace_members(cls, workspace_members, codebase): workspace_package_versions_by_base_purl[base_purl] = version # Update workspace member package information from - # workspace level data + # workspace level data for base_purl, dependency in workspace_dependencies_by_base_purl.items(): extracted_requirement = dependency.get('extracted_requirement') if 'workspace' in extracted_requirement: @@ -1011,6 +1011,14 @@ def parse(cls, location, package_only=False): if '"' in ns_name: ns_name = ns_name.replace('"', '') ns, _ , name = ns_name.rpartition('/') + + # sometimes constraints appear in the form of + # wrap-ansi-cjs "npm:wrap-ansi@^7.0.0" + if '@' in constraint: + # "npm:wrap-ansi" should be appended to `name`, joined + # with an "@" + constraint_package, _, constraint = constraint.partition('@') + name = f'{name}@{constraint_package}' sub_dependencies.append((ns, name, constraint,)) elif line.startswith(' ' * 2): @@ -1112,7 +1120,7 @@ def parse(cls, location, package_only=False): resolved_package=resolved_package_data.to_dict(), ) - if not dep_purl in dependencies_by_purl: + if not dep_purl in dependencies_by_purl: dependencies_by_purl[dep_purl] = dep.to_dict() else: # FIXME: We have duplicate dependencies because of aliases @@ -1176,7 +1184,7 @@ def parse(cls, location, package_only=False): _, name_version = sections elif len(sections) == 3: _, namespace, name_version = sections - + name, version = name_version.split("@") elif major_v == "5" or is_shrinkwrap: if len(sections) == 3: @@ -1264,7 +1272,7 @@ def parse(cls, location, package_only=False): for key in extra_data_fields: value = data.get(key, None) if value is not None: - extra_data_deps[key] = value + extra_data_deps[key] = value dependency_data = models.DependentPackage( purl=purl, @@ -1762,7 +1770,7 @@ def deps_mapper(deps, package, field_name, is_direct=True): deps_by_name[npm_name] = d for fqname, requirement in deps.items(): - # Handle cases in ``resolutions`` with ``**`` + # Handle cases in ``resolutions`` with ``**`` # "resolutions": { # "**/@typescript-eslint/eslint-plugin": "^4.1.1", if fqname.startswith('**'): diff --git a/tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock b/tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock new file mode 100644 index 0000000000..51b688b792 --- /dev/null +++ b/tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock @@ -0,0 +1,14 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +"@isaacs/cliui@^8.0.2": + version "8.0.2" + resolved "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz#b37667b7bc181c168782259bab42474fbf52b550" + integrity sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA== + dependencies: + string-width "^5.1.2" + string-width-cjs "npm:string-width@^4.2.0" + strip-ansi "^7.0.1" + strip-ansi-cjs "npm:strip-ansi@^6.0.1" + wrap-ansi "^8.1.0" diff --git a/tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock-expected b/tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock-expected new file mode 100644 index 0000000000..ac6cf2adff --- /dev/null +++ b/tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock-expected @@ -0,0 +1,157 @@ +[ + { + "type": "npm", + "namespace": null, + "name": null, + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/%40isaacs/cliui@8.0.2", + "extracted_requirement": "^8.0.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "@isaacs", + "name": "cliui", + "version": "8.0.2", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.yarnpkg.com/@isaacs/cliui/-/cliui-8.0.2.tgz", + "size": null, + "sha1": "b37667b7bc181c168782259bab42474fbf52b550", + "md5": null, + "sha256": null, + "sha512": "3bc8dc8da6d76a578e1bd0d0d3e0115d66414df9cfe16340ab3ba224aee5978e009b118abff2763384cf8f18d8df39c109fbc15c5cee726d6dc1dc85c9b16a10", + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": true, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/string-width", + "extracted_requirement": "^5.1.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/string-width-cjs%40%22npm:string-width", + "extracted_requirement": "^4.2.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/strip-ansi", + "extracted_requirement": "^7.0.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/strip-ansi-cjs%40%22npm:strip-ansi", + "extracted_requirement": "^6.0.1", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/wrap-ansi", + "extracted_requirement": "^8.1.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://www.npmjs.com/package/@isaacs/cliui", + "repository_download_url": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "api_data_url": "https://registry.npmjs.org/@isaacs%2fcliui/8.0.2", + "datasource_id": "yarn_lock_v1", + "purl": "pkg:npm/%40isaacs/cliui@8.0.2" + }, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "yarn_lock_v1", + "purl": null + } +] \ No newline at end of file diff --git a/tests/packagedcode/test_npm.py b/tests/packagedcode/test_npm.py index b1bce0b702..cae68b4c13 100644 --- a/tests/packagedcode/test_npm.py +++ b/tests/packagedcode/test_npm.py @@ -360,6 +360,13 @@ def test_npm_yarn_with_package_json_resolve_dependencies(self): expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES ) + def test_npm_yarn_lock_v1_parse_with_other_version_constraint(self): + test_file = self.get_test_loc('npm/yarn-lock/v1-other-constraint/yarn.lock') + expected_loc = self.get_test_loc( + 'npm/yarn-lock/v1-other-constraint/yarn.lock-expected') + packages = npm.YarnLockV1Handler.parse(test_file) + self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES) + def test_is_datafile_pnpm_shrinkwrap_yaml(self): test_file = self.get_test_loc('npm/pnpm/shrinkwrap/v3/vuepack/shrinkwrap.yaml') assert npm.PnpmShrinkwrapYamlHandler.is_datafile(test_file) From c43ec3464dcbcabc8d7c916dfcf4df5b847c628e Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 26 Sep 2024 16:28:22 -0700 Subject: [PATCH 2/6] Use package_adder when assigning workspace package uids Signed-off-by: Jono Yang --- src/packagedcode/npm.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/packagedcode/npm.py b/src/packagedcode/npm.py index a4d1c72b2d..38bcbba287 100644 --- a/src/packagedcode/npm.py +++ b/src/packagedcode/npm.py @@ -276,7 +276,9 @@ def create_packages_from_workspaces( if npm_res.for_packages: continue - npm_res.for_packages = workspace_package_uids + for package_uid in workspace_package_uids: + package_adder(package_uid, npm_res, codebase) + npm_res.save(codebase) @classmethod From e4318f45c0f090d8eb392e7e278211b412b70c82 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Fri, 27 Sep 2024 17:20:42 -0700 Subject: [PATCH 3/6] Handle dependencies with @ in constraint for npm manifest files #3931 Signed-off-by: Jono Yang --- src/packagedcode/npm.py | 3 + .../package.json | 68 ++++++++++++++ .../package.json.expected | 88 +++++++++++++++++++ tests/packagedcode/test_npm.py | 6 ++ 4 files changed, 165 insertions(+) create mode 100755 tests/packagedcode/data/npm/special_extracted_requirements/package.json create mode 100755 tests/packagedcode/data/npm/special_extracted_requirements/package.json.expected diff --git a/src/packagedcode/npm.py b/src/packagedcode/npm.py index 38bcbba287..8b2e786418 100644 --- a/src/packagedcode/npm.py +++ b/src/packagedcode/npm.py @@ -1780,6 +1780,9 @@ def deps_mapper(deps, package, field_name, is_direct=True): ns, name = split_scoped_package_name(fqname) if not name: continue + if '@' in requirement: + requirement_package, _, requirement = requirement.partition('@') + name = f'{name}@{requirement_package}' purl = PackageURL(type='npm', namespace=ns, name=name).to_string() # optionalDependencies override the dependencies with the same name diff --git a/tests/packagedcode/data/npm/special_extracted_requirements/package.json b/tests/packagedcode/data/npm/special_extracted_requirements/package.json new file mode 100755 index 0000000000..fb910a8600 --- /dev/null +++ b/tests/packagedcode/data/npm/special_extracted_requirements/package.json @@ -0,0 +1,68 @@ +{ + "name": "@tapjs/tapjs", + "private": true, + "workspaces": [ + "src/*" + ], + "type": "module", + "prettier": { + "experimentalTernaries": true, + "semi": false, + "printWidth": 70, + "tabWidth": 2, + "useTabs": false, + "singleQuote": true, + "jsxSingleQuote": false, + "bracketSameLine": true, + "arrowParens": "avoid", + "endOfLine": "lf" + }, + "devDependencies": { + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1" + }, + "scripts": { + "start": "npm run start -w src/docs", + "predocsbuild": "npm run bootstrap", + "docsbuild": "npm run build -w src/docs", + "pretest": "rimraf src/test/test-built/dist/node_modules", + "presnap": "rimraf src/test/test-built/dist/node_modules", + "test": "nx run-many -t test", + "test:bootstrap": "bash ./scripts/test-bootstrap.sh", + "snap": "TAP_TYPECHECK=0 TAP_TIMEOUT=240 nx run-many -t snap", + "format": "nx run-many -t format", + "typedoc": "typedoc", + "bootstrap": "bash ./scripts/bootstrap.sh", + "build": "node ./scripts/default-build.mjs", + "pindeps": "node ./scripts/version.mjs pindeps", + "v": "node ./scripts/version.mjs", + "p": "bash ./scripts/bump-changed.sh", + "pub": "npm run v -- pub", + "deploy:docs": "npm run deploy:prod -w src/docs", + "postv": "npm run deploy:docs", + "pj": "node scripts/normalize-package-json.js src/*/package.json" + }, + "repository": "https://github.com/tapjs/tapjs", + "license": "BlueOak-1.0.0", + "engines": { + "node": "20 || >=22" + }, + "overrides": { + "braces@3": "^3.0.3", + "axios@1.0.0 - 1.5.1": "^1.7.2", + "netlify-cli": { + "braces": "^3.0.3", + "micromatch": "^4.0.7", + "chokidar": { + "braces": "^3.0.3" + }, + "http-proxy-middleware": { + "micromatch": { + ".": "^4.0.7", + "braces": "^3.0.3" + } + } + }, + "micromatch@4.0.5": "^4.0.7", + "tar@6.1.11": "6.2" + } +} diff --git a/tests/packagedcode/data/npm/special_extracted_requirements/package.json.expected b/tests/packagedcode/data/npm/special_extracted_requirements/package.json.expected new file mode 100755 index 0000000000..07e89067ca --- /dev/null +++ b/tests/packagedcode/data/npm/special_extracted_requirements/package.json.expected @@ -0,0 +1,88 @@ +[ + { + "type": "npm", + "namespace": "@tapjs", + "name": "tapjs", + "version": null, + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": "https://github.com/tapjs/tapjs", + "copyright": null, + "holder": null, + "declared_license_expression": "blueoak-1.0.0", + "declared_license_expression_spdx": "BlueOak-1.0.0", + "license_detections": [ + { + "license_expression": "blueoak-1.0.0", + "license_expression_spdx": "BlueOak-1.0.0", + "matches": [ + { + "license_expression": "blueoak-1.0.0", + "spdx_license_expression": "BlueOak-1.0.0", + "from_file": null, + "start_line": 1, + "end_line": 1, + "matcher": "1-spdx-id", + "score": 100.0, + "matched_length": 4, + "match_coverage": 100.0, + "rule_relevance": 100, + "rule_identifier": "spdx-license-identifier-blueoak_1_0_0-a27d3d91aab5047de087c05901869c5f4a1f12e7", + "rule_url": null, + "matched_text": "BlueOak-1.0.0" + } + ], + "identifier": "blueoak_1_0_0-3c25fef0-5634-6497-b6c8-0c16ab8320b3" + } + ], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": "- BlueOak-1.0.0\n", + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": true, + "is_virtual": false, + "extra_data": { + "workspaces": [ + "src/*" + ], + "engines": { + "node": "20 || >=22" + } + }, + "dependencies": [ + { + "purl": "pkg:npm/strip-ansi-cjs%40npm:strip-ansi", + "extracted_requirement": "^6.0.1", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": null, + "repository_download_url": null, + "api_data_url": null, + "datasource_id": "npm_package_json", + "purl": "pkg:npm/%40tapjs/tapjs" + } +] \ No newline at end of file diff --git a/tests/packagedcode/test_npm.py b/tests/packagedcode/test_npm.py index cae68b4c13..30acb962e2 100644 --- a/tests/packagedcode/test_npm.py +++ b/tests/packagedcode/test_npm.py @@ -393,6 +393,12 @@ def test_parse_pnpm_shrinkwrap_yaml(self): packages = npm.PnpmLockYamlHandler.parse(test_file) self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES) + def test_parse_package_json_special_dep_requirements(self): + test_file = self.get_test_loc('npm/special_extracted_requirements/package.json') + expected_loc = self.get_test_loc('npm/special_extracted_requirements/package.json.expected') + packages = npm.NpmPackageJsonHandler.parse(test_file) + self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES) + def test_pnpm_scan_with_workspace_package_json(self): test_folder = self.get_test_loc('npm/pnpm/pnpm-lock/v5/cobe/') expected_file = self.get_test_loc('npm/pnpm/pnpm-lock/v5/cobe-scan.expected.json') From 762078c1bf489a757339db14238bb17a838483bc Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 3 Oct 2024 12:00:13 -0700 Subject: [PATCH 4/6] Report package in npm dep requirement #3931 Signed-off-by: Jono Yang --- src/packagedcode/npm.py | 35 +++++++++++++------ .../package.json.expected | 2 +- .../data/npm/workspace/crystal.expected.json | 10 +++--- .../v1-other-constraint/yarn.lock-expected | 4 +-- 4 files changed, 33 insertions(+), 18 deletions(-) diff --git a/src/packagedcode/npm.py b/src/packagedcode/npm.py index 8b2e786418..5fa2292b34 100644 --- a/src/packagedcode/npm.py +++ b/src/packagedcode/npm.py @@ -376,6 +376,18 @@ def update_dependencies_by_purl( if '_' in metadata: requirement, _extra = metadata.split('_') + if ':' in requirement and '@' in requirement: + # dependencies with requirements like this are aliases and should be reported + aliased_package, _, constraint = requirement.rpartition('@') + _, _, aliased_package_name = aliased_package.rpartition(':') + sdns, _ , sdname = aliased_package_name.rpartition('/') + dep_purl = PackageURL( + type=cls.default_package_type, + namespace=sdns, + name=sdname + ).to_string() + requirement = constraint + dep_package = models.DependentPackage( purl=dep_purl, scope=scope, @@ -1014,13 +1026,12 @@ def parse(cls, location, package_only=False): ns_name = ns_name.replace('"', '') ns, _ , name = ns_name.rpartition('/') - # sometimes constraints appear in the form of - # wrap-ansi-cjs "npm:wrap-ansi@^7.0.0" - if '@' in constraint: - # "npm:wrap-ansi" should be appended to `name`, joined - # with an "@" - constraint_package, _, constraint = constraint.partition('@') - name = f'{name}@{constraint_package}' + if ':' in constraint and '@' in constraint: + # dependencies with requirements like this are aliases and should be reported + aliased_package, _, constraint = constraint.rpartition('@') + _, _, aliased_package_name = aliased_package.rpartition(':') + ns, _ , name = aliased_package_name.rpartition('/') + sub_dependencies.append((ns, name, constraint,)) elif line.startswith(' ' * 2): @@ -1780,9 +1791,13 @@ def deps_mapper(deps, package, field_name, is_direct=True): ns, name = split_scoped_package_name(fqname) if not name: continue - if '@' in requirement: - requirement_package, _, requirement = requirement.partition('@') - name = f'{name}@{requirement_package}' + + if ':' in requirement and '@' in requirement: + # dependencies with requirements like this are aliases and should be reported + aliased_package, _, requirement = requirement.rpartition('@') + _, _, aliased_package_name = aliased_package.rpartition(':') + ns, _ , name = aliased_package_name.rpartition('/') + purl = PackageURL(type='npm', namespace=ns, name=name).to_string() # optionalDependencies override the dependencies with the same name diff --git a/tests/packagedcode/data/npm/special_extracted_requirements/package.json.expected b/tests/packagedcode/data/npm/special_extracted_requirements/package.json.expected index 07e89067ca..89bb0db8d8 100755 --- a/tests/packagedcode/data/npm/special_extracted_requirements/package.json.expected +++ b/tests/packagedcode/data/npm/special_extracted_requirements/package.json.expected @@ -68,7 +68,7 @@ }, "dependencies": [ { - "purl": "pkg:npm/strip-ansi-cjs%40npm:strip-ansi", + "purl": "pkg:npm/strip-ansi", "extracted_requirement": "^6.0.1", "scope": "devDependencies", "is_runtime": false, diff --git a/tests/packagedcode/data/npm/workspace/crystal.expected.json b/tests/packagedcode/data/npm/workspace/crystal.expected.json index fd30da2696..09e6eece4f 100644 --- a/tests/packagedcode/data/npm/workspace/crystal.expected.json +++ b/tests/packagedcode/data/npm/workspace/crystal.expected.json @@ -9960,8 +9960,8 @@ "datasource_id": "npm_package_json" }, { - "purl": "pkg:npm/%40localrepo/prettier2-for-jest", - "extracted_requirement": "npm:prettier@^2", + "purl": "pkg:npm/prettier", + "extracted_requirement": "^2", "scope": "devDependencies", "is_runtime": false, "is_optional": true, @@ -9969,7 +9969,7 @@ "is_direct": true, "resolved_package": {}, "extra_data": {}, - "dependency_uid": "pkg:npm/%40localrepo/prettier2-for-jest?uuid=fixed-uid-done-for-testing-5642512d1758", + "dependency_uid": "pkg:npm/prettier?uuid=fixed-uid-done-for-testing-5642512d1758", "for_package_uid": null, "datafile_path": "crystal/package.json", "datasource_id": "npm_package_json" @@ -18249,8 +18249,8 @@ "extra_data": {} }, { - "purl": "pkg:npm/%40localrepo/prettier2-for-jest", - "extracted_requirement": "npm:prettier@^2", + "purl": "pkg:npm/prettier", + "extracted_requirement": "^2", "scope": "devDependencies", "is_runtime": false, "is_optional": true, diff --git a/tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock-expected b/tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock-expected index ac6cf2adff..5c6d955a7e 100644 --- a/tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock-expected +++ b/tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock-expected @@ -95,7 +95,7 @@ "extra_data": {} }, { - "purl": "pkg:npm/string-width-cjs%40%22npm:string-width", + "purl": "pkg:npm/string-width", "extracted_requirement": "^4.2.0", "scope": "dependencies", "is_runtime": true, @@ -117,7 +117,7 @@ "extra_data": {} }, { - "purl": "pkg:npm/strip-ansi-cjs%40%22npm:strip-ansi", + "purl": "pkg:npm/strip-ansi", "extracted_requirement": "^6.0.1", "scope": "dependencies", "is_runtime": true, From 8b63e76f786ee98de0f46e98e99cb7d1ffc8a36f Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Thu, 3 Oct 2024 12:37:40 -0700 Subject: [PATCH 5/6] Add test for package-lock.json #3931 Signed-off-by: Jono Yang --- .../data/npm/alias/package-lock.json | 57 +++++++ .../data/npm/alias/package-lock.json.expected | 149 ++++++++++++++++++ .../package.json | 0 .../package.json.expected | 0 .../yarn.lock | 0 .../yarn.lock-expected | 0 tests/packagedcode/test_npm.py | 19 ++- 7 files changed, 219 insertions(+), 6 deletions(-) create mode 100644 tests/packagedcode/data/npm/alias/package-lock.json create mode 100644 tests/packagedcode/data/npm/alias/package-lock.json.expected rename tests/packagedcode/data/npm/{special_extracted_requirements => alias}/package.json (100%) rename tests/packagedcode/data/npm/{special_extracted_requirements => alias}/package.json.expected (100%) rename tests/packagedcode/data/npm/yarn-lock/{v1-other-constraint => v1-alias}/yarn.lock (100%) rename tests/packagedcode/data/npm/yarn-lock/{v1-other-constraint => v1-alias}/yarn.lock-expected (100%) diff --git a/tests/packagedcode/data/npm/alias/package-lock.json b/tests/packagedcode/data/npm/alias/package-lock.json new file mode 100644 index 0000000000..024ce5c343 --- /dev/null +++ b/tests/packagedcode/data/npm/alias/package-lock.json @@ -0,0 +1,57 @@ +{ + "name": "poshi-language-support", + "version": "0.3.7", + "lockfileVersion": 2, + "requires": true, + "packages": { + "": { + "name": "poshi-language-support", + "version": "0.3.7", + "dependencies": { + "@vscode/ripgrep": "^1.14.2", + "lodash": "^4.17.21" + }, + "devDependencies": { + "@ava/typescript": "^4.1.0", + "@types/glob": "^7.2.0", + "@types/lodash": "^4.14.178", + "@types/mocha": "^9.1.0", + "@types/node": "14.x", + "@types/vscode": "^1.64.0", + "@typescript-eslint/eslint-plugin": "^5.12.1", + "@typescript-eslint/parser": "^5.12.1", + "@vscode/test-electron": "^2.1.2", + "@vscode/vsce": "^2.22.0", + "ava": "^5.3.1", + "esbuild": "^0.18.15", + "eslint": "^8.9.0", + "glob": "^7.2.0", + "mocha": "^9.2.1", + "prettier": "^3.0.0", + "rimraf": "^5.0.1", + "typescript": "^4.5.5" + }, + "engines": { + "node": ">=16", + "vscode": "^1.64.0" + } + }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + } + } +} diff --git a/tests/packagedcode/data/npm/alias/package-lock.json.expected b/tests/packagedcode/data/npm/alias/package-lock.json.expected new file mode 100644 index 0000000000..2bfba64986 --- /dev/null +++ b/tests/packagedcode/data/npm/alias/package-lock.json.expected @@ -0,0 +1,149 @@ +[ + { + "type": "npm", + "namespace": "", + "name": "poshi-language-support", + "version": "0.3.7", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": null, + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": { + "lockfile_version": 2 + }, + "dependencies": [ + { + "purl": "pkg:npm/%40isaacs/cliui@8.0.2", + "extracted_requirement": "8.0.2", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_resolved": true, + "is_direct": false, + "resolved_package": { + "type": "npm", + "namespace": "@isaacs", + "name": "cliui", + "version": "8.0.2", + "qualifiers": {}, + "subpath": null, + "primary_language": "JavaScript", + "description": null, + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": "3bc8dc8da6d76a578e1bd0d0d3e0115d66414df9cfe16340ab3ba224aee5978e009b118abff2763384cf8f18d8df39c109fbc15c5cee726d6dc1dc85c9b16a10", + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [ + [ + { + "path": "node_modules/@isaacs/cliui", + "size": 0, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "extra_data": {} + } + ] + ], + "is_private": false, + "is_virtual": true, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:npm/string-width", + "extracted_requirement": "^4.2.0", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/strip-ansi", + "extracted_requirement": "^6.0.1", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:npm/wrap-ansi", + "extracted_requirement": "^7.0.0", + "scope": "devDependencies", + "is_runtime": false, + "is_optional": true, + "is_resolved": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://www.npmjs.com/package/@isaacs/cliui", + "repository_download_url": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "api_data_url": "https://registry.npmjs.org/@isaacs%2fcliui/8.0.2", + "datasource_id": "npm_package_lock_json", + "purl": "pkg:npm/%40isaacs/cliui@8.0.2" + }, + "extra_data": {} + } + ], + "repository_homepage_url": "https://www.npmjs.com/package/poshi-language-support", + "repository_download_url": "https://registry.npmjs.org/poshi-language-support/-/poshi-language-support-0.3.7.tgz", + "api_data_url": "https://registry.npmjs.org/poshi-language-support/0.3.7", + "datasource_id": "npm_package_lock_json", + "purl": "pkg:npm/poshi-language-support@0.3.7" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/npm/special_extracted_requirements/package.json b/tests/packagedcode/data/npm/alias/package.json similarity index 100% rename from tests/packagedcode/data/npm/special_extracted_requirements/package.json rename to tests/packagedcode/data/npm/alias/package.json diff --git a/tests/packagedcode/data/npm/special_extracted_requirements/package.json.expected b/tests/packagedcode/data/npm/alias/package.json.expected similarity index 100% rename from tests/packagedcode/data/npm/special_extracted_requirements/package.json.expected rename to tests/packagedcode/data/npm/alias/package.json.expected diff --git a/tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock b/tests/packagedcode/data/npm/yarn-lock/v1-alias/yarn.lock similarity index 100% rename from tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock rename to tests/packagedcode/data/npm/yarn-lock/v1-alias/yarn.lock diff --git a/tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock-expected b/tests/packagedcode/data/npm/yarn-lock/v1-alias/yarn.lock-expected similarity index 100% rename from tests/packagedcode/data/npm/yarn-lock/v1-other-constraint/yarn.lock-expected rename to tests/packagedcode/data/npm/yarn-lock/v1-alias/yarn.lock-expected diff --git a/tests/packagedcode/test_npm.py b/tests/packagedcode/test_npm.py index 30acb962e2..2e5d5e4f33 100644 --- a/tests/packagedcode/test_npm.py +++ b/tests/packagedcode/test_npm.py @@ -275,6 +275,13 @@ def test_parse_package_lock_v2_2(self): packages = npm.NpmPackageLockJsonHandler.parse(test_file) self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES) + def test_parse_package_lock_v2_alias(self): + test_file = self.get_test_loc('npm/alias/package-lock.json') + expected_loc = self.get_test_loc( + 'npm/alias/package-lock.json.expected') + packages = npm.NpmPackageLockJsonHandler.parse(test_file) + self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES) + def test_is_datafile_npm_shrinkwrap_json(self): test_file = self.get_test_loc('npm/npm-shrinkwrap/npm-shrinkwrap.json') assert npm.NpmShrinkwrapJsonHandler.is_datafile(test_file) @@ -360,10 +367,10 @@ def test_npm_yarn_with_package_json_resolve_dependencies(self): expected_file, result_file, remove_uuid=True, regen=REGEN_TEST_FIXTURES ) - def test_npm_yarn_lock_v1_parse_with_other_version_constraint(self): - test_file = self.get_test_loc('npm/yarn-lock/v1-other-constraint/yarn.lock') + def test_npm_yarn_lock_v1_parse_alias(self): + test_file = self.get_test_loc('npm/yarn-lock/v1-alias/yarn.lock') expected_loc = self.get_test_loc( - 'npm/yarn-lock/v1-other-constraint/yarn.lock-expected') + 'npm/yarn-lock/v1-alias/yarn.lock-expected') packages = npm.YarnLockV1Handler.parse(test_file) self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES) @@ -393,9 +400,9 @@ def test_parse_pnpm_shrinkwrap_yaml(self): packages = npm.PnpmLockYamlHandler.parse(test_file) self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES) - def test_parse_package_json_special_dep_requirements(self): - test_file = self.get_test_loc('npm/special_extracted_requirements/package.json') - expected_loc = self.get_test_loc('npm/special_extracted_requirements/package.json.expected') + def test_parse_package_json_alias_requirements(self): + test_file = self.get_test_loc('npm/alias/package.json') + expected_loc = self.get_test_loc('npm/alias/package.json.expected') packages = npm.NpmPackageJsonHandler.parse(test_file) self.check_packages_data(packages, expected_loc, regen=REGEN_TEST_FIXTURES) From e0ff11e9bcac1f516377225233f50c091aee9fd6 Mon Sep 17 00:00:00 2001 From: Jono Yang Date: Tue, 8 Oct 2024 12:52:43 -0700 Subject: [PATCH 6/6] Update CHANGELOG.rst Signed-off-by: Jono Yang --- CHANGELOG.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 57cda84a0f..b283b4b08b 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -886,6 +886,8 @@ Package detection: - For Pypi packages, python_requires is treated as a package dependency. +- Update JavaScript package handlers to handle aliases in npm and yarn manifests. + License Clarity Scoring Update: ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~