updates to resolvers. pytests failing

Author: FredHappyface

licensecheck/cli.py

@@ -8,7 +8,8 @@
 from dataclasses import fields
 from pathlib import Path
 from sys import exit as sysexit
-from sys import stdout
+from sys import stdout, stdin
+from typing import TextIO
 
 from fhconfparser import FHConfParser, SimpleConf
 
@@ -30,17 +31,24 @@ def cli() -> None:  # pragma: no cover
 		"-f",
 		help=f"Output format. one of: {', '.join(list(formatter.formatMap))}. default=simple",
 	)
+	parser.add_argument(
+		"--deps-file",
+		"-i",
+		"-d",
+		help="Filename to read from (omit for stdin)",
+		nargs="+",
+	)
+	parser.add_argument(
+		"--groups",
+		"-g",
+		help="Select groups from supported files",
+		nargs="+",
+	)
 	parser.add_argument(
 		"--file",
 		"-o",
 		help="Filename to write to (omit for stdout)",
 	)
-	parser.add_argument(
-		"--using",
-		"-u",
-		help="Environment to use e.g. requirements.txt. one of: "
-		f"{', '.join(get_deps.USINGS)}. default=poetry",
-	)
 	parser.add_argument(
 		"--ignore-packages",
 		help="a list of packages to ignore (compat=True)",
@@ -110,31 +118,34 @@ def main(args: dict) -> int:
 	simpleConf = SimpleConf(configparser, "licensecheck", args)
 
 	# File
-	textIO = (
+	input_file = (
+		simpleConf.get("deps_file")  or stdin
+	)
+	output_file = (
 		stdout
 		if simpleConf.get("file") is None
 		else Path(simpleConf.get("file")).open("w", encoding="utf-8")
 	)
 
 	# Get my license
-	myLiceTxt = args["license"] if args.get("license") else packageinfo.getMyPackageLicense()
-	myLice = license_matrix.licenseType(myLiceTxt)[0]
-
-	# Get list of licenses
-	depsWithLicenses = get_deps.getDepsWithLicenses(
-		simpleConf.get("using", "poetry"),
-		myLice,
-		list(map(types.ucstr, simpleConf.get("ignore_packages", []))),
-		list(map(types.ucstr, simpleConf.get("fail_packages", []))),
-		list(map(types.ucstr, simpleConf.get("ignore_licenses", []))),
-		list(map(types.ucstr, simpleConf.get("fail_licenses", []))),
-		list(map(types.ucstr, simpleConf.get("only_licenses", []))),
-		list(map(types.ucstr, simpleConf.get("skip_dependencies", []))),
+	this_license_text = args["license"] if args.get("license") else packageinfo.getMyPackageLicense()
+	this_license = license_matrix.licenseType(this_license_text)[0]
+
+	def getFromConfig(key: str) -> list[types.ucstr]:
+		return list(map(types.ucstr, simpleConf.get(key, [])))
+
+	incompatible, depsWithLicenses = get_deps.check(
+		input_file=input_file,
+		groups=simpleConf.get("groups", []),
+		this_license=this_license,
+		ignore_packages=getFromConfig("ignore_packages"),
+		fail_packages=getFromConfig("fail_packages"),
+		ignore_licenses=getFromConfig("ignore_licenses"),
+		fail_licenses=getFromConfig("fail_licenses"),
+		only_licenses=getFromConfig("only_licenses"),
+		skip_dependencies=getFromConfig("skip_dependencies"),
 	)
 
-	# Are any licenses incompatible?
-	incompatible = any(not lice.licenseCompat for lice in depsWithLicenses)
-
 	# Format the results
 	hide_output_parameters = [types.ucstr(x) for x in simpleConf.get("hide_output_parameters", [])]
 	available_params = [param.name.upper() for param in fields(types.PackageInfo)]
@@ -147,11 +158,11 @@ def main(args: dict) -> int:
 	if simpleConf.get("format", "simple") in formatter.formatMap:
 		print(
 			formatter.formatMap[simpleConf.get("format", "simple")](
-				myLice,
+				this_license,
 				sorted(depsWithLicenses),
 				hide_output_parameters,
 			),
-			file=textIO,
+			file=output_file,
 		)
 	else:
 		exitCode = 2
@@ -162,5 +173,5 @@ def main(args: dict) -> int:
 
 	# Cleanup + exit
 	if simpleConf.get("file") is not None:
-		textIO.close()
+		output_file.close()
 	return exitCode

licensecheck/get_deps.py

@@ -11,126 +11,89 @@
 from licensecheck.resolvers import uv as res_uv
 from licensecheck.types import JOINS, License, PackageInfo, ucstr
 
-USINGS = ["requirements", "poetry", "PEP631"]
 
 
-def getReqs(using: str, skipDependencies: list[ucstr]) -> set[ucstr]:
-	"""Get requirements for the end user project/ lib.
+def resolve_requirements(
+	requirements_paths: list[str],
+	groups: list[str],
+	skip_dependencies: list[ucstr],
+	) -> set[ucstr]:
 
-	>>> getReqs("poetry")
-	>>> getReqs("poetry:dev")
-	>>> getReqs("requirements")
-	>>> getReqs("requirements:requirements.txt;requirements-dev.txt")
-	>>> getReqs("PEP631")
-	>>> getReqs("PEP631:tests")
-
-	Args:
-	----
-		using (str): use requirements, poetry or PEP631.
-		skipDependencies (list[str]): list of dependencies to skip.
-
-	Returns:
-	-------
-		set[str]: set of requirement packages
-
-	"""
-
-	_ = using.split(":", 1)
-	using = _[0]
-	extras = _[1].split(";") if len(_) > 1 else []
-	if using not in USINGS:
-		using = "poetry"
-
-	# if using poetry or pep621
-	requirementsPaths = ["pyproject.toml"]
-
-	# Requirements
-	if using == "requirements":
-		requirementsPaths = ["requirements.txt"] if len(extras) > 0 else extras
-		extras = []
 
 	try:
 		return res_uv.get_reqs(
-			using=using,
-			skipDependencies=skipDependencies,
-			extras=extras,
-			requirementsPaths=requirementsPaths,
+			skipDependencies=skip_dependencies,
+			extras=groups,
+			requirementsPaths=requirements_paths,
 		)
 
 	except RuntimeError:
 		pyproject = {}
-		if "pyproject.toml" in requirementsPaths:
+		if "pyproject.toml" in requirements_paths:
 			pyproject = tomli.loads(Path("pyproject.toml").read_text("utf-8"))
 
 		# Fallback to the old resolver (hopefully we can deprecate this asap!)
 		return res_native.get_reqs(
-			using=using,
-			skipDependencies=skipDependencies,
-			extras=extras,
+			skipDependencies=skip_dependencies,
+			extras=groups,
 			pyproject=pyproject,
-			requirementsPaths=[Path(x) for x in requirementsPaths],
+			requirementsPaths=[Path(x) for x in requirements_paths],
 		)
 
 
-def getDepsWithLicenses(
-	using: str,
-	myLice: License,
-	ignorePackages: list[ucstr],
-	failPackages: list[ucstr],
-	ignoreLicenses: list[ucstr],
-	failLicenses: list[ucstr],
-	onlyLicenses: list[ucstr],
-	skipDependencies: list[ucstr],
-) -> set[PackageInfo]:
-	"""Get a set of dependencies with licenses and determine license compatibility.
-
-	Args:
-	----
-		using (str): use requirements or poetry
-		myLice (License): user license
-		ignorePackages (list[ucstr]): a list of packages to ignore (compat=True)
-		failPackages (list[ucstr]): a list of packages to fail (compat=False)
-		ignoreLicenses (list[ucstr]): a list of licenses to ignore (skipped, compat may still be
-		False)
-		failLicenses (list[ucstr]): a list of licenses to fail (compat=False)
-		onlyLicenses (list[ucstr]): a list of allowed licenses (any other license will fail)
-		skipDependencies (list[ucstr]): a list of dependencies to skip (compat=False)
-
-	Returns:
-	-------
-		tuple[License, set[PackageInfo]]: tuple of
-			my package license
-			set of updated dependencies with licenseCompat set
-
-	"""
-	reqs = getReqs(using, skipDependencies)
+def check(
+	requirements_paths: list[str],
+	groups: list[str],
+	this_license: License,
+	ignore_packages: list[ucstr] | None = None,
+	fail_packages: list[ucstr] | None = None,
+	ignore_licenses: list[ucstr] | None = None,
+	fail_licenses: list[ucstr] | None = None,
+	only_licenses: list[ucstr] | None = None,
+	skip_dependencies: list[ucstr] | None = None,
+) -> tuple[bool, set[PackageInfo]]:
+
+	# Def values
+	ignore_packages = ignore_packages or []
+	fail_packages = fail_packages or []
+	ignore_licenses = ignore_licenses or []
+	fail_licenses = fail_licenses or []
+	only_licenses = only_licenses or []
+	skip_dependencies = skip_dependencies or []
+
+
+	requirements = resolve_requirements(requirements_paths, groups, skip_dependencies)
 
 	ignoreLicensesType = license_matrix.licenseType(
-		ucstr(JOINS.join(ignoreLicenses)), ignoreLicenses
+		ucstr(JOINS.join(ignore_licenses)), ignore_licenses
 	)
-	failLicensesType = license_matrix.licenseType(ucstr(JOINS.join(failLicenses)), ignoreLicenses)
-	onlyLicensesType = license_matrix.licenseType(ucstr(JOINS.join(onlyLicenses)), ignoreLicenses)
+	failLicensesType = license_matrix.licenseType(ucstr(JOINS.join(fail_licenses)), ignore_licenses)
+	onlyLicensesType = license_matrix.licenseType(ucstr(JOINS.join(only_licenses)), ignore_licenses)
 	# licenseType will always return NO_LICENSE when onlyLicenses is empty          # noqa: ERA001
 	if License.NO_LICENSE in onlyLicensesType:
 		onlyLicensesType.remove(License.NO_LICENSE)
 
 	# Check it is compatible with packages and add a note
-	packages = packageinfo.getPackages(reqs)
+	packages = packageinfo.getPackages(requirements)
 	for package in packages:
 		# Deal with --ignore-packages and --fail-packages
 		package.licenseCompat = False
 		packageName = package.name.upper()
-		if packageName in ignorePackages:
+		if packageName in ignore_packages:
 			package.licenseCompat = True
-		elif packageName in failPackages:
+		elif packageName in fail_packages:
 			pass  # package.licenseCompat = False
 		# Else get compat with myLice
 		else:
 			package.licenseCompat = license_matrix.depCompatWMyLice(
-				myLice,
-				license_matrix.licenseType(package.license, ignoreLicenses),
+				this_license,
+				license_matrix.licenseType(package.license, ignore_licenses),
 				ignoreLicensesType,
 				failLicensesType,
 				onlyLicensesType,
 			)
-	return packages
+
+	# Are any licenses incompatible?
+	incompatible = any(not package.licenseCompat for package in packages)
+
+	return incompatible, packages

licensecheck/resolvers/native.py

@@ -8,12 +8,39 @@
 
 from packaging.requirements import Requirement
 from packaging.utils import canonicalize_name
+import tomli
 
 from licensecheck.session import session
 from licensecheck.types import ucstr
 
 
 def get_reqs(
+	skipDependencies: list[ucstr],
+	extras: list[str],
+	requirementsPaths: list[Path],
+	pyproject: dict[str, Any],
+) -> set[ucstr]:
+
+	using = "[unknown]"
+
+	# determine using based on file type
+	for requirementsPath in requirementsPaths:
+		try:
+			tomli.loads(requirementsPath.read_text("utf-8"))
+			if pyproject.get("project", {}).get("dependencies") is not None:
+				using = "PEP631"
+			if pyproject.get("tool", {}).get("poetry", {}).get("dependencies") is not None:
+				using = "poetry"
+
+		except tomli.TOMLDecodeError:
+			using = "requirements"
+
+
+	return do_get_reqs(using=using, skipDependencies=skipDependencies, extras=extras, pyproject=pyproject, requirementsPaths=requirementsPaths)
+
+
+
+def do_get_reqs(
 	using: str,
 	skipDependencies: list[ucstr],
 	extras: list[str],

licensecheck/resolvers/uv.py

@@ -13,14 +13,10 @@
 
 
 def get_reqs(
-	using: str,
 	skipDependencies: list[ucstr],
 	extras: list[str],
 	requirementsPaths: list[str],
 ) -> set[ucstr]:
-	if using == "requirements" and len(extras) > 0:
-		msg = "You may not use extras with requirements.txt"
-		raise RuntimeError(msg)
 
 	for idx, requirement in enumerate(requirementsPaths):
 		if not Path(requirement).exists():

tests/data/pep631_socks/pyproject.toml

@@ -0,0 +1,40 @@
+[project]
+# Added to example.
+name = "pep631_socks"
+license = "gpl-3.0-or-later"
+
+# Example copied from https://peps.python.org/pep-0631/#example
+dependencies = [
+    'cached-property >= 1.2.0, < 2',
+    'distro >= 1.5.0, < 2',
+    'docker[ssh] >= 4.2.2, < 5',
+    'dockerpty >= 0.4.1, < 1',
+    'docopt >= 0.6.1, < 1',
+    'jsonschema >= 2.5.1, < 4',
+    'PyYAML >= 3.10, < 6',
+    'python-dotenv >= 0.13.0, < 1',
+    'requests >= 2.20.0, < 3',
+    'texttable >= 0.9.0, < 2',
+    'websocket-client >= 0.32.0, < 1',
+    # 'toskip >= 0.0.1',
+
+    # Conditional
+    'backports.shutil_get_terminal_size == 1.0.0; python_version < "3.3"',
+    'backports.ssl_match_hostname >= 3.5, < 4; python_version < "3.5"',
+    'colorama >= 0.4, < 1; sys_platform == "win32"',
+    'enum34 >= 1.0.4, < 2; python_version < "3.4"',
+    'ipaddress >= 1.0.16, < 2; python_version < "3.3"',
+    'subprocess32 >= 3.5.4, < 4; python_version < "3.2"',
+]
+
+[project.optional-dependencies]
+socks = [ 'PySocks >= 1.5.6, != 1.5.7, < 2' ]
+tests = [
+    'ddt >= 1.2.2, < 2',
+    'pytest < 6',
+    'mock >= 1.0.1, < 4; python_version < "3.4"',
+]
+
+# Added to example.
+[tool.licensecheck]
+using = "PEP631:socks"