Skip to content

Commit 4a760ee

Browse files
NicorettiNicoretti
authored
Merge branch 'main' into feature/#149-add-support-for-import-linter-to-lint-tasks
2 parents 4611558 + dec6bd3 commit 4a760ee

File tree

11 files changed

+463
-7
lines changed

11 files changed

+463
-7
lines changed

‎.github/workflows/report.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,4 @@ jobs:
5151
poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY
5252
echo -e "\n\n# Static Code Analysis\n" >> $GITHUB_STEP_SUMMARY
5353
cat .lint.txt >> $GITHUB_STEP_SUMMARY
54+
poetry run tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY

‎doc/changes/unreleased.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
# Unreleased
22

3-
## ✨ Features
3+
## ✨ Added
44

5-
* #149: Added nox task to lint imports
5+
* #149: Added nox task to lint imports
6+
* #248: Added security results to workflow summary
7+
* #233: Added nox task to verify dependency declarations

‎doc/user_guide/getting_started.rst‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,7 @@ You are ready to use the toolbox. With *nox -l* you can list all available tasks
193193
- lint:code -> Runs the static code analyzer on the project
194194
- lint:typing -> Runs the type checker on the project
195195
- lint:security -> Runs the security linter on the project
196+
- lint:dependencies -> Checks if only valid sources of dependencies are used
196197
- docs:multiversion -> Builds the multiversion project documentation
197198
- docs:build -> Builds the project documentation
198199
- docs:open -> Opens the built project documentation

‎exasol/toolbox/nox/_lint.py‎

Lines changed: 76 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
from __future__ import annotations
22

3-
from typing import Iterable
3+
from typing import (
4+
Iterable,
5+
List,
6+
Dict
7+
)
48
import argparse
59
from pathlib import Path
610

@@ -10,6 +14,11 @@
1014
from exasol.toolbox.nox._shared import python_files
1115
from noxconfig import PROJECT_CONFIG
1216

17+
from pathlib import Path
18+
import rich.console
19+
import tomlkit
20+
import sys
21+
1322

1423
def _pylint(session: Session, files: Iterable[str]) -> None:
1524
session.run(
@@ -67,6 +76,7 @@ def _security_lint(session: Session, files: Iterable[str]) -> None:
6776
)
6877

6978

79+
7080
def _import_lint(session: Session, path: Path) -> None:
7181
session.run(
7282
"poetry",
@@ -76,6 +86,60 @@ def _import_lint(session: Session, path: Path) -> None:
7686
path
7787
)
7888

89+
class Dependencies:
90+
def __init__(self, illegal: Dict[str, List[str]] | None):
91+
self._illegal = illegal or {}
92+
93+
@staticmethod
94+
def parse(pyproject_toml: str) -> "Dependencies":
95+
def _source_filter(version) -> bool:
96+
ILLEGAL_SPECIFIERS = ['url', 'git', 'path']
97+
return any(
98+
specifier in version
99+
for specifier in ILLEGAL_SPECIFIERS
100+
)
101+
102+
def find_illegal(part) -> List[str]:
103+
return [
104+
f"{name} = {version}"
105+
for name, version in part.items()
106+
if _source_filter(version)
107+
]
108+
109+
illegal: Dict[str, List[str]] = {}
110+
toml = tomlkit.loads(pyproject_toml)
111+
poetry = toml.get("tool", {}).get("poetry", {})
112+
113+
part = poetry.get("dependencies", {})
114+
if illegal_group := find_illegal(part):
115+
illegal["tool.poetry.dependencies"] = illegal_group
116+
117+
part = poetry.get("dev", {}).get("dependencies", {})
118+
if illegal_group := find_illegal(part):
119+
illegal["tool.poetry.dev.dependencies"] = illegal_group
120+
121+
part = poetry.get("group", {})
122+
for group, content in part.items():
123+
illegal_group = find_illegal(content.get("dependencies", {}))
124+
if illegal_group:
125+
illegal[f"tool.poetry.group.{group}.dependencies"] = illegal_group
126+
return Dependencies(illegal)
127+
128+
@property
129+
def illegal(self) -> Dict[str, List[str]]:
130+
return self._illegal
131+
132+
133+
def report_illegal(illegal: Dict[str, List[str]], console: rich.console.Console):
134+
count = sum(len(deps) for deps in illegal.values())
135+
suffix = "y" if count == 1 else "ies"
136+
console.print(f"{count} illegal dependenc{suffix}\n", style="red")
137+
for section, dependencies in illegal.items():
138+
console.print(f"\\[{section}]", style="red")
139+
for dependency in dependencies:
140+
console.print(dependency, style="red")
141+
console.print("")
142+
79143

80144
@nox.session(name="lint:code", python=False)
81145
def lint(session: Session) -> None:
@@ -98,6 +162,16 @@ def security_lint(session: Session) -> None:
98162
_security_lint(session, list(filter(lambda file: "test" not in file, py_files)))
99163

100164

165+
@nox.session(name="lint:dependencies", python=False)
166+
def dependency_check(session: Session) -> None:
167+
"""Checks if only valid sources of dependencies are used"""
168+
content = Path(PROJECT_CONFIG.root, "pyproject.toml").read_text()
169+
dependencies = Dependencies.parse(content)
170+
console = rich.console.Console()
171+
if illegal := dependencies.illegal:
172+
report_illegal(illegal, console)
173+
sys.exit(1)
174+
101175
@nox.session(name="lint:import", python=False)
102176
def import_lint(session: Session) -> None:
103177
"""(experimental) Runs import linter on the project"""
@@ -124,5 +198,4 @@ def import_lint(session: Session) -> None:
124198
session.error(
125199
"Please make sure you have a configuration file for the importlinter"
126200
)
127-
_import_lint(session=session, path=path)
128-
201+
_import_lint(session=session, path=path)

‎exasol/toolbox/nox/tasks.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,5 +68,6 @@ def check(session: Session) -> None:
6868
python_files,
6969
)
7070

71+
7172
# isort: on
7273
# fmt: on

‎exasol/toolbox/templates/github/workflows/report.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,3 +51,4 @@ jobs:
5151
poetry run coverage report -- --format markdown >> $GITHUB_STEP_SUMMARY
5252
echo -e "\n\n# Static Code Analysis\n" >> $GITHUB_STEP_SUMMARY
5353
cat .lint.txt >> $GITHUB_STEP_SUMMARY
54+
poetry run tbx security pretty-print .security.json >> $GITHUB_STEP_SUMMARY

‎exasol/toolbox/tools/security.py‎

Lines changed: 74 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,8 @@
1616
Iterable,
1717
Tuple,
1818
)
19-
2019
import typer
20+
from pathlib import Path
2121

2222
stdout = print
2323
stderr = partial(print, file=sys.stderr)
@@ -101,6 +101,64 @@ def from_maven(report: str) -> Iterable[Issue]:
101101
)
102102

103103

104+
@dataclass(frozen=True)
105+
class SecurityIssue:
106+
file_name: str
107+
line: int
108+
column: int
109+
cwe: str
110+
test_id: str
111+
description: str
112+
references: tuple
113+
114+
115+
def from_json(report_str: str, prefix: Path) -> Iterable[SecurityIssue]:
116+
report = json.loads(report_str)
117+
issues = report.get("results", {})
118+
for issue in issues:
119+
references = []
120+
if issue["more_info"]:
121+
references.append(issue["more_info"])
122+
if issue.get("issue_cwe", {}).get("link", None):
123+
references.append(issue["issue_cwe"]["link"])
124+
yield SecurityIssue(
125+
file_name=issue["filename"].replace(str(prefix) + "/", ""),
126+
line=issue["line_number"],
127+
column=issue["col_offset"],
128+
cwe=str(issue["issue_cwe"].get("id", "")),
129+
test_id=issue["test_id"],
130+
description=issue["issue_text"],
131+
references=tuple(references)
132+
)
133+
134+
135+
def issues_to_markdown(issues: Iterable[SecurityIssue]) -> str:
136+
template = cleandoc("""
137+
{header}{rows}
138+
""")
139+
140+
def _header():
141+
header = "# Security\n\n"
142+
header += "|File|line/<br>column|Cwe|Test ID|Details|\n"
143+
header += "|---|:-:|:-:|:-:|---|\n"
144+
return header
145+
146+
def _row(issue):
147+
row = "|" + issue.file_name + "|"
148+
row += f"line: {issue.line}<br>column: {issue.column}|"
149+
row += issue.cwe + "|"
150+
row += issue.test_id + "|"
151+
for element in issue.references:
152+
row += element + " ,<br>"
153+
row = row[:-5] + "|"
154+
return row
155+
156+
return template.format(
157+
header=_header(),
158+
rows="\n".join(_row(i) for i in issues)
159+
)
160+
161+
104162
def security_issue_title(issue: Issue) -> str:
105163
return f"🔐 {issue.cve}: {issue.coordinates}"
106164

@@ -159,7 +217,6 @@ def create_security_issue(issue: Issue, project="") -> Tuple[str, str]:
159217
CVE_CLI = typer.Typer()
160218
CLI.add_typer(CVE_CLI, name="cve", help="Work with CVE's")
161219

162-
163220
class Format(str, Enum):
164221
Maven = "maven"
165222

@@ -257,6 +314,21 @@ def create(
257314
stdout(format_jsonl(issue_url, issue))
258315

259316

317+
class PPrintFormats(str, Enum):
318+
markdown = "markdown"
319+
320+
321+
@CLI.command(name="pretty-print")
322+
def json_issue_to_markdown(
323+
json_file: typer.FileText = typer.Argument(mode="r", help="json file with issues to convert"),
324+
path: Path = typer.Argument(default=Path("."), help="path to project root")
325+
) -> None:
326+
content = json_file.read()
327+
issues = from_json(content, path.absolute())
328+
issues = sorted(issues, key=lambda i: (i.file_name, i.cwe, i.test_id))
329+
print(issues_to_markdown(issues))
330+
331+
260332
def format_jsonl(issue_url: str, issue: Issue) -> str:
261333
issue_json = asdict(issue)
262334
issue_json["issue_url"] = issue_url.strip()

‎test/integration/cli/security-pprint-emty.t‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
Create test input
2+
3+
$ cat > .security.json <<EOF
4+
> {
5+
> "result":[
6+
> ]
7+
> }
8+
> EOF
9+
10+
Run test case
11+
12+
$ tbx security pretty-print .security.json
13+
# Security
14+
15+
|File|line/<br>column|Cwe|Test ID|Details|
16+
|---|:-:|:-:|:-:|---|
17+
18+

‎test/integration/cli/security-pprint.t‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
Create test input
2+
3+
$ cat > .security.json <<EOF
4+
> {
5+
> "results":[
6+
> {
7+
> "code": "555 subprocess.check_call(\n556 config.smv_postbuild_command, cwd=current_cwd, shell=True\n557 )\n558 if config.smv_postbuild_export_pattern != \"\":\n559 matches = find_matching_files_and_dirs(\n",
8+
> "col_offset": 16,
9+
> "end_col_offset": 17,
10+
> "filename": "exasol/toolbox/sphinx/multiversion/main.py",
11+
> "issue_confidence": "HIGH",
12+
> "issue_cwe": {
13+
> "id": 78,
14+
> "link": "https://cwe.mitre.org/data/definitions/78.html"
15+
> },
16+
> "issue_severity": "HIGH",
17+
> "issue_text": "subprocess call with shell=True identified, security issue.",
18+
> "line_number": 556,
19+
> "line_range": [
20+
> 555,
21+
> 556,
22+
> 557
23+
> ],
24+
> "more_info": "https://bandit.readthedocs.io/en/1.7.10/plugins/b602_subprocess_popen_with_shell_equals_true.html",
25+
> "test_id": "B602",
26+
> "test_name": "subprocess_popen_with_shell_equals_true"
27+
> },
28+
> {
29+
> "code": "156 )\n157 subprocess.check_call(cmd, cwd=gitroot, stdout=fp)\n158 fp.seek(0)\n",
30+
> "col_offset": 8,
31+
> "end_col_offset": 58,
32+
> "filename": "exasol/toolbox/sphinx/multiversion/git.py",
33+
> "issue_confidence": "HIGH",
34+
> "issue_cwe": {
35+
> "id": 78,
36+
> "link": "https://cwe.mitre.org/data/definitions/78.html"
37+
> },
38+
> "issue_severity": "LOW",
39+
> "issue_text": "subprocess call - check for execution of untrusted input.",
40+
> "line_number": 157,
41+
> "line_range": [
42+
> 157
43+
> ],
44+
> "more_info": "https://bandit.readthedocs.io/en/1.7.10/plugins/b603_subprocess_without_shell_equals_true.html",
45+
> "test_id": "B603",
46+
> "test_name": "subprocess_without_shell_equals_true"
47+
> },
48+
> {
49+
> "code": "159 with tarfile.TarFile(fileobj=fp) as tarfp:\n160 tarfp.extractall(dst)\n",
50+
> "col_offset": 12,
51+
> "end_col_offset": 33,
52+
> "filename": "exasol/toolbox/sphinx/multiversion/git.py",
53+
> "issue_confidence": "HIGH",
54+
> "issue_cwe": {
55+
> "id": 22,
56+
> "link": "https://cwe.mitre.org/data/definitions/22.html"
57+
> },
58+
> "issue_severity": "HIGH",
59+
> "issue_text": "tarfile.extractall used without any validation. Please check and discard dangerous members.",
60+
> "line_number": 160,
61+
> "line_range": [
62+
> 160
63+
> ],
64+
> "more_info": "https://bandit.readthedocs.io/en/1.7.10/plugins/b202_tarfile_unsafe_members.html",
65+
> "test_id": "B202",
66+
> "test_name": "tarfile_unsafe_members"
67+
> }
68+
> ]
69+
> }
70+
> EOF
71+
72+
Run test case
73+
74+
$ tbx security pretty-print .security.json
75+
# Security
76+
77+
|File|line/<br>column|Cwe|Test ID|Details|
78+
|---|:-:|:-:|:-:|---|
79+
|exasol/toolbox/sphinx/multiversion/git.py|line: 160<br>column: 12|22|B202|https://bandit.readthedocs.io/en/1.7.10/plugins/b202_tarfile_unsafe_members.html ,<br>https://cwe.mitre.org/data/definitions/22.html |
80+
|exasol/toolbox/sphinx/multiversion/git.py|line: 157<br>column: 8|78|B603|https://bandit.readthedocs.io/en/1.7.10/plugins/b603_subprocess_without_shell_equals_true.html ,<br>https://cwe.mitre.org/data/definitions/78.html |
81+
|exasol/toolbox/sphinx/multiversion/main.py|line: 556<br>column: 16|78|B602|https://bandit.readthedocs.io/en/1.7.10/plugins/b602_subprocess_popen_with_shell_equals_true.html ,<br>https://cwe.mitre.org/data/definitions/78.html |

0 commit comments

Comments
 (0)