Skip to content
This repository was archived by the owner on Aug 25, 2024. It is now read-only.

Commit f2cfb78

Browse files
Yash-VarshneyYash-Varshney
authored
shouldi: Add cargo audit for rust
- Move downloads to their own folder for caching Signed-off-by: John Andersen <[email protected]>
1 parent d88b3fe commit f2cfb78

File tree

6 files changed

+131
-15
lines changed

6 files changed

+131
-15
lines changed

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1414
- `validate` parameter in `Input` takes `Operation.instance_name`
1515
- New db source can utilize any database that inherits from `BaseDatabase`
1616
- Logistic Regression with SAG optimizer
17+
- shouldi got an operation to run cargo-audit on rust code.
18+
- Moved all the downloads to tests/downloads to speed the CI test.
1719
- Test tensorflow DNNEstimator documentation exaples in CI
1820
- Add python code for tensorflow DNNEstimator
1921
- Ability to run a subflow as if it were an operation using the

‎examples/shouldi/.gitignore‎

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,11 @@ wheelhouse/
1818
*.modeldir
1919
*.db
2020
htmlcov/
21-
tests/golangci-lint-download/
22-
tests/javascript_algo-download/
23-
tests/npm-audit-download/
24-
tests/golang-download/
25-
tests/cri-resource-manager-download/
21+
tests/downloads/golangci-lint-download/
22+
tests/downloads/javascript_algo-download/
23+
tests/downloads/npm-audit-download/
24+
tests/downloads/cargo-audit-download/
25+
tests/downloads/rust-download/
26+
tests/downloads/crates-download/
27+
tests/downloads/golang-download/
28+
tests/downloads/cri-resource-manager-download/

‎examples/shouldi/shouldi/cargo_audit.py‎

Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
import json
2+
import asyncio
3+
from typing import Dict, Any
4+
5+
from dffml.df.base import op
6+
from dffml.df.types import Definition
7+
8+
package_src_dir = Definition(name="package_src_dir", primitive="str")
9+
cargo_audit_output = Definition(
10+
name="golangci_lint_output", primitive="Dict[str, Any]"
11+
)
12+
13+
14+
class CargoAuditError(Exception):
15+
"""
16+
Raised when cargo-audit fails
17+
"""
18+
19+
20+
async def run_cargo_build(pkg_input: str):
21+
22+
new_proc = await asyncio.create_subprocess_exec(
23+
"cargo",
24+
"build",
25+
"--release",
26+
cwd=pkg_input,
27+
stdout=asyncio.subprocess.PIPE,
28+
stderr=asyncio.subprocess.PIPE,
29+
)
30+
stdout, stderr = await new_proc.communicate()
31+
if new_proc.returncode != 0:
32+
raise Exception(stderr.decode())
33+
34+
35+
@op(inputs={"pkg": package_src_dir}, outputs={"report": cargo_audit_output})
36+
async def run_cargo_audit(pkg: str) -> Dict[str, Any]:
37+
"""
38+
CLI usage: dffml service dev run -log debug shouldi.cargo_audit:run_cargo_audit -pkg .
39+
"""
40+
proc = await asyncio.create_subprocess_exec(
41+
"cargo",
42+
"audit",
43+
"--json",
44+
cwd=pkg,
45+
stdout=asyncio.subprocess.PIPE,
46+
stderr=asyncio.subprocess.PIPE,
47+
)
48+
stdout, stderr = await proc.communicate()
49+
if len(stdout) == 0:
50+
raise CargoAuditError(stderr.decode())
51+
52+
cargo_audit_op = stdout.decode()
53+
issues = json.loads(cargo_audit_op)
54+
result = issues["vulnerabilities"]["count"]
55+
56+
return {"report": result}

‎examples/shouldi/tests/test_cargo_audit.py‎

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
import pathlib
2+
3+
from dffml.util.os import prepend_to_path
4+
from dffml.util.net import cached_download_unpack_archive
5+
from dffml.util.asynctestcase import AsyncTestCase
6+
7+
from shouldi.cargo_audit import run_cargo_audit, run_cargo_build
8+
9+
10+
class TestRunCargoAuditOp(AsyncTestCase):
11+
@cached_download_unpack_archive(
12+
"https://static.rust-lang.org/dist/rust-1.42.0-x86_64-unknown-linux-gnu.tar.gz",
13+
pathlib.Path(__file__).parent / "downloads" / "rust.tar.gz",
14+
pathlib.Path(__file__).parent / "downloads" / "rust-download",
15+
"ad2ab72dc407b0f5d34621640555e2da751da8803cbad734396faa54111e03093093f6fa66f14a1948bece8f9e33730d",
16+
)
17+
@cached_download_unpack_archive(
18+
"https://github.com/RustSec/cargo-audit/archive/v0.11.2.tar.gz",
19+
pathlib.Path(__file__).parent / "downloads" / "cargo_audit.tar.gz",
20+
pathlib.Path(__file__).parent / "downloads" / "cargo-audit-download",
21+
"dea36731efaac4d0fd37a295c65520a7e9b23b5faa0a92dce7ab20764f8323fc34856079524c676e4cad1cb065ee6472",
22+
)
23+
@cached_download_unpack_archive(
24+
"https://github.com/rust-lang/crates.io/archive/8c1a7e29073e175f0e69e0e537374269da244cee.tar.gz",
25+
pathlib.Path(__file__).parent / "downloads" / "crates.tar.gz",
26+
pathlib.Path(__file__).parent / "downloads" / "crates-download",
27+
"1bf0c3459373882f51132942872d0dbf8da01eee8d42c3c2090d234e4db99b39d4858c1fd2492c85917d670cae2519ca",
28+
)
29+
async def test_run(self, rust, cargo_audit, crates):
30+
if not (
31+
cargo_audit
32+
/ "cargo-audit-0.11.2"
33+
/ "target"
34+
/ "release"
35+
/ "cargo-audit"
36+
).is_file():
37+
await run_cargo_build(cargo_audit / "cargo-audit-0.11.2")
38+
39+
with prepend_to_path(
40+
rust / "rust-1.42.0-x86_64-unknown-linux-gnu" / "cargo" / "bin",
41+
cargo_audit / "cargo-audit-0.11.2" / "target" / "release",
42+
):
43+
results = await run_cargo_audit(
44+
str(
45+
crates
46+
/ "crates.io-8c1a7e29073e175f0e69e0e537374269da244cee"
47+
)
48+
)
49+
self.assertEqual(type(results["report"]), int)

‎examples/shouldi/tests/test_golangci_lint.py‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -25,20 +25,24 @@ async def tearDown(self):
2525

2626
@cached_download_unpack_archive(
2727
"https://dl.google.com/go/go1.14.linux-amd64.tar.gz",
28-
pathlib.Path(__file__).parent / "golang.tar.gz",
29-
pathlib.Path(__file__).parent / "golang-download",
28+
pathlib.Path(__file__).parent / "downloads" / "golang.tar.gz",
29+
pathlib.Path(__file__).parent / "downloads" / "golang-download",
3030
"5dcc7b2e9049d80ceee9d3a7a4b76b578f42de64eaadabd039f080a9f329f2ad448da710626ed8fb4b070b4555b50e6f",
3131
)
3232
@cached_download_unpack_archive(
3333
"https://github.com/golangci/golangci-lint/releases/download/v1.23.7/golangci-lint-1.23.7-linux-amd64.tar.gz",
34-
pathlib.Path(__file__).parent / "golangci-lint.tar.gz",
35-
pathlib.Path(__file__).parent / "golangci-lint-download",
34+
pathlib.Path(__file__).parent / "downloads" / "golangci-lint.tar.gz",
35+
pathlib.Path(__file__).parent / "downloads" / "golangci-lint-download",
3636
"088a65ae7aa45c8a5695f40cc90672d00dece7f08ce307567fddc8b2d03858cb5baf9d162193922d36c57c504cc52999",
3737
)
3838
@cached_download_unpack_archive(
3939
"https://github.com/intel/cri-resource-manager/archive/c5e6091c79830cf7d076bbdec59c4a253b369d6a.tar.gz",
40-
pathlib.Path(__file__).parent / "cri-resource-manager.tar.gz",
41-
pathlib.Path(__file__).parent / "cri-resource-manager-download",
40+
pathlib.Path(__file__).parent
41+
/ "downloads"
42+
/ "cri-resource-manager.tar.gz",
43+
pathlib.Path(__file__).parent
44+
/ "downloads"
45+
/ "cri-resource-manager-download",
4246
"bdcbc8dadf9c6ee2f7571d10cb54459fe54773036982ad7485f007606efae96d7aaec7da18e2fea806fb6f68eb1722a8",
4347
)
4448
async def test_run(self, golang, golangci_lint, cri_resource_manager):

‎examples/shouldi/tests/test_npm_audit.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,16 @@
1010
class TestRunNPM_AuditOp(AsyncTestCase):
1111
@cached_download_unpack_archive(
1212
"https://nodejs.org/dist/v12.16.1/node-v12.16.1-linux-x64.tar.gz",
13-
pathlib.Path(__file__).parent / "npm.tar.gz",
14-
pathlib.Path(__file__).parent / "npm-audit-download",
13+
pathlib.Path(__file__).parent / "downloads" / "npm.tar.gz",
14+
pathlib.Path(__file__).parent / "downloads" / "npm-audit-download",
1515
"7df0e7b9f0d7e387c866c3b75596d924a63d11233e7a1a850acdeb333729ebbc9dcf01b1724ddf48a48bedf0cf2fddd8",
1616
)
1717
@cached_download_unpack_archive(
1818
"https://github.com/trekhleb/javascript-algorithms/archive/ba2d8dc4a8e27659c1420fe52390cb7981df4a94.tar.gz",
19-
pathlib.Path(__file__).parent / "javascript_algo.tar.gz",
20-
pathlib.Path(__file__).parent / "javascript_algo-download",
19+
pathlib.Path(__file__).parent / "downloads" / "javascript_algo.tar.gz",
20+
pathlib.Path(__file__).parent
21+
/ "downloads"
22+
/ "javascript_algo-download",
2123
"36b3ce51780ee6ea8dcec266c9d09e3a00198868ba1b041569950b82cf45884da0c47ec354dd8514022169849dfe8b7c",
2224
)
2325
async def test_run(self, npm_audit, javascript_algo):

0 commit comments

Comments
 (0)