JULIET: add more commands

Author: sipma

chc/cmdline/chkc

@@ -525,6 +525,20 @@ def parse() -> argparse.Namespace:
 
     julietscore.set_defaults(func=J.juliet_score)
 
+    # --- score-sets
+    julietscoresets = julietparsers.add_parser("score-sets")
+    julietscoresets.add_argument(
+        "--maxprocesses",
+        type=int,
+        default=1,
+        help="maximum number of processors to use")
+    julietscoresets.add_argument(
+        "--cwes",
+        nargs="*",
+        default=[],
+        help="only score the tests with the given cwe's (default is all)")
+    julietscoresets.set_defaults(func=J.juliet_score_sets)
+
     # --- investigate
     julietinvestigate = julietparsers.add_parser("investigate")
     julietinvestigate.add_argument("cwe", help="name of cwe, e.g., CWE121")

chc/cmdline/juliet/JulietTestRef.py

@@ -26,6 +26,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 # ------------------------------------------------------------------------------
+"""Access to the reference results for a Juliet Test ."""
 
 from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING
 
@@ -39,6 +40,7 @@
 
 
 class JulietTestRef:
+
     def __init__(
             self,
             testsetref: "JulietTestSetRef",

chc/cmdline/juliet/JulietTestSetRef.py

@@ -26,6 +26,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 # ------------------------------------------------------------------------------
+"""Access to the reference results for all Juliet Tests."""
 
 from typing import Any, Callable, cast, Dict, List, Optional, TYPE_CHECKING
 

chc/cmdline/juliet/julietutil.py

@@ -24,7 +24,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 # ------------------------------------------------------------------------------
-"""Implementation of juliet commands in the command-line interpreter."""
+"""Implementation of juliet commands in the CLI."""
 
 import argparse
 import json
@@ -38,7 +38,7 @@
 from multiprocessing import Pool
 
 from typing import (
-    Any, Callable, Dict, List, NoReturn, Optional, Tuple, TYPE_CHECKING)
+    Any, Callable, cast, Dict, List, NoReturn, Optional, Tuple, TYPE_CHECKING)
 
 from chc.app.CApplication import CApplication
 
@@ -382,7 +382,7 @@ def save_xrefs(f: "CFile") -> None:
         am.generate_and_check_app("llrvisp", processes=jmaxproc)
         capp.reinitialize_tables()
 
-    def filefilter(filename):
+    def filefilter(filename: str) -> bool:
         return not (filename in ["io", "main_linux", "std_thread"])
 
     contractviolations = capp.get_contract_condition_violations()
@@ -403,7 +403,7 @@ def filefilter(filename):
     exit(0)
 
 
-def analyze_test(testdata):
+def analyze_test(testdata: Tuple[str, str, int]) -> int:
     """CLI command to run a juliet test case.
 
     Note: this function needs to be global for multiprocessing to work.
@@ -427,7 +427,7 @@ def juliet_analyze_sets(args: argparse.Namespace) -> NoReturn:
     pool = Pool(jmaxproc)
     testcases = []
 
-    def excluded(cwe):
+    def excluded(cwe: str) -> bool:
         if len(jcwes) == 0:
             return False
         else:
@@ -616,6 +616,61 @@ def juliet_score(args: argparse.Namespace) -> NoReturn:
     exit(0)
 
 
+def score_test(testdata: Tuple[str, str, int]) -> int:
+    """CLI command to score a juliet test case.
+
+    Note: this function needs to be global for multiprocessing to work.
+    """
+
+    (cwe, testcase, index) = testdata
+    cmd = ["chkc", "juliet", "score", cwe, testcase]
+    result = subprocess.call(cmd, stderr=subprocess.STDOUT)
+    return result
+
+
+def juliet_score_sets(args: argparse.Namespace) -> NoReturn:
+    """Scores all or a subset of the registered juliet tests."""
+
+    # arguments
+    jmaxproc: int = args.maxprocesses
+    jcwes: List[str] = args.cwes
+
+    maxptxt = "" if jmaxproc == 1 else f" (with {jmaxproc} processors)"
+
+    pool = Pool(jmaxproc)
+    testcases = []
+
+    def excluded(cwe: str) -> bool:
+        if len(jcwes) == 0:
+            return False
+        else:
+            return cwe not in jcwes
+
+    with timing("score-sets" + maxptxt):
+        count: int = 0
+        juliettests = UF.get_juliet_testcases()
+        for cwe in sorted(juliettests):
+            if excluded(cwe):
+                continue
+            print(f"Scoring testcases for cwe {cwe}")
+            for subdir in sorted(juliettests[cwe]):
+                for t in juliettests[cwe][subdir]:
+                    testcases.append((cwe, t, count))
+
+        results = pool.map(score_test, testcases)
+
+    print("\n" + ("=" * 80))
+    if len(results) == results.count(0):
+        print("All Juliet tests cases were scored successfully.")
+    else:
+        for x in range(len(results)):
+            if results[x] != 0:
+                print(f"Error in testcase {testcases[x][0]}")
+    print("=" * 80)
+
+    exit(0)
+
+
 def juliet_investigate(args: argparse.Namespace) -> NoReturn:
     """Lists problematic proof obligations."""
 
@@ -978,8 +1033,10 @@ def juliet_project_dashboard(args: argparse.Namespace) -> NoReturn:
             spo_project_totals[pname] = {}
 
             if "stats" in results:
-                projectstats[pname] = results["stats"]
-                analysistimes[pname] = results["timestamp"]
+                # use casts to handle union type in loaded dictionary
+                projectstats[pname] = cast(
+                    Tuple[int, int, int], results["stats"])
+                analysistimes[pname] = cast(int, results["timestamp"])
             else:
                 projectstats[pname] = (0, 0, 0)