JULIET: add more commands

Author: sipma

chc/cmdline/chkc

@@ -200,6 +200,84 @@ The chkc juliet set of commands is used to analyze the juliet test suite cases
 provided in
 
 https://github.com/static-analysis-engineering/CodeHawk-C-Targets-Juliet
+
+All commands in the juliet collection of commands assume that a pointer to
+the location of these (pre-parsed) tests is configured in util/Config.py
+or in util/ConfigLocal.py. To check wheter this is the case, run
+
+> chkc juliet check_config
+
+It should show the analyzer configuration and the location of the juliet test
+case index file.
+
+
+The command
+
+> chkc juliet list
+
+lists the individual test cases provided with the date/time the analyzer was
+last run on this test, and the date/time when the results were compared with
+the accompanying scoring key.
+
+
+The command
+
+> chkc juliet analyze <cwe> <test>   (e.g., CWE121 char_type_overrun_memcpy)
+
+runs the analysis on a juliet test. An individual juliet test typically
+consists of 18-60 files. Analysis of all of these files can be run in parallel
+with the optional commandline option
+
+--maxprocesses <n>
+
+where <n> is the number of processors to be applied.
+
+A summary of the results is saved in the file summaryresults.json for later
+reference. Detailed results are saved in the .cch/a directory, as usual.
+
+
+The command
+
+> chkc juliet analyze-sets
+
+can be used to automatically run all (or a subset) of the juliet tests
+provided. In this case parallelism is applied to the sets, rather than to
+the individual files within a set.
+
+The commands below can be run only on tests that have already been analyzed.
+
+The command
+
+> chkc juliet report <cwe> <test>
+
+outputs a summary of the proof obligation analysis statistics for the given
+test.
+
+
+The command
+
+> chkc juliet report-file <cwe> <test> <filename>   (e.g., x01.c)
+
+outputs the original source code annotated with the proof obligations and
+their analysis results for each line of code for the given source file, as well
+as the proof obligation analysis statistics for that source file.
+
+
+The command
+
+> chkc juliet score <cwe> <test>
+
+compares the analysis results the given test with the expected analysis,
+as provided by the scorekey.json file that comes with each test.
+
+
+The command
+
+> chkc juliet investigate <cwe> <test>
+
+outputs the list of open/violated/delegated proof obligations, together with
+their dependencies and diagnostics for the given test.
+
 """
     print(descr)
     exit(0)
@@ -384,6 +462,20 @@ def parse() -> argparse.Namespace:
 
     julietanalyze.set_defaults(func=J.juliet_analyze)
 
+    # --- analyze-sets
+    julietanalyzesets = julietparsers.add_parser("analyze-sets")
+    julietanalyzesets.add_argument(
+        "--maxprocesses",
+        help="maximum number of processors to use",
+        type=int,
+        default=1)
+    julietanalyzesets.add_argument(
+        "--cwes",
+        nargs="*",
+        help="restrict analysis to these cwe's (default is all)",
+        default=[])
+    julietanalyzesets.set_defaults(func=J.juliet_analyze_sets)
+
     # --- report
     julietreport = julietparsers.add_parser("report")
     julietreport.add_argument("cwe", help="name of cwe, e.g., CWE121")
@@ -396,7 +488,20 @@ def parse() -> argparse.Namespace:
     julietreportfile.add_argument("cwe", help="name of cwe, e.g., CWE121")
     julietreportfile.add_argument(
         "test", help="name of test case, e.g., CWE129_large")
-    julietreportfile.add_argument("filename", help="name of c file, e.g., x01.c")
+    julietreportfile.add_argument(
+        "filename", help="name of c file, e.g., x01.c")
+    julietreportfile.add_argument(
+        "--showcode",
+        action="store_true",
+        help="show the proof obligations on the code")
+    julietreportfile.add_argument(
+        "--open",
+        action="store_true",
+        help="show only open proof obligations on the code")
+    julietreportfile.add_argument(
+        "--showinvariants",
+        action="store_true",
+        help="show invariants for open proof obligations")
     julietreportfile.set_defaults(func=J.juliet_report_file)
 
     # --- score
@@ -420,6 +525,47 @@ def parse() -> argparse.Namespace:
 
     julietscore.set_defaults(func=J.juliet_score)
 
+    # --- investigate
+    julietinvestigate = julietparsers.add_parser("investigate")
+    julietinvestigate.add_argument("cwe", help="name of cwe, e.g., CWE121")
+    julietinvestigate.add_argument(
+        "test", help="name of test case, e.g., CWE129_large")
+    julietinvestigate.add_argument(
+        "--xdelegated",
+        help="exclude delegated proof obligations",
+        action="store_true")
+    julietinvestigate.add_argument(
+        "--xviolated",
+        help="exclude violated proof obligations",
+        action="store_true")
+    julietinvestigate.add_argument(
+        "--predicates",
+        nargs="*",
+        help="predicates of interest (default: all)",
+        default=[])
+    julietinvestigate.set_defaults(func=J.juliet_investigate)
+
+    # --- report-requests
+    julietreprequests = julietparsers.add_parser("report-requests")
+    julietreprequests.add_argument("cwe", help="name of cwe, e.g., CWE121")
+    julietreprequests.add_argument(
+        "test", help="name of test case, e.g., CWE129_large")
+    julietreprequests.set_defaults(func=J.juliet_report_requests)
+
+    # --- dashboard
+    julietdashboard = julietparsers.add_parser("dashboard")
+    julietdashboard.add_argument(
+        "--cwe", help="only report results on the given cwe")
+    julietdashboard.add_argument(
+        "--variant", help="only report results on the given variant")
+    julietdashboard.set_defaults(func=J.juliet_dashboard)
+
+    # --- project-dashboard
+    julietpdashboard = julietparsers.add_parser("project-dashboard")
+    julietpdashboard.add_argument(
+        "--cwe", help="only report results on the given cwe")
+    julietpdashboard.set_defaults(func=J.juliet_project_dashboard)
+
     # --------------------------------------------------------------- c-file ---
     cfilecmd = subparsers.add_parser("c-file")
     cfilecmd.set_defaults(func=cfilecommand)

chc/cmdline/juliet/JulietTestFileRef.py

@@ -28,7 +28,8 @@
 # ------------------------------------------------------------------------------
 """Juliet scoring reference for a juliet test set file."""
 
-from typing import Any, Dict, List, Optional, TYPE_CHECKING
+from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING
+
 
 if TYPE_CHECKING:
     from chc.cmdline.juliet.JulietTestRef import JulietTestRef
@@ -41,9 +42,9 @@ def __init__(
             self,
             testref: "JulietTestRef",
             test: str,
-            d: Dict[str, Any]) -> None:
+            d: Dict[str, Dict[str, List[str]]]) -> None:
         self._testref = testref
-        self._test: str
+        self._test = test
         self._d = d
         self._violations: Optional[Dict[int, List[JulietViolation]]] = None
         self._safecontrols: Optional[Dict[int, List[JulietSafeControl]]] = None
@@ -57,7 +58,7 @@ def test(self) -> str:
         return self._test
 
     @property
-    def violations(self) -> Dict[int, List[JulietViolation]]:
+    def violations(self) -> Dict[int, List["JulietViolation"]]:
         if self._violations is None:
             self._violations = {}
             for line in self._d["violations"]:
@@ -70,7 +71,7 @@ def violations(self) -> Dict[int, List[JulietViolation]]:
         return self._violations
 
     @property
-    def safe_controls(self) -> Dict[int, List[JulietSafeControl]]:
+    def safe_controls(self) -> Dict[int, List["JulietSafeControl"]]:
         if self._safecontrols is None:
             self._safecontrols = {}
             for line in self._d["safe-controls"]:
@@ -85,18 +86,34 @@ def safe_controls(self) -> Dict[int, List[JulietSafeControl]]:
     def expand(self, m: str) -> List[Dict[str, Any]]:
         return self.testref.expand_macro(m)
 
-    def get_violations(self) -> List[JulietViolation]:
+    def get_violations(self) -> List["JulietViolation"]:
         result: List[JulietViolation] = []
         for (line, v) in sorted(self.violations.items()):
             result.extend(v)
         return result
 
-    def get_safe_controls(self) -> List[JulietSafeControl]:
+    def iter_violations(
+            self, f: Callable[[int, "JulietPpo"], None]) -> None:
+        for (line, ppos) in self.violations.items():
+            for ppo in ppos:
+                f(line, ppo)
+
+    def get_safe_controls(self) -> List["JulietSafeControl"]:
         result: List[JulietSafeControl] = []
         for (line, s) in sorted(self.safe_controls.items()):
             result.extend(s)
         return result
 
+    def iter_safe_controls(
+            self, f: Callable[[int, "JulietPpo"], None]) -> None:
+        for (line, ppos) in self.safe_controls.items():
+            for ppo in ppos:
+                f(line, ppo)
+
+    def iter(self, f: Callable[[int, "JulietPpo"], None]) -> None:
+        self.iter_violations(f)
+        self.iter_safe_controls(f)
+
     def __str__(self) -> str:
         lines: List[str] = []
         lines.append("  violations:")
@@ -139,6 +156,14 @@ def test(self) -> str:
     def line(self) -> int:
         return self._line
 
+    @property
+    def is_violation(self) -> bool:
+        return False
+
+    @property
+    def is_safe_control(self) -> bool:
+        return False
+
     @property
     def predicate(self) -> str:
         return self._d["P"]
@@ -248,6 +273,7 @@ def __init__(
             d: Dict[str, Any]) -> None:
         JulietPpo.__init__(self, testfileref, test, line, d)
 
+    @property
     def is_violation(self) -> bool:
         return True
 
@@ -262,5 +288,6 @@ def __init__(
             d: Dict[str, Any]) -> None:
         JulietPpo.__init__(self, testfileref, test, line, d)
 
+    @property
     def is_violation(self) -> bool:
         return False

chc/cmdline/juliet/JulietTestRef.py

@@ -27,7 +27,7 @@
 # SOFTWARE.
 # ------------------------------------------------------------------------------
 
-from typing import Any, Dict, List, Optional, TYPE_CHECKING
+from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING
 
 from chc.cmdline.juliet.JulietTestFileRef import JulietSafeControl
 from chc.cmdline.juliet.JulietTestFileRef import JulietTestFileRef
@@ -66,6 +66,10 @@ def cfiles(self) -> Dict[str, JulietTestFileRef]:
                     self, self.test, self._d["cfiles"][f])
         return self._cfiles
 
+    def iter(self, f: Callable[[str, JulietTestFileRef], None]) -> None:
+        for (filename, fileref) in self.cfiles.items():
+            f(filename, fileref)
+
     def expand_macro(self, m: str) -> List[Dict[str, Any]]:
         return self.testsetref.expand_macro(m)