Add docstring to new submission checker

Author: pgmpablo157321

tools/submission/submission_checker/checks/accuracy_check.py

@@ -7,9 +7,49 @@
 
 
 class AccuracyCheck(BaseCheck):
+    """Checks accuracy-related submission artifacts and reports issues.
+
+    The `AccuracyCheck` class performs a set of validations on submission
+    accuracy outputs. It inspects the parsed MLPerf log and accompanying
+    accuracy artifacts provided via `SubmissionLogs` and the test
+    `Config` to ensure that reported accuracy metrics meet configured
+    targets and limits, that the accuracy JSON exists and is properly
+    truncated, that Loadgen did not report blocking errors, and that the
+    accuracy run covered the expected dataset size.
+
+    Main check methods:
+    - `accuracy_result_check`: Parses `accuracy.txt` lines to validate
+        reported metrics against targets, upper limits, and hash presence.
+    - `accuracy_json_check`: Ensures the accuracy JSON file exists and is
+        within allowed size limits.
+    - `loadgen_errors_check`: Fails if Loadgen reported non-ignored errors.
+    - `dataset_check`: Verifies the reported sample count matches the
+        configured dataset size unless the check is skipped.
+
+    Attributes:
+            submission_logs (SubmissionLogs): Holder for submission log paths
+                    and parsed contents (accuracy logs, results, json, loader data).
+            mlperf_log: Parsed MLPerf log object used to inspect errors and
+                    run metadata.
+            accuracy_result (list[str]): Lines from `accuracy.txt` used to
+                    extract reported accuracy values.
+            accuracy_json (str): Path to the accuracy JSON file.
+            config (Config): Configuration helper providing target values and
+                    dataset sizes.
+    """
+
     def __init__(
         self, log, path, config: Config, submission_logs: SubmissionLogs
     ):
+        """Initialize the accuracy check helper.
+
+        Args:
+            log: Logger instance used to report messages.
+            path: Path to the submission being checked.
+            config (Config): Configuration provider for targets and limits.
+            submission_logs (SubmissionLogs): Parsed submission logs and
+                artifact paths (accuracy logs, results, json, loader data).
+        """
         super().__init__(log, path)
         self.name = "accuracy checks"
         self.submission_logs = submission_logs
@@ -29,12 +69,29 @@ def __init__(
         self.setup_checks()
 
     def setup_checks(self):
+        """Register individual accuracy-related checks.
+
+        Adds the per-submission validation callables to `self.checks` in
+        the order they should be executed.
+        """
         self.checks.append(self.accuracy_result_check)
         self.checks.append(self.accuracy_json_check)
         self.checks.append(self.loadgen_errors_check)
         self.checks.append(self.dataset_check)
 
     def accuracy_result_check(self):
+        """Validate reported accuracy metrics in `accuracy.txt`.
+
+        Parses lines from `self.accuracy_result` using configured patterns
+        and compares found values against targets and optional upper
+        limits. Also ensures a hash value is present and records the
+        observed accuracy metrics in `submission_logs.loader_data`.
+
+        Returns:
+            bool: True if accuracy checks passed (or division is 'open'),
+                False otherwise.
+        """
+
         patterns, acc_targets, acc_types, acc_limits, up_patterns, acc_upper_limit = self.config.get_accuracy_values(
             self.model
         )
@@ -105,6 +162,12 @@ def accuracy_result_check(self):
         return is_valid
 
     def accuracy_json_check(self):
+        """Check that the accuracy JSON exists and is within size limits.
+
+        Returns:
+            bool: True if the JSON file exists and its size does not
+                exceed `MAX_ACCURACY_LOG_SIZE`, False otherwise.
+        """
         if not os.path.exists(self.accuracy_json):
             self.log.error("%s is missing", self.accuracy_json)
             return False
@@ -115,6 +178,15 @@ def accuracy_json_check(self):
         return True
 
     def loadgen_errors_check(self):
+        """Detect Loadgen errors reported in the MLPerf log.
+
+        If errors are present and not ignored by configuration, logs the
+        error messages and returns False to indicate failure.
+
+        Returns:
+            bool: True if no blocking Loadgen errors are present,
+                False otherwise.
+        """
         if self.mlperf_log.has_error():
             if self.config.ignore_uncommited:
                 has_other_errors = False
@@ -133,6 +205,17 @@ def loadgen_errors_check(self):
         return True
 
     def dataset_check(self):
+        """Verify the accuracy run covered the expected dataset size.
+
+        If `skip_dataset_size_check` is enabled in the configuration,
+        this check is skipped and returns True. Otherwise compares the
+        `qsl_reported_total_count` from the MLPerf log to the expected
+        dataset size for the model.
+
+        Returns:
+            bool: True if the dataset sizes match or the check is skipped,
+                False if the reported count differs from expected.
+        """
         if self.config.skip_dataset_size_check:
             self.log.info(
                 "%s Skipping dataset size check", self.path

tools/submission/submission_checker/checks/base.py

@@ -4,7 +4,7 @@
 class BaseCheck(ABC):
     """
     A generic check class meant to be inherited by concrete check implementations.
-    Subclasses must implement the `run()` method.
+    Subclasses must register their check methods into `self.checks`.
     """
 
     def __init__(self, log, path):
@@ -16,9 +16,7 @@ def __init__(self, log, path):
 
     def run_checks(self):
         """
-        Execute the check.
-        Must be implemented by subclasses.
-        Should return a CheckResult instance.
+        Execute all registered checks. Returns True if all checks pass, False otherwise.
         """
         valid = True
         errors = []
@@ -36,6 +34,7 @@ def run_checks(self):
         return valid
 
     def execute(self, check):
+        """Custom execution of a single check method."""
         return check()
 
     def __call__(self):

tools/submission/submission_checker/checks/compliance_check.py

@@ -11,8 +11,32 @@
 
 
 class ComplianceCheck(BaseCheck):
+    """Validate compliance test artifacts for a submission.
+
+    The `ComplianceCheck` class runs a set of validations against the
+    compliance directory produced with a submission. It verifies the
+    presence of required test subdirectories and files, runs delegated
+    performance and accuracy checks for compliance tests, and inspects
+    compliance-specific performance outputs.
+
+    The class delegates some checks to `PerformanceCheck` and
+    `AccuracyCheck` helpers when relevant. Results and file lists are
+    logged via the provided logger.
+    """
+
     def __init__(self, log, path, config: Config,
                  submission_logs: SubmissionLogs):
+        """Initialize the compliance checker.
+
+        Args:
+            log: Logger used to emit informational, warning, and error
+                messages about the compliance checks.
+            path: Filesystem path to the submission root being checked.
+            config (Config): Configuration provider for models and
+                compliance expectations.
+            submission_logs (SubmissionLogs): Parsed submission log
+                artifacts and loader metadata.
+        """
         super().__init__(log, path)
         self.submission_logs = submission_logs
         self.config = config
@@ -28,12 +52,30 @@ def __init__(self, log, path, config: Config,
         self.setup_checks()
 
     def setup_checks(self):
+        """Register the sequence of compliance checks to run.
+
+        Appends the per-submission validation callables to `self.checks` in
+        the order they should be executed by the checking framework.
+        """
         self.checks.append(self.dir_exists_check)
         self.checks.append(self.performance_check)
         self.checks.append(self.accuracy_check)
         self.checks.append(self.compliance_performance_check)
 
     def get_test_list(self, model):
+        """Return the list of compliance tests applicable to `model`.
+
+        The mapping of models to tests is read from the configuration
+        (`self.config.base`) using the pre-defined keys
+        `models_TEST01`, `models_TEST04`, and `models_TEST06`.
+
+        Args:
+            model (str): MLPerf benchmark/model identifier.
+
+        Returns:
+            list[str]: Ordered list of compliance test names to execute.
+        """
+
         test_list = []
         if model in self.config.base["models_TEST01"]:
             test_list.append("TEST01")
@@ -44,6 +86,18 @@ def get_test_list(self, model):
         return test_list
 
     def dir_exists_check(self):
+        """Verify required compliance directories and files exist.
+
+        Skips checks for the 'open' division. For each test in
+        `self.test_list`, ensures the expected test directory exists and
+        that required verification files are present depending on the
+        test type (accuracy/performance files for specific tests).
+
+        Returns:
+            bool: True if all required files and directories are present,
+                False otherwise.
+        """
+
         if self.division.lower() == "open":
             self.log.info(
                 "Compliance tests not needed for open division. Skipping tests on %s",
@@ -86,6 +140,17 @@ def dir_exists_check(self):
         return is_valid
 
     def performance_check(self):
+        """Run performance compliance checks for applicable tests.
+
+        For each test that requires a performance check (TEST01 and
+        TEST04), construct a `SubmissionLogs` object pointing at the
+        test's performance log and delegate to `PerformanceCheck`.
+
+        Returns:
+            bool: True if all delegated performance checks pass, False
+                if any fail.
+        """
+
         if self.division.lower() == "open":
             self.log.info(
                 "Compliance tests not needed for open division. Skipping tests on %s",
@@ -108,6 +173,20 @@ def performance_check(self):
         return is_valid
 
     def accuracy_check(self):
+        """Run accuracy compliance checks for applicable tests.
+
+        For TEST01, verifies deterministic-mode pass lines and checks the
+        `accuracy` directory contents and baseline/compliance accuracy
+        values against model-specific delta thresholds.
+
+        For TEST06, inspects the pre-parsed result lines for first-token,
+        EOS, and sample-length checks.
+
+        Returns:
+            bool: True if all required accuracy checks pass, False
+                otherwise.
+        """
+
         if self.division.lower() == "open":
             self.log.info(
                 "Compliance tests not needed for open division. Skipping tests on %s",
@@ -228,6 +307,17 @@ def accuracy_check(self):
         return is_valid
 
     def compliance_performance_check(self):
+        """Inspect compliance performance verification outputs.
+
+        For TEST01 and TEST04, checks the `verify_performance.txt` file for
+        a passing indicator and ensures the `performance/run_1` directory
+        contains the expected files (with optional exclusions).
+
+        Returns:
+            bool: True if all compliance performance checks pass, False
+                if any check fails.
+        """
+
         if self.division.lower() == "open":
             self.log.info(
                 "Compliance tests not needed for open division. Skipping tests on %s",