[autorevert] Fix bug on job retrival for autorevert lambda (#6979)

After fixing the bug, the new statistics are:

```
==================================================
SUMMARY STATISTICS
==================================================
Workflow(s): Lint, trunk, pull, inductor, linux-binary-manywheel
Timeframe: 4320 hours
Commits checked: 33519
Auto revert patterns detected: 1345
Actual reverts inside auto revert patterns detected (precision): 219 (16.3%)
Total revert commits in period: 585

Revert categories:
  nosignal: 202 (34.5%)
  ghfirst: 156 (26.7%)
  uncategorized: 104 (17.8%)
  ignoredsignal: 68 (11.6%)
  weird: 45 (7.7%)
  landrace: 10 (1.7%)

Total reverts excluding ghfirst: 429
Reverts (excluding ghfirst) that dont match any auto revert pattern detected (recall): 250 (58.3%)
Per workflow precision:
  Lint: 45 reverts out of 75 patterns (60.0%) [excluding ghfirst: 41 (54.7%)]
  trunk: 30 reverts out of 136 patterns (22.1%) [excluding ghfirst: 28 (20.6%)]
  pull: 104 reverts out of 859 patterns (12.1%) [excluding ghfirst: 92 (10.7%)]
  inductor: 39 reverts out of 269 patterns (14.5%) [excluding ghfirst: 38 (14.1%)]
  linux-binary-manywheel: 1 reverts out of 6 patterns (16.7%) [excluding ghfirst: 0 (0.0%)]
```

The main bug is that when checking for commits before/after with the
same job, it actually concatenated all commits+jobs before and after,
instead of only returning the next one.

I added also the ergonomics for lambda invocation

---------

Co-authored-by: Ivan Zaitsev <[email protected]>

Author: jeanschmidt

aws/lambda/pytorch-auto-revert/Makefile

@@ -23,7 +23,7 @@ run-local: venv/bin/python
 
 .PHONY: run-local-workflows
 run-local-workflows: venv/bin/python
-	venv/bin/python -m pytorch_auto_revert workflows pull.yml
+	venv/bin/python -m pytorch_auto_revert autorevert-checker Lint trunk pull inductor linux-binary-manywheel --hours 4320 --ignore-common-errors
 
 deployment.zip:
 	mkdir -p deployment

aws/lambda/pytorch-auto-revert/pytorch_auto_revert/__main__.py

@@ -95,6 +95,11 @@ def get_opts() -> argparse.Namespace:
         action="store_true",
         help="Show what would be restarted without actually doing it (use with --do-restart)",
     )
+    workflow_parser.add_argument(
+        "--ignore-common-errors",
+        action="store_true",
+        help="Ignore common errors in autorevert patterns (e.g., 'No tests found')",
+    )
 
     # workflow-restart-checker subcommand
     workflow_restart_parser = subparsers.add_parser(
@@ -159,15 +164,29 @@ def main(*args, **kwargs) -> None:
             "ClickHouse connection test failed. Please check your configuration."
         )
 
-    if opts.subcommand == "lambda":
-        print("TODO: run lambda flow")
+    if opts.subcommand is None:
+        autorevert_checker(
+            [
+                "Lint",
+                "trunk",
+                "pull",
+                "inductor",
+                "linux-binary-manywheel",
+            ],
+            hours=2,
+            verbose=True,
+            do_restart=True,
+            dry_run=False,
+            ignore_common_errors=True,
+        )
     elif opts.subcommand == "autorevert-checker":
         autorevert_checker(
             opts.workflows,
             hours=opts.hours,
             verbose=opts.verbose,
             do_restart=opts.do_restart,
             dry_run=opts.dry_run,
+            ignore_common_errors=opts.ignore_common_errors,
         )
     elif opts.subcommand == "workflow-restart-checker":
         workflow_restart_checker(opts.workflow, commit=opts.commit, days=opts.days)

aws/lambda/pytorch-auto-revert/pytorch_auto_revert/autorevert_checker.py

@@ -63,14 +63,20 @@ def get_job_base_names(self) -> Set[str]:
 class AutorevertPatternChecker:
     """Detects autorevert patterns in workflow job failures."""
 
-    def __init__(self, workflow_names: List[str] = None, lookback_hours: int = 48):
+    def __init__(
+        self,
+        workflow_names: List[str] = None,
+        lookback_hours: int = 48,
+        ignore_classification_rules: Set[str] = None,
+    ):
         self.workflow_names = workflow_names or []
         self.lookback_hours = lookback_hours
         self._workflow_commits_cache: Dict[str, List[CommitJobs]] = {}
         self._commit_history = None
+        self._ignore_classification_rules = ignore_classification_rules or set()
 
     def get_workflow_commits(self, workflow_name: str) -> List[CommitJobs]:
-        """Get workflow commits for a specific workflow, fetching if needed."""
+        """Get workflow commits for a specific workflow, fetching if needed. From newer to older"""
         if workflow_name not in self._workflow_commits_cache:
             self._fetch_workflow_data()
         return self._workflow_commits_cache.get(workflow_name, [])
@@ -90,7 +96,7 @@ def commit_history(self) -> List[Dict]:
         return self._commit_history or []
 
     def _fetch_workflow_data(self):
-        """Fetch workflow job data from ClickHouse for all workflows in batch."""
+        """Fetch workflow job data from ClickHouse for all workflows in batch. From newer to older"""
         if not self.workflow_names:
             return
 
@@ -203,25 +209,25 @@ def _find_last_commit_with_job(
         self, commits: Iterable[CommitJobs], job_name: str
     ) -> Optional[Tuple[CommitJobs, List[JobResult]]]:
         """
-        Find the last commit in the iterable that has a job with the specified name.
+        Find the first commit (in the provided iteration order) that has a job with the specified name.
 
         Args:
             commits: Iterable of CommitJobs to search
             job_name: The job name to look for
 
         Returns:
-            The last CommitJobs object that contains the specified job, or None if not found
+            The first CommitJobs object (per the iterable's order) that contains the specified job, or None if not found
         """
         job_results = []
         for commit in commits:
             for job in commit.jobs:
                 if job.name.split("(")[0] == job_name:  # Normalize job name
                     job_results.append(job)
-        if job_results:
-            return (
-                commit,
-                job_results,
-            )
+            if job_results:
+                return (
+                    commit,
+                    job_results,
+                )
         return None, None
 
     def detect_autorevert_pattern_workflow(self, workflow_name: str) -> List[Dict]:
@@ -266,6 +272,10 @@ def detect_autorevert_pattern_workflow(self, workflow_name: str) -> List[Dict]:
                 suspected_failure_class_rule,
                 suspected_failure_job_name,
             ) in suspected_failures:
+                if suspected_failure_class_rule in self._ignore_classification_rules:
+                    # Skip ignored classification rules
+                    continue
+
                 newer_commit_same_job, newer_same_jobs = (
                     self._find_last_commit_with_job(
                         (commits[j] for j in range(i - 1, -1, -1)),
@@ -304,18 +314,8 @@ def detect_autorevert_pattern_workflow(self, workflow_name: str) -> List[Dict]:
                     # No older commit with the same job found
                     continue
 
-                if any(
-                    j.name.split("(")[0] != job_name
-                    for j in last_commit_with_same_job.failed_jobs
-                ):
-                    # newr commit has the same job failing
-                    continue
-
-                if any(
-                    j.classification_rule == suspected_failure_class_rule
-                    for j in last_same_jobs
-                ):
-                    # The last commit with the same job has the same failure classification
+                if any(j.classification_rule == failure_rule for j in last_same_jobs):
+                    # The older commit has the same job failing with same rule
                     continue
 
                 patterns.append(

aws/lambda/pytorch-auto-revert/pytorch_auto_revert/testers/autorevert.py

@@ -10,9 +10,20 @@ def autorevert_checker(
     verbose: bool = False,
     do_restart: bool = False,
     dry_run: bool = False,
+    ignore_common_errors=True,
 ):
+    common_errors = {
+        "GHA error",
+        "GHA timeout",
+        "sccache error",
+    }
+
     # Initialize checker
-    checker = AutorevertPatternChecker(workflow_names, hours)
+    checker = AutorevertPatternChecker(
+        workflow_names,
+        hours,
+        ignore_classification_rules=common_errors if ignore_common_errors else set(),
+    )
 
     # Fetch data
     if verbose:
@@ -108,7 +119,7 @@ def autorevert_checker(
                     "category", "uncategorized"
                 )
                 print(
-                    f"✓ REVERTED ({category}): {second_commit[:8]} was reverted by {revert_result['revert_sha'][:8]} "
+                    f"✓ REVERTED ({category}): {second_commit} was reverted by {revert_result['revert_sha'][:8]} "
                     f"after {revert_result['hours_after_target']:.1f} hours"
                 )
                 reverted_patterns.append(pattern)