[autorevert] add mocked tests for autorevert functionality (#7026)

Added mocked unit tests, manually ensured that when key invariants are
broken, tests fail.

Added workflow to test autorevert module.

### Testing
* see [this
job](https://github.com/pytorch/test-infra/actions/runs/17057797640/job/48358574501?pr=7026)

---------

Co-authored-by: Jean Schmidt <[email protected]>

Author: izaitsevfb

.github/workflows/pytorch-auto-revert-tests.yml

@@ -0,0 +1,35 @@
+name: Test pytorch-auto-revert lambda
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/pytorch-auto-revert-tests.yml
+      - aws/lambda/pytorch-auto-revert/**
+  push:
+    branches:
+      - main
+    paths:
+      - .github/workflows/pytorch-auto-revert-tests.yml
+      - aws/lambda/pytorch-auto-revert/**
+
+defaults:
+  run:
+    working-directory: aws/lambda/pytorch-auto-revert/
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: '3.11'
+          cache: pip
+      - name: Install virtualenv
+        run: pip install virtualenv==20.34.0
+      - name: Run autorevert tests
+        env:
+          # GitHub provides this automatically; used by workflow_resolver tests
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          make test

aws/lambda/pytorch-auto-revert/Makefile

@@ -43,3 +43,7 @@ create-deployment-package: deployment.zip
 lintrunner: venv/bin/lintrunner
 	# lintrunner only works properly with virtualenv if you activate it first
 	. venv/bin/activate && lintrunner -a -v --force-color --config ../../../.lintrunner.toml --paths-cmd='git grep -Il .'
+
+.PHONY: test
+test: venv/bin/python
+	venv/bin/python -m unittest discover -vs pytorch_auto_revert/tests -p 'test_*.py'

aws/lambda/pytorch-auto-revert/pytorch_auto_revert/tests/resources/issue_comments.json

@@ -0,0 +1,3 @@
+[
+  [999999999, "Automated revert triggered. -c ghfirst"]
+]

aws/lambda/pytorch-auto-revert/pytorch_auto_revert/tests/resources/push_commits.json

@@ -0,0 +1,7 @@
+[
+  ["aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "Nightly: update submodules", "2025-01-02T14:00:00"],
+  ["bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", "Revert \"Some change\"\n\nThis reverts commit bbc0df1094b5a4dcd2cce83f8402127b07913231.\n\nMore context: https://github.com/pytorch/pytorch/pull/123#issuecomment-999999999 -c ghfirst", "2025-01-02T13:30:00"],
+  ["bbc0df1094b5a4dcd2cce83f8402127b07913231", "Introduce change that breaks ROCm shard2", "2025-01-02T11:00:00"],
+  ["33ec6e3e9aa2b93f7d907236aa10ba2b36355018", "Previous good commit", "2025-01-02T10:00:00"],
+  ["4fd5fabee9b2641440a413adf54f728fe2816375", "Newer commit with flakiness", "2025-01-02T12:00:00"]
+]

aws/lambda/pytorch-auto-revert/pytorch_auto_revert/tests/resources/workflow_job_main.json

@@ -0,0 +1,44 @@
+[
+  ["trunk","4fd5fabee9b2641440a413adf54f728fe2816375","linux-jammy-rocm-py3.10 / build","success","completed","","2025-01-02T12:00:00"],
+  ["trunk","4fd5fabee9b2641440a413adf54f728fe2816375","linux-jammy-rocm-py3.10 / test (default, 1, 6, linux.rocm.gpu.2)","success","completed","GHA error","2025-01-02T12:00:10"],
+  ["trunk","4fd5fabee9b2641440a413adf54f728fe2816375","linux-jammy-rocm-py3.10 / test (default, 2, 6, linux.rocm.gpu.2)","success","completed","pytest failure","2025-01-02T12:00:20"],
+  ["trunk","4fd5fabee9b2641440a413adf54f728fe2816375","linux-jammy-rocm-py3.10 / test (default, 3, 6, linux.rocm.gpu.2)","success","completed","GHA error","2025-01-02T12:00:30"],
+  ["trunk","4fd5fabee9b2641440a413adf54f728fe2816375","linux-jammy-rocm-py3.10 / test (default, 4, 6, linux.rocm.gpu.2)","success","completed","pytest failure","2025-01-02T12:00:40"],
+  ["trunk","4fd5fabee9b2641440a413adf54f728fe2816375","linux-jammy-rocm-py3.10 / test (default, 5, 6, linux.rocm.gpu.2)","failure","completed","pytest failure","2025-01-02T12:00:50"],
+  ["trunk","4fd5fabee9b2641440a413adf54f728fe2816375","linux-jammy-rocm-py3.10 / test (default, 6, 6, linux.rocm.gpu.2)","success","completed","GHA error","2025-01-02T12:01:00"],
+  ["trunk","bbc0df1094b5a4dcd2cce83f8402127b07913231","linux-jammy-rocm-py3.10 / build","success","completed","","2025-01-02T11:00:00"],
+  ["trunk","bbc0df1094b5a4dcd2cce83f8402127b07913231","linux-jammy-rocm-py3.10 / test (default, 1, 6, linux.rocm.gpu.2)","success","completed","GHA error","2025-01-02T11:00:10"],
+  ["trunk","bbc0df1094b5a4dcd2cce83f8402127b07913231","linux-jammy-rocm-py3.10 / test (default, 2, 6, linux.rocm.gpu.2)","failure","completed","pytest failure","2025-01-02T11:00:20"],
+  ["trunk","bbc0df1094b5a4dcd2cce83f8402127b07913231","linux-jammy-rocm-py3.10 / test (default, 3, 6, linux.rocm.gpu.2)","success","completed","pytest failure","2025-01-02T11:00:30"],
+  ["trunk","bbc0df1094b5a4dcd2cce83f8402127b07913231","linux-jammy-rocm-py3.10 / test (default, 4, 6, linux.rocm.gpu.2)","success","completed","GHA error","2025-01-02T11:00:40"],
+  ["trunk","bbc0df1094b5a4dcd2cce83f8402127b07913231","linux-jammy-rocm-py3.10 / test (default, 5, 6, linux.rocm.gpu.2)","success","completed","pytest failure","2025-01-02T11:00:50"],
+  ["trunk","bbc0df1094b5a4dcd2cce83f8402127b07913231","linux-jammy-rocm-py3.10 / test (default, 6, 6, linux.rocm.gpu.2)","success","completed","pytest failure","2025-01-02T11:01:00"],
+  ["trunk","33ec6e3e9aa2b93f7d907236aa10ba2b36355018","linux-jammy-rocm-py3.10 / build","success","completed","","2025-01-02T10:00:00"],
+  ["trunk","33ec6e3e9aa2b93f7d907236aa10ba2b36355018","linux-jammy-rocm-py3.10 / test (default, 1, 6, linux.rocm.gpu.2)","success","completed","GHA error","2025-01-02T10:00:10"],
+  ["trunk","33ec6e3e9aa2b93f7d907236aa10ba2b36355018","linux-jammy-rocm-py3.10 / test (default, 2, 6, linux.rocm.gpu.2)","success","completed","pytest failure","2025-01-02T10:00:20"],
+  ["trunk","33ec6e3e9aa2b93f7d907236aa10ba2b36355018","linux-jammy-rocm-py3.10 / test (default, 3, 6, linux.rocm.gpu.2)","success","completed","GHA error","2025-01-02T10:00:30"],
+  ["trunk","33ec6e3e9aa2b93f7d907236aa10ba2b36355018","linux-jammy-rocm-py3.10 / test (default, 4, 6, linux.rocm.gpu.2)","success","completed","pytest failure","2025-01-02T10:00:40"],
+  ["trunk","33ec6e3e9aa2b93f7d907236aa10ba2b36355018","linux-jammy-rocm-py3.10 / test (default, 5, 6, linux.rocm.gpu.2)","success","completed","GHA error","2025-01-02T10:00:50"],
+  ["trunk","33ec6e3e9aa2b93f7d907236aa10ba2b36355018","linux-jammy-rocm-py3.10 / test (default, 6, 6, linux.rocm.gpu.2)","success","completed","pytest failure","2025-01-02T10:01:00"],
+
+  ["trunk-pending","1111111111111111111111111111111111111111","linux-jammy-rocm-py3.10 / build","success","completed","","2025-01-03T12:00:00"],
+  ["trunk-pending","1111111111111111111111111111111111111111","linux-jammy-rocm-py3.10 / test (default, 2, 6, linux.rocm.gpu.2)","failure","completed","pytest failure","2025-01-03T12:00:10"],
+  ["trunk-pending","2222222222222222222222222222222222222222","linux-jammy-rocm-py3.10 / build","success","completed","","2025-01-03T11:00:00"],
+  ["trunk-pending","2222222222222222222222222222222222222222","linux-jammy-rocm-py3.10 / test (default, 2, 6, linux.rocm.gpu.2)","failure","completed","pytest failure","2025-01-03T11:00:10"],
+  ["trunk-pending","3333333333333333333333333333333333333333","linux-jammy-rocm-py3.10 / build","success","completed","","2025-01-03T10:00:00"],
+  ["trunk-pending","3333333333333333333333333333333333333333","linux-jammy-rocm-py3.10 / test (default, 2, 6, linux.rocm.gpu.2)","success","in_progress","","2025-01-03T10:00:10"],
+
+  ["trunk-baseline-fails","4444444444444444444444444444444444444444","linux-jammy-rocm-py3.10 / build","success","completed","","2025-01-04T12:00:00"],
+  ["trunk-baseline-fails","4444444444444444444444444444444444444444","linux-jammy-rocm-py3.10 / test (default, 2, 6, linux.rocm.gpu.2)","failure","completed","pytest failure","2025-01-04T12:00:10"],
+  ["trunk-baseline-fails","5555555555555555555555555555555555555555","linux-jammy-rocm-py3.10 / build","success","completed","","2025-01-04T11:00:00"],
+  ["trunk-baseline-fails","5555555555555555555555555555555555555555","linux-jammy-rocm-py3.10 / test (default, 2, 6, linux.rocm.gpu.2)","failure","completed","pytest failure","2025-01-04T11:00:10"],
+  ["trunk-baseline-fails","6666666666666666666666666666666666666666","linux-jammy-rocm-py3.10 / build","success","completed","","2025-01-04T10:00:00"],
+  ["trunk-baseline-fails","6666666666666666666666666666666666666666","linux-jammy-rocm-py3.10 / test (default, 2, 6, linux.rocm.gpu.2)","failure","completed","pytest failure","2025-01-04T10:00:10"],
+
+  ["trunk-one-failure","7777777777777777777777777777777777777777","linux-jammy-rocm-py3.10 / build","success","completed","","2025-01-05T12:00:00"],
+  ["trunk-one-failure","7777777777777777777777777777777777777777","linux-jammy-rocm-py3.10 / test (default, 2, 6, linux.rocm.gpu.2)","failure","completed","pytest failure","2025-01-05T12:00:10"],
+  ["trunk-one-failure","8888888888888888888888888888888888888888","linux-jammy-rocm-py3.10 / build","success","completed","","2025-01-05T11:00:00"],
+  ["trunk-one-failure","8888888888888888888888888888888888888888","linux-jammy-rocm-py3.10 / test (default, 2, 6, linux.rocm.gpu.2)","success","completed","","2025-01-05T11:00:10"],
+  ["trunk-one-failure","9999999999999999999999999999999999999999","linux-jammy-rocm-py3.10 / build","success","completed","","2025-01-05T10:00:00"],
+  ["trunk-one-failure","9999999999999999999999999999999999999999","linux-jammy-rocm-py3.10 / test (default, 2, 6, linux.rocm.gpu.2)","success","completed","","2025-01-05T10:00:10"]
+]

aws/lambda/pytorch-auto-revert/pytorch_auto_revert/tests/resources/workflow_job_restarted.json

@@ -0,0 +1,13 @@
+{
+  "bbc0df1094b5a4dcd2cce83f8402127b07913231": [
+    ["linux-jammy-rocm-py3.10 / test (default, 2, 6, linux.rocm.gpu.2)", "failure", "completed", "pytest failure", "2025-01-02T13:10:00"],
+    ["linux-jammy-rocm-py3.10 / build", "success", "completed", "", "2025-01-02T13:00:00"]
+  ],
+  "33ec6e3e9aa2b93f7d907236aa10ba2b36355018": [
+    ["linux-jammy-rocm-py3.10 / test (default, 2, 6, linux.rocm.gpu.2)", "success", "completed", "pytest failure", "2025-01-02T12:10:00"],
+    ["linux-jammy-rocm-py3.10 / build", "success", "completed", "", "2025-01-02T12:00:00"]
+  ],
+  "pending_prev": [
+    ["linux-jammy-rocm-py3.10 / test (default, 2, 6, linux.rocm.gpu.2)", "success", "in_progress", "", "2025-01-02T12:10:00"]
+  ]
+}