[TurnCompleter] Add ruff+mypy to minimono_turn_completer, fix typing

.github/workflows/preparedness_turncompleter_lint.yml

@@ -0,0 +1,51 @@
+name: Run linting on preparedness_turn_completer
+
+on:
+  pull_request:
+    paths:
+      - 'project/preparedness_turn_completer/**'
+    branches:
+      - '**'
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - ready_for_review
+  workflow_dispatch:
+    inputs:
+      test_branch:
+        description: 'Branch to run the tests on'
+        required: false
+        default: 'main'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    env:
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        lfs: true
+        fetch-depth: 0
+
+    - name: Install uv
+      uses: astral-sh/setup-uv@v5
+
+    - name: Install dependencies
+      working-directory: ./project/preparedness_turn_completer
+      env:
+        UV_GIT_LFS: 1
+      run: |
+        uv sync
+
+    - name: run mypy
+      working-directory: ./project/preparedness_turn_completer
+      run: |
+        uv run mypy preparedness_turn_completer
+
+    - name: run ruff-strict
+      working-directory: ./project/preparedness_turn_completer
+      run: |
+        uv run ruff check --output-format=github --no-fix

.pre-commit-config.yaml

@@ -8,7 +8,7 @@ repos:
       # picks up config from [tool.ruff] in pyproject.toml
       - id: ruff-format
         name: ruff-format (black-style)
-        files: (?x)^project/(paperbench|swelancer|compatibility_api|compatibility_api_preparedness)/
+        files: (?x)^project/(paperbench|swelancer|compatibility_api|compatibility_api_preparedness|preparedness_turn_completer)/
 
       # Lenient hook: runs auto‑fix on all files using a lenient config.
       - id: ruff
@@ -25,7 +25,7 @@ repos:
           - --fix
           - --fix-only
           - --exit-non-zero-on-fix
-        files: (?x)^project/(alcatraz|nanoeval|paperbench|swelancer|compatibility_api|compatibility_api_preparedness)/
+        files: (?x)^project/(alcatraz|nanoeval|paperbench|swelancer|compatibility_api|compatibility_api_preparedness|preparedness_turn_completer)/
 
   - repo: https://github.com/astral-sh/uv-pre-commit
     # uv version.
@@ -35,6 +35,10 @@ repos:
         name: uv-lock-paperbench
         args: [--project, project/paperbench]
         files: (?x)^project/paperbench/
+      - id: uv-lock
+        name: uv-lock-preparedness_turn_completer
+        args: [--project, project/preparedness_turn_completer]
+        files: (?x)^project/preparedness_turn_completer/
       - id: uv-lock
         name: uv-lock-swelancer
         args: [--project, project/swelancer]

project/alcatraz/alcatraz/clusters/local.py

@@ -38,6 +38,16 @@
 import pydantic
 import tenacity
 import yaml
+from docker import DockerClient
+from docker.errors import APIError, NotFound
+from docker.models.containers import Container, ExecResult
+from docker.models.networks import Network
+from filelock import Timeout as LockTimeout
+from filelock import UnixFileLock
+from jupyter_client.asynchronous.client import AsyncKernelClient
+from jupyter_client.manager import AsyncKernelManager
+from pydantic import BaseModel, ConfigDict, Extra, Field, field_validator
+from typing_extensions import TypedDict, override
 
 # import diskcache as dc
 from alcatraz.clusters._container_proc import ContainerProc
@@ -50,16 +60,6 @@
     AlcatrazUnexpectedSystemError,
     ExecutionResult,
 )
-from docker import DockerClient
-from docker.errors import APIError, NotFound
-from docker.models.containers import Container, ExecResult
-from docker.models.networks import Network
-from filelock import Timeout as LockTimeout
-from filelock import UnixFileLock
-from jupyter_client.asynchronous.client import AsyncKernelClient
-from jupyter_client.manager import AsyncKernelManager
-from pydantic import BaseModel, ConfigDict, Extra, Field, field_validator
-from typing_extensions import TypedDict, override
 
 logger = logging.getLogger(__name__)
 

project/alcatraz/alcatraz/utils/cmds.py

@@ -1,4 +1,5 @@
 import structlog.stdlib
+
 from alcatraz.clusters.interface import ExecutionResult
 from alcatraz.clusters.local import BaseAlcatrazCluster
 

project/compatibility_api/compatibility_api/task.py

@@ -1,9 +1,10 @@
 from abc import ABC
 from typing import Self
 
+from pydantic import Field, model_validator
+
 from nanoeval.solvers.computer_tasks.code_execution_interface import NetworkMode
 from nanoeval.solvers.computer_tasks.task import ComputerTask
-from pydantic import Field, model_validator
 
 
 class CompatibleComputerTask(ComputerTask, ABC):

project/compatibility_api_preparedness/compatibility_api_preparedness/api.py

@@ -1,6 +1,7 @@
-import chz
 from compatibility_api.api import CompatibilityAPI
 
+import chz
+
 
 @chz.chz
 class CompatibilityAPIPreparedness(CompatibilityAPI):

project/nanoeval/nanoeval/_db.py

@@ -9,6 +9,7 @@
 
 import blobfile as bf
 import dill
+
 from nanoeval._persistent_db import PersistentDb
 from nanoeval.fs_paths import writable_root_dir
 from nanoeval.recorder_protocol import uuid

project/nanoeval/nanoeval/_executor_worker.py

@@ -15,9 +15,11 @@
 import aiodebug.log_slow_callbacks
 import blobfile as bf
 import dill
-import nanoeval._db as db
 import psutil
 import structlog.stdlib
+from structlog.contextvars import bound_contextvars
+
+import nanoeval._db as db
 from nanoeval import monitor as monitor
 from nanoeval._aiomonitor import start_aiomonitor
 from nanoeval._db import as_default_db, default_db
@@ -29,7 +31,6 @@
 from nanoeval.library_config import LibraryConfig, get_library_config, set_library_config
 from nanoeval.recorder import RecorderProtocol, set_default_recorder
 from nanoeval.setup import global_exit_stack, nanoeval_logging, properly_closed_thread_pool
-from structlog.contextvars import bound_contextvars
 
 logger = structlog.stdlib.get_logger(component=__name__)
 

project/nanoeval/nanoeval/_multiprocessing_utils.py

@@ -11,6 +11,7 @@
 from loky import get_reusable_executor, set_loky_pickler
 from loky.backend import reduction
 from loky.backend.context import set_start_method
+
 from nanoeval.setup import global_exit_stack
 
 logger = logging.getLogger(__name__)

project/nanoeval/nanoeval/_persistent_db.py

@@ -10,8 +10,9 @@
 
 import blobfile as bf
 import boostedblob as bbb
-import chz
 from boostedblob.request import RequestFailure
+
+import chz
 from nanoeval.fs_paths import database_dir
 
 logger = logging.getLogger(__name__)

project/nanoeval/nanoeval/asyncio_utils.py

@@ -7,7 +7,6 @@
 from functools import cached_property
 from types import TracebackType
 
-import chz
 from typing_extensions import (
     Any,
     AsyncContextManager,
@@ -18,6 +17,8 @@
     TypeVar,
 )
 
+import chz
+
 T = TypeVar("T")
 
 

project/nanoeval/nanoeval/bin/resume.py

@@ -1,8 +1,9 @@
 import asyncio
 from typing import Any
 
-import chz
 import structlog.stdlib
+
+import chz
 from nanoeval._db import open_run_set_db
 from nanoeval.evaluation import run_eval_in_database
 from nanoeval.setup import nanoeval_entrypoint

project/nanoeval/nanoeval/bin/resume_test.py

@@ -1,15 +1,16 @@
 from tempfile import NamedTemporaryFile
 
+import pytest
+from typing_extensions import override
+
 import chz
 import nanoeval
-import pytest
 from nanoeval._db import conn, open_run_set_db
 from nanoeval.bin.resume import resume
 from nanoeval.eval import EvalSpec, RunnerArgs
 from nanoeval.examples._gpqa import GPQAEval
 from nanoeval.setup import global_exit_stack
 from nanoeval.solvers.mcq import Answer, MCQSolver, MCQTask
-from typing_extensions import override
 
 
 @chz.chz

project/nanoeval/nanoeval/bin/sqlite.py

@@ -1,8 +1,9 @@
 import logging
 import subprocess
 
-import chz
 import litecli
+
+import chz
 from nanoeval._db import open_run_set_db
 from nanoeval.setup import nanoeval_entrypoint
 

project/nanoeval/nanoeval/eval.py

@@ -16,13 +16,14 @@
     TypeVar,
 )
 
+from pydantic import BaseModel, field_validator, model_validator
+from tqdm import tqdm
+
 import chz
 from chz.factories import function
 from nanoeval._multiprocessing_utils import check_multiprocess_safe
 from nanoeval.asyncio_utils import HasAsyncContextManager
 from nanoeval.recorder_protocol import RecorderConfig
-from pydantic import BaseModel, field_validator, model_validator
-from tqdm import tqdm
 
 
 class Task(BaseModel):

project/nanoeval/nanoeval/evaluation.py

@@ -10,18 +10,19 @@
 from pprint import pformat
 from typing import Any
 
-import chz
 import dill
-import nanoeval._db as db
 import structlog.stdlib
+from structlog.contextvars import bound_contextvars
+from tqdm import tqdm
+
+import chz
+import nanoeval._db as db
 from nanoeval._db import cached_deserialize, default_db, open_run_set_db
 from nanoeval._executor_worker import ExecutorExceptionWrapper, ensure_executor_workers_started
 from nanoeval.eval import Eval, EvalSpec, RolloutSystemError, RunnerArgs, Task
 from nanoeval.library_config import get_library_config
 from nanoeval.recorder import RecorderProtocol, dummy_recorder
 from nanoeval.setup import global_exit_stack
-from structlog.contextvars import bound_contextvars
-from tqdm import tqdm
 
 logger = structlog.stdlib.get_logger(component=__name__)
 

project/nanoeval/nanoeval/evaluation_test.py

@@ -5,17 +5,18 @@
 from typing import Any
 from unittest.mock import patch
 
+import numpy as np
+import pytest
+from typing_extensions import override
+
 import chz
 import nanoeval
 import nanoeval._db as db
-import numpy as np
-import pytest
 from nanoeval.asyncio_utils import cancel_task
 from nanoeval.evaluation import EvalSpec, RolloutSystemError, RunnerArgs, Task
 from nanoeval.examples._gpqa import GPQAEval
 from nanoeval.setup import global_exit_stack
 from nanoeval.solvers.mcq import Answer, MCQSolver, MCQTask, MockSolver
-from typing_extensions import override
 
 
 @pytest.mark.asyncio

project/nanoeval/nanoeval/examples/_gpqa.py

@@ -2,11 +2,12 @@
 
 import random
 
-import chz
 import pandas as pd
-from nanoeval.solvers.mcq import MCQEval, Question
 from typing_extensions import override
 
+import chz
+from nanoeval.solvers.mcq import MCQEval, Question
+
 
 @chz.chz
 class GPQAEval(MCQEval):

project/nanoeval/nanoeval/examples/bundled_evals_test.py

@@ -1,4 +1,5 @@
 import pytest
+
 from nanoeval.eval import EvalSpec, RunnerArgs
 from nanoeval.examples._gpqa import GPQAEval
 from nanoeval.solvers.mcq import MockSolver

project/nanoeval/nanoeval/json_recorder_test.py

@@ -1,8 +1,9 @@
 from pathlib import Path
 from tempfile import NamedTemporaryFile
 
-import nanoeval
 import pytest
+
+import nanoeval
 from nanoeval.eval import EvalSpec, RunnerArgs
 from nanoeval.examples._gpqa import GPQAEval
 from nanoeval.json_recorder import JsonRecorderConfig

project/nanoeval/nanoeval/library_config.py

@@ -10,12 +10,13 @@
 from pathlib import Path
 from typing import TYPE_CHECKING, Any, Generator, Literal, Self
 
-import chz
 import pandas as pd
 import structlog
-from nanoeval.recorder_protocol import BasicRunSpec, RecorderConfig, RecorderProtocol
 from structlog.typing import EventDict
 
+import chz
+from nanoeval.recorder_protocol import BasicRunSpec, RecorderConfig, RecorderProtocol
+
 if TYPE_CHECKING:
     from nanoeval.eval import EvalSpec
 

project/nanoeval/nanoeval/metrics/agents.py

@@ -3,6 +3,7 @@
 from typing import Any
 
 import pandas as pd
+
 from nanoeval.eval import RolloutSystemError, Task
 from nanoeval.metrics.standard import compute_default_metrics
 

project/nanoeval/nanoeval/metrics/standard.py

@@ -4,6 +4,7 @@
 
 import pandas as pd
 from more_itertools import flatten
+
 from nanoeval.eval import RolloutSystemError, TResult, TTask
 from nanoeval.library_config import get_library_config
 

project/nanoeval/nanoeval/metrics/standard_test.py

@@ -3,6 +3,7 @@
 
 import pandas as pd
 import pytest
+
 from nanoeval.metrics.standard import (
     compute_default_metrics,
     compute_default_metrics_on_correctness_without_answer_groups,

project/nanoeval/nanoeval/monitor.py

@@ -8,9 +8,10 @@
 
 import dill
 import pandas as pd
-from nanoeval._persistent_db import database_dir
 from pyarrow import ArrowInvalid
 
+from nanoeval._persistent_db import database_dir
+
 logger = logging.getLogger(__name__)