feat: add sandbox logic (#624)

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

* **New Features**
* Added sandbox backend supporting AWS ECS Fargate infrastructure with
session management.
* Enabled interactive command execution and file upload capabilities
within sandboxes.
* Integrated AWS credential validation and comprehensive error handling.

* **Documentation**
* Added API documentation for the sandbox module and ECS Fargate usage.

* **Tests**
  * Added unit tests for sandbox module behavior and edge cases.

<sub>✏️ Tip: You can customize this high-level summary in your review
settings.</sub>

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Signed-off-by: Michal Bien <mbien@nvidia.com>

Author: Glorf

docs/references/api/nemo-evaluator/api/index.rst

@@ -45,5 +45,6 @@ The central point of evaluation is ``evaluate()`` function that takes standarize
 
     api-dataclasses
     nemo-evaluator.adapters <../adapters/adapters>
+    nemo-evaluator.sandbox <../sandbox/index>
 
 

docs/references/api/nemo-evaluator/sandbox/index.rst

@@ -0,0 +1,55 @@
+``nemo_evaluator.sandbox``
+======================================
+
+Sandbox implementations used by evaluation harnesses that need a tmux-like interactive session.
+
+This module is designed to keep dependencies **optional**:
+
+- The ECS Fargate implementation only imports AWS SDKs (``boto3``/``botocore``) when actually used.
+- Using the ECS sandbox also requires the AWS CLI (``aws``) and ``session-manager-plugin`` on the host.
+
+Usage (ECS Fargate)
+-------------------
+
+Typical usage is:
+
+- configure :class:`~nemo_evaluator.sandbox.ecs_fargate.EcsFargateConfig`
+- :meth:`~nemo_evaluator.sandbox.ecs_fargate.EcsFargateSandbox.spin_up` a sandbox context
+- create an interactive :class:`~nemo_evaluator.sandbox.base.NemoSandboxSession`
+
+Example::
+
+    from nemo_evaluator.sandbox import EcsFargateConfig, EcsFargateSandbox
+
+    cfg = EcsFargateConfig(
+        region="us-west-2",
+        cluster="my-ecs-cluster",
+        task_definition="my-task-def:1",
+        container_name="eval",
+        subnets=["subnet-abc"],
+        security_groups=["sg-xyz"],
+        s3_bucket="my-staging-bucket",
+    )
+
+    with EcsFargateSandbox.spin_up(
+        cfg=cfg,
+        task_id="task-001",
+        trial_name="trial-0001",
+        run_id="run-2026-01-12",
+    ) as sandbox:
+        session = sandbox.create_session("main")
+        session.send_keys(["echo hello", "Enter"], block=True)
+        print(session.capture_pane())
+
+Prerequisites / Notes
+---------------------
+
+- The harness host must have **AWS CLI** and **session-manager-plugin** installed.
+- If you use S3-based fallbacks (large uploads / long commands), configure ``s3_bucket``.
+
+.. automodule:: nemo_evaluator.sandbox
+    :members:
+    :undoc-members:
+    :member-order: bysource
+
+

packages/nemo-evaluator/src/nemo_evaluator/sandbox/__init__.py

@@ -0,0 +1,33 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from .base import NemoEvaluatorSandbox, NemoSandboxCommand, NemoSandboxSession
+from .ecs_fargate import (
+    AwsCliMissingError,
+    EcsExecError,
+    EcsFargateConfig,
+    EcsFargateSandbox,
+)
+
+__all__ = [
+    "NemoEvaluatorSandbox",
+    "NemoSandboxCommand",
+    "NemoSandboxSession",
+    "AwsCliMissingError",
+    "EcsExecError",
+    "EcsFargateConfig",
+    "EcsFargateSandbox",
+]

packages/nemo-evaluator/src/nemo_evaluator/sandbox/base.py

@@ -0,0 +1,115 @@
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from pathlib import Path
+from typing import ContextManager, Iterable, Protocol, runtime_checkable
+
+
+@dataclass(frozen=True)
+class NemoSandboxCommand:
+    """
+    TB-independent command model for driving an interactive terminal.
+
+    Mirrors the fields terminal-bench agents commonly use (but does not depend on TB).
+    """
+
+    command: str
+    min_timeout_sec: float = 0.0
+    max_timeout_sec: float = 180.0
+    block: bool = False
+    append_enter: bool = True
+
+
+@runtime_checkable
+class NemoSandboxSession(Protocol):
+    """
+    Minimal session API used by agents/harnesses (tmux-like).
+    """
+
+    def send_keys(
+        self,
+        keys: str | list[str],
+        block: bool = False,
+        min_timeout_sec: float = 0.0,
+        max_timeout_sec: float = 180.0,
+    ) -> None: ...
+
+    def send_command(self, command: NemoSandboxCommand) -> None: ...
+
+    def capture_pane(self, capture_entire: bool = False) -> str: ...
+
+    def is_session_alive(self) -> bool: ...
+
+    def get_incremental_output(self) -> str: ...
+
+    def get_asciinema_timestamp(self) -> float: ...
+
+    def copy_to_sandbox(
+        self,
+        paths: list[Path] | Path,
+        container_dir: str | None = None,
+        container_filename: str | None = None,
+    ) -> None: ...
+
+
+class NemoEvaluatorSandbox(ABC):
+    """
+    Abstract factory for evaluator sandboxes.
+
+    Implementations are responsible for provisioning an isolated environment and exposing
+    a tmux-like session API for agents to interact with it.
+    """
+
+    @classmethod
+    @abstractmethod
+    def spin_up(
+        cls,
+        *,
+        task_id: str,
+        trial_name: str,
+        run_id: str,
+        pre_upload_paths: Iterable[Path] | None = None,
+        upload_dest_dir: str | None = None,
+        **kwargs,
+    ) -> ContextManager["NemoEvaluatorSandbox"]:
+        raise NotImplementedError
+
+    @abstractmethod
+    def create_session(
+        self,
+        session_name: str,
+        is_active_stream: bool = False,
+        as_configured_user: bool = True,
+    ) -> NemoSandboxSession:
+        raise NotImplementedError
+
+    @abstractmethod
+    def copy_to_sandbox(
+        self,
+        *,
+        paths: list[Path] | Path,
+        container_dir: str | None = None,
+        container_filename: str | None = None,
+    ) -> None:
+        raise NotImplementedError
+
+    @abstractmethod
+    def stop(self) -> None:
+        raise NotImplementedError