feat: add OCI exports configuration to evaluation jobs (#90)

* feat: add OCI exports configuration to evaluation jobs

Add EvaluationExports, EvaluationExportsOCI, OCIConnectionConfig, and
OCICoordinates models to support OCI artifact persistence in evaluation
job submissions. Export new models from the public API surface.

Co-Authored-By: Claude <noreply@anthropic.com>

* fix: linting from ruff

* chore: update readme with minimal info

* chore: remove examples in pydantic hint

---------

Co-authored-by: Claude <noreply@anthropic.com>

Author: gnaulak-redhat

README.md

@@ -383,8 +383,16 @@ from evalhub.adapter import (
 **EvalHub Service User:**
 ```python
 # Interacting with EvalHub REST API
-from evalhub.client import EvalHubClient
-from evalhub.models.api import ModelConfig, JobSubmissionRequest, BenchmarkConfig
+from evalhub import (
+    EvalHubClient,
+    BenchmarkConfig,
+    EvaluationExports,
+    EvaluationExportsOCI,
+    JobSubmissionRequest,
+    ModelConfig,
+    OCIConnectionConfig,
+    OCICoordinates,
+)
 ```
 
 ## Complete Example

src/evalhub/__init__.py

@@ -29,6 +29,8 @@
     BenchmarkInfo,
     CollectionRef,
     ErrorResponse,
+    EvaluationExports,
+    EvaluationExportsOCI,
     EvaluationJob,
     EvaluationResponse,
     EvaluationResult,
@@ -40,6 +42,8 @@
     JobStatus,
     JobSubmissionRequest,
     ModelConfig,
+    OCIConnectionConfig,
+    OCICoordinates,
 )
 
 __version__ = "0.1.4"
@@ -52,6 +56,8 @@
     "BenchmarkInfo",
     "CollectionRef",
     "ErrorResponse",
+    "EvaluationExports",
+    "EvaluationExportsOCI",
     "EvaluationJob",
     "EvaluationResponse",
     "EvaluationResult",
@@ -63,6 +69,8 @@
     "JobStatus",
     "JobSubmissionRequest",
     "ModelConfig",
+    "OCIConnectionConfig",
+    "OCICoordinates",
 ]
 
 # Conditional imports based on available extras

src/evalhub/models/__init__.py

@@ -14,6 +14,8 @@
     CollectionRef,
     ErrorInfo,
     ErrorResponse,
+    EvaluationExports,
+    EvaluationExportsOCI,
     EvaluationJob,
     EvaluationJobResource,
     EvaluationJobResults,
@@ -31,6 +33,8 @@
     JobStatus,
     JobSubmissionRequest,
     ModelConfig,
+    OCIConnectionConfig,
+    OCICoordinates,
     PassCriteria,
     PrimaryScore,
     Provider,
@@ -41,8 +45,12 @@
 __all__ = [
     # Job & Evaluation models
     "JobStatus",
+    "EvaluationExports",
+    "EvaluationExportsOCI",
     "EvaluationStatus",
     "ModelConfig",
+    "OCIConnectionConfig",
+    "OCICoordinates",
     "EvaluationResult",
     "EvaluationJob",
     "EvaluationJobResource",

src/evalhub/models/api.py

@@ -281,6 +281,49 @@ class ExperimentConfig(BaseModel):
     )
 
 
+class OCICoordinates(BaseModel):
+    """OCI artifact coordinates for persistence."""
+
+    oci_host: str = Field(..., description="OCI registry host (e.g., 'quay.io')")
+    oci_repository: str = Field(
+        ..., description="OCI repository (e.g., 'my-org/my-repo')"
+    )
+    oci_tag: str | None = Field(default=None, description="OCI tag (e.g., 'eval-123')")
+    oci_subject: str | None = Field(
+        default=None,
+        description="Optional OCI subject identifier (in same registry and repo)",
+    )
+    annotations: dict[str, str] = Field(
+        default_factory=dict, description="Custom annotations"
+    )
+
+
+class OCIConnectionConfig(BaseModel):
+    """K8s connection configuration for OCI registry authentication."""
+
+    connection: str = Field(
+        ...,
+        description="Name of a K8s Secret (type kubernetes.io/dockerconfigjson) for OCI registry auth",
+    )
+
+
+class EvaluationExportsOCI(BaseModel):
+    """OCI export configuration for an evaluation job."""
+
+    coordinates: OCICoordinates = Field(..., description="OCI artifact coordinates")
+    k8s: OCIConnectionConfig | None = Field(
+        default=None, description="K8s connection for OCI registry auth"
+    )
+
+
+class EvaluationExports(BaseModel):
+    """Optional exports configuration for an evaluation job."""
+
+    oci: EvaluationExportsOCI | None = Field(
+        default=None, description="OCI export configuration"
+    )
+
+
 class JobSubmissionRequest(BaseModel):
     """Request to submit an evaluation job.
 
@@ -303,6 +346,10 @@ class JobSubmissionRequest(BaseModel):
         default=None,
         description="MLFlow experiment configuration. When provided, the evaluation job will be tracked in MLFlow.",
     )
+    exports: EvaluationExports | None = Field(
+        default=None,
+        description="Optional exports configuration (e.g., OCI artifact persistence)",
+    )
 
     @model_validator(mode="after")
     def check_benchmarks_or_collection(self) -> "JobSubmissionRequest":
@@ -344,6 +391,10 @@ class EvaluationJob(BaseModel):
         default=None,
         description="MLFlow experiment configuration",
     )
+    exports: EvaluationExports | None = Field(
+        default=None,
+        description="Optional exports configuration",
+    )
 
     # Convenience properties to access nested fields
     @property
@@ -398,37 +449,6 @@ class EvaluationResponse(BaseModel):
     duration_seconds: float = Field(..., description="Total evaluation time")
 
 
-class OCICoordinates(BaseModel):
-    """OCI artifact coordinates for persistence."""
-
-    oci_host: str = Field(
-        ..., description="OCI registry host (e.g., 'quay.io')", examples=["quay.io"]
-    )
-    oci_repository: str = Field(
-        ...,
-        description="OCI repository (e.g., 'my-org/my-repo')",
-        examples=["my-org/my-repo"],
-    )
-    oci_tag: str | None = Field(
-        default=None, description="OCI tag (e.g., 'eval-123')", examples=["eval-123"]
-    )
-    oci_subject: str | None = Field(
-        default=None,
-        description="Optional OCI subject identifier (in same registry and repo)",
-        examples=["quay.io/my-org/my-repo:model"],
-    )
-    annotations: dict[str, str] = Field(
-        default_factory=dict,
-        description="Custom annotations",
-        examples=[
-            {
-                "model": "quay.io/my-org/my-repo:model",
-                "some": "value",
-            }
-        ],
-    )
-
-
 class EvaluationJobFilesLocation(BaseModel):
     """Files location for persisting as OCI artifacts for an evaluation job."""
 

tests/unit/test_evalhub_client.py

@@ -28,10 +28,16 @@
     BaseSyncClient,
 )
 from evalhub.models.api import (
+    BenchmarkConfig,
     CollectionRef,
+    EvaluationExports,
+    EvaluationExportsOCI,
     EvaluationJob,
     JobStatus,
+    JobSubmissionRequest,
     ModelConfig,
+    OCIConnectionConfig,
+    OCICoordinates,
 )
 
 # Environment variable to enable real server testing
@@ -323,6 +329,65 @@ def test_sync_client_submit_job_with_collection(self) -> None:
 
         client.close()
 
+    @pytest.mark.skipif(
+        EVALHUB_TEST_BASE_URL is not None,
+        reason="Skipping in real server mode - would create actual jobs",
+    )
+    def test_sync_client_submit_job_with_exports_oci(self) -> None:
+        """Test that SyncEvalHubClient can submit jobs with OCI exports configuration."""
+        client = SyncEvalHubClient()
+        mock_job_data = {
+            "resource": {
+                "id": "job_oci_1",
+                "tenant": "default",
+                "created_at": "2024-01-01T12:00:00Z",
+                "updated_at": "2024-01-01T12:00:00Z",
+            },
+            "name": "oci-export-eval",
+            "description": "Evaluate with OCI exports",
+            "tags": [],
+            "status": {"state": JobStatus.PENDING.value},
+            "model": {"url": "http://localhost:8000/v1", "name": "test-model"},
+            "benchmarks": [{"id": "mmlu", "provider_id": "lm_eval", "parameters": {}}],
+            "exports": {
+                "oci": {
+                    "coordinates": {
+                        "oci_host": "quay.io",
+                        "oci_repository": "my-org/my-repo",
+                        "oci_tag": "eval-123",
+                    },
+                    "k8s": {"connection": "my-pull-secret"},
+                }
+            },
+        }
+        mock_response = Mock()
+        mock_response.json.return_value = mock_job_data
+
+        with patch.object(client, "_request", return_value=mock_response):
+            request = JobSubmissionRequest(
+                name="oci-export-eval",
+                description="Evaluate with OCI exports",
+                model=ModelConfig(url="http://localhost:8000/v1", name="test-model"),
+                benchmarks=[
+                    BenchmarkConfig(id="mmlu", provider_id="lm_eval", parameters={})
+                ],
+                exports=EvaluationExports(
+                    oci=EvaluationExportsOCI(
+                        coordinates=OCICoordinates(
+                            oci_host="quay.io",
+                            oci_repository="my-org/my-repo",
+                            oci_tag="eval-123",
+                        ),
+                        k8s=OCIConnectionConfig(connection="my-pull-secret"),
+                    ),
+                ),
+            )
+            job = client.jobs.submit(request)
+            assert isinstance(job, EvaluationJob)
+            assert job.name == "oci-export-eval"
+
+        client.close()
+
     def test_sync_client_context_manager(self) -> None:
         """Test SyncEvalHubClient as context manager."""
         with SyncEvalHubClient() as client:

tests/unit/test_models_api.py

@@ -12,6 +12,8 @@
     CollectionRef,
     ErrorInfo,
     ErrorResponse,
+    EvaluationExports,
+    EvaluationExportsOCI,
     EvaluationJob,
     EvaluationResponse,
     EvaluationResult,
@@ -22,6 +24,8 @@
     JobStatus,
     JobSubmissionRequest,
     ModelConfig,
+    OCIConnectionConfig,
+    OCICoordinates,
     ProviderList,
 )
 from pydantic import ValidationError
@@ -318,6 +322,101 @@ def test_submission_excludes_none_on_dump(self) -> None:
         assert "collection" in dumped
         assert dumped["collection"]["id"] == "healthcare_v1"
 
+    def test_submission_with_exports_oci(self) -> None:
+        """Test JobSubmissionRequest with full OCI exports configuration."""
+        request = JobSubmissionRequest(
+            name="test-eval",
+            model=ModelConfig(url="http://localhost:8000/v1", name="test-model"),
+            benchmarks=[
+                BenchmarkConfig(id="mmlu", provider_id="lm_eval", parameters={})
+            ],
+            exports=EvaluationExports(
+                oci=EvaluationExportsOCI(
+                    coordinates=OCICoordinates(
+                        oci_host="quay.io",
+                        oci_repository="my-org/my-repo",
+                        oci_tag="eval-123",
+                        oci_subject="quay.io/my-org/my-repo:model",
+                        annotations={"model": "llama2"},
+                    ),
+                    k8s=OCIConnectionConfig(connection="my-pull-secret"),
+                ),
+            ),
+        )
+        assert request.exports is not None
+        assert request.exports.oci is not None
+        assert request.exports.oci.coordinates.oci_host == "quay.io"
+        assert request.exports.oci.coordinates.oci_repository == "my-org/my-repo"
+        assert request.exports.oci.coordinates.oci_tag == "eval-123"
+        assert request.exports.oci.k8s is not None
+        assert request.exports.oci.k8s.connection == "my-pull-secret"
+
+    def test_submission_with_exports_oci_minimal(self) -> None:
+        """Test JobSubmissionRequest with minimal OCI exports (required fields only)."""
+        request = JobSubmissionRequest(
+            name="test-eval",
+            model=ModelConfig(url="http://localhost:8000/v1", name="test-model"),
+            benchmarks=[
+                BenchmarkConfig(id="mmlu", provider_id="lm_eval", parameters={})
+            ],
+            exports=EvaluationExports(
+                oci=EvaluationExportsOCI(
+                    coordinates=OCICoordinates(
+                        oci_host="quay.io",
+                        oci_repository="my-org/my-repo",
+                    ),
+                ),
+            ),
+        )
+        assert request.exports is not None
+        assert request.exports.oci is not None
+        assert request.exports.oci.coordinates.oci_tag is None
+        assert request.exports.oci.k8s is None
+
+    def test_submission_exports_excluded_when_none_on_dump(self) -> None:
+        """Test that exports is excluded from dump when not set."""
+        request = JobSubmissionRequest(
+            name="test-eval",
+            model=ModelConfig(url="http://localhost:8000/v1", name="test-model"),
+            benchmarks=[
+                BenchmarkConfig(id="mmlu", provider_id="lm_eval", parameters={})
+            ],
+        )
+        dumped = request.model_dump(exclude_none=True)
+        assert "exports" not in dumped
+
+    def test_submission_exports_oci_dump_matches_server_schema(self) -> None:
+        """Test that serialized exports matches the server's expected JSON structure."""
+        request = JobSubmissionRequest(
+            name="test-eval",
+            model=ModelConfig(url="http://localhost:8000/v1", name="test-model"),
+            benchmarks=[
+                BenchmarkConfig(id="mmlu", provider_id="lm_eval", parameters={})
+            ],
+            exports=EvaluationExports(
+                oci=EvaluationExportsOCI(
+                    coordinates=OCICoordinates(
+                        oci_host="quay.io",
+                        oci_repository="my-org/my-repo",
+                        oci_tag="eval-123",
+                    ),
+                    k8s=OCIConnectionConfig(connection="my-pull-secret"),
+                ),
+            ),
+        )
+        dumped = request.model_dump(exclude_none=True)
+        assert dumped["exports"] == {
+            "oci": {
+                "coordinates": {
+                    "oci_host": "quay.io",
+                    "oci_repository": "my-org/my-repo",
+                    "oci_tag": "eval-123",
+                    "annotations": {},
+                },
+                "k8s": {"connection": "my-pull-secret"},
+            }
+        }
+
 
 class TestExperimentConfig:
     """Test cases for ExperimentConfig and ExperimentTag models."""