end to end tlm

Author: ryansingman

tlm/core/types.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import abc
 from typing import Protocol
 
 from pydantic import BaseModel
@@ -49,25 +50,29 @@ class Scorable(Protocol):
     Encapsulates scores and score groups.
     """
 
+    @abc.abstractmethod
     async def acalculate(self, messages: Messages, reference_answer: Completion) -> float | None:
         """Calculate the score."""
         ...
 
+    @abc.abstractmethod
     async def acost(self, messages: Messages, reference_answer_token_cost: TokenCost) -> TokenCost:
         """Get the token cost of the score."""
         ...
 
 
-class Score(Protocol, Scorable):
+class Score(Scorable):
     """A score."""
 
     config: BaseModel
 
+    @abc.abstractmethod
     def __init__(self, config: BaseModel, reference_index: int, **kwargs):
         """Initialize a score."""
         ...
 
     @classmethod
+    @abc.abstractmethod
     def from_config(cls, config: BaseModel, reference_index: int, **kwargs) -> Score:
         """Create a score from a config."""
         ...
@@ -78,19 +83,23 @@ class Reference(Protocol):
 
     config: BaseModel
 
+    @abc.abstractmethod
     def __init__(self, config: BaseModel, reference_index: int, **kwargs):
         """Initialize a reference."""
         ...
 
     @classmethod
+    @abc.abstractmethod
     def from_config(cls, config: BaseModel, reference_index: int, **kwargs) -> Reference:
         """Create a reference from a config."""
         ...
 
+    @abc.abstractmethod
     async def agenerate(self, messages: Messages) -> Completion:
         """Generate a reference answer."""
         ...
 
+    @abc.abstractmethod
     async def acost(self, messages: Messages) -> TokenCost:
         """Get the token cost of the reference answer."""
         ...