Add score cost per model

Author: pierlj

src/flare/schema.py

@@ -133,7 +133,7 @@ class ScorerOutput(FlareModel):
     details: dict[str, Any] = Field(
         default_factory=dict, description="optional details about the scoring process"
     )
-    cost: float = 0
+    cost: dict[str, float] = Field(default_factory=dict)
 
 # TODO: would be better to have subclass of scorer, with custom details as pydantic model
 

src/flare/scorer/majority_vote_model.py

@@ -30,7 +30,7 @@ def __init__(self, message):
 class MajorityVote(FlareModel):
     decision: bool
     raw_responses: dict[str, dict]
-    cost: float = 0
+    cost: dict[str, float]
 
 
 class MajorityVoteEvaluationModel(BaseModel):
@@ -57,7 +57,7 @@ async def majority_vote(
             ]
         )
 
-        evaluation_cost = 0
+        evaluation_cost = {}
         for model, response in zip(self.models, responses):
             try:
                 response_json = extract_json_object(response.choices[0].message.content)
@@ -69,7 +69,7 @@ async def majority_vote(
                     votes[model.litellm_model]["reason"] = response_json["reason"]
 
                 response_cost = response.model_dump()["usage"].get("cost", response._hidden_params.get("response_cost", 0))
-                evaluation_cost += response_cost
+                evaluation_cost[model.litellm_model] = response_cost
 
             except Exception as e:
                 logging.error(