Skip to content

Commit 5d99909

Browse files
authored
fix: add reference to simple scoring (#1758)
fixes: #1758
1 parent a9a0330 commit 5d99909

File tree

2 files changed

+9
-13
lines changed

2 files changed

+9
-13
lines changed

docs/howtos/applications/_metrics_llm_calls.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
## Debug LLM based metrics using tracing
1+
# Explain or debug LLM based metrics using tracing
22

33
While evaluating using LLM based metrics, each metric may make one or more calls to the LLM. These traces are important to understand the results of the metrics and to debug any issues.
44
This notebook demonstrates how to export the LLM traces and analyze them.

src/ragas/metrics/_simple_criteria.py

Lines changed: 8 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,7 @@ class SingleTurnSimpleCriteriaInput(BaseModel):
4949

5050

5151
class MultiTurnSimpleCriteriaInput(BaseModel):
52-
user_input: t.Optional[str] = Field(
53-
description="The input to the model", default=None
54-
)
52+
user_input: str = Field(description="The input to the model")
5553
reference: t.Optional[str] = Field(
5654
description="The reference response", default=None
5755
)
@@ -172,20 +170,18 @@ async def _single_turn_ascore(
172170
async def _ascore(self, row: t.Dict, callbacks: Callbacks) -> float:
173171
assert self.llm is not None, "set LLM before use"
174172

175-
user_input, context, response = (
176-
row["user_input"],
173+
user_input, response, retrieved_contexts, reference = (
174+
row.get("user_input"),
175+
row.get("response"),
177176
row.get("retrieved_contexts"),
178-
row["response"],
177+
row.get("reference"),
179178
)
180179

181-
if context is not None:
182-
if isinstance(context, list):
183-
context = "\n".join(context)
184-
user_input = f"Question: {user_input} Answer using context: {context}"
185-
186180
prompt_input = SingleTurnSimpleCriteriaInput(
187181
user_input=user_input,
188182
response=response,
183+
retrieved_contexts=retrieved_contexts,
184+
reference=reference,
189185
)
190186

191187
response = await self.single_turn_prompt.generate(
@@ -200,11 +196,11 @@ async def _multi_turn_ascore(
200196
self, sample: MultiTurnSample, callbacks: Callbacks
201197
) -> float:
202198
assert self.llm is not None, "LLM is not set"
203-
assert sample.reference is not None, "Reference is not set"
204199

205200
interaction = sample.pretty_repr()
206201
prompt_input = MultiTurnSimpleCriteriaInput(
207202
user_input=interaction,
203+
reference=sample.reference,
208204
)
209205
response = await self.multi_turn_prompt.generate(
210206
data=prompt_input,

0 commit comments

Comments
 (0)