Skip to content

Commit 5109f32

Browse files
transformer issues get fixed
1 parent 29612a5 commit 5109f32

File tree

2 files changed

+28
-9
lines changed

2 files changed

+28
-9
lines changed

libs/langchain/langchain/evaluation/hallucination/detector.py

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,31 @@
1-
from typing import List, Dict
2-
from transformers import pipeline
1+
from typing import TYPE_CHECKING
2+
3+
if TYPE_CHECKING:
4+
from transformers import pipeline as PipelineType
5+
6+
# Lazy import for runtime
7+
try:
8+
from transformers import pipeline
9+
except ImportError:
10+
pipeline = None
11+
312

413
class HallucinationDetector:
5-
"""
6-
Simple Hallucination Detector using NLI models (e.g., facebook/bart-large-mnli).
14+
"""Simple Hallucination Detector using NLI models (e.g., facebook/bart-large-mnli).
715
- Extract claims (basic sentence split)
816
- Verify claims against evidence docs using NLI
917
- Compute hallucination rate
1018
"""
1119

1220
def __init__(self, model_name: str = "facebook/bart-large-mnli"):
21+
if pipeline is None:
22+
raise ImportError(
23+
"The 'transformers' package is required for HallucinationDetector. "
24+
"Install it with `pip install transformers`."
25+
)
1326
self.nli_pipeline = pipeline("text-classification", model=model_name)
1427

15-
def extract_claims(self, text: str) -> List[str]:
28+
def extract_claims(self, text: str) -> list[str]:
1629
"""Naive sentence-based claim extraction"""
1730
return [c.strip() for c in text.split(".") if c.strip()]
1831

@@ -21,14 +34,20 @@ def verify_claim(self, claim: str, evidence: str) -> bool:
2134
result = self.nli_pipeline(f"{claim} </s></s> {evidence}")
2235
return result[0]["label"].lower() == "entailment"
2336

24-
def verify_claim_multi(self, claim: str, evidence_docs: List[str]) -> bool:
37+
def verify_claim_multi(self, claim: str, evidence_docs: list[str]) -> bool:
2538
"""A claim is supported if any evidence doc entails it"""
2639
return any(self.verify_claim(claim, e) for e in evidence_docs)
2740

28-
def compute_hallucination_rate(self, text: str, evidence_docs: List[str]) -> Dict[str, float]:
41+
def compute_hallucination_rate(
42+
self, text: str, evidence_docs: list[str]
43+
) -> dict[str, float]:
2944
claims = self.extract_claims(text)
3045
if not claims:
31-
return {"total_claims": 0, "unsupported_claims": 0, "hallucination_rate": 0.0}
46+
return {
47+
"total_claims": 0,
48+
"unsupported_claims": 0,
49+
"hallucination_rate": 0.0,
50+
}
3251

3352
unsupported = sum(not self.verify_claim_multi(c, evidence_docs) for c in claims)
3453
return {

libs/langchain/tests/integration_tests/evaluation/hallucination/test_detector.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,7 @@ def test_compute_hallucination_rate_mock(detector_mock: HallucinationDetector) -
120120
evidence = [
121121
(
122122
"Barack Obama served as the 44th President of the United States "
123-
"from 2009 to 2017.",
123+
"from 2009 to 2017."
124124
),
125125
"Barack Obama was born in Hawaii, not Kenya.",
126126
]

0 commit comments

Comments
 (0)