Pulling in latest ldp for bulk_evaluate_consensus (#826)

Author: jamesbraza

.pre-commit-config.yaml

@@ -86,7 +86,7 @@ repos:
           - anyio
           - fh-llm-client>=0.0.9 # Match pyproject.toml
           - fhaviary[llm]>=0.14 # Match pyproject.toml
-          - ldp>=0.17 # Match pyproject.toml
+          - ldp>=0.20 # Match pyproject.toml
           - html2text
           - httpx
           - pybtex

paperqa/_ldp_shims.py

@@ -14,8 +14,8 @@
     "SimpleAgentState",
     "UIndexMemoryModel",
     "_Memories",
+    "bulk_evaluate_consensus",
     "discounted_returns",
-    "evaluate_consensus",
     "set_training_mode",
 ]
 
@@ -34,7 +34,7 @@
         Callback,
         ComputeTrajectoryMetricsMixin,
         RolloutManager,
-        evaluate_consensus,
+        bulk_evaluate_consensus,
     )
     from ldp.graph.memory import Memory, UIndexMemoryModel
     from ldp.graph.op_utils import set_training_mode
@@ -63,5 +63,5 @@ class Callback:  # type: ignore[no-redef]
     SimpleAgentState = None  # type: ignore[assignment,misc]
     UIndexMemoryModel = None  # type: ignore[assignment,misc]
     discounted_returns = None  # type: ignore[assignment]
-    evaluate_consensus = None  # type: ignore[assignment]
+    bulk_evaluate_consensus = None  # type: ignore[assignment]
     set_training_mode = None  # type: ignore[assignment]

paperqa/agents/task.py

@@ -36,7 +36,7 @@
 from paperqa._ldp_shims import (
     Callback,
     ComputeTrajectoryMetricsMixin,
-    evaluate_consensus,
+    bulk_evaluate_consensus,
 )
 from paperqa.docs import Docs
 from paperqa.litqa import (
@@ -254,7 +254,7 @@ def extract_ideal(x: GradablePaperQAEnvironment | Frame) -> str:
         return query["ideal_answer"]
 
     try:
-        consensus, accuracy = await evaluate_consensus(
+        consensus, accuracy = await bulk_evaluate_consensus(
             data=data,
             grouping_fn=extract_question,
             extract_answer_fn=extract_answer,

pyproject.toml

@@ -77,7 +77,7 @@ dev = [
     "vcrpy>=6",  # Pin for https://github.com/kevin1024/vcrpy/issues/884
 ]
 ldp = [
-    "ldp>=0.17",  # For evaluate_consensus
+    "ldp>=0.20",  # For bulk_evaluate_consensus
 ]
 local = [
     "sentence-transformers",