fix: change to schema (#1780)

Author: shahules786

src/ragas/dataset_schema.py

@@ -553,7 +553,6 @@ def upload(self, base_url: str = RAGAS_API_URL, verbose: bool = True) -> str:
 class PromptAnnotation(BaseModel):
     prompt_input: t.Dict[str, t.Any]
     prompt_output: t.Dict[str, t.Any]
-    is_accepted: bool
     edited_output: t.Optional[t.Dict[str, t.Any]] = None
 
     def __getitem__(self, key):
@@ -808,6 +807,7 @@ def get_prompt_annotations(self) -> t.Dict[str, t.List[PromptAnnotation]]:
         """
         prompt_annotations = defaultdict(list)
         for sample in self.samples:
-            for prompt_name, prompt_annotation in sample.prompts.items():
-                prompt_annotations[prompt_name].append(prompt_annotation)
+            if sample.is_accepted:
+                for prompt_name, prompt_annotation in sample.prompts.items():
+                    prompt_annotations[prompt_name].append(prompt_annotation)
         return prompt_annotations

src/ragas/metrics/base.py

@@ -311,9 +311,6 @@ def _optimize_demonstration(
             input_examples, output_examples = [], []
             for i, prompt_annotation in enumerate(prompt_annotation_list):
                 try:
-                    # skip if the prompt is not accepted
-                    if not prompt_annotation.is_accepted:
-                        continue
                     input_examples.append(
                         input_model.model_validate(prompt_annotation.prompt_input)
                     )

src/ragas/optimizers/genetic.py

@@ -377,7 +377,7 @@ def feedback_mutation(
         )
         improved_candidates = []
         dataset = dataset.filter(lambda x: x["is_accepted"])
-
+        sample_size = min(sample_size, len(dataset))
         exec = Executor(
             desc="Feedback Mutation",
             raise_exceptions=raise_exceptions,