fixed the sorting bug in the gradients and skipped the good examples in the gradients

Author: liyin2015

adalflow/adalflow/core/generator.py

@@ -642,6 +642,7 @@ def _backward_through_one_predecessor(
         prompt_str: str,
         is_intermediate_node: bool = False,
     ):
+        """Creating gradient/textual feedback for prompt type parameters."""
         if not pred.requires_opt:
             log.debug(
                 f"Generator: Skipping {pred} as it does not require optimization."
@@ -704,7 +705,9 @@ def _backward_through_one_predecessor(
             template=obj_ins_template,
             prompt_kwargs={
                 "response_desc": response.role_desc,
-                "response_gradient": response.get_gradient_and_context_text(),
+                "response_gradient": response.get_gradient_and_context_text(
+                    skip_correct_sample=True
+                ),
                 "instruction_to_backward_engine": pred.instruction_to_backward_engine,
             },
         )()
@@ -716,11 +719,16 @@ def _backward_through_one_predecessor(
         gradient_output: GeneratorOutput = None
         if response._score is not None and float(response._score) > 0.9:
             log.debug(f"EvalFnToTextLoss: Skipping {pred} as the score is high enough.")
-            manual_response = f"You get a high score: {response._score}."
+            # TODO: plus score descriptions
+            manual_response = f"You get score: {response._score}."
             gradient_output = GeneratorOutput(
                 data=manual_response, raw_response=manual_response
             )
         else:
+            # manual_response = f"You get score: {response._score}."
+            # gradient_output = GeneratorOutput(
+            #     data=manual_response, raw_response=manual_response
+            # )
 
             gradient_output: GeneratorOutput = backward_engine(
                 prompt_kwargs=backward_engine_prompt_kwargs

adalflow/adalflow/optim/parameter.py

@@ -212,11 +212,12 @@ def __init__(
 
         # here are used for demo parameter, filled by generator.forward
         self._traces: Dict[str, DataClass] = {}  # id to data items (DynamicDataClass)
+        self._student_traces: Dict[str, DataClass] = {}  # id
+
         self._score: float = (
-            score  # end to end evaluation score, TODO: might have multiple scores if using multiple eval fns
+            score  # end to end evaluation score, TODO: might have multiple scores if using multiple eval fns  # score is set in the gradients in the backward pass
         )
 
-        self._student_traces: Dict[str, DataClass] = {}  # id
         self._demos: List[DataClass] = (
             []
         )  # used for the optimizer to save the proposed demos
@@ -431,10 +432,13 @@ def get_gradients_names(self) -> str:
         names = ", ".join(names)
         return names
 
-    def get_gradient_and_context_text(self) -> str:
+    def get_gradient_and_context_text(self, skip_correct_sample: bool = False) -> str:
         """Aggregates and returns:
         1. the gradients
         2. the context text for which the gradients are computed
+
+        Sort the gradients from the lowest score to the highest score.
+        Highlight the gradients with the lowest score to the optimizer.
         """
         from adalflow.core.prompt_builder import Prompt
 
@@ -444,18 +448,26 @@ def get_gradient_and_context_text(self) -> str:
 
         # sore gradients by the _score from low to high
         self.gradients = sorted(
-            self.gradients, key=lambda x: x._score if x._score else 1
+            self.gradients, key=lambda x: x._score if x._score is not None else 1
         )
+        # print the score for the sorted gradients
+        lowest_score_gradients = []
+        for i, g in enumerate(self.gradients):
+            if skip_correct_sample:
+                if g._score > 0.5:
+                    continue
+            lowest_score_gradients.append(g)
+            print(f"{i} Score: {g._score} for {g.name}, {type(g._score)}")
 
         gradient_context_combined = list(
             zip(
-                self.gradients,
-                [self.gradients_context[g] for g in self.gradients],
+                lowest_score_gradients,
+                [self.gradients_context[g] for g in lowest_score_gradients],
             )
         )
         # set all gradients value to None
-        for g in self.gradients:
-            g.data = None
+        # for g in self.gradients:
+        #     g.data = None
 
         gradient_context_combined_str = Prompt(
             template=COMBINED_GRADIENTS_TEMPLATE,
@@ -520,7 +532,11 @@ def report_cycle(cycle_nodes: List["Parameter"]):
 
     def backward(
         self,
-    ):  # engine should be the llm or customized backwards function to pass feedback
+    ):
+        """
+        Apply backward pass for for all nodes in the graph by reversing the topological order.
+        """
+        # engine should be the llm or customized backwards function to pass feedback
 
         # topological sort of all the predecessors of the current parameter in the graph
         log.debug(f"Backward pass for {self.data}, backward function: {self.grad_fn}")
@@ -543,7 +559,6 @@ def build_topo(node: Parameter):
             if not node.requires_opt:
                 log.debug(f"Skipping {node.name} as it does not require optimization")
                 continue
-            node.gradients = _check_and_reduce_gradients(node)
             log.debug(f"v: {node.data}, grad_fn: {node.grad_fn}, {node.get_grad_fn()}")
             if node.get_grad_fn() is not None:  # gradient function takes in the engine
                 log.debug(f"Calling gradient function for {node.name}")
@@ -949,20 +964,3 @@ def from_dict(cls, data: dict):
     def __repr__(self):
         return f"Parameter(name={self.name}, requires_opt={self.requires_opt}, param_type={self.param_type}, role_desc={self.role_desc}, data={self.data}, predecessors={self.predecessors}, gradients={self.gradients},\
             raw_response={self.raw_response}, input_args={self.input_args}, traces={self._traces})"
-
-
-def _check_and_reduce_gradients(variable: Parameter) -> Set[Parameter]:
-
-    if variable.get_gradient_and_context_text() == "":
-        log.debug(f"No gradients detected for {variable.data}")
-        return variable.gradients
-    if len(variable.gradients) == 1:
-        log.debug(f"Only one gradient, no need to reduce: {variable.gradients}")
-        return variable.gradients
-    else:
-        log.debug(
-            f"Multiple gradients detected for {variable.data}. But we are not reducting them."
-        )
-        return variable.gradients
-
-    # TODO: Implement the reduction logic later

adalflow/adalflow/optim/text_grad/text_loss_with_eval_fn.py

@@ -322,6 +322,7 @@ def _backward_through_one_predecessor(
         )
 
         # backward the end to end score
+        # TODO: not really useful
         pred.set_score(response.data)
         print(f"setting pred name {pred.name} score to {response.data}")
 

adalflow/adalflow/optim/text_grad/tgd_optimizer.py

@@ -37,7 +37,12 @@ class HistoryPrompt(DataClass):
 ####################################################################################################
 # Textual Gradient Descent Optimizer
 ####################################################################################################
-
+# {% if failed_proposals %}
+# Here are the past failed proposals:
+# {% for failed_proposal in failed_proposals %}
+# {{loop.index}}. {{failed_proposal}}
+# {% endfor %}
+# {% endif %}
 TEXT_GRAD_DESC_TEMPLATE = r"""<START_OF_SYSTEM_PROMPT>
 {{optimizer_system_prompt}}
 <END_OF_SYSTEM_PROMPT>
@@ -51,13 +56,6 @@ class HistoryPrompt(DataClass):
 {{loop.index}}. {{history}}
 {% endfor %}
 IMPORTANT: Your goal is to generate new variable values that score higher than all previous iterations.
-
-{% if failed_proposals %}
-Here are the past failed proposals:
-{% for failed_proposal in failed_proposals %}
-{{loop.index}}. {{failed_proposal}}
-{% endfor %}
-{% endif %}
 <END_OF_HISTORY_PERFORMANCE>
 {% endif %}
 Here are the context and feedback for the variable:
@@ -176,7 +174,7 @@ class TGDOptimizer(TextOptimizer):
     params: ParamsT
     constraints: List[str]
     params_history: Dict[str, List[HistoryPrompt]] = {}  # id to history
-    failed_proposals: Dict[str, List[HistoryPrompt]] = {}  # only need the value
+    # failed_proposals: Dict[str, List[HistoryPrompt]] = {}  # only need the value
 
     def __init__(
         self,
@@ -189,7 +187,7 @@ def __init__(
         in_context_examples: List[str] = None,  # TODO: in-context examples
         num_gradient_memory: int = 0,  # TODO: gradient memory and momentum, for now it is not useful
         max_past_history: int = 3,
-        max_failed_proposals: int = 3,
+        # max_failed_proposals: int = 3,
     ):
         from adalflow.core.generator import Generator
         from adalflow.core import Prompt
@@ -229,12 +227,12 @@ def __init__(
         )
 
         self.max_past_history = max_past_history
-        self.max_failed_proposals = max_failed_proposals
+        # self.max_failed_proposals = max_failed_proposals
 
         # initate the past history for each parameter
         for param in self.params:
             self.params_history[param.id] = []
-            self.failed_proposals[param.id] = []
+            # self.failed_proposals[param.id] = []
 
     @property
     def constraint_text(self):
@@ -289,39 +287,39 @@ def render_history(self, param_id: str) -> List[str]:
             history.to_yaml(exclude=["id"]) for history in self.params_history[param_id]
         ]
 
-    def add_failed_proposal(self):
-        """Save a copy of the current value of the parameter in the failed proposals."""
-        for param in self.params:
-            failed_proposal = HistoryPrompt(
-                id=param.id,
-                value=param.data,
-                eval_score=None,
-            )
-            self.failed_proposals[param.id].append(failed_proposal)
-            if len(self.failed_proposals[param.id]) > self.max_failed_proposals:
-                for _ in range(
-                    len(self.failed_proposals[param.id]) - self.max_failed_proposals
-                ):
-                    self.failed_proposals[param.id].pop()
-        # if param_id not in self.failed_proposals:
-        #     self.failed_proposals[param_id] = []
-        # failed_proposal = HistoryPrompt(
-        #     id=param_id,
-        #     value=value,
-        #     eval_score=None,
-        # )
-        # self.failed_proposals[param_id].append(failed_proposal)
-        # if len(self.failed_proposals[param_id]) > self.max_failed_proposals:
-        #     for _ in range(len(self.failed_proposals[param_id]) - self.max_failed_proposals):
-        #         self.failed_proposals[param_id].pop()
-
-    def render_failed_proposals(self, param_id: str) -> List[str]:
-        if param_id not in self.failed_proposals:
-            return []
-        return [
-            history.to_yaml(exclude=["id", "eval_score"])
-            for history in self.failed_proposals[param_id]
-        ]
+    # def add_failed_proposal(self):
+    #     """Save a copy of the current value of the parameter in the failed proposals."""
+    #     for param in self.params:
+    #         failed_proposal = HistoryPrompt(
+    #             id=param.id,
+    #             value=param.data,
+    #             eval_score=None,
+    #         )
+    #         self.failed_proposals[param.id].append(failed_proposal)
+    #         if len(self.failed_proposals[param.id]) > self.max_failed_proposals:
+    #             for _ in range(
+    #                 len(self.failed_proposals[param.id]) - self.max_failed_proposals
+    #             ):
+    #                 self.failed_proposals[param.id].pop()
+    #     # if param_id not in self.failed_proposals:
+    #     #     self.failed_proposals[param_id] = []
+    #     # failed_proposal = HistoryPrompt(
+    #     #     id=param_id,
+    #     #     value=value,
+    #     #     eval_score=None,
+    #     # )
+    #     # self.failed_proposals[param_id].append(failed_proposal)
+    #     # if len(self.failed_proposals[param_id]) > self.max_failed_proposals:
+    #     #     for _ in range(len(self.failed_proposals[param_id]) - self.max_failed_proposals):
+    #     #         self.failed_proposals[param_id].pop()
+
+    # def render_failed_proposals(self, param_id: str) -> List[str]:
+    #     if param_id not in self.failed_proposals:
+    #         return []
+    #     return [
+    #         history.to_yaml(exclude=["id", "eval_score"])
+    #         for history in self.failed_proposals[param_id]
+    #     ]
 
     # TODO: optimize with adalflow template for better readability
     def get_gradient_memory_text(self, param: Parameter) -> str:
@@ -341,7 +339,9 @@ def _get_user_prompt_kwargs(self, param: Parameter) -> Dict[str, str]:
 
         user_prompt_kwargs = {
             "variable_and_peers_info": variable_and_peer_info,
-            "variable_grad": param.get_gradient_and_context_text(),
+            "variable_grad": param.get_gradient_and_context_text(
+                skip_correct_sample=True
+            ),
             # constraints
             "constraint_text": self.constraint_text if self.do_constrained else None,
             # in-context examples
@@ -361,19 +361,19 @@ def _get_user_prompt_kwargs(self, param: Parameter) -> Dict[str, str]:
                 self.render_history(param.id) if self.max_past_history else None
             ),
             # failed proposals
-            "failed_proposals": (
-                self.render_failed_proposals(param.id)
-                if self.max_failed_proposals
-                else None
-            ),
+            # "failed_proposals": (
+            #     self.render_failed_proposals(param.id)
+            #     if self.max_failed_proposals
+            #     else None
+            # ),
         }
 
         return user_prompt_kwargs
 
     # TODO: better way to update the gradient memory
     def update_gradient_memory(self, param: Parameter):
         self.gradient_memory_dict[param.id].append(
-            {"value": param.get_gradient_and_context_text()}
+            {"value": param.get_gradient_and_context_text(skip_correct_sample=True)}
         )
 
     def zero_grad(self):