more graphs with dialogues generated

Author: Yuriy Peshkichev

dev_packages/chatsky_llm_autoconfig/chatsky_llm_autoconfig/algorithms/cycle_graph_generation_pipeline.py

@@ -236,10 +236,14 @@ def generate_and_validate(self, topic: str) -> PipelineResult:
             if not graph.edges_match_nodes():
                 return GenerationError(
                     error_type=ErrorType.INVALID_GRAPH_STRUCTURE,
-                    message="Genrated graph is wrong: edges don't match nodes"
+                    message="Generated graph is wrong: edges don't match nodes"
                 )
             graph = graph.remove_duplicated_nodes()
-
+            if graph is None:
+                return GenerationError(
+                    error_type=ErrorType.INVALID_GRAPH_STRUCTURE,
+                    message="Generated graph is wrong: utterances in nodes doubled"
+                )                
             # 2. Validate cycles
             cycle_validation = self.validate_graph_cycle_requirement(graph, self.min_cycles)
             if not cycle_validation["meets_requirements"]:
@@ -281,10 +285,22 @@ def generate_and_validate(self, topic: str) -> PipelineResult:
                     error_type=ErrorType.INVALID_GRAPH_STRUCTURE,
                     message=f"Found {len(invalid_transitions)} invalid transitions after {transition_validation['validation_details']['attempts_made']} fix attempts"
                 )
+            
+            graph = transition_validation["graph"]
+            print("Sampling dialogues...")
+            sampled_dialogues = self.dialogue_sampler.invoke(graph, 15)
+            print(f"Sampled {len(sampled_dialogues)} dialogues")
+            for s in sampled_dialogues:
+                print(s)
+            if all_utterances_present(graph, sampled_dialogues) != True:
+                return GenerationError(
+                    error_type=ErrorType.SAMPLING_FAILED,
+                    message="Failed to sample valid dialogues - not all utterances are present"
+                )
 
             # All validations passed - return successful result
             return GraphGenerationResult(
-                graph=transition_validation["graph"].graph_dict,
+                graph=graph.graph_dict,
                 topic=topic,
                 dialogues=sampled_dialogues
             )

dev_packages/chatsky_llm_autoconfig/chatsky_llm_autoconfig/algorithms/dialogue_generation.py

@@ -14,9 +14,6 @@
 
 env_settings = EnvSettings()
 
-def list_in(a, b):
-    return any(map(lambda x: b[x:x + len(a)] == a, range(len(b) - len(a) + 1)))
-
 def len_in(a,b):
     return sum([b[x:x + len(a)] == a for x in range(len(b) - len(a) + 1)])
 
@@ -31,17 +28,17 @@ def mix_ends(graph: BaseGraph, ends: list[int], cycles: list[int]):
                 visited.append(c)
     return [e for e in cycles if e not in visited] + ends
 
-def all_paths(graph: BaseGraph, start: int, visited: list, repeats: int):
-    global visited_list
+# def all_paths(graph: BaseGraph, start: int, visited: list, repeats: int):
+#     global visited_list
 
-    # if len(visited) < 1 or len_in([visited[-1],start],visited) < repeats:
-    if len(visited) < repeats or not list_in(visited[-repeats:]+[start],visited):
-        # print("LEN: ", len(visited))
-        visited.append(start)
-        for edge in graph.edge_by_source(start):
-            # print("TARGET: ", edge['target'])
-            all_paths(graph, edge['target'], visited.copy(), repeats)
-    visited_list.append(visited)
+#     # if len(visited) < 1 or len_in([visited[-1],start],visited) < repeats:
+#     if len(visited) < repeats or not list_in(visited[-repeats:]+[start],visited):
+#         # print("LEN: ", len(visited))
+#         visited.append(start)
+#         for edge in graph.edge_by_source(start):
+#             # print("TARGET: ", edge['target'])
+#             all_paths(graph, edge['target'], visited.copy(), repeats)
+#     visited_list.append(visited)
 
 def all_combinations(path: list, start: dict, next: int, visited: list):
     global visited_list
@@ -90,10 +87,12 @@ def get_utts(seq: list[list[dict]]) -> set[tuple[str]]:
     return set(res)
 
 def dialogue_edges(seq: list[list[dict]]) -> set[tuple[str]]:
+
     res = []
     for dialogue in seq:
          assist_texts = [d['text'] for d in dialogue if d['participant']=='assistant']
-         res.extend([(a1,a2) for a1,a2 in zip(assist_texts[:-1],assist_texts[1:])])
+         user_texts = [d['text'] for d in dialogue if d['participant']=='user']         
+         res.extend([(a1,u,a2) for a1,u,a2 in zip(assist_texts[:-1],user_texts[:len(assist_texts)-1],assist_texts[1:])])
     # print("DIA: ", set(res))
     return set(res)
 
@@ -110,7 +109,7 @@ def get_dialogues(graph: BaseGraph, repeats: int, ends: list[int]) -> list[Dialo
     starts = [n for n in graph.graph_dict.get("nodes") if n["is_start"]]
     for s in starts:
         visited_list = [[]]
-        all_paths(graph, s['id'], [], repeats)
+        graph.all_paths(s['id'], [], repeats)
         paths.extend(visited_list)
 
     paths.sort()
@@ -148,16 +147,16 @@ def get_dialogues(graph: BaseGraph, repeats: int, ends: list[int]) -> list[Dialo
         dialogue = [el[1:] for el in visited_list if len(el)==len(f)+1]
         dialogues.extend(dialogue)
 
-    # for d in dialogues:
-    #    print("DGS: ", d)
-    # print("\n")
+    for d in dialogues:
+       print("DGS: ", d)
+    print("\n")
     final = list(k for k,_ in itertools.groupby(dialogues))
     # print("BEFORE: ", len(final))
     final = remove_duplicated_utts(final)
     # print("AFTER: ", len(final))
-    # for f in final:
-    #     print("FINAL: ", f)
-    # print("\n")
+    for f in final:
+        print("FINAL: ", f)
+    print("\n")
     result = [Dialogue().from_list(seq) for seq in final]
     return result
 
@@ -274,9 +273,6 @@ async def ainvoke(self, *args, **kwargs):
 
 # @AlgorithmRegistry.register(input_type=BaseGraph, output_type=Dialogue)
 class RecursiveDialogueSampler(DialogueGenerator):
-    def _list_in(self, a: list, b: list) -> bool:
-        """Check if sequence a exists within sequence b."""
-        return any(map(lambda x: b[x : x + len(a)] == a, range(len(b) - len(a) + 1)))
 
     def invoke(self, graph: BaseGraph, upper_limit: int) -> list[Dialogue]:
         global visited_list

dev_packages/chatsky_llm_autoconfig/chatsky_llm_autoconfig/algorithms/three_stages_graph_generation.py

@@ -1,19 +1,22 @@
+import pandas as pd
 from langchain.prompts import PromptTemplate
 from langchain_openai  import ChatOpenAI
 from langchain.output_parsers import PydanticOutputParser
 from langchain_community.embeddings import HuggingFaceEmbeddings
 
 from chatsky_llm_autoconfig.algorithms.base import GraphGenerator
-from chatsky_llm_autoconfig.graph import BaseGraph, Graph
+from chatsky_llm_autoconfig.metrics.automatic_metrics import (
+    is_same_structure,
+    compare_graphs
+)
 from chatsky_llm_autoconfig.metrics.embedder import nodes2groups
 from chatsky_llm_autoconfig.schemas import DialogueGraph
 from chatsky_llm_autoconfig.dialogue import Dialogue
 from chatsky_llm_autoconfig.autometrics.registry import AlgorithmRegistry
 from chatsky_llm_autoconfig.utils import call_llm_api, nodes2graph, dialogues2list
 from chatsky_llm_autoconfig.settings import EnvSettings
-
 from chatsky_llm_autoconfig.missing_edges_prompt import three_1, three_2
-
+from chatsky_llm_autoconfig.graph import BaseGraph, Graph
 env_settings = EnvSettings()
 
 embeddings = HuggingFaceEmbeddings(model_name=env_settings.EMBEDDER_MODEL, model_kwargs={"device": env_settings.EMBEDDER_DEVICE})
@@ -89,3 +92,16 @@ def invoke(self, dialogues: list[Dialogue] = None, graph: DialogueGraph = None,
 
     async def ainvoke(self, *args, **kwargs):
         return self.invoke(*args, **kwargs)
+    
+    async def evaluate(self, dialogues, target_graph, report_type = "dict"):
+        graph = self.invoke(dialogues)
+        report = {
+            "is_same_structure": is_same_structure(graph, target_graph),
+            "graph_match": compare_graphs(graph, target_graph),
+        }
+        if report_type == "dataframe":
+            report = pd.DataFrame(report, index=[0])
+        elif report_type == "dict":
+            return report
+        else:
+            raise ValueError(f"Invalid report_type: {report_type}")