feature!: knowledge graph flexibility (#2030)

The current implementations of the synthesizers (in particular,
`MultiHopAbstractQuerySynthesizer` and
`MultiHopSpecificQuerySynthesizer`) have hard-coded values for the node-
and relationship- properties/types they look for when generating
scenarios and samples. This works when using the default knowledge graph
transforms, but may cause unexpected behavior (or force users to use the
expected hardcoded values when they do not actually correspond to the
reality of property lineage) when using customized the knowledge graph
transformations.

This PR allows flexibility in defining the node-property,
relationship-property, and relationship-type expected by the
*QuerySynthesizers, while retaining the current behavior as default

The bulk of this PR is nonbreaking, but there may be a breaking typo fix
in the `SingleHopSpecificQuerySynthesizer` default name.

- feat: allow user-defined node- and relationship- properties in
MultiHopAbstractQuerySynthesizer
- Users should be able to set the relationship property to use for
identifying clusters and the node property used for identifying abstract
concepts.
  - This will not change default behavior, but allows users to override.
- feat: allow user-defined node- and relationship- properties in
MultiHopSpecificQuerySynthesizer
- Users should be able to set the relationship type to use for
identifying clusters and the relationship property used for identifying
overlapping concepts within the triple.
  - This will not change default behavior, but allows users to override.
- fix!: typo in SingleHopSpecificQuerySynthesizer name
(`single_hop_specifc_query_synthesizer` ->
`single_hop_specific_query_synthesizer`)
- docs: minor docstring updates to SingleHopSpecificQuerySynthesizer

Author: ahgraber

src/ragas/testset/synthesizers/multi_hop/abstract.py

@@ -31,21 +31,19 @@
 
 @dataclass
 class MultiHopAbstractQuerySynthesizer(MultiHopQuerySynthesizer):
-    """
-    Synthesizes abstract multi-hop queries from given knowledge graph.
-
-    Attributes
-    ----------
-    """
+    """Synthesize abstract multi-hop queries from given knowledge graph."""
 
     name: str = "multi_hop_abstract_query_synthesizer"
+    relation_property: str = "summary_similarity"
+    abstract_property_name: str = "themes"
     concept_combination_prompt: PydanticPrompt = ConceptCombinationPrompt()
     theme_persona_matching_prompt: PydanticPrompt = ThemesPersonasMatchingPrompt()
 
     def get_node_clusters(self, knowledge_graph: KnowledgeGraph) -> t.List[t.Set[Node]]:
+        """Identify clusters of nodes based on the specified relationship condition."""
         node_clusters = knowledge_graph.find_indirect_clusters(
-            relationship_condition=lambda rel: (
-                True if rel.get_property("summary_similarity") else False
+            relationship_condition=lambda rel: bool(
+                rel.get_property(self.relation_property)
             ),
             depth_limit=3,
         )
@@ -60,7 +58,8 @@ async def _generate_scenarios(
         callbacks: Callbacks,
     ) -> t.List[MultiHopScenario]:
         """
-        Generates a list of scenarios on type MultiHopAbstractQuerySynthesizer
+        Generate a list of scenarios of type MultiHopScenario.
+
         Steps to generate scenarios:
         1. Find indirect clusters of nodes based on relationship condition
         2. Calculate the number of samples that should be created per cluster to get n samples in total
@@ -92,7 +91,9 @@ async def _generate_scenarios(
                     nodes.append(node)
 
             base_scenarios = []
-            node_themes = [node.properties.get("themes", []) for node in nodes]
+            node_themes = [
+                node.properties.get(self.abstract_property_name, []) for node in nodes
+            ]
             prompt_input = ConceptsList(
                 lists_of_concepts=node_themes, max_combinations=num_sample_per_cluster
             )
@@ -116,7 +117,7 @@ async def _generate_scenarios(
                 concept_combination.combinations,
                 personas=persona_list,
                 persona_item_mapping=persona_concepts.mapping,
-                property_name="themes",
+                property_name=self.abstract_property_name,
             )
             base_scenarios = self.sample_diverse_combinations(
                 base_scenarios, num_sample_per_cluster

src/ragas/testset/synthesizers/multi_hop/specific.py

@@ -2,6 +2,7 @@
 
 import logging
 import typing as t
+from collections.abc import Iterable
 from dataclasses import dataclass
 
 import numpy as np
@@ -27,27 +28,19 @@
 
 @dataclass
 class MultiHopSpecificQuerySynthesizer(MultiHopQuerySynthesizer):
-    """
-    Synthesizes overlap based queries by choosing specific chunks and generating a
-    keyphrase from them and then generating queries based on that.
-
-    Attributes
-    ----------
-    generate_query_prompt : PydanticPrompt
-        The prompt used for generating the query.
-    """
+    """Synthesize multi-hop queries based on a chunk cluster defined by entity overlap."""
 
     name: str = "multi_hop_specific_query_synthesizer"
-    relation_type: str = "entities_overlap"
     property_name: str = "entities"
+    relation_type: str = "entities_overlap"
+    relation_overlap_property: str = "overlapped_items"
     theme_persona_matching_prompt: PydanticPrompt = ThemesPersonasMatchingPrompt()
     generate_query_reference_prompt: PydanticPrompt = QueryAnswerGenerationPrompt()
 
     def get_node_clusters(self, knowledge_graph: KnowledgeGraph) -> t.List[t.Tuple]:
+        """Identify clusters of nodes based on the specified relationship condition."""
         node_clusters = knowledge_graph.find_two_nodes_single_rel(
-            relationship_condition=lambda rel: (
-                True if rel.type == self.relation_type else False
-            )
+            relationship_condition=lambda rel: rel.type == self.relation_type
         )
         logger.info("found %d clusters", len(node_clusters))
         return node_clusters
@@ -60,7 +53,8 @@ async def _generate_scenarios(
         callbacks: Callbacks,
     ) -> t.List[MultiHopScenario]:
         """
-        Generates a list of scenarios on type MultiHopSpecificQuerySynthesizer
+        Generate a list of scenarios of type MultiHopScenario.
+
         Steps to generate scenarios:
         1. Filter the knowledge graph to find cluster of nodes or defined relation type. Here entities_overlap
         2. Calculate the number of samples that should be created per cluster to get n samples in total
@@ -86,9 +80,18 @@ async def _generate_scenarios(
             if len(scenarios) < n:
                 node_a, node_b = triplet[0], triplet[-1]
                 overlapped_items = []
-                overlapped_items = triplet[1].properties["overlapped_items"]
+                overlapped_items = triplet[1].properties[self.relation_overlap_property]
                 if overlapped_items:
-                    themes = list(dict(overlapped_items).keys())
+                    if not all(
+                        isinstance(item, (str, Iterable)) for item in overlapped_items
+                    ):
+                        logger.debug("Overlapped items are not strings or iterables.")
+                        continue
+                    themes = (
+                        list(overlapped_items.keys())
+                        if isinstance(overlapped_items, dict)
+                        else overlapped_items
+                    )
                     prompt_input = ThemesPersonasInput(
                         themes=themes, personas=persona_list
                     )
@@ -97,10 +100,13 @@ async def _generate_scenarios(
                             data=prompt_input, llm=self.llm, callbacks=callbacks
                         )
                     )
-                    overlapped_items = [list(item) for item in overlapped_items]
+                    combinations = [
+                        [item] if isinstance(item, str) else list(item)
+                        for item in themes
+                    ]
                     base_scenarios = self.prepare_combinations(
                         [node_a, node_b],
-                        overlapped_items,
+                        combinations,
                         personas=persona_list,
                         persona_item_mapping=persona_concepts.mapping,
                         property_name=self.property_name,

src/ragas/testset/synthesizers/single_hop/specific.py

@@ -39,7 +39,7 @@ class SingleHopScenario(BaseScenario):
 
 @dataclass
 class SingleHopSpecificQuerySynthesizer(SingleHopQuerySynthesizer):
-    name: str = "single_hop_specifc_query_synthesizer"
+    name: str = "single_hop_specific_query_synthesizer"
     theme_persona_matching_prompt: PydanticPrompt = ThemesPersonasMatchingPrompt()
     property_name: str = "entities"
 

tests/unit/test_analytics.py

@@ -136,7 +136,7 @@ def test_testset_generation_tracking(monkeypatch):
     )
 
     assert testset_event_payload.model_dump()["evolution_names"] == [
-        "single_hop_specifc_query_synthesizer",
+        "single_hop_specific_query_synthesizer",
         "multi_hop_abstract_query_synthesizer",
         "multi_hop_specific_query_synthesizer",
     ]