test: improve text2sparql indexing

Author: vemonet

compose.text2sparql.yml

@@ -77,5 +77,6 @@ services:
       - DBPEDIA_URL=http://virtuoso-dbpedia:8890/sparql
       - CORPORATE_URL=http://virtuoso-corporate:8890/sparql
       - VECTORDB_URL=http://vectordb:6334/
+      - BENCH_MODEL=openrouter/openai/gpt-oss-120b
     entrypoint: uv run
     command: uvicorn tests.text2sparql.api:app --host 0.0.0.0 --port 8765

tests/text2sparql/api.py

@@ -29,24 +29,20 @@ def get_dataset_id_from_iri(dataset_iri: str) -> str:
     "https://text2sparql.aksw.org/2025/corporate/": os.getenv("CORPORATE_URL", "http://virtuoso-corporate:8890/sparql"),
 }
 
-MODEL = "openrouter/openai/gpt-oss-120b"
-DOCKER_ENDPOINT_URL = "http://text2sparql-virtuoso:8890/sparql/"
-DOCKER_VECTORDB_URL = "http://vectordb:6334"
-ENDPOINT_URL = "http://localhost:8890/sparql/"
-
-# def normalize_docker_to_localhost(url: str) -> str:
-
+MODEL = os.getenv("BENCH_MODEL", "openrouter/openai/gpt-oss-120b")
 
 SCHEMAS = {}
 for dataset_iri in DATASETS_ENDPOINTS.keys():
-    with open(
-        os.path.join("/", "data", f"{get_dataset_id_from_iri(dataset_iri)}_schema.json"),
-        encoding="utf-8",
-    ) as f:
-        SCHEMAS[dataset_iri] = json.load(f)
-    # SCHEMAS[dataset][DOCKER_ENDPOINT_URL] = SCHEMAS[dataset].pop(ENDPOINT_URL)
-    # docker_url =
-    SCHEMAS[dataset_iri][DOCKER_ENDPOINT_URL] = SCHEMAS[dataset_iri].pop(ENDPOINT_URL)
+    try:
+        with open(
+            os.path.join("/", "data", f"{get_dataset_id_from_iri(dataset_iri)}_schema.json"),
+            encoding="utf-8",
+        ) as f:
+            SCHEMAS[dataset_iri] = json.load(f)
+    except FileNotFoundError:
+        print(
+            f"Schema file for dataset {dataset_iri} not found. Please run the indexing script to generate the schema files."
+        )
 
 RAG_PROMPT = """
 
@@ -103,10 +99,11 @@ def get_dataset_id_from_iri(dataset_iri: str) -> str:
 async def get_answer(question: str, dataset: str):
     if dataset not in DATASETS_ENDPOINTS:
         raise fastapi.HTTPException(404, "Unknown dataset ...")
+    endpoint_url = DATASETS_ENDPOINTS[dataset]
     # Retrieve relevant queries
     question_embeddings = next(iter(embedding_model.embed([question])))
     retrieved_queries = vectordb.query_points(
-        collection_name=f"text2sparql-{dataset.split('/')[-2]}",
+        collection_name=f"text2sparql-{get_dataset_id_from_iri(dataset)}",
         query=question_embeddings,
         limit=settings.default_number_of_retrieved_docs,
         query_filter=Filter(
@@ -121,7 +118,7 @@ async def get_answer(question: str, dataset: str):
 
     # Retrieve relevant classes
     retrieved_classes = vectordb.query_points(
-        collection_name=f"text2sparql-{dataset.split('/')[-2]}",
+        collection_name=f"text2sparql-{get_dataset_id_from_iri(dataset)}",
         query=question_embeddings,
         limit=settings.default_number_of_retrieved_docs,
         query_filter=Filter(
@@ -163,14 +160,13 @@ async def get_answer(question: str, dataset: str):
             chat_resp_md = response.model_dump()["content"]
             generated_sparqls = extract_sparql_queries(chat_resp_md)
             generated_sparql = generated_sparqls[-1]["query"].strip()
-            generated_sparql = generated_sparql.replace(ENDPOINT_URL, DOCKER_ENDPOINT_URL)
             # print(f"Generated SPARQL query: {generated_sparql}")
             # print(f"Response message: {resp_msg}")
         except Exception:
             resp_msg += "## No SPARQL query could be extracted from the model response. Please provide a valid SPARQL query based on the provided information and try again.\n"
         if generated_sparql != "":
             try:
-                res = query_sparql(generated_sparql, DOCKER_ENDPOINT_URL)
+                res = query_sparql(generated_sparql, endpoint_url)
                 if res.get("results", {}).get("bindings"):
                     # Successfully generated a query with results
                     if num_of_tries > 0:
@@ -183,7 +179,7 @@ async def get_answer(question: str, dataset: str):
 
             except Exception as e:
                 validation_output = validate_sparql(
-                    query=generated_sparql, endpoint_url=DOCKER_ENDPOINT_URL, endpoints_void_dict=SCHEMAS[dataset]
+                    query=generated_sparql, endpoint_url=endpoint_url, endpoints_void_dict=SCHEMAS[dataset]
                 )
                 if validation_output["errors"]:
                     error_str = "- " + "\n- ".join(validation_output["errors"])

tests/text2sparql/index.py

@@ -15,17 +15,16 @@
 
 def init_vectordb(
     endpoint_url: str,
-    graph: str,
+    dataset_iri: str,
     limit_schema: dict[str, float],
     max_workers: int,
     force_recompute: bool,
-    schema_path: str,
 ) -> None:
     """Initialize the vectordb with example queries and schema information from the SPARQL endpoints"""
     docs: list[Document] = []
 
     # Index example queries
-    examples = ["Generated-CK"] if "corporate" in graph else ["QALD-9+", "LC-QuAD"]
+    examples = ["Generated-CK"] if "corporate" in dataset_iri else ["QALD-9+", "LC-QuAD"]
 
     queries = pd.read_csv(QUERIES_FILE)
     queries = queries[queries["dataset"].isin(examples)].reset_index(drop=True)
@@ -51,11 +50,10 @@ def init_vectordb(
     start_time = time.time()
     schema = EndpointSchema(
         endpoint_url=endpoint_url,
-        # graph=graph,
         limit_schema=limit_schema,
         max_workers=max_workers,
         force_recompute=force_recompute,
-        schema_path=schema_path,
+        schema_path=os.path.join("data", f"{get_dataset_id_from_iri(dataset_iri)}_schema.json"),
     ).get_schema()
 
     docs += schema.apply(
@@ -84,7 +82,7 @@ def init_vectordb(
 
     embeddings = list(embedding_model.embed([d.page_content for d in docs]))
 
-    collection_name = f"text2sparql-{graph.split('/')[-2]}"
+    collection_name = f"text2sparql-{get_dataset_id_from_iri(dataset_iri)}"
     # Ensure collection exists before upserting
     if not qdrant_client.collection_exists(collection_name):
         qdrant_client.create_collection(
@@ -119,13 +117,12 @@ def init_vectordb(
     # Init vectordb for the specified dataset
     init_vectordb(
         endpoint_url=DATASETS_ENDPOINTS[dataset_iri],
-        graph=dataset_iri,
+        dataset_iri=dataset_iri,
         limit_schema={
             "top_classes_percentile": 0,
             "top_n_predicates": 20,
             "top_n_ranges": 1,
         },
         max_workers=4,
         force_recompute=True,
-        schema_path=os.path.join("data", f"{get_dataset_id_from_iri(dataset_iri)}_schema.json"),
     )