Discussion on MultiQueryEmbeddingRetriever component

[davidsbatista]

This is the discussion board for the experimental MultiQueryEmbeddingRetriever component.

Usage example

  from haystack import Document
  from haystack.document_stores.in_memory import InMemoryDocumentStore
  from haystack.document_stores.types import DuplicatePolicy
  from haystack.components.embedders import SentenceTransformersTextEmbedder
  from haystack.components.embedders import SentenceTransformersDocumentEmbedder
  from haystack.components.retrievers import InMemoryEmbeddingRetriever
  from haystack.components.writers import DocumentWriter
  from haystack_experimental.components.retrievers import MultiQueryEmbeddingRetriever

  documents = [
      Document(content="Renewable energy is energy that is collected from renewable resources."),
      Document(content="Solar energy is a type of green energy that is harnessed from the sun."),
      Document(content="Wind energy is another type of green energy that is generated by wind turbines."),
      Document(content="Geothermal energy is heat that comes from the sub-surface of the earth."),
      Document(content="Biomass energy is produced from organic materials, such as plant and animal waste."),
      Document(content="Fossil fuels, such as coal, oil, and natural gas, are non-renewable energy sources."),
  ]

  # Populate the document store
  doc_store = InMemoryDocumentStore()
  doc_embedder = SentenceTransformersDocumentEmbedder(model="sentence-transformers/all-MiniLM-L6-v2")
  doc_embedder.warm_up()
  doc_writer = DocumentWriter(document_store=doc_store, policy=DuplicatePolicy.SKIP)
  documents = doc_embedder.run(documents)["documents"]
  doc_writer.run(documents=documents)

  # Run the multi-query retriever
  in_memory_retriever = InMemoryEmbeddingRetriever(document_store=doc_store, top_k=1)
  query_embedder = SentenceTransformersTextEmbedder(model="sentence-transformers/all-MiniLM-L6-v2")

  multi_query_retriever = MultiQueryEmbeddingRetriever(
      retriever=in_memory_retriever,
      query_embedder=query_embedder,
      max_workers=3
  )

  queries = ["Geothermal energy", "natural gas", "turbines"]
  result = multi_query_retriever.run(queries=queries)
  for doc in result["documents"]:
      print(f"Content: {doc.content}, Score: {doc.score}")
  >> Content: Geothermal energy is heat that comes from the sub-surface of the earth., Score: 0.8509603046266574
  >> Content: Renewable energy is energy that is collected from renewable resources., Score: 0.42763211298893034
  >> Content: Solar energy is a type of green energy that is harnessed from the sun., Score: 0.40077417016494354
  >> Content: Fossil fuels, such as coal, oil, and natural gas, are non-renewable energy sources., Score: 0.3774863680995796
  >> Content: Wind energy is another type of green energy that is generated by wind turbines., Score: 0.3091423972562246
  >> Content: Biomass energy is produced from organic materials, such as plant and animal waste., Score: 0.25173074243668087