wip: Integration

Dockerfile

@@ -42,13 +42,6 @@ ARG MODEL=stablelm2:1.6b-zephyr
 ENV MODEL=${MODEL}
 RUN ollama serve & while ! curl http://localhost:11434; do sleep 1; done; ollama pull $MODEL
 
-# Build a language model
-# ARG MODEL=discolm
-# ENV MODEL=${MODEL}
-# WORKDIR /tmp/model
-# COPY --chmod=644 Modelfile Modelfile
-# RUN curl --location https://huggingface.co/TheBloke/DiscoLM_German_7b_v1-GGUF/resolve/main/discolm_german_7b_v1.Q5_K_S.gguf?download=true --output discolm_german_7b_v1.Q5_K_S.gguf; ollama serve & while ! curl http://localhost:11434; do sleep 1; done; ollama create ${MODEL} -f Modelfile && rm -rf /tmp/model
-
 
 # Setup the custom API and frontend
 WORKDIR /workspace

README.md

@@ -9,9 +9,14 @@
 To build and run the container locally with hot reload on python files do:
 ```
 DOCKER_BUILDKIT=1 docker build . -t gbnc
-docker run  -v "$(pwd)/gswikichat":/workspace/gswikichat \
-            -p 8000:8000 --rm --name gbnc -it gbnc \
-            -e HUGGING_FACE_HUB_TOKEN=$HUGGING_FACE_HUB_TOKEN
+docker run  \
+  -v "$(pwd)/gswikichat":/workspace/gswikichat \
+  -v "$(pwd)/cache":/root/.cache \
+  -e HUGGING_FACE_HUB_TOKEN=$HUGGING_FACE_HUB_TOKEN
+  -p 8000:8000 \
+  --rm -it \
+  --name gbnc \
+  gbnc
 ```
 Point your browser to http://localhost:8000/ and use the frontend.
 
@@ -44,3 +49,52 @@ A [FastAPI](https://fastapi.tiangolo.com/) server is running in the container. I
 ### Frontend
 
 A minimal frontend lets the user input a question and renders the response from the system.
+
+## Sentence Transformers Statistics
+
+```
+basic_transformer_models = [
+    "all-MiniLM-L6-v2",
+    "xlm-clm-ende-1024",
+    "xlm-mlm-ende-1024",
+    "bert-base-german-cased",
+    "bert-base-german-dbmdz-cased",
+    "bert-base-german-dbmdz-uncased",
+    "distilbert-base-german-cased",
+    "xlm-roberta-large-finetuned-conll03-german",
+    "deutsche-telekom/gbert-large-paraphrase-cosine"
+]
+
+https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2
+sentence_transformer_model = "all-MiniLM-L6-v2"
+3 minutes to batch 82
+
+https://huggingface.co/deutsche-telekom/gbert-large-paraphrase-cosine
+sentence_transformer_model = 'deutsche-telekom/gbert-large-paraphrase-cosine'
+76 minutes to batch 82
+
+https://huggingface.co/jinaai/jina-embeddings-v2-base-de
+sentence_transformer_model = 'jinaai/jina-embeddings-v2-base-de'
+Cannot find or load the embedding model
+Unknown minutes to batch 82
+
+https://huggingface.co/aari1995/German_Semantic_STS_V2
+sentence_transformer_model = 'aari1995/German_Semantic_STS_V2'
+75 minutes to batch 82
+
+https://huggingface.co/Sahajtomar/German-semantic
+sentence_transformer_model = 'Sahajtomar/German-semantic'
+72 minutes to batch 82
+
+https://huggingface.co/svalabs/german-gpl-adapted-covid
+ntence_transformer_model = 'svalabs/german-gpl-adapted-covid'
+2 minutes to batch 82
+
+https://huggingface.co/PM-AI/bi-encoder_msmarco_bert-base_german
+sentence_transformer_model = 'PM-AI/bi-encoder_msmarco_bert-base_german'
+14 minutes to batch 82
+
+https://huggingface.co/JoBeer/german-semantic-base
+sentence_transformer_model = 'JoBeer/german-semantic-base'
+22 minutes to batch 82
+```

frontend/src/components/field/FieldAnswer.vue

@@ -12,7 +12,7 @@
     <div v-else>
       <div v-if="response && response.sources">
         <div v-for="s in response.sources" :key="s.id">
-          <div v-if="s.score > 2" class="mb-2">
+          <div v-if="s.score > 0" class="mb-2">
             <details
               class="text-sm cursor-pointer text-light-distinct-text dark:text-dark-distinct-text"
             >

gswikichat/__init__.py

@@ -1,2 +1 @@
 from .api import *
-# from .haystack2beta_tutorial_InMemoryEmbeddingRetriever import *

gswikichat/api.py

@@ -2,7 +2,6 @@
 from fastapi.staticfiles import StaticFiles
 from fastapi import FastAPI
 
-# from .rag import rag_pipeline
 from .rag import embedder, retriever, prompt_builder, llm, answer_builder
 from haystack import Document
 
@@ -22,50 +21,49 @@ async def root():
 
 @app.get("/api")
 async def api(q):
+    print("query: ", q)
 
-    embedder, retriever, prompt_builder, llm, answer_builder
-
-    # query = "How many languages are there?"
     query = Document(content=q)
 
-    result = embedder.run([query])
+    queryEmbedded = embedder.run([query])
+    queryEmbedding = queryEmbedded['documents'][0].embedding
 
-    results = retriever.run(
-        query_embedding=list(result['documents'][0].embedding),
+    retrieverResults = retriever.run(
+        query_embedding=list(queryEmbedding),
         filters=None,
-        top_k=None,
+        top_k=3,
         scale_score=None,
         return_embedding=None
     )
-    # .run(
-    #     result['documents'][0].embedding
-    # )
 
-    prompt = prompt_builder.run(documents=results['documents'])['prompt']
+    print("retriever results:")
+    for retrieverResult in retrieverResults:
+        print(retrieverResult)
 
-    response = llm.run(prompt=prompt, generation_kwargs=None)
-    # reply = response['replies'][0]
+    promptBuild = prompt_builder.run(question=q, documents=retrieverResults['documents'])
+    prompt = promptBuild['prompt']
+
+    print("prompt: ", prompt)
 
-    # rag_pipeline.connect("llm.replies", "answer_builder.replies")
-    # rag_pipeline.connect("llm.metadata", "answer_builder.meta")
-    # rag_pipeline.connect("retriever", "answer_builder.documents")
+    response = llm.run(prompt=prompt, generation_kwargs=None)
 
-    results = answer_builder.run(
+    answerBuild = answer_builder.run(
         query=q,
         replies=response['replies'],
         meta=response['meta'],
-        documents=results['documents'],
+        documents=retrieverResults['documents'],
         pattern=None,
         reference_pattern=None
     )
+    print("answerBuild", answerBuild)
+
+    answer = answerBuild['answers'][0]
+
+    sources= [{ "src": d.meta['src'], "content": d.content, "score": d.score } for d in answer.documents]
 
-    answer = results['answers'][0]
+    print("answer", answer)
 
     return {
         "answer": answer.data,
-        "sources": [{
-            "src": d.meta['src'],
-            "content": d.content,
-            "score": d.score
-        } for d in answer.documents]
+        "sources": sources
     }

gswikichat/vector_store_interface.py

@@ -1,24 +1,25 @@
 import os
 import json
 
-# from sentence_transformers import SentenceTransformer
 from tqdm import tqdm
 
 from haystack import Document  # , Pipeline
 from haystack.components.embedders import SentenceTransformersDocumentEmbedder
-# from haystack.components.embedders import SentenceTransformersTextEmbedder
 from haystack.document_stores.in_memory import InMemoryDocumentStore
-# from haystack.components.retrievers.in_memory import InMemoryBM25Retriever
 from haystack.components.retrievers.in_memory import InMemoryEmbeddingRetriever
-# from haystack.components.writers import DocumentWriter
 from haystack.document_stores.types.policy import DuplicatePolicy
+from haystack.components.preprocessors import DocumentSplitter
+from haystack.components.preprocessors import DocumentCleaner
 
 HUGGING_FACE_HUB_TOKEN = os.environ.get('HUGGING_FACE_HUB_TOKEN')
+EMBEDDING_CACHE_FILE = '/tmp/gbnc_embeddings.json'
+
 top_k = 5
 input_documents = []
 
 json_dir = 'json_input'
-json_fname = 'excellent-articles_10_paragraphs.json'
+json_fname = 'excellent-articles_10.json'
+
 json_fpath = os.path.join(json_dir, json_fname)
 
 if os.path.isfile(json_fpath):
@@ -30,11 +31,11 @@
         for k, v in tqdm(json_obj.items()):
             print(f"Loading {k}")
             input_documents.append(Document(content=v, meta={"src": k}))
+
     elif isinstance(json_obj, list):
         for obj_ in tqdm(json_obj):
             url = obj_['meta']
             content = obj_['content']
-
             input_documents.append(
                 Document(
                     content=content,
@@ -57,112 +58,55 @@
         ),
     ]
 
-# Write documents to InMemoryDocumentStore
+# cleaner = DocumentCleaner(
+#         remove_empty_lines=True,
+#         remove_extra_whitespaces=True,
+#         remove_repeated_substrings=False)
+# input_documents = cleaner.run(input_documents)['documents']
+
+splitter = DocumentSplitter(split_by="sentence", split_length=20, split_overlap=0)
+input_documents = splitter.run(input_documents)['documents']
 
 document_store = InMemoryDocumentStore(
     embedding_similarity_function="cosine",
     # embedding_dim=768,
     # duplicate_documents="overwrite"
 )
-# document_store.write_documents(input_documents)
-
-# TODO Introduce Jina.AI from HuggingFace. Establish env-variable for trust_...
-
-# basic_transformer_models = [
-#     "all-MiniLM-L6-v2",
-#     "xlm-clm-ende-1024",
-#     "xlm-mlm-ende-1024",
-#     "bert-base-german-cased",
-#     "bert-base-german-dbmdz-cased",
-#     "bert-base-german-dbmdz-uncased",
-#     "distilbert-base-german-cased",
-#     "xlm-roberta-large-finetuned-conll03-german",
-#     "deutsche-telekom/gbert-large-paraphrase-cosine"
-# ]
-
-# https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2
-# sentence_transformer_model = "all-MiniLM-L6-v2"
-# 3 minutes to batch 82
-
-# https://huggingface.co/deutsche-telekom/gbert-large-paraphrase-cosine
-# sentence_transformer_model = 'deutsche-telekom/gbert-large-paraphrase-cosine'
-# 76 minutes to batch 82
-
-# https://huggingface.co/jinaai/jina-embeddings-v2-base-de
-# sentence_transformer_model = 'jinaai/jina-embeddings-v2-base-de'
-# Cannot find or load the embedding model
-# Unknown minutes to batch 82
-
-# https://huggingface.co/aari1995/German_Semantic_STS_V2
-# sentence_transformer_model = 'aari1995/German_Semantic_STS_V2'
-# 75 minutes to batch 82
-
-# https://huggingface.co/Sahajtomar/German-semantic
-# sentence_transformer_model = 'Sahajtomar/German-semantic'
-# 72 minutes to batch 82
 
 # https://huggingface.co/svalabs/german-gpl-adapted-covid
 sentence_transformer_model = 'svalabs/german-gpl-adapted-covid'
-# 2 minutes to batch 82
-
-# https://huggingface.co/PM-AI/bi-encoder_msmarco_bert-base_german
-# sentence_transformer_model = 'PM-AI/bi-encoder_msmarco_bert-base_german'
-# 14 minutes to batch 82
-
-# https://huggingface.co/JoBeer/german-semantic-base
-# sentence_transformer_model = 'JoBeer/german-semantic-base'
-# 22 minutes to batch 82
-
-print(f'Sentence Transformer Name:{sentence_transformer_model}')
+print(f'Sentence Transformer Name: {sentence_transformer_model}')
 
 embedder = SentenceTransformersDocumentEmbedder(
     model=sentence_transformer_model,
-    # model="T-Systems-onsite/german-roberta-sentence-transformer-v2",
-    # model="jinaai/jina-embeddings-v2-base-de",
-    # token=HUGGING_FACE_HUB_TOKEN
 )
-
-# hg_embedder = SentenceTransformer(
-#     "jinaai/jina-embeddings-v2-base-de",
-#     token=HUGGING_FACE_HUB_TOKEN
-# )
-
 embedder.warm_up()
 
-documents_with_embeddings = embedder.run(input_documents)
-# documents_with_embeddings = embedder.encode(input_documents)
-
 
-# print('\n\n')
-# # print(documents_with_embeddings['documents'])
-# print(type(documents_with_embeddings['documents']))
-# print(len(documents_with_embeddings['documents']))
-# print(dir(documents_with_embeddings['documents'][0]))
-# print('\n\n')
-# print(type(embedder.model))
-# print('\n\n')
-# # print(dir(hg_embedder))
-
-
-document_store.write_documents(
-    documents=documents_with_embeddings['documents'],
-    policy=DuplicatePolicy.OVERWRITE
-)
+# if os.path.isfile(EMBEDDING_CACHE_FILE):
+#     print("[INFO] Loading embeddings from cache")
+#
+#     with open(EMBEDDING_CACHE_FILE, 'r') as f:
+#         documentsDict = json.load(f)
+#         document_store.write_documents(
+#             documents=[Document.from_dict(d) for d in documentsDict],
+#             policy=DuplicatePolicy.OVERWRITE
+#         )
+#
+# else:
+if True:
+    embedded = embedder.run(input_documents)
+    document_store.write_documents(
+        documents=embedded['documents'],
+        policy=DuplicatePolicy.OVERWRITE
+    )
+
+    with open(EMBEDDING_CACHE_FILE, 'w') as f:
+        documentsDict = [Document.to_dict(d) for d in embedded['documents']]
+        json.dump(documentsDict, f)
 
 retriever = InMemoryEmbeddingRetriever(
-    # embedding_model="sentence-transformers/all-MiniLM-L6-v2",
     document_store=document_store,
     top_k=top_k
 )
 
-# writer = DocumentWriter(document_store=document_store)
-
-# indexing_pipeline = Pipeline()
-# indexing_pipeline.add_component("embedder", embedder)
-# indexing_pipeline.add_component("writer", writer)
-# indexing_pipeline.connect("embedder", "writer")
-# indexing_pipeline.run(
-#     {
-#         "embedder": {"documents": input_documents}
-#     }
-# )

requirements.txt

@@ -32,7 +32,7 @@ python-dotenv==1.0.1
 pytz==2023.3.post1
 PyYAML==6.0.1
 requests==2.31.0
-sentence-transformers>=2.2.0
+sentence-transformers==2.3.1
 six==1.16.0
 sniffio==1.3.0
 starlette==0.35.1