RAG MD Python Code for BYOC and Streamlit application

Author: Gaurav141199

model-deployment/containers/rag_llama2/Inference MD/Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.10
+
+RUN mkdir /app
+
+WORKDIR /app
+
+COPY requirements.txt requirements.txt
+COPY main.py main.py
+COPY start.sh start.sh
+
+# Installing the server dependencies.
+RUN pip3 install -r requirements.txt
+
+EXPOSE 8080
+
+RUN chmod +x start.sh
+
+CMD ["./start.sh"]

model-deployment/containers/rag_llama2/Inference MD/main.py

@@ -0,0 +1,68 @@
+"""The main model serving HTTP server. Creates the following endpoints:
+
+  /predict (POST) - model prediction endpoint
+"""
+from fastapi import FastAPI, Body, Request, Response, status
+from fastapi.responses import HTMLResponse, JSONResponse
+import logging
+from langchain.embeddings import LlamaCppEmbeddings
+from langchain.vectorstores import Qdrant
+
+
+fast_app = FastAPI()
+model_path = "/opt/ds/model/deployed_model/7B/ggml-model-q4_0.bin"
+
+def load_model(model_folder_directory):
+    embedding = LlamaCppEmbeddings(model_path=model_folder_directory)
+    return embedding
+
+try:
+    logging.info("Loading the model")
+    embedding = load_model(model_path)
+except Exception as e:
+    print("Error: %s", e)
+
+url = "https://0ad84320-52a6-407d-9c82-375bf60e1fc6.us-east4-0.gcp.cloud.qdrant.io"
+api_key= "a675QyMVF8SxqY9wNAssu4dwuIpbHGuXj8aZVDPBKX22AJeBGCOhqw"
+
+
+qdrant = None
+text_count = 0
+
+@fast_app.get("/", response_class=HTMLResponse)
+def read_root():
+    return """
+        <h2>Hello! Welcome to the model serving api.</h2>
+        Check the <a href="/docs">api specs</a>.
+    """
+
+@fast_app.post("/predict")
+def model_predict(request: Request, response: Response, data=Body(None)):
+    global embedding, qdrant, text_count, url, api_key
+    text = data.decode("utf-8")
+    try:
+        if qdrant is None:
+            qdrant = Qdrant.from_texts(
+                text,
+                embedding,
+                url=url,
+                api_key=api_key,
+                collection_name="my_documents"
+            )
+        else:
+            qdrant.add_texts(text)
+        text_count += 1
+        result = "Sentence Added: Total sentences count is " + str(text_count)
+    except Exception as e:
+        result = "Error " + str(e)
+    return result
+
+'''
+Health GET endpoint returning the health status
+'''
+@fast_app.get("/health")
+def model_predict1(request: Request, response: Response):
+    return {"status":"success"}
+
+if __name__ == "__main__":
+    uvicorn.run("main:fast_app", port=8080,reload=True)

model-deployment/containers/rag_llama2/Inference MD/requirements.txt

@@ -0,0 +1,6 @@
+langchain
+llama-cpp-python
+requests
+uvicorn
+fastapi
+qdrant-client

model-deployment/containers/rag_llama2/Inference MD/start.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+uvicorn main:fast_app --port 8080 --host=0.0.0.0

model-deployment/containers/rag_llama2/README.md

@@ -0,0 +1 @@
+## TO ADD

model-deployment/containers/rag_llama2/ingestion MD/Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.10
+
+RUN mkdir /app
+
+WORKDIR /app
+
+COPY requirements.txt requirements.txt
+COPY main.py main.py
+COPY start.sh start.sh
+
+# Installing the server dependencies.
+RUN pip3 install -r requirements.txt
+
+EXPOSE 8080
+
+RUN chmod +x start.sh
+
+CMD ["./start.sh"]

model-deployment/containers/rag_llama2/ingestion MD/main.py

@@ -0,0 +1,100 @@
+"""The main model serving HTTP server. Creates the following endpoints:
+
+  /predict (POST) - model prediction endpoint
+"""
+from fastapi import FastAPI, Body, Request, Response, status
+from fastapi.responses import HTMLResponse, JSONResponse
+import logging
+from langchain.embeddings import LlamaCppEmbeddings
+from langchain.chains.question_answering import load_qa_chain
+from langchain.llms import LlamaCpp
+from langchain.prompts.prompt import PromptTemplate
+from langchain.vectorstores import Qdrant
+import qdrant_client
+
+fast_app = FastAPI()
+
+model_path = "/opt/ds/model/deployed_model/7B/ggml-model-q4_0.bin"
+
+def load_model(model_folder_directory):
+    embedding = LlamaCppEmbeddings(model_path=model_folder_directory,n_gpu_layers=15000)
+    return embedding
+
+try:
+    logging.info("Loading the model")
+    embeddings = load_model(model_path)
+except Exception as e: 
+    print("Error: %s", e)
+
+url = "QDRANT_URL"
+api_key= "API_KEY"
+
+template = """You are an assistant to the user, you are given some context below, please answer the query of the user with as detail as possible
+
+Context:\"""
+{context}
+\"""
+
+Question:\"
+{question}
+\"""
+
+Answer:"""
+
+
+
+client = qdrant_client.QdrantClient(
+    url,
+    api_key=api_key
+)
+
+qdrant = Qdrant(
+    client=client, collection_name="my_documents",
+    embeddings=embeddings
+)
+
+qa_prompt = PromptTemplate.from_template(template)
+
+llm = LlamaCpp(model_path=model_path,n_gpu_layers=15000, n_ctx=2048)
+# llm = LlamaCpp(model_path=model_path, n_ctx=2048)
+
+@fast_app.get("/", response_class=HTMLResponse)
+def read_root():
+    return """
+        <h2>Hello! Welcome to the model serving api.</h2>
+        Check the <a href="/docs">api specs</a>.
+    """
+
+@fast_app.post("/predict")
+def model_predict(request: Request, response: Response, data=Body(None)):
+    global llm, embeddings, qa_prompt, qdrant
+    print(data)
+    question = data.decode("utf-8")
+    print(question)
+    chain = load_qa_chain(llm, chain_type="stuff", prompt=qa_prompt)
+    print("OK")
+    if question =="Hi":
+        return "I am able to load the embedding"
+    if question == "Hello":
+        docs = qdrant.similarity_search(question)
+        return docs
+    try:
+        docs = qdrant.similarity_search(question)
+        print(docs)
+    except Exception as e:
+        print(e)
+        return e
+    print(question)
+    answer = chain({"input_documents": docs, "question": question,"context": docs}, return_only_outputs=True)['output_text']
+    return answer
+
+'''
+Health GET endpoint returning the health status
+'''
+@fast_app.get("/health")
+def model_predict1(request: Request, response: Response):
+    return {"status":"success"}
+
+if __name__ == "__main__":
+    uvicorn.run("main:fast_app", port=8080, reload=True)
+

model-deployment/containers/rag_llama2/ingestion MD/requirements.txt

@@ -0,0 +1,7 @@
+langchain==0.0.333
+llama-cpp-python==0.2.15
+oci==2.47.1
+requests==2.25.1
+uvicorn
+fastapi
+qdrant-client==1.6.9

model-deployment/containers/rag_llama2/ingestion MD/start.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+uvicorn main:fast_app --port 8080 --host=0.0.0.0

model-deployment/containers/rag_llama2/streamlit/app.py

@@ -0,0 +1,66 @@
+import streamlit as st
+from streamlit_chat import message
+import oci
+import time
+import os
+import oci
+import requests
+from oci.signer import Signer
+
+# token_file = os.path.expanduser("/Users/gagachau/.oci/sessions/OC1/token")
+# with open(token_file, 'r') as f:
+#         token = f.read()
+# private_key = oci.signer.load_private_key_from_file("/Users/gagachau/.oci/sessions/OC1/oci_api_key.pem")
+# signer = oci.auth.signers.SecurityTokenSigner(token, private_key)
+
+
+def generate_response(prompt):
+    # global signer
+    endpoint = "http://localhost:8080/predict"
+    headers = {"content-type": "application/text"}  # header goes here
+    # response = requests.post(endpoint, data=prompt, auth=signer, headers=headers)
+    response = requests.post(endpoint, data=prompt, headers=headers)
+    res = response.text
+    print(res)
+    res = res.replace('\n', '')
+    res = res.replace("\n", "")
+    res = res.replace('"', "")
+    res = res.replace("'", "")
+    res = res.replace('\\', "")
+    return res
+
+# Create the title and
+st.set_page_config(page_title="SQuAD Chatbot")
+
+# create the header and the line underneath it
+header_html = "<h1 style='text-align: center; margin-bottom: 1px;'>🤖 The SQuAD Chatbot 🤖</h1>"
+line_html = "<hr style='border: 2px solid green; margin-top: 1px; margin-bottom: 0px;'>"
+st.markdown(header_html, unsafe_allow_html=True)
+st.markdown(line_html, unsafe_allow_html=True)
+
+# create lists to store user queries and generated responses
+if "generated" not in st.session_state:
+    st.session_state["generated"] = []
+if "past" not in st.session_state:
+    st.session_state["past"] = []
+
+
+# create input field for user queries
+user_input = st.chat_input("How can I help?")
+
+# generate response when a user prompt is submitted
+if user_input:
+    output = generate_response(prompt=user_input)
+    print(output)
+    st.session_state.past.append(user_input)
+    st.session_state.generated.append(output)
+
+
+# show queries and responses in the user interface
+if st.session_state["generated"]:
+
+    for i in range(len(st.session_state["generated"])):
+        message(st.session_state["past"][i], is_user=True, key=str(i) + "_user")
+        message(st.session_state["generated"][i], key=str(i))
+
+