This is my pipeline an and I am returning the links in the string but I want the urls to looks like when using the integrate knowledge base.

[egouilliard]

from collections.abc import Generator, Iterator
import pandas as pd
import psycopg2
import requests
from scipy.spatial.distance import cosine
from sentence_transformers import SentenceTransformer

#logging.basicConfig(level=logging.DEBUG)

class Pipeline:
"""Pipeline for processing queries.

This pipeline handles database connections, document retrieval,
embedding calculations, and similarity matching for the
pjm_documents collection.
"""

def __init__(self):
    """Initialize the Pipeline with database configuration."""
    self.name = "Optimised Pipeline"
    self.connection_params = {
        "dbname": "openwebui",
        "user": "openwebui",
        "password": "your_secure_password",
        "host": "postgres",  # Hostname defined in docker-compose.yml
        "port": "5432",
    }
    self.data_directory_path = "/app/src/data/docs/PDFs/data_directory.csv"

async def on_startup(self):
    """Initialize required components during startup."""
    self.model = SentenceTransformer(
        "sentence-transformers/all-mpnet-base-v2"
    )
    # logging.info("Embedding model loaded successfully.")
    # llama_test_result = self.test_llama_connection()
    # print(llama_test_result)

async def on_shutdown(self):
    """Perform cleanup operations during shutdown."""
    print("Pipeline shutting down.")

def fetch_id_embeddings(self, connection_params, limit=None, ):
    """Fetch data from the public.pjm_documents table."""
    try:
        conn = psycopg2.connect(**connection_params)
        cur = conn.cursor()

        # Query to fetch data from pjm_documents table

        # Query to fetch data from pjm_documents table
        if limit is None:
            query = "SELECT id, embeddings FROM public.pjm_documents;"
        else:
            query = f"SELECT id, embeddings FROM public.pjm_documents LIMIT {limit};"
            
        cur.execute(query)
        rows = cur.fetchall()

        # Close the cursor and connection
        cur.close()
        conn.close()

        # Format the data for display
        if rows:
            return rows
        else:
            return "No data found in public.pjm_documents."

    except Exception as e:
        print(f"Failed to retrieve data from pjm_documents: {e}")
        return f"Failed to retrieve data from the pjm_documents table. {e}"

def calculate_similarity(self, connection_params, query_embedding, limit=10) -> str:
    """Fetch embeddings and calculate similarity to the query embedding."""
    try:
        rows = self.fetch_id_embeddings(connection_params)

        # Check if rows were retrieved
        if not rows:
            return "No data found in pjm_documents."

        # Calculate cosine similarity and store results
        similarities = []
        for row in rows:
            doc_id, doc_embedding = row
            if not doc_embedding:
                continue

            similarity_score = 1 - cosine(
                query_embedding, doc_embedding
            )  # Cosine similarity
            similarities.append((doc_id, similarity_score))

        # Sort results by similarity score in descending order
        similarities.sort(key=lambda x: x[1], reverse=True)
        top_results = similarities[:limit]
        
        return top_results

    except Exception as e:
            return (
                f"Failed to calculate similarity with the pjm doc embeddings. {e}"
            )
    
def get_links_from_documents(self, documents):
    links= []

    # Get a list that has no duplicates
    documents = list(dict.fromkeys(documents))

    # Go get the dataframe with the links
    path_data_directory = "/app/pipelines/data/data_directory.csv"

    # The columns are called Urls, File_name
    data_directory = pd.read_csv(path_data_directory)

    # Get the links
    for document in documents:
        print(f"document: {document}")
        # get type of document
        link = data_directory[data_directory["File_name"] == document]["Urls"].values[0]
        links.append(link)

    return links

def get_texts_and_links_with_ids(self, connection_params, similarity_result):
    """
    Fetches the texts corresponding to doc_ids in similarity_result and returns them
    concatenated as a single string, separated by a newline.
    """
    try:
        # Step 1: Extract IDs from similarity_result
        doc_ids = [doc_id for doc_id, _ in similarity_result]

        # Step 2: Connect to the database and fetch texts in bulk
        conn = psycopg2.connect(**connection_params)
        cur = conn.cursor()
        query = f"SELECT document, text FROM public.pjm_documents WHERE id IN %s;"
        cur.execute(query, (tuple(doc_ids),))
        rows = cur.fetchall()  # Fetch all matching rows

        # Extract documents and texts into lists
        documents = [row[0] for row in rows]  # Extract 'documents' column
        texts = [row[1] for row in rows]  # Extract 'text' column

        # Get links from the documents
        links = self.get_links_from_documents(documents)

        # Step 3: Close database connection
        cur.close()
        conn.close()

        # Step 4: Format the result
        formatted_texts = [
            f"Text {i+1}: \n {text} \n\n" for i, text in enumerate(texts)
        ]

        # Add the links at the end of the string separated by a newline
        formatted_texts.append("\n".join(links))

        return "\n".join(formatted_texts)

    except Exception as e:
        return f"Failed to retrieve texts from pjm_documents: {e}"
def pipe(
    self,
    user_message: str,
    model_id: str,
    messages: list[dict],
    body: dict,
) -> str | Generator | Iterator: 
    """Process a user message through the pipeline."""
    query_embedding = self.model.encode(user_message)
    
    similarity_texts = self.calculate_similarity(self.connection_params, query_embedding)

    final_result = self.get_texts_and_links_with_ids(self.connection_params, similarity_texts)

    return final_result

[xrh1337]

I might be misunderstanding, but if you'd like to have links in your response just format your links with markdown.
Example docs:
https://www.markdownguide.org/basic-syntax/#links

[egouilliard]

Yes, that is what I am doing currently. However, I want them too look like this.

[egouilliard]

I don't want it in markdown, I want it to be in the same format as when I try to get a file from the knowledge base.