How to retrieve DB Embedding from We Aviate

[lakshya29154]

Checked other resources

• I added a very descriptive title to this question.
• I searched the LangChain documentation with the integrated search.
• I used the GitHub search to find a similar question and didn't find it.

Commit to Help

• I commit to help with one of those options 👆

Example Code

import * as fs from "fs";
import { config } from 'dotenv';
import { createHash } from 'crypto';
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
import { OpenAIEmbeddings } from "@langchain/openai";
import weaviate, { ApiKey } from "weaviate-ts-client";
import { WeaviateStore } from "@langchain/weaviate";

// const { Collection, MetadataQuery } = require("weaviate");
// import {Collection, MetaDataQuery} from "@langchain/weaviate"
// import { API_URL_PROMPT_TEMPLATE } from "./test20";
import { OpenAI } from "@langchain/openai";
import { PromptTemplate } from "@langchain/core/prompts";
config();

const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
const model = new OpenAI({ temperature: 0, apiKey: OPENAI_API_KEY });
const client = (weaviate).client({
    scheme: process.env.WEAVIATE_SCHEME || "https",
    host: "swagger-as4wyy7e.weaviate.network",
    apiKey: new ApiKey('yEhKKW4MLmbvVJT2yw7mlTYutuz5zl22U2p1'),
  });
const API_URL_PROMPT_TEMPLATE1 = `You are given the below the embedding fo some swagger Documentation:
{api_docs}
Using this embeddings, find the relevant API URLs there can be multiple url so give all the url step by step give the method also like get, post , put , delete and give the answer in step1 , step 2 like this.

You should build the API URLs in order to get a response that is as short as possible, while still getting the necessary information to answer the question. Pay attention to deliberately exclude any unnecessary pieces of data in the API call.
Question:{question}
API url:`;
const API_URL_PROMPT_TEMPLATE = /* #__PURE__ */ new PromptTemplate({
    inputVariables: ["api_docs", "question"],
    template: API_URL_PROMPT_TEMPLATE1,
  });
// async function getValidData(apiJson)
// {
//     const API_URL_PROMPT_TEMPLATE2 = `You are given the below the mebedding fo some swagger json data:
//     {api_docs}
//     From this json data you have to fetch only those data which are required to recommend the api like its endpoint description method like You will get a question
//     like you will get a question give me the api for .... so from the data you need to find the api and give it as response so for now from all the swagger data remove the data which is not needed and give it to me
//     API url:`;
//     const API_URL_PROMPT_TEMPLATE3 = /* #__PURE__ */ new PromptTemplate({
//         inputVariables: ["api_docs"],
//         template: API_URL_PROMPT_TEMPLATE2,
//       });
//     const apiUrlPrompt = await API_URL_PROMPT_TEMPLATE3.format({
//         api_docs: apiJson,
//       });
    
//      let finalData =  await model.generate([apiUrlPrompt]);
//      console.log(finalData);
//      return finalData
// }
async function generateHash(text) {
    return createHash('sha256').update(text).digest('hex');
}


async function splitTextIntoChunks(text) {
    const splitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000, chunkOverlap: 200 });
    const documents = await splitter.createDocuments([text]);
    return documents.map(doc => doc.pageContent);
}

async function storeEmbedding(textChunks)
{

    const hashes = textChunks.map(chunk => generateHash(chunk));
    await WeaviateStore.fromTexts(
        textChunks,
        textChunks.map((chunk, index) => ({ hash: hashes[index] })), 
        new OpenAIEmbeddings(),
        {
            client,
            indexName: "Test",
            textKey: "text",
            metadataKeys: ["hash"],
        }
    );
}
async function queryWeaviate(query) {
    try {
      // Retrieve the collection named "Test"
      const reviews = await client.collections.get("Test");
  
      // Perform a query using the nearText method
      const response = await reviews.query.nearText({
        query: query,
        limit: 2,
        targetVector: "text", // Specify the target vector for named vector collections
        returnMetadata: new MetaDataQuery({ distance: true }),
      });
  
      console.log(response); // Print the response
    } catch (error) {
      console.error("Error querying Weaviate:", error);
    }
  }



async function generateEmbeddings(textChunks) {
    const embeddings = new OpenAIEmbeddings({ apiKey: process.env.OPENAI_API_KEY });
    const vectors = [];

    for (const chunk of textChunks) {
        const vector = await embeddings.embedQuery(chunk);
        // console.log("Generated Vector:", vector); // Log to check the vector
        if (vector && vector.length > 0) {
            vectors.push({ chunk, vector });
        } else {
            console.error("Invalid or empty vector generated for chunk:", chunk);
        }
    }

    return vectors;
}
async function queryWeaviateForSimilarEmbeddings(questionEmbedding) {
    const searchQuery = {
        vector: {
            and: [
                {
                    property: "vector",  // Assuming the name of the property storing embeddings is "vector"
                    value: {
                        nearVector: {
                            vector: questionEmbedding,
                            limit: 200  // Retrieve top 10 nearest neighbors
                        }
                    }
                }
            ]
        }
    }
    return searchQuery
}

async function main() {
    try {
        const specFilePath = './swagger.json';
        const jsonData = fs.readFileSync(specFilePath, 'utf8');
        // console.log("This is json data:", jsonData);
        // const validData = await getValidData(jsonData)
        // console.log("This is valid data", validData)
        // Generate SHA256 hash of JSON data
        // const hash = generateHash(jsonData);
        const textChunks = await splitTextIntoChunks(jsonData);
        await storeEmbedding(textChunks);
        console.log("Embedding has been stored successfully")
      let query = "Can you give me the endpoint to place an order for a pet?"
       queryWeaviate(query)
    //    let Nearembedding = await queryWeaviate(query)
        // console.log("Text chunk", textChunks);
        // console.log(typeof(textChunks))
        // textChunks.map((chunk)=>{
        //     console.log("-------------------------------",chunk)
        // })
        //  embeddingStore(textChunks)
        // textChunks.map(chunk => ({ hash }
        //     console.log("-------------------------------",hash
        // ))
        // Generate embeddings for JSON data
        // const dataEmbeddings = await generateEmbeddings(textChunks);
        // console.log("Embeddings", embeddings);
        
        // Initialize Weaviate client
        // const client = (weaviate).client({
        //     scheme: process.env.WEAVIATE_SCHEME || "https",
        //     host: "swagger01-bybjaoyp.grpc.weaviate.network",
        //     apiKey: new ApiKey('jmIEZWKS5RXJbwpQBCtsTDI5afQMQRBCKIA3'),
        //   });

        // Create a store and fill it with some texts + metadata
        // await WeaviateStore.fromTexts(
        //     textChunks,
        //     textChunks.map(chunk => ({ hash })),
        //     new OpenAIEmbeddings(),
        //     {
        //         client,
        //         indexName: "Test",
        //         textKey: "text",
        //         metadataKeys: ["hash"],
        //     }
        // );
    //    const query = "Can you give the api url for whitelist investor and then fetch all wallet?"
    //    let queryEmbedding = await generateEmbeddings(query)
       
    //    console.log("This is query embedding",queryEmbedding)
    //     console.log('Embeddings stored successfully.');
    //    let similarEmbedding = await queryWeaviateForSimilarEmbeddings(queryEmbedding)
    //    console.log("This is similar embedding",similarEmbedding)
        // const result = await client.query(similarEmbedding);   // Trying to get answer
        // // console.log("This is the result",result)   // Trying to get answer
        // const apiUrlPrompt = await API_URL_PROMPT_TEMPLATE.format({
        //     api_docs: similarEmbedding,
        //     question: query,
        //   });
      
          // console.log("API URL Prompt:", apiUrlPrompt);
      
        //   const apiUrlResponse = await model.generate([apiUrlPrompt]);
        //   console.log("API URL Response:", JSON.stringify(apiUrlResponse, null, 2));
      
        //   // Extract the generated URL from the response
        //   const apiUrl = apiUrlResponse.generations[0][0]?.text
        //   console.log("Generated API URL:", apiUrl);
    } catch (error) {
        console.error('Error:', error);
    }
}

main();

Description

I am getting error in retrieving similar embedding Can you please help how to get relevant embedding

System Info

all ok

[dosubot[bot]]

To retrieve similar embeddings from Weaviate using the provided code, you can use the similaritySearch method from the WeaviateStore class. This method performs a similarity search on the stored vectors in the Weaviate index and returns the top k most similar documents.

Here is an example of how you can modify your code to use the similaritySearch method:

import * as fs from "fs";
import { config } from 'dotenv';
import { createHash } from 'crypto';
import { RecursiveCharacterTextSplitter } from "langchain/text_splitter";
import { OpenAIEmbeddings } from "@langchain/openai";
import weaviate, { ApiKey } from "weaviate-ts-client";
import { WeaviateStore } from "@langchain/weaviate";
import { OpenAI } from "@langchain/openai";
import { PromptTemplate } from "@langchain/core/prompts";
config();

const OPENAI_API_KEY = process.env.OPENAI_API_KEY;
const model = new OpenAI({ temperature: 0, apiKey: OPENAI_API_KEY });
const client = (weaviate as any).client({
    scheme: process.env.WEAVIATE_SCHEME || "https",
    host: "swagger-as4wyy7e.weaviate.network",
    apiKey: new ApiKey('yEhKKW4MLmbvVJT2yw7mlTYutuz5zl22U2p1'),
});

const API_URL_PROMPT_TEMPLATE1 = `You are given the below the embedding fo some swagger Documentation:
{api_docs}
Using this embeddings, find the relevant API URLs there can be multiple url so give all the url step by step give the method also like get, post , put , delete and give the answer in step1 , step 2 like this.

You should build the API URLs in order to get a response that is as short as possible, while still getting the necessary information to answer the question. Pay attention to deliberately exclude any unnecessary pieces of data in the API call.
Question:{question}
API url:`;
const API_URL_PROMPT_TEMPLATE = /* #__PURE__ */ new PromptTemplate({
    inputVariables: ["api_docs", "question"],
    template: API_URL_PROMPT_TEMPLATE1,
});

async function generateHash(text) {
    return createHash('sha256').update(text).digest('hex');
}

async function splitTextIntoChunks(text) {
    const splitter = new RecursiveCharacterTextSplitter({ chunkSize: 1000, chunkOverlap: 200 });
    const documents = await splitter.createDocuments([text]);
    return documents.map(doc => doc.pageContent);
}

async function storeEmbedding(textChunks) {
    const hashes = textChunks.map(chunk => generateHash(chunk));
    await WeaviateStore.fromTexts(
        textChunks,
        textChunks.map((chunk, index) => ({ hash: hashes[index] })), 
        new OpenAIEmbeddings(),
        {
            client,
            indexName: "Test",
            textKey: "text",
            metadataKeys: ["hash"],
        }
    );
}

async function queryWeaviate(query) {
    try {
        const store = await WeaviateStore.fromExistingIndex(new OpenAIEmbeddings(), {
            client,
            indexName: "Test",
            metadataKeys: ["hash"],
        });

        const results = await store.similaritySearch(query, 2);
        console.log(results);
    } catch (error) {
        console.error("Error querying Weaviate:", error);
    }
}

async function main() {
    try {
        const specFilePath = './swagger.json';
        const jsonData = fs.readFileSync(specFilePath, 'utf8');
        const textChunks = await splitTextIntoChunks(jsonData);
        await storeEmbedding(textChunks);
        console.log("Embedding has been stored successfully");

        let query = "Can you give me the endpoint to place an order for a pet?";
        await queryWeaviate(query);
    } catch (error) {
        console.error('Error:', error);
    }
}

main();

In this example, the queryWeaviate function is updated to use the similaritySearch method from the WeaviateStore class. This method performs a similarity search on the stored vectors and retrieves the most similar documents based on the query ^[1][2].

To continue talking to Dosu, mention @dosu.