docs and examples

Author: margaretjgu

examples/src/langchain-classic/indexes/vector_stores/elasticsearch/elasticsearch_hybrid.ts

@@ -0,0 +1,140 @@
+import { Client, ClientOptions } from "@elastic/elasticsearch";
+import { OpenAIEmbeddings } from "@langchain/openai";
+import {
+  ElasticClientArgs,
+  ElasticVectorSearch,
+  HybridRetrievalStrategy,
+} from "@langchain/community/vectorstores/elasticsearch";
+import { Document } from "@langchain/core/documents";
+
+/**
+ * Demonstrates hybrid search with Elasticsearch, combining:
+ * - Vector (semantic) search using embeddings
+ * - BM25 (lexical) full-text search
+ * - Reciprocal Rank Fusion (RRF) for result merging
+ * 
+ * Requirements:
+ * - Elasticsearch 8.9+ (for RRF support)
+ * - Run: docker-compose up -d --build (in elasticsearch directory)
+ * - Set ELASTIC_URL, ELASTIC_API_KEY (or ELASTIC_USERNAME/ELASTIC_PASSWORD)
+ */
+export async function run() {
+  // Configure Elasticsearch client
+  const config: ClientOptions = {
+    node: process.env.ELASTIC_URL ?? "http://127.0.0.1:9200",
+  };
+  if (process.env.ELASTIC_API_KEY) {
+    config.auth = {
+      apiKey: process.env.ELASTIC_API_KEY,
+    };
+  } else if (process.env.ELASTIC_USERNAME && process.env.ELASTIC_PASSWORD) {
+    config.auth = {
+      username: process.env.ELASTIC_USERNAME,
+      password: process.env.ELASTIC_PASSWORD,
+    };
+  }
+
+  const embeddings = new OpenAIEmbeddings();
+
+  // Create vector store with hybrid search strategy
+  const clientArgs: ElasticClientArgs = {
+    client: new Client(config),
+    indexName: process.env.ELASTIC_INDEX ?? "test_hybrid_search",
+    strategy: new HybridRetrievalStrategy({
+      rankWindowSize: 100,  // Number of documents to consider for RRF
+      rankConstant: 60,     // RRF constant for score normalization
+      textField: "text",    // Field to use for BM25 search
+    }),
+  };
+
+  const vectorStore = new ElasticVectorSearch(embeddings, clientArgs);
+
+  // Clean up any existing data
+  await vectorStore.deleteIfExists();
+
+  // Add sample documents
+  const docs = [
+    new Document({
+      pageContent: "Running helps build cardiovascular endurance and strengthens leg muscles.",
+      metadata: { category: "fitness", topic: "running" },
+    }),
+    new Document({
+      pageContent: "Marathon training requires consistent mileage and proper recovery.",
+      metadata: { category: "fitness", topic: "running" },
+    }),
+    new Document({
+      pageContent: "Muscle soreness after exercise is caused by microscopic damage to muscle fibers.",
+      metadata: { category: "health", topic: "recovery" },
+    }),
+    new Document({
+      pageContent: "Stretching and foam rolling can help prevent post-workout muscle pain.",
+      metadata: { category: "health", topic: "recovery" },
+    }),
+    new Document({
+      pageContent: "Python is a popular programming language for data science and machine learning.",
+      metadata: { category: "technology", topic: "programming" },
+    }),
+  ];
+
+  console.log("Adding documents to Elasticsearch...");
+  await vectorStore.addDocuments(docs);
+  console.log("Documents added successfully!\n");
+
+  // Example 1: Hybrid search combines semantic + keyword matching
+  console.log("=== Example 1: Hybrid Search ===");
+  const query1 = "How to avoid muscle soreness while running?";
+  console.log(`Query: "${query1}"\n`);
+  
+  const results1 = await vectorStore.similaritySearchWithScore(query1, 3);
+  results1.forEach(([doc, score], i) => {
+    console.log(`${i + 1}. [Score: ${score.toFixed(4)}] ${doc.pageContent}`);
+    console.log(`   Metadata: ${JSON.stringify(doc.metadata)}\n`);
+  });
+
+  // Example 2: Semantic search works well for conceptual queries
+  console.log("\n=== Example 2: Semantic Query ===");
+  const query2 = "tips for preventing pain after workouts";
+  console.log(`Query: "${query2}"\n`);
+  
+  const results2 = await vectorStore.similaritySearchWithScore(query2, 2);
+  results2.forEach(([doc, score], i) => {
+    console.log(`${i + 1}. [Score: ${score.toFixed(4)}] ${doc.pageContent}`);
+    console.log(`   Metadata: ${JSON.stringify(doc.metadata)}\n`);
+  });
+
+  // Example 3: With metadata filters
+  console.log("\n=== Example 3: Hybrid Search with Filters ===");
+  const query3 = "fitness advice";
+  console.log(`Query: "${query3}"`);
+  console.log(`Filter: category = "fitness"\n`);
+  
+  const results3 = await vectorStore.similaritySearchWithScore(
+    query3,
+    3,
+    { category: "fitness" }
+  );
+  results3.forEach(([doc, score], i) => {
+    console.log(`${i + 1}. [Score: ${score.toFixed(4)}] ${doc.pageContent}`);
+    console.log(`   Metadata: ${JSON.stringify(doc.metadata)}\n`);
+  });
+
+  // Clean up
+  console.log("\n=== Cleanup ===");
+  await vectorStore.deleteIfExists();
+  console.log("Index deleted.");
+}
+
+/**
+ * For Elasticsearch 9.2+:
+ * If you need to include vectors in the response, set includeSourceVectors:
+ * 
+ * strategy: new HybridRetrievalStrategy({
+ *   includeSourceVectors: true,
+ *   rankWindowSize: 100,
+ *   rankConstant: 60,
+ * })
+ * 
+ * Note: This is only needed if you're on ES 9.2+ and want vector data
+ * in search responses. ES 9.2+ excludes vectors by default for performance.
+ */
+

libs/langchain-community/src/vectorstores/elasticsearch.ts

@@ -33,9 +33,14 @@ export interface HybridRetrievalStrategyConfig {
   rankConstant?: number;
   textField?: string;
   /**
-   * For Elasticsearch 9.2, set to `false` to include vectors in responses.
+   * Include source vectors in search responses.
+   * 
+   * Elasticsearch 9.2+ excludes vectors from `_source` by default.
+   * Set to `true` to include vectors in responses for ES 9.2+.
+   * 
+   * Note: ES < 8.19 does not support this parameter.
    */
-  excludeSourceVectors?: boolean;
+  includeSourceVectors?: boolean;
 }
 
 /**
@@ -45,13 +50,13 @@ export class HybridRetrievalStrategy {
   public readonly rankWindowSize: number;
   public readonly rankConstant: number;
   public readonly textField: string;
-  public readonly excludeSourceVectors?: boolean;
+  public readonly includeSourceVectors?: boolean;
 
   constructor(config: HybridRetrievalStrategyConfig = {}) {
     this.rankWindowSize = config.rankWindowSize ?? 100;
     this.rankConstant = config.rankConstant ?? 60;
     this.textField = config.textField ?? "text";
-    this.excludeSourceVectors = config.excludeSourceVectors;
+    this.includeSourceVectors = config.includeSourceVectors;
   }
 }
 
@@ -83,10 +88,23 @@ type ElasticMetadataTerms = {
 };
 
 /**
- * Class for interacting with an Elasticsearch database. It extends the
- * VectorStore base class and provides methods for adding documents and
- * vectors to the Elasticsearch database, performing similarity searches,
- * deleting documents, and more.
+ * Elasticsearch vector store supporting vector and hybrid search.
+ * 
+ * Hybrid search combines kNN vector search with BM25 full-text search
+ * using RRF. Enable by passing a `HybridRetrievalStrategy` to the constructor.
+ * 
+ * @example
+ * ```typescript
+ * // Vector search (default)
+ * const vectorStore = new ElasticVectorSearch(embeddings, { client, indexName });
+ * 
+ * // Hybrid search
+ * const hybridStore = new ElasticVectorSearch(embeddings, {
+ *   client,
+ *   indexName,
+ *   strategy: new HybridRetrievalStrategy()
+ * });
+ * ```
  */
 export class ElasticVectorSearch extends VectorStore {
   declare FilterType: ElasticFilter;
@@ -239,6 +257,9 @@ export class ElasticVectorSearch extends VectorStore {
         k,
         num_candidates: this.candidates,
       },
+      ...(this.strategy?.includeSourceVectors === true && {
+        _source: { includes: ["*"] },
+      }),
     });
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -291,6 +312,9 @@ export class ElasticVectorSearch extends VectorStore {
         },
       },
       ...(filterClauses && { query: filterClauses }),
+      ...(this.strategy?.includeSourceVectors === true && {
+        _source: { includes: ["*"] },
+      }),
     });
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
@@ -427,12 +451,6 @@ export class ElasticVectorSearch extends VectorStore {
       },
     };
 
-    if (this.strategy?.excludeSourceVectors !== undefined) {
-      request.settings = {
-        "index.mapping.exclude_source_vectors": this.strategy.excludeSourceVectors,
-      };
-    }
-
     const indexExists = await this.doesIndexExist();
     if (indexExists) return;
 

libs/langchain-community/src/vectorstores/tests/elasticsearch.int.test.ts

@@ -423,37 +423,30 @@ describe("ElasticVectorSearch - ES 9.x Compatibility", () => {
     embeddings = new OpenAIEmbeddings();
   });
 
-  test.skip("Hybrid search with excludeSourceVectors set to false for ES 9.x", async () => {
-    const indexName = "test_es9_exclude_false";
+  test.skip("Hybrid search with includeSourceVectors set to true for ES 9.2+", async () => {
+    const indexName = "test_es9_include_vectors";
     const store = new ElasticVectorSearch(embeddings, {
       client,
       indexName,
       strategy: new HybridRetrievalStrategy({
-        excludeSourceVectors: false,
+        includeSourceVectors: true,
       }),
     });
     await store.deleteIfExists();
 
     await store.addDocuments([
-      new Document({ pageContent: "Document for ES 9.x testing" }),
+      new Document({ pageContent: "Document for ES 9.2+ testing" }),
       new Document({ pageContent: "Another document for compatibility" }),
     ]);
 
     const results = await store.similaritySearch("testing", 2);
 
     expect(results).toHaveLength(2);
     expect(results[0]).toBeInstanceOf(Document);
-
-    const indexSettings = await client.indices.getSettings({
-      index: indexName,
-    });
-    expect(
-      indexSettings[indexName].settings?.index?.mapping?.exclude_source_vectors
-    ).toBe("false");
   });
 
-  test.skip("Hybrid search with excludeSourceVectors undefined uses ES defaults", async () => {
-    const indexName = "test_es_default_exclude";
+  test.skip("Hybrid search with includeSourceVectors undefined uses ES defaults", async () => {
+    const indexName = "test_es_default_include";
     const store = new ElasticVectorSearch(embeddings, {
       client,
       indexName,
@@ -471,13 +464,13 @@ describe("ElasticVectorSearch - ES 9.x Compatibility", () => {
     expect(results[0]).toBeInstanceOf(Document);
   });
 
-  test.skip("Pure vector search with excludeSourceVectors for ES 9.x", async () => {
+  test.skip("Pure vector search with includeSourceVectors for ES 9.2+", async () => {
     const indexName = "test_es9_pure_vector";
     const store = new ElasticVectorSearch(embeddings, {
       client,
       indexName,
       strategy: new HybridRetrievalStrategy({
-        excludeSourceVectors: false,
+        includeSourceVectors: true,
       }),
     });
     await store.deleteIfExists();