refactor: upgrade to latest AI Search SDK

Author: sinedied

package-lock.json

@@ -26,7 +26,7 @@
     "@azure/core-http": "^3.0.5",
     "@azure/identity": "^4.0.1",
     "@azure/monitor-opentelemetry": "^1.0.0-beta.2",
-    "@azure/search-documents": "12.0.0-beta.3",
+    "@azure/search-documents": "^12.1.0",
     "@azure/storage-blob": "^12.15.0",
     "@dqbd/tiktoken": "^1.0.7",
     "@fastify/autoload": "^6.1.0",

packages/indexer/package.json

@@ -52,6 +52,41 @@ export class Indexer {
     } else {
       const index: SearchIndex = {
         name: indexName,
+        vectorSearch: {
+          algorithms: [
+            {
+              name: 'vector-search-algorithm',
+              kind: 'hnsw',
+              parameters: {
+                m: 4,
+                efSearch: 500,
+                metric: 'cosine',
+                efConstruction: 400,
+              },
+            },
+          ],
+          profiles: [
+            {
+              name: 'vector-search-profile',
+              algorithmConfigurationName: 'vector-search-algorithm',
+            },
+          ],
+        },
+        semanticSearch: {
+          defaultConfigurationName: 'semantic-search-config',
+          configurations: [
+            {
+              name: 'semantic-search-config',
+              prioritizedFields: {
+                contentFields: [
+                  {
+                    name: 'content',
+                  },
+                ],
+              },
+            },
+          ],
+        },
         fields: [
           {
             name: 'id',
@@ -73,7 +108,7 @@ export class Indexer {
             sortable: false,
             facetable: false,
             vectorSearchDimensions: 1536,
-            vectorSearchConfiguration: 'default',
+            vectorSearchProfileName: 'vector-search-profile',
           },
           {
             name: 'category',
@@ -94,27 +129,6 @@ export class Indexer {
             facetable: true,
           },
         ],
-        semanticSettings: {
-          configurations: [
-            {
-              name: 'default',
-              prioritizedFields: {
-                prioritizedContentFields: [{ name: 'content' }],
-              },
-            },
-          ],
-        },
-        vectorSearch: {
-          algorithmConfigurations: [
-            {
-              name: 'default',
-              kind: 'hnsw',
-              parameters: {
-                metric: 'cosine',
-              },
-            },
-          ],
-        },
       };
       this.logger.debug(`Creating "${indexName}" search index...`);
       await searchIndexClient.createIndex(index);

packages/indexer/src/lib/indexer.ts

@@ -23,7 +23,7 @@
     "@azure/core-http": "^3.0.5",
     "@azure/identity": "^4.0.1",
     "@azure/monitor-opentelemetry": "^1.0.0-beta.2",
-    "@azure/search-documents": "12.0.0-beta.3",
+    "@azure/search-documents": "^12.1.0",
     "@azure/storage-blob": "^12.15.0",
     "@dqbd/tiktoken": "^1.0.7",
     "@fastify/autoload": "^6.1.0",

packages/search/package.json

@@ -43,29 +43,40 @@ export class ApproachBase {
       ? this.search.search(queryText, {
           filter,
           queryType: 'semantic',
-          queryLanguage: 'en-us',
-          speller: 'lexicon',
-          semanticConfiguration: 'default',
+          semanticSearchOptions: {
+            configurationName: 'semantic-search-config',
+            captions: useSemanticCaption
+              ? {
+                  captionType: 'extractive',
+                  highlight: false,
+                }
+              : undefined,
+          },
           top,
-          captions: useSemanticCaption ? 'extractive|highlight-false' : undefined,
-          vectors: [
-            {
-              value: queryVector,
-              kNearestNeighborsCount: queryVector ? 50 : undefined,
-              fields: queryVector ? ['embedding'] : undefined,
-            },
-          ],
+          vectorSearchOptions: {
+            queries: [
+              {
+                kind: 'vector',
+                vector: queryVector!,
+                kNearestNeighborsCount: queryVector ? 50 : undefined,
+                fields: queryVector ? ['embedding'] : undefined,
+              },
+            ],
+          },
         })
       : this.search.search(queryText, {
           filter,
           top,
-          vectors: [
-            {
-              value: queryVector,
-              kNearestNeighborsCount: queryVector ? 50 : undefined,
-              fields: queryVector ? ['embedding'] : undefined,
-            },
-          ],
+          vectorSearchOptions: {
+            queries: [
+              {
+                kind: 'vector',
+                vector: queryVector!,
+                kNearestNeighborsCount: queryVector ? 50 : undefined,
+                fields: queryVector ? ['embedding'] : undefined,
+              },
+            ],
+          },
         }));
 
     const results: string[] = [];
@@ -87,32 +98,4 @@ export class ApproachBase {
     const content = results.join('\n');
     return { query: queryText ?? '', results, content };
   }
-
-  protected async lookupDocument(query: string): Promise<any> {
-    const searchResults = await this.search.search(query, {
-      top: 1,
-      includeTotalCount: true,
-      queryType: 'semantic',
-      queryLanguage: 'en-us',
-      speller: 'lexicon',
-      semanticConfiguration: 'default',
-      answers: 'extractive|count-1',
-      captions: 'extractive|highlight-false',
-    });
-
-    const answers = await searchResults.answers;
-    if (answers && answers.length > 0) {
-      return answers[0].text;
-    }
-    if (searchResults.count ?? 0 > 0) {
-      const results: string[] = [];
-      for await (const result of searchResults.results) {
-        // TODO: ensure typings
-        const document = result.document as any;
-        results.push(document[this.contentField]);
-      }
-      return results.join('\n');
-    }
-    return undefined;
-  }
 }