Add encodings folder to azure-ai-resources (#34228)

* add encodings

* include encodings folder

* use local embeddings

* use local cache dirs

* ignore unknown words in encodings files

Author: jingyizhu99

sdk/ai/azure-ai-generative/azure/ai/generative/index/_embeddings/openai.py

@@ -235,9 +235,11 @@ def _embed(self, texts: List[str]) -> List[List[float]]:
         """Embed the given texts."""
         import numpy as np
         import tiktoken
+        from azure.ai.generative.index._utils.tokens import tiktoken_cache_dir
 
         try:
-            encoding = tiktoken.encoding_for_model(self.model)
+            with tiktoken_cache_dir():
+                encoding = tiktoken.encoding_for_model(self.model)
         except KeyError:
             logger.warning("Warning: model not found. Using cl100k_base encoding.")
             model = "cl100k_base"

sdk/ai/azure-ai-resources/MANIFEST.in

@@ -4,4 +4,5 @@ include *.md
 include azure/__init__.py
 include azure/ai/__init__.py
 include azure/ai/resources/py.typed
+include azure/ai/resources/_index/_utils/encodings/*
 include azure/ai/common/operations/component-configs/*

sdk/ai/azure-ai-resources/azure/ai/resources/_index/_embeddings/openai.py

@@ -222,9 +222,11 @@ def _embed(self, texts: List[str]) -> List[List[float]]:
         """Embed the given texts."""
         import numpy as np
         import tiktoken
+        from azure.ai.resources._index._utils.tokens import tiktoken_cache_dir
 
         try:
-            encoding = tiktoken.encoding_for_model(self.model)
+            with tiktoken_cache_dir():
+                encoding = tiktoken.encoding_for_model(self.model)
         except KeyError:
             logger.warning("Warning: model not found. Using cl100k_base encoding.")
             model = "cl100k_base"