Cannot run embedding!

[aiddroid]

Auto download model , but not working, is there anything wrong?

from langchain.embeddings import HuggingFaceEmbeddings
from langchain.document_loaders import JSONLoader
from langchain.vectorstores.chroma import Chroma

loader = JSONLoader(
    file_path="database/q1.jsonl",
    jq_schema=".question",
    json_lines=True
)
texts = loader.load()
# print('texts', texts[0], len(texts))

embeddings = HuggingFaceEmbeddings(model_name="THUDM/chatglm3-6b")
docsearch = Chroma.from_documents(texts, embeddings)

error messages:

Loading checkpoint shards: 100%|█████████████████████████████████████████████| 7/7 [00:16<00:00,  2.38s/it]
Traceback (most recent call last):
  File "/Users/allen/dev/demo/huggingface-embeddings.py", line 21, in <module>
    embeddings = HuggingFaceEmbeddings(model_name="THUDM/chatglm3-6b")
  File "/Users/allen/.pyenv/versions/3.10.11/lib/python3.10/site-packages/langchain_community/embeddings/huggingface.py", line 65, in __init__
    self.client = sentence_transformers.SentenceTransformer(
  File "/Users/allen/.pyenv/versions/3.10.11/lib/python3.10/site-packages/sentence_transformers/SentenceTransformer.py", line 97, in __init__
    modules = self._load_auto_model(model_path)
  File "/Users/allen/.pyenv/versions/3.10.11/lib/python3.10/site-packages/sentence_transformers/SentenceTransformer.py", line 806, in _load_auto_model
    transformer_model = Transformer(model_name_or_path)
  File "/Users/allen/.pyenv/versions/3.10.11/lib/python3.10/site-packages/sentence_transformers/models/Transformer.py", line 31, in __init__
    self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path if tokenizer_name_or_path is not None else model_name_or_path, cache_dir=cache_dir, **tokenizer_args)
  File "/Users/allen/.pyenv/versions/3.10.11/lib/python3.10/site-packages/transformers/models/auto/tokenization_auto.py", line 765, in from_pretrained
    raise ValueError(
ValueError: Tokenizer class ChatGLMTokenizer does not exist or is not currently imported.

Works well

from langchain.embeddings import HuggingFaceEmbeddings
from langchain.document_loaders import JSONLoader
from langchain.vectorstores.chroma import Chroma

loader = JSONLoader(
    file_path="database/q1.jsonl",
    jq_schema=".question",
    json_lines=True
)
texts = loader.load()
# print('texts', texts[0], len(texts))

embeddings = HuggingFaceEmbeddings(model_name="nghuyong/ernie-1.0-base-zh")
docsearch = Chroma.from_documents(texts, embeddings)

Works well too

from langchain.embeddings import HuggingFaceEmbeddings
from langchain.document_loaders import JSONLoader
from langchain.vectorstores.chroma import Chroma

loader = JSONLoader(
    file_path="database/q1.jsonl",
    jq_schema=".question",
    json_lines=True
)
texts = loader.load()
# print('texts', texts[0], len(texts))

embeddings = HuggingFaceEmbeddings()
docsearch = Chroma.from_documents(texts, embeddings)

[zRzRzRzRzRzRzR]

#436 discuss here