RuntimeError: Now only support bool/uint8 type

[tom54007]

在尝试使用摩尔线程 mtt s80显卡进行推理时遇到了这个问题，

完整错误信息如下：

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Cell In[2], line 1
----> 1 main()

Cell In[1], line 44, in main()
     42 print("\nChatGLM：", end="")
     43 current_length = 0
---> 44 for response, history, past_key_values in model.stream_chat(tokenizer, query, history=history, top_p=1,
     45                                                             temperature=0.01,
     46                                                             past_key_values=past_key_values,
     47                                                             return_past_key_values=True):
     48     if stop_stream:
     49         stop_stream = False

File ~/miniconda3/envs/CG3/lib/python3.9/site-packages/torch/utils/_contextlib.py:35, in _wrap_generator.<locals>.generator_context(*args, **kwargs)
     32 try:
     33     # Issuing `None` to a generator fires it up
     34     with ctx_factory():
---> 35         response = gen.send(None)
     37     while True:
     38         try:
     39             # Forward the response to our caller and get its next request

File ~/.cache/huggingface/modules/transformers_modules/chatglm3-6b/modeling_chatglm.py:1069, in ChatGLMForConditionalGeneration.stream_chat(self, tokenizer, query, history, role, past_key_values, max_length, do_sample, top_p, temperature, logits_processor, return_past_key_values, **kwargs)
   1067     inputs['attention_mask'] = attention_mask
   1068 history.append({"role": role, "content": query})
-> 1069 for outputs in self.stream_generate(**inputs, past_key_values=past_key_values,
   1070                                     eos_token_id=eos_token_id, return_past_key_values=return_past_key_values,
   1071                                     **gen_kwargs):
   1072     if return_past_key_values:
   1073         outputs, past_key_values = outputs

File ~/miniconda3/envs/CG3/lib/python3.9/site-packages/torch/utils/_contextlib.py:35, in _wrap_generator.<locals>.generator_context(*args, **kwargs)
     32 try:
     33     # Issuing `None` to a generator fires it up
     34     with ctx_factory():
---> 35         response = gen.send(None)
     37     while True:
     38         try:
     39             # Forward the response to our caller and get its next request

File ~/.cache/huggingface/modules/transformers_modules/chatglm3-6b/modeling_chatglm.py:1156, in ChatGLMForConditionalGeneration.stream_generate(self, input_ids, generation_config, logits_processor, stopping_criteria, prefix_allowed_tokens_fn, return_past_key_values, **kwargs)
   1154 model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
   1155 # forward pass to get next token
-> 1156 outputs = self(
   1157     **model_inputs,
   1158     return_dict=True,
   1159     output_attentions=False,
   1160     output_hidden_states=False,
   1161 )
   1163 next_token_logits = outputs.logits[:, -1, :]
   1165 # pre-process distribution

File ~/miniconda3/envs/CG3/lib/python3.9/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File ~/.cache/huggingface/modules/transformers_modules/chatglm3-6b/modeling_chatglm.py:937, in ChatGLMForConditionalGeneration.forward(self, input_ids, position_ids, attention_mask, past_key_values, inputs_embeds, labels, use_cache, output_attentions, output_hidden_states, return_dict, return_last_logit)
    934 use_cache = use_cache if use_cache is not None else self.config.use_cache
    935 return_dict = return_dict if return_dict is not None else self.config.use_return_dict
--> 937 transformer_outputs = self.transformer(
    938     input_ids=input_ids,
    939     position_ids=position_ids,
    940     attention_mask=attention_mask,
    941     past_key_values=past_key_values,
    942     inputs_embeds=inputs_embeds,
    943     use_cache=use_cache,
    944     output_hidden_states=output_hidden_states,
    945     return_dict=return_dict,
    946 )
    948 hidden_states = transformer_outputs[0]
    949 if return_last_logit:

File ~/miniconda3/envs/CG3/lib/python3.9/site-packages/torch/nn/modules/module.py:1501, in Module._call_impl(self, *args, **kwargs)
   1496 # If we don't have any hooks, we want to skip the rest of the logic in
   1497 # this function, and just call forward.
   1498 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1499         or _global_backward_pre_hooks or _global_backward_hooks
   1500         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1501     return forward_call(*args, **kwargs)
   1502 # Do not call functions when jit is used
   1503 full_backward_hooks, non_full_backward_hooks = [], []

File ~/.cache/huggingface/modules/transformers_modules/chatglm3-6b/modeling_chatglm.py:818, in ChatGLMModel.forward(self, input_ids, position_ids, attention_mask, full_attention_mask, past_key_values, inputs_embeds, use_cache, output_hidden_states, return_dict)
    814         attention_mask = torch.cat([attention_mask.new_ones((batch_size, self.pre_seq_len)),
    815                                     attention_mask], dim=-1)
    817 if full_attention_mask is None:
--> 818     if (attention_mask is not None and not attention_mask.all()) or (past_key_values and seq_length != 1):
    819         full_attention_mask = self.get_masks(input_ids, past_key_values, padding_mask=attention_mask)
    821 # Rotary positional embeddings

RuntimeError: Now only support bool/uint8 type

使用的代码是cli_demo.py的代码，具体如下：

import os
import platform
from transformers import AutoTokenizer, AutoModel
import torch_musa
from modelscope import snapshot_download
MODEL_PATH = snapshot_download("ZhipuAI/chatglm3-6b-32k", revision="v1.0.1")
TOKENIZER_PATH = os.environ.get("TOKENIZER_PATH", MODEL_PATH)


DEVICE = 'musa'
tokenizer = AutoTokenizer.from_pretrained(TOKENIZER_PATH, trust_remote_code=True)
model = AutoModel.from_pretrained(MODEL_PATH, trust_remote_code=True).float().to(DEVICE).eval()

os_name = platform.system()
clear_command = 'cls' if os_name == 'Windows' else 'clear'
stop_stream = False

welcome_prompt = "欢迎使用 ChatGLM3-6B 模型，输入内容即可进行对话，clear 清空对话历史，stop 终止程序"


def build_prompt(history):
    prompt = welcome_prompt
    for query, response in history:
        prompt += f"\n\n用户：{query}"
        prompt += f"\n\nChatGLM3-6B：{response}"
    return prompt


def main():
    past_key_values, history = None, []
    global stop_stream
    print(welcome_prompt)
    while True:
        query = input("\n用户：")
        if query.strip() == "stop":
            break
        if query.strip() == "clear":
            past_key_values, history = None, []
            os.system(clear_command)
            print(welcome_prompt)
            continue
        print("\nChatGLM：", end="")
        current_length = 0
        for response, history, past_key_values in model.stream_chat(tokenizer, query, history=history, top_p=1,
                                                                    temperature=0.01,
                                                                    past_key_values=past_key_values,
                                                                    return_past_key_values=True):
            if stop_stream:
                stop_stream = False
                break
            else:
                print(response[current_length:], end="", flush=True)
                current_length = len(response)
        print("")

[zRzRzRzRzRzRzR]

这个卡没有进行实验，没办法测试

[1823616178]

哎，摩尔线程驱动还不行

[1823616178]

如果有进展请转告我

[tom54007]

测试微软的phi-2可以在摩尔线程上跑

[1823616178]

谢谢，有测试过使用摩尔线程的显卡训练模型吗？

[1823616178]

你好，这段时间我又使用S80尝试进行推理，一直没有找到合适的模型，使用chatglm3-6B会爆显存，使用glm-int8出现和你一样的错误，使用phi-2会莫名报错，请问你是如何运行的？是否可以分享下程序。