11# Copyright (c) Alibaba, Inc. and its affiliates.
2- from typing import Tuple
2+ from typing import Iterator , Tuple
33
44from swift .utils import get_logger , get_main
55from .infer import merge_lora , prepare_model_template
@@ -21,7 +21,7 @@ def gradio_generation_demo(args: AppUIArguments) -> None:
2121 else :
2222 model , template = prepare_model_template (args )
2323
24- def model_generation (query : str ) -> str :
24+ def model_generation (query : str ) -> Iterator [ str ] :
2525 if args .infer_backend == 'vllm' :
2626 gen = inference_stream_vllm (llm_engine , template , [{
2727 'query' : query
@@ -64,7 +64,8 @@ def gradio_chat_demo(args: AppUIArguments) -> None:
6464 else :
6565 model , template = prepare_model_template (args )
6666
67- def model_chat (query : str , history : History ) -> Tuple [str , History ]:
67+ def model_chat (query : str ,
68+ history : History ) -> Iterator [Tuple [str , History ]]:
6869 old_history , history = limit_history_length (template , query , history ,
6970 args .max_length )
7071 if args .infer_backend == 'vllm' :
0 commit comments