1. support deepspeed on ui 2. add tools to client_utils (#1446)

Author: tastelikefeet

swift/llm/utils/client_utils.py

@@ -168,6 +168,8 @@ def _pre_inference_client(model_type: str,
                           history: Optional[History] = None,
                           system: Optional[str] = None,
                           images: Optional[List[str]] = None,
+                          tools: Optional[List[Dict[str, Union[str, Dict]]]] = None,
+                          tool_choice: Optional[Union[str, Dict]] = 'auto',
                           *,
                           is_chat_request: Optional[bool] = None,
                           request_config: Optional[XRequestConfig] = None,
@@ -212,7 +214,10 @@ def _pre_inference_client(model_type: str,
     data['model'] = model_type
     if len(images) > 0:
         data['images'] = images
-
+    if tools and len(tools) > 0:
+        data['tools'] = tools
+    if tool_choice:
+        data['tool_choice'] = tool_choice
     return url, data, is_chat_request
 
 
@@ -222,6 +227,8 @@ def inference_client(
     history: Optional[History] = None,
     system: Optional[str] = None,
     images: Optional[List[str]] = None,
+    tools: Optional[List[Dict[str, Union[str, Dict]]]] = None,
+    tool_choice: Optional[Union[str, Dict]] = 'auto',
     *,
     is_chat_request: Optional[bool] = None,
     request_config: Optional[XRequestConfig] = None,
@@ -238,6 +245,8 @@ def inference_client(
         history,
         system,
         images,
+        tools,
+        tool_choice,
         is_chat_request=is_chat_request,
         request_config=request_config,
         host=host,
@@ -280,6 +289,8 @@ async def inference_client_async(
     history: Optional[History] = None,
     system: Optional[str] = None,
     images: Optional[List[str]] = None,
+    tools: Optional[List[Dict[str, Union[str, Dict]]]] = None,
+    tool_choice: Optional[Union[str, Dict]] = 'auto',
     *,
     is_chat_request: Optional[bool] = None,
     request_config: Optional[XRequestConfig] = None,
@@ -296,6 +307,8 @@ async def inference_client_async(
         history,
         system,
         images,
+        tools,
+        tool_choice,
         is_chat_request=is_chat_request,
         request_config=request_config,
         host=host,

swift/trainers/optimizers/galore/utils.py

@@ -187,7 +187,7 @@ def get_optimizer(args: TrainingArguments, config: GaLoreConfig) -> Tuple[Any, A
         optimizer_kwargs.update({'scale_parameter': False, 'relative_step': False})
     elif args.optim in ('adamw_hf', 'adamw_torch'):
         if config.quantize:
-            assert importlib.util.find_spec("q_galore_torch") is not None, \
+            assert importlib.util.find_spec('q_galore_torch') is not None, \
                 'Please install q-galore by `pip install q_galore_torch`'
             from swift.utils import get_dist_setting
             _, _, world_size, _ = get_dist_setting()

swift/ui/llm_eval/llm_eval.py

@@ -10,6 +10,7 @@
 import json
 import torch
 from gradio import Accordion, Tab
+from json import JSONDecodeError
 from modelscope import snapshot_download
 
 from swift.llm import EvalArguments
@@ -39,8 +40,8 @@ class LLMEval(BaseUI):
                 'en': 'More params'
             },
             'info': {
-                'zh': '以json格式填入',
-                'en': 'Fill in with json format'
+                'zh': '以json格式或--xxx xxx命令行格式填入',
+                'en': 'Fill in with json format or --xxx xxx cmd format'
             }
         },
         'evaluate': {
@@ -113,6 +114,7 @@ def eval(cls, *args):
         kwargs_is_list = {}
         other_kwargs = {}
         more_params = {}
+        more_params_cmd = ''
         keys = [key for key, value in cls.elements().items() if not isinstance(value, (Tab, Accordion))]
         for key, value in zip(keys, args):
             compare_value = eval_args.get(key)
@@ -130,7 +132,10 @@ def eval(cls, *args):
             else:
                 other_kwargs[key] = value
             if key == 'more_params' and value:
-                more_params = json.loads(value)
+                try:
+                    more_params = json.loads(value)
+                except (JSONDecodeError or TypeError):
+                    more_params_cmd = value
 
         kwargs.update(more_params)
         if kwargs['model_type'] == cls.locale('checkpoint', cls.lang)['value']:
@@ -152,6 +157,7 @@ def eval(cls, *args):
                 params += f'--{e} {kwargs[e]} '
             else:
                 params += f'--{e} "{kwargs[e]}" '
+        params += more_params_cmd + ' '
         devices = other_kwargs['gpu_id']
         devices = [d for d in devices if d]
         assert (len(devices) == 1 or 'cpu' not in devices)

swift/ui/llm_export/llm_export.py

@@ -10,6 +10,7 @@
 import json
 import torch
 from gradio import Accordion, Tab
+from json import JSONDecodeError
 from modelscope import snapshot_download
 
 from swift.llm import ExportArguments
@@ -37,8 +38,8 @@ class LLMExport(BaseUI):
                 'en': 'More params'
             },
             'info': {
-                'zh': '以json格式填入',
-                'en': 'Fill in with json format'
+                'zh': '以json格式或--xxx xxx命令行格式填入',
+                'en': 'Fill in with json format or --xxx xxx cmd format'
             }
         },
         'export': {
@@ -111,6 +112,7 @@ def export(cls, *args):
         kwargs_is_list = {}
         other_kwargs = {}
         more_params = {}
+        more_params_cmd = ''
         keys = [key for key, value in cls.elements().items() if not isinstance(value, (Tab, Accordion))]
         for key, value in zip(keys, args):
             compare_value = export_args.get(key)
@@ -128,7 +130,10 @@ def export(cls, *args):
             else:
                 other_kwargs[key] = value
             if key == 'more_params' and value:
-                more_params = json.loads(value)
+                try:
+                    more_params = json.loads(value)
+                except (JSONDecodeError or TypeError):
+                    more_params_cmd = value
 
         kwargs.update(more_params)
         if kwargs['model_type'] == cls.locale('checkpoint', cls.lang)['value']:
@@ -151,6 +156,7 @@ def export(cls, *args):
                 params += f'--{e} {kwargs[e]} '
             else:
                 params += f'--{e} "{kwargs[e]}" '
+        params += more_params_cmd + ' '
         devices = other_kwargs['gpu_id']
         devices = [d for d in devices if d]
         assert (len(devices) == 1 or 'cpu' not in devices)

swift/ui/llm_infer/llm_infer.py

@@ -10,6 +10,7 @@
 import json
 import torch
 from gradio import Accordion, Tab
+from json import JSONDecodeError
 from modelscope import GenerationConfig, snapshot_download
 
 from swift.llm import (TEMPLATE_MAPPING, DeployArguments, InferArguments, XRequestConfig, inference_client,
@@ -215,6 +216,7 @@ def deploy(cls, *args):
         kwargs_is_list = {}
         other_kwargs = {}
         more_params = {}
+        more_params_cmd = ''
         keys = [key for key, value in cls.elements().items() if not isinstance(value, (Tab, Accordion))]
         for key, value in zip(keys, args):
             compare_value = deploy_args.get(key)
@@ -232,7 +234,10 @@ def deploy(cls, *args):
             else:
                 other_kwargs[key] = value
             if key == 'more_params' and value:
-                more_params = json.loads(value)
+                try:
+                    more_params = json.loads(value)
+                except (JSONDecodeError or TypeError):
+                    more_params_cmd = value
 
         kwargs.update(more_params)
         if kwargs['model_type'] == cls.locale('checkpoint', cls.lang)['value']:
@@ -263,6 +268,7 @@ def deploy(cls, *args):
                 params += f'--{e} "{kwargs[e]}" '
         if 'port' not in kwargs:
             params += f'--port "{deploy_args.port}" '
+        params += more_params_cmd + ' '
         devices = other_kwargs['gpu_id']
         devices = [d for d in devices if d]
         assert (len(devices) == 1 or 'cpu' not in devices)

swift/ui/llm_infer/model.py

@@ -96,8 +96,8 @@ class Model(BaseUI):
                 'en': 'More params'
             },
             'info': {
-                'zh': '以json格式填入',
-                'en': 'Fill in with json format'
+                'zh': '以json格式或--xxx xxx命令行格式填入',
+                'en': 'Fill in with json format or --xxx xxx cmd format'
             }
         },
         'reset': {

swift/ui/llm_train/advanced.py

@@ -72,8 +72,8 @@ class Advanced(BaseUI):
                 'en': 'Other params'
             },
             'info': {
-                'zh': '以json格式输入其他超参数',
-                'en': 'Input in the json format'
+                'zh': '以json格式或--xxx xxx命令行格式填入',
+                'en': 'Fill in with json format or --xxx xxx cmd format'
             }
         },
         'custom_train_dataset_path': {
@@ -156,6 +156,16 @@ class Advanced(BaseUI):
                 'en': 'Use model.generate/Rouge instead of loss',
             }
         },
+        'deepspeed': {
+            'label': {
+                'zh': 'deepspeed',
+                'en': 'deepspeed',
+            },
+            'info': {
+                'zh': '可以选择下拉列表，也支持传入路径',
+                'en': 'Choose from the dropbox or fill in a valid path',
+            }
+        },
     }
 
     @classmethod
@@ -177,6 +187,11 @@ def do_build_ui(cls, base_tab: Type['BaseUI']):
                     gr.Textbox(elem_id='max_grad_norm', lines=1, scale=20)
                     gr.Checkbox(elem_id='predict_with_generate', scale=20)
                 with gr.Row():
+                    gr.Dropdown(
+                        elem_id='deepspeed',
+                        scale=4,
+                        allow_custom_value=True,
+                        choices=['default-zero2', 'default-zero3', 'zero3-offload'])
                     gr.Textbox(elem_id='gpu_memory_fraction', scale=4)
                 with gr.Row():
                     gr.Textbox(elem_id='more_params', lines=4, scale=20)

swift/ui/llm_train/llm_train.py

@@ -11,6 +11,7 @@
 import json
 import torch
 from gradio import Accordion, Tab
+from json import JSONDecodeError
 
 from swift.llm import RLHFArguments
 from swift.ui.base import BaseUI
@@ -294,6 +295,7 @@ def train(cls, *args):
         kwargs_is_list = {}
         other_kwargs = {}
         more_params = {}
+        more_params_cmd = ''
         keys = [key for key, value in cls.elements().items() if not isinstance(value, (Tab, Accordion))]
         model_type = None
         do_rlhf = False
@@ -311,7 +313,10 @@ def train(cls, *args):
             else:
                 other_kwargs[key] = value
             if key == 'more_params' and value:
-                more_params = json.loads(value)
+                try:
+                    more_params = json.loads(value)
+                except (JSONDecodeError or TypeError):
+                    more_params_cmd = value
 
             if key == 'model_type':
                 model_type = value
@@ -327,6 +332,8 @@ def train(cls, *args):
             raise gr.Error(cls.locale('dataset_alert', cls.lang)['value'])
 
         cmd = 'rlhf' if do_rlhf else 'sft'
+        if kwargs.get('deepspeed'):
+            more_params_cmd += f' --deepspeed {kwargs.pop("deepspeed")} '
         sft_args = RLHFArguments(
             **{
                 key: value.split(' ') if kwargs_is_list.get(key, False) and isinstance(value, str) else value
@@ -341,6 +348,7 @@ def train(cls, *args):
                 params += f'--{e} {kwargs[e]} '
             else:
                 params += f'--{e} "{kwargs[e]}" '
+        params += more_params_cmd + ' '
         params += f'--add_output_dir_suffix False --output_dir {sft_args.output_dir} ' \
                   f'--logging_dir {sft_args.logging_dir} --ignore_args_error True'
         ddp_param = ''