Skip to content

Commit e78dcdb

Browse files
Jintao-HuangJintao-Huang
authored
update readme and fix bug (#167)
1 parent 03b8d9e commit e78dcdb

File tree

10 files changed

+258
-34
lines changed

10 files changed

+258
-34
lines changed

‎README.md‎

Lines changed: 60 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ To see more sh startup scripts, please refer to: [Run SFT and Inference](https:/
115115
```bash
116116
git clone https://github.com/modelscope/swift.git
117117
cd swift
118-
pip install .
118+
pip install -e .
119119
```
120120

121121

@@ -141,19 +141,74 @@ sft_args = SftArguments(
141141
dataset=[DatasetName.blossom_math_zh],
142142
output_dir='output',
143143
gradient_checkpointing=True)
144-
best_ckpt_dir = sft_main(sft_args)['best_model_checkpoint']
145-
print(f'best_ckpt_dir: {best_ckpt_dir}')
144+
result = sft_main(sft_args)
145+
best_model_checkpoint = result['best_model_checkpoint']
146+
print(f'best_model_checkpoint: {best_model_checkpoint}')
146147
torch.cuda.empty_cache()
148+
147149
infer_args = InferArguments(
148-
ckpt_dir=best_ckpt_dir,
150+
ckpt_dir=best_model_checkpoint,
149151
load_args_from_ckpt_dir=True,
150152
stream=True,
151153
show_dataset_sample=5)
152-
infer_main(infer_args)
154+
result = infer_main(infer_args)
155+
print(f'result: {result}')
153156
torch.cuda.empty_cache()
157+
154158
web_ui_main(infer_args)
155159
```
156160

161+
**Single-Sample Inference**:
162+
163+
Inference using LoRA **incremental** weights:
164+
```python
165+
import os
166+
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
167+
168+
from swift.llm import (
169+
get_model_tokenizer, get_template, inference, ModelType, get_default_template_type
170+
)
171+
from swift.tuners import Swift
172+
import torch
173+
174+
model_dir = 'vx_xxx/checkpoint-100'
175+
model_type = ModelType.qwen_7b_chat
176+
template_type = get_default_template_type(model_type)
177+
178+
model, tokenizer = get_model_tokenizer(model_type, torch.bfloat16, {'device_map': 'auto'})
179+
180+
model = Swift.from_pretrained(model, model_dir, inference_mode=True)
181+
template = get_template(template_type, tokenizer)
182+
query = 'xxxxxx'
183+
response, history = inference(model, template, query, verbose=False)
184+
print(f'response: {response}')
185+
print(f'history: {history}')
186+
```
187+
188+
Inference using LoRA **merged** complete weights:
189+
```python
190+
import os
191+
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
192+
193+
from swift.llm import (
194+
get_model_tokenizer, get_template, inference, ModelType, get_default_template_type
195+
)
196+
import torch
197+
198+
model_dir = 'vx_xxx/checkpoint-100-merged'
199+
model_type = ModelType.qwen_7b_chat
200+
template_type = get_default_template_type(model_type)
201+
202+
model, tokenizer = get_model_tokenizer(model_type, torch.bfloat16, {'device_map': 'auto'},
203+
model_dir=model_dir)
204+
205+
template = get_template(template_type, tokenizer)
206+
query = 'xxxxxx'
207+
response, history = inference(model, template, query, verbose=False)
208+
print(f'response: {response}')
209+
print(f'history: {history}')
210+
```
211+
157212
#### Run using Swift CLI
158213
**SFT**:
159214
```bash

‎README_CN.md‎

Lines changed: 60 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ LLM微调的详细使用文档可以查看[这里](https://github.com/modelscope
113113
```bash
114114
git clone https://github.com/modelscope/swift.git
115115
cd swift
116-
pip install .
116+
pip install -e .
117117
```
118118

119119
#### 使用python运行
@@ -138,19 +138,74 @@ sft_args = SftArguments(
138138
dataset=[DatasetName.blossom_math_zh],
139139
output_dir='output',
140140
gradient_checkpointing=True)
141-
best_ckpt_dir = sft_main(sft_args)['best_model_checkpoint']
142-
print(f'best_ckpt_dir: {best_ckpt_dir}')
141+
result = sft_main(sft_args)
142+
best_model_checkpoint = result['best_model_checkpoint']
143+
print(f'best_model_checkpoint: {best_model_checkpoint}')
143144
torch.cuda.empty_cache()
145+
144146
infer_args = InferArguments(
145-
ckpt_dir=best_ckpt_dir,
147+
ckpt_dir=best_model_checkpoint,
146148
load_args_from_ckpt_dir=True,
147149
stream=True,
148150
show_dataset_sample=5)
149-
infer_main(infer_args)
151+
result = infer_main(infer_args)
152+
print(f'result: {result}')
150153
torch.cuda.empty_cache()
154+
151155
web_ui_main(infer_args)
152156
```
153157

158+
**单样本推理**:
159+
160+
使用LoRA**增量**权重进行推理:
161+
```python
162+
import os
163+
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
164+
165+
from swift.llm import (
166+
get_model_tokenizer, get_template, inference, ModelType, get_default_template_type
167+
)
168+
from swift.tuners import Swift
169+
import torch
170+
171+
model_dir = 'vx_xxx/checkpoint-100'
172+
model_type = ModelType.qwen_7b_chat
173+
template_type = get_default_template_type(model_type)
174+
175+
model, tokenizer = get_model_tokenizer(model_type, torch.bfloat16, {'device_map': 'auto'})
176+
177+
model = Swift.from_pretrained(model, model_dir, inference_mode=True)
178+
template = get_template(template_type, tokenizer)
179+
query = 'xxxxxx'
180+
response, history = inference(model, template, query, verbose=False)
181+
print(f'response: {response}')
182+
print(f'history: {history}')
183+
```
184+
185+
使用LoRA **merge**后完整的权重进行推理:
186+
```python
187+
import os
188+
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
189+
190+
from swift.llm import (
191+
get_model_tokenizer, get_template, inference, ModelType, get_default_template_type
192+
)
193+
import torch
194+
195+
model_dir = 'vx_xxx/checkpoint-100-merged'
196+
model_type = ModelType.qwen_7b_chat
197+
template_type = get_default_template_type(model_type)
198+
199+
model, tokenizer = get_model_tokenizer(model_type, torch.bfloat16, {'device_map': 'auto'},
200+
model_dir=model_dir)
201+
202+
template = get_template(template_type, tokenizer)
203+
query = 'xxxxxx'
204+
response, history = inference(model, template, query, verbose=False)
205+
print(f'response: {response}')
206+
print(f'history: {history}')
207+
```
208+
154209
#### 使用Swift CLI运行
155210
**微调**:
156211
```bash

‎examples/pytorch/llm/README.md‎

Lines changed: 61 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ Experimental environment: A10, 3090, V100, A100, ...
6161
pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/
6262
git clone https://github.com/modelscope/swift.git
6363
cd swift
64-
pip install .
64+
pip install -e .
6565
# The following script needs to be executed in this directory.
6666
cd examples/pytorch/llm
6767

@@ -104,19 +104,74 @@ sft_args = SftArguments(
104104
dataset=[DatasetName.blossom_math_zh],
105105
output_dir='output',
106106
gradient_checkpointing=True)
107-
best_ckpt_dir = sft_main(sft_args)['best_model_checkpoint']
108-
print(f'best_ckpt_dir: {best_ckpt_dir}')
107+
result = sft_main(sft_args)
108+
best_model_checkpoint = result['best_model_checkpoint']
109+
print(f'best_model_checkpoint: {best_model_checkpoint}')
109110
torch.cuda.empty_cache()
111+
110112
infer_args = InferArguments(
111-
ckpt_dir=best_ckpt_dir,
113+
ckpt_dir=best_model_checkpoint,
112114
load_args_from_ckpt_dir=True,
113115
stream=True,
114-
val_dataset_sample=5)
115-
infer_main(infer_args)
116+
show_dataset_sample=5)
117+
result = infer_main(infer_args)
118+
print(f'result: {result}')
116119
torch.cuda.empty_cache()
120+
117121
web_ui_main(infer_args)
118122
```
119123

124+
**Single-Sample Inference**:
125+
126+
Inference using LoRA **incremental** weights:
127+
```python
128+
import os
129+
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
130+
131+
from swift.llm import (
132+
get_model_tokenizer, get_template, inference, ModelType, get_default_template_type
133+
)
134+
from swift.tuners import Swift
135+
import torch
136+
137+
model_dir = 'vx_xxx/checkpoint-100'
138+
model_type = ModelType.qwen_7b_chat
139+
template_type = get_default_template_type(model_type)
140+
141+
model, tokenizer = get_model_tokenizer(model_type, torch.bfloat16, {'device_map': 'auto'})
142+
143+
model = Swift.from_pretrained(model, model_dir, inference_mode=True)
144+
template = get_template(template_type, tokenizer)
145+
query = 'xxxxxx'
146+
response, history = inference(model, template, query, verbose=False)
147+
print(f'response: {response}')
148+
print(f'history: {history}')
149+
```
150+
151+
Inference using LoRA **merged** complete weights:
152+
```python
153+
import os
154+
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
155+
156+
from swift.llm import (
157+
get_model_tokenizer, get_template, inference, ModelType, get_default_template_type
158+
)
159+
import torch
160+
161+
model_dir = 'vx_xxx/checkpoint-100-merged'
162+
model_type = ModelType.qwen_7b_chat
163+
template_type = get_default_template_type(model_type)
164+
165+
model, tokenizer = get_model_tokenizer(model_type, torch.bfloat16, {'device_map': 'auto'},
166+
model_dir=model_dir)
167+
168+
template = get_template(template_type, tokenizer)
169+
query = 'xxxxxx'
170+
response, history = inference(model, template, query, verbose=False)
171+
print(f'response: {response}')
172+
print(f'history: {history}')
173+
```
174+
120175
### Run using Swift CLI
121176
**SFT**:
122177
```bash

‎examples/pytorch/llm/README_CN.md‎

Lines changed: 61 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@
6060
pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/
6161
git clone https://github.com/modelscope/swift.git
6262
cd swift
63-
pip install .
63+
pip install -e .
6464
# 下面的脚本需要在此目录下执行
6565
cd examples/pytorch/llm
6666

@@ -103,19 +103,74 @@ sft_args = SftArguments(
103103
dataset=[DatasetName.blossom_math_zh],
104104
output_dir='output',
105105
gradient_checkpointing=True)
106-
best_ckpt_dir = sft_main(sft_args)['best_model_checkpoint']
107-
print(f'best_ckpt_dir: {best_ckpt_dir}')
106+
result = sft_main(sft_args)
107+
best_model_checkpoint = result['best_model_checkpoint']
108+
print(f'best_model_checkpoint: {best_model_checkpoint}')
108109
torch.cuda.empty_cache()
110+
109111
infer_args = InferArguments(
110-
ckpt_dir=best_ckpt_dir,
112+
ckpt_dir=best_model_checkpoint,
111113
load_args_from_ckpt_dir=True,
112114
stream=True,
113-
val_dataset_sample=5)
114-
infer_main(infer_args)
115+
show_dataset_sample=5)
116+
result = infer_main(infer_args)
117+
print(f'result: {result}')
115118
torch.cuda.empty_cache()
119+
116120
web_ui_main(infer_args)
117121
```
118122

123+
**单样本推理**:
124+
125+
使用LoRA**增量**权重进行推理:
126+
```python
127+
import os
128+
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
129+
130+
from swift.llm import (
131+
get_model_tokenizer, get_template, inference, ModelType, get_default_template_type
132+
)
133+
from swift.tuners import Swift
134+
import torch
135+
136+
model_dir = 'vx_xxx/checkpoint-100'
137+
model_type = ModelType.qwen_7b_chat
138+
template_type = get_default_template_type(model_type)
139+
140+
model, tokenizer = get_model_tokenizer(model_type, torch.bfloat16, {'device_map': 'auto'})
141+
142+
model = Swift.from_pretrained(model, model_dir, inference_mode=True)
143+
template = get_template(template_type, tokenizer)
144+
query = 'xxxxxx'
145+
response, history = inference(model, template, query, verbose=False)
146+
print(f'response: {response}')
147+
print(f'history: {history}')
148+
```
149+
150+
使用LoRA **merge**后完整的权重进行推理:
151+
```python
152+
import os
153+
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
154+
155+
from swift.llm import (
156+
get_model_tokenizer, get_template, inference, ModelType, get_default_template_type
157+
)
158+
import torch
159+
160+
model_dir = 'vx_xxx/checkpoint-100-merged'
161+
model_type = ModelType.qwen_7b_chat
162+
template_type = get_default_template_type(model_type)
163+
164+
model, tokenizer = get_model_tokenizer(model_type, torch.bfloat16, {'device_map': 'auto'},
165+
model_dir=model_dir)
166+
167+
template = get_template(template_type, tokenizer)
168+
query = 'xxxxxx'
169+
response, history = inference(model, template, query, verbose=False)
170+
print(f'response: {response}')
171+
print(f'history: {history}')
172+
```
173+
119174
### 使用Swift CLI运行
120175
**微调**:
121176
```bash

‎swift/llm/infer.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -153,11 +153,10 @@ def llm_infer(args: InferArguments) -> None:
153153
_, val_dataset = get_dataset(args.dataset, args.dataset_test_ratio,
154154
args.dataset_seed)
155155
if args.val_dataset_sample >= 0:
156-
mini_val_dataset = val_dataset.select(
156+
val_dataset = val_dataset.select(
157157
range(min(args.val_dataset_sample, val_dataset.shape[0])))
158-
else:
159-
mini_val_dataset = val_dataset
160-
for data in mini_val_dataset:
158+
logger.info(f'val_dataset: {val_dataset}')
159+
for data in val_dataset:
161160
_, history = inference(
162161
model,
163162
template,

‎swift/llm/utils/__init__.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
from .dataset import (DATASET_MAPPING, DatasetName, GetDatasetFunction,
44
get_dataset, get_dataset_from_repo, register_dataset)
55
from .model import (MODEL_MAPPING, GetModelTokenizerFunction, LoRATM,
6-
ModelType, get_model_tokenizer,
6+
ModelType, get_default_template_type, get_model_tokenizer,
77
get_model_tokenizer_from_repo,
88
get_model_tokenizer_from_sdk, register_model)
99
from .preprocess import (AlpacaPreprocessor, ClsPreprocessor,

‎swift/llm/utils/argument.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -538,7 +538,7 @@ def register_custom_dataset(args: Union[SftArguments, InferArguments]) -> None:
538538
get_function=get_custom_dataset)
539539
if args.dataset is None:
540540
args.dataset = ['_custom_dataset']
541-
else:
541+
elif '_custom_dataset' not in args.dataset:
542542
args.dataset.append('_custom_dataset')
543543

544544

‎swift/llm/utils/model.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -905,3 +905,7 @@ def get_model_tokenizer(
905905
model.generation_config = GenerationConfig.from_pretrained(
906906
model_dir)
907907
return model, tokenizer
908+
909+
910+
def get_default_template_type(model_type: str) -> Optional[str]:
911+
return MODEL_MAPPING[model_type].get('template')

0 commit comments

Comments
 (0)