Skip to content

Commit f913bca

Browse files
Jintao-HuangJintao-Huang
authored
Fix windows encoding gbk (#2741)
1 parent c1f10f4 commit f913bca

File tree

16 files changed

+37
-35
lines changed

16 files changed

+37
-35
lines changed

‎scripts/benchmark/exp_utils.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ def assert_gpu_not_overlap(self):
106106

107107
def run(self, exp: Experiment):
108108
if os.path.exists(os.path.join(exp.input_args.save_dir, exp.name + '.json')):
109-
with open(os.path.join(exp.input_args.save_dir, exp.name + '.json'), 'r') as f:
109+
with open(os.path.join(exp.input_args.save_dir, exp.name + '.json'), 'r', encoding='utf-8') as f:
110110
_json = json.load(f)
111111
if exp.eval_dataset and 'eval_result' not in _json['record']:
112112
if not exp.do_eval:
@@ -238,7 +238,7 @@ def _find_free_gpu(self, n):
238238
def prepare_experiments(self, args: Any):
239239
experiments = []
240240
for config_file in args.config:
241-
with open(config_file, 'r') as f:
241+
with open(config_file, 'r', encoding='utf-8') as f:
242242
group = os.path.basename(config_file)
243243
group = group[:-5]
244244
content = json.load(f)
@@ -275,7 +275,7 @@ def prepare_experiments(self, args: Any):
275275
def _get_metric(exp: Experiment):
276276
if exp.do_eval:
277277
if os.path.isfile(os.path.join('exp', f'{exp.name}.eval.log')):
278-
with open(os.path.join('exp', f'{exp.name}.eval.log'), 'r') as f:
278+
with open(os.path.join('exp', f'{exp.name}.eval.log'), 'r', encoding='utf-8') as f:
279279
for line in f.readlines():
280280
if 'Final report:' in line:
281281
return json.loads(line.split('Final report:')[1].replace('\'', '"'))
@@ -301,7 +301,7 @@ def _get_metric(exp: Experiment):
301301
logging_dir = exp.runtime.get('logging_dir')
302302
logging_file = os.path.join(logging_dir, '..', 'logging.jsonl')
303303
if os.path.isfile(logging_file):
304-
with open(logging_file, 'r') as f:
304+
with open(logging_file, 'r', encoding='utf-8') as f:
305305
for line in f.readlines():
306306
if 'model_info' in line:
307307
return json.loads(line)

‎scripts/benchmark/generate_report.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,7 @@ def generate_export_report(outputs: List[ModelOutput]):
284284

285285

286286
def parse_output(file):
287-
with open(file, 'r') as f:
287+
with open(file, 'r', encoding='utf-8') as f:
288288
content = json.load(f)
289289

290290
name = content['name']

‎scripts/utils/run_model_info.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,15 +54,15 @@ def get_model_info_table():
5454
result[i] += text[i]
5555

5656
for i, fpath in enumerate(fpaths):
57-
with open(fpath, 'r') as f:
57+
with open(fpath, 'r', encoding='utf-8') as f:
5858
text = f.read()
5959
llm_start_idx = text.find('| Model ID |')
6060
mllm_start_idx = text[llm_start_idx + 1:].find('| Model ID |') + llm_start_idx + 1
6161
llm_end_idx = text.find(end_words[i][0])
6262
mllm_end_idx = text.find(end_words[i][1])
6363
output = text[:llm_start_idx] + result[0] + '\n\n' + text[llm_end_idx:mllm_start_idx] + result[
6464
1] + '\n\n' + text[mllm_end_idx:]
65-
with open(fpath, 'w') as f:
65+
with open(fpath, 'w', encoding='utf-8') as f:
6666
f.write(output)
6767

6868

‎swift/hub/hub.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -256,7 +256,7 @@ def push_to_hub(cls,
256256
if commit_description:
257257
commit_message = commit_message + '\n' + commit_description
258258
if not os.path.exists(os.path.join(folder_path, 'configuration.json')):
259-
with open(os.path.join(folder_path, 'configuration.json'), 'w') as f:
259+
with open(os.path.join(folder_path, 'configuration.json'), 'w', encoding='utf-8') as f:
260260
f.write('{"framework": "pytorch", "task": "text-generation", "allow_remote": true}')
261261
if ignore_patterns:
262262
ignore_patterns = [p for p in ignore_patterns if p != '_*']

‎swift/llm/argument/base_args/model_args.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ def parse_to_dict(value: Union[str, Dict, None], strict: bool = True) -> Union[s
5252
value = {}
5353
elif isinstance(value, str):
5454
if os.path.exists(value): # local path
55-
with open(value, 'r') as f:
55+
with open(value, 'r', encoding='utf-8') as f:
5656
value = json.load(f)
5757
else: # json str
5858
try:

‎swift/llm/dataset/register.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -157,7 +157,7 @@ def register_dataset_info(dataset_info: Union[str, List[str], None] = None) -> L
157157
if os.path.isfile(dataset_path):
158158
log_msg = dataset_path
159159
base_dir = os.path.dirname(dataset_path)
160-
with open(dataset_path, 'r') as f:
160+
with open(dataset_path, 'r', encoding='utf-8') as f:
161161
dataset_info = json.load(f)
162162
else:
163163
dataset_info = json.loads(dataset_info) # json

‎swift/llm/export/merge_lora.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ def merge_lora(args: ExportArguments, device_map=None, replace_if_exists=False)
6060
'{base_model}', base_model)
6161
try:
6262
yamlfile = os.path.join(tempdir, 'mergekit.yaml')
63-
with open(yamlfile, 'w') as f:
63+
with open(yamlfile, 'w', encoding='utf-8') as f:
6464
f.write(merge_yaml)
6565
logger.info(f'Merging with config: {merge_yaml}')
6666
os.system(f'mergekit-yaml {yamlfile} {mergekit_path}')

‎swift/llm/export/ollama.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ def export_to_ollama(args: ExportArguments):
3636
pt_engine = PtEngine.from_model_template(model, template)
3737
logger.info(f'Using model_dir: {pt_engine.model_dir}')
3838
template_meta = template.template_meta
39-
with open(os.path.join(args.output_dir, 'Modelfile'), 'w') as f:
39+
with open(os.path.join(args.output_dir, 'Modelfile'), 'w', encoding='utf-8') as f:
4040
f.write(f'FROM {pt_engine.model_dir}\n')
4141
f.write(f'TEMPLATE """{{{{ if .System }}}}'
4242
f'{replace_and_concat(template, template_meta.system_prefix, "{{SYSTEM}}", "{{ .System }}")}'

‎swift/plugin/loss_scale.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ def __init__(self):
6969
if self.loss_scale_config is not None:
7070
path = os.path.dirname(os.path.abspath(__file__))
7171
config_path = os.path.join(path, 'agent', self.loss_scale_config)
72-
with open(config_path, 'r') as json_file:
72+
with open(config_path, 'r', encoding='utf-8') as json_file:
7373
self.loss_scale_map = json.load(json_file)
7474
else:
7575
self.loss_scale_map = None

‎swift/tuners/base.py‎

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -322,7 +322,7 @@ def from_pretrained(cls,
322322
raise ValueError(f'Please pass in a local dir or a model id, not a local file: {model_dir}')
323323
extra_state_keys = kwargs.pop('extra_state_keys', None)
324324
if extra_state_keys is None and os.path.isfile(os.path.join(model_dir, cls.EXTRA_STATE_DIR, CONFIG_NAME)):
325-
with open(os.path.join(model_dir, cls.EXTRA_STATE_DIR, CONFIG_NAME), 'r') as file:
325+
with open(os.path.join(model_dir, cls.EXTRA_STATE_DIR, CONFIG_NAME), 'r', encoding='utf-8') as file:
326326
_json = json.load(file)
327327
extra_state_keys = _json.get('extra_state_keys')
328328
if adapter_name is None:
@@ -340,7 +340,7 @@ def from_pretrained(cls,
340340
logger.warning(f'{_name} is not a valid tuner')
341341
continue
342342

343-
with open(config_file, 'r') as file:
343+
with open(config_file, 'r', encoding='utf-8') as file:
344344
json_object = json.load(file)
345345

346346
if SWIFT_TYPE_KEY not in json_object:
@@ -395,7 +395,7 @@ def create_or_update_model_card(self, output_dir: str):
395395
if not os.path.exists(os.path.join(output_dir, 'README.md')):
396396
lines = []
397397
else:
398-
with open(os.path.join(output_dir, 'README.md'), 'r') as f:
398+
with open(os.path.join(output_dir, 'README.md'), 'r', encoding='utf-8') as f:
399399
lines = f.readlines()
400400

401401
quantization_config = None
@@ -426,7 +426,7 @@ def create_or_update_model_card(self, output_dir: str):
426426
lines.append(f'{base_model_heading}\n\n- BaseModel Class {self.base_model.__class__.__name__}\n')
427427

428428
# write the lines back to README.md
429-
with open(os.path.join(output_dir, 'README.md'), 'w') as f:
429+
with open(os.path.join(output_dir, 'README.md'), 'w', encoding='utf-8') as f:
430430
f.writelines(lines)
431431

432432
def add_weighted_adapter(
@@ -587,13 +587,14 @@ def save_pretrained(self,
587587
os.makedirs(os.path.join(save_directory, self.EXTRA_STATE_DIR), exist_ok=True)
588588
self._save_state_dict(output_state_dict, os.path.join(save_directory, self.EXTRA_STATE_DIR),
589589
safe_serialization)
590-
with open(os.path.join(save_directory, self.EXTRA_STATE_DIR, CONFIG_NAME), 'w') as file:
590+
with open(
591+
os.path.join(save_directory, self.EXTRA_STATE_DIR, CONFIG_NAME), 'w', encoding='utf-8') as file:
591592
json.dump({'extra_state_keys': self.extra_state_keys}, file)
592593
else:
593594
logger.error('Full parameter training, save_extra_states will be ignored')
594595

595596
if not os.path.exists(os.path.join(save_directory, 'configuration.json')):
596-
with open(os.path.join(save_directory, 'configuration.json'), 'w') as f:
597+
with open(os.path.join(save_directory, 'configuration.json'), 'w', encoding='utf-8') as f:
597598
f.write('{}')
598599

599600
@staticmethod
@@ -776,7 +777,7 @@ def has_custom_content(_json):
776777
return not LoRAConfig(**_json).can_be_saved_to_peft()
777778

778779
for adapter in adapter_names:
779-
with open(os.path.join(ckpt_dir, adapter, CONFIG_NAME)) as f:
780+
with open(os.path.join(ckpt_dir, adapter, CONFIG_NAME), encoding='utf-8') as f:
780781
_json = json.load(f)
781782
if has_custom_content(_json):
782783
raise AssertionError('Cannot transfer to peft format, '
@@ -802,7 +803,7 @@ def has_custom_content(_json):
802803
state_dict = new_state_dict
803804
SwiftModel._save_state_dict(state_dict, os.path.join(output_dir, adapter), safe_serialization)
804805
from swift import LoRAConfig
805-
with open(os.path.join(output_dir, adapter, CONFIG_NAME)) as f:
806+
with open(os.path.join(output_dir, adapter, CONFIG_NAME), encoding='utf-8') as f:
806807
_json = json.load(f)
807808
peft_config = LoRAConfig(**_json).to_peft_config()
808809
peft_config.save_pretrained(os.path.join(output_dir, adapter))
@@ -836,7 +837,7 @@ def from_pretrained(model: Union[nn.Module, SwiftModel, PeftModel],
836837
model_id = snapshot_download(model_id, revision=revision)
837838
is_peft_model = False
838839
if os.path.exists(os.path.join(model_id, CONFIG_NAME)):
839-
with open(os.path.join(model_id, CONFIG_NAME), 'r') as f:
840+
with open(os.path.join(model_id, CONFIG_NAME), 'r', encoding='utf-8') as f:
840841
_json = json.load(f)
841842
is_peft_model = SWIFT_TYPE_KEY not in _json
842843

@@ -845,7 +846,7 @@ def from_pretrained(model: Union[nn.Module, SwiftModel, PeftModel],
845846
if isinstance(adapter_name, list) else list(adapter_name.keys())[0]
846847
_name = _name or ''
847848
if os.path.exists(os.path.join(model_id, _name, CONFIG_NAME)):
848-
with open(os.path.join(model_id, _name, CONFIG_NAME), 'r') as f:
849+
with open(os.path.join(model_id, _name, CONFIG_NAME), 'r', encoding='utf-8') as f:
849850
_json = json.load(f)
850851
is_peft_model = SWIFT_TYPE_KEY not in _json and 'extra_state_keys' not in _json
851852
if is_peft_model:

0 commit comments

Comments
 (0)