starlight-apk/Qwen
1
0
Fork 0
mirror of https://github.com/QwenLM/Qwen.git synced 2026-05-21 00:45:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update finetune.py

Browse Source
This commit is contained in:
Junyang Lin
2023-10-09 01:11:24 +08:00
committed by GitHub
parent ce1ca46099
commit 00dceebbaa
1 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
finetune.py
View File

@@ -98,6 +98,7 @@ def get_peft_state_maybe_zero_3(named_params, bias):
to_return = {k: maybe_zero_3(v) for k, v in to_return.items()} to_return = {k: maybe_zero_3(v) for k, v in to_return.items()}
return to_return return to_return
local_rank = None local_rank = None
def rank0_print(*args): def rank0_print(*args):
@@ -266,7 +267,7 @@ def train():
) = parser.parse_args_into_dataclasses() ) = parser.parse_args_into_dataclasses()
# This serves for single-gpu qlora. # This serves for single-gpu qlora.
if getattr(training_args, 'deepspeed', None) and getattr(lora_args, 'q_lora', False): if getattr(training_args, 'deepspeed', None) and int(os.environ.get("WORLD_SIZE", 1))==1:
training_args.distributed_state.distributed_type = DistributedType.DEEPSPEED training_args.distributed_state.distributed_type = DistributedType.DEEPSPEED
compute_dtype = ( compute_dtype = (