starlight-apk/Qwen
1
0
Fork 0
mirror of https://github.com/QwenLM/Qwen.git synced 2026-05-21 00:45:48 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #403 from QwenLM/JustinLin610-patch-2

Browse Source 
Update finetune.py
This commit is contained in:
Junyang Lin
2023-10-07 14:50:07 +08:00
committed by GitHub
parent 3876edc42b 75cc160f74
commit f0078d013b
1 changed files with 5 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
finetune.py
View File

@@ -319,6 +319,10 @@ def train():
tokenizer.pad_token_id = tokenizer.eod_id tokenizer.pad_token_id = tokenizer.eod_id
if training_args.use_lora: if training_args.use_lora:
if lora_args.q_lora:
modules_to_save = None
else:
modules_to_save = ["wte", "lm_head"]
lora_config = LoraConfig( lora_config = LoraConfig(
r=lora_args.lora_r, r=lora_args.lora_r,
lora_alpha=lora_args.lora_alpha, lora_alpha=lora_args.lora_alpha,
@@ -326,7 +330,7 @@ def train():
lora_dropout=lora_args.lora_dropout, lora_dropout=lora_args.lora_dropout,
bias=lora_args.lora_bias, bias=lora_args.lora_bias,
task_type="CAUSAL_LM", task_type="CAUSAL_LM",
modules_to_save=["wte", "lm_head"] # This argument serves for adding new tokens. modules_to_save=modules_to_save # This argument serves for adding new tokens.
) )
if lora_args.q_lora: if lora_args.q_lora:
model = prepare_model_for_kbit_training( model = prepare_model_for_kbit_training(