add gradient checkpointing

2026-05-20 16:35:47 +08:00 · 2023-09-12 23:10:59 +08:00
parent c5f7fa9487
commit 73f4040abb
4 changed files with 4 additions and 0 deletions
--- a/finetune/finetune_qlora_ds.sh
+++ b/finetune/finetune_qlora_ds.sh
@@ -46,4 +46,5 @@ torchrun $DISTRIBUTED_ARGS finetune.py \
    --lazy_preprocess True \
    --use_lora \
    --q_lora \
+    --gradient_checkpointing \
    --deepspeed finetune/ds_config_zero2.json