From 00dceebbaaec2bf642ad327f7022ac9b6f8da187 Mon Sep 17 00:00:00 2001
From: Junyang Lin <justinlin930319@hotmail.com>
Date: Mon, 9 Oct 2023 01:11:24 +0800
Subject: [PATCH] Update finetune.py

---
 finetune.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/finetune.py b/finetune.py
index cea3fab..a4fee87 100644
--- a/finetune.py
+++ b/finetune.py
@@ -98,6 +98,7 @@ def get_peft_state_maybe_zero_3(named_params, bias):
     to_return = {k: maybe_zero_3(v) for k, v in to_return.items()}
     return to_return
 
+
 local_rank = None
 
 def rank0_print(*args):
@@ -266,7 +267,7 @@ def train():
     ) = parser.parse_args_into_dataclasses()
 
     # This serves for single-gpu qlora.
-    if getattr(training_args, 'deepspeed', None) and getattr(lora_args, 'q_lora', False):
+    if getattr(training_args, 'deepspeed', None) and int(os.environ.get("WORLD_SIZE", 1))==1:
         training_args.distributed_state.distributed_type = DistributedType.DEEPSPEED
 
     compute_dtype = (