From d1b236ca50cd66b7d3ab94aed7a23990efe5c34d Mon Sep 17 00:00:00 2001 From: Michael Waskom Date: Tue, 6 Feb 2024 14:30:57 -0500 Subject: [PATCH] Update deepspeed config location --- config/codellama.yml | 4 ++-- config/mistral.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/codellama.yml b/config/codellama.yml index e204b9c..32ed3dc 100644 --- a/config/codellama.yml +++ b/config/codellama.yml @@ -54,7 +54,7 @@ learning_rate: 0.0002 train_on_inputs: false group_by_length: false -bf16: true +bf16: auto fp16: false tf32: false @@ -71,7 +71,7 @@ warmup_steps: 10 eval_steps: 0.05 save_steps: debug: True -deepspeed: /root/axolotl/deepspeed/zero3.json +deepspeed: /root/axolotl/deepspeed_configs/zero3_bf16.json weight_decay: 0.0 fsdp: fsdp_config: diff --git a/config/mistral.yml b/config/mistral.yml index 885825c..b2d76d0 100644 --- a/config/mistral.yml +++ b/config/mistral.yml @@ -52,7 +52,7 @@ optimizer: adamw_bnb_8bit lr_scheduler: cosine learning_rate: 0.0002 -bf16: true +bf16: auto fp16: false tf32: false train_on_inputs: false @@ -69,7 +69,7 @@ flash_attention: false warmup_steps: 10 save_steps: debug: -deepspeed: /root/axolotl/deepspeed/zero3.json +deepspeed: /root/axolotl/deepspeed_configs/zero3_bf16.json weight_decay: 0.0 fsdp: fsdp_config: