Try simple torch optimizer

modal-labs · Feb 7, 2024 · 791a5af · 791a5af
1 parent e855a8b
commit 791a5af
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/config/codellama.yml b/config/codellama.yml
@@ -48,7 +48,7 @@ wandb_run_id:
 gradient_accumulation_steps: 1
 micro_batch_size: 32
 num_epochs: 4
-optimizer: adamw_bnb_8bit
+optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 0.0001
 

diff --git a/config/llama-2.yml b/config/llama-2.yml
@@ -48,7 +48,7 @@ wandb_run_id:
 gradient_accumulation_steps: 1
 micro_batch_size: 32
 num_epochs: 4
-optimizer: adamw_bnb_8bit
+optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 0.0001
 

diff --git a/config/mistral.yml b/config/mistral.yml
@@ -48,7 +48,7 @@ wandb_run_id:
 gradient_accumulation_steps: 1
 micro_batch_size: 32
 num_epochs: 4
-optimizer: adamw_bnb_8bit
+optimizer: adamw_torch
 lr_scheduler: cosine
 learning_rate: 0.0001