Skip to content

Commit

Permalink
Those evals don't exist.
Browse files Browse the repository at this point in the history
  • Loading branch information
@dirkgr
dirkgr committed Jan 17, 2025
1 parent 5715ef3 commit 64fc957
Showing 1 changed file with 0 additions and 42 deletions.
42 changes: 0 additions & 42 deletions src/scripts/train/OLMo2-1B.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,48 +76,6 @@ def build_trainer_config(common: CommonComponents) -> TrainerConfig:
cancel_check_interval=10,
),
)
.with_callback(
"downstream_evaluator",
DownstreamEvaluatorCallbackConfig(
tasks=[
"arc_challenge_val_rc_5shot",
"arc_challenge_val_mc_5shot",
"arc_challenge_test_rc_5shot",
"arc_challenge_test_mc_5shot",
"arc_easy_val_rc_5shot",
"arc_easy_val_mc_5shot",
"arc_easy_test_rc_5shot",
"arc_easy_test_mc_5shot",
"boolq_val_rc_5shot",
"boolq_val_mc_5shot",
"csqa_val_rc_5shot",
"csqa_val_mc_5shot",
"hellaswag_val_rc_5shot",
"hellaswag_val_mc_5shot",
"openbookqa_val_rc_5shot",
"openbookqa_val_mc_5shot",
"openbookqa_test_rc_5shot",
"openbookqa_test_mc_5shot",
"piqa_val_rc_5shot",
"piqa_val_mc_5shot",
"socialiqa_val_rc_5shot",
"socialiqa_val_mc_5shot",
"winogrande_val_rc_5shot",
"winogrande_val_mc_5shot",
"mmlu_stem_val_rc_5shot",
"mmlu_stem_val_mc_5shot",
"mmlu_humanities_val_rc_5shot",
"mmlu_humanities_val_mc_5shot",
"mmlu_social_sciences_val_rc_5shot",
"mmlu_social_sciences_val_mc_5shot",
"mmlu_other_val_rc_5shot",
"mmlu_other_val_mc_5shot",
],
tokenizer=common.tokenizer,
eval_batch_size=1024 * 4096,
eval_interval=1000,
),
)
)


Expand Down

0 comments on commit 64fc957

Please sign in to comment.