Skip to content

Commit

Permalink
add in loop gsm
Browse files Browse the repository at this point in the history
  • Loading branch information
davidheineman committed Jan 8, 2025
1 parent 69362b9 commit e485097
Show file tree
Hide file tree
Showing 3 changed files with 2 additions and 1 deletion.
2 changes: 1 addition & 1 deletion olmo/eval/downstream.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
log = logging.getLogger(__name__)

# Map from oe-eval metrics to metrics used here
METRIC_FROM_OE_EVAL = {"acc_raw": "acc", "acc_per_char": "len_norm", "acc_uncond": "pmi_dc"}
METRIC_FROM_OE_EVAL = {"acc_raw": "acc", "acc_per_char": "len_norm", "acc_uncond": "pmi_dc", "logits_per_byte": "bpb"}
LOG_2_OF_E = 1.44269504089


Expand Down
1 change: 1 addition & 0 deletions olmo_data/oe_eval_tasks/gsm8k/gold_bpb_5shot/config.json
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"task_name": "gsm8k", "task_hash": "c9a8b5bfa866f678c3ea4ef06729f149", "task_config": {"task_name": "gsm8k", "task_core": "gsm8k", "limit": null, "split": "test", "num_shots": 8, "fewshot_seed": 1234, "primary_metric": "logits_per_byte", "random_subsample_seed": 1234, "context_kwargs": {"no_cot": false}, "generation_kwargs": {"max_gen_toks": 512, "do_sample": false, "temperature": 0.0, "stop_sequences": ["Question:", "</s>", "<|im_end|>", "\n\n"], "repeats": 1}, "metric_kwargs": {"regexes_to_ignore": [",", "\\$", "(?s).*#### ", "\\.$"]}, "native_id_field": "id", "fewshot_source": "STD:GSM8k", "dataset_path": "gsm8k", "dataset_name": "main", "use_chat_format": null, "version": 0.1, "revision": null, "compute_gold_bpb": true, "metadata": {"alias": "gsm8k::bpb"}}, "current_date": "2025-01-08 21:03:44 UTC", "num_instances": 1319}
Binary file not shown.

0 comments on commit e485097

Please sign in to comment.