Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

increase the max token number to 2000 #3266

Merged
merged 1 commit into from
Jan 13, 2025
Merged

increase the max token number to 2000 #3266

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 8 additions & 8 deletions src/helm/benchmark/run_specs/lite_run_specs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,7 +155,7 @@ def get_mmlu_pro_spec(subject: str, use_chain_of_thought: str = "False", use_few
adapter_spec = AdapterSpec(
method=ADAPT_MULTIPLE_CHOICE_JOINT_CHAIN_OF_THOUGHT,
max_train_instances=max_train_instance_num,
max_tokens=1000,
max_tokens=2000,
input_prefix="What is the correct answer to this question: ",
input_suffix="\nChoices:\n",
output_prefix="",
Expand All @@ -179,7 +179,7 @@ def get_mmlu_pro_spec(subject: str, use_chain_of_thought: str = "False", use_few
adapter_spec = AdapterSpec(
method=ADAPT_MULTIPLE_CHOICE_JOINT,
max_train_instances=max_train_instance_num,
max_tokens=1000,
max_tokens=2000,
input_prefix="What is the correct answer to this question: ",
input_suffix="\nChoices:\n",
output_prefix="",
Expand Down Expand Up @@ -387,7 +387,7 @@ def get_gpqa_spec(subset: str, use_chain_of_thought: str = "False", use_few_shot
if use_chain_of_thought_bool:
adapter_spec = get_multiple_choice_adapter_spec(
method=ADAPT_MULTIPLE_CHOICE_JOINT_CHAIN_OF_THOUGHT,
max_tokens=1000, # following original repo
max_tokens=2000, # original: 1000
max_train_instances=max_train_instance_num,
instructions=(
"Here are some example questions from experts. "
Expand Down Expand Up @@ -426,7 +426,7 @@ def get_gpqa_spec(subset: str, use_chain_of_thought: str = "False", use_few_shot
adapter_spec = AdapterSpec(
method=ADAPT_MULTIPLE_CHOICE_JOINT_CHAIN_OF_THOUGHT,
max_train_instances=max_train_instance_num,
max_tokens=1000,
max_tokens=2000, # original: 1000
input_prefix="What is the correct answer to this question: ",
input_suffix="\nChoices:\n",
output_prefix="",
Expand All @@ -441,7 +441,7 @@ def get_gpqa_spec(subset: str, use_chain_of_thought: str = "False", use_few_shot
adapter_spec = AdapterSpec(
method=ADAPT_MULTIPLE_CHOICE_JOINT,
max_train_instances=max_train_instance_num,
max_tokens=1000,
max_tokens=2000, # original: 1000
input_prefix="What is the correct answer to this question: ",
input_suffix="\nChoices:\n",
output_prefix="",
Expand Down Expand Up @@ -475,7 +475,7 @@ def get_ifeval_spec() -> RunSpec:
scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.ifeval_scenario.IFEvalScenario")

adapter_spec = AdapterSpec(
method=ADAPT_GENERATION, input_prefix="", output_prefix="", max_tokens=1000, num_outputs=1, temperature=0.0
method=ADAPT_GENERATION, input_prefix="", output_prefix="", max_tokens=2000, num_outputs=1, temperature=0.0
)

metric_specs = [MetricSpec(class_name="helm.benchmark.metrics.ifeval_metrics.IFEvalMetric")]
Expand Down Expand Up @@ -528,7 +528,7 @@ def get_bigcodebench_spec(version: str) -> RunSpec:
method=ADAPT_GENERATION,
input_prefix="",
output_prefix="",
max_tokens=1280,
max_tokens=2000, # original: 1280
num_outputs=1,
temperature=0.0,
global_prefix="Please provide a self-contained Python script "
Expand Down Expand Up @@ -558,7 +558,7 @@ def get_omni_math_spec() -> RunSpec:
method=ADAPT_GENERATION,
input_prefix="",
output_prefix="",
max_tokens=1000,
max_tokens=2000,
num_outputs=1,
temperature=0.0,
)
Expand Down
Loading