Auto-merge updates from master branch

mlcommons · Jan 7, 2025 · 787282d · 787282d
2 parents b8074ef + e0d093d
commit 787282d
Show file tree

Hide file tree

Showing 6 changed files with 7 additions and 19 deletions.
diff --git a/language/llama3.1-405b/evaluate-accuracy.py b/language/llama3.1-405b/evaluate-accuracy.py
@@ -138,6 +138,7 @@ def main():
     dataset_path = args.dataset_file
     checkpoint_path = args.checkpoint_path
     nltk.download("punkt")
+    nltk.download('punkt_tab')
 
     tokenizer = AutoTokenizer.from_pretrained(
         checkpoint_path,

diff --git a/language/mixtral-8x7b/README.md b/language/mixtral-8x7b/README.md
@@ -13,11 +13,6 @@ Please see the [new docs site](https://docs.mlcommons.org/inference/benchmarks/l
 
 ## Prepare environment
 
-Copy the mlperf.conf file to this folder.
-```
-cp ../../mlperf.conf .
-```
-
 For a CPU-only run:
 
 ```
@@ -136,7 +131,6 @@ wget https://inference.mlcommons-storage.org/mixtral_8x7b%2F2024.06.06_mixtral_1
 ```
 python -u main.py --scenario Offline \
                 --model-path ${CHECKPOINT_PATH} \
-                --mlperf-conf mlperf.conf \
                 --user-conf user.conf \
                 --total-sample-count 15000 \
                 --device cpu \
@@ -149,7 +143,6 @@ For a GPU-based run:
 ```
 python3 -u main.py --scenario Offline \
         --model-path ${CHECKPOINT_PATH} \
-        --mlperf-conf mlperf.conf \
         --user-conf user.conf \
         --total-sample-count 15000 \
         --dataset-path ${DATASET_PATH} \
@@ -162,7 +155,6 @@ python3 -u main.py --scenario Offline \
 ```
 python -u main.py --scenario Server \
                 --model-path ${CHECKPOINT_PATH} \
-                --mlperf-conf mlperf.conf \
                 --user-conf user.conf \
                 --total-sample-count 15000 \
                 --device cpu \
@@ -184,7 +176,6 @@ mkdir -p "run_outputs"  # The script will dump all the outputs to 'run_outputs'.
 python -u main.py --scenario Offline \
                 --model-path ${CHECKPOINT_PATH} \
                 --accuracy \
-                --mlperf-conf mlperf.conf \
                 --user-conf user.conf \
                 --total-sample-count 15000 \
                 --dataset-path ${DATASET_PATH} \
@@ -221,7 +212,6 @@ OUTPUT_LOG_DIR=server-accuracy-logs
 python -u main.py --scenario Server \
                 --model-path ${CHECKPOINT_PATH} \
                 --accuracy \
-                --mlperf-conf mlperf.conf \
                 --user-conf user.conf \
                 --total-sample-count 15000 \
                 --dataset-path ${DATASET_PATH} \

diff --git a/language/mixtral-8x7b/run_accuracy.sh b/language/mixtral-8x7b/run_accuracy.sh
@@ -6,7 +6,6 @@ mkdir -p "run_outputs"
 python3 -u main.py --scenario Offline \
         --model-path ${CHECKPOINT_PATH} \
         --accuracy \
-        --mlperf-conf mlperf.conf \
         --user-conf user.conf \
         --total-sample-count 15000 \
         --dataset-path ${DATASET_PATH} \

diff --git a/language/mixtral-8x7b/run_offline.sh b/language/mixtral-8x7b/run_offline.sh
@@ -3,7 +3,6 @@ DATASET_PATH="${DATASET_PATH:dataset/2024_06_06_mixtral_15k_v4.pkl}"
 
 python -u main.py --scenario Offline \
 		--model-path ${CHECKPOINT_PATH} \
-		--mlperf-conf mlperf.conf \
 		--user-conf user.conf \
 		--total-sample-count 15000 \
 		--dataset-path ${DATASET_PATH} \

diff --git a/language/mixtral-8x7b/run_server.sh b/language/mixtral-8x7b/run_server.sh
@@ -5,7 +5,6 @@ DATASET_PATH="${DATASET_PATH:dataset/2024_06_06_mixtral_15k_v4.pkl}"
 
 python -u main.py --scenario Server \
 		--model-path ${CHECKPOINT_PATH} \
-		--mlperf-conf mlperf.conf \
 		--user-conf user.conf \
 		--total-sample-count 15000 \
 		--dataset-path ${DATASET_PATH} \

diff --git a/tools/submission/submission_checker.py b/tools/submission/submission_checker.py
@@ -378,17 +378,17 @@
             ),
             "mixtral-8x7b": (
                 "ROUGE1",
-                45.4911 * 0.99,
+                45.5989 * 0.99,
                 "ROUGE2",
-                23.2829 * 0.99,
+                23.3526 * 0.99,
                 "ROUGEL",
-                30.3615 * 0.99,
+                30.4608 * 0.99,
                 "TOKENS_PER_SAMPLE",
-                145.9 * 0.9,
+                144.84 * 0.9,
                 "gsm8k_accuracy",
-                73.78 * 0.99,
+                73.66 * 0.99,
                 "mbxp_accuracy",
-                60.12 * 0.99,
+                60.16 * 0.99,
             ),
             "llama3.1-405b": (
                 "ROUGEL",