ah yes I forgot to call the script

mlabonne · Mar 28, 2024 · eb1b4b9 · eb1b4b9
1 parent 1e7d897
commit eb1b4b9
Showing 1 changed file with 12 additions and 6 deletions.
diff --git a/runpod.sh b/runpod.sh
@@ -84,23 +84,26 @@ elif [ "$BENCHMARK" == "openllm" ]; then
     pip install accelerate
 
     benchmark="arc"
-    accelerate launch --model lm_eval \
+    accelerate launch -m lm_eval \
+        --model hf \
         --model_args pretrained=${MODEL_ID},dtype=auto,gpu_memory_utilization=0.8,trust_remote_code=$TRUST_REMOTE_CODE \
         --tasks arc_challenge \
         --num_fewshot 25 \
         --batch_size auto \
         --output_path ./${benchmark}.json
 
     benchmark="hellaswag"
-    accelerate launch --model lm_eval \
+    accelerate launch -m lm_eval \
+        --model hf \
         --model_args pretrained=${MODEL_ID},dtype=auto,gpu_memory_utilization=0.8,trust_remote_code=$TRUST_REMOTE_CODE \
         --tasks hellaswag \
         --num_fewshot 10 \
         --batch_size auto \
         --output_path ./${benchmark}.json
 
     benchmark="mmlu"
-    accelerate launch --model lm_eval \
+    accelerate launch -m lm_eval \
+        --model hf \
         --model_args pretrained=${MODEL_ID},dtype=auto,gpu_memory_utilization=0.8,trust_remote_code=$TRUST_REMOTE_CODE \
         --tasks mmlu \
         --num_fewshot 5 \
@@ -109,23 +112,26 @@ elif [ "$BENCHMARK" == "openllm" ]; then
         --output_path ./${benchmark}.json
 
     benchmark="truthfulqa"
-    accelerate launch --model lm_eval \
+    accelerate launch -m lm_eval \
+        --model hf \
         --model_args pretrained=${MODEL_ID},dtype=auto,gpu_memory_utilization=0.8,trust_remote_code=$TRUST_REMOTE_CODE \
         --tasks truthfulqa \
         --num_fewshot 0 \
         --batch_size auto \
         --output_path ./${benchmark}.json
 
     benchmark="winogrande"
-    accelerate launch --model lm_eval \
+    accelerate launch -m lm_eval \
+        --model hf \
         --model_args pretrained=${MODEL_ID},dtype=auto,gpu_memory_utilization=0.8,trust_remote_code=$TRUST_REMOTE_CODE \
         --tasks winogrande \
         --num_fewshot 5 \
         --batch_size auto \
         --output_path ./${benchmark}.json
 
     benchmark="gsm8k"
-    accelerate launch --model lm_eval \
+    accelerate launch -m lm_eval \
+        --model hf \
         --model_args pretrained=${MODEL_ID},dtype=auto,gpu_memory_utilization=0.8,trust_remote_code=$TRUST_REMOTE_CODE \
         --tasks gsm8k \
         --num_fewshot 5 \