Evaluate LLM Performance on Cloud GPU

runcrate instances create --name eval --gpu A100 --template ubuntu-devbox
runcrate instances status eval

runcrate ssh eval -- "pip install lm-eval[vllm] vllm"

runcrate ssh eval -- "lm_eval --model vllm \
  --model_args pretrained=meta-llama/Llama-3.1-8B-Instruct,tensor_parallel_size=1 \
  --tasks mmlu \
  --num_fewshot 5 \
  --batch_size auto \
  --output_path /workspace/results/llama-8b-mmlu"

runcrate ssh eval -- "lm_eval --model vllm \
  --model_args pretrained=meta-llama/Llama-3.1-8B-Instruct \
  --tasks mmlu,hellaswag,arc_challenge,truthfulqa_mc2,winogrande,gsm8k \
  --num_fewshot 5 \
  --batch_size auto \
  --output_path /workspace/results/llama-8b-full"

runcrate ssh eval -- "lm_eval --model vllm \
  --model_args pretrained=Qwen/Qwen2.5-7B-Instruct \
  --tasks mmlu,hellaswag,arc_challenge,truthfulqa_mc2,winogrande,gsm8k \
  --num_fewshot 5 \
  --batch_size auto \
  --output_path /workspace/results/qwen-7b-full"

runcrate ssh eval -- "cat /workspace/results/llama-8b-full/results.json | python -m json.tool"
runcrate cp eval:/workspace/results/ ./eval-results/

runcrate instances delete eval