{"benchmark_id":"arena-hard","benchmark_name":"Arena Hard","benchmark_description":"Arena-Hard-Auto is an automatic evaluation benchmark for instruction-tuned LLMs consisting of 500 challenging real-world prompts curated by BenchBuilder. It includes open-ended software engineering problems, mathematical questions, and creative writing tasks. The benchmark uses LLM-as-a-Judge methodology with GPT-4.1 and Gemini-2.5 as automatic judges to approximate human preference. Arena-Hard achieves 98.6% correlation with human preference rankings and provides 3x higher separation of model performances compared to MT-Bench, making it highly effective for distinguishing between models of similar quality.","max_score":1.0,"categories":["chat","reasoning","general","creativity","writing"],"modality":"text","total_models":26,"entries":[{"rank":1,"model_id":"qwen3-235b-a22b","model_name":"Qwen3 235B A22B","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.956,"normalized_score":0.956,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-04-29","announcement_date":"2025-04-29","multimodal":false,"param_count":235000000000,"is_new":false},{"rank":2,"model_id":"qwen3-32b","model_name":"Qwen3 32B","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.938,"normalized_score":0.938,"verified":false,"self_reported":true,"provider_id":"deepinfra","input_cost_per_million":0.1,"output_cost_per_million":0.3,"speed_rps":26.95,"context_window":128000,"release_date":"2025-04-29","announcement_date":"2025-04-29","multimodal":false,"param_count":32800000000,"is_new":false},{"rank":3,"model_id":"qwen3-30b-a3b","model_name":"Qwen3 30B A3B","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.91,"normalized_score":0.91,"verified":false,"self_reported":true,"provider_id":"deepinfra","input_cost_per_million":0.1,"output_cost_per_million":0.3,"speed_rps":82.57,"context_window":128000,"release_date":"2025-04-29","announcement_date":"2025-04-29","multimodal":false,"param_count":30500000000,"is_new":false},{"rank":4,"model_id":"llama-3.3-nemotron-super-49b-v1","model_name":"Llama-3.3 Nemotron Super 49B v1","organization_name":"NVIDIA","organization_id":"nvidia","benchmark_score":0.883,"normalized_score":0.883,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-03-18","announcement_date":"2025-03-18","multimodal":false,"param_count":49900000000,"is_new":false},{"rank":5,"model_id":"mistral-small-24b-instruct-2501","model_name":"Mistral Small 3 24B Instruct","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.876,"normalized_score":0.876,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-01-30","announcement_date":"2025-01-30","multimodal":false,"param_count":24000000000,"is_new":false},{"rank":6,"model_id":"qwen-2.5-72b-instruct","model_name":"Qwen2.5 72B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.812,"normalized_score":0.812,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-09-19","announcement_date":"2024-09-19","multimodal":false,"param_count":72700000000,"is_new":false},{"rank":7,"model_id":"phi-4-reasoning-plus","model_name":"Phi 4 Reasoning Plus","organization_name":"Microsoft","organization_id":"microsoft","benchmark_score":0.79,"normalized_score":0.79,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-04-30","announcement_date":"2025-04-30","multimodal":false,"param_count":14000000000,"is_new":false},{"rank":8,"model_id":"deepseek-v2.5","model_name":"DeepSeek-V2.5","organization_name":"DeepSeek","organization_id":"deepseek","benchmark_score":0.762,"normalized_score":0.762,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-05-08","announcement_date":"2024-05-08","multimodal":false,"param_count":236000000000,"is_new":false},{"rank":9,"model_id":"phi-4","model_name":"Phi 4","organization_name":"Microsoft","organization_id":"microsoft","benchmark_score":0.754,"normalized_score":0.754,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-12-12","announcement_date":"2024-12-12","multimodal":false,"param_count":14700000000,"is_new":false},{"rank":10,"model_id":"phi-4-reasoning","model_name":"Phi 4 Reasoning","organization_name":"Microsoft","organization_id":"microsoft","benchmark_score":0.733,"normalized_score":0.733,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-04-30","announcement_date":"2025-04-30","multimodal":false,"param_count":14000000000,"is_new":false},{"rank":11,"model_id":"ministral-8b-instruct-2410","model_name":"Ministral 8B Instruct","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.709,"normalized_score":0.709,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-10-16","announcement_date":"2024-10-16","multimodal":false,"param_count":8019808256,"is_new":false},{"rank":12,"model_id":"jamba-1.5-large","model_name":"Jamba 1.5 Large","organization_name":"AI21 Labs","organization_id":"ai21","benchmark_score":0.654,"normalized_score":0.654,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-08-22","announcement_date":"2024-08-22","multimodal":false,"param_count":398000000000,"is_new":false},{"rank":13,"model_id":"mistral-small-latest","model_name":"Mistral Small 4","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.583,"normalized_score":0.583,"verified":false,"self_reported":true,"provider_id":"mistral","input_cost_per_million":0.15,"output_cost_per_million":0.6,"speed_rps":null,"context_window":256000,"release_date":"2026-03-16","announcement_date":"2026-03-16","multimodal":true,"param_count":119000000000,"is_new":false},{"rank":14,"model_id":"granite-3.3-8b-instruct","model_name":"Granite 3.3 8B Instruct","organization_name":"IBM","organization_id":"ibm","benchmark_score":0.5756,"normalized_score":0.5756,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-04-16","announcement_date":"2025-04-16","multimodal":true,"param_count":8000000000,"is_new":false},{"rank":15,"model_id":"granite-3.3-8b-base","model_name":"Granite 3.3 8B Base","organization_name":"IBM","organization_id":"ibm","benchmark_score":0.5756,"normalized_score":0.5756,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-04-16","announcement_date":"2025-04-16","multimodal":true,"param_count":8170000000,"is_new":false},{"rank":16,"model_id":"ministral-3-14b-instruct-2512","model_name":"MiniStral 3 (14B Instruct 2512)","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.551,"normalized_score":0.551,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-12-04","announcement_date":"2025-12-04","multimodal":true,"param_count":14000000000,"is_new":false},{"rank":17,"model_id":"mistral-large-3-2509","model_name":"Mistral Large 3","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.551,"normalized_score":0.551,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-09-01","announcement_date":"2025-09-01","multimodal":true,"param_count":675000000000,"is_new":false},{"rank":18,"model_id":"qwen-2.5-7b-instruct","model_name":"Qwen2.5 7B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.52,"normalized_score":0.52,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-09-19","announcement_date":"2024-09-19","multimodal":false,"param_count":7610000000,"is_new":false},{"rank":19,"model_id":"ministral-3-8b-instruct-2512","model_name":"Ministral 3 (8B Instruct 2512)","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.509,"normalized_score":0.509,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-12-04","announcement_date":"2025-12-04","multimodal":true,"param_count":8000000000,"is_new":false},{"rank":20,"model_id":"jamba-1.5-mini","model_name":"Jamba 1.5 Mini","organization_name":"AI21 Labs","organization_id":"ai21","benchmark_score":0.461,"normalized_score":0.461,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-08-22","announcement_date":"2024-08-22","multimodal":false,"param_count":52000000000,"is_new":false},{"rank":21,"model_id":"mistral-small-3.2-24b-instruct-2506","model_name":"Mistral Small 3.2 24B Instruct","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.431,"normalized_score":0.431,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-06-20","announcement_date":"2025-06-20","multimodal":true,"param_count":23600000000,"is_new":false},{"rank":22,"model_id":"phi-3.5-moe-instruct","model_name":"Phi-3.5-MoE-instruct","organization_name":"Microsoft","organization_id":"microsoft","benchmark_score":0.379,"normalized_score":0.379,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-08-23","announcement_date":"2024-08-23","multimodal":false,"param_count":60000000000,"is_new":false},{"rank":23,"model_id":"phi-3.5-mini-instruct","model_name":"Phi-3.5-mini-instruct","organization_name":"Microsoft","organization_id":"microsoft","benchmark_score":0.37,"normalized_score":0.37,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-08-23","announcement_date":"2024-08-23","multimodal":false,"param_count":3800000000,"is_new":false},{"rank":24,"model_id":"phi-4-mini","model_name":"Phi 4 Mini","organization_name":"Microsoft","organization_id":"microsoft","benchmark_score":0.328,"normalized_score":0.328,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-02-01","announcement_date":"2025-02-01","multimodal":false,"param_count":3840000000,"is_new":false},{"rank":25,"model_id":"ministral-3-3b-instruct-2512","model_name":"Ministral 3 (3B Instruct 2512)","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.305,"normalized_score":0.305,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-12-04","announcement_date":"2025-12-04","multimodal":true,"param_count":3000000000,"is_new":false},{"rank":26,"model_id":"granite-4.0-tiny-preview","model_name":"IBM Granite 4.0 Tiny Preview","organization_name":"IBM","organization_id":"ibm","benchmark_score":0.267,"normalized_score":0.267,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-05-02","announcement_date":"2025-05-02","multimodal":false,"param_count":7000000000,"is_new":false}]}