{"benchmark_id":"mt-bench","benchmark_name":"MT-Bench","benchmark_description":"MT-Bench is a challenging multi-turn benchmark that measures the ability of large language models to engage in coherent, informative, and engaging conversations. It uses strong LLMs as judges for scalable and explainable evaluation of multi-turn dialogue capabilities.","max_score":100.0,"categories":["chat","reasoning","roleplay","general","communication","creativity"],"modality":"text","total_models":12,"entries":[{"rank":1,"model_id":"hermes-3-70b","model_name":"Hermes 3 70B","organization_name":"Nous Research","organization_id":"nous-research","benchmark_score":8.99,"normalized_score":0.899,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-08-15","announcement_date":"2024-08-15","multimodal":false,"param_count":70000000000,"is_new":false},{"rank":2,"model_id":"qwen-2.5-72b-instruct","model_name":"Qwen2.5 72B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.935,"normalized_score":0.935,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-09-19","announcement_date":"2024-09-19","multimodal":false,"param_count":72700000000,"is_new":false},{"rank":3,"model_id":"llama-3.3-nemotron-super-49b-v1","model_name":"Llama-3.3 Nemotron Super 49B v1","organization_name":"NVIDIA","organization_id":"nvidia","benchmark_score":0.917,"normalized_score":0.917,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-03-18","announcement_date":"2025-03-18","multimodal":false,"param_count":49900000000,"is_new":false},{"rank":4,"model_id":"deepseek-v2.5","model_name":"DeepSeek-V2.5","organization_name":"DeepSeek","organization_id":"deepseek","benchmark_score":0.902,"normalized_score":0.902,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-05-08","announcement_date":"2024-05-08","multimodal":false,"param_count":236000000000,"is_new":false},{"rank":5,"model_id":"qwen-2.5-7b-instruct","model_name":"Qwen2.5 7B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.875,"normalized_score":0.875,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-09-19","announcement_date":"2024-09-19","multimodal":false,"param_count":7610000000,"is_new":false},{"rank":6,"model_id":"mistral-large-2-2407","model_name":"Mistral Large 2","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.863,"normalized_score":0.863,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-07-24","announcement_date":"2024-07-24","multimodal":false,"param_count":123000000000,"is_new":false},{"rank":7,"model_id":"qwen2-7b-instruct","model_name":"Qwen2 7B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.841,"normalized_score":0.841,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-07-23","announcement_date":"2024-07-23","multimodal":false,"param_count":7620000000,"is_new":false},{"rank":8,"model_id":"mistral-small-24b-instruct-2501","model_name":"Mistral Small 3 24B Instruct","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.835,"normalized_score":0.835,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-01-30","announcement_date":"2025-01-30","multimodal":false,"param_count":24000000000,"is_new":false},{"rank":9,"model_id":"ministral-8b-instruct-2410","model_name":"Ministral 8B Instruct","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.83,"normalized_score":0.83,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-10-16","announcement_date":"2024-10-16","multimodal":false,"param_count":8019808256,"is_new":false},{"rank":10,"model_id":"llama-3.1-nemotron-nano-8b-v1","model_name":"Llama 3.1 Nemotron Nano 8B V1","organization_name":"NVIDIA","organization_id":"nvidia","benchmark_score":0.81,"normalized_score":0.81,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-03-18","announcement_date":"2025-03-18","multimodal":false,"param_count":8000000000,"is_new":false},{"rank":11,"model_id":"pixtral-12b-2409","model_name":"Pixtral-12B","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.768,"normalized_score":0.768,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-09-17","announcement_date":"2024-09-17","multimodal":true,"param_count":12400000000,"is_new":false},{"rank":12,"model_id":"llama-3.1-nemotron-70b-instruct","model_name":"Llama 3.1 Nemotron 70B Instruct","organization_name":"NVIDIA","organization_id":"nvidia","benchmark_score":0.0899,"normalized_score":0.0899,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-10-01","announcement_date":"2024-10-01","multimodal":false,"param_count":70000000000,"is_new":false}]}