{"benchmark_id":"multilingual-mmlu","benchmark_name":"Multilingual MMLU","benchmark_description":"MMLU-ProX is a comprehensive multilingual benchmark covering 29 typologically diverse languages, building upon MMLU-Pro. Each language version consists of 11,829 identical questions enabling direct cross-linguistic comparisons. The benchmark evaluates large language models' reasoning capabilities across linguistic and cultural boundaries through challenging, reasoning-focused questions with 10 answer choices.","max_score":1.0,"categories":["language","reasoning","general"],"modality":"text","total_models":5,"entries":[{"rank":1,"model_id":"o3-mini","model_name":"o3-mini","organization_name":"OpenAI","organization_id":"openai","benchmark_score":0.807,"normalized_score":0.807,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-01-30","announcement_date":"2025-01-30","multimodal":false,"param_count":null,"is_new":false},{"rank":2,"model_id":"ministral-3-14b-base-2512","model_name":"Ministral 3 (14B Base 2512)","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.742,"normalized_score":0.742,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-12-04","announcement_date":"2025-12-04","multimodal":true,"param_count":14000000000,"is_new":false},{"rank":3,"model_id":"ministral-3-8b-base-2512","model_name":"Ministral 3 (8B Base 2512)","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.706,"normalized_score":0.706,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-12-04","announcement_date":"2025-12-04","multimodal":true,"param_count":8000000000,"is_new":false},{"rank":4,"model_id":"ministral-3-3b-base-2512","model_name":"Ministral 3 (3B Base 2512)","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.652,"normalized_score":0.652,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-12-04","announcement_date":"2025-12-04","multimodal":true,"param_count":3000000000,"is_new":false},{"rank":5,"model_id":"phi-4-mini","model_name":"Phi 4 Mini","organization_name":"Microsoft","organization_id":"microsoft","benchmark_score":0.493,"normalized_score":0.493,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-02-01","announcement_date":"2025-02-01","multimodal":false,"param_count":3840000000,"is_new":false}]}