{"benchmark_id":"mmmu-(validation)","benchmark_name":"MMMU (validation)","benchmark_description":"Validation set of the Massive Multi-discipline Multimodal Understanding and Reasoning benchmark. Features college-level multimodal questions across 6 core disciplines (Art & Design, Business, Science, Health & Medicine, Humanities & Social Science, Tech & Engineering) spanning 30 subjects and 183 subfields with diverse image types including charts, diagrams, maps, and tables.","max_score":1.0,"categories":["multimodal","reasoning","general","healthcare","vision"],"modality":"multimodal","total_models":4,"entries":[{"rank":1,"model_id":"claude-opus-4-5-20251101","model_name":"Claude Opus 4.5","organization_name":"Anthropic","organization_id":"anthropic","benchmark_score":0.807,"normalized_score":0.807,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-11-24","announcement_date":"2025-11-24","multimodal":true,"param_count":null,"is_new":false},{"rank":2,"model_id":"claude-opus-4-1-20250805","model_name":"Claude Opus 4.1","organization_name":"Anthropic","organization_id":"anthropic","benchmark_score":0.771,"normalized_score":0.771,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-08-05","announcement_date":"2025-08-05","multimodal":true,"param_count":null,"is_new":false},{"rank":3,"model_id":"claude-opus-4-20250514","model_name":"Claude Opus 4","organization_name":"Anthropic","organization_id":"anthropic","benchmark_score":0.765,"normalized_score":0.765,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-05-22","announcement_date":"2025-05-22","multimodal":true,"param_count":null,"is_new":false},{"rank":4,"model_id":"claude-haiku-4-5-20251001","model_name":"Claude Haiku 4.5","organization_name":"Anthropic","organization_id":"anthropic","benchmark_score":0.732,"normalized_score":0.732,"verified":false,"self_reported":true,"provider_id":"anthropic","input_cost_per_million":1.0,"output_cost_per_million":5.0,"speed_rps":100.0,"context_window":200000,"release_date":"2025-10-15","announcement_date":"2025-10-15","multimodal":true,"param_count":null,"is_new":false}]}