{"benchmark_id":"mmbench","benchmark_name":"MMBench","benchmark_description":"A bilingual benchmark for assessing multi-modal capabilities of vision-language models through multiple-choice questions in both English and Chinese, providing systematic evaluation across diverse vision-language tasks with robust metrics.","max_score":1.0,"categories":["multimodal","reasoning","vision"],"modality":"multimodal","total_models":9,"entries":[{"rank":1,"model_id":"step3-vl-10b","model_name":"Step3-VL-10B","organization_name":"StepFun","organization_id":"stepfun","benchmark_score":0.918,"normalized_score":0.918,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2026-01-15","announcement_date":"2026-01-15","multimodal":true,"param_count":10000000000,"is_new":false},{"rank":2,"model_id":"qwen2.5-vl-72b","model_name":"Qwen2.5 VL 72B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.88,"normalized_score":0.88,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-01-26","announcement_date":"2025-01-26","multimodal":true,"param_count":72000000000,"is_new":false},{"rank":3,"model_id":"phi-4-multimodal-instruct","model_name":"Phi-4-multimodal-instruct","organization_name":"Microsoft","organization_id":"microsoft","benchmark_score":0.867,"normalized_score":0.867,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-02-01","announcement_date":"2025-02-01","multimodal":true,"param_count":5600000000,"is_new":false},{"rank":4,"model_id":"qwen2-vl-72b","model_name":"Qwen2-VL-72B-Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.865,"normalized_score":0.865,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-08-29","announcement_date":"2024-08-29","multimodal":true,"param_count":73400000000,"is_new":false},{"rank":5,"model_id":"qwen2.5-vl-7b","model_name":"Qwen2.5 VL 7B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.843,"normalized_score":0.843,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-01-26","announcement_date":"2025-01-26","multimodal":true,"param_count":8290000000,"is_new":false},{"rank":6,"model_id":"phi-3.5-vision-instruct","model_name":"Phi-3.5-vision-instruct","organization_name":"Microsoft","organization_id":"microsoft","benchmark_score":0.819,"normalized_score":0.819,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-08-23","announcement_date":"2024-08-23","multimodal":true,"param_count":4200000000,"is_new":false},{"rank":7,"model_id":"deepseek-vl2-small","model_name":"DeepSeek VL2 Small","organization_name":"DeepSeek","organization_id":"deepseek","benchmark_score":0.803,"normalized_score":0.803,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-12-13","announcement_date":"2024-12-13","multimodal":true,"param_count":16000000000,"is_new":false},{"rank":8,"model_id":"deepseek-vl2","model_name":"DeepSeek VL2","organization_name":"DeepSeek","organization_id":"deepseek","benchmark_score":0.796,"normalized_score":0.796,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-12-13","announcement_date":"2024-12-13","multimodal":true,"param_count":27000000000,"is_new":false},{"rank":9,"model_id":"deepseek-vl2-tiny","model_name":"DeepSeek VL2 Tiny","organization_name":"DeepSeek","organization_id":"deepseek","benchmark_score":0.692,"normalized_score":0.692,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-12-13","announcement_date":"2024-12-13","multimodal":true,"param_count":3000000000,"is_new":false}]}