{"benchmark_id":"mvbench","benchmark_name":"MVBench","benchmark_description":"A comprehensive multi-modal video understanding benchmark covering 20 challenging video tasks that require temporal understanding beyond single-frame analysis. Tasks span from perception to cognition, including action recognition, temporal reasoning, spatial reasoning, object interaction, scene transition, and counterfactual inference. Uses a novel static-to-dynamic method to systematically generate video tasks from existing annotations.","max_score":1.0,"categories":["multimodal","reasoning","spatial_reasoning","video","vision"],"modality":"multimodal","total_models":18,"entries":[{"rank":1,"model_id":"glm-5.3-flash","model_name":"GLM-5.3-Flash","organization_name":"Zhipu AI","organization_id":"zai-org","benchmark_score":0.778,"normalized_score":0.778,"verified":false,"self_reported":true,"provider_id":"novita","input_cost_per_million":0.15,"output_cost_per_million":0.5,"speed_rps":null,"context_window":1048576,"release_date":"2026-08-26","announcement_date":"2026-08-26","multimodal":true,"param_count":320000000000,"is_new":true},{"rank":2,"model_id":"qwen3.5-122b-a10b","model_name":"Qwen3.5-122B-A10B","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.766,"normalized_score":0.766,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2026-02-24","announcement_date":"2026-02-24","multimodal":true,"param_count":122000000000,"is_new":false},{"rank":3,"model_id":"qwen3.6-27b","model_name":"Qwen3.6-27B","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.755,"normalized_score":0.755,"verified":false,"self_reported":true,"provider_id":"novita","input_cost_per_million":0.6,"output_cost_per_million":3.6,"speed_rps":null,"context_window":262144,"release_date":"2026-04-21","announcement_date":"2026-04-21","multimodal":true,"param_count":27781427952,"is_new":false},{"rank":4,"model_id":"qwen3.5-35b-a3b","model_name":"Qwen3.5-35B-A3B","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.748,"normalized_score":0.748,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2026-02-24","announcement_date":"2026-02-24","multimodal":true,"param_count":35000000000,"is_new":false},{"rank":5,"model_id":"qwen3.6-35b-a3b","model_name":"Qwen3.6-35B-A3B","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.746,"normalized_score":0.746,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2026-04-16","announcement_date":"2026-04-16","multimodal":true,"param_count":35000000000,"is_new":false},{"rank":6,"model_id":"qwen3.5-27b","model_name":"Qwen3.5-27B","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.746,"normalized_score":0.746,"verified":false,"self_reported":true,"provider_id":"novita","input_cost_per_million":0.3,"output_cost_per_million":2.4,"speed_rps":null,"context_window":262144,"release_date":"2026-02-24","announcement_date":"2026-02-24","multimodal":true,"param_count":27000000000,"is_new":false},{"rank":7,"model_id":"qwen2-vl-72b","model_name":"Qwen2-VL-72B-Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.736,"normalized_score":0.736,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-08-29","announcement_date":"2024-08-29","multimodal":true,"param_count":73400000000,"is_new":false},{"rank":8,"model_id":"qwen3-vl-32b-thinking","model_name":"Qwen3 VL 32B Thinking","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.732,"normalized_score":0.732,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-09-22","announcement_date":"2025-09-22","multimodal":true,"param_count":33000000000,"is_new":false},{"rank":9,"model_id":"qwen3-vl-32b-instruct","model_name":"Qwen3 VL 32B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.728,"normalized_score":0.728,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-09-22","announcement_date":"2025-09-22","multimodal":true,"param_count":33000000000,"is_new":false},{"rank":10,"model_id":"qwen3-vl-30b-a3b-instruct","model_name":"Qwen3 VL 30B A3B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.723,"normalized_score":0.723,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-09-22","announcement_date":"2025-09-22","multimodal":true,"param_count":31000000000,"is_new":false},{"rank":11,"model_id":"qwen3-vl-30b-a3b-thinking","model_name":"Qwen3 VL 30B A3B Thinking","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.72,"normalized_score":0.72,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-09-22","announcement_date":"2025-09-22","multimodal":true,"param_count":31000000000,"is_new":false},{"rank":12,"model_id":"qwen2.5-vl-72b","model_name":"Qwen2.5 VL 72B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.704,"normalized_score":0.704,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-01-26","announcement_date":"2025-01-26","multimodal":true,"param_count":72000000000,"is_new":false},{"rank":13,"model_id":"qwen2.5-omni-7b","model_name":"Qwen2.5-Omni-7B","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.703,"normalized_score":0.703,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-03-27","announcement_date":"2025-03-27","multimodal":true,"param_count":7000000000,"is_new":false},{"rank":14,"model_id":"qwen2.5-vl-7b","model_name":"Qwen2.5 VL 7B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.696,"normalized_score":0.696,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-01-26","announcement_date":"2025-01-26","multimodal":true,"param_count":8290000000,"is_new":false},{"rank":15,"model_id":"qwen3-vl-4b-thinking","model_name":"Qwen3 VL 4B Thinking","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.693,"normalized_score":0.693,"verified":false,"self_reported":true,"provider_id":"deepinfra","input_cost_per_million":0.1,"output_cost_per_million":1.0,"speed_rps":null,"context_window":262144,"release_date":"2025-09-22","announcement_date":"2025-09-22","multimodal":true,"param_count":4000000000,"is_new":false},{"rank":16,"model_id":"qwen3-vl-8b-thinking","model_name":"Qwen3 VL 8B Thinking","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.69,"normalized_score":0.69,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-09-22","announcement_date":"2025-09-22","multimodal":true,"param_count":9000000000,"is_new":false},{"rank":17,"model_id":"qwen3-vl-4b-instruct","model_name":"Qwen3 VL 4B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.689,"normalized_score":0.689,"verified":false,"self_reported":true,"provider_id":"deepinfra","input_cost_per_million":0.1,"output_cost_per_million":0.6,"speed_rps":null,"context_window":262144,"release_date":"2025-09-22","announcement_date":"2025-09-22","multimodal":true,"param_count":4000000000,"is_new":false},{"rank":18,"model_id":"qwen3-vl-8b-instruct","model_name":"Qwen3 VL 8B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.687,"normalized_score":0.687,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-09-22","announcement_date":"2025-09-22","multimodal":true,"param_count":9000000000,"is_new":false}]}