{"benchmark_id":"chartqa","benchmark_name":"ChartQA","benchmark_description":"ChartQA is a large-scale benchmark comprising 9.6K human-written questions and 23.1K questions generated from human-written chart summaries, designed to evaluate models' abilities in visual and logical reasoning over charts.","max_score":1.0,"categories":["multimodal","reasoning","vision"],"modality":"multimodal","total_models":26,"entries":[{"rank":1,"model_id":"claude-3-5-sonnet-20241022","model_name":"Claude 3.5 Sonnet","organization_name":"Anthropic","organization_id":"anthropic","benchmark_score":0.908,"normalized_score":0.908,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-10-22","announcement_date":"2024-10-22","multimodal":true,"param_count":null,"is_new":false},{"rank":2,"model_id":"llama-4-maverick","model_name":"Llama 4 Maverick","organization_name":"Meta","organization_id":"meta","benchmark_score":0.9,"normalized_score":0.9,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-04-05","announcement_date":"2025-04-05","multimodal":true,"param_count":400000000000,"is_new":false},{"rank":3,"model_id":"qwen2.5-vl-72b","model_name":"Qwen2.5 VL 72B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.895,"normalized_score":0.895,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-01-26","announcement_date":"2025-01-26","multimodal":true,"param_count":72000000000,"is_new":false},{"rank":4,"model_id":"nova-pro","model_name":"Nova Pro","organization_name":"Amazon","organization_id":"amazon","benchmark_score":0.892,"normalized_score":0.892,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-11-20","announcement_date":"2024-11-20","multimodal":true,"param_count":null,"is_new":false},{"rank":5,"model_id":"llama-4-scout","model_name":"Llama 4 Scout","organization_name":"Meta","organization_id":"meta","benchmark_score":0.888,"normalized_score":0.888,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-04-05","announcement_date":"2025-04-05","multimodal":true,"param_count":109000000000,"is_new":false},{"rank":6,"model_id":"qwen2-vl-72b","model_name":"Qwen2-VL-72B-Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.883,"normalized_score":0.883,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-08-29","announcement_date":"2024-08-29","multimodal":true,"param_count":73400000000,"is_new":false},{"rank":7,"model_id":"pixtral-large","model_name":"Pixtral Large","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.881,"normalized_score":0.881,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-11-18","announcement_date":"2024-11-18","multimodal":true,"param_count":124000000000,"is_new":false},{"rank":8,"model_id":"mistral-small-3.2-24b-instruct-2506","model_name":"Mistral Small 3.2 24B Instruct","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.874,"normalized_score":0.874,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-06-20","announcement_date":"2025-06-20","multimodal":true,"param_count":23600000000,"is_new":false},{"rank":9,"model_id":"qwen2.5-vl-7b","model_name":"Qwen2.5 VL 7B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.873,"normalized_score":0.873,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-01-26","announcement_date":"2025-01-26","multimodal":true,"param_count":8290000000,"is_new":false},{"rank":10,"model_id":"nova-lite","model_name":"Nova Lite","organization_name":"Amazon","organization_id":"amazon","benchmark_score":0.868,"normalized_score":0.868,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-11-20","announcement_date":"2024-11-20","multimodal":true,"param_count":null,"is_new":false},{"rank":11,"model_id":"deepseek-vl2","model_name":"DeepSeek VL2","organization_name":"DeepSeek","organization_id":"deepseek","benchmark_score":0.86,"normalized_score":0.86,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-12-13","announcement_date":"2024-12-13","multimodal":true,"param_count":27000000000,"is_new":false},{"rank":12,"model_id":"gpt-4o-2024-08-06","model_name":"GPT-4o","organization_name":"OpenAI","organization_id":"openai","benchmark_score":0.857,"normalized_score":0.857,"verified":false,"self_reported":true,"provider_id":"openai","input_cost_per_million":2.5,"output_cost_per_million":10.0,"speed_rps":132.0,"context_window":128000,"release_date":"2024-08-06","announcement_date":"2024-08-06","multimodal":true,"param_count":null,"is_new":false},{"rank":13,"model_id":"llama-3.2-90b-instruct","model_name":"Llama 3.2 90B Instruct","organization_name":"Meta","organization_id":"meta","benchmark_score":0.855,"normalized_score":0.855,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-09-25","announcement_date":"2024-09-25","multimodal":true,"param_count":90000000000,"is_new":false},{"rank":14,"model_id":"qwen2.5-omni-7b","model_name":"Qwen2.5-Omni-7B","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.853,"normalized_score":0.853,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-03-27","announcement_date":"2025-03-27","multimodal":true,"param_count":7000000000,"is_new":false},{"rank":15,"model_id":"deepseek-vl2-small","model_name":"DeepSeek VL2 Small","organization_name":"DeepSeek","organization_id":"deepseek","benchmark_score":0.845,"normalized_score":0.845,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-12-13","announcement_date":"2024-12-13","multimodal":true,"param_count":16000000000,"is_new":false},{"rank":16,"model_id":"llama-3.2-11b-instruct","model_name":"Llama 3.2 11B Instruct","organization_name":"Meta","organization_id":"meta","benchmark_score":0.834,"normalized_score":0.834,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-09-25","announcement_date":"2024-09-25","multimodal":true,"param_count":10600000000,"is_new":false},{"rank":17,"model_id":"pixtral-12b-2409","model_name":"Pixtral-12B","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.818,"normalized_score":0.818,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-09-17","announcement_date":"2024-09-17","multimodal":true,"param_count":12400000000,"is_new":false},{"rank":18,"model_id":"phi-3.5-vision-instruct","model_name":"Phi-3.5-vision-instruct","organization_name":"Microsoft","organization_id":"microsoft","benchmark_score":0.818,"normalized_score":0.818,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-08-23","announcement_date":"2024-08-23","multimodal":true,"param_count":4200000000,"is_new":false},{"rank":19,"model_id":"phi-4-multimodal-instruct","model_name":"Phi-4-multimodal-instruct","organization_name":"Microsoft","organization_id":"microsoft","benchmark_score":0.814,"normalized_score":0.814,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-02-01","announcement_date":"2025-02-01","multimodal":true,"param_count":5600000000,"is_new":false},{"rank":20,"model_id":"lfm-2.5-vl-3b","model_name":"LFM2.5-VL-3B","organization_name":"Liquid AI","organization_id":"liquid","benchmark_score":0.813,"normalized_score":0.813,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2026-08-12","announcement_date":"2026-08-12","multimodal":true,"param_count":3123483888,"is_new":false},{"rank":21,"model_id":"deepseek-vl2-tiny","model_name":"DeepSeek VL2 Tiny","organization_name":"DeepSeek","organization_id":"deepseek","benchmark_score":0.81,"normalized_score":0.81,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-12-13","announcement_date":"2024-12-13","multimodal":true,"param_count":3000000000,"is_new":false},{"rank":22,"model_id":"north-micro-vision-instruct","model_name":"North Micro Vision Instruct","organization_name":"Cohere","organization_id":"cohere","benchmark_score":0.808,"normalized_score":0.808,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2026-08-12","announcement_date":"2026-08-12","multimodal":true,"param_count":2400000000,"is_new":false},{"rank":23,"model_id":"gemma-3-27b-it","model_name":"Gemma 3 27B","organization_name":"Google","organization_id":"google","benchmark_score":0.78,"normalized_score":0.78,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-03-12","announcement_date":"2025-03-12","multimodal":true,"param_count":27000000000,"is_new":false},{"rank":24,"model_id":"grok-1.5v","model_name":"Grok-1.5V","organization_name":"xAI","organization_id":"xai","benchmark_score":0.761,"normalized_score":0.761,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-04-12","announcement_date":"2024-04-12","multimodal":true,"param_count":null,"is_new":false},{"rank":25,"model_id":"gemma-3-12b-it","model_name":"Gemma 3 12B","organization_name":"Google","organization_id":"google","benchmark_score":0.757,"normalized_score":0.757,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-03-12","announcement_date":"2025-03-12","multimodal":true,"param_count":12000000000,"is_new":false},{"rank":26,"model_id":"gemma-3-4b-it","model_name":"Gemma 3 4B","organization_name":"Google","organization_id":"google","benchmark_score":0.688,"normalized_score":0.688,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-03-12","announcement_date":"2025-03-12","multimodal":true,"param_count":4000000000,"is_new":false}]}