{"benchmark_id":"charxiv-r","name":"CharXiv-R","parent_benchmark":null,"categories":["multimodal","reasoning","vision"],"modality":"multimodal","multilingual":false,"max_score":1.0,"language":"en","description":"CharXiv-R is the reasoning component of the CharXiv benchmark, focusing on complex reasoning questions that require synthesizing information across visual chart elements. It evaluates multimodal large language models on their ability to understand and reason about scientific charts from arXiv papers through various reasoning tasks.","paper_link":"https://arxiv.org/abs/2406.18521","implementation_link":null,"verified":false,"created_at":"2026-05-07T16:53:22.340340+00:00","updated_at":"2026-07-05T18:27:45.184918+00:00","statistics":{"total_models":43,"average_score":0.7176511627906975,"min_score":0.397,"max_score":0.932,"score_stddev":0.14837739587200796,"verified_count":0,"self_reported_count":43},"child_benchmarks":[],"linked_dataset":null,"models":[{"rank":1,"model_id":"claude-mythos-preview","model_name":"Claude Mythos Preview","organization_id":"anthropic","organization_name":"Anthropic","organization_country":"US","score":0.932,"normalized_score":0.932,"verified":false,"self_reported":true,"self_reported_source":"https://www.anthropic.com/claude-mythos-preview-system-card","analysis_method":"CharXiv Reasoning with tools. Without tools: 86.1%. Opus 4.6: 78.9% (with tools), 61.5% (without tools).","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-04-07","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":2,"model_id":"claude-opus-4-7","model_name":"Claude Opus 4.7","organization_id":"anthropic","organization_name":"Anthropic","organization_country":"US","score":0.91,"normalized_score":0.91,"verified":false,"self_reported":true,"self_reported_source":"https://www.anthropic.com/news/claude-opus-4-7","analysis_method":"CharXiv Reasoning. Visual reasoning with tools: 91.0%. Without tools: 82.1%.","verification_date":null,"provider_id":"anthropic","input_cost_per_million":5.0,"output_cost_per_million":25.0,"context_window":1000000,"announcement_date":"2026-04-16","param_count":null,"is_open_source":false,"is_new":false,"best_latency":0.5,"latency_provider":"Anthropic","best_throughput":42.0,"throughput_provider":"Anthropic","context_provider":"Anthropic"},{"rank":3,"model_id":"claude-opus-4-8","model_name":"Claude Opus 4.8","organization_id":"anthropic","organization_name":"Anthropic","organization_country":"US","score":0.899,"normalized_score":0.899,"verified":false,"self_reported":true,"self_reported_source":"https://www.anthropic.com/news/claude-opus-4-8","analysis_method":"CharXiv Reasoning. Adaptive thinking at max effort with Python tools: 89.9%. Without tools: 80.5%. 1,000 validation questions averaged over 5 runs.","verification_date":null,"provider_id":"anthropic","input_cost_per_million":5.0,"output_cost_per_million":25.0,"context_window":1000000,"announcement_date":"2026-05-28","param_count":null,"is_open_source":false,"is_new":false,"best_latency":0.5,"latency_provider":"Anthropic","best_throughput":42.0,"throughput_provider":"Anthropic","context_provider":"Anthropic"},{"rank":4,"model_id":"claude-sonnet-5","model_name":"Claude Sonnet 5","organization_id":"anthropic","organization_name":"Anthropic","organization_country":"US","score":0.883,"normalized_score":0.883,"verified":false,"self_reported":true,"self_reported_source":"https://www.anthropic.com/news/claude-sonnet-5","analysis_method":"CharXiv Reasoning (1,000 validation questions). With Python tools: 88.3%. Without tools: 77.0%. Averaged over 5 runs, Claude Sonnet 4.6 grader.","verification_date":null,"provider_id":"anthropic","input_cost_per_million":3.0,"output_cost_per_million":15.0,"context_window":1000000,"announcement_date":"2026-06-30","param_count":null,"is_open_source":false,"is_new":true,"best_latency":0.4,"latency_provider":"Anthropic","best_throughput":42.0,"throughput_provider":"Anthropic","context_provider":"Anthropic"},{"rank":5,"model_id":"kimi-k2.6","model_name":"Kimi K2.6","organization_id":"moonshotai","organization_name":"Moonshot AI","organization_country":"CN","score":0.867,"normalized_score":0.867,"verified":false,"self_reported":true,"self_reported_source":"https://www.kimi.com/blog/kimi-k2-6","analysis_method":"w/ python","verification_date":null,"provider_id":"deepinfra","input_cost_per_million":0.75,"output_cost_per_million":3.5,"context_window":262144,"announcement_date":"2026-04-20","param_count":1000000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":"DeepInfra","best_throughput":null,"throughput_provider":"DeepInfra","context_provider":"Together"},{"rank":6,"model_id":"seed-2.1-pro","model_name":"Seed 2.1 Pro","organization_id":"bytedance","organization_name":"ByteDance","organization_country":"CN","score":0.864,"normalized_score":0.864,"verified":false,"self_reported":true,"self_reported_source":"https://seed.bytedance.com/en/seed2_1","analysis_method":"With tools","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-06-24","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":6,"model_id":"muse-spark","model_name":"Muse Spark","organization_id":"meta","organization_name":"Meta","organization_country":"US","score":0.864,"normalized_score":0.864,"verified":false,"self_reported":true,"self_reported_source":"https://ai.meta.com/static-resource/muse-spark-eval-methodology","analysis_method":"Thinking mode","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-04-08","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":8,"model_id":"qwen3.7-plus","model_name":"Qwen3.7-Plus","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.859,"normalized_score":0.859,"verified":false,"self_reported":true,"self_reported_source":"https://qwen.ai/blog?id=qwen3.7-plus","analysis_method":"With CI","verification_date":null,"provider_id":"together","input_cost_per_million":0.32,"output_cost_per_million":1.28,"context_window":1000000,"announcement_date":"2026-05-31","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":"Together","best_throughput":null,"throughput_provider":"Together","context_provider":"Together"},{"rank":9,"model_id":"gemini-3.5-flash","model_name":"Gemini 3.5 Flash","organization_id":"google","organization_name":"Google","organization_country":"US","score":0.842,"normalized_score":0.842,"verified":false,"self_reported":true,"self_reported_source":"https://deepmind.google/models/evals-methodology/gemini-3-5-flash/","analysis_method":"No tools","verification_date":null,"provider_id":"google","input_cost_per_million":1.5,"output_cost_per_million":9.0,"context_window":1048576,"announcement_date":"2026-05-19","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":"Google","best_throughput":null,"throughput_provider":"Google","context_provider":"Google"},{"rank":10,"model_id":"seed-2.1-turbo","model_name":"Seed 2.1 Turbo","organization_id":"bytedance","organization_name":"ByteDance","organization_country":"CN","score":0.836,"normalized_score":0.836,"verified":false,"self_reported":true,"self_reported_source":"https://seed.bytedance.com/en/seed2_1","analysis_method":"With tools","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-06-24","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":11,"model_id":"gpt-5.2-2025-12-11","model_name":"GPT-5.2","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.821,"normalized_score":0.821,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/introducing-gpt-5-2/","analysis_method":"GPT-5.2 Thinking - CharXiv reasoning (no tools).","verification_date":null,"provider_id":"openai","input_cost_per_million":1.75,"output_cost_per_million":14.0,"context_window":400000,"announcement_date":"2025-12-11","param_count":null,"is_open_source":false,"is_new":false,"best_latency":2.0,"latency_provider":"OpenAI","best_throughput":100.0,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":12,"model_id":"gpt-5.5-instant","model_name":"GPT-5.5 Instant","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.816,"normalized_score":0.816,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/gpt-5-5-instant/","analysis_method":"CharXiv reasoning scientific chart reasoning accuracy.","verification_date":null,"provider_id":"openai","input_cost_per_million":5.0,"output_cost_per_million":30.0,"context_window":400000,"announcement_date":"2026-05-05","param_count":null,"is_open_source":false,"is_new":false,"best_latency":2.0,"latency_provider":"OpenAI","best_throughput":100.0,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":13,"model_id":"qwen3.6-plus","model_name":"Qwen3.6 Plus","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.815,"normalized_score":0.815,"verified":false,"self_reported":true,"self_reported_source":"https://qwen.ai/blog?id=qwen3.6","analysis_method":null,"verification_date":null,"provider_id":"together","input_cost_per_million":0.5,"output_cost_per_million":3.0,"context_window":1000000,"announcement_date":"2026-04-02","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":"Together","best_throughput":null,"throughput_provider":"Together","context_provider":"Together"},{"rank":14,"model_id":"gemini-3-pro-preview","model_name":"Gemini 3 Pro","organization_id":"google","organization_name":"Google","organization_country":"US","score":0.814,"normalized_score":0.814,"verified":false,"self_reported":true,"self_reported_source":"https://blog.google/products/gemini/gemini-3","analysis_method":null,"verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2025-11-18","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":15,"model_id":"gpt-5-2025-08-07","model_name":"GPT-5","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.811,"normalized_score":0.811,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/gpt-5/","analysis_method":"GPT-5 with thinking mode - Scientific figure reasoning and interpretation.","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2025-08-07","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":16,"model_id":"mimo-v2.5","model_name":"MiMo-V2.5","organization_id":"xiaomi","organization_name":"Xiaomi","organization_country":"CN","score":0.81,"normalized_score":0.81,"verified":false,"self_reported":true,"self_reported_source":"https://mimo.xiaomi.com/mimo-v2-5/","analysis_method":"RQ","verification_date":null,"provider_id":"novita","input_cost_per_million":0.168,"output_cost_per_million":0.336,"context_window":1048576,"announcement_date":"2026-04-22","param_count":310775040000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":"Novita","best_throughput":null,"throughput_provider":"Novita","context_provider":"Novita"},{"rank":17,"model_id":"gemini-3-flash-preview","model_name":"Gemini 3 Flash","organization_id":"google","organization_name":"Google","organization_country":"US","score":0.803,"normalized_score":0.803,"verified":false,"self_reported":true,"self_reported_source":"https://blog.google/products/gemini/gemini-3-flash/","analysis_method":"No tools","verification_date":null,"provider_id":"google","input_cost_per_million":0.5,"output_cost_per_million":3.0,"context_window":1000000,"announcement_date":"2025-12-17","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":"Google","best_throughput":null,"throughput_provider":"Google","context_provider":"Google"},{"rank":18,"model_id":"qwen3.5-27b","model_name":"Qwen3.5-27B","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.795,"normalized_score":0.795,"verified":false,"self_reported":true,"self_reported_source":"https://qwen.ai/blog?id=qwen3.5","analysis_method":"RQ","verification_date":null,"provider_id":"novita","input_cost_per_million":0.3,"output_cost_per_million":2.4,"context_window":262144,"announcement_date":"2026-02-24","param_count":27000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":"Novita","best_throughput":null,"throughput_provider":"Novita","context_provider":"Novita"},{"rank":19,"model_id":"o3-2025-04-16","model_name":"o3","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.786,"normalized_score":0.786,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/gpt-5/","analysis_method":"OpenAI o3 with thinking mode - Scientific figure reasoning and interpretation.","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2025-04-16","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":20,"model_id":"qwen3.6-27b","model_name":"Qwen3.6-27B","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.784,"normalized_score":0.784,"verified":false,"self_reported":true,"self_reported_source":"https://qwen.ai/blog?id=qwen3.6-27b","analysis_method":"RQ","verification_date":null,"provider_id":"novita","input_cost_per_million":0.6,"output_cost_per_million":3.6,"context_window":262144,"announcement_date":"2026-04-21","param_count":27781427952,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":"Novita","best_throughput":null,"throughput_provider":"Novita","context_provider":"Novita"},{"rank":21,"model_id":"qwen3.6-35b-a3b","model_name":"Qwen3.6-35B-A3B","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.78,"normalized_score":0.78,"verified":false,"self_reported":true,"self_reported_source":"https://qwen.ai/blog?id=qwen3.6-35b-a3b","analysis_method":"RQ","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-04-16","param_count":35000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":22,"model_id":"qwen3.5-35b-a3b","model_name":"Qwen3.5-35B-A3B","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.775,"normalized_score":0.775,"verified":false,"self_reported":true,"self_reported_source":"https://qwen.ai/blog?id=qwen3.5","analysis_method":"RQ","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-02-24","param_count":35000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":22,"model_id":"kimi-k2.5","model_name":"Kimi K2.5","organization_id":"moonshotai","organization_name":"Moonshot AI","organization_country":"CN","score":0.775,"normalized_score":0.775,"verified":false,"self_reported":true,"self_reported_source":"https://www.kimi.com/blog/kimi-k2-5.html","analysis_method":"Image","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-01-27","param_count":1000000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":24,"model_id":"claude-opus-4-6","model_name":"Claude Opus 4.6","organization_id":"anthropic","organization_name":"Anthropic","organization_country":"US","score":0.774,"normalized_score":0.774,"verified":false,"self_reported":true,"self_reported_source":"https://www-cdn.anthropic.com/0dd865075ad3132672ee0ab40b05a53f14cf5288.pdf","analysis_method":"CharXiv Reasoning. With adaptive thinking, image cropping tool, and max effort. Scores averaged over 5 runs. Without image cropping tool: 68.7%.","verification_date":null,"provider_id":"anthropic","input_cost_per_million":5.0,"output_cost_per_million":25.0,"context_window":1000000,"announcement_date":"2026-02-05","param_count":null,"is_open_source":false,"is_new":false,"best_latency":0.5,"latency_provider":"Anthropic","best_throughput":42.0,"throughput_provider":"Anthropic","context_provider":"Anthropic"},{"rank":25,"model_id":"qwen3.5-122b-a10b","model_name":"Qwen3.5-122B-A10B","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.772,"normalized_score":0.772,"verified":false,"self_reported":true,"self_reported_source":"https://qwen.ai/blog?id=qwen3.5","analysis_method":"RQ","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-02-24","param_count":122000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":26,"model_id":"gemini-3.1-flash-lite-preview","model_name":"Gemini 3.1 Flash-Lite","organization_id":"google","organization_name":"Google","organization_country":"US","score":0.732,"normalized_score":0.732,"verified":false,"self_reported":true,"self_reported_source":"https://deepmind.google/models/model-cards/gemini-3-1-flash-lite/","analysis_method":null,"verification_date":null,"provider_id":"google","input_cost_per_million":0.25,"output_cost_per_million":1.5,"context_window":1000000,"announcement_date":"2026-03-03","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":"Google","best_throughput":null,"throughput_provider":"Google","context_provider":"Google"},{"rank":27,"model_id":"o4-mini","model_name":"o4-mini","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.72,"normalized_score":0.72,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/introducing-o3-and-o4-mini/","analysis_method":"accuracy","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2025-04-16","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":28,"model_id":"qwen3-vl-235b-a22b-thinking","model_name":"Qwen3 VL 235B A22B Thinking","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.661,"normalized_score":0.661,"verified":false,"self_reported":true,"self_reported_source":null,"analysis_method":"RQ","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2025-09-22","param_count":236000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":29,"model_id":"qwen3-vl-32b-thinking","model_name":"Qwen3 VL 32B Thinking","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.652,"normalized_score":0.652,"verified":false,"self_reported":true,"self_reported_source":null,"analysis_method":null,"verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2025-09-22","param_count":33000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":30,"model_id":"qwen3-vl-32b-instruct","model_name":"Qwen3 VL 32B Instruct","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.628,"normalized_score":0.628,"verified":false,"self_reported":true,"self_reported_source":null,"analysis_method":"RO","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2025-09-22","param_count":33000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":31,"model_id":"qwen3-vl-235b-a22b-instruct","model_name":"Qwen3 VL 235B A22B Instruct","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.621,"normalized_score":0.621,"verified":false,"self_reported":true,"self_reported_source":null,"analysis_method":"RQ","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2025-09-22","param_count":236000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":32,"model_id":"gpt-4o-2024-08-06","model_name":"GPT-4o","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.588,"normalized_score":0.588,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/gpt-5/","analysis_method":"GPT-4o without thinking mode - Scientific figure reasoning and interpretation.","verification_date":null,"provider_id":"openai","input_cost_per_million":2.5,"output_cost_per_million":10.0,"context_window":128000,"announcement_date":"2024-08-06","param_count":null,"is_open_source":false,"is_new":false,"best_latency":0.5,"latency_provider":"OpenAI","best_throughput":132.0,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":33,"model_id":"gpt-4.1-mini-2025-04-14","model_name":"GPT-4.1 mini","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.568,"normalized_score":0.568,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/introducing-gpt-5-for-developers/","analysis_method":"Standard benchmark","verification_date":null,"provider_id":"openai","input_cost_per_million":0.4,"output_cost_per_million":1.6,"context_window":1047576,"announcement_date":"2025-04-14","param_count":null,"is_open_source":false,"is_new":false,"best_latency":5.0,"latency_provider":"OpenAI","best_throughput":150.0,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":34,"model_id":"gpt-4.1-2025-04-14","model_name":"GPT-4.1","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.567,"normalized_score":0.567,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/introducing-gpt-5-for-developers/","analysis_method":"Standard benchmark","verification_date":null,"provider_id":"openai","input_cost_per_million":2.0,"output_cost_per_million":8.0,"context_window":1047576,"announcement_date":"2025-04-14","param_count":null,"is_open_source":false,"is_new":false,"best_latency":10.0,"latency_provider":"OpenAI","best_throughput":100.0,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":35,"model_id":"qwen3-vl-30b-a3b-thinking","model_name":"Qwen3 VL 30B A3B Thinking","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.566,"normalized_score":0.566,"verified":false,"self_reported":true,"self_reported_source":null,"analysis_method":"RQ","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2025-09-22","param_count":31000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":36,"model_id":"gpt-4.5","model_name":"GPT-4.5","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.554,"normalized_score":0.554,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/gpt-4-1/","analysis_method":"Accuracy","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2025-02-27","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":37,"model_id":"qwen3-vl-8b-thinking","model_name":"Qwen3 VL 8B Thinking","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.53,"normalized_score":0.53,"verified":false,"self_reported":true,"self_reported_source":null,"analysis_method":"RQ","verification_date":null,"provider_id":"deepinfra","input_cost_per_million":0.18,"output_cost_per_million":2.09,"context_window":262144,"announcement_date":"2025-09-22","param_count":9000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":"DeepInfra","best_throughput":null,"throughput_provider":"DeepInfra","context_provider":"DeepInfra"},{"rank":38,"model_id":"command-a-plus-05-2026","model_name":"Command A+","organization_id":"cohere","organization_name":"Cohere","organization_country":"CA","score":0.527,"normalized_score":0.527,"verified":false,"self_reported":true,"self_reported_source":"https://cohere.com/blog/command-a-plus","analysis_method":"CharXiv reasoning; standard methodology reported by Cohere","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-05-20","param_count":218000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":39,"model_id":"qwen3-vl-4b-thinking","model_name":"Qwen3 VL 4B Thinking","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.503,"normalized_score":0.503,"verified":false,"self_reported":true,"self_reported_source":null,"analysis_method":"RQ","verification_date":null,"provider_id":"deepinfra","input_cost_per_million":0.1,"output_cost_per_million":1.0,"context_window":262144,"announcement_date":"2025-09-22","param_count":4000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":"DeepInfra","best_throughput":null,"throughput_provider":"DeepInfra","context_provider":"DeepInfra"},{"rank":40,"model_id":"qwen3-vl-30b-a3b-instruct","model_name":"Qwen3 VL 30B A3B Instruct","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.489,"normalized_score":0.489,"verified":false,"self_reported":true,"self_reported_source":null,"analysis_method":"RQ","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2025-09-22","param_count":31000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":41,"model_id":"qwen3-vl-8b-instruct","model_name":"Qwen3 VL 8B Instruct","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.464,"normalized_score":0.464,"verified":false,"self_reported":true,"self_reported_source":null,"analysis_method":"RQ","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2025-09-22","param_count":9000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":42,"model_id":"gpt-4.1-nano-2025-04-14","model_name":"GPT-4.1 nano","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.405,"normalized_score":0.405,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/gpt-4-1/","analysis_method":"Standard benchmark","verification_date":null,"provider_id":"openai","input_cost_per_million":0.1,"output_cost_per_million":0.4,"context_window":1047576,"announcement_date":"2025-04-14","param_count":null,"is_open_source":false,"is_new":false,"best_latency":2.0,"latency_provider":"OpenAI","best_throughput":200.0,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":43,"model_id":"qwen3-vl-4b-instruct","model_name":"Qwen3 VL 4B Instruct","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.397,"normalized_score":0.397,"verified":false,"self_reported":true,"self_reported_source":null,"analysis_method":"RQ","verification_date":null,"provider_id":"deepinfra","input_cost_per_million":0.1,"output_cost_per_million":0.6,"context_window":262144,"announcement_date":"2025-09-22","param_count":4000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":"DeepInfra","best_throughput":null,"throughput_provider":"DeepInfra","context_provider":"DeepInfra"}]}