{"benchmark_id":"toolathlon","name":"Toolathlon","parent_benchmark":null,"categories":["reasoning","agents","tool_calling"],"modality":"text","multilingual":false,"max_score":1.0,"language":"en","description":"Tool Decathlon is a comprehensive benchmark for evaluating AI agents' ability to use multiple tools across diverse task categories. It measures proficiency in tool selection, sequencing, and execution across ten different tool-use scenarios.","paper_link":null,"implementation_link":null,"verified":false,"created_at":"2026-05-07T16:53:26.562829+00:00","updated_at":"2026-07-17T06:43:10.845373+00:00","statistics":{"total_models":29,"average_score":0.47579310344827586,"min_score":0.269,"max_score":0.732,"score_stddev":0.09502419966601923,"verified_count":0,"self_reported_count":29},"child_benchmarks":[],"linked_dataset":null,"models":[{"rank":1,"model_id":"kimi-k3","model_name":"Kimi K3","organization_id":"moonshotai","organization_name":"Moonshot AI","organization_country":"CN","score":0.732,"normalized_score":0.732,"verified":false,"self_reported":true,"self_reported_source":"https://www.kimi.com/blog/kimi-k3","analysis_method":"Toolathlon-Verified; max reasoning effort","verification_date":null,"provider_id":"moonshot","input_cost_per_million":3.0,"output_cost_per_million":15.0,"context_window":1048576,"announcement_date":"2026-07-16","param_count":2800000000000,"is_open_source":true,"is_new":true,"best_latency":null,"latency_provider":"Moonshot AI","best_throughput":null,"throughput_provider":"Moonshot AI","context_provider":"Moonshot AI"},{"rank":2,"model_id":"claude-opus-4-8","model_name":"Claude Opus 4.8","organization_id":"anthropic","organization_name":"Anthropic","organization_country":"US","score":0.599,"normalized_score":0.599,"verified":false,"self_reported":true,"self_reported_source":"https://www.anthropic.com/news/claude-opus-4-8","analysis_method":"Pass@1 averaged over 3 trials across all 108 tasks. Internal harness with adaptive thinking at max effort. Pass@3: 67.6%.","verification_date":null,"provider_id":"anthropic","input_cost_per_million":5.0,"output_cost_per_million":25.0,"context_window":1000000,"announcement_date":"2026-05-28","param_count":null,"is_open_source":false,"is_new":false,"best_latency":0.5,"latency_provider":"Anthropic","best_throughput":42.0,"throughput_provider":"Anthropic","context_provider":"Anthropic"},{"rank":3,"model_id":"gpt-5.6-sol","model_name":"GPT-5.6 Sol","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.58,"normalized_score":0.58,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/gpt-5-6/","analysis_method":"Toolathlon. Max reasoning effort.","verification_date":null,"provider_id":"openai","input_cost_per_million":5.0,"output_cost_per_million":30.0,"context_window":1050000,"announcement_date":"2026-07-09","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":"OpenAI","best_throughput":null,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":4,"model_id":"gemini-3.5-flash","model_name":"Gemini 3.5 Flash","organization_id":"google","organization_name":"Google","organization_country":"US","score":0.565,"normalized_score":0.565,"verified":false,"self_reported":true,"self_reported_source":"https://deepmind.google/models/evals-methodology/gemini-3-5-flash/","analysis_method":null,"verification_date":null,"provider_id":"google","input_cost_per_million":1.5,"output_cost_per_million":9.0,"context_window":1048576,"announcement_date":"2026-05-19","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":"Google","best_throughput":null,"throughput_provider":"Google","context_provider":"Google"},{"rank":5,"model_id":"gpt-5.5","model_name":"GPT-5.5","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.556,"normalized_score":0.556,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/introducing-gpt-5-5/","analysis_method":"Toolathlon tool-use agentic tasks benchmark. Reasoning effort xhigh.","verification_date":null,"provider_id":"openai","input_cost_per_million":5.0,"output_cost_per_million":30.0,"context_window":1050000,"announcement_date":"2026-04-23","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":"OpenAI","best_throughput":null,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":6,"model_id":"gpt-5.4","model_name":"GPT-5.4","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.546,"normalized_score":0.546,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/introducing-gpt-5-4/","analysis_method":"Toolathlon tool-use agentic tasks benchmark. Reasoning effort xhigh.","verification_date":null,"provider_id":"openai","input_cost_per_million":2.5,"output_cost_per_million":15.0,"context_window":1000000,"announcement_date":"2026-03-05","param_count":null,"is_open_source":false,"is_new":false,"best_latency":3.0,"latency_provider":"OpenAI","best_throughput":50.0,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":7,"model_id":"claude-sonnet-5","model_name":"Claude Sonnet 5","organization_id":"anthropic","organization_name":"Anthropic","organization_country":"US","score":0.543,"normalized_score":0.543,"verified":false,"self_reported":true,"self_reported_source":"https://www.anthropic.com/news/claude-sonnet-5","analysis_method":"Pass@1 averaged over 3 trials across all 108 tasks. Internal harness with adaptive thinking at max effort. Pass@3: 63.0%, Pass^3: 40.7%.","verification_date":null,"provider_id":"anthropic","input_cost_per_million":3.0,"output_cost_per_million":15.0,"context_window":1000000,"announcement_date":"2026-06-30","param_count":null,"is_open_source":false,"is_new":false,"best_latency":0.4,"latency_provider":"Anthropic","best_throughput":42.0,"throughput_provider":"Anthropic","context_provider":"Anthropic"},{"rank":8,"model_id":"gpt-5.6-luna","model_name":"GPT-5.6 Luna","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.534,"normalized_score":0.534,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/gpt-5-6/","analysis_method":"Toolathlon. Max reasoning effort.","verification_date":null,"provider_id":"openai","input_cost_per_million":1.0,"output_cost_per_million":6.0,"context_window":1050000,"announcement_date":"2026-07-09","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":"OpenAI","best_throughput":null,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":9,"model_id":"gpt-5.6-terra","model_name":"GPT-5.6 Terra","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.531,"normalized_score":0.531,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/gpt-5-6/","analysis_method":"Toolathlon. Max reasoning effort.","verification_date":null,"provider_id":"openai","input_cost_per_million":2.5,"output_cost_per_million":15.0,"context_window":1050000,"announcement_date":"2026-07-09","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":"OpenAI","best_throughput":null,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":10,"model_id":"deepseek-v4-pro-max","model_name":"DeepSeek-V4-Pro-Max","organization_id":"deepseek","organization_name":"DeepSeek","organization_country":"CN","score":0.518,"normalized_score":0.518,"verified":false,"self_reported":true,"self_reported_source":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro","analysis_method":"Pass@1","verification_date":null,"provider_id":"novita","input_cost_per_million":1.6,"output_cost_per_million":3.2,"context_window":1048576,"announcement_date":"2026-04-23","param_count":1600000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":"Novita","best_throughput":null,"throughput_provider":"Novita","context_provider":"DeepInfra"},{"rank":11,"model_id":"seed-2.1-pro","model_name":"Seed 2.1 Pro","organization_id":"bytedance","organization_name":"ByteDance","organization_country":"CN","score":0.506,"normalized_score":0.506,"verified":false,"self_reported":true,"self_reported_source":"https://seed.bytedance.com/en/seed2_1","analysis_method":null,"verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-06-24","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":12,"model_id":"kimi-k2.6","model_name":"Kimi K2.6","organization_id":"moonshotai","organization_name":"Moonshot AI","organization_country":"CN","score":0.5,"normalized_score":0.5,"verified":false,"self_reported":true,"self_reported_source":"https://www.kimi.com/blog/kimi-k2-6","analysis_method":null,"verification_date":null,"provider_id":"deepinfra","input_cost_per_million":0.75,"output_cost_per_million":3.5,"context_window":262144,"announcement_date":"2026-04-20","param_count":1000000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":"DeepInfra","best_throughput":null,"throughput_provider":"DeepInfra","context_provider":"Fireworks"},{"rank":13,"model_id":"gemini-3-flash-preview","model_name":"Gemini 3 Flash","organization_id":"google","organization_name":"Google","organization_country":"US","score":0.494,"normalized_score":0.494,"verified":false,"self_reported":true,"self_reported_source":"https://blog.google/products/gemini/gemini-3-flash/","analysis_method":null,"verification_date":null,"provider_id":"google","input_cost_per_million":0.5,"output_cost_per_million":3.0,"context_window":1000000,"announcement_date":"2025-12-17","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":"Google","best_throughput":null,"throughput_provider":"Google","context_provider":"Google"},{"rank":14,"model_id":"seed-2.1-turbo","model_name":"Seed 2.1 Turbo","organization_id":"bytedance","organization_name":"ByteDance","organization_country":"CN","score":0.491,"normalized_score":0.491,"verified":false,"self_reported":true,"self_reported_source":"https://seed.bytedance.com/en/seed2_1","analysis_method":null,"verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-06-24","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":15,"model_id":"hy3","model_name":"Hy3","organization_id":"tencent","organization_name":"Tencent","organization_country":"CN","score":0.485,"normalized_score":0.485,"verified":false,"self_reported":true,"self_reported_source":"https://huggingface.co/tencent/Hy3","analysis_method":null,"verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-07-06","param_count":295000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":16,"model_id":"glm-5.2","model_name":"GLM-5.2","organization_id":"zai-org","organization_name":"Zhipu AI","organization_country":"CN","score":0.482,"normalized_score":0.482,"verified":false,"self_reported":true,"self_reported_source":"https://z.ai/blog/glm-5.2","analysis_method":null,"verification_date":null,"provider_id":"deepinfra","input_cost_per_million":0.95,"output_cost_per_million":3.0,"context_window":1048576,"announcement_date":"2026-06-16","param_count":753000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":"Fireworks","best_throughput":null,"throughput_provider":"Fireworks","context_provider":"DeepInfra"},{"rank":17,"model_id":"deepseek-v4-flash-max","model_name":"DeepSeek-V4-Flash-Max","organization_id":"deepseek","organization_name":"DeepSeek","organization_country":"CN","score":0.478,"normalized_score":0.478,"verified":false,"self_reported":true,"self_reported_source":"https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash","analysis_method":"Pass@1","verification_date":null,"provider_id":"deepinfra","input_cost_per_million":0.1,"output_cost_per_million":0.2,"context_window":1048576,"announcement_date":"2026-04-23","param_count":284000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":"DeepInfra","best_throughput":null,"throughput_provider":"DeepInfra","context_provider":"DeepSeek"},{"rank":18,"model_id":"gpt-5.2-2025-12-11","model_name":"GPT-5.2","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.463,"normalized_score":0.463,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/introducing-gpt-5-2/","analysis_method":"GPT-5.2 Thinking - Tool Decathlon.","verification_date":null,"provider_id":"openai","input_cost_per_million":1.75,"output_cost_per_million":14.0,"context_window":400000,"announcement_date":"2025-12-11","param_count":null,"is_open_source":false,"is_new":false,"best_latency":2.0,"latency_provider":"OpenAI","best_throughput":100.0,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":18,"model_id":"minimax-m2.7","model_name":"MiniMax M2.7","organization_id":"minimax","organization_name":"MiniMax","organization_country":"CN","score":0.463,"normalized_score":0.463,"verified":false,"self_reported":true,"self_reported_source":"https://www.minimax.io/models/text/m27","analysis_method":null,"verification_date":null,"provider_id":"novita","input_cost_per_million":0.3,"output_cost_per_million":1.2,"context_window":204800,"announcement_date":"2026-03-18","param_count":null,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":"Novita","best_throughput":null,"throughput_provider":"Novita","context_provider":"Novita"},{"rank":20,"model_id":"minimax-m2.1","model_name":"MiniMax M2.1","organization_id":"minimax","organization_name":"MiniMax","organization_country":"CN","score":0.435,"normalized_score":0.435,"verified":false,"self_reported":true,"self_reported_source":"https://www.minimax.io/news/minimax-m2-1","analysis_method":null,"verification_date":null,"provider_id":"minimax","input_cost_per_million":0.3,"output_cost_per_million":1.2,"context_window":1000000,"announcement_date":"2025-12-23","param_count":230000000000,"is_open_source":true,"is_new":false,"best_latency":3.0,"latency_provider":"MiniMax","best_throughput":100.0,"throughput_provider":"MiniMax","context_provider":"MiniMax"},{"rank":21,"model_id":"gpt-5.4-mini","model_name":"GPT-5.4 mini","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.429,"normalized_score":0.429,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/introducing-gpt-5-4-mini-and-nano/","analysis_method":"Toolathlon tool-use benchmark. Reasoning effort xhigh.","verification_date":null,"provider_id":"openai","input_cost_per_million":0.75,"output_cost_per_million":4.5,"context_window":400000,"announcement_date":"2026-03-17","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":"OpenAI","best_throughput":null,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":22,"model_id":"glm-5.1","model_name":"GLM-5.1","organization_id":"zai-org","organization_name":"Zhipu AI","organization_country":"CN","score":0.407,"normalized_score":0.407,"verified":false,"self_reported":true,"self_reported_source":"https://z.ai/blog/glm-5.1","analysis_method":null,"verification_date":null,"provider_id":"friendli","input_cost_per_million":1.4,"output_cost_per_million":4.4,"context_window":200000,"announcement_date":"2026-04-07","param_count":754000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":"FriendliAI","best_throughput":null,"throughput_provider":"FriendliAI","context_provider":"FriendliAI"},{"rank":23,"model_id":"qwen3.6-plus","model_name":"Qwen3.6 Plus","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.398,"normalized_score":0.398,"verified":false,"self_reported":true,"self_reported_source":"https://qwen.ai/blog?id=qwen3.6","analysis_method":null,"verification_date":null,"provider_id":"together","input_cost_per_million":0.5,"output_cost_per_million":3.0,"context_window":1000000,"announcement_date":"2026-04-02","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":"Together","best_throughput":null,"throughput_provider":"Together","context_provider":"Together"},{"rank":24,"model_id":"qwen3.5-397b-a17b","model_name":"Qwen3.5-397B-A17B","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.383,"normalized_score":0.383,"verified":false,"self_reported":true,"self_reported_source":"https://qwenlm.github.io/blog/qwen3.5/","analysis_method":null,"verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-02-16","param_count":397000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":25,"model_id":"gpt-5.4-nano","model_name":"GPT-5.4 nano","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.355,"normalized_score":0.355,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/introducing-gpt-5-4-mini-and-nano/","analysis_method":"Toolathlon tool-use benchmark. Reasoning effort xhigh.","verification_date":null,"provider_id":"openai","input_cost_per_million":0.2,"output_cost_per_million":1.25,"context_window":400000,"announcement_date":"2026-03-17","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":"OpenAI","best_throughput":null,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":26,"model_id":"deepseek-v3.2","model_name":"DeepSeek-V3.2","organization_id":"deepseek","organization_name":"DeepSeek","organization_country":"CN","score":0.352,"normalized_score":0.352,"verified":false,"self_reported":true,"self_reported_source":"https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/main/assets/paper.pdf","analysis_method":"Pass@1 (Agentic Tool Use)","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2025-12-01","param_count":685000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":26,"model_id":"deepseek-reasoner","model_name":"DeepSeek-V3.2 (Thinking)","organization_id":"deepseek","organization_name":"DeepSeek","organization_country":"CN","score":0.352,"normalized_score":0.352,"verified":false,"self_reported":true,"self_reported_source":"https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/main/assets/paper.pdf","analysis_method":"Pass@1","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2025-12-01","param_count":685000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":26,"model_id":"deepseek-v3.2-speciale","model_name":"DeepSeek-V3.2-Speciale","organization_id":"deepseek","organization_name":"DeepSeek","organization_country":"CN","score":0.352,"normalized_score":0.352,"verified":false,"self_reported":true,"self_reported_source":"https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/main/assets/paper.pdf","analysis_method":"Pass@1","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2025-12-01","param_count":685000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":29,"model_id":"qwen3.6-35b-a3b","model_name":"Qwen3.6-35B-A3B","organization_id":"qwen","organization_name":"Alibaba Cloud / Qwen Team","organization_country":"CN","score":0.269,"normalized_score":0.269,"verified":false,"self_reported":true,"self_reported_source":"https://qwen.ai/blog?id=qwen3.6-35b-a3b","analysis_method":null,"verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-04-16","param_count":35000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null}]}