{"benchmark_id":"tau-bench","benchmark_name":"Tau-bench","benchmark_description":"τ-bench: A benchmark for tool-agent-user interaction in real-world domains. Tests language agents' ability to interact with users and follow domain-specific rules through dynamic conversations using API tools and policy guidelines across retail and airline domains. Evaluates consistency and reliability of agent behavior over multiple trials.","max_score":1.0,"categories":["reasoning","general","agents","tool_calling"],"modality":"text","total_models":6,"entries":[{"rank":1,"model_id":"step-3.5-flash","model_name":"Step-3.5-Flash","organization_name":"StepFun","organization_id":"stepfun","benchmark_score":0.882,"normalized_score":0.882,"verified":false,"self_reported":true,"provider_id":"stepfun","input_cost_per_million":0.1,"output_cost_per_million":0.4,"speed_rps":150.0,"context_window":65536,"release_date":"2026-02-02","announcement_date":"2026-02-02","multimodal":false,"param_count":196000000000,"is_new":false},{"rank":2,"model_id":"glm-4.7","model_name":"GLM-4.7","organization_name":"Zhipu AI","organization_id":"zai-org","benchmark_score":0.874,"normalized_score":0.874,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-12-22","announcement_date":"2025-12-22","multimodal":true,"param_count":358000000000,"is_new":false},{"rank":3,"model_id":"mimo-v2-flash","model_name":"MiMo-V2-Flash","organization_name":"Xiaomi","organization_id":"xiaomi","benchmark_score":0.803,"normalized_score":0.803,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-12-16","announcement_date":"2025-12-16","multimodal":false,"param_count":309000000000,"is_new":false},{"rank":4,"model_id":"glm-4.7-flash","model_name":"GLM-4.7-Flash","organization_name":"Zhipu AI","organization_id":"zai-org","benchmark_score":0.795,"normalized_score":0.795,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2026-01-19","announcement_date":"2026-01-19","multimodal":false,"param_count":30000000000,"is_new":false},{"rank":5,"model_id":"minimax-m2","model_name":"MiniMax M2","organization_name":"MiniMax","organization_id":"minimax","benchmark_score":0.772,"normalized_score":0.772,"verified":false,"self_reported":true,"provider_id":"novita","input_cost_per_million":0.3,"output_cost_per_million":1.2,"speed_rps":null,"context_window":204800,"release_date":"2025-10-27","announcement_date":"2025-10-27","multimodal":false,"param_count":230000000000,"is_new":false},{"rank":6,"model_id":"o3-2025-04-16","model_name":"o3","organization_name":"OpenAI","organization_id":"openai","benchmark_score":0.63,"normalized_score":0.63,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-04-16","announcement_date":"2025-04-16","multimodal":true,"param_count":null,"is_new":false}]}