{"benchmark_id":"bigcodebench","benchmark_name":"BigCodeBench","benchmark_description":"A benchmark that challenges LLMs to invoke multiple function calls as tools from 139 libraries and 7 domains for 1,140 fine-grained programming tasks. Evaluates code generation with diverse function calls and complex instructions, featuring two variants: Complete (code completion based on comprehensive docstrings) and Instruct (generating code from natural language instructions).","max_score":1.0,"categories":["reasoning","general"],"modality":"text","total_models":2,"entries":[{"rank":1,"model_id":"gemini-diffusion","model_name":"Gemini Diffusion","organization_name":"Google","organization_id":"google","benchmark_score":0.454,"normalized_score":0.454,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-05-20","announcement_date":"2025-05-20","multimodal":false,"param_count":null,"is_new":false},{"rank":2,"model_id":"qwen-2.5-coder-7b-instruct","model_name":"Qwen2.5-Coder 7B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.41,"normalized_score":0.41,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-09-19","announcement_date":"2024-09-19","multimodal":false,"param_count":7000000000,"is_new":false}]}