{"benchmark_id":"bird-sql-(dev)","benchmark_name":"Bird-SQL (dev)","benchmark_description":"BIRD (BIg Bench for LaRge-scale Database Grounded Text-to-SQLs) is a comprehensive text-to-SQL benchmark containing 12,751 question-SQL pairs across 95 databases (33.4 GB total) spanning 37+ professional domains. It evaluates large language models' ability to convert natural language to executable SQL queries in real-world scenarios with complex database schemas and dirty data.","max_score":1.0,"categories":["reasoning"],"modality":"text","total_models":7,"entries":[{"rank":1,"model_id":"gemini-2.0-flash-lite","model_name":"Gemini 2.0 Flash-Lite","organization_name":"Google","organization_id":"google","benchmark_score":0.574,"normalized_score":0.574,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-02-05","announcement_date":"2025-02-05","multimodal":true,"param_count":null,"is_new":false},{"rank":2,"model_id":"gemini-2.0-flash","model_name":"Gemini 2.0 Flash","organization_name":"Google","organization_id":"google","benchmark_score":0.569,"normalized_score":0.569,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-12-01","announcement_date":"2024-12-01","multimodal":true,"param_count":null,"is_new":false},{"rank":3,"model_id":"gemma-3-27b-it","model_name":"Gemma 3 27B","organization_name":"Google","organization_id":"google","benchmark_score":0.544,"normalized_score":0.544,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-03-12","announcement_date":"2025-03-12","multimodal":true,"param_count":27000000000,"is_new":false},{"rank":4,"model_id":"gemma-3-12b-it","model_name":"Gemma 3 12B","organization_name":"Google","organization_id":"google","benchmark_score":0.479,"normalized_score":0.479,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-03-12","announcement_date":"2025-03-12","multimodal":true,"param_count":12000000000,"is_new":false},{"rank":5,"model_id":"nemotron-3-super-120b-a12b","model_name":"Nemotron 3 Super (120B A12B)","organization_name":"NVIDIA","organization_id":"nvidia","benchmark_score":0.418,"normalized_score":0.418,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2026-03-11","announcement_date":"2026-03-11","multimodal":false,"param_count":120000000000,"is_new":false},{"rank":6,"model_id":"gemma-3-4b-it","model_name":"Gemma 3 4B","organization_name":"Google","organization_id":"google","benchmark_score":0.363,"normalized_score":0.363,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-03-12","announcement_date":"2025-03-12","multimodal":true,"param_count":4000000000,"is_new":false},{"rank":7,"model_id":"gemma-3-1b-it","model_name":"Gemma 3 1B","organization_name":"Google","organization_id":"google","benchmark_score":0.064,"normalized_score":0.064,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-03-12","announcement_date":"2025-03-12","multimodal":false,"param_count":1000000000,"is_new":false}]}