{"benchmark_id":"natural-questions","benchmark_name":"Natural Questions","benchmark_description":"Natural Questions is a question answering dataset featuring real anonymized queries issued to Google search engine. It contains 307,373 training examples where annotators provide long answers (passages) and short answers (entities) from Wikipedia pages, or mark them as unanswerable.","max_score":1.0,"categories":["reasoning","search","general"],"modality":"text","total_models":7,"entries":[{"rank":1,"model_id":"gemma-2-27b-it","model_name":"Gemma 2 27B","organization_name":"Google","organization_id":"google","benchmark_score":0.345,"normalized_score":0.345,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-06-27","announcement_date":"2024-06-27","multimodal":false,"param_count":27200000000,"is_new":false},{"rank":2,"model_id":"mistral-nemo-instruct-2407","model_name":"Mistral NeMo Instruct","organization_name":"Mistral AI","organization_id":"mistral","benchmark_score":0.312,"normalized_score":0.312,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-07-18","announcement_date":"2024-07-18","multimodal":false,"param_count":12000000000,"is_new":false},{"rank":3,"model_id":"gemma-2-9b-it","model_name":"Gemma 2 9B","organization_name":"Google","organization_id":"google","benchmark_score":0.292,"normalized_score":0.292,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-06-27","announcement_date":"2024-06-27","multimodal":false,"param_count":9240000000,"is_new":false},{"rank":4,"model_id":"gemma-3n-e4b","model_name":"Gemma 3n E4B","organization_name":"Google","organization_id":"google","benchmark_score":0.209,"normalized_score":0.209,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-06-26","announcement_date":"2025-06-26","multimodal":true,"param_count":8000000000,"is_new":false},{"rank":5,"model_id":"gemma-3n-e4b-it-litert-preview","model_name":"Gemma 3n E4B Instructed LiteRT Preview","organization_name":"Google","organization_id":"google","benchmark_score":0.209,"normalized_score":0.209,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-05-20","announcement_date":"2025-05-20","multimodal":true,"param_count":1910000000,"is_new":false},{"rank":6,"model_id":"gemma-3n-e2b","model_name":"Gemma 3n E2B","organization_name":"Google","organization_id":"google","benchmark_score":0.155,"normalized_score":0.155,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-06-26","announcement_date":"2025-06-26","multimodal":true,"param_count":8000000000,"is_new":false},{"rank":7,"model_id":"gemma-3n-e2b-it-litert-preview","model_name":"Gemma 3n E2B Instructed LiteRT (Preview)","organization_name":"Google","organization_id":"google","benchmark_score":0.155,"normalized_score":0.155,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-05-20","announcement_date":"2025-05-20","multimodal":true,"param_count":1910000000,"is_new":false}]}