{"benchmark_id":"mrcr","benchmark_name":"MRCR","benchmark_description":"MRCR (Multi-Round Coreference Resolution) is a synthetic long-context reasoning task where models must navigate long conversations to reproduce specific model outputs. It tests the ability to distinguish between similar requests and reason about ordering while maintaining attention across extended contexts.","max_score":1.0,"categories":["long_context","reasoning","general"],"modality":"text","total_models":7,"entries":[{"rank":1,"model_id":"gemini-2.5-pro","model_name":"Gemini 2.5 Pro","organization_name":"Google","organization_id":"google","benchmark_score":0.93,"normalized_score":0.93,"verified":false,"self_reported":true,"provider_id":"google","input_cost_per_million":1.25,"output_cost_per_million":10.0,"speed_rps":85.0,"context_window":1048576,"release_date":"2025-05-20","announcement_date":"2025-05-20","multimodal":true,"param_count":null,"is_new":false},{"rank":2,"model_id":"gemini-1.5-pro","model_name":"Gemini 1.5 Pro","organization_name":"Google","organization_id":"google","benchmark_score":0.826,"normalized_score":0.826,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-05-01","announcement_date":"2024-05-01","multimodal":true,"param_count":null,"is_new":false},{"rank":3,"model_id":"gemini-1.5-flash","model_name":"Gemini 1.5 Flash","organization_name":"Google","organization_id":"google","benchmark_score":0.719,"normalized_score":0.719,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-05-01","announcement_date":"2024-05-01","multimodal":true,"param_count":null,"is_new":false},{"rank":4,"model_id":"gemini-2.0-flash","model_name":"Gemini 2.0 Flash","organization_name":"Google","organization_id":"google","benchmark_score":0.692,"normalized_score":0.692,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-12-01","announcement_date":"2024-12-01","multimodal":true,"param_count":null,"is_new":false},{"rank":5,"model_id":"gemini-1.5-flash-8b","model_name":"Gemini 1.5 Flash 8B","organization_name":"Google","organization_id":"google","benchmark_score":0.547,"normalized_score":0.547,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-03-15","announcement_date":"2024-03-15","multimodal":true,"param_count":8000000000,"is_new":false},{"rank":6,"model_id":"mimo-v2-flash","model_name":"MiMo-V2-Flash","organization_name":"Xiaomi","organization_id":"xiaomi","benchmark_score":0.457,"normalized_score":0.457,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-12-16","announcement_date":"2025-12-16","multimodal":false,"param_count":309000000000,"is_new":false},{"rank":7,"model_id":"gemini-2.5-flash","model_name":"Gemini 2.5 Flash","organization_name":"Google","organization_id":"google","benchmark_score":0.32,"normalized_score":0.32,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-05-20","announcement_date":"2025-05-20","multimodal":true,"param_count":null,"is_new":false}]}