{"benchmark_id":"openai-mrcr:-2-needle-1m","name":"OpenAI-MRCR: 2 needle 1M","parent_benchmark":null,"categories":["reasoning","long_context"],"modality":"text","multilingual":false,"max_score":1.0,"language":"en","description":"Multi-Round Co-reference Resolution benchmark that tests an LLM's ability to distinguish between multiple similar needles hidden in long conversations. Models must reproduce specific instances of content (e.g., 'Return the 2nd poem about tapirs') from multi-turn synthetic conversations, requiring reasoning about context, ordering, and subtle differences between similar outputs.","paper_link":"https://arxiv.org/abs/2409.12640","implementation_link":null,"verified":false,"created_at":"2026-05-07T16:53:25.617739+00:00","updated_at":"2026-07-05T18:27:58.030749+00:00","statistics":{"total_models":5,"average_score":0.4128,"min_score":0.12,"max_score":0.586,"score_stddev":0.19162385029009307,"verified_count":0,"self_reported_count":5},"child_benchmarks":[],"linked_dataset":null,"models":[{"rank":1,"model_id":"minimax-m1-40k","model_name":"MiniMax M1 40K","organization_id":"minimax","organization_name":"MiniMax","organization_country":"CN","score":0.586,"normalized_score":0.586,"verified":false,"self_reported":true,"self_reported_source":null,"analysis_method":null,"verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2025-06-16","param_count":456000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":2,"model_id":"minimax-m1-80k","model_name":"MiniMax M1 80K","organization_id":"minimax","organization_name":"MiniMax","organization_country":"CN","score":0.562,"normalized_score":0.562,"verified":false,"self_reported":true,"self_reported_source":null,"analysis_method":null,"verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2025-06-16","param_count":456000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":3,"model_id":"gpt-4.1-2025-04-14","model_name":"GPT-4.1","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.463,"normalized_score":0.463,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/introducing-gpt-5-for-developers/","analysis_method":"Internal benchmark","verification_date":null,"provider_id":"openai","input_cost_per_million":2.0,"output_cost_per_million":8.0,"context_window":1047576,"announcement_date":"2025-04-14","param_count":null,"is_open_source":false,"is_new":false,"best_latency":10.0,"latency_provider":"OpenAI","best_throughput":100.0,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":4,"model_id":"gpt-4.1-mini-2025-04-14","model_name":"GPT-4.1 mini","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.333,"normalized_score":0.333,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/introducing-gpt-5-for-developers/","analysis_method":"Internal benchmark","verification_date":null,"provider_id":"openai","input_cost_per_million":0.4,"output_cost_per_million":1.6,"context_window":1047576,"announcement_date":"2025-04-14","param_count":null,"is_open_source":false,"is_new":false,"best_latency":5.0,"latency_provider":"OpenAI","best_throughput":150.0,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":5,"model_id":"gpt-4.1-nano-2025-04-14","model_name":"GPT-4.1 nano","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.12,"normalized_score":0.12,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/gpt-4-1/","analysis_method":"Internal benchmark","verification_date":null,"provider_id":"openai","input_cost_per_million":0.1,"output_cost_per_million":0.4,"context_window":1047576,"announcement_date":"2025-04-14","param_count":null,"is_open_source":false,"is_new":false,"best_latency":2.0,"latency_provider":"OpenAI","best_throughput":200.0,"throughput_provider":"OpenAI","context_provider":"OpenAI"}]}