{"benchmark_id":"swe-lancer-(ic-diamond-subset)","benchmark_name":"SWE-Lancer (IC-Diamond subset)","benchmark_description":"SWE-Lancer (IC-Diamond subset) is a benchmark of real-world freelance software engineering tasks from Upwork, ranging from $50 bug fixes to $32,000 feature implementations. It evaluates AI models on independent engineering tasks using end-to-end tests triple-verified by experienced software engineers, and includes managerial tasks where models choose between technical implementation proposals.","max_score":1.0,"categories":["reasoning","code"],"modality":"text","total_models":6,"entries":[{"rank":1,"model_id":"gpt-5-2025-08-07","model_name":"GPT-5","organization_name":"OpenAI","organization_id":"openai","benchmark_score":1.0,"normalized_score":1.0,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-08-07","announcement_date":"2025-08-07","multimodal":true,"param_count":null,"is_new":false},{"rank":2,"model_id":"gpt-5.3-codex","model_name":"GPT-5.3 Codex","organization_name":"OpenAI","organization_id":"openai","benchmark_score":0.814,"normalized_score":0.814,"verified":false,"self_reported":true,"provider_id":"openai","input_cost_per_million":1.75,"output_cost_per_million":14.0,"speed_rps":50.0,"context_window":400000,"release_date":"2026-02-05","announcement_date":"2026-02-05","multimodal":true,"param_count":null,"is_new":false},{"rank":3,"model_id":"gpt-5.2-2025-12-11","model_name":"GPT-5.2","organization_name":"OpenAI","organization_id":"openai","benchmark_score":0.746,"normalized_score":0.746,"verified":false,"self_reported":true,"provider_id":"openai","input_cost_per_million":1.75,"output_cost_per_million":14.0,"speed_rps":100.0,"context_window":400000,"release_date":"2025-12-11","announcement_date":"2025-12-11","multimodal":true,"param_count":null,"is_new":false},{"rank":4,"model_id":"gpt-4.5","model_name":"GPT-4.5","organization_name":"OpenAI","organization_id":"openai","benchmark_score":0.174,"normalized_score":0.174,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-02-27","announcement_date":"2025-02-27","multimodal":true,"param_count":null,"is_new":false},{"rank":5,"model_id":"gpt-4o-2024-08-06","model_name":"GPT-4o","organization_name":"OpenAI","organization_id":"openai","benchmark_score":0.124,"normalized_score":0.124,"verified":false,"self_reported":true,"provider_id":"openai","input_cost_per_million":2.5,"output_cost_per_million":10.0,"speed_rps":132.0,"context_window":128000,"release_date":"2024-08-06","announcement_date":"2024-08-06","multimodal":true,"param_count":null,"is_new":false},{"rank":6,"model_id":"o3-mini","model_name":"o3-mini","organization_name":"OpenAI","organization_id":"openai","benchmark_score":0.074,"normalized_score":0.074,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-01-30","announcement_date":"2025-01-30","multimodal":false,"param_count":null,"is_new":false}]}