{"benchmark_id":"swe-lancer","benchmark_name":"SWE-Lancer","benchmark_description":"A benchmark for evaluating large language models on real-world freelance software engineering tasks from Upwork. Contains over 1,400 tasks valued at $1 million USD total, ranging from $50 bug fixes to $32,000 feature implementations. Includes both independent engineering tasks graded via end-to-end tests and managerial tasks assessed against original engineering managers' choices.","max_score":1.0,"categories":["reasoning","code"],"modality":"text","total_models":4,"entries":[{"rank":1,"model_id":"gpt-5.1-codex","model_name":"GPT-5.1 Codex","organization_name":"OpenAI","organization_id":"openai","benchmark_score":0.663,"normalized_score":0.663,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-11-19","announcement_date":"2025-11-19","multimodal":true,"param_count":null,"is_new":false},{"rank":2,"model_id":"gpt-4.5","model_name":"GPT-4.5","organization_name":"OpenAI","organization_id":"openai","benchmark_score":0.373,"normalized_score":0.373,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-02-27","announcement_date":"2025-02-27","multimodal":true,"param_count":null,"is_new":false},{"rank":3,"model_id":"gpt-4o-2024-08-06","model_name":"GPT-4o","organization_name":"OpenAI","organization_id":"openai","benchmark_score":0.326,"normalized_score":0.326,"verified":false,"self_reported":true,"provider_id":"openai","input_cost_per_million":2.5,"output_cost_per_million":10.0,"speed_rps":132.0,"context_window":128000,"release_date":"2024-08-06","announcement_date":"2024-08-06","multimodal":true,"param_count":null,"is_new":false},{"rank":4,"model_id":"o3-mini","model_name":"o3-mini","organization_name":"OpenAI","organization_id":"openai","benchmark_score":0.18,"normalized_score":0.18,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2025-01-30","announcement_date":"2025-01-30","multimodal":false,"param_count":null,"is_new":false}]}