{"benchmark_id":"cybersecurity-ctfs","name":"Cybersecurity CTFs","parent_benchmark":null,"categories":["safety"],"modality":"text","multilingual":false,"max_score":1.0,"language":"en","description":"Cybersecurity Capture the Flag (CTF) benchmark for evaluating LLMs in offensive security challenges. Contains diverse cybersecurity tasks including cryptography, web exploitation, binary analysis, and forensics to assess AI capabilities in cybersecurity problem-solving.","paper_link":"https://arxiv.org/abs/2406.05590","implementation_link":null,"verified":false,"created_at":"2026-05-07T16:53:22.764474+00:00","updated_at":"2026-07-05T18:27:46.696959+00:00","statistics":{"total_models":3,"average_score":0.5105833333333333,"min_score":0.287,"max_score":0.776,"score_stddev":0.24716951841465673,"verified_count":0,"self_reported_count":3},"child_benchmarks":[],"linked_dataset":null,"models":[{"rank":1,"model_id":"gpt-5.3-codex","model_name":"GPT-5.3 Codex","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.776,"normalized_score":0.776,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/introducing-gpt-5-3-codex/","analysis_method":"Cybersecurity Capture The Flag Challenges. xhigh effort.","verification_date":null,"provider_id":"openai","input_cost_per_million":1.75,"output_cost_per_million":14.0,"context_window":400000,"announcement_date":"2026-02-05","param_count":null,"is_open_source":false,"is_new":false,"best_latency":3.0,"latency_provider":"OpenAI","best_throughput":50.0,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":2,"model_id":"claude-haiku-4-5-20251001","model_name":"Claude Haiku 4.5","organization_id":"anthropic","organization_name":"Anthropic","organization_country":"US","score":0.46875,"normalized_score":0.46875,"verified":false,"self_reported":true,"self_reported_source":"https://assets.anthropic.com/m/99128ddd009bdcb/original/Claude-Haiku-4-5-System-Card.pdf","analysis_method":"32-challenge subset","verification_date":null,"provider_id":"anthropic","input_cost_per_million":1.0,"output_cost_per_million":5.0,"context_window":200000,"announcement_date":"2025-10-15","param_count":null,"is_open_source":false,"is_new":false,"best_latency":0.3,"latency_provider":"Anthropic","best_throughput":100.0,"throughput_provider":"Anthropic","context_provider":"Anthropic"},{"rank":3,"model_id":"o1-mini","model_name":"o1-mini","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.287,"normalized_score":0.287,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/openai-o1-mini-advancing-cost-efficient-reasoning/","analysis_method":"Pass@12 accuracy","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2024-09-12","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null}]}