{"benchmark_id":"cybergym","name":"CyberGym","parent_benchmark":null,"categories":["safety","agents","code"],"modality":"text","multilingual":false,"max_score":1.0,"language":"en","description":"CyberGym is a benchmark for evaluating AI agents on cybersecurity tasks, testing their ability to identify vulnerabilities, perform security analysis, and complete security-related challenges in a controlled environment.","paper_link":null,"implementation_link":null,"verified":false,"created_at":"2026-05-07T16:53:22.731535+00:00","updated_at":"2026-07-17T06:42:48.369761+00:00","statistics":{"total_models":9,"average_score":0.7086666666666667,"min_score":0.413,"max_score":0.831,"score_stddev":0.12449899597988734,"verified_count":0,"self_reported_count":9},"child_benchmarks":[],"linked_dataset":null,"models":[{"rank":1,"model_id":"claude-mythos-preview","model_name":"Claude Mythos Preview","organization_id":"anthropic","organization_name":"Anthropic","organization_country":"US","score":0.831,"normalized_score":0.831,"verified":false,"self_reported":true,"self_reported_source":"https://www.anthropic.com/news/project-glasswing","analysis_method":"Cybersecurity vulnerability reproduction.","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-04-07","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":2,"model_id":"gpt-5.5","model_name":"GPT-5.5","organization_id":"openai","organization_name":"OpenAI","organization_country":"US","score":0.818,"normalized_score":0.818,"verified":false,"self_reported":true,"self_reported_source":"https://openai.com/index/introducing-gpt-5-5/","analysis_method":"CyberGym. Reasoning effort xhigh.","verification_date":null,"provider_id":"openai","input_cost_per_million":5.0,"output_cost_per_million":30.0,"context_window":1050000,"announcement_date":"2026-04-23","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":"OpenAI","best_throughput":null,"throughput_provider":"OpenAI","context_provider":"OpenAI"},{"rank":3,"model_id":"claude-opus-4-8","model_name":"Claude Opus 4.8","organization_id":"anthropic","organization_name":"Anthropic","organization_country":"US","score":0.788,"normalized_score":0.788,"verified":false,"self_reported":true,"self_reported_source":"https://www.anthropic.com/news/claude-opus-4-8","analysis_method":"Targeted vulnerability reproduction (pass@1) over 1,507 tasks without deployed safeguards. With Tier-3 safeguards: 1.0%.","verification_date":null,"provider_id":"anthropic","input_cost_per_million":5.0,"output_cost_per_million":25.0,"context_window":1000000,"announcement_date":"2026-05-28","param_count":null,"is_open_source":false,"is_new":false,"best_latency":0.5,"latency_provider":"Anthropic","best_throughput":42.0,"throughput_provider":"Anthropic","context_provider":"Anthropic"},{"rank":4,"model_id":"claude-opus-4-6","model_name":"Claude Opus 4.6","organization_id":"anthropic","organization_name":"Anthropic","organization_country":"US","score":0.738,"normalized_score":0.738,"verified":false,"self_reported":true,"self_reported_source":"https://www.anthropic.com/news/claude-opus-4-6","analysis_method":"Cybersecurity vulnerability reproduction. No thinking, default effort, temperature, and top_p. Given a 'think' tool for interleaved thinking in multi-turn evaluations.","verification_date":null,"provider_id":"anthropic","input_cost_per_million":5.0,"output_cost_per_million":25.0,"context_window":1000000,"announcement_date":"2026-02-05","param_count":null,"is_open_source":false,"is_new":false,"best_latency":0.5,"latency_provider":"Anthropic","best_throughput":42.0,"throughput_provider":"Anthropic","context_provider":"Anthropic"},{"rank":5,"model_id":"claude-opus-4-7","model_name":"Claude Opus 4.7","organization_id":"anthropic","organization_name":"Anthropic","organization_country":"US","score":0.731,"normalized_score":0.731,"verified":false,"self_reported":true,"self_reported_source":"https://www.anthropic.com/news/claude-opus-4-7","analysis_method":"Cybersecurity vulnerability reproduction. Opus 4.6's score was updated from the originally reported 66.6 to 73.8 after harness parameter updates to better elicit cyber capability.","verification_date":null,"provider_id":"anthropic","input_cost_per_million":5.0,"output_cost_per_million":25.0,"context_window":1000000,"announcement_date":"2026-04-16","param_count":null,"is_open_source":false,"is_new":false,"best_latency":0.5,"latency_provider":"Anthropic","best_throughput":42.0,"throughput_provider":"Anthropic","context_provider":"Anthropic"},{"rank":6,"model_id":"seed-2.1-pro","model_name":"Seed 2.1 Pro","organization_id":"bytedance","organization_name":"ByteDance","organization_country":"CN","score":0.702,"normalized_score":0.702,"verified":false,"self_reported":true,"self_reported_source":"https://seed.bytedance.com/en/seed2_1","analysis_method":null,"verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-06-24","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":7,"model_id":"glm-5.1","model_name":"GLM-5.1","organization_id":"zai-org","organization_name":"Zhipu AI","organization_country":"CN","score":0.687,"normalized_score":0.687,"verified":false,"self_reported":true,"self_reported_source":"https://z.ai/blog/glm-5.1","analysis_method":"Claude Code","verification_date":null,"provider_id":"friendli","input_cost_per_million":1.4,"output_cost_per_million":4.4,"context_window":200000,"announcement_date":"2026-04-07","param_count":754000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":"FriendliAI","best_throughput":null,"throughput_provider":"FriendliAI","context_provider":"FriendliAI"},{"rank":8,"model_id":"seed-2.1-turbo","model_name":"Seed 2.1 Turbo","organization_id":"bytedance","organization_name":"ByteDance","organization_country":"CN","score":0.67,"normalized_score":0.67,"verified":false,"self_reported":true,"self_reported_source":"https://seed.bytedance.com/en/seed2_1","analysis_method":null,"verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-06-24","param_count":null,"is_open_source":false,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null},{"rank":9,"model_id":"kimi-k2.5","model_name":"Kimi K2.5","organization_id":"moonshotai","organization_name":"Moonshot AI","organization_country":"CN","score":0.413,"normalized_score":0.413,"verified":false,"self_reported":true,"self_reported_source":"https://www.kimi.com/blog/kimi-k2-5.html","analysis_method":"Coding","verification_date":null,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"context_window":null,"announcement_date":"2026-01-27","param_count":1000000000000,"is_open_source":true,"is_new":false,"best_latency":null,"latency_provider":null,"best_throughput":null,"throughput_provider":null,"context_provider":null}]}