{"benchmark_id":"theoremqa","benchmark_name":"TheoremQA","benchmark_description":"A theorem-driven question answering dataset containing 800 high-quality questions covering 350+ theorems from Math, Physics, EE&CS, and Finance. Designed to evaluate AI models' capabilities to apply theorems to solve challenging university-level science problems.","max_score":1.0,"categories":["math","physics","reasoning","finance"],"modality":"text","total_models":6,"entries":[{"rank":1,"model_id":"qwen2-72b-instruct","model_name":"Qwen2 72B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.444,"normalized_score":0.444,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-07-23","announcement_date":"2024-07-23","multimodal":false,"param_count":72000000000,"is_new":false},{"rank":2,"model_id":"qwen-2.5-32b-instruct","model_name":"Qwen2.5 32B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.441,"normalized_score":0.441,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-09-19","announcement_date":"2024-09-19","multimodal":false,"param_count":32500000000,"is_new":false},{"rank":3,"model_id":"qwen-2.5-coder-32b-instruct","model_name":"Qwen2.5-Coder 32B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.431,"normalized_score":0.431,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-09-19","announcement_date":"2024-09-19","multimodal":false,"param_count":32000000000,"is_new":false},{"rank":4,"model_id":"qwen-2.5-14b-instruct","model_name":"Qwen2.5 14B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.43,"normalized_score":0.43,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-09-19","announcement_date":"2024-09-19","multimodal":false,"param_count":14700000000,"is_new":false},{"rank":5,"model_id":"qwen-2.5-coder-7b-instruct","model_name":"Qwen2.5-Coder 7B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.34,"normalized_score":0.34,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-09-19","announcement_date":"2024-09-19","multimodal":false,"param_count":7000000000,"is_new":false},{"rank":6,"model_id":"qwen2-7b-instruct","model_name":"Qwen2 7B Instruct","organization_name":"Alibaba Cloud / Qwen Team","organization_id":"qwen","benchmark_score":0.253,"normalized_score":0.253,"verified":false,"self_reported":true,"provider_id":null,"input_cost_per_million":null,"output_cost_per_million":null,"speed_rps":null,"context_window":null,"release_date":"2024-07-23","announcement_date":"2024-07-23","multimodal":false,"param_count":7620000000,"is_new":false}]}