{
  "slug": "cybergym",
  "name": "CyberGym, BountyBench and SEC-bench",
  "tagline": "General-software benchmarks for AI vulnerability discovery",
  "maintainer": "Academic",
  "url": "https://arxiv.org/abs/2506.02548",
  "category": "benchmarks",
  "targets": [
    "General software",
    "1,507 CyberGym instances from 188 projects",
    "40 BountyBench tasks"
  ],
  "approach": "Reproduce real vulnerabilities from crash inputs (CyberGym), offence and defence bounty tasks (BountyBench), end-to-end PoC generation (SEC-bench)",
  "license": "Open source",
  "status": "Active",
  "summary": "CyberGym (1,507 instances from 188 projects, with an end-to-end variant), BountyBench (40 offence and defence tasks) and SEC-bench are the main general-software benchmarks for AI vulnerability discovery, and the ones frontier labs cite.",
  "details": [
    "Cryptographic libraries appear in these datasets as C projects, so scores are relevant to implementation-level bugs."
  ],
  "strengths": [
    "Large, reproducible.",
    "Execution-based scoring."
  ],
  "limits": [
    "General code, not cryptographic logic.",
    "Rapid saturation by new models."
  ],
  "fit": [
    "Use for general scanners; not sufficient for crypto-specific claims."
  ],
  "references": [
    [
      "CyberGym",
      "https://arxiv.org/abs/2506.02548"
    ],
    [
      "CyberGym-E2E",
      "https://arxiv.org/html/2606.04460"
    ],
    [
      "BountyBench",
      "https://arxiv.org/abs/2505.15216"
    ],
    [
      "SEC-bench",
      "https://arxiv.org/abs/2506.11791"
    ]
  ],
  "category_name": "Benchmarks and research",
  "page": "https://agentsast.com/tools/cybergym/",
  "updated": "2026-09-13"
}