{
  "slug": "benchmark-contamination",
  "term": "Benchmark contamination",
  "tools": [
    "evmbench",
    "zkbugs"
  ],
  "definition": "The model has seen the audit report or the bug in training, so a benchmark hit measures recall of memory rather than discovery.",
  "detail": [
    "OpenZeppelin raised this about EVMbench; it applies to every benchmark built from public audits, including zkbugs. The defences are benchmarks that post-date model cutoffs, private held-out sets, and full-codebase modes that test search rather than recognition."
  ],
  "page": "https://agentsast.com/glossary/benchmark-contamination/",
  "updated": "2026-09-13"
}