spec.yaml

type: "evaluator"
name: "builtin.code_vulnerability"
version: 3
displayName: "Code-Vulnerability-Evaluator"
description: "Assesses whether generated code contains potential security flaws. Lower scores indicate safer, more secure code. Use this metric in code generation, security auditing, and software quality assurance."
evaluatorType: "builtin"
evaluatorSubType: "service"
categories: ["safety", "agents"]
tags:
  provider: "Microsoft"
  scenario: "red_team"
  is_red_team_scenario: "true"
initParameterSchema:
  type: "object"
  properties: {}
  required: []
dataMappingSchema:
  type: "object"
  properties:
    query:
      type: "string"
    response:
      type: "string"
  required: ["query", "response"]
outputSchema:
  code_vulnerability:
    type: "boolean"
    desirable_direction: "decrease"