Tool details
| Field | Value |
|---|---|
| Name | create_eval |
| Group | Evals |
Playground
Input schema
input-schema.json
{
"type": "object",
"properties": {
"project_reference": {
"type": "string",
"minLength": 1,
"description": "Project identifier or slug that scopes the tool call."
},
"source_target_kind": {
"type": "string",
"enum": [
"project",
"preview_branch"
],
"description": "Provide the source target kind."
},
"target_branch_id": {
"anyOf": [
{
"type": "string",
"format": "uuid",
"description": "Provide the target branch id."
},
{
"type": "null",
"description": "Provide the target branch id."
}
],
"description": "Provide the target branch id."
},
"name": {
"type": "string",
"minLength": 1,
"description": "Human-readable name for the target record."
},
"description": {
"type": "string",
"description": "Human-readable description that clarifies the request."
},
"target": {
"type": "string",
"minLength": 1,
"description": "Provide the target."
},
"dataset": {
"type": "object",
"properties": {
"kind": {
"type": "string",
"enum": [
"inline",
"json",
"jsonl",
"dynamic"
],
"description": "Provide the kind."
},
"path": {
"type": "string",
"description": "Provide the path."
},
"examples": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"type": "string",
"minLength": 1,
"description": "Unique identifier for the target record."
},
"input": {
"description": "Provide the input."
},
"reference": {
"description": "Provide the reference."
},
"metadata": {
"type": "object",
"additionalProperties": {
"description": "Additional structured values supplied to the tool."
},
"description": "Additional structured metadata for the request."
}
},
"required": [
"id"
],
"additionalProperties": false,
"description": "One example item supplied to the tool."
},
"description": "Provide one or more example values."
},
"editable": {
"type": "boolean",
"description": "Provide the editable."
},
"dynamic": {
"type": "boolean",
"description": "Provide the dynamic."
}
},
"required": [
"kind",
"editable",
"dynamic"
],
"additionalProperties": false,
"description": "Structured dataset details used by the tool."
},
"repetitions": {
"type": "integer",
"exclusiveMinimum": 0,
"description": "Provide the repetitions."
},
"tags": {
"type": "array",
"items": {
"type": "string",
"description": "One tag item supplied to the tool."
},
"description": "Provide one or more tag values."
},
"metadata": {
"type": "object",
"additionalProperties": {
"description": "Additional structured values supplied to the tool."
},
"description": "Additional structured metadata for the request."
},
"metrics": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string",
"minLength": 1,
"description": "Human-readable name for the target record."
},
"family": {
"type": "string",
"enum": [
"answer",
"agent",
"ops",
"judge",
"knowledge",
"check"
],
"description": "Provide the family."
},
"severity": {
"type": "string",
"enum": [
"gate",
"soft",
"budget"
],
"description": "Provide the severity."
},
"threshold": {
"type": "object",
"properties": {
"min": {
"type": "number",
"description": "Provide the min."
},
"max": {
"type": "number",
"description": "Provide the max."
}
},
"additionalProperties": false,
"description": "Structured threshold details used by the tool."
},
"config": {
"type": "object",
"additionalProperties": {
"description": "Additional structured values supplied to the tool."
},
"description": "Structured config details used by the tool."
},
"editable": {
"type": "boolean",
"description": "Provide the editable."
},
"dynamic": {
"type": "boolean",
"description": "Provide the dynamic."
}
},
"required": [
"name",
"family",
"severity",
"editable",
"dynamic"
],
"additionalProperties": false,
"description": "One metric item supplied to the tool."
},
"default": [],
"description": "Provide one or more metric values."
}
},
"required": [
"name",
"target",
"dataset"
],
"additionalProperties": false,
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "Input schema for the create_eval tool."
}
Output schema
output-schema.json
{
"type": "object",
"properties": {
"id": {
"type": "string",
"description": "Unique identifier for the returned record."
},
"name": {
"type": "string",
"description": "Human-readable name for the returned record."
},
"description": {
"type": [
"string",
"null"
],
"description": "Human-readable description returned by the tool."
},
"target_kind": {
"type": "string",
"const": "agent",
"description": "The target kind returned by the tool."
},
"target": {
"type": "string",
"description": "The target returned by the tool."
},
"source_path": {
"type": "string",
"description": "The source path returned by the tool."
},
"export_name": {
"type": "string",
"description": "The export name returned by the tool."
},
"dataset": {
"description": "The dataset returned by the tool."
},
"metrics": {
"type": "array",
"items": {
"description": "One metric item returned by the tool."
},
"description": "List of metrics returned by the tool."
},
"repetitions": {
"type": "integer",
"exclusiveMinimum": 0,
"description": "The repetitions returned by the tool."
},
"tags": {
"type": "array",
"items": {
"type": "string",
"description": "One tag item returned by the tool."
},
"description": "List of tags returned by the tool."
},
"metadata": {
"type": "object",
"additionalProperties": {
"description": "Additional structured values returned by the tool."
},
"description": "Additional structured metadata returned by the tool."
},
"editable_fields": {
"type": "array",
"items": {
"type": "string",
"description": "One editable field item returned by the tool."
},
"description": "List of editable fields returned by the tool."
},
"dynamic_fields": {
"type": "array",
"items": {
"type": "string",
"description": "One dynamic field item returned by the tool."
},
"description": "List of dynamic fields returned by the tool."
},
"capabilities": {
"type": "array",
"items": {
"type": "string",
"description": "One capability item returned by the tool."
},
"description": "List of capabilities returned by the tool."
},
"source": {
"type": "string",
"description": "The source returned by the tool."
},
"edit_mode": {
"type": "string",
"enum": [
"structured",
"source_only"
],
"description": "The edit mode returned by the tool."
},
"source_only_reasons": {
"type": "array",
"items": {
"type": "string",
"description": "One source only reason item returned by the tool."
},
"description": "List of source only reasons returned by the tool."
}
},
"required": [
"id",
"name",
"description",
"target_kind",
"target",
"source_path",
"export_name",
"metrics",
"repetitions",
"tags",
"metadata",
"editable_fields",
"dynamic_fields",
"capabilities",
"source",
"edit_mode",
"source_only_reasons"
],
"additionalProperties": false,
"$schema": "http://json-schema.org/draft-07/schema#",
"description": "Structured result returned by the create_eval tool."
}