POST /projects/{project_reference}/evalsCreate an eval source file in the project source or a preview branch to test an agent against a dataset and metrics; returns the eval source document. Requires project.evals.write.
Servers
[
{
"url": "https://api.veryfront.com"
}
]Authentication
[
{
"bearerAuth": []
},
{
"apiKeyAuth": []
}
]{
"bearerAuth": {
"type": "http",
"scheme": "bearer",
"bearerFormat": "JWT or API Key",
"description": "Use a JWT bearer token or a Veryfront API key in the `Authorization` header."
},
"apiKeyAuth": {
"type": "apiKey",
"in": "header",
"name": "X-API-Key",
"description": "Alternative API key header for `vf_<prefix>_<secret>` tokens."
}
}Parameters
Parameter contracts
[
{
"schema": {
"$ref": "#/components/schemas/ProjectReference"
},
"required": true,
"description": "Project slug, UUID, or domain that identifies the project.",
"name": "project_reference",
"in": "path"
},
{
"schema": {
"type": "string",
"description": "Source to use: project or preview_branch. Defaults to project."
},
"required": false,
"description": "Source to use: project or preview_branch. Defaults to project.",
"name": "source_target_kind",
"in": "query"
},
{
"schema": {
"type": [
"string",
"null"
],
"format": "uuid",
"description": "Preview branch to use when the target kind is preview_branch."
},
"required": false,
"description": "Preview branch to use when the target kind is preview_branch.",
"name": "target_branch_id",
"in": "query"
}
]Request body
{
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"name": {
"type": "string",
"minLength": 1,
"description": "Human-readable eval name; the eval id is derived from it."
},
"description": {
"type": "string",
"description": "Optional short description of what the eval checks."
},
"target": {
"type": "string",
"minLength": 1,
"description": "Identifier of the agent the eval runs against."
},
"dataset": {
"type": "object",
"properties": {
"kind": {
"type": "string",
"enum": [
"inline",
"json",
"jsonl",
"dynamic"
]
},
"path": {
"type": "string"
},
"examples": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"type": "string",
"minLength": 1
},
"input": {},
"reference": {},
"metadata": {
"type": "object",
"additionalProperties": {}
}
},
"required": [
"id"
]
}
},
"editable": {
"type": "boolean"
},
"dynamic": {
"type": "boolean"
}
},
"required": [
"kind",
"editable",
"dynamic"
],
"description": "The examples the eval runs: inline examples or a dataset file."
},
"metrics": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string",
"minLength": 1
},
"family": {
"type": "string",
"enum": [
"answer",
"agent",
"ops",
"judge",
"knowledge",
"check"
]
},
"severity": {
"type": "string",
"enum": [
"gate",
"soft",
"budget"
]
},
"threshold": {
"type": "object",
"properties": {
"min": {
"type": "number"
},
"max": {
"type": "number"
}
}
},
"config": {
"type": "object",
"additionalProperties": {}
},
"editable": {
"type": "boolean"
},
"dynamic": {
"type": "boolean"
}
},
"required": [
"name",
"family",
"severity",
"editable",
"dynamic"
]
},
"default": [],
"description": "The metrics that score each example. Defaults to none."
},
"repetitions": {
"type": "integer",
"exclusiveMinimum": 0,
"default": 1,
"description": "How often each example runs. Defaults to 1."
},
"tags": {
"type": "array",
"items": {
"type": "string"
},
"default": [],
"description": "Labels for the eval. Defaults to none."
},
"metadata": {
"type": "object",
"additionalProperties": {},
"default": {},
"description": "Free-form JSON object stored with the eval. Defaults to empty."
}
},
"required": [
"name",
"target",
"dataset"
],
"additionalProperties": false
}
}
}
}Responses
201
Created a structured evaluation source document in the selected source target.
{
"description": "Created a structured evaluation source document in the selected source target.",
"content": {
"application/json": {
"schema": {
"allOf": [
{
"$ref": "#/components/schemas/EvalSourceDocument"
},
{
"type": "object",
"properties": {
"edit_mode": {
"type": "string",
"enum": [
"structured",
"source_only"
],
"description": "Editing mode of the eval."
},
"source_only_reasons": {
"type": "array",
"items": {
"type": "string"
},
"description": "Why the eval can only be edited as source."
}
},
"required": [
"edit_mode",
"source_only_reasons"
]
}
]
}
}
}
}400
Bad request
{
"description": "Bad request",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/PrimitiveCrudProblem"
}
}
}
}401
Unauthorized
{
"description": "Unauthorized",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/PrimitiveCrudProblem"
}
}
}
}403
Forbidden
{
"description": "Forbidden",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/PrimitiveCrudProblem"
}
}
}
}409
Conflict
{
"description": "Conflict",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/PrimitiveCrudProblem"
}
}
}
}500
Server error
{
"description": "Server error",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/PrimitiveCrudProblem"
}
}
}
}503
internal: authentication state is temporarily unavailable and the request was not processed; retry shortly.
{
"description": "internal: authentication state is temporarily unavailable and the request was not processed; retry shortly.",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/Problem"
}
}
}
}Referenced contracts
components/parameters/ProjectReference
{
"schema": {
"$ref": "#/components/schemas/ProjectReference"
},
"required": true,
"description": "Project slug, UUID, or domain that identifies the project.",
"name": "project_reference",
"in": "path"
}components/schemas/ProjectReference
{
"type": "string"
}components/schemas/EvalSourceDocument
{
"type": "object",
"properties": {
"kind": {
"type": "string",
"enum": [
"eval-source-document"
]
},
"id": {
"type": "string"
},
"name": {
"type": "string"
},
"description": {
"type": "string"
},
"target": {
"type": "string"
},
"source": {
"type": "object",
"properties": {
"content": {
"type": "string"
}
}
},
"dataset": {
"type": "object",
"properties": {
"kind": {
"type": "string",
"enum": [
"inline",
"json",
"jsonl",
"dynamic"
]
},
"path": {
"type": "string"
},
"examples": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"type": "string"
},
"input": {},
"reference": {},
"metadata": {
"type": "object",
"additionalProperties": {}
}
},
"required": [
"id"
]
}
},
"editable": {
"type": "boolean"
},
"dynamic": {
"type": "boolean"
}
},
"required": [
"kind",
"editable",
"dynamic"
]
},
"metrics": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string"
},
"family": {
"type": "string",
"enum": [
"answer",
"agent",
"ops",
"judge",
"knowledge",
"check"
]
},
"severity": {
"type": "string",
"enum": [
"gate",
"soft",
"budget"
]
},
"threshold": {
"type": "object",
"properties": {
"min": {
"type": "number"
},
"max": {
"type": "number"
}
}
},
"config": {
"type": "object",
"additionalProperties": {}
},
"editable": {
"type": "boolean"
},
"dynamic": {
"type": "boolean"
}
},
"required": [
"name",
"family",
"severity",
"editable",
"dynamic"
]
}
},
"repetitions": {
"type": "integer",
"exclusiveMinimum": 0
},
"tags": {
"type": "array",
"items": {
"type": "string"
}
},
"metadata": {
"type": "object",
"additionalProperties": {}
},
"capabilities": {
"type": "array",
"items": {
"type": "string",
"enum": [
"project.evals.read",
"project.evals.write",
"project.evals.run"
]
}
},
"target_kind": {
"type": "string",
"enum": [
"agent"
],
"description": "What the eval runs against: always agent."
},
"source_path": {
"type": "string",
"description": "Source file path where the eval definition lives."
},
"export_name": {
"type": "string",
"description": "Name of the export that declares the eval in its source file."
},
"editable_fields": {
"type": "array",
"items": {
"type": "string",
"enum": [
"name",
"description",
"target",
"dataset",
"repetitions",
"tags",
"metadata",
"metrics",
"check"
]
},
"description": "Fields that update_eval can change."
},
"dynamic_fields": {
"type": "array",
"items": {
"type": "string",
"enum": [
"name",
"description",
"target",
"dataset",
"repetitions",
"tags",
"metadata",
"metrics",
"check"
]
},
"description": "Fields computed in source, which update_eval cannot change."
},
"edit_mode": {
"type": "string",
"enum": [
"structured",
"source_only"
],
"description": "Editing mode of the eval. Returned with a source document, not in the list."
},
"source_only_reasons": {
"type": "array",
"items": {
"type": "string"
},
"description": "Why the eval can only be edited as source. Returned with a source document, not in the list."
}
},
"required": [
"kind",
"id",
"name",
"target",
"source",
"dataset",
"metrics",
"repetitions",
"tags",
"metadata",
"capabilities",
"target_kind",
"source_path",
"export_name",
"editable_fields",
"dynamic_fields"
]
}components/schemas/PrimitiveCrudProblem
{
"allOf": [
{
"$ref": "#/components/schemas/Problem"
},
{
"type": "object",
"properties": {
"edit_mode": {
"type": "string",
"enum": [
"structured",
"source_only"
],
"description": "Edit mode of the definition, on a source_only_conflict."
},
"source_only_reasons": {
"type": "array",
"items": {
"type": "string"
},
"description": "Why the definition can only be edited as raw source, on a source_only_conflict."
},
"source_path": {
"type": "string",
"description": "Source file of the definition, on a source_only_conflict."
}
}
}
]
}components/schemas/Problem
{
"type": "object",
"properties": {
"cause": {
"type": "string",
"enum": [
"schedule_concurrency_forbidden",
"schedule_fire_in_progress",
"schedule_run_limit_reached"
],
"description": "Schedule admission refusal."
},
"idempotency": {
"type": "object",
"properties": {
"state": {
"type": "string",
"enum": [
"in_progress",
"payload_mismatch",
"outcome_unknown"
]
},
"operation": {
"type": "string",
"minLength": 1
},
"retryable": {
"type": "boolean"
},
"retry_after_seconds": {
"type": [
"integer",
"null"
],
"minimum": 0
},
"expires_at": {
"type": [
"string",
"null"
]
},
"recovery": {
"type": "object",
"properties": {
"available": {
"type": "boolean"
},
"description": {
"type": "string"
},
"resource_url": {
"type": "string"
}
},
"required": [
"available",
"description"
]
}
},
"required": [
"state",
"operation",
"retryable",
"retry_after_seconds",
"expires_at"
],
"description": "Replay status, retry timing, and recovery lookup."
},
"type": {
"type": "string",
"description": "URI reference identifying the problem type",
"example": "https://api.veryfront.com/errors/validation-failed"
},
"title": {
"type": "string",
"description": "Short human-readable summary of the problem",
"example": "Validation Failed"
},
"status": {
"type": "integer",
"minimum": 100,
"maximum": 599,
"description": "HTTP status code",
"example": 400
},
"code": {
"type": "string",
"description": "Stable machine-readable error code",
"example": "RUN_INVALID_INPUT"
},
"detail": {
"type": "string",
"description": "Human-readable explanation specific to this occurrence",
"example": "The 'email' field must be a valid email address"
},
"instance": {
"type": "string",
"description": "URI reference identifying the specific occurrence",
"example": "/projects/my-project/members"
},
"slug": {
"type": "string",
"description": "Stable identifier for the problem type",
"example": "validation-failed"
},
"category": {
"type": "string",
"enum": [
"AUTH",
"RESOURCE",
"VALIDATION",
"DATABASE",
"SUBSCRIPTION",
"INTEGRATION",
"INTERNAL"
],
"description": "API error category",
"example": "VALIDATION"
},
"suggestion": {
"type": "string",
"description": "Suggested action to resolve the problem",
"example": "Check the request body and query parameters against the API documentation."
},
"errors": {
"type": "array",
"items": {
"type": "object",
"properties": {
"field": {
"type": "string",
"description": "Field path that caused the error",
"example": "email"
},
"message": {
"type": "string",
"description": "Error message for this field",
"example": "Invalid email format"
},
"code": {
"type": "string",
"description": "Error code for this field",
"example": "INVALID_FORMAT"
}
},
"required": [
"field",
"message"
]
},
"description": "Detailed field-level errors for validation failures"
}
},
"required": [
"type",
"title",
"status"
]
}Operation contract
{
"operationId": "createEval",
"tags": [
"evaluations/Evaluations/Evaluations"
],
"summary": "Create eval",
"description": "Create an eval source file in the project source or a preview branch to test an agent against a dataset and metrics; returns the eval source document. Requires project.evals.write.",
"parameters": [
{
"$ref": "#/components/parameters/ProjectReference"
},
{
"schema": {
"type": "string",
"description": "Source to use: project or preview_branch. Defaults to project."
},
"required": false,
"description": "Source to use: project or preview_branch. Defaults to project.",
"name": "source_target_kind",
"in": "query"
},
{
"schema": {
"type": [
"string",
"null"
],
"format": "uuid",
"description": "Preview branch to use when the target kind is preview_branch."
},
"required": false,
"description": "Preview branch to use when the target kind is preview_branch.",
"name": "target_branch_id",
"in": "query"
}
],
"requestBody": {
"required": true,
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"name": {
"type": "string",
"minLength": 1,
"description": "Human-readable eval name; the eval id is derived from it."
},
"description": {
"type": "string",
"description": "Optional short description of what the eval checks."
},
"target": {
"type": "string",
"minLength": 1,
"description": "Identifier of the agent the eval runs against."
},
"dataset": {
"type": "object",
"properties": {
"kind": {
"type": "string",
"enum": [
"inline",
"json",
"jsonl",
"dynamic"
]
},
"path": {
"type": "string"
},
"examples": {
"type": "array",
"items": {
"type": "object",
"properties": {
"id": {
"type": "string",
"minLength": 1
},
"input": {},
"reference": {},
"metadata": {
"type": "object",
"additionalProperties": {}
}
},
"required": [
"id"
]
}
},
"editable": {
"type": "boolean"
},
"dynamic": {
"type": "boolean"
}
},
"required": [
"kind",
"editable",
"dynamic"
],
"description": "The examples the eval runs: inline examples or a dataset file."
},
"metrics": {
"type": "array",
"items": {
"type": "object",
"properties": {
"name": {
"type": "string",
"minLength": 1
},
"family": {
"type": "string",
"enum": [
"answer",
"agent",
"ops",
"judge",
"knowledge",
"check"
]
},
"severity": {
"type": "string",
"enum": [
"gate",
"soft",
"budget"
]
},
"threshold": {
"type": "object",
"properties": {
"min": {
"type": "number"
},
"max": {
"type": "number"
}
}
},
"config": {
"type": "object",
"additionalProperties": {}
},
"editable": {
"type": "boolean"
},
"dynamic": {
"type": "boolean"
}
},
"required": [
"name",
"family",
"severity",
"editable",
"dynamic"
]
},
"default": [],
"description": "The metrics that score each example. Defaults to none."
},
"repetitions": {
"type": "integer",
"exclusiveMinimum": 0,
"default": 1,
"description": "How often each example runs. Defaults to 1."
},
"tags": {
"type": "array",
"items": {
"type": "string"
},
"default": [],
"description": "Labels for the eval. Defaults to none."
},
"metadata": {
"type": "object",
"additionalProperties": {},
"default": {},
"description": "Free-form JSON object stored with the eval. Defaults to empty."
}
},
"required": [
"name",
"target",
"dataset"
],
"additionalProperties": false
}
}
}
},
"responses": {
"201": {
"description": "Created a structured evaluation source document in the selected source target.",
"content": {
"application/json": {
"schema": {
"allOf": [
{
"$ref": "#/components/schemas/EvalSourceDocument"
},
{
"type": "object",
"properties": {
"edit_mode": {
"type": "string",
"enum": [
"structured",
"source_only"
],
"description": "Editing mode of the eval."
},
"source_only_reasons": {
"type": "array",
"items": {
"type": "string"
},
"description": "Why the eval can only be edited as source."
}
},
"required": [
"edit_mode",
"source_only_reasons"
]
}
]
}
}
}
},
"400": {
"description": "Bad request",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/PrimitiveCrudProblem"
}
}
}
},
"401": {
"description": "Unauthorized",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/PrimitiveCrudProblem"
}
}
}
},
"403": {
"description": "Forbidden",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/PrimitiveCrudProblem"
}
}
}
},
"409": {
"description": "Conflict",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/PrimitiveCrudProblem"
}
}
}
},
"500": {
"description": "Server error",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/PrimitiveCrudProblem"
}
}
}
},
"503": {
"description": "internal: authentication state is temporarily unavailable and the request was not processed; retry shortly.",
"content": {
"application/problem+json": {
"schema": {
"$ref": "#/components/schemas/Problem"
}
}
}
}
},
"x-documentation-source-tag": "Evaluation Definitions",
"x-canonical-documentation": {
"domain": "evaluations",
"resource": "Evaluations",
"subgroup": "Evaluations",
"name": "Create eval",
"description": "Create an eval source file in the project source or a preview branch to test an agent against a dataset and metrics; returns the eval source document. Requires project.evals.write.",
"operationId": "createEval"
},
"x-mint": {
"href": "/cloud/rest/api-reference/evaluations-evaluations-evaluations/create-eval"
}
}