Create eval

http
POST /projects/{project_reference}/evals

Create an eval source file in the project source or a preview branch to test an agent against a dataset and metrics; returns the eval source document. Requires project.evals.write.

Servers

json
[
  {
    "url": "https://api.veryfront.com"
  }
]

Authentication

json
[
  {
    "bearerAuth": []
  },
  {
    "apiKeyAuth": []
  }
]
json
{
  "bearerAuth": {
    "type": "http",
    "scheme": "bearer",
    "bearerFormat": "JWT or API Key",
    "description": "Use a JWT bearer token or a Veryfront API key in the `Authorization` header."
  },
  "apiKeyAuth": {
    "type": "apiKey",
    "in": "header",
    "name": "X-API-Key",
    "description": "Alternative API key header for `vf_<prefix>_<secret>` tokens."
  }
}

Parameters

NameLocationRequiredDescription
project_referencepathYesProject slug, UUID, or domain that identifies the project.
source_target_kindqueryNoSource to use: project or preview_branch. Defaults to project.
target_branch_idqueryNoPreview branch to use when the target kind is preview_branch.

Parameter contracts

json
[
  {
    "schema": {
      "$ref": "#/components/schemas/ProjectReference"
    },
    "required": true,
    "description": "Project slug, UUID, or domain that identifies the project.",
    "name": "project_reference",
    "in": "path"
  },
  {
    "schema": {
      "type": "string",
      "description": "Source to use: project or preview_branch. Defaults to project."
    },
    "required": false,
    "description": "Source to use: project or preview_branch. Defaults to project.",
    "name": "source_target_kind",
    "in": "query"
  },
  {
    "schema": {
      "type": [
        "string",
        "null"
      ],
      "format": "uuid",
      "description": "Preview branch to use when the target kind is preview_branch."
    },
    "required": false,
    "description": "Preview branch to use when the target kind is preview_branch.",
    "name": "target_branch_id",
    "in": "query"
  }
]

Request body

json
{
  "required": true,
  "content": {
    "application/json": {
      "schema": {
        "type": "object",
        "properties": {
          "name": {
            "type": "string",
            "minLength": 1,
            "description": "Human-readable eval name; the eval id is derived from it."
          },
          "description": {
            "type": "string",
            "description": "Optional short description of what the eval checks."
          },
          "target": {
            "type": "string",
            "minLength": 1,
            "description": "Identifier of the agent the eval runs against."
          },
          "dataset": {
            "type": "object",
            "properties": {
              "kind": {
                "type": "string",
                "enum": [
                  "inline",
                  "json",
                  "jsonl",
                  "dynamic"
                ]
              },
              "path": {
                "type": "string"
              },
              "examples": {
                "type": "array",
                "items": {
                  "type": "object",
                  "properties": {
                    "id": {
                      "type": "string",
                      "minLength": 1
                    },
                    "input": {},
                    "reference": {},
                    "metadata": {
                      "type": "object",
                      "additionalProperties": {}
                    }
                  },
                  "required": [
                    "id"
                  ]
                }
              },
              "editable": {
                "type": "boolean"
              },
              "dynamic": {
                "type": "boolean"
              }
            },
            "required": [
              "kind",
              "editable",
              "dynamic"
            ],
            "description": "The examples the eval runs: inline examples or a dataset file."
          },
          "metrics": {
            "type": "array",
            "items": {
              "type": "object",
              "properties": {
                "name": {
                  "type": "string",
                  "minLength": 1
                },
                "family": {
                  "type": "string",
                  "enum": [
                    "answer",
                    "agent",
                    "ops",
                    "judge",
                    "knowledge",
                    "check"
                  ]
                },
                "severity": {
                  "type": "string",
                  "enum": [
                    "gate",
                    "soft",
                    "budget"
                  ]
                },
                "threshold": {
                  "type": "object",
                  "properties": {
                    "min": {
                      "type": "number"
                    },
                    "max": {
                      "type": "number"
                    }
                  }
                },
                "config": {
                  "type": "object",
                  "additionalProperties": {}
                },
                "editable": {
                  "type": "boolean"
                },
                "dynamic": {
                  "type": "boolean"
                }
              },
              "required": [
                "name",
                "family",
                "severity",
                "editable",
                "dynamic"
              ]
            },
            "default": [],
            "description": "The metrics that score each example. Defaults to none."
          },
          "repetitions": {
            "type": "integer",
            "exclusiveMinimum": 0,
            "default": 1,
            "description": "How often each example runs. Defaults to 1."
          },
          "tags": {
            "type": "array",
            "items": {
              "type": "string"
            },
            "default": [],
            "description": "Labels for the eval. Defaults to none."
          },
          "metadata": {
            "type": "object",
            "additionalProperties": {},
            "default": {},
            "description": "Free-form JSON object stored with the eval. Defaults to empty."
          }
        },
        "required": [
          "name",
          "target",
          "dataset"
        ],
        "additionalProperties": false
      }
    }
  }
}

Responses

201

Created a structured evaluation source document in the selected source target.

json
{
  "description": "Created a structured evaluation source document in the selected source target.",
  "content": {
    "application/json": {
      "schema": {
        "allOf": [
          {
            "$ref": "#/components/schemas/EvalSourceDocument"
          },
          {
            "type": "object",
            "properties": {
              "edit_mode": {
                "type": "string",
                "enum": [
                  "structured",
                  "source_only"
                ],
                "description": "Editing mode of the eval."
              },
              "source_only_reasons": {
                "type": "array",
                "items": {
                  "type": "string"
                },
                "description": "Why the eval can only be edited as source."
              }
            },
            "required": [
              "edit_mode",
              "source_only_reasons"
            ]
          }
        ]
      }
    }
  }
}

400

Bad request

json
{
  "description": "Bad request",
  "content": {
    "application/problem+json": {
      "schema": {
        "$ref": "#/components/schemas/PrimitiveCrudProblem"
      }
    }
  }
}

401

Unauthorized

json
{
  "description": "Unauthorized",
  "content": {
    "application/problem+json": {
      "schema": {
        "$ref": "#/components/schemas/PrimitiveCrudProblem"
      }
    }
  }
}

403

Forbidden

json
{
  "description": "Forbidden",
  "content": {
    "application/problem+json": {
      "schema": {
        "$ref": "#/components/schemas/PrimitiveCrudProblem"
      }
    }
  }
}

409

Conflict

json
{
  "description": "Conflict",
  "content": {
    "application/problem+json": {
      "schema": {
        "$ref": "#/components/schemas/PrimitiveCrudProblem"
      }
    }
  }
}

500

Server error

json
{
  "description": "Server error",
  "content": {
    "application/problem+json": {
      "schema": {
        "$ref": "#/components/schemas/PrimitiveCrudProblem"
      }
    }
  }
}

503

internal: authentication state is temporarily unavailable and the request was not processed; retry shortly.

json
{
  "description": "internal: authentication state is temporarily unavailable and the request was not processed; retry shortly.",
  "content": {
    "application/problem+json": {
      "schema": {
        "$ref": "#/components/schemas/Problem"
      }
    }
  }
}

Referenced contracts

components/parameters/ProjectReference

json
{
  "schema": {
    "$ref": "#/components/schemas/ProjectReference"
  },
  "required": true,
  "description": "Project slug, UUID, or domain that identifies the project.",
  "name": "project_reference",
  "in": "path"
}

components/schemas/ProjectReference

json
{
  "type": "string"
}

components/schemas/EvalSourceDocument

json
{
  "type": "object",
  "properties": {
    "kind": {
      "type": "string",
      "enum": [
        "eval-source-document"
      ]
    },
    "id": {
      "type": "string"
    },
    "name": {
      "type": "string"
    },
    "description": {
      "type": "string"
    },
    "target": {
      "type": "string"
    },
    "source": {
      "type": "object",
      "properties": {
        "content": {
          "type": "string"
        }
      }
    },
    "dataset": {
      "type": "object",
      "properties": {
        "kind": {
          "type": "string",
          "enum": [
            "inline",
            "json",
            "jsonl",
            "dynamic"
          ]
        },
        "path": {
          "type": "string"
        },
        "examples": {
          "type": "array",
          "items": {
            "type": "object",
            "properties": {
              "id": {
                "type": "string"
              },
              "input": {},
              "reference": {},
              "metadata": {
                "type": "object",
                "additionalProperties": {}
              }
            },
            "required": [
              "id"
            ]
          }
        },
        "editable": {
          "type": "boolean"
        },
        "dynamic": {
          "type": "boolean"
        }
      },
      "required": [
        "kind",
        "editable",
        "dynamic"
      ]
    },
    "metrics": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "name": {
            "type": "string"
          },
          "family": {
            "type": "string",
            "enum": [
              "answer",
              "agent",
              "ops",
              "judge",
              "knowledge",
              "check"
            ]
          },
          "severity": {
            "type": "string",
            "enum": [
              "gate",
              "soft",
              "budget"
            ]
          },
          "threshold": {
            "type": "object",
            "properties": {
              "min": {
                "type": "number"
              },
              "max": {
                "type": "number"
              }
            }
          },
          "config": {
            "type": "object",
            "additionalProperties": {}
          },
          "editable": {
            "type": "boolean"
          },
          "dynamic": {
            "type": "boolean"
          }
        },
        "required": [
          "name",
          "family",
          "severity",
          "editable",
          "dynamic"
        ]
      }
    },
    "repetitions": {
      "type": "integer",
      "exclusiveMinimum": 0
    },
    "tags": {
      "type": "array",
      "items": {
        "type": "string"
      }
    },
    "metadata": {
      "type": "object",
      "additionalProperties": {}
    },
    "capabilities": {
      "type": "array",
      "items": {
        "type": "string",
        "enum": [
          "project.evals.read",
          "project.evals.write",
          "project.evals.run"
        ]
      }
    },
    "target_kind": {
      "type": "string",
      "enum": [
        "agent"
      ],
      "description": "What the eval runs against: always agent."
    },
    "source_path": {
      "type": "string",
      "description": "Source file path where the eval definition lives."
    },
    "export_name": {
      "type": "string",
      "description": "Name of the export that declares the eval in its source file."
    },
    "editable_fields": {
      "type": "array",
      "items": {
        "type": "string",
        "enum": [
          "name",
          "description",
          "target",
          "dataset",
          "repetitions",
          "tags",
          "metadata",
          "metrics",
          "check"
        ]
      },
      "description": "Fields that update_eval can change."
    },
    "dynamic_fields": {
      "type": "array",
      "items": {
        "type": "string",
        "enum": [
          "name",
          "description",
          "target",
          "dataset",
          "repetitions",
          "tags",
          "metadata",
          "metrics",
          "check"
        ]
      },
      "description": "Fields computed in source, which update_eval cannot change."
    },
    "edit_mode": {
      "type": "string",
      "enum": [
        "structured",
        "source_only"
      ],
      "description": "Editing mode of the eval. Returned with a source document, not in the list."
    },
    "source_only_reasons": {
      "type": "array",
      "items": {
        "type": "string"
      },
      "description": "Why the eval can only be edited as source. Returned with a source document, not in the list."
    }
  },
  "required": [
    "kind",
    "id",
    "name",
    "target",
    "source",
    "dataset",
    "metrics",
    "repetitions",
    "tags",
    "metadata",
    "capabilities",
    "target_kind",
    "source_path",
    "export_name",
    "editable_fields",
    "dynamic_fields"
  ]
}

components/schemas/PrimitiveCrudProblem

json
{
  "allOf": [
    {
      "$ref": "#/components/schemas/Problem"
    },
    {
      "type": "object",
      "properties": {
        "edit_mode": {
          "type": "string",
          "enum": [
            "structured",
            "source_only"
          ],
          "description": "Edit mode of the definition, on a source_only_conflict."
        },
        "source_only_reasons": {
          "type": "array",
          "items": {
            "type": "string"
          },
          "description": "Why the definition can only be edited as raw source, on a source_only_conflict."
        },
        "source_path": {
          "type": "string",
          "description": "Source file of the definition, on a source_only_conflict."
        }
      }
    }
  ]
}

components/schemas/Problem

json
{
  "type": "object",
  "properties": {
    "cause": {
      "type": "string",
      "enum": [
        "schedule_concurrency_forbidden",
        "schedule_fire_in_progress",
        "schedule_run_limit_reached"
      ],
      "description": "Schedule admission refusal."
    },
    "idempotency": {
      "type": "object",
      "properties": {
        "state": {
          "type": "string",
          "enum": [
            "in_progress",
            "payload_mismatch",
            "outcome_unknown"
          ]
        },
        "operation": {
          "type": "string",
          "minLength": 1
        },
        "retryable": {
          "type": "boolean"
        },
        "retry_after_seconds": {
          "type": [
            "integer",
            "null"
          ],
          "minimum": 0
        },
        "expires_at": {
          "type": [
            "string",
            "null"
          ]
        },
        "recovery": {
          "type": "object",
          "properties": {
            "available": {
              "type": "boolean"
            },
            "description": {
              "type": "string"
            },
            "resource_url": {
              "type": "string"
            }
          },
          "required": [
            "available",
            "description"
          ]
        }
      },
      "required": [
        "state",
        "operation",
        "retryable",
        "retry_after_seconds",
        "expires_at"
      ],
      "description": "Replay status, retry timing, and recovery lookup."
    },
    "type": {
      "type": "string",
      "description": "URI reference identifying the problem type",
      "example": "https://api.veryfront.com/errors/validation-failed"
    },
    "title": {
      "type": "string",
      "description": "Short human-readable summary of the problem",
      "example": "Validation Failed"
    },
    "status": {
      "type": "integer",
      "minimum": 100,
      "maximum": 599,
      "description": "HTTP status code",
      "example": 400
    },
    "code": {
      "type": "string",
      "description": "Stable machine-readable error code",
      "example": "RUN_INVALID_INPUT"
    },
    "detail": {
      "type": "string",
      "description": "Human-readable explanation specific to this occurrence",
      "example": "The 'email' field must be a valid email address"
    },
    "instance": {
      "type": "string",
      "description": "URI reference identifying the specific occurrence",
      "example": "/projects/my-project/members"
    },
    "slug": {
      "type": "string",
      "description": "Stable identifier for the problem type",
      "example": "validation-failed"
    },
    "category": {
      "type": "string",
      "enum": [
        "AUTH",
        "RESOURCE",
        "VALIDATION",
        "DATABASE",
        "SUBSCRIPTION",
        "INTEGRATION",
        "INTERNAL"
      ],
      "description": "API error category",
      "example": "VALIDATION"
    },
    "suggestion": {
      "type": "string",
      "description": "Suggested action to resolve the problem",
      "example": "Check the request body and query parameters against the API documentation."
    },
    "errors": {
      "type": "array",
      "items": {
        "type": "object",
        "properties": {
          "field": {
            "type": "string",
            "description": "Field path that caused the error",
            "example": "email"
          },
          "message": {
            "type": "string",
            "description": "Error message for this field",
            "example": "Invalid email format"
          },
          "code": {
            "type": "string",
            "description": "Error code for this field",
            "example": "INVALID_FORMAT"
          }
        },
        "required": [
          "field",
          "message"
        ]
      },
      "description": "Detailed field-level errors for validation failures"
    }
  },
  "required": [
    "type",
    "title",
    "status"
  ]
}

Operation contract

json
{
  "operationId": "createEval",
  "tags": [
    "evaluations/Evaluations/Evaluations"
  ],
  "summary": "Create eval",
  "description": "Create an eval source file in the project source or a preview branch to test an agent against a dataset and metrics; returns the eval source document. Requires project.evals.write.",
  "parameters": [
    {
      "$ref": "#/components/parameters/ProjectReference"
    },
    {
      "schema": {
        "type": "string",
        "description": "Source to use: project or preview_branch. Defaults to project."
      },
      "required": false,
      "description": "Source to use: project or preview_branch. Defaults to project.",
      "name": "source_target_kind",
      "in": "query"
    },
    {
      "schema": {
        "type": [
          "string",
          "null"
        ],
        "format": "uuid",
        "description": "Preview branch to use when the target kind is preview_branch."
      },
      "required": false,
      "description": "Preview branch to use when the target kind is preview_branch.",
      "name": "target_branch_id",
      "in": "query"
    }
  ],
  "requestBody": {
    "required": true,
    "content": {
      "application/json": {
        "schema": {
          "type": "object",
          "properties": {
            "name": {
              "type": "string",
              "minLength": 1,
              "description": "Human-readable eval name; the eval id is derived from it."
            },
            "description": {
              "type": "string",
              "description": "Optional short description of what the eval checks."
            },
            "target": {
              "type": "string",
              "minLength": 1,
              "description": "Identifier of the agent the eval runs against."
            },
            "dataset": {
              "type": "object",
              "properties": {
                "kind": {
                  "type": "string",
                  "enum": [
                    "inline",
                    "json",
                    "jsonl",
                    "dynamic"
                  ]
                },
                "path": {
                  "type": "string"
                },
                "examples": {
                  "type": "array",
                  "items": {
                    "type": "object",
                    "properties": {
                      "id": {
                        "type": "string",
                        "minLength": 1
                      },
                      "input": {},
                      "reference": {},
                      "metadata": {
                        "type": "object",
                        "additionalProperties": {}
                      }
                    },
                    "required": [
                      "id"
                    ]
                  }
                },
                "editable": {
                  "type": "boolean"
                },
                "dynamic": {
                  "type": "boolean"
                }
              },
              "required": [
                "kind",
                "editable",
                "dynamic"
              ],
              "description": "The examples the eval runs: inline examples or a dataset file."
            },
            "metrics": {
              "type": "array",
              "items": {
                "type": "object",
                "properties": {
                  "name": {
                    "type": "string",
                    "minLength": 1
                  },
                  "family": {
                    "type": "string",
                    "enum": [
                      "answer",
                      "agent",
                      "ops",
                      "judge",
                      "knowledge",
                      "check"
                    ]
                  },
                  "severity": {
                    "type": "string",
                    "enum": [
                      "gate",
                      "soft",
                      "budget"
                    ]
                  },
                  "threshold": {
                    "type": "object",
                    "properties": {
                      "min": {
                        "type": "number"
                      },
                      "max": {
                        "type": "number"
                      }
                    }
                  },
                  "config": {
                    "type": "object",
                    "additionalProperties": {}
                  },
                  "editable": {
                    "type": "boolean"
                  },
                  "dynamic": {
                    "type": "boolean"
                  }
                },
                "required": [
                  "name",
                  "family",
                  "severity",
                  "editable",
                  "dynamic"
                ]
              },
              "default": [],
              "description": "The metrics that score each example. Defaults to none."
            },
            "repetitions": {
              "type": "integer",
              "exclusiveMinimum": 0,
              "default": 1,
              "description": "How often each example runs. Defaults to 1."
            },
            "tags": {
              "type": "array",
              "items": {
                "type": "string"
              },
              "default": [],
              "description": "Labels for the eval. Defaults to none."
            },
            "metadata": {
              "type": "object",
              "additionalProperties": {},
              "default": {},
              "description": "Free-form JSON object stored with the eval. Defaults to empty."
            }
          },
          "required": [
            "name",
            "target",
            "dataset"
          ],
          "additionalProperties": false
        }
      }
    }
  },
  "responses": {
    "201": {
      "description": "Created a structured evaluation source document in the selected source target.",
      "content": {
        "application/json": {
          "schema": {
            "allOf": [
              {
                "$ref": "#/components/schemas/EvalSourceDocument"
              },
              {
                "type": "object",
                "properties": {
                  "edit_mode": {
                    "type": "string",
                    "enum": [
                      "structured",
                      "source_only"
                    ],
                    "description": "Editing mode of the eval."
                  },
                  "source_only_reasons": {
                    "type": "array",
                    "items": {
                      "type": "string"
                    },
                    "description": "Why the eval can only be edited as source."
                  }
                },
                "required": [
                  "edit_mode",
                  "source_only_reasons"
                ]
              }
            ]
          }
        }
      }
    },
    "400": {
      "description": "Bad request",
      "content": {
        "application/problem+json": {
          "schema": {
            "$ref": "#/components/schemas/PrimitiveCrudProblem"
          }
        }
      }
    },
    "401": {
      "description": "Unauthorized",
      "content": {
        "application/problem+json": {
          "schema": {
            "$ref": "#/components/schemas/PrimitiveCrudProblem"
          }
        }
      }
    },
    "403": {
      "description": "Forbidden",
      "content": {
        "application/problem+json": {
          "schema": {
            "$ref": "#/components/schemas/PrimitiveCrudProblem"
          }
        }
      }
    },
    "409": {
      "description": "Conflict",
      "content": {
        "application/problem+json": {
          "schema": {
            "$ref": "#/components/schemas/PrimitiveCrudProblem"
          }
        }
      }
    },
    "500": {
      "description": "Server error",
      "content": {
        "application/problem+json": {
          "schema": {
            "$ref": "#/components/schemas/PrimitiveCrudProblem"
          }
        }
      }
    },
    "503": {
      "description": "internal: authentication state is temporarily unavailable and the request was not processed; retry shortly.",
      "content": {
        "application/problem+json": {
          "schema": {
            "$ref": "#/components/schemas/Problem"
          }
        }
      }
    }
  },
  "x-documentation-source-tag": "Evaluation Definitions",
  "x-canonical-documentation": {
    "domain": "evaluations",
    "resource": "Evaluations",
    "subgroup": "Evaluations",
    "name": "Create eval",
    "description": "Create an eval source file in the project source or a preview branch to test an agent against a dataset and metrics; returns the eval source document. Requires project.evals.write.",
    "operationId": "createEval"
  },
  "x-mint": {
    "href": "/cloud/rest/api-reference/evaluations-evaluations-evaluations/create-eval"
  }
}