Create eval
curl --request POST \
--url https://api.veryfront.com/projects/{project_reference}/evals \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "<string>",
"target": "<string>",
"dataset": {
"editable": true,
"dynamic": true,
"path": "<string>",
"examples": [
{
"id": "<string>",
"input": "<unknown>",
"reference": "<unknown>",
"metadata": {}
}
]
},
"description": "<string>",
"metrics": [],
"repetitions": 1,
"tags": [],
"metadata": {}
}
'import requests
url = "https://api.veryfront.com/projects/{project_reference}/evals"
payload = {
"name": "<string>",
"target": "<string>",
"dataset": {
"editable": True,
"dynamic": True,
"path": "<string>",
"examples": [
{
"id": "<string>",
"input": "<unknown>",
"reference": "<unknown>",
"metadata": {}
}
]
},
"description": "<string>",
"metrics": [],
"repetitions": 1,
"tags": [],
"metadata": {}
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
target: '<string>',
dataset: {
editable: true,
dynamic: true,
path: '<string>',
examples: [{id: '<string>', input: '<unknown>', reference: '<unknown>', metadata: {}}]
},
description: '<string>',
metrics: [],
repetitions: 1,
tags: [],
metadata: {}
})
};
fetch('https://api.veryfront.com/projects/{project_reference}/evals', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.veryfront.com/projects/{project_reference}/evals",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => '<string>',
'target' => '<string>',
'dataset' => [
'editable' => true,
'dynamic' => true,
'path' => '<string>',
'examples' => [
[
'id' => '<string>',
'input' => '<unknown>',
'reference' => '<unknown>',
'metadata' => [
]
]
]
],
'description' => '<string>',
'metrics' => [
],
'repetitions' => 1,
'tags' => [
],
'metadata' => [
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.veryfront.com/projects/{project_reference}/evals"
payload := strings.NewReader("{\n \"name\": \"<string>\",\n \"target\": \"<string>\",\n \"dataset\": {\n \"editable\": true,\n \"dynamic\": true,\n \"path\": \"<string>\",\n \"examples\": [\n {\n \"id\": \"<string>\",\n \"input\": \"<unknown>\",\n \"reference\": \"<unknown>\",\n \"metadata\": {}\n }\n ]\n },\n \"description\": \"<string>\",\n \"metrics\": [],\n \"repetitions\": 1,\n \"tags\": [],\n \"metadata\": {}\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.veryfront.com/projects/{project_reference}/evals")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"<string>\",\n \"target\": \"<string>\",\n \"dataset\": {\n \"editable\": true,\n \"dynamic\": true,\n \"path\": \"<string>\",\n \"examples\": [\n {\n \"id\": \"<string>\",\n \"input\": \"<unknown>\",\n \"reference\": \"<unknown>\",\n \"metadata\": {}\n }\n ]\n },\n \"description\": \"<string>\",\n \"metrics\": [],\n \"repetitions\": 1,\n \"tags\": [],\n \"metadata\": {}\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.veryfront.com/projects/{project_reference}/evals")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"<string>\",\n \"target\": \"<string>\",\n \"dataset\": {\n \"editable\": true,\n \"dynamic\": true,\n \"path\": \"<string>\",\n \"examples\": [\n {\n \"id\": \"<string>\",\n \"input\": \"<unknown>\",\n \"reference\": \"<unknown>\",\n \"metadata\": {}\n }\n ]\n },\n \"description\": \"<string>\",\n \"metrics\": [],\n \"repetitions\": 1,\n \"tags\": [],\n \"metadata\": {}\n}"
response = http.request(request)
puts response.read_body{
"kind": "eval-source-document",
"id": "<string>",
"name": "<string>",
"targetKind": "agent",
"target": "<string>",
"source": {
"filePath": "<string>",
"exportName": "<string>",
"content": "<string>"
},
"dataset": {
"kind": "inline",
"editable": true,
"dynamic": true,
"path": "<string>",
"examples": [
{
"id": "<string>",
"input": "<unknown>",
"reference": "<unknown>",
"metadata": {}
}
]
},
"metrics": [
{
"name": "<string>",
"family": "answer",
"severity": "gate",
"editable": true,
"dynamic": true,
"threshold": {
"min": 123,
"max": 123
},
"config": {}
}
],
"repetitions": 123,
"tags": [
"<string>"
],
"metadata": {},
"editableFields": [
"name"
],
"dynamicFields": [
"name"
],
"capabilities": [
"project.evals.read"
],
"target_kind": "agent",
"source_path": "<string>",
"export_name": "<string>",
"editable_fields": [
"name"
],
"dynamic_fields": [
"name"
],
"description": "<string>",
"edit_mode": "structured",
"source_only_reasons": [
"<string>"
]
}{
"type": "<string>",
"title": "<string>",
"status": 123,
"detail": "<string>",
"error": "<string>",
"message": "<string>",
"edit_mode": "structured",
"source_only_reasons": [
"<string>"
],
"source_path": "<string>"
}{
"type": "<string>",
"title": "<string>",
"status": 123,
"detail": "<string>",
"error": "<string>",
"message": "<string>",
"edit_mode": "structured",
"source_only_reasons": [
"<string>"
],
"source_path": "<string>"
}{
"type": "<string>",
"title": "<string>",
"status": 123,
"detail": "<string>",
"error": "<string>",
"message": "<string>",
"edit_mode": "structured",
"source_only_reasons": [
"<string>"
],
"source_path": "<string>"
}{
"type": "<string>",
"title": "<string>",
"status": 123,
"detail": "<string>",
"error": "<string>",
"message": "<string>",
"edit_mode": "structured",
"source_only_reasons": [
"<string>"
],
"source_path": "<string>"
}Create eval
Create an eval source file in the project source or a preview branch to test an agent against a dataset and metrics; returns the eval source document. Requires project.evals.write.
POST
/
projects
/
{project_reference}
/
evals
Create eval
curl --request POST \
--url https://api.veryfront.com/projects/{project_reference}/evals \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "<string>",
"target": "<string>",
"dataset": {
"editable": true,
"dynamic": true,
"path": "<string>",
"examples": [
{
"id": "<string>",
"input": "<unknown>",
"reference": "<unknown>",
"metadata": {}
}
]
},
"description": "<string>",
"metrics": [],
"repetitions": 1,
"tags": [],
"metadata": {}
}
'import requests
url = "https://api.veryfront.com/projects/{project_reference}/evals"
payload = {
"name": "<string>",
"target": "<string>",
"dataset": {
"editable": True,
"dynamic": True,
"path": "<string>",
"examples": [
{
"id": "<string>",
"input": "<unknown>",
"reference": "<unknown>",
"metadata": {}
}
]
},
"description": "<string>",
"metrics": [],
"repetitions": 1,
"tags": [],
"metadata": {}
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
target: '<string>',
dataset: {
editable: true,
dynamic: true,
path: '<string>',
examples: [{id: '<string>', input: '<unknown>', reference: '<unknown>', metadata: {}}]
},
description: '<string>',
metrics: [],
repetitions: 1,
tags: [],
metadata: {}
})
};
fetch('https://api.veryfront.com/projects/{project_reference}/evals', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.veryfront.com/projects/{project_reference}/evals",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => '<string>',
'target' => '<string>',
'dataset' => [
'editable' => true,
'dynamic' => true,
'path' => '<string>',
'examples' => [
[
'id' => '<string>',
'input' => '<unknown>',
'reference' => '<unknown>',
'metadata' => [
]
]
]
],
'description' => '<string>',
'metrics' => [
],
'repetitions' => 1,
'tags' => [
],
'metadata' => [
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.veryfront.com/projects/{project_reference}/evals"
payload := strings.NewReader("{\n \"name\": \"<string>\",\n \"target\": \"<string>\",\n \"dataset\": {\n \"editable\": true,\n \"dynamic\": true,\n \"path\": \"<string>\",\n \"examples\": [\n {\n \"id\": \"<string>\",\n \"input\": \"<unknown>\",\n \"reference\": \"<unknown>\",\n \"metadata\": {}\n }\n ]\n },\n \"description\": \"<string>\",\n \"metrics\": [],\n \"repetitions\": 1,\n \"tags\": [],\n \"metadata\": {}\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.veryfront.com/projects/{project_reference}/evals")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"<string>\",\n \"target\": \"<string>\",\n \"dataset\": {\n \"editable\": true,\n \"dynamic\": true,\n \"path\": \"<string>\",\n \"examples\": [\n {\n \"id\": \"<string>\",\n \"input\": \"<unknown>\",\n \"reference\": \"<unknown>\",\n \"metadata\": {}\n }\n ]\n },\n \"description\": \"<string>\",\n \"metrics\": [],\n \"repetitions\": 1,\n \"tags\": [],\n \"metadata\": {}\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.veryfront.com/projects/{project_reference}/evals")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"<string>\",\n \"target\": \"<string>\",\n \"dataset\": {\n \"editable\": true,\n \"dynamic\": true,\n \"path\": \"<string>\",\n \"examples\": [\n {\n \"id\": \"<string>\",\n \"input\": \"<unknown>\",\n \"reference\": \"<unknown>\",\n \"metadata\": {}\n }\n ]\n },\n \"description\": \"<string>\",\n \"metrics\": [],\n \"repetitions\": 1,\n \"tags\": [],\n \"metadata\": {}\n}"
response = http.request(request)
puts response.read_body{
"kind": "eval-source-document",
"id": "<string>",
"name": "<string>",
"targetKind": "agent",
"target": "<string>",
"source": {
"filePath": "<string>",
"exportName": "<string>",
"content": "<string>"
},
"dataset": {
"kind": "inline",
"editable": true,
"dynamic": true,
"path": "<string>",
"examples": [
{
"id": "<string>",
"input": "<unknown>",
"reference": "<unknown>",
"metadata": {}
}
]
},
"metrics": [
{
"name": "<string>",
"family": "answer",
"severity": "gate",
"editable": true,
"dynamic": true,
"threshold": {
"min": 123,
"max": 123
},
"config": {}
}
],
"repetitions": 123,
"tags": [
"<string>"
],
"metadata": {},
"editableFields": [
"name"
],
"dynamicFields": [
"name"
],
"capabilities": [
"project.evals.read"
],
"target_kind": "agent",
"source_path": "<string>",
"export_name": "<string>",
"editable_fields": [
"name"
],
"dynamic_fields": [
"name"
],
"description": "<string>",
"edit_mode": "structured",
"source_only_reasons": [
"<string>"
]
}{
"type": "<string>",
"title": "<string>",
"status": 123,
"detail": "<string>",
"error": "<string>",
"message": "<string>",
"edit_mode": "structured",
"source_only_reasons": [
"<string>"
],
"source_path": "<string>"
}{
"type": "<string>",
"title": "<string>",
"status": 123,
"detail": "<string>",
"error": "<string>",
"message": "<string>",
"edit_mode": "structured",
"source_only_reasons": [
"<string>"
],
"source_path": "<string>"
}{
"type": "<string>",
"title": "<string>",
"status": 123,
"detail": "<string>",
"error": "<string>",
"message": "<string>",
"edit_mode": "structured",
"source_only_reasons": [
"<string>"
],
"source_path": "<string>"
}{
"type": "<string>",
"title": "<string>",
"status": 123,
"detail": "<string>",
"error": "<string>",
"message": "<string>",
"edit_mode": "structured",
"source_only_reasons": [
"<string>"
],
"source_path": "<string>"
}Authorizations
bearerAuthapiKeyAuth
Use a JWT bearer token or a Veryfront API key in the Authorization header.
Path Parameters
Project slug, UUID, or domain that identifies the project.
Body
application/json
- Option 1
- Option 2
Human-readable eval name; the eval id is derived from it.
Minimum string length:
1Identifier of the agent the eval runs against.
Minimum string length:
1The examples the eval runs: inline examples or a dataset file.
Show child attributes
Show child attributes
Optional short description of what the eval checks.
The metrics that score each example. Defaults to none.
Show child attributes
Show child attributes
How often each example runs. Defaults to 1.
Labels for the eval. Defaults to none.
Free-form JSON object stored with the eval. Defaults to empty.
Show child attributes
Show child attributes
Response
Created a structured evaluation source document in the selected source target.
Available options:
eval-source-document Deprecated: use target_kind.
Available options:
agent Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Show child attributes
Deprecated: use editable_fields.
Available options:
name, description, target, dataset, repetitions, tags, metadata, metrics, check Deprecated: use dynamic_fields.
Available options:
name, description, target, dataset, repetitions, tags, metadata, metrics, check Available options:
project.evals.read, project.evals.write, project.evals.run The same value as targetKind.
Available options:
agent The same value as source.filePath.
The same value as source.exportName.
The same value as editableFields.
Available options:
name, description, target, dataset, repetitions, tags, metadata, metrics, check The same value as dynamicFields.
Available options:
name, description, target, dataset, repetitions, tags, metadata, metrics, check Editing mode of the eval. Returned with a source document, not in the list.
Available options:
structured, source_only Why the eval can only be edited as source. Returned with a source document, not in the list.