Get Evaluation Results

GET

https://api.captain.dev/v3/evals/:eval_id

GET

/v3/evals/:eval_id

import time
import requests
BASE_URL = "https://api.captain.dev"
API_KEY = "your_api_key"
EVAL_ID = "eval_019b4c2f11d04c8e9a7b3d5c2e1840ff"
headers = {
    "Authorization": f"Bearer {API_KEY}",
}
while True:
    response = requests.get(
        f"{BASE_URL}/v3/evals/{EVAL_ID}",
        headers=headers,
        params={"items_limit": 100},
        timeout=30.0,
    )
    data = response.json()
    if data["status"] in ("completed", "completed_with_errors", "failed"):
        break
    print(f"{data['status']}: {data['progress']['percent']}%")
    time.sleep(10)
for name, card in data["scorecards"].items():
    print(name, card["recall_at_1"], card["mrr"], card["ndcg_at_10"], card["latency_p50_ms"])

{
  "collection_name": "customer_forms",
  "configs": [
    {
      "explicit_fields": [],
      "name": "baseline",
      "query": {
        "limit": 10,
        "rerank": {
          "enabled": false,
          "candidate_limit": null,
          "model": null
        },
        "semantic_ratio": 0.5,
        "boost": [],
        "exclude_chunk_types": [],
        "filter": null,
        "include": {
          "document": true,
          "metadata": true,
          "regions": false,
          "relations": false,
          "document_metadata": false,
          "related_chunks": false,
          "neighboring_chunks": false,
          "archived": false
        },
        "max_chunks_per_document": null,
        "relation_direction": "outgoing",
        "relation_types": null
      }
    },
    {
      "explicit_fields": [
        "exclude_chunk_types",
        "filter",
        "limit",
        "max_chunks_per_document",
        "rerank",
        "semantic_ratio"
      ],
      "name": "rerank-deep",
      "query": {
        "limit": 10,
        "rerank": {
          "enabled": true,
          "candidate_limit": 40,
          "model": "voyage-rerank-2.5"
        },
        "semantic_ratio": 0.35,
        "boost": [],
        "exclude_chunk_types": [
          "page_header",
          "page_footer",
          "footnote"
        ],
        "filter": {
          "review_status": {
            "$in": [
              "approved",
              "in_review"
            ]
          }
        },
        "include": {
          "document": true,
          "metadata": true,
          "regions": false,
          "relations": false,
          "document_metadata": false,
          "related_chunks": false,
          "neighboring_chunks": false,
          "archived": false
        },
        "max_chunks_per_document": 2,
        "relation_direction": "outgoing",
        "relation_types": null
      }
    }
  ],
  "created_at": "2026-09-12T17:30:01Z",
  "eval_id": "eval_019b4c2f11d04c8e9a7b3d5c2e1840ff",
  "idempotency_key": "eval-payments-2026-09-12-a",
  "items_page": {
    "limit": 100,
    "total": 3,
    "next_cursor": null
  },
  "preview": {
    "billable_units": 4,
    "cases": 3,
    "cases_error": 1,
    "configs": 2,
    "units": 6
  },
  "progress": {
    "cases_error": 1,
    "cases_total": 3,
    "configs_failed": 0,
    "configs_total": 2,
    "elapsed_seconds": 78,
    "percent": 100,
    "units_completed": 4,
    "units_error": 2,
    "units_failed": 0,
    "units_total": 6,
    "eta_seconds": 0,
    "qps": 1.8
  },
  "request_id": "req_20260912_eval001_get",
  "status": "completed_with_errors",
  "updated_at": "2026-09-12T17:31:22Z",
  "upload_id": "evu_019b4c2e7a3f70d2a91c4e8b6f1023aa",
  "billing": {
    "billable_units": 4,
    "skipped_units": 2,
    "failed_configs": []
  },
  "completed_at": "2026-09-12T17:31:22Z",
  "environment": "development",
  "error_code": null,
  "error_message": null,
  "items": [
    {
      "expected_files": [
        "payment-authorization-form.pdf"
      ],
      "id": "auth-window",
      "query": "What language authorizes the merchant to charge the card on file?",
      "status": "scored",
      "error_code": null,
      "error_message": null,
      "expected_document_ids": [
        "a1b2c3d4e5f60718293a4b5c6d7e8f90"
      ],
      "filters": {
        "policy_area": "payments"
      },
      "results": {
        "baseline": {
          "status": "scored",
          "error_code": null,
          "error_message": null,
          "hit": true,
          "latency_ms": 405,
          "rank": 2,
          "retrieved_document_ids": [
            "9f8e7d6c5b4a3928170695a4b3c2d1e0",
            "a1b2c3d4e5f60718293a4b5c6d7e8f90",
            "11223344556677889900aabbccddeeff"
          ]
        },
        "rerank-deep": {
          "status": "scored",
          "error_code": null,
          "error_message": null,
          "hit": true,
          "latency_ms": 972,
          "rank": 1,
          "retrieved_document_ids": [
            "a1b2c3d4e5f60718293a4b5c6d7e8f90",
            "9f8e7d6c5b4a3928170695a4b3c2d1e0",
            "11223344556677889900aabbccddeeff"
          ]
        }
      }
    },
    {
      "expected_files": [
        "f4a91c7e8b2d4a0c9e6f13b5a8d02744"
      ],
      "id": "recurring-terms",
      "query": "Where are cancellation terms for recurring payments written?",
      "status": "scored",
      "error_code": null,
      "error_message": null,
      "expected_document_ids": [
        "f4a91c7e8b2d4a0c9e6f13b5a8d02744"
      ],
      "filters": null,
      "results": {
        "baseline": {
          "status": "scored",
          "error_code": null,
          "error_message": null,
          "hit": true,
          "latency_ms": 419,
          "rank": 1,
          "retrieved_document_ids": [
            "f4a91c7e8b2d4a0c9e6f13b5a8d02744",
            "00112233445566778899aabbccddeeff"
          ]
        },
        "rerank-deep": {
          "status": "scored",
          "error_code": null,
          "error_message": null,
          "hit": true,
          "latency_ms": 1011,
          "rank": 1,
          "retrieved_document_ids": [
            "f4a91c7e8b2d4a0c9e6f13b5a8d02744",
            "00112233445566778899aabbccddeeff"
          ]
        }
      }
    },
    {
      "expected_files": [
        "billing-policy-controls.xlsx"
      ],
      "id": "case_3",
      "query": "Which spreadsheet rows list approval required for recurring payments?",
      "status": "error",
      "error_code": "GOLD_UNRESOLVED",
      "error_message": "expected_files entry matched 0 documents",
      "expected_document_ids": null,
      "filters": {
        "policy_area": {
          "$eq": "payments"
        }
      },
      "results": {}
    }
  ],
  "scorecards": {
    "baseline": {
      "chunk_count": 19402,
      "collection_name": "customer_forms",
      "document_count": 1284,
      "error": 1,
      "explicit_fields": [],
      "failed": 0,
      "name": "baseline",
      "query": {
        "limit": 10,
        "rerank": {
          "enabled": false,
          "candidate_limit": null,
          "model": null
        },
        "semantic_ratio": 0.5,
        "boost": [],
        "exclude_chunk_types": [],
        "filter": null,
        "include": {
          "document": true,
          "metadata": true,
          "regions": false,
          "relations": false,
          "document_metadata": false,
          "related_chunks": false,
          "neighboring_chunks": false,
          "archived": false
        },
        "max_chunks_per_document": null,
        "relation_direction": "outgoing",
        "relation_types": null
      },
      "question_set_size": 3,
      "scored": 2,
      "status": "scored",
      "environment": "development",
      "error_code": null,
      "error_message": null,
      "latency_p50_ms": 412,
      "latency_p95_ms": 690,
      "mrr": 0.75,
      "ndcg_at_10": 0.88,
      "recall_at_1": 0.5,
      "recall_at_10": 1,
      "recall_at_3": 1
    },
    "rerank-deep": {
      "chunk_count": 19402,
      "collection_name": "customer_forms",
      "document_count": 1284,
      "error": 1,
      "explicit_fields": [
        "exclude_chunk_types",
        "filter",
        "limit",
        "max_chunks_per_document",
        "rerank",
        "semantic_ratio"
      ],
      "failed": 0,
      "name": "rerank-deep",
      "query": {
        "limit": 10,
        "rerank": {
          "enabled": true,
          "candidate_limit": 40,
          "model": "voyage-rerank-2.5"
        },
        "semantic_ratio": 0.35,
        "boost": [],
        "exclude_chunk_types": [
          "page_header",
          "page_footer",
          "footnote"
        ],
        "filter": {
          "review_status": {
            "$in": [
              "approved",
              "in_review"
            ]
          }
        },
        "include": {
          "document": true,
          "metadata": true,
          "regions": false,
          "relations": false,
          "document_metadata": false,
          "related_chunks": false,
          "neighboring_chunks": false,
          "archived": false
        },
        "max_chunks_per_document": 2,
        "relation_direction": "outgoing",
        "relation_types": null
      },
      "question_set_size": 3,
      "scored": 2,
      "status": "scored",
      "environment": "development",
      "error_code": null,
      "error_message": null,
      "latency_p50_ms": 980,
      "latency_p95_ms": 1410,
      "mrr": 1,
      "ndcg_at_10": 1,
      "recall_at_1": 1,
      "recall_at_10": 1,
      "recall_at_3": 1
    }
  },
  "started_at": "2026-09-12T17:30:04Z"
}

Status, progress, per-config scorecards, billing and per-case results for an eval. scorecards is {} and billing is null until the eval is terminal (completed, completed_with_errors, or failed). Items are paged by items_limit (default 100, max 500) and items_cursor; each item carries a result object per config name. Recall, MRR and nDCG are over scored units only; a failed query is counted in failed, never scored as a miss. Evals of a deleted collection return 404. Metric definitions are on the Evaluations guide.

Path parameters

eval_idstringRequired

Query parameters

items_limitintegerOptional1-500Defaults to 100

items_cursorstring or nullOptional<=256 characters

Response

The eval, with items paged.

collection_namestring

configslist of objects

created_atstring

eval_idstring

idempotency_keystring

items_pageobject

previewobject

progressobject

request_idstring

statusenum

Allowed values:

updated_atstring

upload_idstring

billingobject or nullOptional

completed_atstring or nullOptional

environmentstring or nullOptional

error_codestring or nullOptional

error_messagestring or nullOptional

itemslist of objectsOptional

scorecardsmap from strings to objectsOptional

started_atstring or nullOptional

Errors

400

Bad Request Error

401

Unauthorized Error

403

Forbidden Error

404

Not Found Error