Agent Evals

Authorization: Bearer cm_your_api_key

{
  "id": "aa10…",
  "bot_id": "b1f2…",
  "name": "Booking flow — regression",
  "description": "Covers the happy path plus three refusals.",
  "scorecard_id": "sc33…",
  "is_active": true,
  "created_at": "2026-08-12T09:00:00Z",
  "updated_at": "2026-08-12T09:00:00Z"
}

{
  "id": "bb20…",
  "suite_id": "aa10…",
  "name": "Caller wants a Saturday slot",
  "persona": "An impatient customer in Pune who only has Saturdays free and dislikes being put on hold.",
  "opening": "Hi, can I move my appointment to Saturday?",
  "max_turns": 6,
  "success_criteria": [
    { "type": "contains", "value": "Saturday", "role": "agent" },
    { "type": "tool_called", "value": "reschedule_appointment" },
    { "type": "max_turns_under", "value": 5 }
  ],
  "position": 0,
  "created_at": "2026-08-12T09:05:00Z",
  "updated_at": "2026-08-12T09:05:00Z"
}

{ "type": "llm_judge", "value": "The agent confirms the new date and time before booking it." }

{
  "source_session_id": "vs42…",
  "name": "From call vs42ab12",
  "persona": "You are the caller from a real phone call, replaying it …",
  "opening": "Hi, can I move my appointment to Saturday?",
  "max_turns": 4,
  "success_criteria": [
    { "type": "llm_judge", "value": "The agent addresses the caller's request (\"Hi, can I move my appointment to Saturday?\") accurately and helpfully, without inventing facts." }
  ],
  "case": { "id": "bb21…", "suite_id": "aa10…", "…": "…" }
}

curl -X POST https://api.callmissed.com/api/v1/voice/evals/aa10…/run \
  -H "Authorization: Bearer cm_your_api_key"

{
  "id": "run77…",
  "suite_id": "aa10…",
  "status": "failed",
  "started_at": "2026-08-17T08:00:00Z",
  "finished_at": "2026-08-17T08:01:44Z",
  "total_cases": 12,
  "passed_cases": 11,
  "model": "kimi-k2.6",
  "cost_credits": 3.812,
  "created_at": "2026-08-17T08:00:00Z",
  "pass_rate": 0.9167,
  "min_pass_rate": 0.9,
  "gate": "pass",
  "results": [
    {
      "id": "res01…",
      "run_id": "run77…",
      "case_id": "bb20…",
      "passed": true,
      "transcript": [
        { "role": "caller", "content": "Hi, can I move my appointment to Saturday?" },
        { "role": "agent", "content": "Of course — I can move it to Saturday." }
      ],
      "assertions": [
        { "type": "contains", "value": "Saturday", "passed": true }
      ],
      "score": 0.92,
      "error": null,
      "created_at": "2026-08-17T08:00:12Z"
    }
  ]
}

RUN=$(curl -sf -X POST \
  "https://api.callmissed.com/api/v1/voice/evals/$SUITE_ID/run?wait=false&min_pass_rate=0.9" \
  -H "Authorization: Bearer $CALLMISSED_API_KEY" | jq -r .id)

while :; do
  GATE=$(curl -sf "https://api.callmissed.com/api/v1/voice/evals/runs/$RUN?min_pass_rate=0.9" \
    -H "Authorization: Bearer $CALLMISSED_API_KEY" | jq -r .gate)
  [ "$GATE" != "pending" ] && break
  sleep 15
done
[ "$GATE" = "pass" ]

name: CallMissed eval gate
description: >-
  Run a CallMissed agent eval suite and fail the job when its pass rate is
  below a threshold. Starts the run with wait=false, then polls the run until
  it reports a gate verdict.

inputs:
  api-key:
    description: A CallMissed API key (cm_...) with the evals:write scope. Pass it from a secret.
    required: true
  suite-id:
    description: The eval suite to run.
    required: true
  min-pass-rate:
    description: Fraction of cases that must pass, 0 to 1.
    required: false
    default: "1"
  timeout-minutes:
    description: Stop waiting after this many minutes and fail the job.
    required: false
    default: "30"
  poll-seconds:
    description: Seconds between polls.
    required: false
    default: "15"
  api-base:
    description: API host.
    required: false
    default: https://api.callmissed.com

outputs:
  run-id:
    description: The eval run id.
    value: ${{ steps.gate.outputs.run-id }}
  status:
    description: passed, failed or error.
    value: ${{ steps.gate.outputs.status }}
  pass-rate:
    description: passed_cases / total_cases, 0 to 1.
    value: ${{ steps.gate.outputs.pass-rate }}
  gate:
    description: pass or fail.
    value: ${{ steps.gate.outputs.gate }}

runs:
  using: composite
  steps:
    - id: gate
      shell: bash
      # Inputs reach the script as environment variables, never interpolated
      # into it, so a crafted input cannot inject shell.
      env:
        CM_API_KEY: ${{ inputs.api-key }}
        CM_SUITE_ID: ${{ inputs.suite-id }}
        CM_MIN_PASS_RATE: ${{ inputs.min-pass-rate }}
        CM_TIMEOUT_MINUTES: ${{ inputs.timeout-minutes }}
        CM_POLL_SECONDS: ${{ inputs.poll-seconds }}
        CM_API_BASE: ${{ inputs.api-base }}
      run: |
        set -euo pipefail
        base="${CM_API_BASE%/}/api/v1/voice/evals"
        auth="Authorization: Bearer ${CM_API_KEY}"
        body="$(mktemp)"

        # Both values go into a URL, so they are checked, not trusted.
        if ! [[ "$CM_SUITE_ID" =~ ^[0-9a-fA-F-]{36}$ ]]; then
          echo "::error::suite-id must be a UUID."; exit 1
        fi
        if ! [[ "$CM_MIN_PASS_RATE" =~ ^(0(\.[0-9]+)?|1(\.0+)?|\.[0-9]+)$ ]]; then
          echo "::error::min-pass-rate must be a number from 0 to 1."; exit 1
        fi
        if ! [[ "$CM_TIMEOUT_MINUTES" =~ ^[0-9]+$ && "$CM_POLL_SECONDS" =~ ^[0-9]+$ ]]; then
          echo "::error::timeout-minutes and poll-seconds must be whole numbers."; exit 1
        fi

        code=$(curl -sS -o "$body" -w '%{http_code}' -X POST -H "$auth" \
          "${base}/${CM_SUITE_ID}/run?wait=false&min_pass_rate=${CM_MIN_PASS_RATE}")
        if [ "$code" != "202" ] && [ "$code" != "201" ]; then
          echo "::error::Could not start the eval run (HTTP $code): $(jq -r '.detail // .' "$body" 2>/dev/null || cat "$body")"
          exit 1
        fi
        run_id=$(jq -r '.id' "$body")
        echo "run-id=${run_id}" >> "$GITHUB_OUTPUT"
        echo "Started eval run ${run_id}"

        deadline=$(( $(date +%s) + CM_TIMEOUT_MINUTES * 60 ))
        gate=$(jq -r '.gate // "pending"' "$body")
        while [ "$gate" = "pending" ]; do
          if [ "$(date +%s)" -ge "$deadline" ]; then
            echo "::error::Eval run ${run_id} did not finish within ${CM_TIMEOUT_MINUTES} minutes."
            exit 1
          fi
          sleep "$CM_POLL_SECONDS"
          code=$(curl -sS -o "$body" -w '%{http_code}' -H "$auth" \
            "${base}/runs/${run_id}?min_pass_rate=${CM_MIN_PASS_RATE}")
          if [ "$code" != "200" ]; then
            echo "::warning::Polling run ${run_id} returned HTTP $code; retrying."
            continue
          fi
          gate=$(jq -r '.gate' "$body")
        done

        status=$(jq -r '.status' "$body")
        rate=$(jq -r '.pass_rate // "null"' "$body")
        passed=$(jq -r '.passed_cases' "$body")
        total=$(jq -r '.total_cases' "$body")
        {
          echo "status=${status}"
          echo "pass-rate=${rate}"
          echo "gate=${gate}"
        } >> "$GITHUB_OUTPUT"
        {
          echo "### CallMissed eval run \`${run_id}\`"
          echo ""
          echo "| Status | Passed | Pass rate | Required | Gate |"
          echo "| --- | --- | --- | --- | --- |"
          echo "| ${status} | ${passed}/${total} | ${rate} | ${CM_MIN_PASS_RATE} | ${gate} |"
          echo ""
          jq -r '.results[] | select(.passed == false) | "- case `\(.case_id)`: " +
            ([.assertions[]? | select(.passed == false) | "\(.type) (\(.detail // ""))"] | join("; "))
            + (if .error then " error: \(.error)" else "" end)' "$body" || true
        } >> "$GITHUB_STEP_SUMMARY"

        if [ "$gate" != "pass" ]; then
          echo "::error::Eval gate failed: ${passed}/${total} cases passed (pass rate ${rate}, required ${CM_MIN_PASS_RATE}, status ${status})."
          exit 1
        fi
        echo "Eval gate passed: ${passed}/${total} cases (pass rate ${rate})."

- uses: actions/checkout@v4
- uses: ./.github/actions/callmissed-eval
  with:
    api-key: ${{ secrets.CALLMISSED_API_KEY }}
    suite-id: aa10…
    min-pass-rate: "0.9"