Agentic Evaluator · run report · protocol 0.4
运行模式 lowest-cost 下推荐 sonnet-5。 chosen candidate is the control (sonnet-5)。 lowest generation cost per successful task among eligible: $0.00755。
3 inputs; fewer than 10；no minimum meaningful difference declared
合格：sonnet-5、kimi-k3。成对胜率仍展示但不进入选择器。
| 候选 | 任务结果 | 必过检查 | 成对胜率 | 绝对分 1–5 | 每次成功生成成本 | p50 耗时 | 完成状态 |
|---|---|---|---|---|---|---|---|
| sonnet-5推荐anthropic/claude-sonnet-5 · openrouter/claude-platform-on-aws | 9 success0 failure0 undetermined |
9 / 9 | 50%6 场 | 5.0 | $0.0076 | 5.4 s | 9 success |
| deepseek-v4-pro未过门deepseek/deepseek-v4-pro · openrouter/deepinfra, openrouter/gmicloud, openrouter/alibaba, openrouter/streamlake | 8 success1 failure0 undetermined |
8 / 9 | 25%6 场 | 4.9 | $0.0033 | 24.6 s | 9 success |
| kimi-k3moonshotai/kimi-k3 · openrouter/together, openrouter/digitalocean, openrouter/deepinfra | 9 success0 failure0 undetermined |
9 / 9 | 75%6 场 | 4.7 | $0.0425 | 26.1 s | 9 success |
必过检查的分母是检查实际执行的次数。成对胜率只用每格第一次成功输出，tie 计 0.5。每次成功生成成本不含裁判费用，全口径见成本分账。对照候选：sonnet-5。
| 评估器 | 版本 | 应执行 | pass / fail | 未评估 | 评估器错误 |
|---|---|---|---|---|---|
| tsc-noemit | tsc-6.0.3+scaffold-46888556 | 27 | 26 / 1 | 0 | 0 |
| pairwise-swap | google/gemini-3.1-pro-preview+rubric-e5842052+tpl-01fdb36e | 9 | 9 / 0 | 0 | 0 |
| absolute-1-5 | google/gemini-3.1-pro-preview+rubric-e5842052+tpl-02138c61 | 27 | 27 / 0 | 0 | 0 |
本次没有评估器错误。
| 候选生成 · 27 次 | $0.4771 |
| 成对裁判 | $0.3749 |
| 绝对打分 | $0.4689 |
| 确定性检查 | $0.0000 |
| 重试与恢复 | $0.0000 |
| 合计 | $1.3209 |
| 每次成功的全口径成本 | $0.0508 |
评估花费是生成的 1.8 倍。
ttft_ms: non-streaming calls; no time to first token.cache: OpenRouter reported cached prompt tokens on some calls; recorded under tokens.cached, no cache cost breakdown.deployment_ref.region / tier: provider name observed via OpenRouter; region and service tier are not exposed.human_intervention: single-call producers, no interactive execution; recorded as not applicable.tool_calls: producers make no tool calls this milestone.checkpoint: not applicable to single-call trials.{
"run": "codegen-w38-2026-09-16T05-50-19-377Z",
"step": "codegen",
"candidate": "sonnet-5",
"model_ref": "anthropic/claude-sonnet-5",
"deployment_ref": "openrouter/claude-platform-on-aws",
"input": "code-utils",
"trial": 0,
"trial_hash": "83680e7835c66c66202b5732d8ffc159c675d7e836a2a76aa50e78dca12a2cfc",
"completion_state": "success",
"truncated": false,
"task_outcome": "success",
"outcome_reasons": [
"tsc-noemit: pass"
],
"rule_version": "success-v1",
"checks": {
"tsc-noemit": {
"state": "pass",
"version": "tsc-6.0.3+scaffold-46888556"
}
},
"judge": {
"pairwise": [
{
"vs": "deepseek-v4-pro",
"resolved": "win"
},
{
"vs": "kimi-k3",
"resolved": "win"
}
],
"absolute_overall": 5,
"absolute_dimensions": {
"meets-spec": 5,
"correctness": 5,
"type-quality": 5,
"simplicity": 5,
"robustness": 5
}
},
"cost": {
"generation": 0.006776,
"source": "provider-reported"
},
"tokens": {
"prompt": 938,
"completion": 490,
"cached": 0
},
"ms": 4887,
"ttft_ms": null,
"cache": null,
"finish_reason": "stop",
"error": null,
"legacy_status": "ok",
"legacy_check_passed": true,
"reused_from": null
}{
"protocol_version": "0.4",
"run_id": "codegen-w38-2026-09-16T05-50-19-377Z",
"run_name": "codegen-w38",
"step": {
"id": "codegen",
"version": "1"
},
"spec": {
"path": "specs/codegen-w38.yaml",
"sha256": "b2f624ede18531b6d244d4bfcb4f5734b32c06cda0f1b8dd242f7c133a1881e3"
},
"test_set": {
"id": "code-v1",
"inputs": {
"code-utils": "deddbd434c436f80116137fc9fd1414c1eb34a869deeeb2bdcb528f15fac7e24",
"code-component": "947235496e4cbeb45c67f6a106c649456cdd1f68041736d6b31c0c01f75fca14",
"code-schema": "8e0e42e20813a75bc534662c2ca5d60d1ef6ff246ebb6d422cdd7d553ba49fe4"
}
},
"candidates": {
"sonnet-5": {
"id": "sonnet-5",
"model": "anthropic/claude-sonnet-5",
"provider_route": "openrouter",
"generation_settings": {
"temperature": 0.2
}
},
"deepseek-v4-pro": {
"id": "deepseek-v4-pro",
"model": "deepseek/deepseek-v4-pro",
"provider_route": "openrouter",
"generation_settings": {
"temperature": 0.2
}
},
"kimi-k3": {
"id": "kimi-k3",
"model": "moonshotai/kimi-k3",
"provider_route": "openrouter",
"generation_settings": {
"temperature": 0.2
}
}
},
"control_candidate": "sonnet-5",
"judge": {
"model": "google/gemini-3.1-pro-preview",
"provider_route": "openrouter"
},
"evaluators": {
"required_checks": [
"tsc-noemit"
],
"check_version": "tsc-6.0.3+scaffold-46888556",
"rubric_sha256": "e5842052772fae734cfc2999bd87879b6e49776e634c1f345ffb4f8a4cfca724",
"producer_template_sha256": "55362aba271c69fc9045bddda71c4f84becf85cbb0ec41243a9fd5a0b01c45b6",
"judge_template_sha256": "01fdb36e0a299a8c7f4286d085d6035424b1419512bf88a4d753b8c18b3955b4",
"score_template_sha256": "02138c61f94c42f69a50d3bfb125ca89077b4a315fb41e2357d68007cc29a176"
},
"success_criteria": {
"mandatory_checks": "all"
},
"operating_mode": "lowest-cost",
"minimum_meaningful_difference": null,
"execution": {
"trials_per_case": 3,
"concurrency": 3,
"timeout_ms": 180000,
"benchmark_mode": "capability-neutral",
"cache_mode": "cold"
},
"harness": {
"name": "agentic-evaluator",
"version": "0.1.0",
"git_sha": "ddb57c7",
"node": "v24.14.0"
},
"started_at": "2026-09-16T05:50:19.377Z",
"finished_at": "2026-09-16T05:57:03.387Z"
}