Agentic Evaluator · run report · protocol 0.4
步骤 prd 没有合格候选，选择器不给出配置推荐。 no candidate passed eligibility gates。
no candidate passed eligibility gates
合格：无。成对胜率仍展示但不进入选择器。
| 候选 | 任务结果 | 必过检查 | 成对胜率 | 绝对分 1–5 | 每次成功生成成本 | p50 耗时 | 完成状态 |
|---|---|---|---|---|---|---|---|
| sonnet-5未过门anthropic/claude-sonnet-5 · openrouter/claude-platform-on-aws | 0 success0 failure4 undetermined |
— | 25%4 场 | 5.0 | — | 22.5 s | 4 success |
| deepseek-v4-pro未过门deepseek/deepseek-v4-pro · openrouter/streamlake, openrouter/atlascloud, openrouter/siliconflow | 0 success0 failure4 undetermined |
— | 38%4 场 | 5.0 | — | 62.1 s | 4 success |
| kimi-k3未过门moonshotai/kimi-k3 · openrouter/together | 0 success0 failure4 undetermined |
— | 88%4 场 | 5.0 | — | 49.7 s | 4 success |
必过检查的分母是检查实际执行的次数。成对胜率只用每格第一次成功输出，tie 计 0.5。每次成功生成成本不含裁判费用，全口径见成本分账。对照候选：sonnet-5。
| 评估器 | 版本 | 应执行 | pass / fail | 未评估 | 评估器错误 |
|---|---|---|---|---|---|
| pairwise-swap | google/gemini-3.1-pro-preview+rubric-7357ecab+tpl-0e52a472 | 6 | 6 / 0 | 0 | 0 |
| absolute-1-5 | google/gemini-3.1-pro-preview+rubric-7357ecab+tpl-0f5ef8d8 | 12 | 12 / 0 | 0 | 0 |
本次没有评估器错误。
| 候选生成 · 12 次 | $0.3747 |
| 成对裁判 | $0.2654 |
| 绝对打分 | $0.1752 |
| 确定性检查 | $0.0000 |
| 重试与恢复 | $0.0000 |
| 合计 | $0.8153 |
| 每次成功的全口径成本 | — |
评估花费是生成的 1.2 倍。没有成功的任务，每次成功成本无定义。
ttft_ms: non-streaming calls; no time to first token.cache: OpenRouter reported cached prompt tokens on some calls; recorded under tokens.cached, no cache cost breakdown.deployment_ref.region / tier: provider name observed via OpenRouter; region and service tier are not exposed.human_intervention: single-call producers, no interactive execution; recorded as not applicable.tool_calls: producers make no tool calls this milestone.checkpoint: not applicable to single-call trials.{
"run": "prd-w38-2026-09-16T05-57-03-777Z",
"step": "prd",
"candidate": "sonnet-5",
"model_ref": "anthropic/claude-sonnet-5",
"deployment_ref": "openrouter/claude-platform-on-aws",
"input": "todo-app",
"trial": 0,
"trial_hash": "9c3e3c4c7495b3bac7a51d4e8cb13aa0f0dda90a3a9e4bf527f6ad2a5afc96ff",
"completion_state": "success",
"truncated": false,
"task_outcome": "undetermined",
"outcome_reasons": [
"no required checks declared for this step; success cannot be decided"
],
"rule_version": "success-v1",
"checks": {},
"judge": {
"pairwise": [
{
"vs": "deepseek-v4-pro",
"resolved": "tie"
},
{
"vs": "kimi-k3",
"resolved": "loss"
}
],
"absolute_overall": 5,
"absolute_dimensions": {
"completeness": 5,
"testability": 5,
"no-hallucination": 5,
"structure": 5
}
},
"cost": {
"generation": 0.021344,
"source": "provider-reported"
},
"tokens": {
"prompt": 257,
"completion": 2083,
"cached": 0
},
"ms": 22460,
"ttft_ms": null,
"cache": null,
"finish_reason": "stop",
"error": null,
"legacy_status": "ok",
"legacy_check_passed": null,
"reused_from": null
}{
"protocol_version": "0.4",
"run_id": "prd-w38-2026-09-16T05-57-03-777Z",
"run_name": "prd-w38",
"step": {
"id": "prd",
"version": "1"
},
"spec": {
"path": "specs/prd-w38.yaml",
"sha256": "6c49df82e9c7c16b99c3cbfc612c5fcec2c65e1733c1d686393fc126038fce46"
},
"test_set": {
"id": "briefs-v1",
"inputs": {
"todo-app": "a0f3bb264a1543f6e2da31c9bb55f0cbd899bccaaed167be7ef44120ce840608",
"chat-app": "63b941965d16231b7203aaf9b8577d5d95909a62e0be53b7853e55898d486f1b"
}
},
"candidates": {
"sonnet-5": {
"id": "sonnet-5",
"model": "anthropic/claude-sonnet-5",
"provider_route": "openrouter",
"generation_settings": {
"temperature": 0.3
}
},
"deepseek-v4-pro": {
"id": "deepseek-v4-pro",
"model": "deepseek/deepseek-v4-pro",
"provider_route": "openrouter",
"generation_settings": {
"temperature": 0.3
}
},
"kimi-k3": {
"id": "kimi-k3",
"model": "moonshotai/kimi-k3",
"provider_route": "openrouter",
"generation_settings": {
"temperature": 0.3
}
}
},
"control_candidate": "sonnet-5",
"judge": {
"model": "google/gemini-3.1-pro-preview",
"provider_route": "openrouter"
},
"evaluators": {
"required_checks": [],
"check_version": null,
"rubric_sha256": "7357ecab4ab33f23b6d6be06a7e7a10634927d24c28178fa4c87a02aba860169",
"producer_template_sha256": "03f9dd2e8cc9681772d37434fdfeb68777fed5e4dd5a3156206c8132493e6174",
"judge_template_sha256": "0e52a472e59ed2740bc8bc45e8f84593ef69727be56239249cc62840d0f316d5",
"score_template_sha256": "0f5ef8d84d888847a0c73d3f25d68578b0b0fd04894383953763264d7206026c"
},
"success_criteria": null,
"operating_mode": "highest-assurance",
"minimum_meaningful_difference": null,
"execution": {
"trials_per_case": 2,
"concurrency": 3,
"timeout_ms": 240000,
"benchmark_mode": "capability-neutral",
"cache_mode": "cold"
},
"harness": {
"name": "agentic-evaluator",
"version": "0.1.0",
"git_sha": "b8ad780",
"node": "v24.14.0"
},
"started_at": "2026-09-16T05:57:03.777Z",
"finished_at": "2026-09-16T06:02:17.181Z"
}