Agentic Evaluator · run report · protocol 0.4
运行模式 lowest-cost 下推荐 sonnet-5。 lowest generation cost per successful task among eligible: $0.006906。
1 inputs; fewer than 10；no minimum meaningful difference declared
合格：sonnet-5、deepseek-v4-pro。成对胜率仍展示但不进入选择器。
| 候选 | 任务结果 | 必过检查 | 成对胜率 | 绝对分 1–5 | 每次成功生成成本 | p50 耗时 | 完成状态 |
|---|---|---|---|---|---|---|---|
| sonnet-5推荐anthropic/claude-sonnet-5 · openrouter/Claude Platform on AWS | 1 success0 failure0 undetermined |
1 / 1 | 100%1 场 | 5.0 | $0.0069 | 5.7 s | 1 success |
| deepseek-v4-prodeepseek/deepseek-v4-pro · openrouter/SiliconFlow | 1 success0 failure0 undetermined |
1 / 1 | 0%1 场 | 5.0 | $0.0088 | 53.1 s | 1 success |
必过检查的分母是检查实际执行的次数。成对胜率只用每格第一次成功输出，tie 计 0.5。每次成功生成成本不含裁判费用，全口径见成本分账。
| 评估器 | 版本 | 应执行 | pass / fail | 未评估 | 评估器错误 |
|---|---|---|---|---|---|
| tsc-noemit | tsc-6.0.3+scaffold-46888556 | 2 | 2 / 0 | 0 | 0 |
| pairwise-swap | google/gemini-3.1-pro-preview+rubric-e5842052+tpl-01fdb36e | 1 | 1 / 0 | 0 | 0 |
| absolute-1-5 | google/gemini-3.1-pro-preview+rubric-e5842052+tpl-02138c61 | 2 | 2 / 0 | 0 | 0 |
本次没有评估器错误。
| 候选生成 · 2 次 | $0.0157 |
| 成对裁判 | $0.0475 |
| 绝对打分 | $0.0401 |
| 确定性检查 | $0.0000 |
| 重试与恢复 | $0.0000 |
| 合计 | $0.1033 |
| 每次成功的全口径成本 | $0.0516 |
评估花费是生成的 5.6 倍。
ttft_ms: non-streaming calls; no time to first token.cache: no cached prompt tokens reported on any call; cache economics not observable.deployment_ref.region / tier: provider name observed via OpenRouter; region and service tier are not exposed.human_intervention: single-call producers, no interactive execution; recorded as not applicable.tool_calls: producers make no tool calls this milestone.checkpoint: not applicable to single-call trials.{
"run": "smoke-codegen-2026-09-16T03-27-54-631Z",
"step": "codegen",
"candidate": "sonnet-5",
"model_ref": "anthropic/claude-sonnet-5",
"deployment_ref": "openrouter/Claude Platform on AWS",
"input": "code-utils",
"trial": 0,
"trial_hash": "83680e7835c66c66202b5732d8ffc159c675d7e836a2a76aa50e78dca12a2cfc",
"completion_state": "success",
"truncated": false,
"task_outcome": "success",
"outcome_reasons": [
"tsc-noemit: pass"
],
"rule_version": "success-v1",
"checks": {
"tsc-noemit": {
"state": "pass",
"version": "tsc-6.0.3+scaffold-46888556"
}
},
"judge": {
"pairwise": [
{
"vs": "deepseek-v4-pro",
"resolved": "win"
}
],
"absolute_overall": 5,
"absolute_dimensions": {
"meets-spec": 5,
"correctness": 4,
"type-quality": 5,
"simplicity": 5,
"robustness": 5
}
},
"cost": {
"generation": 0.006906,
"source": "provider-reported"
},
"tokens": {
"prompt": 938,
"completion": 503,
"cached": 0
},
"ms": 5707,
"ttft_ms": null,
"cache": null,
"finish_reason": "stop",
"error": null,
"legacy_status": "ok",
"legacy_check_passed": true,
"reused_from": null
}{
"protocol_version": "0.4",
"run_id": "smoke-codegen-2026-09-16T03-27-54-631Z",
"run_name": "smoke-codegen",
"step": {
"id": "codegen",
"version": "1"
},
"spec": {
"path": "specs/smoke-codegen.yaml",
"sha256": "d3705868c887550a2915b1eaba1f013e7996e5caca9b45d5c38b1d251fae951d"
},
"test_set": {
"id": "code-v1",
"inputs": {
"code-utils": "deddbd434c436f80116137fc9fd1414c1eb34a869deeeb2bdcb528f15fac7e24"
}
},
"candidates": {
"sonnet-5": {
"id": "sonnet-5",
"model": "anthropic/claude-sonnet-5",
"provider_route": "openrouter",
"generation_settings": {
"temperature": 0.2
}
},
"deepseek-v4-pro": {
"id": "deepseek-v4-pro",
"model": "deepseek/deepseek-v4-pro",
"provider_route": "openrouter",
"generation_settings": {
"temperature": 0.2
}
}
},
"control_candidate": null,
"judge": {
"model": "google/gemini-3.1-pro-preview",
"provider_route": "openrouter"
},
"evaluators": {
"required_checks": [
"tsc-noemit"
],
"check_version": "tsc-6.0.3+scaffold-46888556",
"rubric_sha256": "e5842052772fae734cfc2999bd87879b6e49776e634c1f345ffb4f8a4cfca724",
"producer_template_sha256": "55362aba271c69fc9045bddda71c4f84becf85cbb0ec41243a9fd5a0b01c45b6",
"judge_template_sha256": "01fdb36e0a299a8c7f4286d085d6035424b1419512bf88a4d753b8c18b3955b4",
"score_template_sha256": "02138c61f94c42f69a50d3bfb125ca89077b4a315fb41e2357d68007cc29a176"
},
"success_criteria": {
"mandatory_checks": "all"
},
"operating_mode": "lowest-cost",
"minimum_meaningful_difference": null,
"execution": {
"trials_per_case": 1,
"concurrency": 2,
"timeout_ms": 180000,
"benchmark_mode": "capability-neutral",
"cache_mode": "cold"
},
"harness": {
"name": "agentic-evaluator",
"version": "0.1.0",
"git_sha": "8e847db",
"node": "v24.14.0"
},
"started_at": "2026-09-16T03:27:54.631Z",
"finished_at": "2026-09-16T03:29:39.300Z"
}