逐任务真实成本:上游用量事件 + 价目表计价
此前每任务只记一条 estimated(字符数/4,金额为空),注释称 host 协议不暴露真实 token 数。本轮实测推翻了该前提: - account/usage/read 在中继上是死的(要求 ChatGPT 账号登录), 不是来源,已在 cost.py docstring 写明死胡同 - app-server 每上游补全发 thread/tokenUsage/updated,last 是该次 补全的用量;中继 5 调用回合实测 last 之和等于最后 total (input/output/cached/reasoning 四路都对得上) 实现:host.py run_turn 从事件窗口按 threadId+turnId 汇总用量, 重试/goal 各轮累加,dispatch/run JSON 与 goal 用量行透传; sandbox 三路解析(dispatch JSON 解析改用完整 stdout,顺带修 了长 text 下 failed 被误判 completed 的潜 bug);worker 每任务 记一条 actual(重试/fallback 用量累加进同一条),无用量时回落 estimated;cost.py 价目表(默认 OpenAI 挂牌价,env 可覆盖, 无价模型 usd 记 None、报表进 unpriced_actual_tokens 而非 0 美元), cost_records 加 cached_input_tokens 列(SQLite/PG 两支迁移); 报表与按天行新增 cached_input_tokens/unpriced_actual_tokens。 测试:host 32、cloud 483 全过。
This commit is contained in:
1 parent
353d6c1aa8
commit
63823fc2e3
11 files changed
+709
-67
No files matched your search
@@ -8,6 +8,7 @@
|
||||
top-level 目录无关。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
@@ -385,5 +386,99 @@ class TestGoalExecutors(unittest.TestCase):
|
||||
self.assertIn("test -f x", seen[0])
|
||||
|
||||
|
||||
class TestUsageParsing(unittest.TestCase):
|
||||
def test_result_from_runner_usage(self):
|
||||
usage = {"input_tokens": 1000, "cached_input_tokens": 400,
|
||||
"cache_write_input_tokens": 0, "output_tokens": 100,
|
||||
"reasoning_output_tokens": 20, "total_tokens": 1100,
|
||||
"completions": 2}
|
||||
fake = SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"status": "completed", "text": "done",
|
||||
"usage": usage}),
|
||||
stderr="")
|
||||
result = LocalExecutor("/fake/host.py",
|
||||
runner=lambda cmd: fake
|
||||
)._result_from_runner("t1", fake)
|
||||
self.assertEqual(result["usage"], usage)
|
||||
self.assertEqual(result["status"], "completed")
|
||||
|
||||
def test_result_from_runner_usage_survives_long_text(self):
|
||||
# usage 在 JSON 末尾:旧的截头解析在 text 很长时会丢 usage、
|
||||
# 还会把 failed 误判成 completed;解析必须用完整 stdout。
|
||||
usage = {"input_tokens": 100, "cached_input_tokens": 0,
|
||||
"cache_write_input_tokens": 0, "output_tokens": 10,
|
||||
"reasoning_output_tokens": 0, "total_tokens": 110,
|
||||
"completions": 1}
|
||||
fake = SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"status": "failed", "text": "x" * 5000,
|
||||
"usage": usage}),
|
||||
stderr="")
|
||||
result = LocalExecutor("/fake/host.py",
|
||||
runner=lambda cmd: fake
|
||||
)._result_from_runner("t1", fake)
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertEqual(result["usage"], usage)
|
||||
|
||||
def test_result_from_runner_dirty_usage_dropped(self):
|
||||
for bad in ({"input_tokens": "abc"},
|
||||
{"input_tokens": 0, "output_tokens": 0,
|
||||
"total_tokens": 0},
|
||||
"not-a-dict"):
|
||||
fake = SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"status": "completed", "usage": bad}),
|
||||
stderr="")
|
||||
result = LocalExecutor("/fake/host.py",
|
||||
runner=lambda cmd: fake
|
||||
)._result_from_runner("t1", fake)
|
||||
self.assertIsNone(result["usage"])
|
||||
|
||||
def test_goal_usage_line(self):
|
||||
fake = SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout="round 1: turn=completed verify_rc=0 verify_out=ok\n"
|
||||
"goal met after rounds\n"
|
||||
"usage input=100 cached=40 cache_write=0 output=50 "
|
||||
"reasoning=10 total=150 completions=2\n",
|
||||
stderr="")
|
||||
result = LocalExecutor("/fake/host.py",
|
||||
runner=lambda cmd: fake
|
||||
)._goal_result_from_runner("t1", fake)
|
||||
self.assertEqual(result["usage"], {
|
||||
"input_tokens": 100, "cached_input_tokens": 40,
|
||||
"cache_write_input_tokens": 0, "output_tokens": 50,
|
||||
"reasoning_output_tokens": 10, "total_tokens": 150,
|
||||
"completions": 2})
|
||||
fake2 = SimpleNamespace(
|
||||
returncode=0, stdout="goal met after rounds\n", stderr="")
|
||||
result2 = LocalExecutor("/fake/host.py",
|
||||
runner=lambda cmd: fake2
|
||||
)._goal_result_from_runner("t1", fake2)
|
||||
self.assertIsNone(result2["usage"])
|
||||
|
||||
def test_docker_goal_usage_line(self):
|
||||
def fake_runner(cmd, timeout=None):
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout="round 1: turn=completed verify_rc=0 verify_out=ok\n"
|
||||
"goal met after rounds\n"
|
||||
"usage input=100 cached=40 cache_write=0 output=50 "
|
||||
"reasoning=10 total=150 completions=2\n",
|
||||
stderr="")
|
||||
|
||||
with mock.patch("sandbox.shutil.which",
|
||||
return_value="/usr/bin/docker"):
|
||||
executor = DockerExecutor(
|
||||
"/fake/host.py", runner=fake_runner,
|
||||
env={"XCM_API_KEY": "placeholder-not-a-real-key"})
|
||||
result = executor.goal_task(
|
||||
{"id": "t1", "prompt": "do it",
|
||||
"verify_command": "test -f x"})
|
||||
self.assertEqual(result["usage"]["total_tokens"], 150)
|
||||
self.assertEqual(result["usage"]["cached_input_tokens"], 40)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user