逐任务真实成本：上游用量事件 + 价目表计价
此前每任务只记一条 estimated（字符数/4，金额为空），注释称 host 协议不暴露真实 token 数。本轮实测推翻了该前提： - account/usage/read 在中继上是死的（要求 ChatGPT 账号登录）， 不是来源，已在 cost.py docstring 写明死胡同 - app-server 每上游补全发 thread/tokenUsage/updated，last 是该次 补全的用量；中继 5 调用回合实测 last 之和等于最后 total （input/output/cached/reasoning 四路都对得上） 实现：host.py run_turn 从事件窗口按 threadId+turnId 汇总用量， 重试/goal 各轮累加，dispatch/run JSON 与 goal 用量行透传； sandbox 三路解析（dispatch JSON 解析改用完整 stdout，顺带修 了长 text 下 failed 被误判 completed 的潜 bug）；worker 每任务 记一条 actual（重试/fallback 用量累加进同一条），无用量时回落 estimated；cost.py 价目表（默认 OpenAI 挂牌价，env 可覆盖， 无价模型 usd 记 None、报表进 unpriced_actual_tokens 而非 0 美元）， cost_records 加 cached_input_tokens 列（SQLite/PG 两支迁移）； 报表与按天行新增 cached_input_tokens/unpriced_actual_tokens。 测试：host 32、cloud 483 全过。
This commit is contained in:
host committed
1 parent
353d6c1aa8
commit
63823fc2e3
11 files changed
+709
-67
No files matched your search
@@ -8,13 +8,15 @@ import os
|
||||
import sys
|
||||
import unittest
|
||||
from datetime import datetime, timezone
|
||||
from unittest import mock
|
||||
|
||||
# cloud/ has no __init__.py by design (the coordinator creates it), so
|
||||
# make the project root importable regardless of the discover top level.
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(
|
||||
os.path.dirname(os.path.abspath(__file__)))))
|
||||
|
||||
from cloud.cost import CostStore, estimate_tokens, fetch_account_usage
|
||||
from cloud.cost import (CostStore, estimate_tokens, fetch_account_usage,
|
||||
price_usd_micros)
|
||||
from cloud.models import FALLBACKABLE, ModelRouter, is_fallbackable, record_fallback
|
||||
|
||||
|
||||
@@ -192,8 +194,80 @@ class TestCostStore(unittest.TestCase):
|
||||
"fallback_count": 0,
|
||||
"actual_tokens": 0,
|
||||
"estimated_tokens": 0,
|
||||
"cached_input_tokens": 0,
|
||||
"unpriced_actual_tokens": 0,
|
||||
})
|
||||
|
||||
|
||||
class TestModelPrices(unittest.TestCase):
|
||||
def test_list_prices(self):
|
||||
# 2026-10-26 中继实测 5 调用回合：(58243-45960)*2
|
||||
# + 45960*0.10 + 1138*10 = 40542 微美元。
|
||||
self.assertEqual(
|
||||
price_usd_micros("gpt-6.1-sol", 58243, 45960, 1138), 40542)
|
||||
self.assertEqual(
|
||||
price_usd_micros("gpt-6-sol", 1000, 200, 100),
|
||||
int(round((800 * 2.00 + 200 * 0.20 + 100 * 10.00))))
|
||||
|
||||
def test_unknown_model_unpriced(self):
|
||||
self.assertIsNone(price_usd_micros("gpt-5.6-sol", 100, 0, 100))
|
||||
self.assertIsNone(price_usd_micros(None, 100, 0, 100))
|
||||
|
||||
def test_cached_clamped_to_input(self):
|
||||
self.assertEqual(
|
||||
price_usd_micros("gpt-6.1-sol", 100, 500, 100),
|
||||
price_usd_micros("gpt-6.1-sol", 100, 100, 100))
|
||||
|
||||
def test_env_override(self):
|
||||
with mock.patch.dict(os.environ, {
|
||||
"CLOUD_MODEL_PRICES_JSON":
|
||||
'{"gpt-6.1-sol": [4.00, 0.20, 20.00]}'}):
|
||||
self.assertEqual(
|
||||
price_usd_micros("gpt-6.1-sol", 1000, 0, 0), 4000)
|
||||
with mock.patch.dict(os.environ, {
|
||||
"CLOUD_MODEL_PRICES_JSON": 'not json'}):
|
||||
# 坏覆盖不炸，回落默认表。
|
||||
self.assertEqual(
|
||||
price_usd_micros("gpt-6.1-sol", 1000, 0, 0), 2000)
|
||||
|
||||
|
||||
class TestCachedColumn(unittest.TestCase):
|
||||
def test_record_keeps_cached_and_unpriced(self):
|
||||
store = CostStore(":memory:")
|
||||
store.record("t1", "p1", "gpt-6.1-sol", input_tokens=1000,
|
||||
output_tokens=100, cached_input_tokens=400,
|
||||
usd_micros=2240, source="actual")
|
||||
store.record("t2", "p1", "gpt-5.6-sol", input_tokens=500,
|
||||
output_tokens=50, cached_input_tokens=0,
|
||||
usd_micros=None, source="actual")
|
||||
report = store.report_by_project("p1")
|
||||
self.assertEqual(report["cached_input_tokens"], 400)
|
||||
self.assertEqual(report["unpriced_actual_tokens"], 550)
|
||||
self.assertEqual(report["usd_micros"], 2240)
|
||||
|
||||
def test_migration_adds_cached_column_to_old_table(self):
|
||||
import sqlite3
|
||||
import tempfile
|
||||
path = os.path.join(tempfile.mkdtemp(), "c.db")
|
||||
conn = sqlite3.connect(path)
|
||||
conn.execute(
|
||||
"CREATE TABLE cost_records (id INTEGER PRIMARY KEY"
|
||||
" AUTOINCREMENT, task_id TEXT, project_id TEXT, model TEXT,"
|
||||
" input_tokens INTEGER, output_tokens INTEGER,"
|
||||
" total_tokens INTEGER, usd_micros INTEGER, source TEXT,"
|
||||
" fallback_count INTEGER DEFAULT 0, day TEXT,"
|
||||
" created_at REAL)")
|
||||
conn.commit()
|
||||
conn.close()
|
||||
store = CostStore(path)
|
||||
cols = {row[1] for row in store._conn.execute(
|
||||
"PRAGMA table_info(cost_records)").fetchall()}
|
||||
self.assertIn("cached_input_tokens", cols)
|
||||
store.record("t1", "p1", "m", input_tokens=10, output_tokens=5,
|
||||
cached_input_tokens=3, source="actual")
|
||||
self.assertEqual(store.report_by_task("t1")["cached_input_tokens"],
|
||||
3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -8,6 +8,7 @@
|
||||
top-level 目录无关。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
@@ -385,5 +386,99 @@ class TestGoalExecutors(unittest.TestCase):
|
||||
self.assertIn("test -f x", seen[0])
|
||||
|
||||
|
||||
class TestUsageParsing(unittest.TestCase):
|
||||
def test_result_from_runner_usage(self):
|
||||
usage = {"input_tokens": 1000, "cached_input_tokens": 400,
|
||||
"cache_write_input_tokens": 0, "output_tokens": 100,
|
||||
"reasoning_output_tokens": 20, "total_tokens": 1100,
|
||||
"completions": 2}
|
||||
fake = SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"status": "completed", "text": "done",
|
||||
"usage": usage}),
|
||||
stderr="")
|
||||
result = LocalExecutor("/fake/host.py",
|
||||
runner=lambda cmd: fake
|
||||
)._result_from_runner("t1", fake)
|
||||
self.assertEqual(result["usage"], usage)
|
||||
self.assertEqual(result["status"], "completed")
|
||||
|
||||
def test_result_from_runner_usage_survives_long_text(self):
|
||||
# usage 在 JSON 末尾：旧的截头解析在 text 很长时会丢 usage、
|
||||
# 还会把 failed 误判成 completed；解析必须用完整 stdout。
|
||||
usage = {"input_tokens": 100, "cached_input_tokens": 0,
|
||||
"cache_write_input_tokens": 0, "output_tokens": 10,
|
||||
"reasoning_output_tokens": 0, "total_tokens": 110,
|
||||
"completions": 1}
|
||||
fake = SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"status": "failed", "text": "x" * 5000,
|
||||
"usage": usage}),
|
||||
stderr="")
|
||||
result = LocalExecutor("/fake/host.py",
|
||||
runner=lambda cmd: fake
|
||||
)._result_from_runner("t1", fake)
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertEqual(result["usage"], usage)
|
||||
|
||||
def test_result_from_runner_dirty_usage_dropped(self):
|
||||
for bad in ({"input_tokens": "abc"},
|
||||
{"input_tokens": 0, "output_tokens": 0,
|
||||
"total_tokens": 0},
|
||||
"not-a-dict"):
|
||||
fake = SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"status": "completed", "usage": bad}),
|
||||
stderr="")
|
||||
result = LocalExecutor("/fake/host.py",
|
||||
runner=lambda cmd: fake
|
||||
)._result_from_runner("t1", fake)
|
||||
self.assertIsNone(result["usage"])
|
||||
|
||||
def test_goal_usage_line(self):
|
||||
fake = SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout="round 1: turn=completed verify_rc=0 verify_out=ok\n"
|
||||
"goal met after rounds\n"
|
||||
"usage input=100 cached=40 cache_write=0 output=50 "
|
||||
"reasoning=10 total=150 completions=2\n",
|
||||
stderr="")
|
||||
result = LocalExecutor("/fake/host.py",
|
||||
runner=lambda cmd: fake
|
||||
)._goal_result_from_runner("t1", fake)
|
||||
self.assertEqual(result["usage"], {
|
||||
"input_tokens": 100, "cached_input_tokens": 40,
|
||||
"cache_write_input_tokens": 0, "output_tokens": 50,
|
||||
"reasoning_output_tokens": 10, "total_tokens": 150,
|
||||
"completions": 2})
|
||||
fake2 = SimpleNamespace(
|
||||
returncode=0, stdout="goal met after rounds\n", stderr="")
|
||||
result2 = LocalExecutor("/fake/host.py",
|
||||
runner=lambda cmd: fake2
|
||||
)._goal_result_from_runner("t1", fake2)
|
||||
self.assertIsNone(result2["usage"])
|
||||
|
||||
def test_docker_goal_usage_line(self):
|
||||
def fake_runner(cmd, timeout=None):
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout="round 1: turn=completed verify_rc=0 verify_out=ok\n"
|
||||
"goal met after rounds\n"
|
||||
"usage input=100 cached=40 cache_write=0 output=50 "
|
||||
"reasoning=10 total=150 completions=2\n",
|
||||
stderr="")
|
||||
|
||||
with mock.patch("sandbox.shutil.which",
|
||||
return_value="/usr/bin/docker"):
|
||||
executor = DockerExecutor(
|
||||
"/fake/host.py", runner=fake_runner,
|
||||
env={"XCM_API_KEY": "placeholder-not-a-real-key"})
|
||||
result = executor.goal_task(
|
||||
{"id": "t1", "prompt": "do it",
|
||||
"verify_command": "test -f x"})
|
||||
self.assertEqual(result["usage"]["total_tokens"], 150)
|
||||
self.assertEqual(result["usage"]["cached_input_tokens"], 40)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -374,6 +374,75 @@ class WorkerTestCase(unittest.TestCase):
|
||||
self.assertEqual(results[0]["status"], "failed")
|
||||
self.assertIn("不支持 goal", results[0]["error"])
|
||||
|
||||
# -- 成本：真实用量 --------------------------------------------------
|
||||
def test_cost_actual_from_usage(self):
|
||||
"""执行器带回用量时记 actual：token 全是实测值，usd 按
|
||||
价目表算（(1000-400)*2 + 400*0.10 + 100*10 = 2240 微美元）。"""
|
||||
_insert_task(self.db_path, "A")
|
||||
usage = {"input_tokens": 1000, "cached_input_tokens": 400,
|
||||
"cache_write_input_tokens": 0, "output_tokens": 100,
|
||||
"reasoning_output_tokens": 20, "total_tokens": 1100,
|
||||
"completions": 1}
|
||||
|
||||
def runner(cmd):
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"status": "completed", "text": "done",
|
||||
"usage": usage}),
|
||||
stderr="")
|
||||
|
||||
worker = self._worker(runner=runner, model="gpt-6.1-sol")
|
||||
results = worker.run_all()
|
||||
self.assertEqual([r["status"] for r in results], ["completed"])
|
||||
from cloud.cost import CostStore
|
||||
report = CostStore(self.db_path).report_by_task("A")
|
||||
self.assertEqual(report["actual_tokens"], 1100)
|
||||
self.assertEqual(report["cached_input_tokens"], 400)
|
||||
self.assertEqual(report["estimated_tokens"], 0)
|
||||
self.assertEqual(report["usd_micros"], 2240)
|
||||
self.assertEqual(report["unpriced_actual_tokens"], 0)
|
||||
|
||||
def test_cost_actual_unpriced_model(self):
|
||||
"""无价模型：token 照样记 actual，usd 记 None、报表里
|
||||
进 unpriced_actual_tokens，绝不按 0 美元处理。"""
|
||||
_insert_task(self.db_path, "A")
|
||||
usage = {"input_tokens": 1000, "cached_input_tokens": 0,
|
||||
"cache_write_input_tokens": 0, "output_tokens": 100,
|
||||
"reasoning_output_tokens": 0, "total_tokens": 1100,
|
||||
"completions": 1}
|
||||
|
||||
def runner(cmd):
|
||||
return SimpleNamespace(
|
||||
returncode=0,
|
||||
stdout=json.dumps({"status": "completed", "text": "done",
|
||||
"usage": usage}),
|
||||
stderr="")
|
||||
|
||||
worker = self._worker(runner=runner, model="gpt-5.6-sol")
|
||||
results = worker.run_all()
|
||||
self.assertEqual([r["status"] for r in results], ["completed"])
|
||||
from cloud.cost import CostStore
|
||||
report = CostStore(self.db_path).report_by_task("A")
|
||||
self.assertEqual(report["actual_tokens"], 1100)
|
||||
self.assertEqual(report["usd_micros"], 0)
|
||||
self.assertEqual(report["unpriced_actual_tokens"], 1100)
|
||||
|
||||
def test_cost_estimated_when_no_usage(self):
|
||||
"""执行器没带回用量时回落字符估算记 estimated（旧行为）。"""
|
||||
_insert_task(self.db_path, "A")
|
||||
|
||||
def runner(cmd):
|
||||
return SimpleNamespace(returncode=0, stdout="plain text",
|
||||
stderr="")
|
||||
|
||||
worker = self._worker(runner=runner, model="gpt-6.1-sol")
|
||||
results = worker.run_all()
|
||||
self.assertEqual([r["status"] for r in results], ["completed"])
|
||||
from cloud.cost import CostStore
|
||||
report = CostStore(self.db_path).report_by_task("A")
|
||||
self.assertEqual(report["actual_tokens"], 0)
|
||||
self.assertGreater(report["estimated_tokens"], 0)
|
||||
|
||||
# -- 取消 ------------------------------------------------------------
|
||||
def test_cancel_queued_task_marks_interrupted(self):
|
||||
_insert_task(self.db_path, "A")
|
||||
|
||||
Reference in new issue
Block a user