feat: 任务实时链路 SSE (#61)

host.py:回合通知摘要成链路条目,增量追加到 <state>/host-traces/<task>.jsonl(上限 5000 行);dispatch/run/goal/delegate 设链路文件
cloud/server.py:GET /tasks/{id}/trace 快照 + /trace/stream SSE(snapshot → entry → done;worktree 在别机时空链路 + 说明)
console:详情链路区,运行中 fetch 流式渲染(EventSource 不能带 X-API-Key 头),终态走快照;DOM 上限 1000 条
单测:test_host_trace 7 + test_task_trace 6;host 49、cloud 525、deploy 14 全过

Co-authored-by: host <host@test.local>
This commit is contained in:
openqjandhost authored and GitHub committed 2026-10-27 02:27:15 +08:00
1 parent 6022f1dada
commit 799f7c3edb
6 files changed
+696

No files matched your search

+106
View File
@@ -137,6 +137,8 @@
<span id="actionMsg" class="muted"></span>
</div>
<pre id="detailBody" style="display:none"></pre>
<h3>实时链路</h3>
<div id="traceBox" class="muted">点任务行查看详情后,这里显示过程链路</div>
<div id="costBox"></div>
<script>
@@ -144,6 +146,10 @@
var $ = function (s) { return document.querySelector(s); };
var KEY_STORE = "codex-console-key";
var currentTask = null;
var traceAbort = null;
var TRACE_KIND = { exec: "命令", file: "文件", call: "调用", tool: "工具", message: "消息", reasoning: "思考", plan: "计划" };
var TRACE_LIMIT = 1000;
var TRACE_TERMINAL = ["completed", "failed", "interrupted", "merged", "merge-failed"];
function esc(s) {
return String(s == null ? "" : s).replace(/&/g, "&amp;").replace(/</g, "&lt;")
@@ -256,7 +262,106 @@ async function loadTasks() {
} catch (e) { $("#taskRows").innerHTML = "<tr><td colspan='7' class='err'>加载失败:" + esc(e.message) + "</td></tr>"; }
}
function stopTrace() {
if (traceAbort) { try { traceAbort.abort(); } catch (e) {} traceAbort = null; }
}
function traceLine(e) {
var k = TRACE_KIND[e.kind] || e.kind || "?";
var body = "";
if (e.kind === "exec") body = esc(e.command || "") + "(exit=" + esc(e.exitCode) + "," + esc(e.durationMs) + "ms,cwd=" + esc(e.cwd || "") + ")";
else if (e.kind === "file") body = esc(e.path || "") + " " + esc(e.change || "") + " " + esc(e.status || "");
else if (e.kind === "message") body = (e.phase ? "[" + esc(e.phase) + "] " : "") + esc(e.text || "");
else if (e.kind === "reasoning") body = esc(e.summary || "");
else if (e.kind === "tool") body = esc(e.server || "") + "/" + esc(e.tool || "") + " " + esc(e.status || "");
else if (e.kind === "call") body = esc(e.name || "");
else if (e.kind === "plan") body = esc(e.text || "");
else body = esc(JSON.stringify(e).slice(0, 300));
return "[" + k + "] " + body;
}
function renderTrace(entries, note, doneStatus) {
var box = $("#traceBox");
var html = "";
if (note) html += "<span class='err'>" + esc(note) + "</span><br>";
if (!entries.length) html += "(暂无链路条目)";
else {
var shown = entries.slice(-TRACE_LIMIT);
if (entries.length > TRACE_LIMIT) html += "(仅显示最近 " + TRACE_LIMIT + " 条,共 " + entries.length + " 条)<br>";
html += shown.map(traceLine).join("<br>");
}
if (doneStatus) html += "<br><span class='muted'>链路已结束(" + esc(doneStatus) + ")</span>";
box.innerHTML = html;
}
async function startTrace(taskId, status) {
stopTrace();
var box = $("#traceBox");
if (TRACE_TERMINAL.indexOf(status) >= 0) {
try {
var d = await api("/tasks/" + encodeURIComponent(taskId) + "/trace", "GET");
renderTrace(d.entries || [], d.note || "", d.status);
} catch (e) { box.innerHTML = "<span class='err'>链路加载失败:" + esc(e.message) + "</span>"; }
return;
}
// 运行中:fetch 流式读 SSE(EventSource 不能带 X-API-Key 头,故不用它)。
var entries = [];
var index = {};
var note = "";
var done = false;
var doneStatus = "";
box.innerHTML = "链路连接中…";
var ctrl = new AbortController();
traceAbort = ctrl;
try {
var headers = {};
var key = getKey();
if (key) headers["X-API-Key"] = key;
var r = await fetch("/tasks/" + encodeURIComponent(taskId) + "/trace/stream", { headers: headers, signal: ctrl.signal });
if (!r.ok) {
var msg = "HTTP " + r.status;
try { var dj = await r.json(); if (dj && dj.error) msg = dj.error; } catch (e) {}
if (r.status === 401) msg += "(key 未填或无效)";
throw new Error(msg);
}
var reader = r.body.getReader();
var decoder = new TextDecoder();
var buf = "";
while (true) {
var chunk = await reader.read();
if (chunk.done) break;
buf += decoder.decode(chunk.value, { stream: true });
var parts = buf.split("\n\n");
buf = parts.pop();
parts.forEach(function (part) {
var line = part.split("\n").filter(function (l) { return l.indexOf("data: ") === 0; }).map(function (l) { return l.slice(6); }).join("\n");
if (!line) return;
try {
var m = JSON.parse(line);
if (m.type === "snapshot") {
entries = m.entries || []; index = {};
entries.forEach(function (e, i) { index[e.id] = i; });
note = m.note || "";
renderTrace(entries, note, "");
} else if (m.type === "entry") {
var e = m.entry;
if (e.id in index) entries[index[e.id]] = e; else { index[e.id] = entries.length; entries.push(e); }
renderTrace(entries, note, "");
} else if (m.type === "done") {
done = true; doneStatus = m.status;
renderTrace(entries, note, doneStatus);
}
} catch (e) {}
});
if (done) break;
}
} catch (e) {
if (e && e.name === "AbortError") return;
box.innerHTML = "<span class='err'>链路流中断:" + esc(e.message) + "</span>";
return;
}
if (done) { loadTasks(); showDetail(taskId); }
}
async function showDetail(taskId) {
stopTrace();
try {
var t = await api("/tasks/" + encodeURIComponent(taskId), "GET");
currentTask = t;
@@ -280,6 +385,7 @@ async function showDetail(taskId) {
$("#detailActions").style.display = "flex";
$("#actionMsg").textContent = "";
$("#costBox").innerHTML = "";
startTrace(taskId, t.status);
} catch (e) { $("#detail").innerHTML = "<span class='err'>加载失败:" + esc(e.message) + "</span>"; }
}
+152
View File
@@ -42,6 +42,7 @@ import html
import json
import logging
import os
import re
import socket
import sqlite3
import threading
@@ -126,6 +127,53 @@ _server_log = logging.getLogger("cloud.server")
TERMINAL_STATUSES = frozenset(
{"completed", "failed", "interrupted", "merged", "merge-failed"}
)
# 实时链路(任务过程事件流):host 在回合进行中把 app-server
# 通知摘要成条目追加写 jsonl,服务端 tail 文件经 SSE 推送。
TRACE_STREAM_MAX_SECONDS = 3600
TRACE_STREAM_POLL_SECONDS = 1.0
TRACE_DEFAULT_DIR = "/opt/codex-app-server/work/host-traces"
def _trace_file_name(task_id):
"""云任务 id -> 链路文件名:与 cloud.sandbox 的 _host_name
同款清洗(只保留 [A-Za-z0-9_.-],其余替 -),两边算出同一名字。"""
return re.sub(r"[^A-Za-z0-9_.-]", "-", str(task_id))
def _trace_dir_for(server):
"""链路目录解析:server 属性(单测注入)> 环境变量 > 默认路径。"""
d = getattr(server, "trace_dir", None)
if d:
return str(d)
return os.environ.get("CODEX_TRACE_DIR") or TRACE_DEFAULT_DIR
def _read_trace_entries(path):
"""读 host 写的 jsonl 链路文件:跳过坏行;按 id 去重,后到的
完成态覆盖先到的开始态,保首次顺序。文件不存在返回空列表。"""
entries = []
index = {}
try:
with open(path, "r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
entry = json.loads(line)
except ValueError:
continue
if not isinstance(entry, dict) or "id" not in entry:
continue
eid = entry["id"]
if eid in index:
entries[index[eid]] = entry
else:
index[eid] = len(entries)
entries.append(entry)
except OSError:
return []
return entries
DEFAULT_SWEEP_THRESHOLD_SECONDS = 300
# 限流默认值:每 IP / 每 API key 每分钟 120 次;环境变量
# CLOUD_RATE_LIMIT_PER_MINUTE / CLOUD_RATE_LIMIT_WINDOW_SECONDS 可调。
@@ -848,6 +896,11 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._get_task(user_id, rest)
elif rest.endswith("/cost") and method == "GET":
self._task_cost(user_id, rest[: -len("/cost")])
elif rest.endswith("/trace/stream") and method == "GET":
self._task_trace_stream(
user_id, rest[: -len("/trace/stream")])
elif rest.endswith("/trace") and method == "GET":
self._task_trace(user_id, rest[: -len("/trace")])
elif rest.endswith("/cancel") and method == "POST":
self._cancel_task(user_id, rest[: -len("/cancel")])
elif rest.endswith("/merge") and method == "POST":
@@ -1553,6 +1606,105 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
return
self._send_json(200, task)
# -- 任务实时链路 --------------------------------------------------
def _trace_snapshot(self, task):
"""当前链路快照 (entries, note):worktree 在别机时读不到
文件,返回空链路 + 说明(多机诚实降级,不谎报无过程)。"""
worktree_host = task.get("worktree_host")
if worktree_host and worktree_host != socket.gethostname():
return [], ("链路文件在 worktree 主机 %s 上,本机读不到"
% worktree_host)
path = os.path.join(
_trace_dir_for(self.server),
_trace_file_name(task["id"]) + ".jsonl")
return _read_trace_entries(path), ""
def _task_trace(self, user_id, task_id):
row = self._fetch_task(task_id)
if row is None:
self._send_json(404, {"error": "task not found"})
return
task = _task_to_dict(row)
with _auth_store(self.server.db_path) as store:
if not store.check_permission(user_id, task["project_id"], "read"):
self._send_json(403, {"error": "forbidden"})
return
entries, note = self._trace_snapshot(task)
self._send_json(200, {"task_id": task_id, "status": task["status"],
"entries": entries, "note": note})
def _task_trace_stream(self, user_id, task_id):
row = self._fetch_task(task_id)
if row is None:
self._send_json(404, {"error": "task not found"})
return
task = _task_to_dict(row)
with _auth_store(self.server.db_path) as store:
if not store.check_permission(user_id, task["project_id"], "read"):
self._send_json(403, {"error": "forbidden"})
return
entries, note = self._trace_snapshot(task)
try:
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Cache-Control", "no-cache")
self.send_header("Connection", "keep-alive")
self.end_headers()
except (BrokenPipeError, ConnectionResetError):
return
# SSE 无 Content-Length,done 后靠关连接界定流结束;注意
# send_header("Connection", "keep-alive") 会把
# close_connection 改回 False,所以必须在 end_headers 之后
# 再置 True(CPython send_header 的 Connection 分支行为)。
self.close_connection = True
sent = {}
def _emit(obj):
try:
self.wfile.write(
("data: " + json.dumps(obj, ensure_ascii=False,
default=_json_default) +
"\n\n").encode("utf-8"))
self.wfile.flush()
except (BrokenPipeError, ConnectionResetError):
return False
return True
if not _emit({"type": "snapshot", "task_id": task_id,
"status": task["status"], "entries": entries,
"note": note}):
return
for entry in entries:
sent[entry["id"]] = json.dumps(entry, sort_keys=True,
ensure_ascii=False)
if task["status"] in TERMINAL_STATUSES:
_emit({"type": "done", "status": task["status"]})
return
deadline = time.time() + TRACE_STREAM_MAX_SECONDS
try:
while time.time() < deadline:
time.sleep(TRACE_STREAM_POLL_SECONDS)
row = self._fetch_task(task_id)
if row is None:
_emit({"type": "done", "status": "deleted"})
return
task = _task_to_dict(row)
entries, _note = self._trace_snapshot(task)
for entry in entries:
blob = json.dumps(entry, sort_keys=True,
ensure_ascii=False)
if sent.get(entry["id"]) != blob:
sent[entry["id"]] = blob
if not _emit({"type": "entry", "entry": entry}):
return
if task["status"] in TERMINAL_STATUSES:
_emit({"type": "done", "status": task["status"]})
return
except (BrokenPipeError, ConnectionResetError):
return
_emit({"type": "done", "status": task["status"],
"reason": "timeout"})
def _cancel_task(self, user_id, task_id):
row = self._fetch_task(task_id)
if row is None:
+178
View File
@@ -0,0 +1,178 @@
"""任务实时链路(服务端)单测:GET /tasks/{id}/trace 与 /trace/stream。
覆盖:鉴权(401/403/404)、快照去重与坏行跳过、worktree 在别机
时的空链路 + 说明、已结束任务的流(snapshot 后立刻 done 关流)。
不跑真实模型,不启动 worker;链路目录经 server.trace_dir 注入。
"""
import http.client
import json
import os
import socket
import sqlite3
import sys
import tempfile
import threading
import unittest
from pathlib import Path
try:
from cloud.auth import AuthStore
from cloud.server import create_server
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.auth import AuthStore
from cloud.server import create_server
class TaskTraceTestCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
self.trace_dir = os.path.join(self.tmp.name, "host-traces")
os.makedirs(self.trace_dir, exist_ok=True)
self.auth = AuthStore(self.db_path)
self.owner_id, self.owner_key = self.auth.create_user("owner")
self.other_id, self.other_key = self.auth.create_user("other")
self.server = create_server(self.db_path, "127.0.0.1", 0)
self.server.trace_dir = self.trace_dir
self.port = self.server.server_address[1]
self.thread = threading.Thread(
target=self.server.serve_forever, daemon=True)
self.thread.start()
status, self.project = self._req(
"POST", "/projects", {"name": "demo"}, self.owner_key)
self.assertEqual(status, 201)
status, self.task = self._req(
"POST", "/tasks",
{"project_id": self.project["id"], "prompt": "hi"},
self.owner_key)
self.assertEqual(status, 201)
self.task_id = self.task["id"]
def tearDown(self):
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=5)
try:
self.auth.conn.close()
except Exception:
pass
self.tmp.cleanup()
def _req(self, method, path, body=None, key=None):
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
headers = {"Content-Type": "application/json"}
if key:
headers["X-API-Key"] = key
conn.request(method, path,
json.dumps(body) if body is not None else None, headers)
resp = conn.getresponse()
raw = resp.read()
conn.close()
try:
data = json.loads(raw)
except ValueError:
data = None
return resp.status, data
def _set_task(self, **fields):
conn = sqlite3.connect(self.db_path)
try:
for k, v in fields.items():
conn.execute(
"UPDATE tasks SET %s = ? WHERE id = ?" % k,
(v, self.task_id))
conn.commit()
finally:
conn.close()
def _write_trace(self, lines):
path = os.path.join(self.trace_dir, self.task_id + ".jsonl")
with open(path, "w", encoding="utf-8") as f:
for line in lines:
f.write(line + "\n")
def test_trace_auth_and_not_found(self):
status, _ = self._req(
"GET", "/tasks/%s/trace" % self.task_id)
self.assertEqual(status, 401)
status, _ = self._req(
"GET", "/tasks/%s/trace" % self.task_id, key=self.other_key)
self.assertEqual(status, 403)
status, _ = self._req(
"GET", "/tasks/nope/trace", key=self.owner_key)
self.assertEqual(status, 404)
def test_trace_snapshot_dedup_and_bad_lines(self):
self._write_trace([
json.dumps({"kind": "exec", "id": "c1", "command": "ls",
"exitCode": None}),
"this is not json",
json.dumps({"kind": "exec", "id": "c1", "command": "ls",
"exitCode": 0, "durationMs": 5}),
json.dumps({"kind": "file", "id": "f1:a.py", "path": "a.py"}),
])
status, data = self._req(
"GET", "/tasks/%s/trace" % self.task_id, key=self.owner_key)
self.assertEqual(status, 200)
self.assertEqual(data["status"], "queued")
# 同 id 后到的完成态覆盖先到的开始态,保首次顺序。
self.assertEqual([e["id"] for e in data["entries"]],
["c1", "f1:a.py"])
self.assertEqual(data["entries"][0]["exitCode"], 0)
self.assertEqual(data["note"], "")
def test_trace_missing_file_is_empty(self):
status, data = self._req(
"GET", "/tasks/%s/trace" % self.task_id, key=self.owner_key)
self.assertEqual(status, 200)
self.assertEqual(data["entries"], [])
def test_trace_remote_worktree_host_is_empty_with_note(self):
self._write_trace([
json.dumps({"kind": "exec", "id": "c1", "command": "ls"})])
self._set_task(worktree_host="some-other-host")
status, data = self._req(
"GET", "/tasks/%s/trace" % self.task_id, key=self.owner_key)
self.assertEqual(status, 200)
self.assertEqual(data["entries"], [])
self.assertIn("some-other-host", data["note"])
def _read_sse(self, path, key):
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
headers = {}
if key:
headers["X-API-Key"] = key
conn.request("GET", path, headers=headers)
resp = conn.getresponse()
raw = resp.read()
conn.close()
events = []
for block in raw.decode("utf-8").split("\n\n"):
for line in block.split("\n"):
if line.startswith("data: "):
events.append(json.loads(line[len("data: "):]))
return resp.status, events
def test_stream_finished_task_snapshot_then_done(self):
self._write_trace([
json.dumps({"kind": "exec", "id": "c1", "command": "ls",
"exitCode": 0})])
self._set_task(status="completed")
status, events = self._read_sse(
"/tasks/%s/trace/stream" % self.task_id, self.owner_key)
self.assertEqual(status, 200)
self.assertEqual([e["type"] for e in events], ["snapshot", "done"])
self.assertEqual(events[0]["entries"][0]["id"], "c1")
self.assertEqual(events[1]["status"], "completed")
def test_stream_auth(self):
status, events = self._read_sse(
"/tasks/%s/trace/stream" % self.task_id, None)
self.assertEqual(status, 401)
self.assertEqual(events, [])
if __name__ == "__main__":
unittest.main()