feat: 任务实时链路 SSE (#61)
host.py：回合通知摘要成链路条目，增量追加到 <state>/host-traces/<task>.jsonl（上限 5000 行）；dispatch/run/goal/delegate 设链路文件
cloud/server.py：GET /tasks/{id}/trace 快照 + /trace/stream SSE（snapshot → entry → done；worktree 在别机时空链路 + 说明）
console：详情链路区，运行中 fetch 流式渲染（EventSource 不能带 X-API-Key 头），终态走快照；DOM 上限 1000 条
单测：test_host_trace 7 + test_task_trace 6；host 49、cloud 525、deploy 14 全过
Co-authored-by: host <host@test.local>
This commit is contained in:
openqjandhost authored and GitHub committed
1 parent
6022f1dada
commit
799f7c3edb
6 files changed
+696
No files matched your search
+152
@@ -42,6 +42,7 @@ import html
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import socket
|
||||
import sqlite3
|
||||
import threading
|
||||
@@ -126,6 +127,53 @@ _server_log = logging.getLogger("cloud.server")
|
||||
TERMINAL_STATUSES = frozenset(
|
||||
{"completed", "failed", "interrupted", "merged", "merge-failed"}
|
||||
)
|
||||
# 实时链路（任务过程事件流）：host 在回合进行中把 app-server
|
||||
# 通知摘要成条目追加写 jsonl，服务端 tail 文件经 SSE 推送。
|
||||
TRACE_STREAM_MAX_SECONDS = 3600
|
||||
TRACE_STREAM_POLL_SECONDS = 1.0
|
||||
TRACE_DEFAULT_DIR = "/opt/codex-app-server/work/host-traces"
|
||||
|
||||
|
||||
def _trace_file_name(task_id):
|
||||
"""云任务 id -> 链路文件名：与 cloud.sandbox 的 _host_name
|
||||
同款清洗（只保留 [A-Za-z0-9_.-]，其余替 -），两边算出同一名字。"""
|
||||
return re.sub(r"[^A-Za-z0-9_.-]", "-", str(task_id))
|
||||
|
||||
|
||||
def _trace_dir_for(server):
|
||||
"""链路目录解析：server 属性（单测注入）> 环境变量 > 默认路径。"""
|
||||
d = getattr(server, "trace_dir", None)
|
||||
if d:
|
||||
return str(d)
|
||||
return os.environ.get("CODEX_TRACE_DIR") or TRACE_DEFAULT_DIR
|
||||
|
||||
|
||||
def _read_trace_entries(path):
|
||||
"""读 host 写的 jsonl 链路文件：跳过坏行；按 id 去重，后到的
|
||||
完成态覆盖先到的开始态，保首次顺序。文件不存在返回空列表。"""
|
||||
entries = []
|
||||
index = {}
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
if not isinstance(entry, dict) or "id" not in entry:
|
||||
continue
|
||||
eid = entry["id"]
|
||||
if eid in index:
|
||||
entries[index[eid]] = entry
|
||||
else:
|
||||
index[eid] = len(entries)
|
||||
entries.append(entry)
|
||||
except OSError:
|
||||
return []
|
||||
return entries
|
||||
DEFAULT_SWEEP_THRESHOLD_SECONDS = 300
|
||||
# 限流默认值：每 IP / 每 API key 每分钟 120 次；环境变量
|
||||
# CLOUD_RATE_LIMIT_PER_MINUTE / CLOUD_RATE_LIMIT_WINDOW_SECONDS 可调。
|
||||
@@ -848,6 +896,11 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
self._get_task(user_id, rest)
|
||||
elif rest.endswith("/cost") and method == "GET":
|
||||
self._task_cost(user_id, rest[: -len("/cost")])
|
||||
elif rest.endswith("/trace/stream") and method == "GET":
|
||||
self._task_trace_stream(
|
||||
user_id, rest[: -len("/trace/stream")])
|
||||
elif rest.endswith("/trace") and method == "GET":
|
||||
self._task_trace(user_id, rest[: -len("/trace")])
|
||||
elif rest.endswith("/cancel") and method == "POST":
|
||||
self._cancel_task(user_id, rest[: -len("/cancel")])
|
||||
elif rest.endswith("/merge") and method == "POST":
|
||||
@@ -1553,6 +1606,105 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
|
||||
return
|
||||
self._send_json(200, task)
|
||||
|
||||
# -- 任务实时链路 --------------------------------------------------
|
||||
def _trace_snapshot(self, task):
|
||||
"""当前链路快照 (entries, note)：worktree 在别机时读不到
|
||||
文件，返回空链路 + 说明（多机诚实降级，不谎报无过程）。"""
|
||||
worktree_host = task.get("worktree_host")
|
||||
if worktree_host and worktree_host != socket.gethostname():
|
||||
return [], ("链路文件在 worktree 主机 %s 上，本机读不到"
|
||||
% worktree_host)
|
||||
path = os.path.join(
|
||||
_trace_dir_for(self.server),
|
||||
_trace_file_name(task["id"]) + ".jsonl")
|
||||
return _read_trace_entries(path), ""
|
||||
|
||||
def _task_trace(self, user_id, task_id):
|
||||
row = self._fetch_task(task_id)
|
||||
if row is None:
|
||||
self._send_json(404, {"error": "task not found"})
|
||||
return
|
||||
task = _task_to_dict(row)
|
||||
with _auth_store(self.server.db_path) as store:
|
||||
if not store.check_permission(user_id, task["project_id"], "read"):
|
||||
self._send_json(403, {"error": "forbidden"})
|
||||
return
|
||||
entries, note = self._trace_snapshot(task)
|
||||
self._send_json(200, {"task_id": task_id, "status": task["status"],
|
||||
"entries": entries, "note": note})
|
||||
|
||||
def _task_trace_stream(self, user_id, task_id):
|
||||
row = self._fetch_task(task_id)
|
||||
if row is None:
|
||||
self._send_json(404, {"error": "task not found"})
|
||||
return
|
||||
task = _task_to_dict(row)
|
||||
with _auth_store(self.server.db_path) as store:
|
||||
if not store.check_permission(user_id, task["project_id"], "read"):
|
||||
self._send_json(403, {"error": "forbidden"})
|
||||
return
|
||||
entries, note = self._trace_snapshot(task)
|
||||
try:
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/event-stream")
|
||||
self.send_header("Cache-Control", "no-cache")
|
||||
self.send_header("Connection", "keep-alive")
|
||||
self.end_headers()
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
return
|
||||
# SSE 无 Content-Length，done 后靠关连接界定流结束；注意
|
||||
# send_header("Connection", "keep-alive") 会把
|
||||
# close_connection 改回 False，所以必须在 end_headers 之后
|
||||
# 再置 True（CPython send_header 的 Connection 分支行为）。
|
||||
self.close_connection = True
|
||||
sent = {}
|
||||
|
||||
def _emit(obj):
|
||||
try:
|
||||
self.wfile.write(
|
||||
("data: " + json.dumps(obj, ensure_ascii=False,
|
||||
default=_json_default) +
|
||||
"\n\n").encode("utf-8"))
|
||||
self.wfile.flush()
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
return False
|
||||
return True
|
||||
|
||||
if not _emit({"type": "snapshot", "task_id": task_id,
|
||||
"status": task["status"], "entries": entries,
|
||||
"note": note}):
|
||||
return
|
||||
for entry in entries:
|
||||
sent[entry["id"]] = json.dumps(entry, sort_keys=True,
|
||||
ensure_ascii=False)
|
||||
if task["status"] in TERMINAL_STATUSES:
|
||||
_emit({"type": "done", "status": task["status"]})
|
||||
return
|
||||
deadline = time.time() + TRACE_STREAM_MAX_SECONDS
|
||||
try:
|
||||
while time.time() < deadline:
|
||||
time.sleep(TRACE_STREAM_POLL_SECONDS)
|
||||
row = self._fetch_task(task_id)
|
||||
if row is None:
|
||||
_emit({"type": "done", "status": "deleted"})
|
||||
return
|
||||
task = _task_to_dict(row)
|
||||
entries, _note = self._trace_snapshot(task)
|
||||
for entry in entries:
|
||||
blob = json.dumps(entry, sort_keys=True,
|
||||
ensure_ascii=False)
|
||||
if sent.get(entry["id"]) != blob:
|
||||
sent[entry["id"]] = blob
|
||||
if not _emit({"type": "entry", "entry": entry}):
|
||||
return
|
||||
if task["status"] in TERMINAL_STATUSES:
|
||||
_emit({"type": "done", "status": task["status"]})
|
||||
return
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
return
|
||||
_emit({"type": "done", "status": task["status"],
|
||||
"reason": "timeout"})
|
||||
|
||||
def _cancel_task(self, user_id, task_id):
|
||||
row = self._fetch_task(task_id)
|
||||
if row is None:
|
||||
|
||||
Reference in new issue
Block a user