feat: 计划模式 + 后续消息 + 会话分支接云端 (#65)
- 计划模式:tasks 加 plan 列;plan 与 approval=full / goal 冲突时
400;执行器在 approval 参数后追加 --sandbox read-only(本地
dispatch/run/goal/popen 与 Docker 内层命令全覆盖),host 任务
存下 sandbox 后续轮次自动继承只读。
- 后续消息:新表 task_followups;POST/GET /tasks/{id}/followups
(仅 queued/running 的非 goal 任务;direct 立即往链路追加
phase=followup-direct 条目,本轮跑完后优先于排队执行;排队
按 direct 优先、队内先来先服务逐条再跑一轮,同一 thread;
一条失败即停、剩余 pending 留给重试入口;取消时回 pending)。
- 会话分支:POST /tasks/{id}/fork(仅终态任务;继承 executor/
模型/强度/权限/计划,goal 字段置空);worker 先 fork 再跑首轮,
首轮失败不自动重试、不重复分叉,失败后走重试入口重跑首轮。
- 修任务级模型被覆盖:worker 此前无条件写成主模型,控制台
选的模型实际不生效;现为任务级 > worker 主模型 > 配置。
- Docker 租户准备抽成 _docker_task_context,供普通尝试与 fork 共用。
- console:新建任务计划勾选(含冲突拦截)、详情计划徽标与
分支来源、分支按钮与表单、后续消息区、列表 [计划]/[分支] 标记。
- 测试:cloud/tests/test_task_links.py 17 个(server 校验与继承、
worker 模型优先、只读命令、排空顺序与停排、goal 不排空、
fork 单次性);全量 cloud 542、host 49、deploy 14。
Co-authored-by: host <host@test.local>
This commit is contained in:
1 parent
6da3745127
commit
f4443dca4f
7 files changed
+1120
-74
No files matched your search
+225
-63
@@ -238,6 +238,8 @@ def _task_from_row(row):
|
||||
"model": _col("model"),
|
||||
"effort": _col("effort"),
|
||||
"approval": _col("approval"),
|
||||
"plan": _col("plan"),
|
||||
"forked_from": _col("forked_from"),
|
||||
}
|
||||
|
||||
|
||||
@@ -563,7 +565,9 @@ class Worker:
|
||||
"worktree_host": "TEXT",
|
||||
"model": "TEXT",
|
||||
"effort": "TEXT",
|
||||
"approval": "TEXT"}.items():
|
||||
"approval": "TEXT",
|
||||
"plan": "INTEGER",
|
||||
"forked_from": "TEXT"}.items():
|
||||
if name not in cols:
|
||||
try:
|
||||
conn.execute(
|
||||
@@ -1155,6 +1159,158 @@ class Worker:
|
||||
return None
|
||||
|
||||
# -- 执行 ----------------------------------------------------------
|
||||
def _docker_task_context(self, task):
|
||||
"""Docker 任务的租户挂载/网络/代理准备(_run_attempt 与
|
||||
_run_fork 共用)。成功返回 (True, mounts, network,
|
||||
proxy_env, None);失败返回 (False, None, None, None,
|
||||
error_text)。网络建失败绝不降级默认网络跑容器(降级会
|
||||
让隔离悄悄失效)。"""
|
||||
tenant_mounts = None
|
||||
tenant_network = None
|
||||
proxy_env = None
|
||||
if self.tenant_isolation:
|
||||
try:
|
||||
tenant_mounts = self._tenant_mounts(
|
||||
task.get("project_id"))
|
||||
tenant_network = self._tenant_network(
|
||||
task.get("project_id"))
|
||||
except (ValueError, RuntimeError) as exc:
|
||||
return (False, None, None, None,
|
||||
"租户隔离准备失败: %s" % exc)
|
||||
if not self.executor.ensure_network(tenant_network):
|
||||
return (False, None, None, None,
|
||||
"租户 Docker 网络创建失败: %s" % tenant_network)
|
||||
if self.egress_proxy_port:
|
||||
# 代理在宿主上,容器经所在网络的网关访问;租户模式
|
||||
# 用项目网络,非租户用默认 bridge。
|
||||
gw = self.executor.network_gateway(tenant_network)
|
||||
if not gw:
|
||||
return (False, None, None, None,
|
||||
"egress 代理网关解析失败,拒绝无代理"
|
||||
"运行(避免绕过出口白名单)")
|
||||
proxy_url = "http://%s:%s" % (gw, self.egress_proxy_port)
|
||||
proxy_env = {
|
||||
"HTTP_PROXY": proxy_url,
|
||||
"HTTPS_PROXY": proxy_url,
|
||||
"http_proxy": proxy_url,
|
||||
"https_proxy": proxy_url,
|
||||
"NO_PROXY": "localhost,127.0.0.1",
|
||||
"no_proxy": "localhost,127.0.0.1",
|
||||
}
|
||||
return (True, tenant_mounts, tenant_network, proxy_env, None)
|
||||
|
||||
def _run_fork(self, task):
|
||||
"""分支任务执行:host.py fork 建分支 thread,再跑首轮。
|
||||
返回 (result, error),语义同 _run_attempt。fork 本身不
|
||||
跑模型;首轮走 _run_attempt(dispatch 撞已存在 → run
|
||||
续跑,三路取消/超时处理与普通任务一致)。首轮失败不
|
||||
自动重试/fallback:失败后走重试入口重跑首轮(已存在
|
||||
续跑),不重复 fork。"""
|
||||
task_id = task["id"]
|
||||
source_id = task.get("forked_from")
|
||||
if not hasattr(self.executor, "fork_task"):
|
||||
return ({"executor": self._executor_name(),
|
||||
"task_id": task_id,
|
||||
"status": "failed", "stdout": "",
|
||||
"returncode": None, "host_status": None,
|
||||
"stderr": ""},
|
||||
"executor 不支持 fork")
|
||||
is_docker = isinstance(self.executor, DockerExecutor)
|
||||
if is_docker:
|
||||
(ok, tenant_mounts, tenant_network, proxy_env,
|
||||
prep_err) = self._docker_task_context(task)
|
||||
if not ok:
|
||||
return ({"executor": "docker",
|
||||
"task_id": task_id,
|
||||
"status": "failed", "stdout": "",
|
||||
"returncode": None, "host_status": None,
|
||||
"stderr": ""},
|
||||
prep_err)
|
||||
else:
|
||||
tenant_mounts = tenant_network = proxy_env = None
|
||||
try:
|
||||
fork_res = self.executor.fork_task(
|
||||
task, source_id,
|
||||
timeout=(self.task_timeout if is_docker else None),
|
||||
extra_mounts=tenant_mounts,
|
||||
network=tenant_network,
|
||||
extra_env=proxy_env)
|
||||
except Exception as exc:
|
||||
return ({"executor": self._executor_name(),
|
||||
"task_id": task_id,
|
||||
"status": "failed", "stdout": "",
|
||||
"returncode": None, "host_status": None,
|
||||
"stderr": ""},
|
||||
str(exc))
|
||||
if (fork_res or {}).get("status") != "completed":
|
||||
err = ((fork_res or {}).get("stderr", "") or
|
||||
(fork_res or {}).get("stdout", "") or "fork failed")
|
||||
err = err.strip()
|
||||
return (fork_res,
|
||||
err[-1000:] if err else "fork failed")
|
||||
return self._run_attempt(task)
|
||||
|
||||
def _claim_next_followup(self, task_id):
|
||||
"""取下一条后续消息:direct 优先、队内按创建时间先来
|
||||
先服务。CAS 认领(pending → running),抢不到或表不
|
||||
在返回 None(表由 server init_db 建;worker 直连老库
|
||||
时缺表不炸 run_one,只当没有后续消息)。"""
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
except Exception:
|
||||
return None
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM task_followups"
|
||||
" WHERE task_id = ? AND status = 'pending'"
|
||||
" ORDER BY CASE WHEN mode = 'direct' THEN 0 ELSE 1 END,"
|
||||
" created_at, id LIMIT 1",
|
||||
(task_id,)).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
cur = conn.execute(
|
||||
"UPDATE task_followups SET status = 'running',"
|
||||
" updated_at = ? WHERE id = ? AND status = 'pending'",
|
||||
(time.time(), row["id"]))
|
||||
conn.commit()
|
||||
if cur.rowcount != 1:
|
||||
return None
|
||||
return {"id": row["id"], "body": row["body"],
|
||||
"mode": row["mode"]}
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def _finish_followup(self, fu_id, status):
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"UPDATE task_followups SET status = ?,"
|
||||
" updated_at = ? WHERE id = ?",
|
||||
(status, time.time(), fu_id))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _restore_followup(self, fu_id):
|
||||
"""取消时把跑中的后续消息放回 pending,留给重试入口。"""
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"UPDATE task_followups SET status = 'pending',"
|
||||
" updated_at = ? WHERE id = ? AND status = 'running'",
|
||||
(time.time(), fu_id))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _run_attempt(self, task):
|
||||
"""单次执行尝试(不含认领、key 门、重试、成本与最终审计)。
|
||||
|
||||
@@ -1214,62 +1370,20 @@ class Worker:
|
||||
deadline_seconds=self.task_timeout)
|
||||
else:
|
||||
if is_docker:
|
||||
tenant_mounts = None
|
||||
tenant_network = None
|
||||
proxy_env = None
|
||||
if self.tenant_isolation:
|
||||
try:
|
||||
tenant_mounts = self._tenant_mounts(
|
||||
task.get("project_id"))
|
||||
tenant_network = self._tenant_network(
|
||||
task.get("project_id"))
|
||||
except (ValueError, RuntimeError) as exc:
|
||||
# 提前返回会跳过下面的 finally(清容器
|
||||
# 标记),这里先手动清掉,否则 _running
|
||||
# 里永远残留该任务,心跳与退出清理
|
||||
# 都会把它当在跑任务。
|
||||
self._running_clear(task_id)
|
||||
return ({"executor": "docker",
|
||||
"task_id": task_id,
|
||||
"status": "failed", "stdout": "",
|
||||
"returncode": None, "host_status": None,
|
||||
"stderr": ""},
|
||||
"租户隔离准备失败: %s" % exc)
|
||||
# 网络建失败绝不降级默认网络跑容器(降级会
|
||||
# 让隔离悄悄失效);直接让本次尝试 failed,
|
||||
# 调用方按普通失败处理(重试/fallback 照常)。
|
||||
if not self.executor.ensure_network(tenant_network):
|
||||
self._running_clear(task_id)
|
||||
return ({"executor": "docker",
|
||||
"task_id": task_id,
|
||||
"status": "failed", "stdout": "",
|
||||
"returncode": None, "host_status": None,
|
||||
"stderr": ""},
|
||||
"租户 Docker 网络创建失败: %s"
|
||||
% tenant_network)
|
||||
if self.egress_proxy_port:
|
||||
# 代理在宿主上,容器经所在网络的网关访问;
|
||||
# 租户模式用项目网络,非租户用默认 bridge。
|
||||
gw = self.executor.network_gateway(tenant_network)
|
||||
if not gw:
|
||||
self._running_clear(task_id)
|
||||
return ({"executor": "docker",
|
||||
"task_id": task_id,
|
||||
"status": "failed", "stdout": "",
|
||||
"returncode": None, "host_status": None,
|
||||
"stderr": ""},
|
||||
"egress 代理网关解析失败,拒绝无代理"
|
||||
"运行(避免绕过出口白名单)")
|
||||
proxy_url = "http://%s:%s" % (
|
||||
gw, self.egress_proxy_port)
|
||||
proxy_env = {
|
||||
"HTTP_PROXY": proxy_url,
|
||||
"HTTPS_PROXY": proxy_url,
|
||||
"http_proxy": proxy_url,
|
||||
"https_proxy": proxy_url,
|
||||
"NO_PROXY": "localhost,127.0.0.1",
|
||||
"no_proxy": "localhost,127.0.0.1",
|
||||
}
|
||||
(ok, tenant_mounts, tenant_network, proxy_env,
|
||||
prep_err) = self._docker_task_context(task)
|
||||
if not ok:
|
||||
# 提前返回会跳过下面的 finally(清容器
|
||||
# 标记),这里先手动清掉,否则 _running
|
||||
# 里永远残留该任务,心跳与退出清理
|
||||
# 都会把它当在跑任务。
|
||||
self._running_clear(task_id)
|
||||
return ({"executor": "docker",
|
||||
"task_id": task_id,
|
||||
"status": "failed", "stdout": "",
|
||||
"returncode": None, "host_status": None,
|
||||
"stderr": ""},
|
||||
prep_err)
|
||||
if goal_mode:
|
||||
result = self.executor.goal_task(
|
||||
task, timeout=effective_docker_timeout,
|
||||
@@ -1484,12 +1598,14 @@ class Worker:
|
||||
actual_model = self.model
|
||||
fallback_model = None
|
||||
first_error = None
|
||||
# 模型经任务的 model 字段传给执行器(执行器再经 host.py 的
|
||||
# 顶层 --model 参数传给 dispatch/run);不设时 host.py 用
|
||||
# 配置文件里的 model。这里显式设成 worker 的主模型,让
|
||||
# --model 覆盖配置真正生效,fallback 时再换成新模型。
|
||||
# 模型优先级:任务级 > worker 主模型 > 配置文件(host.py
|
||||
# 再兜底)。此前这里无条件写成 worker 主模型,导致控制
|
||||
# 台选的模型被悄悄覆盖(任务级模型实际不生效);修掉。
|
||||
# fallback 时下面再把 task["model"] 换成 fallback 模型。
|
||||
task = dict(task)
|
||||
task["model"] = self.model
|
||||
if not task.get("model"):
|
||||
task["model"] = self.model
|
||||
actual_model = task["model"]
|
||||
# 有限重试与真实 fallback 的优先级:取消不重试;一次尝试失败
|
||||
# 且可 fallback、且显式配了 fallback_models 时,优先真换模型
|
||||
# 再跑一次(最多 1 次,同一任务、同一 worktree/分支,dispatch
|
||||
@@ -1501,7 +1617,16 @@ class Worker:
|
||||
# 各次尝试的真实用量都要算进成本(失败尝试花的钱也是钱):
|
||||
# 主尝试与 fallback 尝试的结果都先累加,最后只记一条汇总。
|
||||
usage_total = None
|
||||
for attempt in range(self.max_retries + 1):
|
||||
# 分支任务:先 fork 再跑首轮(见 _run_fork);首轮失败
|
||||
# 不自动重试/fallback,失败后走重试入口重跑首轮(已存在
|
||||
# 续跑),不重复 fork。range(0) 让下面的尝试循环跳过,
|
||||
# 直接进统一的收尾(超时/取消/成本/审计/通知)逻辑。
|
||||
if task.get("forked_from"):
|
||||
result, error = self._run_fork(task)
|
||||
usage_total = sum_usage(usage_total,
|
||||
(result or {}).get("usage"))
|
||||
for attempt in range(0 if task.get("forked_from")
|
||||
else self.max_retries + 1):
|
||||
result, error = self._run_attempt(task)
|
||||
usage_total = sum_usage(usage_total, (result or {}).get("usage"))
|
||||
if self._was_cancelled(task_id) or \
|
||||
@@ -1566,6 +1691,43 @@ class Worker:
|
||||
error = "task timeout after %s seconds" % self.task_timeout
|
||||
self._audit(task["project_id"], "worker_timeout", task_id)
|
||||
|
||||
# 后续消息(排队/直接发送):首轮 completed 且非 goal 时,
|
||||
# 按 direct 优先、队内先来先服务逐条再跑一轮(同一 thread,
|
||||
# 历史与 worktree 改动都在)。直接发送在 POST 时已进链路,
|
||||
# 这里只负责把轮次跑出来。一条失败即停:任务记 failed,
|
||||
# 剩余 pending 留给重试入口(重试走已存在续跑,跑完继续
|
||||
# 排空剩余消息);取消时该条回 pending,任务走中断收尾。
|
||||
if (result is not None and result.get("status") == "completed"
|
||||
and not task.get("verify_command")):
|
||||
while True:
|
||||
fu = self._claim_next_followup(task_id)
|
||||
if fu is None:
|
||||
break
|
||||
fu_task = dict(task)
|
||||
fu_task["prompt"] = fu["body"]
|
||||
fu_result, fu_error = self._run_attempt(fu_task)
|
||||
usage_total = sum_usage(
|
||||
usage_total, (fu_result or {}).get("usage"))
|
||||
if self._was_cancelled(task_id) or \
|
||||
(fu_result or {}).get("status") == "cancelled":
|
||||
self._restore_followup(fu["id"])
|
||||
result = fu_result
|
||||
break
|
||||
if (fu_result or {}).get("status") == "timeout":
|
||||
fu_result = dict(fu_result, status="failed")
|
||||
fu_error = ("task timeout after %s seconds"
|
||||
% self.task_timeout)
|
||||
if (fu_result or {}).get("status") == "completed":
|
||||
self._finish_followup(fu["id"], "done")
|
||||
result = fu_result
|
||||
error = None
|
||||
continue
|
||||
self._finish_followup(fu["id"], "failed")
|
||||
result = fu_result
|
||||
error = fu_error or self._attempt_error(
|
||||
fu_result, None, fu_result.get("host_status"))
|
||||
break
|
||||
|
||||
# 取消优先:执行期间被 cancel 的任务保持 interrupted,
|
||||
# 绝不用执行结果覆盖它(见模块 docstring 的取消说明)。
|
||||
if self._was_cancelled(task_id) or result.get("status") == "cancelled":
|
||||
|
||||
Reference in new issue
Block a user