feat: goal 断点续跑端点（G10）+ 摘要持久化与推送 outbox（G11/G12）
G10：POST /tasks/{id}/goal-resume，仅 interrupted 的 goal 任务，
走 host.py goal-resume（无 --goal-text）从 goal_rounds_done 下一轮
断点续跑，保留原线程与 worktree，状态回 queued 并置
goal_resume_pending（worker 认领后清掉），写 task_goal_resumed
审计；goal-met/unmet、failed、非 goal 一律 409；/continue 开新
循环语义不动。Local/Docker 两路新增 goal_resume_breakpoint_task，
不支持断点的执行器明确失败，绝不回落成 goal 或开新循环。
G11：digest 非失败事件持久化进 push_digest_events，worker 循环
每小时按（项目、订阅）分组汇总发一条，失败类仍即时发。
G12：webhook/slack 投递失败写 push_outbox，worker 循环后台重试
（最多 3 次、默认 60 秒退避），3 次仍失败删行并记日志，不丢
任务状态；缺凭证与无 sender 不进 outbox；文档写明 at-least-once
下 webhook/slack 可能重复，接收方须去重。
测试：cloud 769（4 跳过）、host 68、deploy 19 全过。
This commit is contained in:
host committed
1 parent
ac553509ea
commit
5883169c82
8 files changed
+958
-21
No files matched your search
+111
-7
@@ -338,6 +338,8 @@ def _task_from_row(row):
|
||||
"environment_id": _col("environment_id"),
|
||||
# 续跑消息：老行缺列时 None = 没有待续跑消息。
|
||||
"resume_message": _col("resume_message"),
|
||||
# G10：goal 断点续跑标记，老行缺列时 0 = 普通排队。
|
||||
"goal_resume_pending": _col("goal_resume_pending") or 0,
|
||||
# 协调员：老行缺列时 coordinator_parts 为 []（= 普通任务），
|
||||
# parent_task_id/coordinator_part 缺列时为 None。
|
||||
"coordinator_parts": _coord_parse_parts(_col("coordinator_parts")),
|
||||
@@ -367,11 +369,23 @@ class Worker:
|
||||
tenant_template_home=None, tenant_template_repo=None,
|
||||
egress_proxy_port=0, claim_executor_only=False,
|
||||
host_name=None, pr_jobs_only=False,
|
||||
upload_dir=None):
|
||||
upload_dir=None,
|
||||
digest_interval_seconds=3600,
|
||||
outbox_backoff_seconds=60,
|
||||
outbox_max_attempts=3):
|
||||
self.db_path = str(db_path)
|
||||
self.host_py_path = host_py_path
|
||||
self.config_path = config_path
|
||||
self.repo = repo
|
||||
# G11/G12：摘要与 outbox 的后台节奏。digest_interval 是
|
||||
# worker 循环里两次摘要发送的最短间隔（默认 3600 秒，
|
||||
# 与 push.DIGEST_INTERVAL_SECONDS 同值）；outbox 的重试
|
||||
# 节奏由每行 next_attempt_at 控（默认 60 秒退避），
|
||||
# 这里的两个参数只透传给 process_outbox_due。
|
||||
self.digest_interval_seconds = digest_interval_seconds
|
||||
self.outbox_backoff_seconds = outbox_backoff_seconds
|
||||
self.outbox_max_attempts = outbox_max_attempts
|
||||
self._last_digest_sent = 0.0
|
||||
# env 只用于 key 存在性检查：默认接进程环境，单测可注入
|
||||
# 一个普通 dict；worker 永远不把 env 里的值写出。
|
||||
self._env = env if env is not None else os.environ
|
||||
@@ -757,7 +771,8 @@ class Worker:
|
||||
"root_turn_id": "TEXT",
|
||||
"coordinator_parts": "TEXT",
|
||||
"parent_task_id": "TEXT",
|
||||
"coordinator_part": "TEXT"}.items():
|
||||
"coordinator_part": "TEXT",
|
||||
"goal_resume_pending": "INTEGER DEFAULT 0"}.items():
|
||||
if name not in cols:
|
||||
try:
|
||||
conn.execute(
|
||||
@@ -1590,7 +1605,8 @@ class Worker:
|
||||
"host_status": None}
|
||||
fake = SimpleNamespace(returncode=proc.returncode,
|
||||
stdout=out or "", stderr=err or "")
|
||||
if kind in ("goal", "goal-resume") and hasattr(
|
||||
if kind in ("goal", "goal-resume", "goal-resume-breakpoint") \
|
||||
and hasattr(
|
||||
self.executor, "_goal_result_from_runner"):
|
||||
return self.executor._goal_result_from_runner(task_id, fake)
|
||||
if hasattr(self.executor, "_result_from_runner"):
|
||||
@@ -2277,7 +2293,10 @@ class Worker:
|
||||
try:
|
||||
gateway_provider, gateway_env = self._ensure_gateway(task)
|
||||
if self._use_popen():
|
||||
if goal_mode and task.get("_resume"):
|
||||
if goal_mode and task.get("_goal_resume_breakpoint"):
|
||||
# G10 断点：无 --goal-text，不开新循环。
|
||||
popen_kind = "goal-resume-breakpoint"
|
||||
elif goal_mode and task.get("_resume"):
|
||||
popen_kind = "goal-resume"
|
||||
elif goal_mode:
|
||||
popen_kind = "goal"
|
||||
@@ -2315,7 +2334,16 @@ class Worker:
|
||||
"stderr": ""},
|
||||
prep_err)
|
||||
if goal_mode:
|
||||
if task.get("_resume"):
|
||||
if task.get("_goal_resume_breakpoint"):
|
||||
# G10 断点：Docker 路用无 --goal-text 的
|
||||
# goal-resume-breakpoint 命令。
|
||||
result = self.executor.goal_resume_breakpoint_task(
|
||||
task,
|
||||
timeout=effective_docker_timeout,
|
||||
extra_mounts=tenant_mounts,
|
||||
network=tenant_network,
|
||||
extra_env=proxy_env)
|
||||
elif task.get("_resume"):
|
||||
# P1 断点续跑传 None（不传 --goal-text，
|
||||
# host 沿用原 goal+verify）；开新目标
|
||||
# 才传本轮 prompt 当新 goal_text。
|
||||
@@ -2362,7 +2390,25 @@ class Worker:
|
||||
% type(self.executor).__name__)
|
||||
if goal_mode:
|
||||
|
||||
if task.get("_resume") and isinstance(
|
||||
if task.get("_goal_resume_breakpoint") and isinstance(
|
||||
self.executor, LocalExecutor):
|
||||
result = self.executor.goal_resume_breakpoint_task(
|
||||
task)
|
||||
elif task.get("_goal_resume_breakpoint"):
|
||||
# 假执行器/CloudExecutor 没有断点方法时
|
||||
# 绝不回落成 goal（会撞已存在）或
|
||||
# goal-resume --goal-text（会开新循环）：
|
||||
# 明确失败，等支持断点的执行器再跑。
|
||||
result = {"executor": self._executor_name(),
|
||||
"task_id": task_id,
|
||||
"status": "failed", "stdout": "",
|
||||
"returncode": None,
|
||||
"host_status": None, "stderr": "",
|
||||
"goal_met": None,
|
||||
"goal_error": "executor 不支持"
|
||||
" goal 断点续跑"}
|
||||
error = "executor 不支持 goal 断点续跑"
|
||||
elif task.get("_resume") and isinstance(
|
||||
self.executor, LocalExecutor):
|
||||
_resume_text = None if task.get(
|
||||
"_resume_kind") == "breakpoint" \
|
||||
@@ -2926,6 +2972,30 @@ class Worker:
|
||||
return {"dispatched": False, "reason": "already_claimed",
|
||||
"task_id": task_id}
|
||||
task = self.get_task(task_id) or task
|
||||
# G10：goal 断点续跑标记——认领后立即清掉库里标记（本轮
|
||||
# 已认领为 running，不会重复触发；本轮若再被杀回
|
||||
# interrupted，用户需重新调 /goal-resume 才会再置标记，
|
||||
# 不会因旧标记残留而误把一次普通重跑当成断点续跑）。
|
||||
# 断点优先于续跑消息：本轮不把 prompt 换成 resume_message，
|
||||
# 也不消费它（局部变量置 None，终局清理不删库里那条），
|
||||
# 避免断点续跑把一条待续跑消息悄悄吃掉。
|
||||
if task.get("goal_resume_pending"):
|
||||
task = dict(task)
|
||||
task["_goal_resume_breakpoint"] = True
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"UPDATE tasks SET goal_resume_pending = 0"
|
||||
" WHERE id = ?", (task_id,))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
resume_message = None
|
||||
else:
|
||||
resume_message = task.get("resume_message")
|
||||
# 续跑消息：旧任务继续会话时本轮要说的话——本轮 prompt 换
|
||||
# 成它（只换内存里的副本，原 prompt 不动）；_resume 标记
|
||||
# 让 _run_attempt 在 goal 模式下走 goal-resume 而不是 goal
|
||||
@@ -2943,7 +3013,6 @@ class Worker:
|
||||
# task.goal 写成该消息，这里与之对齐）。
|
||||
# - 非 goal 任务：沿用旧行为（prompt 换成续跑消息，走
|
||||
# 已存在续跑 run 路径）。
|
||||
resume_message = task.get("resume_message")
|
||||
if resume_message:
|
||||
task = dict(task)
|
||||
if task.get("verify_command"):
|
||||
@@ -3800,6 +3869,38 @@ class Worker:
|
||||
break
|
||||
return results
|
||||
|
||||
def _process_push_due(self, now=None):
|
||||
"""G11/G12 后台投递：每轮循环调用，绝不中断在跑任务。
|
||||
|
||||
outbox 每轮都试到期的行（重试节奏由 next_attempt_at
|
||||
控，默认 60 秒退避、最多 3 次，到上限后删行并记日志，
|
||||
不丢任务状态）；摘要只在距上次发送满 digest_interval
|
||||
后才发一次（默认每小时），避免每轮都把摘要提前发掉。
|
||||
全部异常只记日志，不抛、不炸循环。force 语义由调用方
|
||||
通过把 _last_digest_sent 置 0 实现（单测用）。
|
||||
"""
|
||||
now = time.time() if now is None else now
|
||||
try:
|
||||
from cloud.push import process_outbox_due, send_digest_due
|
||||
out = process_outbox_due(
|
||||
self.db_path, now=now,
|
||||
max_attempts=self.outbox_max_attempts,
|
||||
backoff_seconds=self.outbox_backoff_seconds)
|
||||
for failed in out.get("failed", []):
|
||||
log_event(self.logger, "worker_push_outbox_dropped",
|
||||
message="push outbox dropped after retries: %s"
|
||||
% failed)
|
||||
if now - self._last_digest_sent >= \
|
||||
self.digest_interval_seconds:
|
||||
self._last_digest_sent = now
|
||||
digest = send_digest_due(self.db_path, now=now)
|
||||
for err in digest.get("errors", []):
|
||||
log_event(self.logger, "worker_push_digest_failed",
|
||||
message="push digest send failed: %s" % err)
|
||||
except Exception as exc:
|
||||
log_event(self.logger, "worker_push_due_failed",
|
||||
message="push due processing failed: %s" % exc)
|
||||
|
||||
def run_loop(self, interval=300, max_seconds=None, stop_event=None,
|
||||
drain_timeout=60, sweep_threshold=300):
|
||||
"""常驻循环：每 interval 秒跑一轮 run_all，空闲也不退出。
|
||||
@@ -3879,6 +3980,9 @@ class Worker:
|
||||
message="worker_loop_error: %s" % exc)
|
||||
results = []
|
||||
all_results.extend(results)
|
||||
# G11/G12 后台投递：摘要（每小时）与 outbox 重试，
|
||||
# 轮次边界做，不中断在跑任务，失败只记日志。
|
||||
self._process_push_due()
|
||||
# 自动陈旧回收：死 worker 的 running 任务在这里被
|
||||
# 标 interrupted；活任务心跳新鲜，不会被扫到。
|
||||
if sweep_threshold is not None:
|
||||
|
||||
Reference in new issue
Block a user