feat: 任务附件——server/worker/console 链路打通（+ popen goal 新语义补漏）
server：POST /uploads（multipart，413 预检、路径/数量/大小校验、 24h 过期包、used 防一包多用）+ 建任务绑定（同事务写附件行 + 包置 used，cloud 执行器 400）+ 列表/详情带 attachments + 取消删暂存 + 分支带走附件清单；db.py PG 建表 + init_db 补表。 worker：_task_seed_dir（无附件/目录已删静默 None）+ dispatch/ goal 三路（同步本地、popen、Docker 只读挂载 /tmp/codex-seed） + cloud 执行器兜底失败 + 终局删暂存。 console：文件/文件夹/图片三路输入，先上传拿 upload_id 再建任务， 列表 [附件×n]、详情列附件路径。 附带：popen 的 goal 命令此前强制要求 verify_command，goal 新语义 任务到本地 worker（走 popen）会直接报错，改成与同步 goal_task 一致（无验收不传 --verify，轮次 0 原样传）。 测试：新增 test_task_attachments.py 15 个。
This commit is contained in:
host committed
1 parent
9537e6f0d2
commit
7c444a688b
8 files changed
+869
-32
No files matched your search
+65
-8
@@ -122,6 +122,7 @@ import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
@@ -140,6 +141,7 @@ from cloud.pr_exec import (claim_pr_job, execute_task_pr, finish_pr_job)
|
||||
from cloud.queue import budget_exceeded, ready_tasks
|
||||
from cloud.repos import RepoError, checkout_dir, resolve_repo
|
||||
from cloud.sandbox import CloudExecutor, DockerExecutor, LocalExecutor
|
||||
from cloud.uploads import upload_root as _upload_root
|
||||
from cloud.sandbox import _default_runner as _sandbox_default_runner
|
||||
from cloud.workers import heartbeat_worker, register_worker
|
||||
|
||||
@@ -268,7 +270,8 @@ class Worker:
|
||||
tenant_isolation=False, tenants_root=None,
|
||||
tenant_template_home=None, tenant_template_repo=None,
|
||||
egress_proxy_port=0, claim_executor_only=False,
|
||||
host_name=None, pr_jobs_only=False):
|
||||
host_name=None, pr_jobs_only=False,
|
||||
upload_dir=None):
|
||||
self.db_path = str(db_path)
|
||||
self.host_py_path = host_py_path
|
||||
self.config_path = config_path
|
||||
@@ -378,6 +381,9 @@ class Worker:
|
||||
# 只跑 pr 任务（HA 阶段 1 的跨主机 /pr 演练与专用 pr-runner
|
||||
# 用）：不认领普通任务，只处理本主机的 pr_jobs。
|
||||
self.pr_jobs_only = bool(pr_jobs_only)
|
||||
# 附件暂存根：显式传入 > 环境变量 > 默认路径（与 server 侧
|
||||
# 同款解析；两边不一致时 worker 找不到包，派发会静默无附件）。
|
||||
self.upload_dir = str(upload_dir) if upload_dir else _upload_root()
|
||||
# 运维心跳：路径优先显式参数，其次环境变量，默认当前目录下
|
||||
# 的 worker-heartbeat.json；主机路径不在代码里硬编码，主机
|
||||
# 部署时用 --heartbeat-file 或环境变量传入（见 deploy/README）。
|
||||
@@ -928,8 +934,31 @@ class Worker:
|
||||
is _sandbox_default_runner
|
||||
and hasattr(self.executor, "popen_task"))
|
||||
|
||||
def _task_seed_dir(self, task):
|
||||
"""任务附件暂存目录（派发时 seed 用）：无附件返回 None；
|
||||
有附件行但目录已不在（如终局后已清理）也返回 None——
|
||||
续跑任务不需要重 seed（文件已在 worktree 里），静默跳过
|
||||
比报错更正确（重试走已存在续跑时包早已删了）。"""
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT bundle_id FROM task_attachments"
|
||||
" WHERE task_id = ? LIMIT 1",
|
||||
(task["id"],),
|
||||
).fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception:
|
||||
return None
|
||||
if row is None:
|
||||
return None
|
||||
bundle_id = row[0] if isinstance(row, tuple) else row["bundle_id"]
|
||||
bundle_dir = os.path.join(self.upload_dir, bundle_id)
|
||||
return bundle_dir if os.path.isdir(bundle_dir) else None
|
||||
|
||||
def _execute_via_process(self, task, kind="dispatch",
|
||||
deadline_seconds=None):
|
||||
deadline_seconds=None, seed_host_dir=None):
|
||||
"""进程模式执行一轮：启动、轮询、取消时杀进程组。
|
||||
|
||||
返回与同步路径同款 result dict（status/stdout/stderr/
|
||||
@@ -942,7 +971,11 @@ class Worker:
|
||||
if self.process_factory is not None:
|
||||
proc = self.process_factory(task)
|
||||
else:
|
||||
proc = self.executor.popen_task(task, kind=kind)
|
||||
# seed 只对 dispatch/goal 首轮有意义：run 续跑不传
|
||||
# （文件已在 worktree 里）；popen_task 内部同样只在
|
||||
# dispatch/goal 命令上追加 --seed-dir。
|
||||
proc = self.executor.popen_task(
|
||||
task, kind=kind, seed_dir=seed_host_dir)
|
||||
self._running_set(task_id, proc=proc)
|
||||
try:
|
||||
self._set_child_pid(task_id, getattr(proc, "pid", None))
|
||||
@@ -1564,6 +1597,16 @@ class Worker:
|
||||
# 的已存在续跑（run）对它没有意义（goal 没有 run 语义）。
|
||||
goal_mode = bool(task.get("goal_mode")
|
||||
or task.get("verify_command"))
|
||||
# 附件 seed：dispatch/goal 首轮才用；run 续跑不传（文件已
|
||||
# 在 worktree 里）。cloud 执行器没有 worktree 可 seed，
|
||||
# 创建时本应已拒绝，这里是兜底防线（直接失败，不静默跑）。
|
||||
seed_host_dir = self._task_seed_dir(task)
|
||||
if seed_host_dir and isinstance(self.executor, CloudExecutor):
|
||||
return ({"executor": "cloud", "task_id": task_id,
|
||||
"status": "failed", "stdout": "",
|
||||
"returncode": None, "host_status": None,
|
||||
"stderr": ""},
|
||||
"cloud 执行器不支持附件")
|
||||
# Docker 同步路径：dispatch 阻塞在 docker run 里时，取消
|
||||
# 线程靠容器名杀容器（见 cancel()）。容器名按任务派生，
|
||||
# 与执行器本身无关，所以这里只记名不碰执行器状态。
|
||||
@@ -1593,7 +1636,8 @@ class Worker:
|
||||
if self._use_popen():
|
||||
result = self._execute_via_process(
|
||||
task, kind="goal" if goal_mode else "dispatch",
|
||||
deadline_seconds=self.task_timeout)
|
||||
deadline_seconds=self.task_timeout,
|
||||
seed_host_dir=seed_host_dir)
|
||||
combined = (result.get("stdout", "") or "") + \
|
||||
(result.get("stderr", "") or "")
|
||||
if not goal_mode and result["status"] not in (
|
||||
@@ -1623,18 +1667,22 @@ class Worker:
|
||||
task, timeout=effective_docker_timeout,
|
||||
extra_mounts=tenant_mounts,
|
||||
network=tenant_network,
|
||||
extra_env=proxy_env)
|
||||
extra_env=proxy_env,
|
||||
seed_host_dir=seed_host_dir)
|
||||
else:
|
||||
result = self.executor.dispatch_task(
|
||||
task, timeout=effective_docker_timeout,
|
||||
extra_mounts=tenant_mounts,
|
||||
network=tenant_network,
|
||||
extra_env=proxy_env)
|
||||
extra_env=proxy_env,
|
||||
seed_host_dir=seed_host_dir)
|
||||
else:
|
||||
if goal_mode:
|
||||
result = self.executor.goal_task(task)
|
||||
result = self.executor.goal_task(
|
||||
task, seed_dir=seed_host_dir)
|
||||
else:
|
||||
result = self.executor.dispatch_task(task)
|
||||
result = self.executor.dispatch_task(
|
||||
task, seed_dir=seed_host_dir)
|
||||
combined = (result.get("stdout", "") or "") + \
|
||||
(result.get("stderr", "") or "")
|
||||
if not goal_mode and result["status"] != "completed" and \
|
||||
@@ -2030,6 +2078,15 @@ class Worker:
|
||||
task_id, result_text=stdout, error_text=error,
|
||||
actual_model=actual_model, started_at=started_at,
|
||||
finished_at=finished_at)
|
||||
# 附件暂存清理：任务终局后删 bundle 目录（重试走已存在
|
||||
# 续跑，文件已在 worktree 里，不再需要暂存；取消走 server
|
||||
# 侧删，这里删不到也不报错；删失败不影响任务结果本身）。
|
||||
try:
|
||||
_seed = self._task_seed_dir(task)
|
||||
if _seed:
|
||||
shutil.rmtree(_seed, ignore_errors=True)
|
||||
except Exception:
|
||||
pass
|
||||
with self._db_lock:
|
||||
self.usage_turns += 1
|
||||
# 成本：执行器带回上游真实用量时记 actual（input/output/
|
||||
|
||||
Reference in new issue
Block a user