feat: 命令台云端环境——选、建、管 + 任务绑定环境
环境是项目仓库的命名基准检出（项目 + 仓库快照 + 分支 + 执行器）， 不是官方 Codex 云端环境（无公开调用口，不宣称能建它）。 - DB：environments 表（creating/ready/failed/archived，部分唯一索引 保证同项目未归档名唯一、归档名可重用）+ tasks.environment_id， server/worker/db 三处迁移同步（含 #67 的 goal_mode，不丢新语义）。 - API：POST/GET /environments、GET /environments/{id}、 POST /environments/{id}/archive；任务创建选环境须本项目已 ready、 执行器一致或继承、cloud 执行器拒绝。 - Worker：run_all 每轮开头准备 creating 环境（真实 git clone + checkout 分支，幂等、无 preparing 卡死态，准备期归档 CAS 不覆盖）； 环境任务派发时 task[repo] 指到环境检出（租户 Docker 容器内路径 + envs 整目录挂载），任务仍一任务一 worktree。 - Console：环境卡（列表/新建/归档，未 ready 置灰）+ 新建任务环境下拉 + 任务列表 [环境] 标记；公网 nginx 须同批代理 /environments。 - 单测：新增 test_task_environments.py 14 个；本地实弹（真实 git clone octocat/Hello-World，不跑模型）项目→环境→任务→归档全通。
This commit is contained in:
host committed
1 parent
c86b450851
commit
3d45edc8b4
6 files changed
+1224
-11
No files matched your search
+240
-3
@@ -138,7 +138,7 @@ from cloud.models import ModelRouter, is_fallbackable, record_fallback
|
||||
from cloud.pr import PrSkipped, PrStepError
|
||||
from cloud.pr_exec import (claim_pr_job, execute_task_pr, finish_pr_job)
|
||||
from cloud.queue import budget_exceeded, ready_tasks
|
||||
from cloud.repos import RepoError, resolve_repo
|
||||
from cloud.repos import RepoError, checkout_dir, resolve_repo
|
||||
from cloud.sandbox import CloudExecutor, DockerExecutor, LocalExecutor
|
||||
from cloud.sandbox import _default_runner as _sandbox_default_runner
|
||||
from cloud.workers import heartbeat_worker, register_worker
|
||||
@@ -243,6 +243,8 @@ def _task_from_row(row):
|
||||
"approval": _col("approval"),
|
||||
"plan": _col("plan"),
|
||||
"forked_from": _col("forked_from"),
|
||||
# 环境：老行缺列时 None = 不指定环境，走项目仓库路径。
|
||||
"environment_id": _col("environment_id"),
|
||||
}
|
||||
|
||||
|
||||
@@ -470,7 +472,7 @@ class Worker:
|
||||
"SELECT id, project_id, prompt, status, idempotency_key,"
|
||||
" depends_on, priority, created_by, created_at, updated_at,"
|
||||
" last_heartbeat, executor, goal, verify_command,"
|
||||
" goal_max_rounds, goal_mode FROM tasks"
|
||||
" goal_max_rounds, goal_mode, environment_id FROM tasks"
|
||||
" WHERE status IN ('queued', 'completed', 'merged')"
|
||||
" ORDER BY created_at, id").fetchall()
|
||||
finally:
|
||||
@@ -571,7 +573,8 @@ class Worker:
|
||||
"effort": "TEXT",
|
||||
"approval": "TEXT",
|
||||
"plan": "INTEGER",
|
||||
"forked_from": "TEXT"}.items():
|
||||
"forked_from": "TEXT",
|
||||
"environment_id": "TEXT"}.items():
|
||||
if name not in cols:
|
||||
try:
|
||||
conn.execute(
|
||||
@@ -1087,12 +1090,231 @@ class Worker:
|
||||
"%s:/var/lib/codex-app-server/repos/%s:rw"
|
||||
% (dirs["repo_checkout"],
|
||||
self._tenant_safe_name(project_id)))
|
||||
# 环境检出：整目录挂载（按项目分，与 repo_checkout 同款
|
||||
# 隔离语义）。环境任务派发前 worker 已准备好检出，目录
|
||||
# 存在时才挂载；不存在说明该项目暂无环境，不硬挂载。
|
||||
# base = work 的父目录（即租户项目根目录）。
|
||||
envs_dir = os.path.join(os.path.dirname(dirs["work"]), "envs")
|
||||
if os.path.isdir(envs_dir):
|
||||
mounts.append(
|
||||
"%s:/var/lib/codex-app-server/envs:rw" % envs_dir)
|
||||
return mounts
|
||||
|
||||
def _tenant_network(self, project_id):
|
||||
"""某项目的 Docker 网络名（容器跨项目网络隔离用）。"""
|
||||
return "codex-" + self._tenant_safe_name(project_id)
|
||||
|
||||
# -- 环境 ----------------------------------------------------------
|
||||
def _env_host_dir(self, env):
|
||||
"""环境的宿主检出目录（worker 准备检出时写这里）。
|
||||
|
||||
非租户：repos_dir/envs/<env_id>（与项目检出同根，Docker
|
||||
非租户挂载 repos 根目录时它自然在容器可见路径下）。
|
||||
租户 + Docker：tenants_root/<项目>/envs/<env_id>，由
|
||||
_tenant_mounts 挂到容器 /var/lib/codex-app-server/envs。
|
||||
env_id 是 server 生成的 hex，仍按 checkout_dir 同款规则
|
||||
过滤安全字符，防路径穿越。"""
|
||||
if self.tenant_isolation and isinstance(
|
||||
self.executor, DockerExecutor):
|
||||
safe_proj = self._tenant_safe_name(env["project_id"])
|
||||
return checkout_dir(
|
||||
os.path.join(self.tenants_root, safe_proj, "envs"),
|
||||
env["id"])
|
||||
return checkout_dir(
|
||||
os.path.join(self.repos_dir, "envs"), env["id"])
|
||||
|
||||
def _env_repo_path(self, env):
|
||||
"""任务派发时传给执行器的 repo 路径：租户 Docker 用容器
|
||||
内路径，其余用宿主路径——与 _apply_project_repo 的项目
|
||||
检出路径约定完全一致，执行器不用为环境另起分支。"""
|
||||
if self.tenant_isolation and isinstance(
|
||||
self.executor, DockerExecutor):
|
||||
return "/var/lib/codex-app-server/envs/%s" % env["id"]
|
||||
return self._env_host_dir(env)
|
||||
|
||||
def _fetch_environment(self, env_id):
|
||||
"""读环境行；表不存在（老库未跑 server init_db）或查不到
|
||||
时返回 None，调用方按“环境不存在”处理，绝不炸 worker。"""
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT * FROM environments WHERE id = ?",
|
||||
(env_id,)).fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception:
|
||||
return None
|
||||
if row is None:
|
||||
return None
|
||||
|
||||
def _get(name):
|
||||
try:
|
||||
return row[name]
|
||||
except (KeyError, IndexError, TypeError):
|
||||
return None
|
||||
|
||||
try:
|
||||
return {
|
||||
"id": row["id"],
|
||||
"project_id": row["project_id"],
|
||||
"name": row["name"],
|
||||
"branch": _get("branch"),
|
||||
"executor": _get("executor"),
|
||||
"status": row["status"],
|
||||
"error_text": _get("error_text"),
|
||||
"repo_url": _get("repo_url"),
|
||||
}
|
||||
except (KeyError, IndexError, TypeError):
|
||||
return None
|
||||
|
||||
def _run_git(self, cmd):
|
||||
"""跑一条 git 命令：repos_runner 可注入（单测用假 runner，
|
||||
接受 cmd 或 (cmd, timeout) 两种签名，与 repos._run 同款
|
||||
兼容）；缺省走真实 subprocess。返回带 returncode/stdout/
|
||||
stderr 的结果对象。"""
|
||||
if self.repos_runner is not None:
|
||||
try:
|
||||
return self.repos_runner(cmd)
|
||||
except TypeError:
|
||||
return self.repos_runner(cmd, timeout=None)
|
||||
return subprocess.run(cmd, capture_output=True, text=True)
|
||||
|
||||
def _prepare_environment(self, env):
|
||||
"""准备一个环境的检出，返回错误文本或 None。
|
||||
|
||||
幂等：目录不存在时 clone，存在时先校验它是 git 仓库且
|
||||
origin 与环境 repo_url 一致（不一致绝不强制改 remote，
|
||||
直接失败），再 fetch；分支非空时 checkout 该分支（不存
|
||||
在则失败，绝不自动建分支——分支名是用户选的，不应由
|
||||
准备过程发明）。检出本身不做 reset/clean：任务 worktree
|
||||
由 host.py 从检出 HEAD 新建，检出保持干净即可。"""
|
||||
repo_url = env.get("repo_url")
|
||||
if not repo_url:
|
||||
return "environment has no repo_url"
|
||||
target = self._env_host_dir(env)
|
||||
if not os.path.exists(target):
|
||||
os.makedirs(os.path.dirname(target), exist_ok=True)
|
||||
result = self._run_git(
|
||||
["git", "clone", repo_url, target])
|
||||
if getattr(result, "returncode", 1) != 0:
|
||||
err = (getattr(result, "stderr", "") or "").strip()
|
||||
return "git clone 失败: %s" % (err or repo_url)
|
||||
else:
|
||||
git_path = os.path.join(target, ".git")
|
||||
if not (os.path.isdir(git_path)
|
||||
or os.path.isfile(git_path)):
|
||||
return "检出目录已存在但不是 git 仓库，拒绝覆盖: %s" \
|
||||
% target
|
||||
origin = self._run_git(
|
||||
["git", "-C", target, "remote", "get-url", "origin"])
|
||||
if getattr(origin, "returncode", 1) != 0:
|
||||
return "检出目录 %s 取不到 remote origin，拒绝准备" \
|
||||
% target
|
||||
if (getattr(origin, "stdout", "") or "").strip() != repo_url:
|
||||
return "检出目录 origin 与环境 repo_url 不一致，拒绝准备"
|
||||
fetched = self._run_git(
|
||||
["git", "-C", target, "fetch", "origin"])
|
||||
if getattr(fetched, "returncode", 1) != 0:
|
||||
err = (getattr(fetched, "stderr", "") or "").strip()
|
||||
return "git fetch origin 失败: %s" % (err or target)
|
||||
branch = env.get("branch")
|
||||
if branch:
|
||||
checked = self._run_git(
|
||||
["git", "-C", target, "checkout", branch])
|
||||
if getattr(checked, "returncode", 1) != 0:
|
||||
err = (getattr(checked, "stderr", "") or "").strip()
|
||||
return "git checkout %s 失败: %s" % (branch, err or target)
|
||||
return None
|
||||
|
||||
def _prepare_pending_environments(self, limit=5):
|
||||
"""准备所有 status=creating 的环境（run_all 每轮开头调）。
|
||||
|
||||
每条环境独立 try：一条失败只把自己标 failed，不挡其它
|
||||
环境和任务。状态写回用 CAS（WHERE status='creating'）：
|
||||
准备期间用户归档了环境时 CAS 改不到行，准备结果就此
|
||||
作废、状态保持 archived，绝不把归档覆盖回 ready。worker
|
||||
中途死了、环境仍是 creating 时下一轮会重新准备（准备
|
||||
本身幂等），没有“preparing 卡死”这种中间态。返回
|
||||
[(env_id, status, error_or_None), ...] 供日志与单测。"""
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM environments WHERE status = 'creating'"
|
||||
" ORDER BY created_at, id LIMIT ?",
|
||||
(limit,)).fetchall()
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
log_event(self.logger, "worker_env_list_failed",
|
||||
message="worker env list failed: %s" % exc)
|
||||
return []
|
||||
results = []
|
||||
for row in rows:
|
||||
try:
|
||||
env = {
|
||||
"id": row["id"],
|
||||
"project_id": row["project_id"],
|
||||
"name": row["name"],
|
||||
"branch": row["branch"] if "branch" in row.keys()
|
||||
else None,
|
||||
"executor": row["executor"] if "executor" in row.keys()
|
||||
else None,
|
||||
"repo_url": row["repo_url"] if "repo_url" in row.keys()
|
||||
else None,
|
||||
}
|
||||
except Exception:
|
||||
continue
|
||||
try:
|
||||
error = self._prepare_environment(env)
|
||||
except Exception as exc:
|
||||
error = str(exc)
|
||||
status = "failed" if error else "ready"
|
||||
try:
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
cur = conn.execute(
|
||||
"UPDATE environments SET status = ?,"
|
||||
" error_text = ?, updated_at = ?"
|
||||
" WHERE id = ? AND status = 'creating'",
|
||||
(status, error, time.time(), env["id"]))
|
||||
conn.commit()
|
||||
if cur.rowcount == 0:
|
||||
# 准备期间被归档：状态以归档为准，不覆盖。
|
||||
status = "archived"
|
||||
finally:
|
||||
conn.close()
|
||||
except Exception as exc:
|
||||
log_event(self.logger, "worker_env_update_failed",
|
||||
message="worker env update failed: %s" % exc)
|
||||
continue
|
||||
results.append((env["id"], status, error))
|
||||
return results
|
||||
|
||||
def _apply_environment_repo(self, task):
|
||||
"""任务带 environment_id 时的派发：环境必须已 ready 且
|
||||
属于本任务项目，执行器冲突时拒绝；通过后 task["repo"]
|
||||
指到环境检出（租户 Docker 为容器内路径），后续执行器
|
||||
与无环境任务完全同款——host.py 从检出 HEAD 为任务建
|
||||
worktree，任务隔离、合并、提 PR 都不变。环境未 ready
|
||||
时返回错误文本，调用方把任务标 failed（仓库解析失败
|
||||
绝不重试，见 _apply_project_repo 的约定）。"""
|
||||
env_id = task.get("environment_id")
|
||||
env = self._fetch_environment(env_id)
|
||||
if env is None:
|
||||
return "environment not found: %s" % env_id
|
||||
if env["project_id"] != task.get("project_id"):
|
||||
return "environment belongs to another project"
|
||||
if env["status"] != "ready":
|
||||
return "environment not ready: %s" % env["status"]
|
||||
task_executor = task.get("executor")
|
||||
if env["executor"] and task_executor \
|
||||
and env["executor"] != task_executor:
|
||||
return "executor does not match environment executor"
|
||||
task["repo"] = self._env_repo_path(env)
|
||||
return None
|
||||
|
||||
# -- 项目仓库 ------------------------------------------------------
|
||||
def _project_repo_url(self, project_id):
|
||||
"""查任务所属项目的 repo_url；查不到项目、列缺失、表缺失时
|
||||
@@ -1132,7 +1354,13 @@ class Worker:
|
||||
resolve 失败返回错误文本，调用方据此把任务置 failed
|
||||
（仓库解析失败绝不标 completed，也不在此重试，避免对
|
||||
不可用的仓库反复 clone 烧时间）。
|
||||
|
||||
任务带 environment_id 时不走项目仓库路径，直接走
|
||||
_apply_environment_repo（环境检出已由 worker 准备好，
|
||||
这里只校验 ready/归属/执行器并指过去，不再 clone）。
|
||||
"""
|
||||
if task.get("environment_id"):
|
||||
return self._apply_environment_repo(task)
|
||||
use_tenant = self.tenant_isolation and \
|
||||
isinstance(self.executor, DockerExecutor)
|
||||
repo_url = self._project_repo_url(task.get("project_id"))
|
||||
@@ -1945,6 +2173,15 @@ class Worker:
|
||||
停机信号（_loop_stop/stop_event）只阻止新认领，不会
|
||||
丢弃已提交的任务；排空等待由 run_loop 的 finally 负责。
|
||||
"""
|
||||
# 环境准备先跑：每轮先把 creating 的环境准备好（clone +
|
||||
# checkout 分支），再认领普通任务；一条环境失败只标它
|
||||
# 自己 failed，不挡任务——准备函数内部已隔离异常，这里
|
||||
# 再包一层，防 DB 结构异常（如老库缺表）带走整轮任务。
|
||||
try:
|
||||
self._prepare_pending_environments()
|
||||
except Exception as exc:
|
||||
log_event(self.logger, "worker_env_prepare_failed",
|
||||
message="worker env prepare failed: %s" % exc)
|
||||
# pr 任务先跑（HA 阶段 1）：每轮先把本主机的 queued pr 任务
|
||||
# 跑完再认领普通任务；pr_jobs_only 时只跑 pr 任务。
|
||||
self._run_pr_jobs()
|
||||
|
||||
Reference in new issue
Block a user