diff --git a/cloud/skill_seed.py b/cloud/skill_seed.py new file mode 100644 index 0000000..e0b87c6 --- /dev/null +++ b/cloud/skill_seed.py @@ -0,0 +1,117 @@ +"""Seed ChatGPT-Codex-alignment skills into a tenant codex-home. + +Mirrors the websearch MCP seeding pattern in cloud/websearch.py, +but for skills, not MCP config: + +- Source is cloud/skills//SKILL.md (the batch-1 skill pack). +- Destination is /.agents/skills//SKILL.md. + Why this path: Codex documents personal skills at + $HOME/.agents/skills and project skills at .agents/skills + (scanned from the working directory up to the repo root). + A Docker tenant task must not pollute its task repo/worktree + (seeding into the worktree would show up in the task diff), + so the tenant codex-home -- the per-project persistent home + that is mounted into the container at + /var/lib/codex-app-server/codex-home -- is the personal-level + analogue. This is the same channel websearch MCP seeding uses. +- Idempotent and non-destructive by design: + - a missing destination file is copied; + - an identical destination file is left alone; + - a differing destination file is NEVER overwritten (that is + a tenant-modified skill) -- it is reported in + ``skills_skipped_differ`` and left alone; + - a skill directory without SKILL.md is skipped, never invented. +- No config.toml change, no key, no network: skills are plain + instruction files, and the helper never writes secrets. + +Honest boundary (see docs/chatgpt-codex-alignment.md): the +destination path follows the documented personal-skills pattern +mapped onto the tenant home. Whether a given Codex build picks +these skills up in a Docker tenant task still needs a live +verification task (skills/list + an explicit $live-info call), +the same kind of e2e probe websearch MCP got after its seeding +PR. Until that probe passes, do not claim the skills are live +in production tasks -- this helper only guarantees the files +land in the tenant home, idempotently, without clobbering +tenant edits. +""" + +import os +import shutil +from pathlib import Path + +# cloud/skills//SKILL.md, next to this module. +SKILLS_DIR = Path(__file__).resolve().parent / "skills" +# Personal-level skills inside a codex-home. +DEST_SUBDIR = Path(".agents") / "skills" + + +def skill_source_dir(skills_dir=None): + """Skill pack directory: explicit override or cloud/skills/.""" + if skills_dir is not None: + return Path(skills_dir) + return SKILLS_DIR + + +def list_skills(skills_dir=None): + """[(name, SKILL.md path)] for every skill in the pack.""" + src = skill_source_dir(skills_dir) + out = [] + if not src.is_dir(): + return out + for entry in sorted(src.iterdir()): + if not entry.is_dir(): + continue + skill_file = entry / "SKILL.md" + if skill_file.is_file(): + out.append((entry.name, skill_file)) + return out + + +def ensure_codex_home_skills(codex_home_dir, skills_dir=None): + """Copy the skill pack into a tenant codex-home. + + Returns {"dest_dir", "skills_copied", "skills_skipped_differ", + "skills_skipped_missing"}: copied = names newly written, + skipped_differ = names whose destination SKILL.md differs + from the pack (tenant-modified, left alone), skipped_missing + = skill directories in the pack without a SKILL.md file + (should not happen; the batch-1 test guards it). + A missing source directory yields all-empty lists, never + an exception: skills are optional, a missing pack must not + block a task the way a missing websearch config would. + """ + home = Path(codex_home_dir) + dest_root = home / DEST_SUBDIR + dest_root.mkdir(parents=True, exist_ok=True) + copied = [] + skipped_differ = [] + skipped_missing = [] + src = skill_source_dir(skills_dir) + if not src.is_dir(): + return {"dest_dir": str(dest_root), "skills_copied": copied, + "skills_skipped_differ": skipped_differ, + "skills_skipped_missing": skipped_missing} + # Directories in the pack, including ones without SKILL.md, + # so a broken pack is reported instead of silently dropped. + for entry in sorted(src.iterdir()): + if not entry.is_dir(): + continue + skill_file = entry / "SKILL.md" + if not skill_file.is_file(): + skipped_missing.append(entry.name) + continue + dest_dir = dest_root / entry.name + dest_file = dest_dir / "SKILL.md" + src_bytes = skill_file.read_bytes() + if dest_file.is_file(): + if dest_file.read_bytes() == src_bytes: + continue + skipped_differ.append(entry.name) + continue + dest_dir.mkdir(parents=True, exist_ok=True) + shutil.copyfile(skill_file, dest_file) + copied.append(entry.name) + return {"dest_dir": str(dest_root), "skills_copied": copied, + "skills_skipped_differ": skipped_differ, + "skills_skipped_missing": skipped_missing} diff --git a/cloud/tests/test_skill_seed_worker.py b/cloud/tests/test_skill_seed_worker.py new file mode 100644 index 0000000..a5025a2 --- /dev/null +++ b/cloud/tests/test_skill_seed_worker.py @@ -0,0 +1,132 @@ +"""Worker wiring for ChatGPT-Codex-alignment skills: tenant seeding. + +Mirrors test_websearch_worker.py: no Docker, no network, no real +model. _tenant_dirs runs against temp dirs; the skill pack used +here is the real cloud/skills/ pack, so a broken pack fails here +before it ever reaches a tenant. +""" + +import os +import sys +import tempfile +import unittest +from pathlib import Path + +try: + from cloud.server import init_db + from cloud.worker import Worker + from cloud.skill_seed import ensure_codex_home_skills, list_skills +except ImportError: + sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + from cloud.server import init_db + from cloud.worker import Worker + from cloud.skill_seed import ensure_codex_home_skills, list_skills + + +class SkillSeedHelperTestCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + + def tearDown(self): + self.tmp.cleanup() + + def _src(self): + src = os.path.join(self.tmp.name, "pack") + for name in ("live-info", "document-tools"): + d = os.path.join(src, name) + os.makedirs(d, exist_ok=True) + open(os.path.join(d, "SKILL.md"), "w").write( + "---\nname: %s\ndescription: test\n---\nbody\n" % name) + return src + + def test_copy_idempotent_and_missing_source(self): + home = os.path.join(self.tmp.name, "home") + out = ensure_codex_home_skills(home, self._src()) + self.assertEqual(sorted(out["skills_copied"]), + ["document-tools", "live-info"]) + self.assertEqual(out["skills_skipped_differ"], []) + out2 = ensure_codex_home_skills(home, self._src()) + self.assertEqual(out2["skills_copied"], []) + out3 = ensure_codex_home_skills( + os.path.join(self.tmp.name, "home2"), + os.path.join(self.tmp.name, "no-such-pack")) + self.assertEqual(out3["skills_copied"], []) + + def test_tenant_modified_skill_is_never_overwritten(self): + home = os.path.join(self.tmp.name, "home") + src = self._src() + ensure_codex_home_skills(home, src) + dest = os.path.join(home, ".agents", "skills", + "live-info", "SKILL.md") + open(dest, "w").write("tenant edited version\n") + out = ensure_codex_home_skills(home, src) + self.assertEqual(out["skills_skipped_differ"], ["live-info"]) + self.assertEqual(open(dest).read(), "tenant edited version\n") + + def test_real_pack_lists_three_skills(self): + names = [n for n, _p in list_skills()] + self.assertEqual(names, + ["document-tools", "image-workflow", + "live-info"]) + + +class WorkerSkillSeedTestCase(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.db_path = os.path.join(self.tmp.name, "control.db") + init_db(self.db_path) + self.template_home = os.path.join(self.tmp.name, "tpl-home") + os.makedirs(self.template_home) + open(os.path.join(self.template_home, "config.toml"), + "w").write('[model_provider.xcmapi]\n' + 'base_url="https://xcmapi.org/v1"\n') + self.template_repo = os.path.join(self.tmp.name, "tpl-repo") + os.makedirs(os.path.join(self.template_repo, ".git")) + open(os.path.join(self.template_repo, "sample.py"), + "w").write("def add(a, b):\n return a + b\n") + + def tearDown(self): + self.tmp.cleanup() + + def _worker(self, **kwargs): + kwargs.setdefault("env", {"XCM_API_KEY": "test-placeholder"}) + kwargs.setdefault("tenants_root", + os.path.join(self.tmp.name, "tenants")) + kwargs.setdefault("tenant_template_home", self.template_home) + kwargs.setdefault("tenant_template_repo", self.template_repo) + kwargs.setdefault("tenant_isolation", True) + return Worker(self.db_path, "/fake/host.py", **kwargs) + + def _skill_path(self, dirs, name): + return os.path.join(dirs["codex_home"], ".agents", "skills", + name, "SKILL.md") + + def test_flag_off_is_old_behaviour(self): + worker = self._worker() + self.assertFalse(worker.seed_skills) + dirs = worker._tenant_dirs("projA") + self.assertFalse(os.path.exists( + os.path.join(dirs["codex_home"], ".agents", "skills"))) + + def test_flag_on_seeds_real_pack(self): + worker = self._worker(seed_skills=True) + dirs = worker._tenant_dirs("projA") + for name in ("live-info", "document-tools", "image-workflow"): + path = self._skill_path(dirs, name) + self.assertTrue(os.path.exists(path), path) + self.assertIn("name: %s" % name, open(path).read()) + + def test_old_tenant_merges_and_user_edit_survives(self): + worker = self._worker() + dirs = worker._tenant_dirs("projA") + worker2 = self._worker(seed_skills=True) + worker2._tenant_dirs("projA") + path = self._skill_path(dirs, "live-info") + self.assertTrue(os.path.exists(path)) + open(path, "w").write("tenant edited version\n") + worker2._tenant_dirs("projA") + self.assertEqual(open(path).read(), "tenant edited version\n") + + +if __name__ == "__main__": + unittest.main() diff --git a/cloud/worker.py b/cloud/worker.py index 797535d..49a1bd7 100644 --- a/cloud/worker.py +++ b/cloud/worker.py @@ -166,6 +166,7 @@ from cloud.peer_messages import peer_message_prefix as _peer_prefix from cloud.peer_messages import row_get as _row_get from cloud.sub2api import ensure_user_gateway_key as _ensure_sub2api_key from cloud.websearch import ensure_codex_home_websearch as _ensure_websearch_mcp +from cloud.skill_seed import ensure_codex_home_skills as _ensure_skills from cloud.sub2api import gateway_name as _gateway_name from cloud.sub2api import gateway_origin as _gateway_origin from cloud.sub2api import list_gateway_models as _list_gateway_models @@ -412,7 +413,7 @@ class Worker: tenant_isolation=False, tenants_root=None, tenant_template_home=None, tenant_template_repo=None, egress_proxy_port=0, claim_executor_only=False, - websearch_mcp=False, + websearch_mcp=False, seed_skills=False, host_name=None, pr_jobs_only=False, upload_dir=None, digest_interval_seconds=3600, @@ -565,6 +566,14 @@ class Worker: # [mcp_servers.websearch](容器内路径,见 _tenant_dirs)。 # 老租户下一次任务自动合并,已有 section 不重写。 self.websearch_mcp = bool(websearch_mcp) + # 技能播种:默认关(老行为:租户 codex-home 只有模板 + # config.toml + 可选的 websearch MCP)。开启后 + # _tenant_dirs 在 websearch 播种之后调用 skill_seed + # 的播种助手,把 cloud/skills/ 的 SKILL.md 拷进租户 + # codex-home/.agents/skills/(个人级技能路径,见 + # skill_seed.py 的路径说明)。老租户下一次任务自动 + # 合并;租户改过的技能文件绝不覆盖,只上报 differs。 + self.seed_skills = bool(seed_skills) # HA 阶段 1:本 worker 所在主机名。认领任务时写进 # tasks.worktree_host,/pr 据此判断 worktree 在哪台主机。 # 容器内 worker 的 socket.gethostname() 是容器名、不是宿主名, @@ -2403,6 +2412,24 @@ class Worker: "python3", "/var/lib/codex-app-server/codex-home/" "websearch_mcp.py"]) + if self.seed_skills: + # 技能播种失败绝不阻塞任务:技能是可选的流程 + # 说明,不是 websearch 那种“已宣传就必须有”的 + # 工具;播种助手本身对缺包、已存在、租户改动 + # 都是静默处理,只在文件系统本身出错时才抛, + # 那种情况下任务仍应派发(技能缺失只影响流程, + # 不影响 agent 本身跑起来)。 + try: + _ensure_skills(dirs["codex_home"]) + except Exception as exc: + # 只记日志不阻塞:技能缺失只影响流程说明, + # 不应把一次可选播种失败变成任务派发失败。 + # 但绝不静默吞掉——日志里要留下来供排查。 + try: + self.logger.warning( + "skill seeding failed: %s", exc) + except Exception: + pass if not os.path.exists(os.path.join(dirs["test_repo"], ".git")) \ and not os.path.exists(dirs["test_repo"] + "/sample.py"): # 目标为空目录(刚 mkdir 出来的)才播种;已有内容 @@ -5672,6 +5699,12 @@ def main(argv=None): " [mcp_servers.websearch] 并拷入" " websearch.py / websearch_mcp.py;" "默认关,老租户不受影响") + parser.add_argument("--seed-skills", action="store_true", + help="租户 codex-home 自动播种对齐技能包:" + "_tenant_dirs 把 cloud/skills/ 的 SKILL.md" + " 拷进租户 codex-home/.agents/skills/;" + "默认关,老租户不受影响;租户改过的" + "技能文件绝不覆盖") parser.add_argument("--task-timeout", type=float, default=1800, help="每任务截止时间(秒,默认 1800);0 表示不限时" "(与 host --turn-timeout 0 同约定,worker 级" @@ -5733,7 +5766,8 @@ def main(argv=None): tenant_template_repo=args.tenant_template_repo, egress_proxy_port=args.egress_proxy_port, claim_executor_only=args.claim_executor_only, - websearch_mcp=args.websearch_mcp) + websearch_mcp=args.websearch_mcp, + seed_skills=args.seed_skills) # CLI 启动清理兜底(常驻与单次都做):清掉上一轮残留容器。 worker.cleanup_stale_containers() # CLI 起 worker 前先建表/迁移:全新库直接起循环不应因缺表而炸。 diff --git a/docs/chatgpt-codex-alignment.md b/docs/chatgpt-codex-alignment.md index fc6a8fc..7f8c332 100644 --- a/docs/chatgpt-codex-alignment.md +++ b/docs/chatgpt-codex-alignment.md @@ -42,6 +42,33 @@ 播种进租户 codex-home(仿 websearch MCP 的播种方式) 是下一批,需要单独评审 worker 改动。 +## 第二批:技能播种进租户(本分支新增) + +- `cloud/skill_seed.py`:仿 websearch MCP 播种,把 + `cloud/skills/` 的 SKILL.md 拷进租户 + `codex-home/.agents/skills/<名>/SKILL.md`。路径依据: + Codex 个人技能在 `$HOME/.agents/skills`,项目技能在 + `.agents/skills`(从工作目录向上扫到仓库根);租户任务 + 绝不能往任务 worktree 里播种(会污染任务 diff),所以 + 用租户 codex-home 这个个人级类比位置——这也是 + websearch MCP 播种用的通道。 +- `cloud/worker.py`:新增 `--seed-skills`(默认关,仿 + `--websearch-mcp`),只在 `_tenant_dirs` 里调用播种; + 老租户下一次任务自动合并;租户改过的技能文件绝不 + 覆盖(只上报 differs);播种失败只记日志、不阻塞任务 + (技能是可选流程说明,不是已宣传就必须有的工具)。 +- `cloud/tests/test_skill_seed_worker.py`:播种助手 + (拷、幂等、缺包、租户改动不覆盖)+ worker 开关 + (关 = 老行为、开 = 播真实包、老租户合并 + 改动存活)。 +- 诚实边界:播种只保证文件落进租户 home、幂等、不覆盖 + 租户改动。某个 Codex 版本在 Docker 租户任务里是否真 + 从这个位置发现技能,仍需一次真实任务实测 + (skills/list + 显式 `$live-info` 调用,仿 websearch + e2e 探针的做法),实测通过前不要宣称“生产任务已能用 + 这些技能”。本批不改生产配置、不部署,autoscaler 的 + worker 参数里加 `--seed-skills` 是部署时另行应用的 + 主机步骤(unit 文件不在 deploy tar 里,仿 websearch)。 + ## 不照搬(诚实边界) - CUA / 桌面操作:需要浏览器自动化 + 桌面操作基础设施,