align: 第二批:把技能包播种进租户 codex-home
仿 websearch MCP 播种,默认关: - cloud/skill_seed.py:cloud/skills/ 的 SKILL.md 拷进租户 codex-home/.agents/skills/(个人级技能路径,见模块 文档的路径依据与诚实边界);幂等,租户改过的文件 绝不覆盖(上报 differs),缺包不阻塞任务。 - cloud/worker.py:新增 --seed-skills,_tenant_dirs 里 调用播种;老租户下一次任务自动合并;播种失败只记 日志、不阻塞任务(技能是可选流程说明)。 - cloud/tests/test_skill_seed_worker.py:播种助手 + worker 开关,仿 test_websearch_worker.py 的结构。 - docs/chatgpt-codex-alignment.md:补第二批说明与实测 待办(真实 Docker 任务 skills/list + $live-info 调用, 通过前不宣称生产任务已能用这些技能)。 验证:skill_seed + chatgpt skills + websearch worker 共 14 项全过。本批不改生产配置、不部署,autoscaler 加 --seed-skills 是部署时另行的主机步骤。
This commit is contained in:
1 parent
8a26d1973c
commit
cf9d17f035
4 files changed
+312
-2
No files matched your search
@@ -0,0 +1,117 @@
|
||||
"""Seed ChatGPT-Codex-alignment skills into a tenant codex-home.
|
||||
|
||||
Mirrors the websearch MCP seeding pattern in cloud/websearch.py,
|
||||
but for skills, not MCP config:
|
||||
|
||||
- Source is cloud/skills/<name>/SKILL.md (the batch-1 skill pack).
|
||||
- Destination is <codex-home>/.agents/skills/<name>/SKILL.md.
|
||||
Why this path: Codex documents personal skills at
|
||||
$HOME/.agents/skills and project skills at .agents/skills
|
||||
(scanned from the working directory up to the repo root).
|
||||
A Docker tenant task must not pollute its task repo/worktree
|
||||
(seeding into the worktree would show up in the task diff),
|
||||
so the tenant codex-home -- the per-project persistent home
|
||||
that is mounted into the container at
|
||||
/var/lib/codex-app-server/codex-home -- is the personal-level
|
||||
analogue. This is the same channel websearch MCP seeding uses.
|
||||
- Idempotent and non-destructive by design:
|
||||
- a missing destination file is copied;
|
||||
- an identical destination file is left alone;
|
||||
- a differing destination file is NEVER overwritten (that is
|
||||
a tenant-modified skill) -- it is reported in
|
||||
``skills_skipped_differ`` and left alone;
|
||||
- a skill directory without SKILL.md is skipped, never invented.
|
||||
- No config.toml change, no key, no network: skills are plain
|
||||
instruction files, and the helper never writes secrets.
|
||||
|
||||
Honest boundary (see docs/chatgpt-codex-alignment.md): the
|
||||
destination path follows the documented personal-skills pattern
|
||||
mapped onto the tenant home. Whether a given Codex build picks
|
||||
these skills up in a Docker tenant task still needs a live
|
||||
verification task (skills/list + an explicit $live-info call),
|
||||
the same kind of e2e probe websearch MCP got after its seeding
|
||||
PR. Until that probe passes, do not claim the skills are live
|
||||
in production tasks -- this helper only guarantees the files
|
||||
land in the tenant home, idempotently, without clobbering
|
||||
tenant edits.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
# cloud/skills/<name>/SKILL.md, next to this module.
|
||||
SKILLS_DIR = Path(__file__).resolve().parent / "skills"
|
||||
# Personal-level skills inside a codex-home.
|
||||
DEST_SUBDIR = Path(".agents") / "skills"
|
||||
|
||||
|
||||
def skill_source_dir(skills_dir=None):
|
||||
"""Skill pack directory: explicit override or cloud/skills/."""
|
||||
if skills_dir is not None:
|
||||
return Path(skills_dir)
|
||||
return SKILLS_DIR
|
||||
|
||||
|
||||
def list_skills(skills_dir=None):
|
||||
"""[(name, SKILL.md path)] for every skill in the pack."""
|
||||
src = skill_source_dir(skills_dir)
|
||||
out = []
|
||||
if not src.is_dir():
|
||||
return out
|
||||
for entry in sorted(src.iterdir()):
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
skill_file = entry / "SKILL.md"
|
||||
if skill_file.is_file():
|
||||
out.append((entry.name, skill_file))
|
||||
return out
|
||||
|
||||
|
||||
def ensure_codex_home_skills(codex_home_dir, skills_dir=None):
|
||||
"""Copy the skill pack into a tenant codex-home.
|
||||
|
||||
Returns {"dest_dir", "skills_copied", "skills_skipped_differ",
|
||||
"skills_skipped_missing"}: copied = names newly written,
|
||||
skipped_differ = names whose destination SKILL.md differs
|
||||
from the pack (tenant-modified, left alone), skipped_missing
|
||||
= skill directories in the pack without a SKILL.md file
|
||||
(should not happen; the batch-1 test guards it).
|
||||
A missing source directory yields all-empty lists, never
|
||||
an exception: skills are optional, a missing pack must not
|
||||
block a task the way a missing websearch config would.
|
||||
"""
|
||||
home = Path(codex_home_dir)
|
||||
dest_root = home / DEST_SUBDIR
|
||||
dest_root.mkdir(parents=True, exist_ok=True)
|
||||
copied = []
|
||||
skipped_differ = []
|
||||
skipped_missing = []
|
||||
src = skill_source_dir(skills_dir)
|
||||
if not src.is_dir():
|
||||
return {"dest_dir": str(dest_root), "skills_copied": copied,
|
||||
"skills_skipped_differ": skipped_differ,
|
||||
"skills_skipped_missing": skipped_missing}
|
||||
# Directories in the pack, including ones without SKILL.md,
|
||||
# so a broken pack is reported instead of silently dropped.
|
||||
for entry in sorted(src.iterdir()):
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
skill_file = entry / "SKILL.md"
|
||||
if not skill_file.is_file():
|
||||
skipped_missing.append(entry.name)
|
||||
continue
|
||||
dest_dir = dest_root / entry.name
|
||||
dest_file = dest_dir / "SKILL.md"
|
||||
src_bytes = skill_file.read_bytes()
|
||||
if dest_file.is_file():
|
||||
if dest_file.read_bytes() == src_bytes:
|
||||
continue
|
||||
skipped_differ.append(entry.name)
|
||||
continue
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(skill_file, dest_file)
|
||||
copied.append(entry.name)
|
||||
return {"dest_dir": str(dest_root), "skills_copied": copied,
|
||||
"skills_skipped_differ": skipped_differ,
|
||||
"skills_skipped_missing": skipped_missing}
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Worker wiring for ChatGPT-Codex-alignment skills: tenant seeding.
|
||||
|
||||
Mirrors test_websearch_worker.py: no Docker, no network, no real
|
||||
model. _tenant_dirs runs against temp dirs; the skill pack used
|
||||
here is the real cloud/skills/ pack, so a broken pack fails here
|
||||
before it ever reaches a tenant.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from cloud.server import init_db
|
||||
from cloud.worker import Worker
|
||||
from cloud.skill_seed import ensure_codex_home_skills, list_skills
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.server import init_db
|
||||
from cloud.worker import Worker
|
||||
from cloud.skill_seed import ensure_codex_home_skills, list_skills
|
||||
|
||||
|
||||
class SkillSeedHelperTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _src(self):
|
||||
src = os.path.join(self.tmp.name, "pack")
|
||||
for name in ("live-info", "document-tools"):
|
||||
d = os.path.join(src, name)
|
||||
os.makedirs(d, exist_ok=True)
|
||||
open(os.path.join(d, "SKILL.md"), "w").write(
|
||||
"---\nname: %s\ndescription: test\n---\nbody\n" % name)
|
||||
return src
|
||||
|
||||
def test_copy_idempotent_and_missing_source(self):
|
||||
home = os.path.join(self.tmp.name, "home")
|
||||
out = ensure_codex_home_skills(home, self._src())
|
||||
self.assertEqual(sorted(out["skills_copied"]),
|
||||
["document-tools", "live-info"])
|
||||
self.assertEqual(out["skills_skipped_differ"], [])
|
||||
out2 = ensure_codex_home_skills(home, self._src())
|
||||
self.assertEqual(out2["skills_copied"], [])
|
||||
out3 = ensure_codex_home_skills(
|
||||
os.path.join(self.tmp.name, "home2"),
|
||||
os.path.join(self.tmp.name, "no-such-pack"))
|
||||
self.assertEqual(out3["skills_copied"], [])
|
||||
|
||||
def test_tenant_modified_skill_is_never_overwritten(self):
|
||||
home = os.path.join(self.tmp.name, "home")
|
||||
src = self._src()
|
||||
ensure_codex_home_skills(home, src)
|
||||
dest = os.path.join(home, ".agents", "skills",
|
||||
"live-info", "SKILL.md")
|
||||
open(dest, "w").write("tenant edited version\n")
|
||||
out = ensure_codex_home_skills(home, src)
|
||||
self.assertEqual(out["skills_skipped_differ"], ["live-info"])
|
||||
self.assertEqual(open(dest).read(), "tenant edited version\n")
|
||||
|
||||
def test_real_pack_lists_three_skills(self):
|
||||
names = [n for n, _p in list_skills()]
|
||||
self.assertEqual(names,
|
||||
["document-tools", "image-workflow",
|
||||
"live-info"])
|
||||
|
||||
|
||||
class WorkerSkillSeedTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
init_db(self.db_path)
|
||||
self.template_home = os.path.join(self.tmp.name, "tpl-home")
|
||||
os.makedirs(self.template_home)
|
||||
open(os.path.join(self.template_home, "config.toml"),
|
||||
"w").write('[model_provider.xcmapi]\n'
|
||||
'base_url="https://xcmapi.org/v1"\n')
|
||||
self.template_repo = os.path.join(self.tmp.name, "tpl-repo")
|
||||
os.makedirs(os.path.join(self.template_repo, ".git"))
|
||||
open(os.path.join(self.template_repo, "sample.py"),
|
||||
"w").write("def add(a, b):\n return a + b\n")
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _worker(self, **kwargs):
|
||||
kwargs.setdefault("env", {"XCM_API_KEY": "test-placeholder"})
|
||||
kwargs.setdefault("tenants_root",
|
||||
os.path.join(self.tmp.name, "tenants"))
|
||||
kwargs.setdefault("tenant_template_home", self.template_home)
|
||||
kwargs.setdefault("tenant_template_repo", self.template_repo)
|
||||
kwargs.setdefault("tenant_isolation", True)
|
||||
return Worker(self.db_path, "/fake/host.py", **kwargs)
|
||||
|
||||
def _skill_path(self, dirs, name):
|
||||
return os.path.join(dirs["codex_home"], ".agents", "skills",
|
||||
name, "SKILL.md")
|
||||
|
||||
def test_flag_off_is_old_behaviour(self):
|
||||
worker = self._worker()
|
||||
self.assertFalse(worker.seed_skills)
|
||||
dirs = worker._tenant_dirs("projA")
|
||||
self.assertFalse(os.path.exists(
|
||||
os.path.join(dirs["codex_home"], ".agents", "skills")))
|
||||
|
||||
def test_flag_on_seeds_real_pack(self):
|
||||
worker = self._worker(seed_skills=True)
|
||||
dirs = worker._tenant_dirs("projA")
|
||||
for name in ("live-info", "document-tools", "image-workflow"):
|
||||
path = self._skill_path(dirs, name)
|
||||
self.assertTrue(os.path.exists(path), path)
|
||||
self.assertIn("name: %s" % name, open(path).read())
|
||||
|
||||
def test_old_tenant_merges_and_user_edit_survives(self):
|
||||
worker = self._worker()
|
||||
dirs = worker._tenant_dirs("projA")
|
||||
worker2 = self._worker(seed_skills=True)
|
||||
worker2._tenant_dirs("projA")
|
||||
path = self._skill_path(dirs, "live-info")
|
||||
self.assertTrue(os.path.exists(path))
|
||||
open(path, "w").write("tenant edited version\n")
|
||||
worker2._tenant_dirs("projA")
|
||||
self.assertEqual(open(path).read(), "tenant edited version\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+36
-2
@@ -166,6 +166,7 @@ from cloud.peer_messages import peer_message_prefix as _peer_prefix
|
||||
from cloud.peer_messages import row_get as _row_get
|
||||
from cloud.sub2api import ensure_user_gateway_key as _ensure_sub2api_key
|
||||
from cloud.websearch import ensure_codex_home_websearch as _ensure_websearch_mcp
|
||||
from cloud.skill_seed import ensure_codex_home_skills as _ensure_skills
|
||||
from cloud.sub2api import gateway_name as _gateway_name
|
||||
from cloud.sub2api import gateway_origin as _gateway_origin
|
||||
from cloud.sub2api import list_gateway_models as _list_gateway_models
|
||||
@@ -412,7 +413,7 @@ class Worker:
|
||||
tenant_isolation=False, tenants_root=None,
|
||||
tenant_template_home=None, tenant_template_repo=None,
|
||||
egress_proxy_port=0, claim_executor_only=False,
|
||||
websearch_mcp=False,
|
||||
websearch_mcp=False, seed_skills=False,
|
||||
host_name=None, pr_jobs_only=False,
|
||||
upload_dir=None,
|
||||
digest_interval_seconds=3600,
|
||||
@@ -565,6 +566,14 @@ class Worker:
|
||||
# [mcp_servers.websearch](容器内路径,见 _tenant_dirs)。
|
||||
# 老租户下一次任务自动合并,已有 section 不重写。
|
||||
self.websearch_mcp = bool(websearch_mcp)
|
||||
# 技能播种:默认关(老行为:租户 codex-home 只有模板
|
||||
# config.toml + 可选的 websearch MCP)。开启后
|
||||
# _tenant_dirs 在 websearch 播种之后调用 skill_seed
|
||||
# 的播种助手,把 cloud/skills/ 的 SKILL.md 拷进租户
|
||||
# codex-home/.agents/skills/(个人级技能路径,见
|
||||
# skill_seed.py 的路径说明)。老租户下一次任务自动
|
||||
# 合并;租户改过的技能文件绝不覆盖,只上报 differs。
|
||||
self.seed_skills = bool(seed_skills)
|
||||
# HA 阶段 1:本 worker 所在主机名。认领任务时写进
|
||||
# tasks.worktree_host,/pr 据此判断 worktree 在哪台主机。
|
||||
# 容器内 worker 的 socket.gethostname() 是容器名、不是宿主名,
|
||||
@@ -2403,6 +2412,24 @@ class Worker:
|
||||
"python3",
|
||||
"/var/lib/codex-app-server/codex-home/"
|
||||
"websearch_mcp.py"])
|
||||
if self.seed_skills:
|
||||
# 技能播种失败绝不阻塞任务:技能是可选的流程
|
||||
# 说明,不是 websearch 那种“已宣传就必须有”的
|
||||
# 工具;播种助手本身对缺包、已存在、租户改动
|
||||
# 都是静默处理,只在文件系统本身出错时才抛,
|
||||
# 那种情况下任务仍应派发(技能缺失只影响流程,
|
||||
# 不影响 agent 本身跑起来)。
|
||||
try:
|
||||
_ensure_skills(dirs["codex_home"])
|
||||
except Exception as exc:
|
||||
# 只记日志不阻塞:技能缺失只影响流程说明,
|
||||
# 不应把一次可选播种失败变成任务派发失败。
|
||||
# 但绝不静默吞掉——日志里要留下来供排查。
|
||||
try:
|
||||
self.logger.warning(
|
||||
"skill seeding failed: %s", exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not os.path.exists(os.path.join(dirs["test_repo"], ".git")) \
|
||||
and not os.path.exists(dirs["test_repo"] + "/sample.py"):
|
||||
# 目标为空目录(刚 mkdir 出来的)才播种;已有内容
|
||||
@@ -5672,6 +5699,12 @@ def main(argv=None):
|
||||
" [mcp_servers.websearch] 并拷入"
|
||||
" websearch.py / websearch_mcp.py;"
|
||||
"默认关,老租户不受影响")
|
||||
parser.add_argument("--seed-skills", action="store_true",
|
||||
help="租户 codex-home 自动播种对齐技能包:"
|
||||
"_tenant_dirs 把 cloud/skills/ 的 SKILL.md"
|
||||
" 拷进租户 codex-home/.agents/skills/;"
|
||||
"默认关,老租户不受影响;租户改过的"
|
||||
"技能文件绝不覆盖")
|
||||
parser.add_argument("--task-timeout", type=float, default=1800,
|
||||
help="每任务截止时间(秒,默认 1800);0 表示不限时"
|
||||
"(与 host --turn-timeout 0 同约定,worker 级"
|
||||
@@ -5733,7 +5766,8 @@ def main(argv=None):
|
||||
tenant_template_repo=args.tenant_template_repo,
|
||||
egress_proxy_port=args.egress_proxy_port,
|
||||
claim_executor_only=args.claim_executor_only,
|
||||
websearch_mcp=args.websearch_mcp)
|
||||
websearch_mcp=args.websearch_mcp,
|
||||
seed_skills=args.seed_skills)
|
||||
# CLI 启动清理兜底(常驻与单次都做):清掉上一轮残留容器。
|
||||
worker.cleanup_stale_containers()
|
||||
# CLI 起 worker 前先建表/迁移:全新库直接起循环不应因缺表而炸。
|
||||
|
||||
Reference in new issue
Block a user