Merge pull request #155 from openqj/feat-chatgpt-codex-align1
ChatGPT Codex 对齐:技能包 + 租户播种 + web_search Docker 联网修法
This commit is contained in:
12 files changed
+964
-5
No files matched your search
@@ -0,0 +1,124 @@
|
||||
"""Seed ChatGPT-Codex-alignment skills into a tenant codex-home.
|
||||
|
||||
Mirrors the websearch MCP seeding pattern in cloud/websearch.py,
|
||||
but for skills, not MCP config:
|
||||
|
||||
- Source is cloud/skills/<name>/SKILL.md (the batch-1 skill pack).
|
||||
- Destination is <codex-home>/skills/<name>/SKILL.md.
|
||||
Why this path: verified live on 2026-10-30 with the real
|
||||
Codex binary (host.py skills-list against a temp codex-home
|
||||
on the production host, production untouched). The first
|
||||
attempt -- <codex-home>/.agents/skills/ -- was NOT discovered
|
||||
(skills/list returned only the 5 system skills). Copying the
|
||||
same SKILL.md files to <codex-home>/skills/ made them appear
|
||||
as scope=user (5 system + 3 ours, count 8). System skills
|
||||
themselves live at <codex-home>/skills/.system/, which is
|
||||
the tell that this is the user-skills directory for a
|
||||
CODEX_HOME. A Docker tenant task must not pollute its task
|
||||
repo/worktree (seeding into the worktree would show up in
|
||||
the task diff), so the tenant codex-home -- the per-project
|
||||
persistent home that is mounted into the container at
|
||||
/var/lib/codex-app-server/codex-home -- is the right channel.
|
||||
This is the same channel websearch MCP seeding uses.
|
||||
- Idempotent and non-destructive by design:
|
||||
- a missing destination file is copied;
|
||||
- an identical destination file is left alone;
|
||||
- a differing destination file is NEVER overwritten (that is
|
||||
a tenant-modified skill) -- it is reported in
|
||||
``skills_skipped_differ`` and left alone;
|
||||
- a skill directory without SKILL.md is skipped, never invented.
|
||||
- No config.toml change, no key, no network: skills are plain
|
||||
instruction files, and the helper never writes secrets.
|
||||
|
||||
Honest boundary (see docs/chatgpt-codex-alignment.md): discovery
|
||||
itself is now live-verified (skills/list, scope=user, on a temp
|
||||
home with the real Codex binary). What is NOT yet verified is
|
||||
model invocation inside a real Docker tenant task (an explicit
|
||||
$live-info call that actually runs and returns a result, and
|
||||
whether the relay model honours it). Until a task-level probe
|
||||
passes, do not claim the skills are live in production tasks --
|
||||
this helper only guarantees the files land in the tenant home,
|
||||
idempotently, without clobbering tenant edits, and that
|
||||
skills/list sees them there.
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
# cloud/skills/<name>/SKILL.md, next to this module.
|
||||
SKILLS_DIR = Path(__file__).resolve().parent / "skills"
|
||||
# User-level skills inside a codex-home (live-verified 2026-10-30:
|
||||
# .agents/skills inside a codex-home is NOT discovered).
|
||||
DEST_SUBDIR = Path("skills")
|
||||
|
||||
|
||||
def skill_source_dir(skills_dir=None):
|
||||
"""Skill pack directory: explicit override or cloud/skills/."""
|
||||
if skills_dir is not None:
|
||||
return Path(skills_dir)
|
||||
return SKILLS_DIR
|
||||
|
||||
|
||||
def list_skills(skills_dir=None):
|
||||
"""[(name, SKILL.md path)] for every skill in the pack."""
|
||||
src = skill_source_dir(skills_dir)
|
||||
out = []
|
||||
if not src.is_dir():
|
||||
return out
|
||||
for entry in sorted(src.iterdir()):
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
skill_file = entry / "SKILL.md"
|
||||
if skill_file.is_file():
|
||||
out.append((entry.name, skill_file))
|
||||
return out
|
||||
|
||||
|
||||
def ensure_codex_home_skills(codex_home_dir, skills_dir=None):
|
||||
"""Copy the skill pack into a tenant codex-home.
|
||||
|
||||
Returns {"dest_dir", "skills_copied", "skills_skipped_differ",
|
||||
"skills_skipped_missing"}: copied = names newly written,
|
||||
skipped_differ = names whose destination SKILL.md differs
|
||||
from the pack (tenant-modified, left alone), skipped_missing
|
||||
= skill directories in the pack without a SKILL.md file
|
||||
(should not happen; the batch-1 test guards it).
|
||||
A missing source directory yields all-empty lists, never
|
||||
an exception: skills are optional, a missing pack must not
|
||||
block a task the way a missing websearch config would.
|
||||
"""
|
||||
home = Path(codex_home_dir)
|
||||
dest_root = home / DEST_SUBDIR
|
||||
dest_root.mkdir(parents=True, exist_ok=True)
|
||||
copied = []
|
||||
skipped_differ = []
|
||||
skipped_missing = []
|
||||
src = skill_source_dir(skills_dir)
|
||||
if not src.is_dir():
|
||||
return {"dest_dir": str(dest_root), "skills_copied": copied,
|
||||
"skills_skipped_differ": skipped_differ,
|
||||
"skills_skipped_missing": skipped_missing}
|
||||
# Directories in the pack, including ones without SKILL.md,
|
||||
# so a broken pack is reported instead of silently dropped.
|
||||
for entry in sorted(src.iterdir()):
|
||||
if not entry.is_dir():
|
||||
continue
|
||||
skill_file = entry / "SKILL.md"
|
||||
if not skill_file.is_file():
|
||||
skipped_missing.append(entry.name)
|
||||
continue
|
||||
dest_dir = dest_root / entry.name
|
||||
dest_file = dest_dir / "SKILL.md"
|
||||
src_bytes = skill_file.read_bytes()
|
||||
if dest_file.is_file():
|
||||
if dest_file.read_bytes() == src_bytes:
|
||||
continue
|
||||
skipped_differ.append(entry.name)
|
||||
continue
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copyfile(skill_file, dest_file)
|
||||
copied.append(entry.name)
|
||||
return {"dest_dir": str(dest_root), "skills_copied": copied,
|
||||
"skills_skipped_differ": skipped_differ,
|
||||
"skills_skipped_missing": skipped_missing}
|
||||
@@ -0,0 +1,55 @@
|
||||
---
|
||||
name: document-tools
|
||||
description: 创建或编译 LaTeX 项目,处理 Word、PDF、表格和演示文稿时使用。先核查本机工具有没有,缺工具就明说,绝不假装产出了文件。
|
||||
---
|
||||
|
||||
# document-tools:文档处理
|
||||
|
||||
## 何时用
|
||||
|
||||
用户要做 LaTeX 编译、Word(.docx)读写、PDF 读写、表格
|
||||
(.xlsx / .csv)处理、演示文稿(.pptx)制作时用。
|
||||
|
||||
## 何时不用
|
||||
|
||||
- 只是读一段纯文本、改代码、写 Markdown:不用本技能,
|
||||
直接用文件工具。
|
||||
- 用户没说要产出文档,只是问文档里写了什么:先读文件,
|
||||
不用走编译流程。
|
||||
|
||||
## 工具前提(先核查,不满足就明说)
|
||||
|
||||
在工作区 shell 里先查,缺什么就报什么,不要跳过核查直接写:
|
||||
|
||||
- LaTeX:`latexmk` / `pdflatex` / `xelatex` 是否存在。
|
||||
- 通用转换:`pandoc` 是否存在。
|
||||
- Python 库:`python3 -c "import docx, openpyxl, pptx, fitz"`,
|
||||
哪个 import 失败就说明哪个缺失。
|
||||
- PDF 阅读:`pdftotext` 或 Python 的 `fitz` / `pypdf`。
|
||||
|
||||
核查命令本身不产出文件,核查失败时回答里写清“缺 XX,
|
||||
本环境无法编译 / 无法处理该格式”,并给出可替代的方案
|
||||
(例如只产出 Markdown / CSV),不要伪造一个文件路径。
|
||||
|
||||
## 步骤
|
||||
|
||||
1. 先确认输入和输出:输入文件路径、输出格式、输出路径、
|
||||
语言(中文 LaTeX 优先 xelatex + CJK 字体,缺字体先报)。
|
||||
2. 选最小工具链:能用 pandoc 就不用手写 LaTeX 模板;
|
||||
能用 Python 库读写就不用先转格式。
|
||||
3. 编译 / 生成后必须验证产出:
|
||||
- 文件确实存在,大小不是 0。
|
||||
- LaTeX:编译日志无致命错误(error / Emergency stop),
|
||||
页数符合预期。
|
||||
- docx / xlsx / pptx:用对应库重新打开一次,能读回
|
||||
刚才写的内容(标题、表头、页数/幻灯片数)。
|
||||
- PDF:能用 `pdftotext` 或 `fitz` 读回文本,确认不是空白页。
|
||||
4. 验证失败时不要把半成品当成品交,先贴出报错,再修。
|
||||
|
||||
## 输出要求
|
||||
|
||||
- 交代产出文件的真实路径、页数 / 幻灯片数 / 表格行数,
|
||||
以及用哪条命令验证过。
|
||||
- 不要编造页数、字数、文件大小;写不出的就写“未验证”。
|
||||
- 涉及用户数据的文件不要写到工作区以外,不要上传到
|
||||
第三方服务,除非用户明确要求。
|
||||
@@ -0,0 +1,52 @@
|
||||
---
|
||||
name: image-workflow
|
||||
description: 查看、裁剪、缩放、格式转换已有图片,或在图片生成工具已配置时按描述生成/编辑图片时使用。未配置生成工具时必须明说无法生成,绝不把已有图片或占位图当成生成结果。
|
||||
---
|
||||
|
||||
# image-workflow:图片查看与处理
|
||||
|
||||
## 何时用
|
||||
|
||||
- 查看已有图片、裁剪、缩放、转格式、读图片尺寸和格式。
|
||||
- 用户明确要求“按描述生成一张图”或“按描述改这张图”,
|
||||
且本环境已配置图片生成工具 / API 时。
|
||||
|
||||
## 何时不用
|
||||
|
||||
- 只是写代码、改文本、查文档:不用本技能。
|
||||
- 用户要生成图片,但本环境没有生成工具:不要硬做,
|
||||
走下面的“未配置”分支。
|
||||
|
||||
## 工具前提(先核查,不满足就明说)
|
||||
|
||||
1. 查看和处理已有图片:查 `python3 -c "import PIL"`,
|
||||
以及 `convert` / `magick`(ImageMagick)是否存在。
|
||||
有其一即可做裁剪、缩放、转格式;都没有就明说。
|
||||
2. 生成 / 编辑图片:先查任务环境里是否已配置图片生成
|
||||
工具或 MCP / API(包括需要什么 key、走哪个 endpoint)。
|
||||
没有配置时,回答必须是“本环境未配置图片生成工具,
|
||||
无法按描述生成”,并给出替代方案(例如用 shell 画示意图、
|
||||
或用户提供已有图片后做裁剪/缩放),绝不能:
|
||||
- 把网上找来的图、占位图、已有图片说成是刚生成的;
|
||||
- 编造一个图片路径或图片内容描述。
|
||||
|
||||
## 步骤(已有图片处理)
|
||||
|
||||
1. 先读原图信息:格式、尺寸、文件大小,确认输入路径存在。
|
||||
2. 按用户要求做裁剪 / 缩放 / 转格式,产出到用户指定的
|
||||
路径(未指定时用工作区内的新文件名,不覆盖原图)。
|
||||
3. 验证产出:重新打开产出图,确认格式、尺寸符合要求,
|
||||
文件大小不是 0。
|
||||
|
||||
## 步骤(生成 / 编辑,仅当工具已配置时)
|
||||
|
||||
1. 把用户描述原样记下来,不要自己加没说的元素。
|
||||
2. 调已配置的生成工具,记录工具名、提示词、产出路径。
|
||||
3. 验证产出图存在且能打开,再交给用户;失败时贴出
|
||||
工具报错,不要换一张别的图顶替。
|
||||
|
||||
## 输出要求
|
||||
|
||||
- 每个产出图都要给真实路径和尺寸,生成类还要给提示词。
|
||||
- 未配置时不要绕弯子说“已生成”,直接说缺什么、
|
||||
用户补什么才能生成。
|
||||
@@ -0,0 +1,55 @@
|
||||
---
|
||||
name: live-info
|
||||
description: 查询天气、时区、股票价格、体育赛程等实时公开信息时使用。只用可查证的公开来源,绝不编造数字,回答必须带来源和快照时间。
|
||||
---
|
||||
|
||||
# live-info:实时公开信息查询
|
||||
|
||||
## 何时用
|
||||
|
||||
用户问天气、时区换算、某只股票的当前价、某场比赛的赛程或比分等
|
||||
“现在是多少”类问题时用。编程任务里需要查官方文档、发布说明、
|
||||
最新版本时,同样走本技能的查证流程,不要凭记忆给数字。
|
||||
|
||||
## 何时不用
|
||||
|
||||
- 用户只问概念、历史、代码怎么写:不用本技能。
|
||||
- 已经有任务内文件、数据库、日志能直接回答的:以任务内证据为准,
|
||||
不用再上网。
|
||||
|
||||
## 工具前提(先核查,不满足就明说)
|
||||
|
||||
1. 先确认本任务里是否有网页搜索工具(codex-app-server 里是
|
||||
websearch MCP 的 `web_search` / `web_fetch`,生产 worker 需
|
||||
带 `--websearch-mcp` 才播种)。没有搜索工具时,直接告诉用户
|
||||
“本环境无联网查询工具,无法核实实时数字”,不要编一个。
|
||||
2. 有搜索工具时,优先查官方或一级来源:气象部门、交易所、
|
||||
联赛/球队官网、时区数据库说明。聚合新闻、论坛、二手转载
|
||||
只能当线索,不能当唯一来源。
|
||||
|
||||
## 步骤
|
||||
|
||||
1. 把问题拆成可查的实体 + 时间 + 地点:例如“北京明天天气”
|
||||
→ 城市 = 北京,日期 = 明天(换算成具体日期并写出来),
|
||||
指标 = 气温/降水/风力。
|
||||
2. 用 `web_search` 查 1-2 个一级来源,打开来源页核对数字,
|
||||
不要只抄搜索摘要。
|
||||
3. 股票:只报快照价 + 涨跌 + 快照时间,并说明“快照价不是
|
||||
投资建议,盘中价格会变”。不要给买入/卖出建议。
|
||||
4. 时区:先写出两个时区的 UTC 偏移和夏令时状态,再换算,
|
||||
不要只给换算结果。
|
||||
5. 体育赛程:写清联赛、主客队、开赛时间(含时区)、比分
|
||||
是“已结束 / 进行中 / 未开始”哪一种。
|
||||
|
||||
## 输出要求
|
||||
|
||||
- 数字后面必须跟来源和时间,例如“来源:XX 官网,查询时间
|
||||
2026-10-30 00:00 CST”。
|
||||
- 查不到、来源冲突、或时间已过期时,明说查不到 / 冲突,
|
||||
不要选一个看起来顺眼的数字。
|
||||
- 不要把预测(天气预报、分析师目标价)写成事实。
|
||||
|
||||
## 校验
|
||||
|
||||
回答前自查:每个数字都有来源吗?时间写了吗?时区写了吗?
|
||||
有一项没有,就补查或删掉该数字。
|
||||
@@ -0,0 +1,52 @@
|
||||
"""ChatGPT Codex 对齐第一批:技能包校验。
|
||||
|
||||
只校验技能文件本身,不启动 worker、不跑模型、不改生产行为:
|
||||
每个 cloud/skills/<名>/SKILL.md 必须有 name + description
|
||||
frontmatter、名字与目录一致、正文非空、不含密钥占位。
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
SKILLS_DIR = Path(__file__).resolve().parents[1] / "skills"
|
||||
EXPECTED = {"live-info", "document-tools", "image-workflow"}
|
||||
|
||||
|
||||
def _frontmatter(text):
|
||||
m = re.match(r"^---\n(.*?)\n---\n", text, re.DOTALL)
|
||||
if not m:
|
||||
return None
|
||||
out = {}
|
||||
for line in m.group(1).splitlines():
|
||||
if ":" in line:
|
||||
k, v = line.split(":", 1)
|
||||
out[k.strip()] = v.strip()
|
||||
return out
|
||||
|
||||
|
||||
class ChatGptCodexSkillsTestCase(unittest.TestCase):
|
||||
def test_expected_skills_exist(self):
|
||||
names = {p.name for p in SKILLS_DIR.iterdir() if p.is_dir()}
|
||||
self.assertEqual(names, EXPECTED)
|
||||
|
||||
def test_skill_frontmatter_and_body(self):
|
||||
for name in sorted(EXPECTED):
|
||||
path = SKILLS_DIR / name / "SKILL.md"
|
||||
self.assertTrue(path.is_file(), str(path))
|
||||
text = path.read_text(encoding="utf-8")
|
||||
fm = _frontmatter(text)
|
||||
self.assertIsNotNone(fm, name)
|
||||
self.assertEqual(fm.get("name"), name)
|
||||
desc = fm.get("description", "")
|
||||
self.assertTrue(desc, name)
|
||||
self.assertLessEqual(len(desc), 200, name)
|
||||
body = text.split("---", 2)[-1].strip()
|
||||
self.assertTrue(len(body) > 200, name)
|
||||
for bad in ("API_KEY", "sk-", "password"):
|
||||
self.assertNotIn(bad, text, name)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Worker wiring for ChatGPT-Codex-alignment skills: tenant seeding.
|
||||
|
||||
Mirrors test_websearch_worker.py: no Docker, no network, no real
|
||||
model. _tenant_dirs runs against temp dirs; the skill pack used
|
||||
here is the real cloud/skills/ pack, so a broken pack fails here
|
||||
before it ever reaches a tenant.
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from cloud.server import init_db
|
||||
from cloud.worker import Worker
|
||||
from cloud.skill_seed import ensure_codex_home_skills, list_skills
|
||||
except ImportError:
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.server import init_db
|
||||
from cloud.worker import Worker
|
||||
from cloud.skill_seed import ensure_codex_home_skills, list_skills
|
||||
|
||||
|
||||
class SkillSeedHelperTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _src(self):
|
||||
src = os.path.join(self.tmp.name, "pack")
|
||||
for name in ("live-info", "document-tools"):
|
||||
d = os.path.join(src, name)
|
||||
os.makedirs(d, exist_ok=True)
|
||||
open(os.path.join(d, "SKILL.md"), "w").write(
|
||||
"---\nname: %s\ndescription: test\n---\nbody\n" % name)
|
||||
return src
|
||||
|
||||
def test_copy_idempotent_and_missing_source(self):
|
||||
home = os.path.join(self.tmp.name, "home")
|
||||
out = ensure_codex_home_skills(home, self._src())
|
||||
self.assertEqual(sorted(out["skills_copied"]),
|
||||
["document-tools", "live-info"])
|
||||
self.assertEqual(out["skills_skipped_differ"], [])
|
||||
out2 = ensure_codex_home_skills(home, self._src())
|
||||
self.assertEqual(out2["skills_copied"], [])
|
||||
out3 = ensure_codex_home_skills(
|
||||
os.path.join(self.tmp.name, "home2"),
|
||||
os.path.join(self.tmp.name, "no-such-pack"))
|
||||
self.assertEqual(out3["skills_copied"], [])
|
||||
|
||||
def test_tenant_modified_skill_is_never_overwritten(self):
|
||||
home = os.path.join(self.tmp.name, "home")
|
||||
src = self._src()
|
||||
ensure_codex_home_skills(home, src)
|
||||
dest = os.path.join(home, "skills",
|
||||
"live-info", "SKILL.md")
|
||||
open(dest, "w").write("tenant edited version\n")
|
||||
out = ensure_codex_home_skills(home, src)
|
||||
self.assertEqual(out["skills_skipped_differ"], ["live-info"])
|
||||
self.assertEqual(open(dest).read(), "tenant edited version\n")
|
||||
|
||||
def test_real_pack_lists_three_skills(self):
|
||||
names = [n for n, _p in list_skills()]
|
||||
self.assertEqual(names,
|
||||
["document-tools", "image-workflow",
|
||||
"live-info"])
|
||||
|
||||
|
||||
class WorkerSkillSeedTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
init_db(self.db_path)
|
||||
self.template_home = os.path.join(self.tmp.name, "tpl-home")
|
||||
os.makedirs(self.template_home)
|
||||
open(os.path.join(self.template_home, "config.toml"),
|
||||
"w").write('[model_provider.xcmapi]\n'
|
||||
'base_url="https://xcmapi.org/v1"\n')
|
||||
self.template_repo = os.path.join(self.tmp.name, "tpl-repo")
|
||||
os.makedirs(os.path.join(self.template_repo, ".git"))
|
||||
open(os.path.join(self.template_repo, "sample.py"),
|
||||
"w").write("def add(a, b):\n return a + b\n")
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _worker(self, **kwargs):
|
||||
kwargs.setdefault("env", {"XCM_API_KEY": "test-placeholder"})
|
||||
kwargs.setdefault("tenants_root",
|
||||
os.path.join(self.tmp.name, "tenants"))
|
||||
kwargs.setdefault("tenant_template_home", self.template_home)
|
||||
kwargs.setdefault("tenant_template_repo", self.template_repo)
|
||||
kwargs.setdefault("tenant_isolation", True)
|
||||
return Worker(self.db_path, "/fake/host.py", **kwargs)
|
||||
|
||||
def _skill_path(self, dirs, name):
|
||||
return os.path.join(dirs["codex_home"], "skills",
|
||||
name, "SKILL.md")
|
||||
|
||||
def test_flag_off_is_old_behaviour(self):
|
||||
worker = self._worker()
|
||||
self.assertFalse(worker.seed_skills)
|
||||
dirs = worker._tenant_dirs("projA")
|
||||
self.assertFalse(os.path.exists(
|
||||
os.path.join(dirs["codex_home"], "skills")))
|
||||
|
||||
def test_flag_on_seeds_real_pack(self):
|
||||
worker = self._worker(seed_skills=True)
|
||||
dirs = worker._tenant_dirs("projA")
|
||||
for name in ("live-info", "document-tools", "image-workflow"):
|
||||
path = self._skill_path(dirs, name)
|
||||
self.assertTrue(os.path.exists(path), path)
|
||||
self.assertIn("name: %s" % name, open(path).read())
|
||||
|
||||
def test_old_tenant_merges_and_user_edit_survives(self):
|
||||
worker = self._worker()
|
||||
dirs = worker._tenant_dirs("projA")
|
||||
worker2 = self._worker(seed_skills=True)
|
||||
worker2._tenant_dirs("projA")
|
||||
path = self._skill_path(dirs, "live-info")
|
||||
self.assertTrue(os.path.exists(path))
|
||||
open(path, "w").write("tenant edited version\n")
|
||||
worker2._tenant_dirs("projA")
|
||||
self.assertEqual(open(path).read(), "tenant edited version\n")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,71 @@
|
||||
"""ensure_websearch_mcp_env: MCP env lives in config.toml, because
|
||||
Codex's MCP subprocess does not inherit the Codex process env
|
||||
(2026-10-30: Codex bash had SEARXNG_URL + proxy, MCP chain was
|
||||
still ["duckduckgo"] only; adding the config env section made
|
||||
the same task return real SearXNG results)."""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
from cloud.websearch import ensure_websearch_mcp_env # noqa: E402
|
||||
|
||||
|
||||
BASE = ("model = \"gpt-6.1-sol\"\n\n"
|
||||
"[mcp_servers.websearch]\n"
|
||||
"command = \"python3\"\n"
|
||||
"args = [\"/var/lib/codex-app-server/codex-home/"
|
||||
"websearch_mcp.py\"]\n")
|
||||
|
||||
|
||||
class TestMcpEnv(unittest.TestCase):
|
||||
def _home(self, text):
|
||||
tmp = tempfile.mkdtemp()
|
||||
with open(os.path.join(tmp, "config.toml"), "w") as fh:
|
||||
fh.write(text)
|
||||
return tmp
|
||||
|
||||
def _read(self, home):
|
||||
with open(os.path.join(home, "config.toml")) as fh:
|
||||
return fh.read()
|
||||
|
||||
def test_insert_env_after_websearch_section(self):
|
||||
home = self._home(BASE)
|
||||
self.assertTrue(ensure_websearch_mcp_env(
|
||||
home, {"SEARXNG_URL": "http://searx.test",
|
||||
"http_proxy": "http://10.0.0.1:8888"}))
|
||||
text = self._read(home)
|
||||
self.assertIn("[mcp_servers.websearch.env]", text)
|
||||
self.assertIn('SEARXNG_URL = "http://searx.test"', text)
|
||||
self.assertIn('http_proxy = "http://10.0.0.1:8888"', text)
|
||||
self.assertIn('command = "python3"', text)
|
||||
|
||||
def test_replace_existing_env_preserves_other_sections(self):
|
||||
home = self._home(BASE + "[mcp_servers.websearch.env]\n"
|
||||
"SEARXNG_URL = \"http://old.test\"\n\n"
|
||||
"[other]\nfoo = 1\n")
|
||||
self.assertTrue(ensure_websearch_mcp_env(
|
||||
home, {"SEARXNG_URL": "http://new.test"}))
|
||||
text = self._read(home)
|
||||
self.assertIn('SEARXNG_URL = "http://new.test"', text)
|
||||
self.assertNotIn("old.test", text)
|
||||
self.assertIn("[other]\nfoo = 1", text)
|
||||
|
||||
def test_empty_env_writes_nothing(self):
|
||||
home = self._home(BASE)
|
||||
self.assertFalse(ensure_websearch_mcp_env(home, {}))
|
||||
self.assertEqual(self._read(home), BASE)
|
||||
|
||||
def test_idempotent_second_write(self):
|
||||
home = self._home(BASE)
|
||||
env = {"SEARXNG_URL": "http://searx.test"}
|
||||
self.assertTrue(ensure_websearch_mcp_env(home, env))
|
||||
self.assertFalse(ensure_websearch_mcp_env(home, env))
|
||||
self.assertEqual(self._read(home).count(
|
||||
"[mcp_servers.websearch.env]"), 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,52 @@
|
||||
"""SearXNG provider: engines are pinned, not left to the default
|
||||
general category (2026-10-30: default took 20s in a Docker task,
|
||||
pinned bing,360search took 0.47s via the same egress proxy)."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import unittest
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
|
||||
from cloud.websearch import _provider_searxng # noqa: E402
|
||||
|
||||
|
||||
class _FakeHttp:
|
||||
def __init__(self):
|
||||
self.urls = []
|
||||
|
||||
def get(self, url, headers=None):
|
||||
self.urls.append(url)
|
||||
body = json.dumps({"results": [
|
||||
{"title": "Beijing weather", "url": "https://example.test/w",
|
||||
"content": "sunny"}]}).encode("utf-8")
|
||||
return 200, body, "application/json"
|
||||
|
||||
|
||||
class TestSearXNGEngines(unittest.TestCase):
|
||||
def test_default_pins_bing_360search(self):
|
||||
http = _FakeHttp()
|
||||
results, _ = _provider_searxng(
|
||||
"Beijing weather", 3, {"SEARXNG_URL": "http://searx.test"},
|
||||
http)
|
||||
self.assertEqual(len(results), 1)
|
||||
self.assertIn("engines=bing%2C360search", http.urls[0])
|
||||
|
||||
def test_env_overrides_pin(self):
|
||||
http = _FakeHttp()
|
||||
_provider_searxng(
|
||||
"q", 3, {"SEARXNG_URL": "http://searx.test",
|
||||
"SEARXNG_ENGINES": "bing"}, http)
|
||||
self.assertIn("engines=bing", http.urls[0])
|
||||
self.assertNotIn("360search", http.urls[0])
|
||||
|
||||
def test_empty_env_restores_unpinned(self):
|
||||
http = _FakeHttp()
|
||||
_provider_searxng(
|
||||
"q", 3, {"SEARXNG_URL": "http://searx.test",
|
||||
"SEARXNG_ENGINES": ""}, http)
|
||||
self.assertNotIn("engines=", http.urls[0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+92
-2
@@ -267,8 +267,22 @@ def _provider_searxng(query, limit, env, http):
|
||||
base = (env.get("SEARXNG_URL") or "").rstrip("/")
|
||||
if not base:
|
||||
raise WebSearchError("searxng not configured (SEARXNG_URL unset)")
|
||||
url = (base + "/search?"
|
||||
+ urllib.parse.urlencode({"q": query, "format": "json"}))
|
||||
# Engines are pinned, not left to SearXNG's default "general"
|
||||
# category. Measured 2026-10-30 on the production host:
|
||||
# default general search took 20.0s in a Docker task container
|
||||
# (wikipedia / wikidata / brave / duckduckgo / google engines
|
||||
# all timing out, only bing + 360search returning), which hits
|
||||
# the MCP's 20s timeout and then falls through to DuckDuckGo,
|
||||
# which itself times out via the egress proxy -- so the whole
|
||||
# chain fails even though SearXNG can answer. Pinning to the
|
||||
# two proven-fast engines cuts the same query to 0.47s via
|
||||
# the same proxy. SEARXNG_ENGINES overrides the pin; an empty
|
||||
# value restores the old unpinned behaviour.
|
||||
params = {"q": query, "format": "json"}
|
||||
engines = env.get("SEARXNG_ENGINES", "bing,360search")
|
||||
if engines:
|
||||
params["engines"] = engines
|
||||
url = base + "/search?" + urllib.parse.urlencode(params)
|
||||
_status, body, _ctype = http.get(url, _default_headers(env, "searxng"))
|
||||
data = json.loads(body.decode("utf-8", errors="replace"))
|
||||
results = []
|
||||
@@ -633,3 +647,79 @@ def ensure_codex_home_websearch(codex_home_dir, command_args=None,
|
||||
"config_changed": config_changed,
|
||||
"script_changed": script_changed,
|
||||
"script_differs": script_differs}
|
||||
|
||||
|
||||
def ensure_websearch_mcp_env(codex_home_dir, env):
|
||||
"""Write [mcp_servers.websearch.env] into a tenant config.toml.
|
||||
|
||||
Why this exists (measured 2026-10-30, not guessed): Codex's
|
||||
MCP server subprocess does NOT inherit the Codex process env.
|
||||
A Docker task whose Codex bash env contained SEARXNG_URL and
|
||||
http_proxy still ran web_search with chain ["duckduckgo"]
|
||||
only, failing instantly with "Temporary failure in name
|
||||
resolution" -- SEARXNG_URL never reached the MCP process, and
|
||||
without a proxy the container has no direct DNS. Adding an
|
||||
explicit ``env`` table to the MCP config made the same task
|
||||
return real SearXNG results. Container env (docker -e) is
|
||||
therefore NOT sufficient for MCP tools; the env must live in
|
||||
config.toml, where Codex actually reads it for MCP servers.
|
||||
|
||||
Idempotent: an existing env subsection is replaced in place,
|
||||
other sections are left untouched, and a missing websearch
|
||||
section is created first (command/args only, no env values
|
||||
invented). Values are written as TOML basic strings via
|
||||
json.dumps (safe for URLs and proxy URLs). Returns True when
|
||||
the file was written, False when env is empty or the file
|
||||
could not be written (caller logs, never crashes the task).
|
||||
"""
|
||||
if not env:
|
||||
return False
|
||||
home = os.path.abspath(codex_home_dir)
|
||||
config_path = os.path.join(home, "config.toml")
|
||||
try:
|
||||
existing = ""
|
||||
if os.path.exists(config_path):
|
||||
with open(config_path, encoding="utf-8") as fh:
|
||||
existing = fh.read()
|
||||
if "[mcp_servers.websearch]" not in existing:
|
||||
existing += ("\n[mcp_servers.websearch]\n"
|
||||
"command = \"python3\"\n"
|
||||
"args = [\"/var/lib/codex-app-server/"
|
||||
"codex-home/websearch_mcp.py\"]\n")
|
||||
env_lines = ["[mcp_servers.websearch.env]"]
|
||||
for key in sorted(env):
|
||||
env_lines.append("%s = %s" % (key, json.dumps(env[key])))
|
||||
env_block = "\n".join(env_lines) + "\n"
|
||||
lines = existing.splitlines(keepends=True)
|
||||
start = end = None
|
||||
for i, line in enumerate(lines):
|
||||
if line.strip() == "[mcp_servers.websearch.env]":
|
||||
start = i
|
||||
elif start is not None and line.startswith("["):
|
||||
end = i
|
||||
break
|
||||
if start is not None:
|
||||
if end is None:
|
||||
end = len(lines)
|
||||
lines[start:end] = [env_block]
|
||||
new_text = "".join(lines)
|
||||
else:
|
||||
insert_at = None
|
||||
in_section = False
|
||||
for i, line in enumerate(lines):
|
||||
if line.strip() == "[mcp_servers.websearch]":
|
||||
in_section = True
|
||||
elif in_section and line.startswith("["):
|
||||
insert_at = i
|
||||
break
|
||||
if insert_at is None:
|
||||
insert_at = len(lines)
|
||||
lines.insert(insert_at, env_block)
|
||||
new_text = "".join(lines)
|
||||
if new_text == existing:
|
||||
return False
|
||||
with open(config_path, "w", encoding="utf-8") as fh:
|
||||
fh.write(new_text)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
+55
-3
@@ -166,6 +166,8 @@ from cloud.peer_messages import peer_message_prefix as _peer_prefix
|
||||
from cloud.peer_messages import row_get as _row_get
|
||||
from cloud.sub2api import ensure_user_gateway_key as _ensure_sub2api_key
|
||||
from cloud.websearch import ensure_codex_home_websearch as _ensure_websearch_mcp
|
||||
from cloud.websearch import ensure_websearch_mcp_env as _ensure_websearch_mcp_env
|
||||
from cloud.skill_seed import ensure_codex_home_skills as _ensure_skills
|
||||
from cloud.sub2api import gateway_name as _gateway_name
|
||||
from cloud.sub2api import gateway_origin as _gateway_origin
|
||||
from cloud.sub2api import list_gateway_models as _list_gateway_models
|
||||
@@ -412,7 +414,7 @@ class Worker:
|
||||
tenant_isolation=False, tenants_root=None,
|
||||
tenant_template_home=None, tenant_template_repo=None,
|
||||
egress_proxy_port=0, claim_executor_only=False,
|
||||
websearch_mcp=False,
|
||||
websearch_mcp=False, seed_skills=False,
|
||||
host_name=None, pr_jobs_only=False,
|
||||
upload_dir=None,
|
||||
digest_interval_seconds=3600,
|
||||
@@ -565,6 +567,14 @@ class Worker:
|
||||
# [mcp_servers.websearch](容器内路径,见 _tenant_dirs)。
|
||||
# 老租户下一次任务自动合并,已有 section 不重写。
|
||||
self.websearch_mcp = bool(websearch_mcp)
|
||||
# 技能播种:默认关(老行为:租户 codex-home 只有模板
|
||||
# config.toml + 可选的 websearch MCP)。开启后
|
||||
# _tenant_dirs 在 websearch 播种之后调用 skill_seed
|
||||
# 的播种助手,把 cloud/skills/ 的 SKILL.md 拷进租户
|
||||
# codex-home/skills/(用户级技能路径,实测见
|
||||
# skill_seed.py 的路径说明)。老租户下一次任务自动
|
||||
# 合并;租户改过的技能文件绝不覆盖,只上报 differs。
|
||||
self.seed_skills = bool(seed_skills)
|
||||
# HA 阶段 1:本 worker 所在主机名。认领任务时写进
|
||||
# tasks.worktree_host,/pr 据此判断 worktree 在哪台主机。
|
||||
# 容器内 worker 的 socket.gethostname() 是容器名、不是宿主名,
|
||||
@@ -2403,6 +2413,24 @@ class Worker:
|
||||
"python3",
|
||||
"/var/lib/codex-app-server/codex-home/"
|
||||
"websearch_mcp.py"])
|
||||
if self.seed_skills:
|
||||
# 技能播种失败绝不阻塞任务:技能是可选的流程
|
||||
# 说明,不是 websearch 那种“已宣传就必须有”的
|
||||
# 工具;播种助手本身对缺包、已存在、租户改动
|
||||
# 都是静默处理,只在文件系统本身出错时才抛,
|
||||
# 那种情况下任务仍应派发(技能缺失只影响流程,
|
||||
# 不影响 agent 本身跑起来)。
|
||||
try:
|
||||
_ensure_skills(dirs["codex_home"])
|
||||
except Exception as exc:
|
||||
# 只记日志不阻塞:技能缺失只影响流程说明,
|
||||
# 不应把一次可选播种失败变成任务派发失败。
|
||||
# 但绝不静默吞掉——日志里要留下来供排查。
|
||||
try:
|
||||
self.logger.warning(
|
||||
"skill seeding failed: %s", exc)
|
||||
except Exception:
|
||||
pass
|
||||
if not os.path.exists(os.path.join(dirs["test_repo"], ".git")) \
|
||||
and not os.path.exists(dirs["test_repo"] + "/sample.py"):
|
||||
# 目标为空目录(刚 mkdir 出来的)才播种;已有内容
|
||||
@@ -2789,12 +2817,29 @@ class Worker:
|
||||
# 走 extra_env(容器内进程本就需要它调 API)。
|
||||
search_env = {
|
||||
k: os.environ[k]
|
||||
for k in ("SEARXNG_URL", "PERPLEXITY_API_KEY", "EXA_API_KEY",
|
||||
for k in ("SEARXNG_URL", "SEARXNG_ENGINES",
|
||||
"PERPLEXITY_API_KEY", "EXA_API_KEY",
|
||||
"TAVILY_API_KEY", "BRAVE_API_KEY")
|
||||
if os.environ.get(k)
|
||||
}
|
||||
if search_env:
|
||||
proxy_env = _merge_env(proxy_env, search_env)
|
||||
# MCP env must live in config.toml, not just docker -e:
|
||||
# Codex's MCP subprocess does not inherit the Codex
|
||||
# process env (2026-10-30实测:Codex bash 里有
|
||||
# SEARXNG_URL + proxy,MCP 链却只有 duckduckgo,
|
||||
# DNS 瞬间失败;给 config 加 [mcp_servers.websearch.env]
|
||||
# 后同一任务立刻返回真实 SearXNG 结果)。网关在此时
|
||||
# 才已知,所以这里、容器启动前写租户 config,绝不
|
||||
# 在播种时写死一个网关。写失败只记日志,不阻塞任务。
|
||||
if self.tenant_isolation and proxy_env:
|
||||
try:
|
||||
home = self._tenant_dirs(
|
||||
task.get("project_id"))["codex_home"]
|
||||
_ensure_websearch_mcp_env(home, proxy_env)
|
||||
except Exception as exc:
|
||||
self.logger.warning(
|
||||
"websearch MCP env write failed: %s" % exc)
|
||||
return (True, tenant_mounts, tenant_network, proxy_env, None)
|
||||
|
||||
def _run_fork(self, task, quota_check=None):
|
||||
@@ -5672,6 +5717,12 @@ def main(argv=None):
|
||||
" [mcp_servers.websearch] 并拷入"
|
||||
" websearch.py / websearch_mcp.py;"
|
||||
"默认关,老租户不受影响")
|
||||
parser.add_argument("--seed-skills", action="store_true",
|
||||
help="租户 codex-home 自动播种对齐技能包:"
|
||||
"_tenant_dirs 把 cloud/skills/ 的 SKILL.md"
|
||||
" 拷进租户 codex-home/skills/;"
|
||||
"默认关,老租户不受影响;租户改过的"
|
||||
"技能文件绝不覆盖")
|
||||
parser.add_argument("--task-timeout", type=float, default=1800,
|
||||
help="每任务截止时间(秒,默认 1800);0 表示不限时"
|
||||
"(与 host --turn-timeout 0 同约定,worker 级"
|
||||
@@ -5733,7 +5784,8 @@ def main(argv=None):
|
||||
tenant_template_repo=args.tenant_template_repo,
|
||||
egress_proxy_port=args.egress_proxy_port,
|
||||
claim_executor_only=args.claim_executor_only,
|
||||
websearch_mcp=args.websearch_mcp)
|
||||
websearch_mcp=args.websearch_mcp,
|
||||
seed_skills=args.seed_skills)
|
||||
# CLI 启动清理兜底(常驻与单次都做):清掉上一轮残留容器。
|
||||
worker.cleanup_stale_containers()
|
||||
# CLI 起 worker 前先建表/迁移:全新库直接起循环不应因缺表而炸。
|
||||
|
||||
Reference in new issue
Block a user