Files
codex-app-server/cloud/tests/test_chv_gitea_sync.py
T
host e7f699c62b feat(chv-guest): Gitea 同步变量经 seed 进 guest-env(执行器集成)
- 新增 cloud/chv/gitea_sync.py:6 键白名单,三源合并
  (宿主 CHV_GITEA_* < 任务 gitea_sync < extra_env),
  必填 4 键缺失即 fail-closed,空即不同步
- ssh_transport:GITEA_TOKEN 进密钥集(argv 拒绝),其余
  5 键进 guest-env 集(split_guest_env)
- hostkeys.write_seed:guest_env 与 secret_env 合并写同一
  条目,仍无 runcmd、0600
- config:_gitea_fragment 产 gitea_sync_env(函数内 import,
  叶子模块顶层只许标准库)
- executor.wake_vm:起 VMM 前合并+校验(gitea_sync_incomplete),
  guest-env 随 seed 投递;provision_host_key 透传 guest_env
- worker:_run_attempt 仅对 CHV 执行器把同步变量并进 gateway_env
- 测试:cloud/tests/test_chv_gitea_sync.py 20 项
- 边界:tasks 表无 JSON 列,DB 往返丢任务 gitea_sync;
  生产主来源是宿主 CHV_GITEA_* 环境变量
2026-11-06 16:55:32 +00:00

496 lines
20 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""CHV guest-env 集成测试:Gitea 同步变量供给链路。
只用标准库与 mock runner/launcher/api_client,绝不启动真实
VM、不联网、不含任何真实 token(测试值全是假值)。
覆盖:
- gitea_sync 纯函数:三源合并优先级、白名单清洗、完整性
校验(fail-closed)与空=不同步;
- ssh_transport:GITEA_TOKEN 进密钥集(argv 拒绝)、其余
5 键进 guest-env 集;
- hostkeys.write_seed:guest_env 与 secret_env 合并写同一
条目、仍无 runcmd、0600;
- 执行器 wake_vm:完整同步变量进 seed user-data;缺失必填
键时起 VMM 前 ChvFailure 且不启动;无同步变量时正常启动
且不写 guest-env;seed 投递后 build_ssh_command 才能用
GITEA_TOKEN 且 argv 里绝无 token 值;
- worker:_gitea_sync_env 取值;_run_attempt 只对 CHV 执行器
把同步变量并进 gateway_env。
"""
import os
import sys
import tempfile
import unittest
from pathlib import Path
from types import SimpleNamespace
import yaml
try:
from cloud.chv.gitea_sync import (
GITEA_SYNC_KEYS,
config_gitea_sync_env,
extract_gitea_sync_env,
validate_gitea_sync_env,
)
from cloud.chv.hostkeys import write_seed
from cloud.chv.ssh_transport import (
_CHV_SECRET_ENV_NAMES,
build_ssh_command,
remote_command,
split_guest_env,
split_secret_env,
)
from cloud.sandbox import ChvFailure, CloudHypervisorExecutor
from cloud.worker import Worker
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.chv.gitea_sync import (
GITEA_SYNC_KEYS,
config_gitea_sync_env,
extract_gitea_sync_env,
validate_gitea_sync_env,
)
from cloud.chv.hostkeys import write_seed
from cloud.chv.ssh_transport import (
_CHV_SECRET_ENV_NAMES,
build_ssh_command,
remote_command,
split_guest_env,
split_secret_env,
)
from cloud.sandbox import ChvFailure, CloudHypervisorExecutor
from cloud.worker import Worker
FAKE_SYNC = {
"GITEA_URL": "https://gitea.tose.top",
"GITEA_USER": "admin",
"GITEA_TOKEN": "fake-token-123",
"SYNC_REPO": "admin/chv-sync-test",
"SYNC_BRANCH": "main",
"GITEA_EMAIL": "admin@tose.top",
}
class FakeResult:
def __init__(self, returncode=0, stdout="", stderr=""):
self.returncode = returncode
self.stdout = stdout
self.stderr = stderr
class FakeProc:
def __init__(self, alive=True):
self._alive = alive
def poll(self):
return None if self._alive else 1
def terminate(self):
self._alive = False
def wait(self, timeout=None):
return 0
def kill(self):
self._alive = False
def _write(path, content=b"x"):
with open(path, "wb") as fh:
fh.write(content)
os.chmod(path, 0o755)
return path
def _seed_user_text(state_dir, vm_name):
with open(os.path.join(state_dir, vm_name, "seed", "user-data")) as fh:
return fh.read()
def _guest_env_lines(user_text):
parsed = yaml.safe_load(user_text)
entries = parsed["write_files"]
guest = [e for e in entries if e["path"] == "/etc/chv/guest-env"]
assert len(guest) == 1, entries
return guest[0]["content"], guest[0]["permissions"]
# -- (1) gitea_sync 纯函数 ---------------------------------------------
class GiteaSyncPureTest(unittest.TestCase):
def test_keys_are_six(self):
self.assertEqual(GITEA_SYNC_KEYS, set(FAKE_SYNC))
def test_config_mapping(self):
env = {"CHV_GITEA_URL": "https://gitea.tose.top",
"CHV_GITEA_USER": "admin",
"CHV_GITEA_TOKEN": "tok",
"CHV_SYNC_REPO": "admin/chv-sync-test",
"CHV_SYNC_BRANCH": "main",
"CHV_GITEA_EMAIL": "admin@tose.top",
"CHV_UNRELATED": "x"}
self.assertEqual(config_gitea_sync_env(env), {
"GITEA_URL": "https://gitea.tose.top",
"GITEA_USER": "admin", "GITEA_TOKEN": "tok",
"SYNC_REPO": "admin/chv-sync-test",
"SYNC_BRANCH": "main", "GITEA_EMAIL": "admin@tose.top"})
def test_extract_priority_config_lt_task_lt_extra_env(self):
task = {"id": "t1", "gitea_sync": {
"GITEA_URL": "https://task.example",
"GITEA_USER": "task-user", "GITEA_TOKEN": "task-tok",
"SYNC_REPO": "task/repo"}}
out = extract_gitea_sync_env(
task=task,
extra_env={"SYNC_REPO": "extra/repo"},
config_sync={"GITEA_URL": "https://config.example",
"GITEA_USER": "config-user",
"GITEA_TOKEN": "config-tok",
"SYNC_REPO": "config/repo",
"SYNC_BRANCH": "dev"})
self.assertEqual(out["GITEA_URL"], "https://task.example")
self.assertEqual(out["GITEA_USER"], "task-user")
self.assertEqual(out["GITEA_TOKEN"], "task-tok")
self.assertEqual(out["SYNC_REPO"], "extra/repo")
self.assertEqual(out["SYNC_BRANCH"], "dev")
def test_extract_drops_unknown_and_empty_and_nonstr(self):
task = {"id": "t1", "gitea_sync": {
"GITEA_URL": "https://gitea.tose.top",
"EVIL": "nope", "SYNC_REPO": " ",
"GITEA_TOKEN": 12345}}
out = extract_gitea_sync_env(task=task)
self.assertEqual(out, {"GITEA_URL": "https://gitea.tose.top"})
self.assertNotIn("gitea_sync", out)
def test_build_chv_config_maps_gitea_env(self):
from cloud.chv.config import build_chv_config
cfg = build_chv_config({}, {
"CHV_GITEA_URL": "https://gitea.tose.top",
"CHV_GITEA_USER": "admin",
"CHV_GITEA_TOKEN": "tok",
"CHV_SYNC_REPO": "admin/chv-sync-test"})
self.assertEqual(cfg.gitea_sync_env, {
"GITEA_URL": "https://gitea.tose.top",
"GITEA_USER": "admin", "GITEA_TOKEN": "tok",
"SYNC_REPO": "admin/chv-sync-test"})
cfg2 = build_chv_config({}, {})
self.assertEqual(cfg2.gitea_sync_env, {})
def test_validate_empty_passes_partial_fails_with_names_only(self):
validate_gitea_sync_env({})
validate_gitea_sync_env(None)
validate_gitea_sync_env(dict(FAKE_SYNC))
with self.assertRaises(ValueError) as ctx:
validate_gitea_sync_env({"GITEA_URL": "https://gitea.tose.top",
"SYNC_REPO": "admin/chv-sync-test"})
msg = str(ctx.exception)
self.assertIn("GITEA_TOKEN", msg)
self.assertIn("GITEA_USER", msg)
self.assertNotIn("gitea.tose.top", msg)
self.assertNotIn("chv-sync-test", msg)
# -- (2) ssh_transport -------------------------------------------------
class SshTransportSyncTest(unittest.TestCase):
def test_gitea_token_is_secret(self):
self.assertIn("GITEA_TOKEN", _CHV_SECRET_ENV_NAMES)
self.assertIn("SUB2API_API_KEY", _CHV_SECRET_ENV_NAMES)
secrets, plain = split_secret_env(
{"GITEA_TOKEN": "tok", "GITEA_URL": "https://x",
"OTHER": "1"})
self.assertEqual(secrets, {"GITEA_TOKEN": "tok"})
self.assertEqual(plain, {"GITEA_URL": "https://x", "OTHER": "1"})
def test_split_guest_env_covers_all_six_keys(self):
guest, plain = split_guest_env(dict(FAKE_SYNC, OTHER="1"))
self.assertEqual(set(guest), set(FAKE_SYNC))
self.assertEqual(guest["GITEA_TOKEN"], "fake-token-123")
# 非密钥子集仍含同步普通变量(任务期 SSH 前缀照常带它们)。
self.assertIn("GITEA_URL", plain)
self.assertNotIn("GITEA_TOKEN", plain)
self.assertIn("OTHER", plain)
self.assertNotIn("OTHER", guest)
def test_remote_command_rejects_gitea_token(self):
with self.assertRaises(ValueError):
remote_command(["true"], extra_env={"GITEA_TOKEN": "tok"})
with self.assertRaises(ValueError):
build_ssh_command(2222, "/tmp/key", "root", None,
"/tmp/known_hosts", ["true"],
extra_env={"GITEA_TOKEN": "tok"})
# -- (3) hostkeys.write_seed -------------------------------------------
class WriteSeedGuestEnvTest(unittest.TestCase):
def test_guest_env_and_secret_merge_into_one_entry_no_runcmd(self):
with tempfile.TemporaryDirectory() as tmp:
seed_dir = os.path.join(tmp, "seed")
write_seed(seed_dir, "vm1", "PRIV", "ssh-ed25519 AAAAx",
secret_env={"SUB2API_API_KEY": "gw-key"},
guest_env={"GITEA_URL": "https://gitea.tose.top",
"GITEA_USER": "admin",
"SYNC_REPO": "admin/chv-sync-test",
"GITEA_TOKEN": "dup-should-lose"})
with open(os.path.join(seed_dir, "user-data")) as fh:
content = fh.read()
self.assertEqual(content.count("write_files:"), 1)
parsed = yaml.safe_load(content)
self.assertNotIn("runcmd", parsed)
self.assertEqual(len(parsed["write_files"]), 3)
text, perms = _guest_env_lines(content)
self.assertEqual(perms, "0600")
self.assertIn("GITEA_URL=https://gitea.tose.top", text)
self.assertIn("SUB2API_API_KEY=gw-key", text)
# 同名键以 secret_env 为准,不出现两行 GITEA_TOKEN。
self.assertEqual(text.count("GITEA_TOKEN="), 1)
self.assertIn("GITEA_TOKEN=dup-should-lose", text)
def test_guest_env_only_still_no_runcmd(self):
with tempfile.TemporaryDirectory() as tmp:
seed_dir = os.path.join(tmp, "seed")
write_seed(seed_dir, "vm1", "PRIV", "ssh-ed25519 AAAAx",
guest_env={"GITEA_URL": "https://gitea.tose.top"})
with open(os.path.join(seed_dir, "user-data")) as fh:
content = fh.read()
parsed = yaml.safe_load(content)
self.assertNotIn("runcmd", parsed)
self.assertEqual(len(parsed["write_files"]), 3)
# -- (4) 执行器 wake_vm -------------------------------------------------
class WakeGuestEnvCase(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.dir = self.tmp.name
self.binary = _write(os.path.join(self.dir, "cloud-hypervisor"))
self.ch_remote = _write(os.path.join(self.dir, "ch-remote"))
self.kernel = _write(os.path.join(self.dir, "vmlinux"))
self.rootfs = _write(os.path.join(self.dir, "disk.raw"),
b"R" * 100)
self.ssh_key = _write(os.path.join(self.dir, "id_chv"))
self.kvm = _write(os.path.join(self.dir, "kvm"))
self.cpuinfo = os.path.join(self.dir, "cpuinfo")
with open(self.cpuinfo, "w") as fh:
fh.write("flags : fpu vmx sse\n")
self.state_dir = os.path.join(self.dir, "state")
self.calls = []
self.audit = []
self.launches = []
def tearDown(self):
self.tmp.cleanup()
def _runner(self, cmd, timeout=None):
self.calls.append(list(cmd))
if cmd and cmd[0] == "ssh-keygen":
priv = cmd[cmd.index("-f") + 1]
with open(priv, "w") as fh:
fh.write("-----BEGIN OPENSSH PRIVATE KEY-----\nFAKE\n")
os.chmod(priv, 0o600)
with open(priv + ".pub", "w") as fh:
fh.write("ssh-ed25519 AAAAfake1 vm-key\n")
return FakeResult(0, "", "")
if cmd and cmd[0] == "genisoimage":
out = cmd[cmd.index("-output") + 1]
with open(out, "wb") as fh:
fh.write(b"ISO")
return FakeResult(0, "", "")
return FakeResult(0, "", "")
def _launcher(self, cmd):
self.launches.append(list(cmd))
return FakeProc(alive=True)
def make_exec(self, env_extra=None):
env = {"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
"CHV_HOST_KEY_MODE": "provisioned",
"CHV_SSH_FORWARD": "1"}
if env_extra:
env.update(env_extra)
ex = CloudHypervisorExecutor(
binary=self.binary, ch_remote=self.ch_remote,
kernel=self.kernel, rootfs=self.rootfs,
ssh_key=self.ssh_key, kvm_path=self.kvm,
cpuinfo_path=self.cpuinfo, state_dir=self.state_dir,
runner=self._runner, launcher=self._launcher,
api_client=lambda s, m, p, payload=None: (200, {}),
sleeper=lambda s: None, env=env)
ex.audit_callback = lambda e, t, d, k=None: self.audit.append(
(e, t.get("id") if isinstance(t, dict) else t, d, k))
return ex
def test_wake_with_task_gitea_sync_writes_all_six_to_seed(self):
ex = self.make_exec()
task = {"id": "sync1", "gitea_sync": dict(FAKE_SYNC)}
out = ex.wake_vm(task)
self.assertEqual(out["state"], "running")
text = _seed_user_text(self.state_dir, "chv-sync1")
self.assertNotIn("runcmd", yaml.safe_load(text))
guest_text, perms = _guest_env_lines(text)
self.assertEqual(perms, "0600")
for key, value in FAKE_SYNC.items():
self.assertIn("%s=%s" % (key, value), guest_text)
# seed.iso 已制盘且 0600。
seed_iso = os.path.join(self.state_dir, "chv-sync1", "seed.iso")
self.assertTrue(os.path.exists(seed_iso))
self.assertEqual(oct(os.stat(seed_iso).st_mode & 0o777), "0o600")
def test_wake_with_config_env_sync(self):
ex = self.make_exec(env_extra={
"CHV_GITEA_URL": "https://gitea.tose.top",
"CHV_GITEA_USER": "admin",
"CHV_GITEA_TOKEN": "fake-token-123",
"CHV_SYNC_REPO": "admin/chv-sync-test",
"CHV_SYNC_BRANCH": "main"})
self.assertEqual(ex.gitea_sync_env["SYNC_REPO"],
"admin/chv-sync-test")
out = ex.wake_vm({"id": "sync2"})
self.assertEqual(out["state"], "running")
guest_text, _ = _guest_env_lines(
_seed_user_text(self.state_dir, "chv-sync2"))
self.assertIn("GITEA_TOKEN=fake-token-123", guest_text)
self.assertIn("SYNC_BRANCH=main", guest_text)
def test_wake_partial_sync_fails_closed_before_launch(self):
ex = self.make_exec()
task = {"id": "sync3", "gitea_sync": {
"GITEA_URL": "https://gitea.tose.top",
"SYNC_REPO": "admin/chv-sync-test"}}
with self.assertRaises(ChvFailure) as ctx:
ex.wake_vm(task)
self.assertEqual(ctx.exception.kind, "gitea_sync_incomplete")
self.assertEqual(self.launches, [])
self.assertFalse(os.path.exists(
os.path.join(self.state_dir, "chv-sync3", "seed.iso")))
kinds = [a[0] for a in self.audit]
self.assertIn("vm_failed", kinds)
def test_wake_partial_sync_from_config_also_fails_closed(self):
ex = self.make_exec(env_extra={
"CHV_GITEA_URL": "https://gitea.tose.top"})
with self.assertRaises(ChvFailure) as ctx:
ex.wake_vm({"id": "sync4"})
self.assertEqual(ctx.exception.kind, "gitea_sync_incomplete")
self.assertEqual(self.launches, [])
def test_wake_without_sync_writes_no_guest_env(self):
ex = self.make_exec()
out = ex.wake_vm({"id": "plain1"})
self.assertEqual(out["state"], "running")
parsed = yaml.safe_load(
_seed_user_text(self.state_dir, "chv-plain1"))
self.assertEqual(len(parsed["write_files"]), 2)
def test_build_ssh_command_after_wake_sources_guest_env(self):
ex = self.make_exec()
ex.wake_vm({"id": "sync5", "gitea_sync": dict(FAKE_SYNC)})
cmd = ex.build_ssh_command(
{"id": "sync5"}, ["true"],
extra_env={"GITEA_TOKEN": "fake-token-123"})
joined = " ".join(cmd)
self.assertNotIn("fake-token-123", joined)
self.assertIn("/etc/chv/guest-env", joined)
# 未 wake 投递的 VM 用密钥即拒绝。
with self.assertRaises(ChvFailure) as ctx:
ex.build_ssh_command(
{"id": "other"}, ["true"],
extra_env={"GITEA_TOKEN": "fake-token-123"})
self.assertEqual(ctx.exception.kind, "secret_env_not_delivered")
def test_wake_keyscan_mode_with_token_fails_closed(self):
ex = self.make_exec(env_extra={"CHV_HOST_KEY_MODE": "keyscan"})
with self.assertRaises(ChvFailure):
ex.wake_vm({"id": "sync6", "gitea_sync": dict(FAKE_SYNC)})
self.assertEqual(self.launches, [])
# -- (5) worker ---------------------------------------------------------
class WorkerGiteaSyncTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
from cloud.server import init_db
init_db(self.db_path)
def tearDown(self):
self.tmp.cleanup()
def _worker(self, executor, env_extra=None):
env = {"XCM_API_KEY": "test-placeholder"}
if env_extra:
env.update(env_extra)
return Worker(self.db_path, "/fake/host.py",
executor=executor, env=env)
def test_gitea_sync_env_from_worker_env_and_task(self):
worker = self._worker(None, env_extra={
"CHV_GITEA_URL": "https://gitea.tose.top",
"CHV_GITEA_USER": "admin",
"CHV_GITEA_TOKEN": "env-tok",
"CHV_SYNC_REPO": "admin/chv-sync-test"})
out = worker._gitea_sync_env({"id": "t1"})
self.assertEqual(out["GITEA_TOKEN"], "env-tok")
out2 = worker._gitea_sync_env(
{"id": "t1", "gitea_sync": {"GITEA_TOKEN": "task-tok"}})
self.assertEqual(out2["GITEA_TOKEN"], "task-tok")
self.assertEqual(out2["SYNC_REPO"], "admin/chv-sync-test")
worker2 = self._worker(None)
self.assertEqual(worker2._gitea_sync_env({"id": "t1"}), {})
def test_run_attempt_merges_sync_env_only_for_chv(self):
import tempfile as _tf
case_dir = _tf.mkdtemp()
binary = _write(os.path.join(case_dir, "cloud-hypervisor"))
ch_remote = _write(os.path.join(case_dir, "ch-remote"))
kernel = _write(os.path.join(case_dir, "vmlinux"))
rootfs = _write(os.path.join(case_dir, "disk.raw"), b"R" * 100)
ssh_key = _write(os.path.join(case_dir, "id_chv"))
kvm = _write(os.path.join(case_dir, "kvm"))
cpuinfo = os.path.join(case_dir, "cpuinfo")
with open(cpuinfo, "w") as fh:
fh.write("flags : fpu vmx sse\n")
ex = CloudHypervisorExecutor(
binary=binary, ch_remote=ch_remote, kernel=kernel,
rootfs=rootfs, ssh_key=ssh_key, kvm_path=kvm,
cpuinfo_path=cpuinfo,
state_dir=os.path.join(case_dir, "state"),
runner=lambda cmd, timeout=None: FakeResult(0, "", ""),
launcher=lambda cmd: FakeProc(alive=True),
api_client=lambda s, m, p, payload=None: (200, {}),
sleeper=lambda s: None,
env={"CHV_EGRESS_PROXY": "http://10.0.0.1:8888",
"CHV_HOST_KEY_MODE": "provisioned",
"CHV_SSH_FORWARD": "1"})
seen = {}
orig_dispatch = ex.dispatch_task
def fake_dispatch(task, extra_env=None, model_provider=None):
seen["extra_env"] = dict(extra_env or {})
return {"executor": "cloud-hypervisor", "task_id": task["id"],
"status": "completed", "stdout": "", "stderr": "",
"returncode": 0, "host_status": "completed",
"usage": None}
ex.dispatch_task = fake_dispatch
worker = self._worker(ex, env_extra={
"CHV_GITEA_URL": "https://gitea.tose.top",
"CHV_GITEA_USER": "admin",
"CHV_GITEA_TOKEN": "fake-token-123",
"CHV_SYNC_REPO": "admin/chv-sync-test"})
worker._ensure_gateway = lambda task: (None, {})
task = {"id": "w1", "project_id": "p1", "prompt": "做事"}
result, error = worker._run_attempt(task)
self.assertIsNone(error)
self.assertEqual(result["status"], "completed")
self.assertEqual(seen["extra_env"]["GITEA_TOKEN"],
"fake-token-123")
self.assertEqual(seen["extra_env"]["SYNC_REPO"],
"admin/chv-sync-test")
worker.close()
if __name__ == "__main__":
unittest.main()