fix(chv-hybrid): p2 第三轮:attach 轮询等挂盘落地 + detach 走 sudo 窄规则 umount
ci / test (push) Failing after 47m51s
ci / test (push) Failing after 47m51s
第 2 轮复测(0dc8328)p2 仍 FAIL 的两处根因: 1. 挂盘时序竞态:add-disk 返回 06:59:10,root hotplug 路径 (cloud-init-hotplugd → cloud-final runcmd → chv-data-mount) 06:59:14–16 才挂上;旧 attach 以 chv 调 chv-data-mount 并判 rc==0,但 /dev/vd? 属 root:disk、chv 的 blkid/mkfs 必败, 脚本又静默 exit 0,rc 检查是空洞的成功假象。 2. 卸载权限:挂载是 root 做的,chv 的 plain umount 恒 rc=32 must be superuser,release 按现写法永远失败、VM 留置。 修法选 (a) 轮询,不选 (b) root 特权挂载:挂盘本身已有 root 路径在跑,host 侧不另开新的 root 挂载通道(新特权面只扩 攻击面);轮询以 chv 只读判定挂载态,零新增特权。 - attach:删 chv 调 chv-data-mount,改为轮询 mountpoint -q /home/chv/workspace && test -w(默认 60 秒/2 秒,可经 CHV_DATA_MOUNT_TIMEOUT_SECONDS / CHV_DATA_MOUNT_INTERVAL_SECONDS 注入短值);超时记 data_disk_mount_timeout 审计并抛同名 kind,不写归属标记。attach 返回成功 = 挂盘真实发生。 resume_seconds 因此真实包含挂盘等待(§2.1 口径),≤2 秒 目标不改口径、不改目标。 - detach:改走 sudo -n /usr/bin/umount /home/chv/workspace; 失败即 False,release 留置 VM 不回收(fail-closed)。 - chv-data-mount.sh:盘已找到后 mkfs/mount 失败改 exit 1 (原全部失败路径 exit 0);无盘仍 exit 0(启动期 no-op, 不挡 boot)。本脚本只由 root 跑(service/runcmd)。 - build.sh:apt 加 sudo;写 /etc/sudoers.d/chv-data-umount 窄规则(chv 仅能以 root 跑 umount /home/chv/workspace 这一条精确 argv,NOPASSWD,无 shell/其它命令/挂载权), 0440 + visudo -cf 作构建门;buildinfo/lockfile 登记 sudo 与 chv-data-umount。 特权边界:本轮唯一新增提权面就是上述一条 sudoers 规则; attach 零提权(只读轮询),挂载权仍只在 root 的 hotplug/service 路径手里。旧镜像(无 sudo/无规则)上 detach 返回 False、release 留置,须重建镜像才恢复回收。 测试:新增 10 项(attach 等三次轮询才成功、超时 loud 失败、 永不以 chv 调 chv-data-mount、detach 精确 sudo argv、 sudo 失败 False、卸载失败 release 留置、客机脚本 exit 码、 build.sh sudo/窄规则/buildinfo 登记);另修 2 处旧断言 (attach 不再调挂载脚本)与 stage4 lockfile 夹具(补 sudo dpkg 条目),并把 attach 热插段抽为 _hotplug_tenant_disk (50 行函数上限)。全量 Ran 4688,OK (skipped=15,均为 环境 skip:无 PG DSN/驱动、缺 SearXNG_URL、fallback state 文件在位)。
This commit is contained in:
1 parent
0dc8328ecc
commit
a859d731d5
8 files changed
+351
-37
No files matched your search
@@ -33,6 +33,8 @@ try:
|
||||
write_persistent_vm,
|
||||
)
|
||||
from cloud.chv.executor import (
|
||||
DATA_MOUNT_POLL_INTERVAL_SECONDS,
|
||||
DATA_MOUNT_TIMEOUT_SECONDS,
|
||||
HYBRID_COLD_RESTORE_TARGET_SECONDS,
|
||||
HYBRID_QUEUE_TIMEOUT_SECONDS,
|
||||
HYBRID_SNAPSHOT_PER_TENANT,
|
||||
@@ -51,6 +53,8 @@ except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.p
|
||||
write_persistent_vm,
|
||||
)
|
||||
from cloud.chv.executor import (
|
||||
DATA_MOUNT_POLL_INTERVAL_SECONDS,
|
||||
DATA_MOUNT_TIMEOUT_SECONDS,
|
||||
HYBRID_COLD_RESTORE_TARGET_SECONDS,
|
||||
HYBRID_QUEUE_TIMEOUT_SECONDS,
|
||||
HYBRID_SNAPSHOT_PER_TENANT,
|
||||
@@ -492,9 +496,15 @@ class DataDiskAttachTest(HybridACase):
|
||||
# CHV v54 已移除盘格式自动探测:热插盘 body 须与 create
|
||||
# 时盘同款带 image_type(租户数据盘是 raw 稀疏盘 → Raw)。
|
||||
self.assertEqual(payload["image_type"], "Raw")
|
||||
# 客内挂载脚本被调过(chv-data-mount)。
|
||||
# p2 第三轮:attach 不再以 chv 调 chv-data-mount(那条
|
||||
# 路恒因权限失败、脚本又静默 exit 0),改为轮询等
|
||||
# root hotplug 路径把盘挂上:ssh 里须出现 mountpoint
|
||||
# 轮询,且绝不能出现 chv-data-mount 调用。
|
||||
self.assertTrue(any(
|
||||
"/usr/local/bin/chv-data-mount" in call
|
||||
"mountpoint -q /home/chv/workspace" in " ".join(call)
|
||||
for call in self.calls if call and call[0] == "ssh"))
|
||||
self.assertFalse(any(
|
||||
"/usr/local/bin/chv-data-mount" in " ".join(call)
|
||||
for call in self.calls if call and call[0] == "ssh"))
|
||||
self.assertEqual(
|
||||
read_vm_tenant(self.state_dir, "chv-pool-0"), "tenant-a")
|
||||
@@ -519,9 +529,9 @@ class DataDiskAttachTest(HybridACase):
|
||||
self.assertEqual(len(details), 1)
|
||||
self.assertIn("Image type required for disk", details[0])
|
||||
self.assertIn("status=400", details[0])
|
||||
# 挂载脚本没跑、归属标记没写。
|
||||
# 挂载轮询没跑(add-disk 已失败)、归属标记没写。
|
||||
self.assertFalse(any(
|
||||
"/usr/local/bin/chv-data-mount" in call
|
||||
"mountpoint -q /home/chv/workspace" in " ".join(call)
|
||||
for call in self.calls if call and call[0] == "ssh"))
|
||||
self.assertIsNone(read_vm_tenant(self.state_dir, "chv-pool-0"))
|
||||
|
||||
@@ -761,5 +771,183 @@ class DispatchWiringTest(HybridACase):
|
||||
self.assertNotIn("route", out)
|
||||
|
||||
|
||||
# -- p2 第三轮:挂盘等待 + 卸载提权(2026-11-09 第 2 轮复测)---------
|
||||
class DataMountWaitAndPrivilegedUmountTest(HybridACase):
|
||||
"""覆盖第 2 轮复测的两个根因:
|
||||
- 挂盘时序竞态:attach 须等客内 workspace 真挂上才返回
|
||||
成功,超时 loud 失败;且绝不再以 chv 调 chv-data-mount
|
||||
并判 rc==0(静默 exit 0 的成功假象)。
|
||||
- 卸载权限:detach 须走 sudo -n 窄规则 umount,plain
|
||||
umount 恒 rc=32 的老路不得再出现;提权失败即 False,
|
||||
release 留置 VM 不回收。
|
||||
只 mock 成功/失败路径的 SSH 返回码,不启动真实 VM。"""
|
||||
|
||||
def _poll_calls(self):
|
||||
return [call for call in self.calls
|
||||
if call and call[0] == "ssh"
|
||||
and "mountpoint -q /home/chv/workspace"
|
||||
in " ".join(call)]
|
||||
|
||||
def _mount_runner(self, succeed_after):
|
||||
"""轮询专用 runner:前 succeed_after-1 次轮询报
|
||||
未挂上(rc=1,第 2 轮真机:attach 返回时盘尚未挂),
|
||||
第 succeed_after 次起报已挂上;非轮询 ssh 一律 rc=0。
|
||||
succeed_after=None 即永远未挂上。"""
|
||||
state = {"n": 0}
|
||||
|
||||
def runner(cmd, timeout=None):
|
||||
self.calls.append(list(cmd))
|
||||
if cmd and cmd[0] == "ssh" and any(
|
||||
"mountpoint -q" in str(tok) for tok in cmd):
|
||||
state["n"] += 1
|
||||
if succeed_after is not None \
|
||||
and state["n"] >= succeed_after:
|
||||
return FakeResult(0, "", "")
|
||||
return FakeResult(1, "", "")
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
return runner
|
||||
|
||||
def test_mount_poll_defaults(self):
|
||||
self.assertEqual(DATA_MOUNT_TIMEOUT_SECONDS, 60)
|
||||
self.assertEqual(DATA_MOUNT_POLL_INTERVAL_SECONDS, 2)
|
||||
|
||||
def test_attach_waits_for_mount_before_returning(self):
|
||||
# 真机时序复现:add-disk 成功后前两次轮询盘未挂,
|
||||
# 第三次才挂上——attach 必须等到第三次才返回成功,
|
||||
# 而不是 add-disk 一成功就返回。
|
||||
ex = self.make_exec(
|
||||
runner=self._mount_runner(succeed_after=3),
|
||||
env_extra={"CHV_DATA_MOUNT_TIMEOUT_SECONDS": "10",
|
||||
"CHV_DATA_MOUNT_INTERVAL_SECONDS": "1"})
|
||||
out = ex.attach_tenant_data_disk(
|
||||
self._task(project="tenant-a"), "chv-pool-0")
|
||||
self.assertEqual(out["tenant_id"], "tenant-a")
|
||||
self.assertEqual(len(self._poll_calls()), 3)
|
||||
self.assertEqual(
|
||||
read_vm_tenant(self.state_dir, "chv-pool-0"), "tenant-a")
|
||||
|
||||
def test_attach_mount_timeout_fails_loud(self):
|
||||
# 盘一直不挂:超时后抛 data_disk_mount_timeout(kind
|
||||
# 同名),记同名审计事件,不写归属标记,不谎报成功。
|
||||
ex = self.make_exec(
|
||||
runner=self._mount_runner(succeed_after=None),
|
||||
env_extra={"CHV_DATA_MOUNT_TIMEOUT_SECONDS": "3",
|
||||
"CHV_DATA_MOUNT_INTERVAL_SECONDS": "1"})
|
||||
with self.assertRaises(ChvFailure) as ctx:
|
||||
ex.attach_tenant_data_disk(
|
||||
self._task(project="tenant-a"), "chv-pool-0")
|
||||
self.assertEqual(ctx.exception.kind, "data_disk_mount_timeout")
|
||||
self.assertIn("3.0", str(ctx.exception))
|
||||
self.assertEqual(len(self._poll_calls()), 3)
|
||||
details = self._event_details("data_disk_mount_timeout")
|
||||
self.assertEqual(len(details), 1)
|
||||
self.assertIn("chv-pool-0", details[0])
|
||||
self.assertIsNone(read_vm_tenant(self.state_dir, "chv-pool-0"))
|
||||
|
||||
def test_attach_never_runs_data_mount_as_chv(self):
|
||||
# 回归:静默 exit 0 的成功假象不得再出现——attach 全程
|
||||
# 不得以 chv 身份调 /usr/local/bin/chv-data-mount。
|
||||
ex = self.make_exec()
|
||||
ex.attach_tenant_data_disk(
|
||||
self._task(project="tenant-a"), "chv-pool-0")
|
||||
self.assertFalse(any(
|
||||
"/usr/local/bin/chv-data-mount" in " ".join(call)
|
||||
for call in self.calls if call and call[0] == "ssh"))
|
||||
|
||||
def test_detach_uses_privileged_umount(self):
|
||||
ex = self.make_exec()
|
||||
self.assertTrue(ex.detach_tenant_data_disk(
|
||||
self._task(project="tenant-a"), "chv-pool-0"))
|
||||
sudo_calls = [call for call in self.calls
|
||||
if call and call[0] == "ssh"
|
||||
and "sudo" in call]
|
||||
self.assertEqual(len(sudo_calls), 1)
|
||||
call = sudo_calls[0]
|
||||
# 精确 argv:sudo -n /usr/bin/umount /home/chv/workspace,
|
||||
# 且同一调用里不得出现不带 sudo 的 plain umount。
|
||||
self.assertIn("sudo", call)
|
||||
self.assertIn("-n", call)
|
||||
self.assertIn("/usr/bin/umount", call)
|
||||
self.assertIn("/home/chv/workspace", call)
|
||||
self.assertLess(call.index("sudo"),
|
||||
call.index("/usr/bin/umount"))
|
||||
self.assertFalse(any(
|
||||
call and call[0] == "ssh" and "sudo" not in call
|
||||
and "umount" in call for call in self.calls))
|
||||
|
||||
def test_detach_privilege_failure_returns_false(self):
|
||||
# sudo 规则缺失/镜像未重建(sudo 报 rc=1)即 False,
|
||||
# 不谎报卸载成功。
|
||||
def runner(cmd, timeout=None):
|
||||
self.calls.append(list(cmd))
|
||||
if cmd and cmd[0] == "ssh" and "sudo" in cmd:
|
||||
return FakeResult(1, "", "sudo: not found")
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
ex = self.make_exec(runner=runner)
|
||||
self.assertFalse(ex.detach_tenant_data_disk(
|
||||
self._task(project="tenant-a"), "chv-pool-0"))
|
||||
|
||||
def test_release_retains_vm_when_unmount_fails(self):
|
||||
# 第 2 轮根因 2 的终局:卸载失败 → release 返回
|
||||
# released=False + reason=unmount_failed,记审计,
|
||||
# VM 不销毁(不丢改动后静默回收)。
|
||||
def runner(cmd, timeout=None):
|
||||
self.calls.append(list(cmd))
|
||||
if cmd and cmd[0] == "ssh" and "sudo" in cmd:
|
||||
return FakeResult(32, "",
|
||||
"umount: must be superuser")
|
||||
return FakeResult(0, "", "")
|
||||
|
||||
ex = self.make_exec(runner=runner)
|
||||
out = ex.release_pool_vm(
|
||||
self._task(project="tenant-a"), "chv-pool-0")
|
||||
self.assertFalse(out["released"])
|
||||
self.assertEqual(out["reason"], "unmount_failed")
|
||||
self.assertIn("warm_pool_release_failed", self._event_names())
|
||||
details = self._event_details("warm_pool_release_failed")
|
||||
self.assertTrue(any("unmount_failed" in d for d in details))
|
||||
|
||||
|
||||
class DataMountGuestWiringTest(unittest.TestCase):
|
||||
"""客机侧接线(纯文本断言,不启动 VM):chv-data-mount
|
||||
失败 loud、build.sh 装 sudo + 写窄规则。"""
|
||||
|
||||
def setUp(self):
|
||||
self.root = Path(__file__).resolve().parents[2]
|
||||
|
||||
def test_data_mount_failures_are_loud(self):
|
||||
text = (self.root / "chv-guest"
|
||||
/ "chv-data-mount.sh").read_text()
|
||||
# 盘已找到后 mkfs/mount 失败必须 exit 1,不再静默 0。
|
||||
self.assertIn('mkfs failed on $DEV"; exit 1', text)
|
||||
self.assertIn('mount failed"; exit 1', text)
|
||||
# 无盘仍是启动期 no-op:exit 0,不挡 boot。
|
||||
self.assertIn('no data disk found, skip"; exit 0', text)
|
||||
|
||||
def test_build_sh_installs_sudo_and_narrow_rule(self):
|
||||
text = (self.root / "chv-guest" / "build.sh").read_text()
|
||||
self.assertIn("iptables sudo", text)
|
||||
self.assertIn(
|
||||
"chv ALL=(root) NOPASSWD: "
|
||||
"/usr/bin/umount /home/chv/workspace", text)
|
||||
self.assertIn("/etc/sudoers.d/chv-data-umount", text)
|
||||
self.assertIn("chmod 0440 /etc/sudoers.d/chv-data-umount",
|
||||
text)
|
||||
self.assertIn("visudo -cf /etc/sudoers.d/chv-data-umount",
|
||||
text)
|
||||
# 窄规则只放行 umount 这一条:不得出现放宽到 mount、
|
||||
# ALL 命令或带 shell 的写法。
|
||||
idx = text.find("/etc/sudoers.d/chv-data-umount")
|
||||
block = text[max(0, idx - 1500):idx + 500]
|
||||
self.assertNotIn("NOPASSWD: ALL", block)
|
||||
self.assertNotIn("/usr/bin/mount", block)
|
||||
|
||||
def test_buildinfo_lists_sudo_and_umount_rule(self):
|
||||
text = (self.root / "chv-guest" / "buildinfo.py").read_text()
|
||||
self.assertIn('"sudo", "chv-data-umount"', text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -533,6 +533,7 @@ class BuildWiringTest(unittest.TestCase):
|
||||
"openssh-server 1:9.6p1-3ubuntu13",
|
||||
"python3 3.12.3-0ubuntu2",
|
||||
"python3-pip 24.0+dfsg-1ubuntu1",
|
||||
"sudo 1.9.15p5-3ubuntu5",
|
||||
"systemd-container 255.4-1ubuntu8",
|
||||
"xz-utils 5.6.1+really5.4.5-1build0.1"],
|
||||
"pip": ["playwright==1.49.1",
|
||||
|
||||
Reference in new issue
Block a user