host
9ea0a05fb1
2026-10-29 task 2 lost its host-state entry (version 390 -> single digits) while running under Docker/gVisor. Root cause, proven live: - save_state tmp name contained only pid, and every containerized host.py runs as pid 1, so concurrent writers shared one tmp file and interleaved writes produced corrupt JSON; - fcntl.flock does not exclude across runsc containers sharing a bind-mounted lock file (both entered at 0.0s in a live probe); - _read_state_file silently treated corrupt JSON as empty state, so the next save legitimately rewrote the file with only the surviving writer's tasks. Fix: - tmp name now hostname + pid + thread + uuid4 (backup tmp same); - mkdir atomic lock replaces flock (atomic on the host fs, holds across bind mounts/gofer), with stale-lock recovery after 120s and a loud timeout instead of unlocked writes; reads need no lock because publication is tmp + os.replace; - save path uses a strict disk read: corrupt JSON, non-object top level, or wrong-shaped tasks/automations now raise instead of overwriting (recovery stays with restore_state + .bak). Tests: tests/test_host_state_lock.py, 7 items (tmp uniqueness in process and across processes, 8-process concurrent saves all survive with version 8, stale lock recovery, fresh lock timeout raises, corrupt/wrong-shaped disk refuses to overwrite). Full suites: cloud 1168 (skipped 4), host 104, deploy 39, all green.
124 lines
4.6 KiB
Python
124 lines
4.6 KiB
Python
"""host-state 并发保存回归测试（2026-10-29 断点丢失实弹）。
|
|
|
|
旧写法两处同时失效：tmp 文件名只含 pid（容器内 host.py 全是
|
|
pid 1，并发保存写同一个 tmp 交错出坏 JSON）；flock 在 runsc
|
|
容器间不互斥（实测双双 0 秒拿到排他锁）。坏 JSON 又被
|
|
_read_state_file 静默当成空 state，下一次保存合法地重写，
|
|
全部任务条目一起丢、版本从 390 跌回个位数。
|
|
|
|
本文件锁死三条修法：tmp 名每个写者唯一、mkdir 原子锁跨进程
|
|
互斥、磁盘已损坏时保存直接拒绝覆盖。
|
|
"""
|
|
|
|
import glob
|
|
import json
|
|
import multiprocessing
|
|
import os
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[1]
|
|
sys.path.insert(0, str(REPO_ROOT))
|
|
|
|
import host
|
|
|
|
|
|
def _save_one(args):
|
|
path, name = args
|
|
state = {"tasks": {name: {"status": "running"}}, "automations": []}
|
|
host.save_state(path, state)
|
|
return name
|
|
|
|
|
|
def _tmp_name(_ignored):
|
|
return host._state_tmp_path("/tmp/x/host-state.json")
|
|
|
|
|
|
class StateTmpNameTest(unittest.TestCase):
|
|
def test_tmp_names_unique_in_process(self):
|
|
names = {host._state_tmp_path("/tmp/x/host-state.json")
|
|
for _ in range(100)}
|
|
self.assertEqual(len(names), 100)
|
|
|
|
def test_tmp_names_unique_across_processes(self):
|
|
ctx = multiprocessing.get_context("fork")
|
|
with ctx.Pool(8) as pool:
|
|
names = pool.map(_tmp_name, range(8))
|
|
self.assertEqual(len(set(names)), 8)
|
|
|
|
|
|
class StateMkdirLockTest(unittest.TestCase):
|
|
def test_concurrent_saves_all_tasks_survive(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "host-state.json")
|
|
ctx = multiprocessing.get_context("fork")
|
|
names = [f"task-{i}" for i in range(8)]
|
|
with ctx.Pool(8) as pool:
|
|
pool.map(_save_one, [(path, n) for n in names])
|
|
with open(path, encoding="utf-8") as f:
|
|
data = json.load(f)
|
|
self.assertEqual(set(data["tasks"]), set(names))
|
|
self.assertEqual(data["version"], 8)
|
|
self.assertFalse(glob.glob(path + ".tmp.*"))
|
|
self.assertFalse(os.path.exists(path + ".lockdir"))
|
|
|
|
def test_stale_lock_is_recovered(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "host-state.json")
|
|
lock_dir = path + ".lockdir"
|
|
os.mkdir(lock_dir)
|
|
old = time.time() - 1000
|
|
os.utime(lock_dir, (old, old))
|
|
host.save_state(path, {"tasks": {"a": {}}, "automations": []})
|
|
self.assertFalse(os.path.exists(lock_dir))
|
|
with open(path, encoding="utf-8") as f:
|
|
self.assertIn("a", json.load(f)["tasks"])
|
|
|
|
def test_fresh_lock_timeout_raises_instead_of_writing_unlocked(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "host-state.json")
|
|
os.mkdir(path + ".lockdir")
|
|
old_timeout = host._STATE_LOCK_TIMEOUT
|
|
host._STATE_LOCK_TIMEOUT = 0.3
|
|
try:
|
|
with self.assertRaises(RuntimeError):
|
|
host.save_state(path, {"tasks": {"a": {}},
|
|
"automations": []})
|
|
finally:
|
|
host._STATE_LOCK_TIMEOUT = old_timeout
|
|
self.assertFalse(os.path.exists(path))
|
|
|
|
|
|
class StateCorruptRefusalTest(unittest.TestCase):
|
|
def test_corrupt_disk_refuses_to_overwrite(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "host-state.json")
|
|
garbage = b"{not json, torn write"
|
|
with open(path, "wb") as f:
|
|
f.write(garbage)
|
|
with self.assertRaises(RuntimeError):
|
|
host.save_state(path, {"tasks": {"a": {}},
|
|
"automations": []})
|
|
with open(path, "rb") as f:
|
|
self.assertEqual(f.read(), garbage)
|
|
self.assertFalse(os.path.exists(path + ".bak"))
|
|
self.assertFalse(os.path.exists(path + ".lockdir"))
|
|
|
|
def test_wrong_shaped_disk_refuses_to_overwrite(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
path = os.path.join(tmp, "host-state.json")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
json.dump({"tasks": ["not", "an", "object"],
|
|
"automations": []}, f)
|
|
with self.assertRaises(RuntimeError):
|
|
host.save_state(path, {"tasks": {"a": {}},
|
|
"automations": []})
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|