host: phase2 daemon + health + auto-recovery (restore, recover, double-fork)
This commit is contained in:
1 parent
e9735d1fad
commit
240eaacf50
4 files changed
+434
-1
No files matched your search
@@ -297,6 +297,82 @@ class ServerRestartTest(unittest.TestCase):
|
||||
pass
|
||||
|
||||
|
||||
class HealthRestoreRecoverTest(unittest.TestCase):
|
||||
class FakeServer:
|
||||
def initialize(self):
|
||||
return {}
|
||||
|
||||
def list_threads(self):
|
||||
return {"data": []}
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
def test_corrupt_state_detected_and_restored_from_bak(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "host-state.json")
|
||||
good = {"tasks": {"a": {"status": "completed"}},
|
||||
"automations": [], "version": 3}
|
||||
with open(path + ".bak", "w") as f:
|
||||
json.dump(good, f)
|
||||
with open(path, "w") as f:
|
||||
f.write("{not json!!!")
|
||||
strict, err = host.load_state_strict(path)
|
||||
self.assertIsNone(strict)
|
||||
self.assertIsNotNone(err)
|
||||
ok, msg = host.restore_state(path)
|
||||
self.assertTrue(ok, msg)
|
||||
with open(path) as f:
|
||||
restored = json.load(f)
|
||||
self.assertEqual(restored["version"], 3)
|
||||
self.assertTrue(os.path.exists(path + ".corrupt.") or
|
||||
any(fn.startswith("host-state.json.corrupt.")
|
||||
for fn in os.listdir(tmp)))
|
||||
|
||||
def test_restore_without_backup_fails_without_overwriting(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "host-state.json")
|
||||
with open(path, "w") as f:
|
||||
f.write("{bad")
|
||||
ok, msg = host.restore_state(path)
|
||||
self.assertFalse(ok)
|
||||
# 坏文件被改名留底,主文件已不在(不是被空 state 覆盖)
|
||||
self.assertFalse(os.path.exists(path))
|
||||
|
||||
def test_recover_marks_old_running_only(self):
|
||||
import time as _time
|
||||
now = _time.time()
|
||||
state = {"tasks": {
|
||||
"old": {"status": "running", "created_at": now - 7200,
|
||||
"last_turn_at": now - 7200},
|
||||
"fresh": {"status": "running", "created_at": now - 60,
|
||||
"last_turn_at": now - 60},
|
||||
"done": {"status": "completed", "created_at": now - 7200},
|
||||
}, "automations": []}
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "host-state.json")
|
||||
recovered = host.recover_interrupted(state, path,
|
||||
older_than=3600, cfg={})
|
||||
self.assertEqual(recovered, ["old"])
|
||||
self.assertEqual(state["tasks"]["old"]["status"], "interrupted")
|
||||
self.assertEqual(state["tasks"]["fresh"]["status"], "running")
|
||||
|
||||
def test_health_with_fake_server(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = os.path.join(tmp, "host-state.json")
|
||||
with open(path, "w") as f:
|
||||
json.dump({"tasks": {}, "automations": [], "version": 1}, f)
|
||||
with open(path + ".bak", "w") as f:
|
||||
json.dump({"tasks": {}, "automations": [], "version": 1}, f)
|
||||
report = host.run_health_check(
|
||||
path, {}, "/bin/true", tmp, "deny",
|
||||
server_factory=lambda: self.FakeServer())
|
||||
self.assertTrue(report["ok"], report)
|
||||
self.assertTrue(report["checks"]["state_valid"])
|
||||
self.assertTrue(report["checks"]["backup_exists"])
|
||||
self.assertTrue(report["checks"]["server_ok"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in new issue
Block a user