feat(chv): 阶段 2 客机运行时三分 daemon/execd/syncd + restore 留 seed.iso + tap 退避
- 客机三分（chv-guest/chv-daemon.py、chv-execd.py、chv-syncd.py， 仅标准库；systemd Restart=always；build.sh 拷入 + 单元 + enable； buildinfo/check/generate lockfile 登记）： daemon=心跳/资源上报/快照配合（prepare-snapshot 问 execd， 有运行中任务或联系不上即拒绝）；execd=shell/文件读写/codex 托管（chv-cell 起 host.py，setsid 脱离，任务登记落盘， 重启按 pid 认领；SO_PEERCRED 只认 uid 0/1000；codex env 白名单）；syncd=常驻循环只调 chv-sync pull/push，规则一条 不改（先 pull --ff-only 再 push，分叉即停、不 --force、 不 rebase；仓不存在记 needs-boot 不擅自 clone）。 host.py 仍是唯一任务/codex 引擎，broker/browser/cell 分工 不变，不另起第二套 agent。 - 阶段 1 遗留修复：restore_vm 失败路径 stop_vm(keep_disk=True)， seed.iso 与任务盘都留，重试有输入；vm.boot/restore 的 tap 触点遇 Resource busy 等 tap 空闲后重试一次（_wait_tap_idle 看 ip link state，_ch_remote 报错带 stderr 尾巴以便识别）； 诊断不用 pgrep -x（15 字符上限）。 - 单测 cloud/tests/test_chv_opt_stage2.py 25 项；P2 干跑夹具 与 enable 行断言随构建接线更新。 - 全量 cloud/tests 2001 全过（skipped=8）。
This commit is contained in:
host committed
1 parent
f809014bef
commit
9d77ae55a7
10 files changed
+1932
-14
No files matched your search
+16
-1
@@ -186,6 +186,14 @@ for p2_script in chv-cell.py chv-broker.py chv-browser.py; do
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
# feat-chv-opt-landing（阶段 2）：客机运行时三分——daemon /
|
||||
# execd / syncd 三脚本缺一即失败（与 chv-sync.sh 同款门）。
|
||||
for stage2_script in chv-daemon.py chv-execd.py chv-syncd.py; do
|
||||
if [ ! -f "$SCRIPT_DIR/$stage2_script" ]; then
|
||||
echo "缺少 $stage2_script（阶段 2 客机服务）：$SCRIPT_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
if [ "$DRY_RUN" = "0" ]; then
|
||||
if [ ! -e /dev/kvm ]; then
|
||||
echo "CHV_GUEST_BUILD needs a KVM host: /dev/kvm 不存在" >&2
|
||||
@@ -294,6 +302,10 @@ VC_ARGS=(-a "$OUT/disk.raw"
|
||||
--copy-in "$SCRIPT_DIR/chv-broker.py:/usr/local/bin/"
|
||||
--copy-in "$SCRIPT_DIR/chv-browser.py:/usr/local/bin/"
|
||||
--run-command 'mv /usr/local/bin/chv-cell.py /usr/local/bin/chv-cell && mv /usr/local/bin/chv-broker.py /usr/local/bin/chv-broker && mv /usr/local/bin/chv-browser.py /usr/local/bin/chv-browser && chmod +x /usr/local/bin/chv-cell /usr/local/bin/chv-broker /usr/local/bin/chv-browser'
|
||||
--copy-in "$SCRIPT_DIR/chv-daemon.py:/usr/local/bin/"
|
||||
--copy-in "$SCRIPT_DIR/chv-execd.py:/usr/local/bin/"
|
||||
--copy-in "$SCRIPT_DIR/chv-syncd.py:/usr/local/bin/"
|
||||
--run-command 'mv /usr/local/bin/chv-daemon.py /usr/local/bin/chv-daemon && mv /usr/local/bin/chv-execd.py /usr/local/bin/chv-execd && mv /usr/local/bin/chv-syncd.py /usr/local/bin/chv-syncd && chmod +x /usr/local/bin/chv-daemon /usr/local/bin/chv-execd /usr/local/bin/chv-syncd'
|
||||
--run-command 'printf "runcmd:\n - systemctl restart ssh || service ssh restart || true\n - test -f /etc/chv/guest-env && chown chv:chv /etc/chv/guest-env || true\n - /usr/local/bin/chv-data-mount >>/var/log/chv-data-mount.log 2>&1 || true\n - /usr/local/bin/chv-sync boot >>/var/log/chv-sync.log 2>&1 || true\n" > /etc/cloud/cloud.cfg.d/99-chv-sync.cfg'
|
||||
--run-command 'git config --system user.name chv && git config --system user.email chv@localhost && git config --system credential.helper store && git config --system init.defaultBranch main'
|
||||
--run-command "$GT_INSTALL_CMD"
|
||||
@@ -304,7 +316,10 @@ VC_ARGS=(-a "$OUT/disk.raw"
|
||||
--run-command 'useradd --system --home-dir /var/lib/chv-browser --shell /usr/sbin/nologin chv-browser && mkdir -p /var/lib/chv-browser /run/chv/shots && chown chv-browser:chv-browser /var/lib/chv-browser /run/chv/shots && chmod 0755 /run/chv /run/chv/shots'
|
||||
--run-command 'printf "[Unit]\nDescription=CHV cell broker (Muse P2)\nAfter=network.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 /usr/local/bin/chv-broker\nRestart=on-failure\nRestartSec=2\n\n[Install]\nWantedBy=multi-user.target\n" > /etc/systemd/system/chv-broker.service'
|
||||
--run-command 'printf "[Unit]\nDescription=CHV browser sub-agent (Muse P2)\nAfter=network.target\n\n[Service]\nType=simple\nUser=chv-browser\nGroup=chv-browser\nExecStart=/usr/bin/python3 /usr/local/bin/chv-browser\nRestart=on-failure\nRestartSec=2\n\n[Install]\nWantedBy=multi-user.target\n" > /etc/systemd/system/chv-browser.service'
|
||||
--run-command 'printf "d /run/chv 0755 root root -\nd /run/chv/shots 0755 chv-browser chv-browser -\n" > /etc/tmpfiles.d/chv.conf && systemctl enable chv-broker chv-browser || true')
|
||||
--run-command 'printf "[Unit]\nDescription=CHV guest runtime daemon (stage 2: heartbeat/resources/snapshot)\nAfter=network.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 /usr/local/bin/chv-daemon\nRestart=always\nRestartSec=2\n\n[Install]\nWantedBy=multi-user.target\n" > /etc/systemd/system/chv-daemon.service'
|
||||
--run-command 'printf "[Unit]\nDescription=CHV guest exec daemon (stage 2: shell/files/codex hosting)\nAfter=network.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 /usr/local/bin/chv-execd\nRestart=always\nRestartSec=2\n\n[Install]\nWantedBy=multi-user.target\n" > /etc/systemd/system/chv-execd.service'
|
||||
--run-command 'printf "[Unit]\nDescription=CHV Gitea sync daemon (stage 2: resident pull/push loop)\nAfter=network.target\n\n[Service]\nType=simple\nUser=chv\nGroup=chv\nExecStart=/usr/bin/python3 /usr/local/bin/chv-syncd\nRestart=always\nRestartSec=2\n\n[Install]\nWantedBy=multi-user.target\n" > /etc/systemd/system/chv-syncd.service'
|
||||
--run-command 'printf "d /run/chv 0755 root root -\nd /run/chv/shots 0755 chv-browser chv-browser -\nd /run/chv/tasks 0755 root root -\n" > /etc/tmpfiles.d/chv.conf && systemctl enable chv-broker chv-browser chv-daemon chv-execd chv-syncd || true')
|
||||
# P1 交接（Muse P1 的 vsock_notify.py）：文件在仓库里存在时拷入
|
||||
# 镜像（P1 分支才有，本分支尚无）；不存在时只警告、不失败——
|
||||
# 合并后自动生效，P2 不改该脚本内容。
|
||||
|
||||
@@ -41,6 +41,8 @@ BASE_CONTENTS = [
|
||||
"systemd-container", "chv-cell", "chv-broker", "chv-browser",
|
||||
# feat-chv-opt-landing（阶段 1）：数据盘挂载脚本。
|
||||
"chv-data-mount",
|
||||
# feat-chv-opt-landing（阶段 2）：客机运行时三分服务。
|
||||
"chv-daemon", "chv-execd", "chv-syncd",
|
||||
]
|
||||
|
||||
# 执行器默认内核 cmdline（config.py），构建产物须与它一起记录（B-12），
|
||||
|
||||
@@ -33,6 +33,11 @@ DECLARED_SOURCES = {
|
||||
# feat-chv-opt-landing（阶段 1）：数据盘挂载脚本，版本随
|
||||
# repo_commit（与 chv-sync 同款）。
|
||||
"chv-data-mount": "repo_commit",
|
||||
# feat-chv-opt-landing（阶段 2）：客机运行时三分服务，版本
|
||||
# 随 repo_commit（与 chv-sync 同款）。
|
||||
"chv-daemon": "repo_commit",
|
||||
"chv-execd": "repo_commit",
|
||||
"chv-syncd": "repo_commit",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,315 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chv-daemon：客机运行时 daemon（阶段 2 三分之一）。
|
||||
|
||||
职责（只做这三件，不碰任务执行与同步）：
|
||||
1. 生命周期/心跳：每 10 秒写 /run/chv/heartbeat.json（atomic），
|
||||
含 pid、启动时间、seq、uptime、loadavg、内存、workspace
|
||||
余量、execd 任务数、sync 最近状态摘要。seq 跨重启连续
|
||||
（启动时读旧文件续号），宿主侧据此判断“心跳丢了还是
|
||||
daemon 重启了”。
|
||||
2. 资源上报：同上，心跳文件即上报载体（宿主经 SSH 读文件，
|
||||
不另开通道；控制通道仍是 SSH，见 executor.py 定稿）。
|
||||
3. 快照配合：Unix socket /run/chv/daemon.sock，op
|
||||
prepare-snapshot——问 execd 是否有运行中任务；有则
|
||||
拒绝（{ok:false, reason:task-running}），无则写
|
||||
/run/chv/snapshot-ready.json 标记并返回 ok；execd
|
||||
联系不上时同样拒绝（fail-closed，不写标记）。
|
||||
宿主侧快照流程暂未调它（阶段 2 不改 executor 快照流），
|
||||
接口先行、供宿主侧后续接线与本阶段 live 演练。
|
||||
|
||||
与既有分工的映射（不要第二套 agent）：
|
||||
- host.py 仍是唯一的任务/ codex 执行引擎（CLI，经 SSH 或
|
||||
execd 的 codex op 调）；daemon 从不跑 host.py。
|
||||
- chv-broker/chv-browser 仍是单元内外 IPC 与浏览器子代理，
|
||||
daemon 不碰它们的 socket。
|
||||
- sync 仍是 chv-sync.sh + chv-syncd 常驻循环，daemon 只读
|
||||
sync-status.json 作摘要，不调 git。
|
||||
|
||||
只用标准库。以 root 运行（读 /proc 与各服务状态文件无权限
|
||||
问题；socket 0600 root:root，只接受本机 root 客户）。
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
HEARTBEAT_PATH = "/run/chv/heartbeat.json"
|
||||
DAEMON_SOCK = "/run/chv/daemon.sock"
|
||||
SNAPSHOT_READY_PATH = "/run/chv/snapshot-ready.json"
|
||||
EXECD_SOCK = "/run/chv/execd.sock"
|
||||
EXECD_TASKS = "/run/chv/execd-tasks.json"
|
||||
SYNC_STATUS = "/run/chv/sync-status.json"
|
||||
WORKSPACE = "/home/chv/workspace"
|
||||
HEARTBEAT_INTERVAL = 10.0
|
||||
MAX_REQUEST_BYTES = 65536
|
||||
|
||||
|
||||
def _atomic_write(path, data: bytes):
|
||||
tmp = "%s.tmp.%d" % (path, os.getpid())
|
||||
with open(tmp, "wb") as fh:
|
||||
fh.write(data)
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
os.rename(tmp, path)
|
||||
|
||||
|
||||
def _read_json(path):
|
||||
try:
|
||||
with open(path) as fh:
|
||||
return json.load(fh)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _loadavg():
|
||||
try:
|
||||
with open("/proc/loadavg") as fh:
|
||||
parts = fh.read().split()
|
||||
return [float(parts[0]), float(parts[1]), float(parts[2])]
|
||||
except (OSError, ValueError, IndexError):
|
||||
return None
|
||||
|
||||
|
||||
def _meminfo():
|
||||
mem = {}
|
||||
try:
|
||||
with open("/proc/meminfo") as fh:
|
||||
for line in fh:
|
||||
if ":" not in line:
|
||||
continue
|
||||
key, val = line.split(":", 1)
|
||||
val = val.strip().split()
|
||||
if val and val[0].isdigit():
|
||||
mem[key.strip()] = int(val[0])
|
||||
except OSError:
|
||||
return None
|
||||
if not mem:
|
||||
return None
|
||||
return {"total_kb": mem.get("MemTotal"),
|
||||
"available_kb": mem.get("MemAvailable")}
|
||||
|
||||
|
||||
def _workspace_free():
|
||||
try:
|
||||
st = os.statvfs(WORKSPACE)
|
||||
return st.f_bavail * st.f_frsize
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _execd_task_count():
|
||||
data = _read_json(EXECD_TASKS)
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
tasks = data.get("tasks")
|
||||
if not isinstance(tasks, list):
|
||||
return None
|
||||
running = [t for t in tasks
|
||||
if isinstance(t, dict) and t.get("state") == "running"]
|
||||
return {"total": len(tasks), "running": len(running)}
|
||||
|
||||
|
||||
def _sync_summary():
|
||||
data = _read_json(SYNC_STATUS)
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
return {"last": data.get("last"), "at": data.get("at"),
|
||||
"head": data.get("head"),
|
||||
"error": data.get("error")}
|
||||
|
||||
|
||||
def build_heartbeat(pid, started_at, seq):
|
||||
"""组装一次心跳 dict（纯函数，便于单测与 --once）。"""
|
||||
now = time.time()
|
||||
return {"service": "chv-daemon", "pid": pid,
|
||||
"started_at": started_at, "seq": seq,
|
||||
"at": now, "uptime_s": round(now - started_at, 1),
|
||||
"loadavg": _loadavg(), "mem": _meminfo(),
|
||||
"workspace_free_bytes": _workspace_free(),
|
||||
"execd_tasks": _execd_task_count(),
|
||||
"sync": _sync_summary()}
|
||||
|
||||
|
||||
def _ask_execd_status(timeout=2.0):
|
||||
"""问 execd 要任务状态；联系不上返回 (None, reason)。"""
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
try:
|
||||
sock.settimeout(timeout)
|
||||
sock.connect(EXECD_SOCK)
|
||||
sock.sendall(json.dumps({"op": "status"}).encode() + b"\n")
|
||||
buf = b""
|
||||
while len(buf) < MAX_REQUEST_BYTES:
|
||||
chunk = sock.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
buf += chunk
|
||||
if buf.endswith(b"\n"):
|
||||
break
|
||||
data = json.loads(buf.decode())
|
||||
if not data.get("ok"):
|
||||
return None, "execd-status-not-ok"
|
||||
return data, None
|
||||
except (OSError, ValueError) as exc:
|
||||
return None, "execd-unreachable: %s" % exc
|
||||
finally:
|
||||
sock.close()
|
||||
|
||||
|
||||
def prepare_snapshot():
|
||||
"""快照配合：execd 有运行中任务或联系不上即拒绝。"""
|
||||
data, reason = _ask_execd_status()
|
||||
if data is None:
|
||||
return {"ok": False, "reason": reason}
|
||||
tasks = data.get("tasks") or []
|
||||
running = [t for t in tasks
|
||||
if isinstance(t, dict) and t.get("state") == "running"]
|
||||
if running:
|
||||
return {"ok": False, "reason": "task-running",
|
||||
"running": [t.get("id") for t in running]}
|
||||
marker = {"at": time.time(), "pid": os.getpid(),
|
||||
"note": "no running execd task"}
|
||||
_atomic_write(SNAPSHOT_READY_PATH,
|
||||
(json.dumps(marker) + "\n").encode())
|
||||
return {"ok": True, "marker": SNAPSHOT_READY_PATH}
|
||||
|
||||
|
||||
def handle_request(req):
|
||||
"""socket 请求分发（纯函数，便于单测）。"""
|
||||
if not isinstance(req, dict):
|
||||
return {"ok": False, "error": "request must be an object"}
|
||||
op = req.get("op")
|
||||
if op == "ping":
|
||||
return {"ok": True, "service": "chv-daemon",
|
||||
"pid": os.getpid()}
|
||||
if op == "status":
|
||||
hb = _read_json(HEARTBEAT_PATH)
|
||||
return {"ok": True, "heartbeat": hb}
|
||||
if op == "prepare-snapshot":
|
||||
try:
|
||||
return prepare_snapshot()
|
||||
except OSError as exc:
|
||||
return {"ok": False, "reason": "marker-write-failed: %s"
|
||||
% exc}
|
||||
return {"ok": False, "error": "unknown op: %s" % (op,)}
|
||||
|
||||
|
||||
class Daemon:
|
||||
def __init__(self, interval=HEARTBEAT_INTERVAL):
|
||||
self.interval = interval
|
||||
self.pid = os.getpid()
|
||||
self.started_at = time.time()
|
||||
self.seq = 0
|
||||
old = _read_json(HEARTBEAT_PATH)
|
||||
if isinstance(old, dict) and isinstance(old.get("seq"), int):
|
||||
# 续号：宿主据 seq 连续判断心跳丢包还是 daemon 重启。
|
||||
self.seq = old["seq"]
|
||||
self._stop = threading.Event()
|
||||
self._server = None
|
||||
|
||||
def write_heartbeat(self):
|
||||
self.seq += 1
|
||||
hb = build_heartbeat(self.pid, self.started_at, self.seq)
|
||||
_atomic_write(HEARTBEAT_PATH,
|
||||
(json.dumps(hb) + "\n").encode())
|
||||
return hb
|
||||
|
||||
def _serve(self):
|
||||
if os.path.exists(DAEMON_SOCK):
|
||||
os.remove(DAEMON_SOCK)
|
||||
srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
srv.bind(DAEMON_SOCK)
|
||||
os.chmod(DAEMON_SOCK, 0o600)
|
||||
srv.listen(8)
|
||||
srv.settimeout(0.5)
|
||||
self._server = srv
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
conn, _ = srv.accept()
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
break
|
||||
with conn:
|
||||
try:
|
||||
conn.settimeout(5.0)
|
||||
buf = b""
|
||||
while len(buf) < MAX_REQUEST_BYTES:
|
||||
chunk = conn.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
buf += chunk
|
||||
if buf.endswith(b"\n"):
|
||||
break
|
||||
req = json.loads(buf.decode())
|
||||
resp = handle_request(req)
|
||||
except (OSError, ValueError) as exc:
|
||||
resp = {"ok": False, "error": str(exc)}
|
||||
try:
|
||||
conn.sendall((json.dumps(resp) + "\n").encode())
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
srv.close()
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
os.remove(DAEMON_SOCK)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def run(self):
|
||||
thread = threading.Thread(target=self._serve, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
while not self._stop.is_set():
|
||||
self.write_heartbeat()
|
||||
self._stop.wait(self.interval)
|
||||
finally:
|
||||
self._stop.set()
|
||||
thread.join(timeout=5)
|
||||
|
||||
def stop(self):
|
||||
self._stop.set()
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = list(sys.argv[1:] if argv is None else argv)
|
||||
interval = HEARTBEAT_INTERVAL
|
||||
once = False
|
||||
i = 0
|
||||
while i < len(argv):
|
||||
if argv[i] == "--interval" and i + 1 < len(argv):
|
||||
interval = float(argv[i + 1])
|
||||
i += 2
|
||||
elif argv[i] == "--once":
|
||||
once = True
|
||||
i += 1
|
||||
else:
|
||||
print("用法：chv-daemon [--interval 秒] [--once]",
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
daemon = Daemon(interval=interval)
|
||||
if once:
|
||||
daemon.write_heartbeat()
|
||||
print(json.dumps({"ok": True,
|
||||
"heartbeat": HEARTBEAT_PATH,
|
||||
"seq": daemon.seq}))
|
||||
return 0
|
||||
print("chv-daemon pid=%d heartbeat=%s" % (daemon.pid,
|
||||
HEARTBEAT_PATH),
|
||||
flush=True)
|
||||
|
||||
def _on_signal(signum, _frame):
|
||||
daemon.stop()
|
||||
|
||||
signal.signal(signal.SIGTERM, _on_signal)
|
||||
signal.signal(signal.SIGINT, _on_signal)
|
||||
daemon.run()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,485 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chv-execd：客机运行时 execd（阶段 2 三分之一）。
|
||||
|
||||
职责（只做这三件）：
|
||||
1. shell 命令：op exec（同步跑完返回）/ spawn（脱离式后台跑，
|
||||
立即返回 id）。一律 list argv、绝不用 shell=True；工作区
|
||||
内命令以 chv 身份跑（runuser -u chv），codex 除外。
|
||||
2. 文件读写：op read/write，路径只允许 /home/chv/workspace
|
||||
之内（realpath + commonpath 校验），越界即拒绝。
|
||||
3. codex 进程托管：op codex——经 chv-cell run 起
|
||||
`python3 /app/host.py <args>`（单元隔离口径不变，不绕
|
||||
cell 直跑 host.py）；子进程 setsid 脱离，execd 崩溃不
|
||||
带走它；任务登记落盘 /run/chv/execd-tasks.json，execd
|
||||
重启后按 pid 重新认领（reattach），任务状态不丢。
|
||||
|
||||
客户端鉴权：SO_PEERCRED 的 uid 须是 0 或 1000（root / chv），
|
||||
其余一律拒绝。socket /run/chv/execd.sock 0600 root:root。
|
||||
调用方是宿主执行器经 SSH（root）或客机内 root 服务；单元内
|
||||
codex 不直接调 execd（走 chv-broker 白名单）。
|
||||
|
||||
任务状态机：running → completed（rc 已收）/ failed（spawn
|
||||
失败）/ killed / unknown（execd 重启后 pid 已死且退出码
|
||||
不可考——init 已收走，如实记 unknown，不编 rc）。
|
||||
spawn 输出进 /run/chv/tasks/<id>.log（runuser 以 chv 写，
|
||||
codex 以 root 写，均 0644）。
|
||||
|
||||
只用标准库。以 root 运行（要起 chv-cell + runuser）；单租户
|
||||
灰度口径——与今天“执行器经 SSH 以 root 跑 host.py”同权，
|
||||
多租户收紧待安全再审翻绿后另议。
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
EXECD_SOCK = "/run/chv/execd.sock"
|
||||
TASKS_PATH = "/run/chv/execd-tasks.json"
|
||||
TASKS_LOG_DIR = "/run/chv/tasks"
|
||||
WORKSPACE = "/home/chv/workspace"
|
||||
HOST_PY = "/app/host.py"
|
||||
CELL_BIN = "/usr/local/bin/chv-cell"
|
||||
ALLOWED_UIDS = (0, 1000)
|
||||
MAX_REQUEST_BYTES = 65536
|
||||
MAX_OUTPUT_BYTES = 65536
|
||||
MAX_FILE_BYTES = 1 << 20
|
||||
DEFAULT_TIMEOUT = 120.0
|
||||
|
||||
# codex 子进程可透传的环境变量白名单（前缀匹配；调用方
|
||||
# 传什么别的都丢弃。模型网关 key 走这个口径，不走 argv，
|
||||
# 绝不进任务登记文件与日志）。
|
||||
CODEX_ENV_PREFIXES = ("OPENAI_", "CODEX_", "CHV_MODEL_",
|
||||
"PLAYWRIGHT_")
|
||||
|
||||
|
||||
def _atomic_write(path, data: bytes):
|
||||
tmp = "%s.tmp.%d" % (path, os.getpid())
|
||||
with open(tmp, "wb") as fh:
|
||||
fh.write(data)
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
os.rename(tmp, path)
|
||||
|
||||
|
||||
def _read_json(path):
|
||||
try:
|
||||
with open(path) as fh:
|
||||
return json.load(fh)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _peer_uid(conn):
|
||||
"""SO_PEERCRED 取对端 uid；取不到即拒绝（不降级）。"""
|
||||
try:
|
||||
creds = conn.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED,
|
||||
struct.calcsize("3i"))
|
||||
_pid, uid, _gid = struct.unpack("3i", creds)
|
||||
return uid
|
||||
except (OSError, struct.error):
|
||||
return None
|
||||
|
||||
|
||||
def _within_workspace(path):
|
||||
"""路径是否在 WORKSPACE 内（realpath 后 commonpath 校验）。"""
|
||||
try:
|
||||
real = os.path.realpath(path)
|
||||
except OSError:
|
||||
return False
|
||||
return os.path.commonpath([real, WORKSPACE]) == WORKSPACE
|
||||
|
||||
|
||||
def _truncate(text, limit=MAX_OUTPUT_BYTES):
|
||||
if len(text) > limit:
|
||||
return text[:limit] + "\n...<truncated %d bytes>" % (
|
||||
len(text) - limit)
|
||||
return text
|
||||
|
||||
|
||||
def _codex_env(extra):
|
||||
"""调用方 env 过白名单；只留前缀命中的项。"""
|
||||
env = {"PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"),
|
||||
"HOME": "/home/chv",
|
||||
"PLAYWRIGHT_BROWSERS_PATH":
|
||||
"/usr/local/share/ms-playwright"}
|
||||
if isinstance(extra, dict):
|
||||
for key, val in extra.items():
|
||||
if isinstance(key, str) and isinstance(val, str) \
|
||||
and key.startswith(CODEX_ENV_PREFIXES):
|
||||
env[key] = val
|
||||
return env
|
||||
|
||||
|
||||
class TaskStore:
|
||||
"""任务登记（落盘 JSON，execd 重启后重读认领）。"""
|
||||
|
||||
def __init__(self, path=TASKS_PATH):
|
||||
self.path = path
|
||||
self._lock = threading.Lock()
|
||||
self.tasks = {}
|
||||
data = _read_json(path)
|
||||
if isinstance(data, dict) and isinstance(
|
||||
data.get("tasks"), list):
|
||||
for item in data["tasks"]:
|
||||
if isinstance(item, dict) and item.get("id"):
|
||||
self.tasks[item["id"]] = item
|
||||
self._reattach()
|
||||
|
||||
def _reattach(self):
|
||||
"""重启后认领：running 任务按 pid 查存活；已死记
|
||||
unknown（退出码不可考，不编），活着的保持 running。"""
|
||||
for task in self.tasks.values():
|
||||
if task.get("state") != "running":
|
||||
continue
|
||||
pid = task.get("pid")
|
||||
alive = False
|
||||
if isinstance(pid, int):
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
alive = True
|
||||
except (OSError, ProcessLookupError):
|
||||
alive = False
|
||||
except PermissionError:
|
||||
alive = True
|
||||
if alive:
|
||||
task["note"] = "reattached after execd restart"
|
||||
else:
|
||||
task["state"] = "unknown"
|
||||
task["note"] = ("execd restarted, exit code "
|
||||
"unrecoverable (reaped by init)")
|
||||
self._save_locked()
|
||||
|
||||
def _save_locked(self):
|
||||
data = {"tasks": [self.tasks[k] for k in sorted(self.tasks)]}
|
||||
_atomic_write(self.path,
|
||||
(json.dumps(data) + "\n").encode())
|
||||
|
||||
def add(self, task):
|
||||
with self._lock:
|
||||
self.tasks[task["id"]] = task
|
||||
self._save_locked()
|
||||
return task
|
||||
|
||||
def update(self, task_id, **fields):
|
||||
with self._lock:
|
||||
task = self.tasks.get(task_id)
|
||||
if task is None:
|
||||
return None
|
||||
task.update(fields)
|
||||
self._save_locked()
|
||||
return task
|
||||
|
||||
def get(self, task_id):
|
||||
with self._lock:
|
||||
task = self.tasks.get(task_id)
|
||||
return dict(task) if task else None
|
||||
|
||||
def list(self):
|
||||
with self._lock:
|
||||
return [dict(self.tasks[k]) for k in sorted(self.tasks)]
|
||||
|
||||
|
||||
class Execd:
|
||||
def __init__(self, tasks_path=TASKS_PATH):
|
||||
self.store = TaskStore(tasks_path)
|
||||
self._stop = threading.Event()
|
||||
self._procs = {}
|
||||
self._procs_lock = threading.Lock()
|
||||
self._reaper = threading.Thread(target=self._reap_loop,
|
||||
daemon=True)
|
||||
|
||||
def _register_proc(self, task_id, proc):
|
||||
with self._procs_lock:
|
||||
self._procs[task_id] = proc
|
||||
|
||||
def _reap_loop(self):
|
||||
"""收割已退出的子进程，登记 rc；未知 pid 的直接丢弃。
|
||||
重启后认领的 running 任务不在 _procs 里：轮询 pid
|
||||
存活，死了记 unknown（退出码不可考，不编）。"""
|
||||
while not self._stop.is_set():
|
||||
with self._procs_lock:
|
||||
items = list(self._procs.items())
|
||||
tracked = set()
|
||||
for task_id, proc in items:
|
||||
tracked.add(task_id)
|
||||
rc = proc.poll()
|
||||
if rc is None:
|
||||
continue
|
||||
with self._procs_lock:
|
||||
self._procs.pop(task_id, None)
|
||||
task = self.store.get(task_id)
|
||||
if task and task.get("state") == "running":
|
||||
self.store.update(task_id, state="completed",
|
||||
rc=rc, ended_at=time.time())
|
||||
for task in self.store.list():
|
||||
if task.get("state") != "running" \
|
||||
or task.get("id") in tracked:
|
||||
continue
|
||||
pid = task.get("pid")
|
||||
alive = False
|
||||
if isinstance(pid, int):
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
alive = True
|
||||
except OSError:
|
||||
alive = False
|
||||
if not alive:
|
||||
self.store.update(
|
||||
task["id"], state="unknown",
|
||||
note=("execd restarted, exit code "
|
||||
"unrecoverable (reaped by init)"),
|
||||
ended_at=time.time())
|
||||
self._stop.wait(1.0)
|
||||
|
||||
# -- op 实现 ---------------------------------------------------------
|
||||
def op_exec(self, req):
|
||||
argv = req.get("argv")
|
||||
if not isinstance(argv, list) or not argv \
|
||||
or not all(isinstance(a, str) for a in argv):
|
||||
return {"ok": False, "error": "argv must be a non-empty list of strings"}
|
||||
cwd = req.get("cwd") or WORKSPACE
|
||||
if not _within_workspace(cwd) and cwd != "/home/chv":
|
||||
return {"ok": False, "error": "cwd outside workspace: %s" % cwd}
|
||||
try:
|
||||
timeout = float(req.get("timeout") or DEFAULT_TIMEOUT)
|
||||
except (TypeError, ValueError):
|
||||
return {"ok": False, "error": "bad timeout"}
|
||||
timeout = min(max(timeout, 1.0), 600.0)
|
||||
cmd = ["runuser", "-u", "chv", "--"] + argv
|
||||
try:
|
||||
proc = subprocess.run(cmd, cwd=cwd, timeout=timeout,
|
||||
capture_output=True, text=True,
|
||||
errors="replace")
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
return {"ok": False, "error": "timeout after %.0fs" % timeout,
|
||||
"stdout": _truncate(exc.stdout or ""),
|
||||
"stderr": _truncate(exc.stderr or "")}
|
||||
except OSError as exc:
|
||||
return {"ok": False, "error": "spawn failed: %s" % exc}
|
||||
return {"ok": True, "rc": proc.returncode,
|
||||
"stdout": _truncate(proc.stdout or ""),
|
||||
"stderr": _truncate(proc.stderr or "")}
|
||||
|
||||
def op_spawn(self, req, kind="exec"):
|
||||
argv = req.get("argv")
|
||||
if not isinstance(argv, list) or not argv \
|
||||
or not all(isinstance(a, str) for a in argv):
|
||||
return {"ok": False, "error": "argv must be a non-empty list of strings"}
|
||||
cwd = req.get("cwd") or WORKSPACE
|
||||
if not _within_workspace(cwd) and cwd != "/home/chv":
|
||||
return {"ok": False, "error": "cwd outside workspace: %s" % cwd}
|
||||
if kind == "codex":
|
||||
if not os.path.exists(CELL_BIN):
|
||||
return {"ok": False, "error": "chv-cell not found"}
|
||||
full = [CELL_BIN, "run", "--", "python3", HOST_PY] + argv
|
||||
env = _codex_env(req.get("env"))
|
||||
as_user = None
|
||||
else:
|
||||
full = ["runuser", "-u", "chv", "--"] + argv
|
||||
env = None
|
||||
as_user = "chv"
|
||||
task_id = uuid.uuid4().hex[:12]
|
||||
os.makedirs(TASKS_LOG_DIR, exist_ok=True)
|
||||
log_path = os.path.join(TASKS_LOG_DIR, task_id + ".log")
|
||||
try:
|
||||
log_fh = open(log_path, "ab", buffering=0)
|
||||
except OSError as exc:
|
||||
return {"ok": False, "error": "log open failed: %s" % exc}
|
||||
try:
|
||||
proc = subprocess.Popen(
|
||||
full, cwd=cwd, stdout=log_fh, stderr=log_fh,
|
||||
stdin=subprocess.DEVNULL, start_new_session=True,
|
||||
env=env, close_fds=True)
|
||||
except OSError as exc:
|
||||
log_fh.close()
|
||||
return {"ok": False, "error": "spawn failed: %s" % exc}
|
||||
task = {"id": task_id, "kind": kind,
|
||||
"argv_preview": argv[:4], "pid": proc.pid,
|
||||
"state": "running", "started_at": time.time(),
|
||||
"log": log_path,
|
||||
"as_user": as_user or "root-via-cell"}
|
||||
self.store.add(task)
|
||||
self._register_proc(task_id, proc)
|
||||
return {"ok": True, "id": task_id, "pid": proc.pid,
|
||||
"log": log_path}
|
||||
|
||||
def op_read(self, req):
|
||||
path = req.get("path")
|
||||
if not isinstance(path, str) or not path:
|
||||
return {"ok": False, "error": "path required"}
|
||||
if not _within_workspace(path):
|
||||
return {"ok": False, "error": "path outside workspace"}
|
||||
try:
|
||||
with open(path, "rb") as fh:
|
||||
data = fh.read(MAX_FILE_BYTES + 1)
|
||||
except OSError as exc:
|
||||
return {"ok": False, "error": "read failed: %s" % exc}
|
||||
truncated = len(data) > MAX_FILE_BYTES
|
||||
return {"ok": True, "path": path,
|
||||
"content": data[:MAX_FILE_BYTES].decode(
|
||||
"utf-8", errors="replace"),
|
||||
"truncated": truncated}
|
||||
|
||||
def op_write(self, req):
|
||||
path = req.get("path")
|
||||
content = req.get("content")
|
||||
if not isinstance(path, str) or not path:
|
||||
return {"ok": False, "error": "path required"}
|
||||
if not isinstance(content, str):
|
||||
return {"ok": False, "error": "content must be a string"}
|
||||
if not _within_workspace(path):
|
||||
return {"ok": False, "error": "path outside workspace"}
|
||||
try:
|
||||
parent = os.path.dirname(path)
|
||||
if parent:
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
with open(path, "w") as fh:
|
||||
fh.write(content)
|
||||
except OSError as exc:
|
||||
return {"ok": False, "error": "write failed: %s" % exc}
|
||||
return {"ok": True, "path": path, "bytes": len(content)}
|
||||
|
||||
def op_task_status(self, req):
|
||||
task_id = req.get("id")
|
||||
if task_id:
|
||||
task = self.store.get(task_id)
|
||||
if task is None:
|
||||
return {"ok": False, "error": "no such task: %s" % task_id}
|
||||
return {"ok": True, "task": task}
|
||||
return {"ok": True, "tasks": self.store.list()}
|
||||
|
||||
def op_task_kill(self, req):
|
||||
task_id = req.get("id")
|
||||
if not task_id:
|
||||
return {"ok": False, "error": "id required"}
|
||||
task = self.store.get(task_id)
|
||||
if task is None:
|
||||
return {"ok": False, "error": "no such task: %s" % task_id}
|
||||
if task.get("state") != "running":
|
||||
return {"ok": False,
|
||||
"error": "task not running: %s" % task.get("state")}
|
||||
pid = task.get("pid")
|
||||
sig = signal.SIGTERM
|
||||
if req.get("signal") == "KILL":
|
||||
sig = signal.SIGKILL
|
||||
try:
|
||||
os.kill(pid, sig)
|
||||
except OSError as exc:
|
||||
return {"ok": False, "error": "kill failed: %s" % exc}
|
||||
self.store.update(task_id, state="killed",
|
||||
ended_at=time.time())
|
||||
return {"ok": True, "id": task_id, "pid": pid}
|
||||
|
||||
def handle(self, req):
|
||||
if not isinstance(req, dict):
|
||||
return {"ok": False, "error": "request must be an object"}
|
||||
op = req.get("op")
|
||||
if op == "ping":
|
||||
return {"ok": True, "service": "chv-execd",
|
||||
"pid": os.getpid()}
|
||||
if op == "status":
|
||||
return {"ok": True, "tasks": self.store.list()}
|
||||
if op == "exec":
|
||||
return self.op_exec(req)
|
||||
if op == "spawn":
|
||||
return self.op_spawn(req, kind="exec")
|
||||
if op == "codex":
|
||||
return self.op_spawn(req, kind="codex")
|
||||
if op == "read":
|
||||
return self.op_read(req)
|
||||
if op == "write":
|
||||
return self.op_write(req)
|
||||
if op == "task-status":
|
||||
return self.op_task_status(req)
|
||||
if op == "task-kill":
|
||||
return self.op_task_kill(req)
|
||||
return {"ok": False, "error": "unknown op: %s" % (op,)}
|
||||
|
||||
# -- socket 服务 ------------------------------------------------------
|
||||
def serve(self):
|
||||
if os.path.exists(EXECD_SOCK):
|
||||
os.remove(EXECD_SOCK)
|
||||
srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
srv.bind(EXECD_SOCK)
|
||||
os.chmod(EXECD_SOCK, 0o600)
|
||||
srv.listen(8)
|
||||
srv.settimeout(0.5)
|
||||
self._reaper.start()
|
||||
print("chv-execd pid=%d sock=%s" % (os.getpid(), EXECD_SOCK),
|
||||
flush=True)
|
||||
try:
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
conn, _ = srv.accept()
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
break
|
||||
with conn:
|
||||
uid = _peer_uid(conn)
|
||||
if uid not in ALLOWED_UIDS:
|
||||
resp = {"ok": False,
|
||||
"error": "uid %s not allowed" % (uid,)}
|
||||
try:
|
||||
conn.sendall(
|
||||
(json.dumps(resp) + "\n").encode())
|
||||
except OSError:
|
||||
pass
|
||||
continue
|
||||
try:
|
||||
conn.settimeout(10.0)
|
||||
buf = b""
|
||||
while len(buf) < MAX_REQUEST_BYTES:
|
||||
chunk = conn.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
buf += chunk
|
||||
if buf.endswith(b"\n"):
|
||||
break
|
||||
req = json.loads(buf.decode())
|
||||
resp = self.handle(req)
|
||||
except (OSError, ValueError) as exc:
|
||||
resp = {"ok": False, "error": str(exc)}
|
||||
try:
|
||||
conn.sendall((json.dumps(resp) + "\n").encode())
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
self._stop.set()
|
||||
try:
|
||||
srv.close()
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
os.remove(EXECD_SOCK)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def stop(self):
|
||||
self._stop.set()
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = list(sys.argv[1:] if argv is None else argv)
|
||||
if argv:
|
||||
print("用法：chv-execd（无参数）", file=sys.stderr)
|
||||
return 2
|
||||
execd = Execd()
|
||||
|
||||
def _on_signal(signum, _frame):
|
||||
execd.stop()
|
||||
|
||||
signal.signal(signal.SIGTERM, _on_signal)
|
||||
signal.signal(signal.SIGINT, _on_signal)
|
||||
execd.serve()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chv-syncd：客机运行时 sync 常驻（阶段 2 三分之一）。
|
||||
|
||||
职责：常驻循环调用 /usr/local/bin/chv-sync 做 Gitea 双向同步，
|
||||
状态落盘 /run/chv/sync-status.json，socket /run/chv/syncd.sock
|
||||
接受 ping/status/sync-now。
|
||||
|
||||
同步规则一条不改（与 chv-sync.sh 同款，复用它不重写 git 逻辑）：
|
||||
- Gitea https://gitea.tose.top 是权威远端；启动 clone 选定仓
|
||||
最新 HEAD（仍由 cloud-init runcmd 的 chv-sync boot 做；
|
||||
syncd 只在仓已 clone 的前提下循环，仓不存在记 needs-boot
|
||||
不擅自 clone——boot 须 root 写 credential，syncd 以 chv
|
||||
身份跑，无权也不该碰）。
|
||||
- 每轮先 pull --ff-only 再 push；pull 分叉即停住、跳过 push，
|
||||
状态记 error、绝不 --force、不自动 rebase、不覆盖任何一方。
|
||||
- 工作树脏时 pull 拒绝，同样记 error 等下一轮，不丢改动。
|
||||
- push 只推已提交的提交；未提交改动不参与同步。
|
||||
- token 只走 /home/chv/.git-credentials（0600），syncd
|
||||
自身不读 guest-env、不打印、不记入状态文件与日志。
|
||||
|
||||
以 chv 身份运行。默认每 300 秒一轮（CHV_SYNC_INTERVAL，
|
||||
下限 60 秒）；sync-now 立即触发一轮（轮内重复触发跳过）。
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
|
||||
SYNC_SOCK = "/run/chv/syncd.sock"
|
||||
STATUS_PATH = "/run/chv/sync-status.json"
|
||||
WORKSPACE = "/home/chv/workspace"
|
||||
SYNC_BIN = "/usr/local/bin/chv-sync"
|
||||
DEFAULT_INTERVAL = 300.0
|
||||
MIN_INTERVAL = 60.0
|
||||
MAX_REQUEST_BYTES = 65536
|
||||
|
||||
|
||||
def _atomic_write(path, data: bytes):
|
||||
tmp = "%s.tmp.%d" % (path, os.getpid())
|
||||
with open(tmp, "wb") as fh:
|
||||
fh.write(data)
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
os.rename(tmp, path)
|
||||
|
||||
|
||||
def _read_json(path):
|
||||
try:
|
||||
with open(path) as fh:
|
||||
return json.load(fh)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def resolve_repo(workspace=WORKSPACE):
|
||||
"""WORKSPACE 下唯一的 git 仓库；多个/零个返回 (None, reason)。"""
|
||||
try:
|
||||
names = sorted(os.listdir(workspace))
|
||||
except OSError as exc:
|
||||
return None, "workspace unreadable: %s" % exc
|
||||
found = [os.path.join(workspace, name) for name in names
|
||||
if os.path.isdir(os.path.join(workspace, name, ".git"))]
|
||||
if len(found) == 1:
|
||||
return found[0], None
|
||||
if not found:
|
||||
return None, "needs-boot: no cloned repo in workspace"
|
||||
return None, "ambiguous: %d repos, explicit repo required" % len(found)
|
||||
|
||||
|
||||
def _run_sync(action, repo):
|
||||
"""跑 chv-sync pull/push；返回 (ok, tail)。tail 只取末行
|
||||
（git 报错不含 token：remote URL 是干净 https，credential
|
||||
走 store 文件），截断 500 字符。"""
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
[SYNC_BIN, action, repo], capture_output=True,
|
||||
text=True, errors="replace", timeout=300)
|
||||
except OSError as exc:
|
||||
return False, "sync binary failed: %s" % exc
|
||||
except subprocess.TimeoutExpired:
|
||||
return False, "sync %s timeout after 300s" % action
|
||||
tail = ((proc.stdout or "") + (proc.stderr or "")).strip()
|
||||
tail = tail.splitlines()[-1] if tail else ""
|
||||
return proc.returncode == 0, tail[:500]
|
||||
|
||||
|
||||
def _git_head(repo):
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
["git", "-C", repo, "rev-parse", "HEAD"],
|
||||
capture_output=True, text=True, errors="replace",
|
||||
timeout=30)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
return None
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
class Syncd:
|
||||
def __init__(self, interval=DEFAULT_INTERVAL,
|
||||
status_path=STATUS_PATH, workspace=WORKSPACE):
|
||||
self.interval = max(interval, MIN_INTERVAL)
|
||||
self.status_path = status_path
|
||||
self.workspace = workspace
|
||||
self._stop = threading.Event()
|
||||
self._cycle_lock = threading.Lock()
|
||||
self._cycle_running = False
|
||||
self._status = _read_json(status_path) or {
|
||||
"service": "chv-syncd", "last": "never",
|
||||
"at": None, "head": None, "error": None,
|
||||
"consecutive_errors": 0, "cycles": 0}
|
||||
|
||||
def _write_status(self):
|
||||
_atomic_write(self.status_path,
|
||||
(json.dumps(self._status) + "\n").encode())
|
||||
|
||||
def run_cycle(self):
|
||||
"""一轮同步：先 pull 再 push；任一步失败即停住记 error。"""
|
||||
if not self._cycle_lock.acquire(blocking=False):
|
||||
return {"ok": False, "error": "cycle already running"}
|
||||
try:
|
||||
self._cycle_running = True
|
||||
return self._cycle()
|
||||
finally:
|
||||
self._cycle_running = False
|
||||
self._cycle_lock.release()
|
||||
|
||||
def _cycle(self):
|
||||
repo, reason = resolve_repo(self.workspace)
|
||||
now = time.time()
|
||||
self._status["cycles"] = self._status.get("cycles", 0) + 1
|
||||
if repo is None:
|
||||
self._status.update(
|
||||
{"last": "error", "at": now, "error": reason,
|
||||
"consecutive_errors":
|
||||
self._status.get("consecutive_errors", 0) + 1})
|
||||
self._write_status()
|
||||
return {"ok": False, "error": reason}
|
||||
head_before = _git_head(repo)
|
||||
ok, tail = _run_sync("pull", repo)
|
||||
if not ok:
|
||||
self._status.update(
|
||||
{"last": "error", "at": now, "repo": repo,
|
||||
"head": head_before,
|
||||
"error": "pull failed: %s" % tail,
|
||||
"consecutive_errors":
|
||||
self._status.get("consecutive_errors", 0) + 1})
|
||||
self._write_status()
|
||||
return {"ok": False, "error": "pull failed: %s" % tail}
|
||||
ok, tail = _run_sync("push", repo)
|
||||
head_after = _git_head(repo)
|
||||
if not ok:
|
||||
self._status.update(
|
||||
{"last": "error", "at": now, "repo": repo,
|
||||
"head": head_after,
|
||||
"error": "push failed: %s" % tail,
|
||||
"consecutive_errors":
|
||||
self._status.get("consecutive_errors", 0) + 1})
|
||||
self._write_status()
|
||||
return {"ok": False, "error": "push failed: %s" % tail}
|
||||
self._status.update(
|
||||
{"last": "ok", "at": now, "repo": repo,
|
||||
"head": head_after, "head_before": head_before,
|
||||
"error": None, "consecutive_errors": 0})
|
||||
self._write_status()
|
||||
return {"ok": True, "repo": repo, "head": head_after}
|
||||
|
||||
def handle(self, req):
|
||||
if not isinstance(req, dict):
|
||||
return {"ok": False, "error": "request must be an object"}
|
||||
op = req.get("op")
|
||||
if op == "ping":
|
||||
return {"ok": True, "service": "chv-syncd",
|
||||
"pid": os.getpid()}
|
||||
if op == "status":
|
||||
return {"ok": True, "status": self._status}
|
||||
if op == "sync-now":
|
||||
return self.run_cycle()
|
||||
return {"ok": False, "error": "unknown op: %s" % (op,)}
|
||||
|
||||
def _serve(self):
|
||||
if os.path.exists(SYNC_SOCK):
|
||||
os.remove(SYNC_SOCK)
|
||||
srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
srv.bind(SYNC_SOCK)
|
||||
os.chmod(SYNC_SOCK, 0o600)
|
||||
srv.listen(8)
|
||||
srv.settimeout(0.5)
|
||||
try:
|
||||
while not self._stop.is_set():
|
||||
try:
|
||||
conn, _ = srv.accept()
|
||||
except socket.timeout:
|
||||
continue
|
||||
except OSError:
|
||||
break
|
||||
with conn:
|
||||
try:
|
||||
conn.settimeout(10.0)
|
||||
buf = b""
|
||||
while len(buf) < MAX_REQUEST_BYTES:
|
||||
chunk = conn.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
buf += chunk
|
||||
if buf.endswith(b"\n"):
|
||||
break
|
||||
req = json.loads(buf.decode())
|
||||
resp = self.handle(req)
|
||||
except (OSError, ValueError) as exc:
|
||||
resp = {"ok": False, "error": str(exc)}
|
||||
try:
|
||||
conn.sendall((json.dumps(resp) + "\n").encode())
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
try:
|
||||
srv.close()
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
os.remove(SYNC_SOCK)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def run(self):
|
||||
thread = threading.Thread(target=self._serve, daemon=True)
|
||||
thread.start()
|
||||
print("chv-syncd pid=%d interval=%.0fs status=%s"
|
||||
% (os.getpid(), self.interval, self.status_path),
|
||||
flush=True)
|
||||
try:
|
||||
while not self._stop.is_set():
|
||||
self.run_cycle()
|
||||
self._stop.wait(self.interval)
|
||||
finally:
|
||||
self._stop.set()
|
||||
thread.join(timeout=5)
|
||||
|
||||
def stop(self):
|
||||
self._stop.set()
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
argv = list(sys.argv[1:] if argv is None else argv)
|
||||
interval = float(os.environ.get("CHV_SYNC_INTERVAL")
|
||||
or DEFAULT_INTERVAL)
|
||||
i = 0
|
||||
while i < len(argv):
|
||||
if argv[i] == "--interval" and i + 1 < len(argv):
|
||||
interval = float(argv[i + 1])
|
||||
i += 2
|
||||
else:
|
||||
print("用法：chv-syncd [--interval 秒]", file=sys.stderr)
|
||||
return 2
|
||||
syncd = Syncd(interval=interval)
|
||||
|
||||
def _on_signal(signum, _frame):
|
||||
syncd.stop()
|
||||
|
||||
signal.signal(signal.SIGTERM, _on_signal)
|
||||
signal.signal(signal.SIGINT, _on_signal)
|
||||
syncd.run()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -54,6 +54,11 @@ DECLARED_SOURCES = {
|
||||
# feat-chv-opt-landing（阶段 1）：数据盘挂载脚本，版本随
|
||||
# repo_commit（与 chv-sync 同款）。
|
||||
"chv-data-mount": "repo_commit",
|
||||
# feat-chv-opt-landing（阶段 2）：客机运行时三分服务，版本
|
||||
# 随 repo_commit（与 chv-sync 同款）。
|
||||
"chv-daemon": "repo_commit",
|
||||
"chv-execd": "repo_commit",
|
||||
"chv-syncd": "repo_commit",
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user