feat: 沙箱规模——/tmp 顺序加严、注册表寻址与失联精确清理、超时边界抽取
- Docker /tmp tmpfs 移到全部 -v 之前（父挂载在前，seed 子挂载在后）； 宿主级 bwrap /tmp 仍宣告未隔离，口径见 tmp_isolation_status() - workers 表补 host/pid/address（老库幂等补列），心跳透传与保留语义， _claim 写 claimed_by_worker；orphaned_containers 精确识别失联容器， Worker.cleanup_orphaned_containers 默认只杀 pid_verified 的精确容器名 - docker_run_timeout() 纯函数抽取（行为与原内联一致），README 补边界与 未验证清单；新增 test_sandbox_scale.py 20 项
This commit is contained in:
host committed
1 parent
e079b9b44a
commit
b42b650fb7
7 files changed
+897
-48
No files matched your search
+214
-8
@@ -15,6 +15,21 @@
|
||||
存在就 UPDATE，绝不因“未注册”返回 404 报错。
|
||||
- alive 定义：last_seen 在阈值内（默认 120 秒，可经环境变量
|
||||
CLOUD_WORKER_ALIVE_SECONDS 调，测试可传参覆盖）。
|
||||
- 寻址信息（本轮新增）：host（worker 所在主机名，worker 的
|
||||
--host-name，容器内 worker 绝不能用容器名冒充宿主名）、
|
||||
pid（worker 进程号，owner 死活的本地验证依据）、address
|
||||
（可选寻址串，如运维侧约定的 host:port，调用方显式给才
|
||||
有，绝不从环境变量里猜密钥类地址）。老行三列为 NULL，
|
||||
按“未知”处理，绝不猜成活着或死了。
|
||||
- 失联识别（本轮新增）：stale_workers() 列心跳过期行；
|
||||
orphaned_containers() 把“容器名 codex-<任务 id>”精确映射
|
||||
到 tasks.claimed_by_worker（认领时 worker 写），只认领到
|
||||
失联 worker 且状态仍为 running/interrupted 的任务容器才
|
||||
算失联容器；名字不匹配任务 id 正则、任务行缺失、无认领
|
||||
记录、owner 仍存活的一律不算——绝不按 "codex-" 前缀删。
|
||||
pid_verified 另算：本机 host + 已知 pid 时用 pid_alive
|
||||
校验（默认 os.kill(pid, 0)），远端 host 或 pid 未知时
|
||||
为 False（列出来给运维看，不自动杀）。
|
||||
- 本模块不存、不打任何密钥：注册信息只有 worker_id、executor、
|
||||
capacity、last_seen，没有任何凭证字段。
|
||||
- host/pid/address（本轨新增）：新库建表直接含三列；老库由
|
||||
@@ -46,6 +61,7 @@
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import socket
|
||||
import time
|
||||
@@ -78,6 +94,54 @@ def alive_threshold_seconds(override=None):
|
||||
return float(DEFAULT_ALIVE_SECONDS)
|
||||
|
||||
|
||||
def _existing_worker_columns(conn):
|
||||
"""workers 表现有列名集合；查不到时返回 None（调用方跳过补列）。
|
||||
|
||||
PG 走 information_schema，SQLite 走 PRAGMA table_info；两种
|
||||
查询失败都只回滚、不抛异常（老库迁移兜底，绝不因查列炸掉
|
||||
注册/心跳主路径）。
|
||||
"""
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT column_name FROM information_schema.columns "
|
||||
"WHERE table_name = 'workers'").fetchall()
|
||||
return {r[0] for r in rows}
|
||||
except Exception:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
rows = conn.execute("PRAGMA table_info(workers)").fetchall()
|
||||
return {r[1] for r in rows}
|
||||
except Exception:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _ensure_worker_address_columns(conn):
|
||||
"""老库补 host/pid/address 三列（缺哪个补哪个，幂等）。"""
|
||||
cols = _existing_worker_columns(conn)
|
||||
if cols is None:
|
||||
return
|
||||
for name, coltype in (("host", "TEXT"), ("pid", "INTEGER"),
|
||||
("address", "TEXT")):
|
||||
if name in cols:
|
||||
continue
|
||||
try:
|
||||
conn.execute(
|
||||
"ALTER TABLE workers ADD COLUMN %s %s" % (name, coltype))
|
||||
conn.commit()
|
||||
except Exception:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def ensure_workers_table(db_path):
|
||||
"""建 workers 表（不存在时建，已存在时不动，老库补精度）。
|
||||
|
||||
@@ -107,6 +171,7 @@ def ensure_workers_table(db_path):
|
||||
"created_at DOUBLE PRECISION)"
|
||||
)
|
||||
conn.commit()
|
||||
_ensure_worker_address_columns(conn)
|
||||
_ensure_running_column(conn)
|
||||
_ensure_hostpid_columns(conn)
|
||||
# 老库升精度只在确实是 Postgres REAL 列时做一次：每次心跳都
|
||||
@@ -501,6 +566,21 @@ def reap_stale_workers(db_path, stale_seconds=None, local_host=None,
|
||||
return results
|
||||
|
||||
|
||||
def _coerce_pid(pid):
|
||||
"""pid 归一化：None/空串 -> None；可转 int 的转 int；否则 None。
|
||||
|
||||
绝不把非法 pid 写进库（写进去了后续的本地存活校验会拿
|
||||
它当进程号查，错 pid 比缺 pid 更危险）。
|
||||
"""
|
||||
if pid is None or pid == "":
|
||||
return None
|
||||
if isinstance(pid, bool):
|
||||
return None
|
||||
try:
|
||||
return int(pid)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
def register_worker(db_path, worker_id, executor=None, capacity=1,
|
||||
running=None, host=None, pid=None, address=None):
|
||||
"""注册（或刷新）一个 worker：不存在 INSERT，存在 UPDATE。
|
||||
@@ -509,8 +589,7 @@ def register_worker(db_path, worker_id, executor=None, capacity=1,
|
||||
（server 的 heartbeat 端点）据此返回 400。新列语义：新行
|
||||
按给定值写（未给为 NULL）；已存在行上给定值覆盖、未给
|
||||
保留旧值——绝不因一次不带 host/pid 的注册把已有的身份
|
||||
洗成 NULL，也不编造 host/pid/address。
|
||||
"""
|
||||
洗成 NULL，也不编造 host/pid/address。 """
|
||||
if not worker_id or not isinstance(worker_id, str):
|
||||
raise ValueError("worker_id required")
|
||||
pid = _normalize_pid(pid)
|
||||
@@ -541,8 +620,7 @@ def register_worker(db_path, worker_id, executor=None, capacity=1,
|
||||
" VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(worker_id, executor, capacity,
|
||||
running if running is not None else 0,
|
||||
host, pid, address, now, now),
|
||||
)
|
||||
host, pid, address, now, now), )
|
||||
conn.commit()
|
||||
row = conn.execute(
|
||||
"SELECT * FROM workers WHERE worker_id = ?",
|
||||
@@ -562,6 +640,7 @@ def heartbeat_worker(db_path, worker_id, executor=None, capacity=None,
|
||||
应传本机 host 与本进程 pid，见 worker._registry_heartbeat），
|
||||
未给则保留旧值；address 没有新来源，绝不由本函数推导
|
||||
（例如不从 worker_id 或 socket 猜地址），只存显式给的值。
|
||||
显式给 None 与未给在心跳里无法区分（均按未给保留处理）。
|
||||
"""
|
||||
if not worker_id or not isinstance(worker_id, str):
|
||||
raise ValueError("worker_id required")
|
||||
@@ -580,8 +659,7 @@ def heartbeat_worker(db_path, worker_id, executor=None, capacity=None,
|
||||
" VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(worker_id, executor, capacity if capacity is not None
|
||||
else 1, running if running is not None else 0,
|
||||
host, pid, address, now, now),
|
||||
)
|
||||
host, pid, address, now, now), )
|
||||
else:
|
||||
old = _worker_to_dict(row)
|
||||
conn.execute(
|
||||
@@ -594,8 +672,7 @@ def heartbeat_worker(db_path, worker_id, executor=None, capacity=None,
|
||||
host if host is not None else old["host"],
|
||||
pid if pid is not None else old["pid"],
|
||||
address if address is not None else old["address"],
|
||||
now, worker_id),
|
||||
)
|
||||
now, worker_id), )
|
||||
conn.commit()
|
||||
row = conn.execute(
|
||||
"SELECT * FROM workers WHERE worker_id = ?",
|
||||
@@ -643,4 +720,133 @@ def _worker_to_dict(row):
|
||||
"address": _get("address"),
|
||||
"last_seen": _get("last_seen"),
|
||||
"created_at": _get("created_at"),
|
||||
"host": _get("host"),
|
||||
"pid": _get("pid"),
|
||||
"address": _get("address"),
|
||||
}
|
||||
|
||||
|
||||
def stale_workers(db_path, alive_seconds=None):
|
||||
"""心跳过期（失联）的 worker 行列表（每行已含 alive=False）。
|
||||
|
||||
只按 last_seen 判，不查 pid：pid 校验是 orphaned_containers
|
||||
里对“能不能自动杀”的加严，不是失联本身的定义（失联的
|
||||
定义与 sweep_stale_tasks 的任务心跳过期同款语义）。
|
||||
"""
|
||||
return [w for w in list_workers(db_path, alive_seconds)
|
||||
if not w.get("alive")]
|
||||
|
||||
|
||||
def default_pid_alive(pid):
|
||||
"""本地进程存活检查：os.kill(pid, 0) 成功或无权限都算活着
|
||||
（PermissionError 说明进程存在，只是归属别人）；只有
|
||||
查无此进程算死了；其它异常算未知（返回 None，不猜）。
|
||||
pid 非法时返回 None。"""
|
||||
pid = _coerce_pid(pid)
|
||||
if pid is None:
|
||||
return None
|
||||
try:
|
||||
os.kill(pid, 0)
|
||||
return True
|
||||
except ProcessLookupError:
|
||||
return False
|
||||
except PermissionError:
|
||||
return True
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _is_local_host(host):
|
||||
"""host 是否本机：空值按“无法确认”处理，不按本机猜。
|
||||
|
||||
注意与 pid_verified 的关系：host 为空时即使 pid 给了也
|
||||
不能断定该 pid 是本机的——跨主机同 pid 号毫无意义，
|
||||
所以空 host 一律 unverified。
|
||||
"""
|
||||
if not host or not isinstance(host, str):
|
||||
return False
|
||||
try:
|
||||
return host == socket.gethostname()
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
_TASK_CONTAINER_RE = re.compile(r"^codex-([0-9a-f]{32})$")
|
||||
|
||||
|
||||
def orphaned_containers(db_path, container_names,
|
||||
alive_seconds=None, pid_alive=None):
|
||||
"""识别失联 owner 的任务容器（纯识别，不杀任何容器）。
|
||||
|
||||
返回 list，每项 {container, task_id, worker_id, pid_verified}：
|
||||
- container：精确的容器名（原样返回，调用方按此名杀，
|
||||
绝不按前缀展开）；
|
||||
- task_id：从容器名正则里精确抠出的 32 位 hex（不是前缀
|
||||
切出来的任意串）；
|
||||
- worker_id：tasks.claimed_by_worker（认领时写）；
|
||||
- pid_verified：True = 本机 host + 已知 pid 且 pid_alive
|
||||
明确报死（可自动杀）；False = 未验证（远端 host、pid
|
||||
未知、pid_alive 未给或返回未知），只列不自动杀。
|
||||
排除项（任一即排除）：容器名不匹配任务正则；tasks 表或
|
||||
claimed_by_worker 列不存在（无法映射时宁可漏，不猜）；
|
||||
任务行缺失；任务状态不在 running/interrupted；owner 行
|
||||
缺失或仍存活（alive）。调用方杀之前仍须按返回的精确
|
||||
容器名逐个杀，绝不把本函数的返回再按前缀展开。
|
||||
"""
|
||||
if pid_alive is None:
|
||||
pid_alive = default_pid_alive
|
||||
stale = {w["worker_id"]: w for w in stale_workers(
|
||||
db_path, alive_seconds)}
|
||||
if not stale:
|
||||
return []
|
||||
conn = _db_connect(str(db_path))
|
||||
try:
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT id, status, claimed_by_worker FROM tasks"
|
||||
).fetchall()
|
||||
except Exception:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
finally:
|
||||
conn.close()
|
||||
tasks = {}
|
||||
for row in rows:
|
||||
try:
|
||||
tasks[row["id"]] = (row["status"], row["claimed_by_worker"])
|
||||
except (KeyError, IndexError, TypeError):
|
||||
continue
|
||||
out = []
|
||||
for name in container_names or []:
|
||||
if not isinstance(name, str):
|
||||
continue
|
||||
m = _TASK_CONTAINER_RE.match(name)
|
||||
if not m:
|
||||
continue
|
||||
task_id = m.group(1)
|
||||
info = tasks.get(task_id)
|
||||
if info is None:
|
||||
continue
|
||||
status, claimed = info
|
||||
if status not in ("running", "interrupted"):
|
||||
continue
|
||||
if not claimed or claimed not in stale:
|
||||
continue
|
||||
worker = stale[claimed]
|
||||
verified = False
|
||||
pid = worker.get("pid")
|
||||
if _is_local_host(worker.get("host")) and pid is not None:
|
||||
try:
|
||||
verified = pid_alive(pid) is False
|
||||
except Exception:
|
||||
verified = False
|
||||
out.append({
|
||||
"container": name,
|
||||
"task_id": task_id,
|
||||
"worker_id": claimed,
|
||||
"pid_verified": verified,
|
||||
})
|
||||
return out
|
||||
Reference in new issue
Block a user