feat(chv): 阶段 5 多会话并行——具名会话独立 context + storage-state 落盘 + execd 组杀
This commit is contained in:
host committed
1 parent
f723eb8a88
commit
6466ea6024
5 files changed
+723
-24
No files matched your search
+220
-15
@@ -12,9 +12,21 @@
|
||||
未挂载时回落到 /var/lib/chv-browser/profile（本机盘，不跨
|
||||
恢复保留——status 会明说当前 profile 在哪，绝不假装在数据盘）。
|
||||
服务 crash/重启后重新打开同一 profile，cookie/登录态仍在。
|
||||
* 多会话：同一浏览器上下文内每会话一页（session id → page），
|
||||
会话 A/B 各自导航、互不干扰；登录态（cookie）同上下文共享，
|
||||
是既定取舍（方案：同 VM 内 Bot 共享登录态，隔离边界是 VM 本身）。
|
||||
* 多会话：默认会话（"default"）走单一持久上下文（profile 在
|
||||
数据盘，登录态跨浏览器重启保留，阶段 3 口径不变）；具名
|
||||
会话各建一个独立 BrowserContext（同浏览器、不同存储分区，
|
||||
cookie/localStorage 互不串），每会话的 storage-state 落
|
||||
数据盘 .chv-browser-sessions/<会话>/storage-state.json，
|
||||
open 后与 close 时各存一次；重开同一会话 id 时从文件恢复，
|
||||
不同会话 id 永远看不到对方的态。登录态跨会话保留的含义
|
||||
是“同一会话 id 重开能读回自己的态”，不是“新会话能读别
|
||||
人的态”（那正是隔离要防的）。localStorage 的隔离与
|
||||
cookie 同源：同一浏览器的不同 context 自带独立存储分区。
|
||||
* 阶段 5（feat-chv-opt-landing）：具名会话独立 context +
|
||||
storage-state 落盘恢复（见上）；会话 id 只允许
|
||||
[A-Za-z0-9_-]（首字符须为字母数字，最长 64），防路径穿越
|
||||
进 storage-state 目录；close/kill 只动该会话自己的
|
||||
context 与任务，不动其它会话。
|
||||
* 双浏览器：默认 chromium-headless-shell（随 base 常驻）；完整
|
||||
Chromium 按需挂载（op `mount_full`：切到 full 引擎）、用完
|
||||
卸载（op `unmount_full`：切回 shell）。完整版不 bake 进 base；
|
||||
@@ -113,6 +125,14 @@ PLAYWRIGHT_BROWSERS_PATH = "/usr/local/share/ms-playwright"
|
||||
WORKSPACE_MOUNT = "/home/chv/workspace"
|
||||
PROFILE_DATA_DIR = "/home/chv/workspace/.chv-browser-profile"
|
||||
PROFILE_FALLBACK_DIR = "/var/lib/chv-browser/profile"
|
||||
# 具名会话的 storage-state 目录：profile 在数据盘上时跟 profile
|
||||
# 走数据盘（与 profile 同级）；回落本机盘时跟回落 profile 走
|
||||
# /var/lib/chv-browser/sessions（chv-browser 家目录，服务可写）。
|
||||
SESSIONS_FALLBACK_DIR = "/var/lib/chv-browser/sessions"
|
||||
# 会话 id 白名单（阶段 5）：只允许字母数字/_/-，首字符须为字母
|
||||
# 数字，最长 64——storage-state 按会话 id 落子目录，非法 id
|
||||
# 绝不能拼出 ../ 穿越。
|
||||
SESSION_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$")
|
||||
# 完整 Chromium 不 bake 进 base、不装根盘：根盘只剩 ~600MB，
|
||||
# playwright install chromium 会 ENOSPC（2026-11-08 构建 #8 驱动
|
||||
# 实证：109 秒后写满失败）。按需装到数据盘（9.3G 空闲），用完可删。
|
||||
@@ -223,6 +243,36 @@ def resolve_full_browsers_dir(profile_on_data_disk, env=None):
|
||||
return FULL_BROWSERS_FALLBACK_DIR
|
||||
|
||||
|
||||
def resolve_sessions_dir(profile_dir, profile_on_data_disk, env=None):
|
||||
"""具名会话 storage-state 的根目录（阶段 5）。
|
||||
|
||||
env 显式指定（CHV_BROWSER_SESSIONS_DIR）时优先；否则
|
||||
profile 在数据盘上时取 profile 的同级目录
|
||||
/home/chv/workspace/.chv-browser-sessions（数据盘，随
|
||||
快照恢复保留），回落时取 /var/lib/chv-browser/sessions。"""
|
||||
if env is None:
|
||||
env = os.environ
|
||||
explicit = (env.get("CHV_BROWSER_SESSIONS_DIR") or "").strip()
|
||||
if explicit:
|
||||
return explicit
|
||||
if profile_on_data_disk:
|
||||
return os.path.join(os.path.dirname(profile_dir),
|
||||
".chv-browser-sessions")
|
||||
return SESSIONS_FALLBACK_DIR
|
||||
|
||||
|
||||
def validate_session_id(session):
|
||||
"""会话 id 校验（阶段 5）：空/None 视为 default；非法返回
|
||||
(False, error)，合法返回 (True, "")。storage-state 按 id
|
||||
落子目录，校验是防穿越的唯一关口。"""
|
||||
if session is None or session == "":
|
||||
return True, ""
|
||||
if not isinstance(session, str) or not SESSION_ID_RE.match(session):
|
||||
return False, ("非法会话 id：%r（只允许字母数字/_/-，"
|
||||
"首字符须为字母数字，最长 64）" % (session,))
|
||||
return True, ""
|
||||
|
||||
|
||||
def truncate_a11y(node, max_nodes=MAX_A11Y_NODES,
|
||||
max_chars=MAX_A11Y_CHARS):
|
||||
"""截断无障碍树：节点数超上限时剪掉多余子树。返回 (tree, truncated)。
|
||||
@@ -429,7 +479,8 @@ class BrowserAgent:
|
||||
_load_playwright()。"""
|
||||
|
||||
def __init__(self, playwright_factory=None, shots_dir=SHOTS_DIR,
|
||||
profile_dir=None, cdp_port=None, full_browsers_dir=None):
|
||||
profile_dir=None, cdp_port=None, full_browsers_dir=None,
|
||||
sessions_dir=None):
|
||||
self._factory = playwright_factory or _load_playwright
|
||||
self._shots_dir = shots_dir
|
||||
self._profile_explicit = bool(profile_dir)
|
||||
@@ -440,6 +491,10 @@ class BrowserAgent:
|
||||
else:
|
||||
self._profile_dir, self._profile_on_data_disk = \
|
||||
resolve_profile_dir()
|
||||
self._sessions_dir = (
|
||||
sessions_dir or resolve_sessions_dir(
|
||||
self._profile_dir, self._profile_on_data_disk))
|
||||
self._sessions_explicit = bool(sessions_dir)
|
||||
self._full_browsers_explicit = bool(full_browsers_dir)
|
||||
self._full_browsers_dir = (
|
||||
full_browsers_dir or resolve_full_browsers_dir(
|
||||
@@ -453,6 +508,13 @@ class BrowserAgent:
|
||||
self._entered = None
|
||||
self._context = None
|
||||
self._pages = {}
|
||||
# 阶段 5：具名会话的独立 context（session → context）与
|
||||
# 每页所属 context 种类（"shared" 默认持久上下文 /
|
||||
# "isolated" 独立 context）；_storage_restored 记本次
|
||||
# context 创建时是否从 storage-state 文件恢复过。
|
||||
self._contexts = {}
|
||||
self._page_context = {}
|
||||
self._storage_restored = {}
|
||||
self._engine = "shell"
|
||||
self._mounted_full = False
|
||||
self._last_shot = {}
|
||||
@@ -463,6 +525,18 @@ class BrowserAgent:
|
||||
|
||||
# -- 浏览器生命周期 ----------------------------------------------
|
||||
def _close_context(self):
|
||||
# 阶段 5：先把各具名会话的 storage-state 落盘，再关它们
|
||||
# 的独立 context——引擎切换/服务退出时态不丢，下次重开
|
||||
# 同一会话 id 能恢复；关不掉的只跳过，不挡持久上下文关闭。
|
||||
for session in list(self._contexts):
|
||||
self._save_storage_state(session)
|
||||
ctx = self._contexts.pop(session)
|
||||
try:
|
||||
ctx.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._page_context = {}
|
||||
self._storage_restored = {}
|
||||
for page in list(self._pages.values()):
|
||||
try:
|
||||
page.close()
|
||||
@@ -476,6 +550,78 @@ class BrowserAgent:
|
||||
pass
|
||||
self._context = None
|
||||
|
||||
def _norm_session(self, session):
|
||||
"""会话 id 归一化 + 校验：返回 (session, error)。
|
||||
error 非空时调用方直接返回 {"ok": False, "error": error}。"""
|
||||
session = session or DEFAULT_SESSION
|
||||
ok, err = validate_session_id(session)
|
||||
if not ok:
|
||||
return None, err
|
||||
return session, ""
|
||||
|
||||
def _session_storage_path(self, session):
|
||||
"""具名会话的 storage-state 文件路径（阶段 5）。"""
|
||||
return os.path.join(self._sessions_dir, session,
|
||||
"storage-state.json")
|
||||
|
||||
def _save_storage_state(self, session):
|
||||
"""把具名会话的 context 状态落盘（阶段 5）。
|
||||
成功返回 True；无独立 context 或写失败返回 False（调用
|
||||
方只记录，不因此谎报 open/close 失败——存储失败由
|
||||
storage_saved 字段如实报告）。"""
|
||||
ctx = self._contexts.get(session)
|
||||
if ctx is None:
|
||||
return False
|
||||
try:
|
||||
path = self._session_storage_path(session)
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
ctx.storage_state(path=path)
|
||||
except (OSError, ValueError, TypeError, AttributeError):
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
return True
|
||||
|
||||
def _ensure_isolated_context(self, session):
|
||||
"""具名会话的独立 context（阶段 5）：已建则复用；未建则
|
||||
从持久上下文的 browser 上 new_context（同浏览器、独立
|
||||
存储分区），storage-state 文件在位时恢复。返回
|
||||
(context, restored, error)。"""
|
||||
ctx = self._contexts.get(session)
|
||||
if ctx is not None:
|
||||
return ctx, self._storage_restored.get(session, False), ""
|
||||
if self._context is None:
|
||||
return None, False, "浏览器尚未启动"
|
||||
browser = getattr(self._context, "browser", None)
|
||||
if callable(browser):
|
||||
browser = browser()
|
||||
if browser is None or not hasattr(browser, "new_context"):
|
||||
return None, False, ("当前浏览器不支持独立 context"
|
||||
"（无 browser.new_context）")
|
||||
kwargs = {"viewport": {"width": VIEWPORT_WIDTH,
|
||||
"height": VIEWPORT_HEIGHT}}
|
||||
state_path = self._session_storage_path(session)
|
||||
restored = os.path.isfile(state_path)
|
||||
if restored:
|
||||
kwargs["storage_state"] = state_path
|
||||
try:
|
||||
ctx = browser.new_context(**kwargs)
|
||||
except Exception as e:
|
||||
return None, False, "创建独立 context 失败：%s" % e
|
||||
self._contexts[session] = ctx
|
||||
self._storage_restored[session] = restored
|
||||
return ctx, restored, ""
|
||||
|
||||
def _context_for(self, session):
|
||||
"""会话对应的 context：default 走共享持久上下文，具名走
|
||||
独立 context。返回 (context, kind, error)。"""
|
||||
if session == DEFAULT_SESSION:
|
||||
return self._context, "shared", ""
|
||||
ctx, restored, err = self._ensure_isolated_context(session)
|
||||
if err:
|
||||
return None, "isolated", err
|
||||
return ctx, "isolated", ""
|
||||
|
||||
def _ensure_browser(self, engine=None):
|
||||
"""确保浏览器上下文已起（指定引擎时切换）。返回 (ok, error)。
|
||||
切换时先关旧上下文再起新上下文——两个引擎共用同一 profile
|
||||
@@ -494,6 +640,11 @@ class BrowserAgent:
|
||||
# 上下文不受影响（上面已提前返回）。
|
||||
self._profile_dir, self._profile_on_data_disk = \
|
||||
resolve_profile_dir()
|
||||
if not self._sessions_explicit:
|
||||
# 阶段 5：sessions 目录跟 profile 走——迟挂载自愈
|
||||
# 到数据盘时，storage-state 也落数据盘而非回落盘。
|
||||
self._sessions_dir = resolve_sessions_dir(
|
||||
self._profile_dir, self._profile_on_data_disk)
|
||||
if not self._full_browsers_explicit:
|
||||
self._full_browsers_dir = resolve_full_browsers_dir(
|
||||
self._profile_on_data_disk)
|
||||
@@ -563,10 +714,15 @@ class BrowserAgent:
|
||||
ok, err = validate_url(url)
|
||||
if not ok:
|
||||
return {"ok": False, "error": err}
|
||||
session = session or DEFAULT_SESSION
|
||||
session, err = self._norm_session(session)
|
||||
if err:
|
||||
return {"ok": False, "error": err}
|
||||
ok, err = self._ensure_browser()
|
||||
if not ok:
|
||||
return {"ok": False, "error": err}
|
||||
ctx, kind, err = self._context_for(session)
|
||||
if err:
|
||||
return {"ok": False, "error": err}
|
||||
try:
|
||||
old = self._pages.get(session)
|
||||
if old is not None:
|
||||
@@ -574,8 +730,9 @@ class BrowserAgent:
|
||||
old.close()
|
||||
except Exception:
|
||||
pass
|
||||
page = self._context.new_page()
|
||||
page = ctx.new_page()
|
||||
self._pages[session] = page
|
||||
self._page_context[session] = kind
|
||||
# 注意：open 不清 diff 基线——diff_shot 永远与“上一帧”
|
||||
# 逐像素比对，不管上一帧是哪一页；导航后调 diff_shot
|
||||
# 得到的正是新旧两页渲染的真实差异区（跨页局部变化
|
||||
@@ -587,10 +744,26 @@ class BrowserAgent:
|
||||
except Exception as e:
|
||||
return {"ok": False,
|
||||
"error": "打开 %s 失败：%s" % (url[:80], e)}
|
||||
return self.snapshot(session)
|
||||
# 阶段 5：具名会话导航成功后立即把 storage-state 落盘——
|
||||
# 导航里刚置的 cookie 当场进文件，不等 close；浏览器
|
||||
# crash 后重开同一会话 id 仍能恢复（Chromium 刷盘竞态
|
||||
# 绕过去了：storage_state() 读的是 context 的实时态）。
|
||||
saved = False
|
||||
if kind == "isolated":
|
||||
saved = self._save_storage_state(session)
|
||||
resp = self.snapshot(session)
|
||||
if resp.get("ok"):
|
||||
resp["context"] = kind
|
||||
if kind == "isolated":
|
||||
resp["storage_saved"] = saved
|
||||
resp["storage_restored"] = \
|
||||
self._storage_restored.get(session, False)
|
||||
return resp
|
||||
|
||||
def snapshot(self, session=None):
|
||||
session = session or DEFAULT_SESSION
|
||||
session, err = self._norm_session(session)
|
||||
if err:
|
||||
return {"ok": False, "error": err}
|
||||
page = self._page(session)
|
||||
if page is None:
|
||||
return {"ok": False,
|
||||
@@ -624,10 +797,13 @@ class BrowserAgent:
|
||||
"a11y": tree, "truncated": truncated,
|
||||
"a11y_roles": roles, "a11y_interactive": interactive,
|
||||
"a11y_chars": chars, "a11y_hash": digest,
|
||||
"session": session}
|
||||
"session": session,
|
||||
"context": self._page_context.get(session, "shared")}
|
||||
|
||||
def screenshot(self, session=None, diff_only=False):
|
||||
session = session or DEFAULT_SESSION
|
||||
session, err = self._norm_session(session)
|
||||
if err:
|
||||
return {"ok": False, "error": err}
|
||||
page = self._page(session)
|
||||
if page is None:
|
||||
return {"ok": False,
|
||||
@@ -674,7 +850,9 @@ class BrowserAgent:
|
||||
首帧（本会话无上一帧）视为整帧变化，bbox 为全视口。
|
||||
Pillow 缺失时 ok:false 并指引回落 screenshot 全帧，
|
||||
不计入计数器、不落盘、不假装裁剪。"""
|
||||
session = session or DEFAULT_SESSION
|
||||
session, err = self._norm_session(session)
|
||||
if err:
|
||||
return {"ok": False, "error": err}
|
||||
page = self._page(session)
|
||||
if page is None:
|
||||
return {"ok": False,
|
||||
@@ -747,7 +925,9 @@ class BrowserAgent:
|
||||
def verify_shot(self, session=None):
|
||||
"""三级全屏校验（阶段 4）：全帧 JPEG q70，只用于检查点。
|
||||
计 shots_verify。日常动作后用 diff_shot，检查点才调它。"""
|
||||
session = session or DEFAULT_SESSION
|
||||
session, err = self._norm_session(session)
|
||||
if err:
|
||||
return {"ok": False, "error": err}
|
||||
page = self._page(session)
|
||||
if page is None:
|
||||
return {"ok": False,
|
||||
@@ -778,16 +958,37 @@ class BrowserAgent:
|
||||
"session": session}
|
||||
|
||||
def close(self, session=None):
|
||||
session = session or DEFAULT_SESSION
|
||||
session, err = self._norm_session(session)
|
||||
if err:
|
||||
return {"ok": False, "error": err}
|
||||
page = self._pages.pop(session, None)
|
||||
if page is not None:
|
||||
try:
|
||||
page.close()
|
||||
except Exception:
|
||||
pass
|
||||
kind = self._page_context.pop(session, "shared")
|
||||
saved = False
|
||||
if kind == "isolated":
|
||||
# 阶段 5：关会话先落盘 storage-state，再关独立
|
||||
# context——态进文件，下次重开同一会话 id 恢复；
|
||||
# 只动本会话的 context，其它会话的页与 context
|
||||
# 原样不动。
|
||||
saved = self._save_storage_state(session)
|
||||
ctx = self._contexts.pop(session, None)
|
||||
if ctx is not None:
|
||||
try:
|
||||
ctx.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._storage_restored.pop(session, None)
|
||||
self._last_shot.pop(session, None)
|
||||
self._last_png.pop(session, None)
|
||||
return {"ok": True, "closed": True, "session": session}
|
||||
resp = {"ok": True, "closed": True, "session": session,
|
||||
"context": kind}
|
||||
if kind == "isolated":
|
||||
resp["storage_saved"] = saved
|
||||
return resp
|
||||
|
||||
def sessions(self):
|
||||
out = []
|
||||
@@ -800,7 +1001,9 @@ class BrowserAgent:
|
||||
url = page.url
|
||||
except Exception:
|
||||
url = ""
|
||||
out.append({"session": sid, "url": url, "title": title})
|
||||
out.append({"session": sid, "url": url, "title": title,
|
||||
"context": self._page_context.get(
|
||||
sid, "shared")})
|
||||
return {"ok": True, "sessions": out}
|
||||
|
||||
# -- 服务化：状态 / CDP / 引擎 --------------------------------------
|
||||
@@ -817,6 +1020,8 @@ class BrowserAgent:
|
||||
"cdp_port": self._cdp_port,
|
||||
"sessions": [s["session"] for s in
|
||||
self.sessions()["sessions"]],
|
||||
"isolated_sessions": sorted(self._contexts),
|
||||
"sessions_dir": self._sessions_dir,
|
||||
"mounted_full": self._mounted_full,
|
||||
"full_present": full_chromium_present(
|
||||
self._full_browsers_dir),
|
||||
|
||||
Reference in new issue
Block a user