Compare commits
137
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
aba06d9a24 | ||
|
|
5d0651e70a | ||
|
|
7524a779f2 | ||
|
|
29af9c7bbf | ||
|
|
145aa2eb5f | ||
|
|
b4270dc117 | ||
|
|
aa6179b5e1 | ||
|
|
5ca642c86e | ||
|
|
055da3c2dd | ||
|
|
34b9a34896 | ||
|
|
22ac542760 | ||
|
|
9e56c1d266 | ||
|
|
63abb3953b | ||
|
|
de925bff3b | ||
|
|
368363e096 | ||
|
|
f406398ce2 | ||
|
|
f8a6a9d258 | ||
|
|
19fe4fc7e1 | ||
|
|
8fedd7fba0 | ||
|
|
be81d658b6 | ||
|
|
3978dbbbf2 | ||
|
|
ae012a8e7c | ||
|
|
b7e2f29868 | ||
|
|
83135c283d | ||
|
|
488aedbcad | ||
|
|
bd55599859 | ||
|
|
e00c255011 | ||
|
|
e32391fd85 | ||
|
|
e131e8658d | ||
|
|
e7acc143db | ||
|
|
31edc5cda9 | ||
|
|
5428b1defe | ||
|
|
0d51c1d334 | ||
|
|
4919e107a6 | ||
|
|
f4a5ab9600 | ||
|
|
be2962525b | ||
|
|
c33bfd240e | ||
|
|
ee8d1e1e28 | ||
|
|
19b9767478 | ||
|
|
5db89d1b06 | ||
|
|
328ffdb7c6 | ||
|
|
32620c4517 | ||
|
|
b0d51b139c | ||
|
|
39ec2b4435 | ||
|
|
34a19ab9e2 | ||
|
|
6bceed484c | ||
|
|
12e72dab9b | ||
|
|
773945d136 | ||
|
|
2dc35bd331 | ||
|
|
741eb387ed | ||
|
|
6e0660e97b | ||
|
|
935b530f84 | ||
|
|
c900a0f78d | ||
|
|
cdb481092d | ||
|
|
765cff29ff | ||
|
|
dc5df30a07 | ||
|
|
24517d922b | ||
|
|
dc556b0c19 | ||
|
|
436a92ba1d | ||
|
|
72d343b2c7 | ||
|
|
24adb568da | ||
|
|
fddd08546d | ||
|
|
48e66d6c14 | ||
|
|
1e024f175f | ||
|
|
88e16ee8ac | ||
|
|
45be478d70 | ||
|
|
1156d3c754 | ||
|
|
e768959c93 | ||
|
|
b9466e556b | ||
|
|
3eac2a702e | ||
|
|
91b61e5302 | ||
|
|
a433460343 | ||
|
|
8006a6fcba | ||
|
|
0db51f8145 | ||
|
|
238ebbdbd2 | ||
|
|
f1f8c8bf62 | ||
|
|
dba6536ac2 | ||
|
|
382ec6d62e | ||
|
|
eeb10118d6 | ||
|
|
c0ab995ccc | ||
|
|
e921db3472 | ||
|
|
41e8c98512 | ||
|
|
2720acef46 | ||
|
|
be41ad2c0a | ||
|
|
06710cf47b | ||
|
|
143ab30094 | ||
|
|
0072e1ae6a | ||
|
|
d973e63b11 | ||
|
|
b4a69e1e71 | ||
|
|
1accfac844 | ||
|
|
f96960c9c5 | ||
|
|
a9e2f70d9e | ||
|
|
8ea54cf66b | ||
|
|
452fc8ea7e | ||
|
|
4e554b5c07 | ||
|
|
61e08a1a8c | ||
|
|
d553f9549c | ||
|
|
44616a03dc | ||
|
|
a605ff2f92 | ||
|
|
34ed438d0e | ||
|
|
ef0052d15a | ||
|
|
49a2f7881b | ||
|
|
0d7fc867b5 | ||
|
|
12f56cd8c8 | ||
|
|
3dc4ebe273 | ||
|
|
3b7d45c9a2 | ||
|
|
21a5fd3a2a | ||
|
|
0e665d616f | ||
|
|
4cd9b5eaa4 | ||
|
|
903f138fd9 | ||
|
|
1e9c1b36ee | ||
|
|
3d2df7dfe6 | ||
|
|
576fd50ee3 | ||
|
|
81a58daca2 | ||
|
|
f39ed3654e | ||
|
|
7ba5bfaf81 | ||
|
|
c127beaa98 | ||
|
|
7b537c31a6 | ||
|
|
82d0e84371 | ||
|
|
b391fe9320 | ||
|
|
60997b138f | ||
|
|
6525bcafd0 | ||
|
|
23725f92e6 | ||
|
|
561b3891fb | ||
|
|
0722ca3fd8 | ||
|
|
622c51e7fb | ||
|
|
3a56f83114 | ||
|
|
0602d06136 | ||
|
|
e9fa43dd8b | ||
|
|
125b436ab3 | ||
|
|
1141d0d0f8 | ||
|
|
2a569ac6e3 | ||
|
|
847d1280c4 | ||
|
|
da7a9950fb | ||
|
|
397a46f850 | ||
|
|
27f48fe1ed | ||
|
|
08484302ac |
No files matched your search
+28
@@ -0,0 +1,28 @@
|
||||
# Coverage 配置:只计量生产代码,测试本身、工作区、示例不计。
|
||||
# 阈值不在此文件写死,写在 workflow 的 --fail-under 里,便于在
|
||||
# workflow diff 中一眼看到门槛变化。
|
||||
# source 注意:不能写 host.py / client.py / run_parallel.py 这种文件
|
||||
# 路径——coverage 会把它们当模块名处理、静默丢弃(实测:那样 host.py
|
||||
# 等 3 个根文件根本不进报告,总数少 2223 行)。正确写法是目录源
|
||||
# “.”(含根模块与 cloud 包)+ deploy(非包目录,需单独列),
|
||||
# 实测恰好 58 个生产文件、22392 行,与 mypy 生产文件数一致。
|
||||
[run]
|
||||
source =
|
||||
.
|
||||
deploy
|
||||
omit =
|
||||
*/tests/*
|
||||
*/test_*.py
|
||||
cloud/tests/*
|
||||
deploy/tests/*
|
||||
tests/*
|
||||
work/*
|
||||
examples/*
|
||||
*/run_tests.py
|
||||
|
||||
[report]
|
||||
show_missing = False
|
||||
exclude_lines =
|
||||
pragma: no cover
|
||||
def __repr__
|
||||
raise NotImplementedError
|
||||
@@ -3,6 +3,23 @@ on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
# CI 扩项(审计加固第 2 步,基线 origin/main aebc623):
|
||||
# 每个新检查的生效范围与存量如实写在步骤注释里,不写空转检查:
|
||||
# - lint 全量 E,F@120:基线 111 项,见 PR 摘要存量清单,第一阶段
|
||||
# 只报告不阻塞(continue-on-error),语法 E9 仍是硬门禁;
|
||||
# - mypy 默认模式、生产范围:基线 24 项,第一阶段只报告不阻塞;
|
||||
# - bandit 生产范围:基线 287 项(High 5),第一阶段只报告不阻塞;
|
||||
# - pip-audit:只审计 requirements-ci.txt(仓库无生产依赖清单,
|
||||
# 生产代码除宿主延迟加载的 psycopg 驱动外只用标准库,见摘要),
|
||||
# 该文件 0 已知漏洞时硬通过,有漏洞即硬失败;
|
||||
# - coverage:阈值见 coverage 作业的 --fail-under,取实测值向下取整,
|
||||
# 防回归不冒充高覆盖;收紧计划见 PR 摘要。
|
||||
# 账单说明:CI 目前仍是 GitHub 账单故障秒级失败模式,账单恢复是
|
||||
# 用户在 GitHub Billing 侧的动作;本 workflow 的改动不声称改了账单、
|
||||
# 也不为变绿绕过任何检查,账单恢复后上述检查才实际生效。
|
||||
# 失败即停:同一作业内步骤按顺序执行,前一步硬失败则后一步不跑;
|
||||
# 报告类步骤用 continue-on-error + always() 上传报告,作业本身仍绿,
|
||||
# 但报告 artifact 里是失败的原始输出,不是“通过”。
|
||||
jobs:
|
||||
verify:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -11,6 +28,127 @@ jobs:
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: python3 -m unittest discover -s tests -v
|
||||
- run: python3 -m unittest discover -s cloud/tests -v
|
||||
- run: python3 -m unittest discover -s deploy/tests -v
|
||||
# 测试入口统一走 run_tests.py:它在运行器层面加仓库根路径
|
||||
# (deploy 套件再加 deploy/ 目录,见 run_tests.py docstring),
|
||||
# 测试文件本身不再写 sys.path 前置 import(ruff E402 清零轨)。
|
||||
# coverage 作业仍用下面三条 discover 原命令:运行器是子进程
|
||||
# 分套件,coverage 测不到子进程里的测试。
|
||||
- run: python3 run_tests.py -v
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: pip install -r requirements-ci.txt
|
||||
# 硬门禁:语法错误。基线 0 项,通过才继续;不是空集,
|
||||
# E9 覆盖 SyntaxError / IndentationError 等编译级错误。
|
||||
- name: lint 硬门禁(ruff E9 语法)
|
||||
run: ruff check --select E9 .
|
||||
# 报告类:全量 E,F@120,基线 111 项(F401 43、E702 40、E501 13、
|
||||
# E402 8、F841 4、F811 1、F821 1、E741 1)。第二阶段清零存量后
|
||||
# 删掉 continue-on-error,转硬门禁;删之前不声称 lint 已通过。
|
||||
- name: lint 全量报告(ruff E,F)
|
||||
continue-on-error: true
|
||||
run: ruff check . | tee ruff-report.txt
|
||||
- name: 上传 lint 报告
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ruff-report
|
||||
path: ruff-report.txt
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
typecheck:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: pip install -r requirements-ci.txt
|
||||
# 报告类:mypy 默认模式、生产范围(mypy.ini 排除 tests/work)。
|
||||
# 基线 24 项(8 文件,见 PR 摘要)。第二阶段清零后删掉
|
||||
# continue-on-error,转硬门禁;不改生产代码凑通过。
|
||||
- name: 类型检查报告(mypy 生产范围)
|
||||
continue-on-error: true
|
||||
run: mypy host.py cloud deploy client.py run_parallel.py | tee mypy-report.txt
|
||||
- name: 上传类型检查报告
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: mypy-report
|
||||
path: mypy-report.txt
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
security:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: pip install -r requirements-ci.txt
|
||||
# 报告类:bandit 生产范围,测试目录排除(测试故意用 shell/tmp)。
|
||||
# 基线 287 项:Low 235、Medium 47、High 5。High 5 = host.py 的
|
||||
# B602 shell=True 4 处(审计已知 P0)+ cloud/harness2.py 的
|
||||
# B324 sha1 非安全用途 1 处。先修 High 再转硬门禁;不通过
|
||||
# --skip 跳过 B602/B324 来假装通过。
|
||||
- name: 安全扫描报告(bandit 生产范围)
|
||||
continue-on-error: true
|
||||
# bandit 有发现即 exit 1:该失败是步骤的真实状态,作业靠
|
||||
# continue-on-error 继续到上传;不要用 `; cat` 吞掉 exit 码。
|
||||
run: bandit -r host.py cloud deploy client.py run_parallel.py -x cloud/tests,tests,deploy/tests -f txt -o bandit-report.txt
|
||||
- name: 上传安全扫描报告
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: bandit-report
|
||||
path: bandit-report.txt
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
depscan:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: pip install -r requirements-ci.txt
|
||||
# 硬门禁:审计 CI 工具链清单本身。基线 0 已知漏洞。
|
||||
# 局限如实说明:仓库没有生产依赖清单可审计——生产代码只用
|
||||
# 标准库,psycopg/psycopg2 只在宿主侧延迟加载、版本不在仓库
|
||||
# 里;给生产依赖建清单并审计是第二阶段项,不是本作业的覆盖
|
||||
# 范围,本作业不声称覆盖了生产依赖。
|
||||
- name: 依赖扫描(pip-audit,CI 工具链清单)
|
||||
run: pip-audit -r requirements-ci.txt
|
||||
coverage:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
- run: pip install -r requirements-ci.txt
|
||||
# 硬门禁:三套 unittest 在 coverage 下重跑(与 verify 作业同款
|
||||
# 测试,只是加了计量),生产代码覆盖率低于门槛即失败。
|
||||
# 门槛取实测值向下取整到 5 的倍数:本轨实测 85.2565%
|
||||
# (22437 行,已覆盖 19129 行,余量 57 行),故门槛 85%,见 PR 摘要。
|
||||
# 目的是防回归,不是 0 空转;余量仅 57 行,合轨后如回落到 85% 以下
|
||||
# 不要靠删检查或加 skip 补,先看交集处的新缺口。
|
||||
- name: 覆盖率计量与门槛
|
||||
run: |
|
||||
coverage run -m unittest discover -s tests
|
||||
coverage run --append -m unittest discover -s cloud/tests
|
||||
coverage run --append -m unittest discover -s deploy/tests
|
||||
coverage report --fail-under=85
|
||||
coverage xml -o coverage.xml
|
||||
- name: 上传覆盖率报告
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: coverage-xml
|
||||
path: coverage.xml
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
@@ -42,6 +42,16 @@ jobs:
|
||||
python3 -m unittest discover -s tests -v
|
||||
python3 -m unittest discover -s cloud/tests -v
|
||||
python3 -m unittest discover -s deploy/tests -v
|
||||
# CI 扩项在发布链路的最小硬门禁:语法 E9 + CI 工具链依赖扫描。
|
||||
# 全量 lint/mypy/bandit 报告与覆盖率门槛只在 host-verify 里跑
|
||||
# (PR 与 main 推送都会跑,走到 tag 的提交此前已过 host-verify),
|
||||
# 这里不重复跑三套报告,避免发布作业与 host-verify 的检查漂移;
|
||||
# 这里的两项是硬失败,失败则不产发布包。
|
||||
- name: CI 扩项硬门禁(语法 + 依赖扫描)
|
||||
run: |
|
||||
pip install -r requirements-ci.txt
|
||||
ruff check --select E9 .
|
||||
pip-audit -r requirements-ci.txt
|
||||
- name: 构建发布包(git archive + DEPLOYED_COMMIT + sha256)
|
||||
id: build
|
||||
run: |
|
||||
|
||||
@@ -199,19 +199,6 @@ python3 -m cloud.server # 默认 127.0.0.1:8765,生产经 nginx + TLS
|
||||
# 除 GET /health、/webhooks/*、/oauth/* 外,所有端点都要 X-API-Key 请求头
|
||||
# 任务实时链路:GET /tasks/{id}/trace(快照)与 GET /tasks/{id}/trace/stream(SSE)
|
||||
# 命令台:GET /console(每次现读 cloud/console.html,无需重启)
|
||||
# CHV 前端入口(实验,默认关闭):新建任务与新建环境的执行器下拉里有
|
||||
# cloud-hypervisor 选项;任务详情与任务列表的执行器筛选同样展示它。
|
||||
# 前端只传 executor 值,安全边界仍在后端白名单(任务接口 allowlist 含
|
||||
# cloud-hypervisor,环境接口 allowlist 为 local/docker/cloud-hypervisor);
|
||||
# 需后端有 KVM 宿主上的 CHV worker 才会认领,否则任务一直排队,带附件
|
||||
# 时建任务即被后端拒绝,拒绝文案经控制台原样展示(“提交失败:…”)。
|
||||
# 生产级前端新增:新建任务页与 Workers 区用现有 /workers 展示 CHV worker
|
||||
# 在线数(非管理员只能看到聚合数时写未知、不写成 0);零 CHV worker 在线
|
||||
# 时选 CHV 出强提示但不硬拦提交(worker 随后上线可认领,后端是最终权威);
|
||||
# 选 CHV 时附件输入禁用并可一键清空;排队中的 CHV 任务在列表与详情里给
|
||||
# “可能原因:无 CHV worker 在线”提示(基于 /workers 推断)。
|
||||
# 详情页只有实验提示,没有 VM 暂停/恢复/关机/快照等控制按钮;VM 状态、
|
||||
# IP、快照、预热池字段后端任务接口不返回,前端不展示、不编造。
|
||||
```
|
||||
|
||||
- **任务状态机**:`queued → running → completed / failed /
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
# Cloud Hypervisor 客机镜像(chv-guest)——内容清单与版本化构建说明
|
||||
|
||||
> 本目录有清单与真实构建步骤(build.sh,补缺口轨已实做),
|
||||
> 但镜像本身不在本轨内实际构建发布。原因写在前面,不藏着:
|
||||
> 当前没有带 /dev/kvm 的宿主可以构建并实测 VM 镜像(生产
|
||||
> 主机是 CVM、无嵌套虚拟化;本 VM 容器内同样无 /dev/kvm)。
|
||||
> 构建与实测见 `docs/chv-production-readiness.md` 的“需镜像
|
||||
> 构建 / 需 KVM 宿主实测”项,绝不在本轨声称镜像已建成、
|
||||
> 已发布、已跑通。
|
||||
|
||||
## 1. 镜像定位
|
||||
|
||||
- 基础:Ubuntu Minimal 24.04 官方云镜像(轻量口径,2026-11-06
|
||||
用户定:停掉 5.5G 全量版,不再对齐 Manus,不做三面板/
|
||||
多租户/生产级全套;只装 codex + Python + SSH + cloud-init
|
||||
+ 薄客机 agent + git + gt + 最小中文字体)。
|
||||
- 载体:Cloud Hypervisor 直接内核启动(vmlinux)+ raw 磁盘
|
||||
镜像,执行器用 `CHV_KERNEL` / `CHV_ROOTFS` 指向它们。
|
||||
- 客内 host.py 位置:`/app/host.py`(与 Docker 的 `/app:ro`
|
||||
同款约定,只是载体从容器只读挂载换成 VM 镜像内置代码;
|
||||
代码版本 = 构建时仓库 commit,见 §3 版本化)。
|
||||
- 绝不进镜像的东西:任何模型 key、系统 key、用户账号数据、
|
||||
生产凭证。模型 key 由客机 cloud-init 预置的运行环境变量名
|
||||
注入(与 Docker 的 `-e <名>` 同款惯例),不写进镜像层。
|
||||
|
||||
## 2. 内容清单(轻量口径,2026-11-06 用户定)
|
||||
|
||||
| 类别 | 内容 | 来源/版本口径 |
|
||||
| --- | --- | --- |
|
||||
| 语言与工具链 | Python 3.12、git、curl、ca-certificates、xz-utils、fontconfig、openssh-client(Node.js 不装:Playwright 驱动是 Python playwright,见 PLAYWRIGHT=1 分支;任务镜像 Dockerfile 里也没有 Node.js,本清单不虚构它) | 构建时钉小版本并记入构建日志;apt 用 --no-install-recommends(B-21:默认装 Recommends 会连带拉入 gcc 全套工具链顶满根分区) |
|
||||
| 文档/图片工具 | 已裁剪:pandoc、poppler-utils、latexmk、texlive-latex-base、python-docx、openpyxl、python-pptx、PyMuPDF、Pillow 不在轻量镜像内 | 全量版曾照抄 `cloud/docker/Dockerfile` 的 apt/pip 清单,轻量版按用户决定砍掉 |
|
||||
| agent 引擎 | codex、codex-code-mode-host、codex-resources/bwrap(成套,约 367M,缺 sibling 直接构建失败) | codex 0.164.0-alpha.9(与生产任务镜像一致);`CODEX_BIN_DIR` 由构建方提供,bin/ 不进仓库 |
|
||||
| 浏览器 | Playwright + chromium-headless-shell(默认装,不要完整 Chromium) | Python playwright 装当时的 headless-shell,版本以 ms-playwright 缓存目录名记入 BUILDINFO.versions.chromium;浏览器在系统路径 /usr/local/share/ms-playwright(B-22:virt-customize 里 HOME=/,默认缓存会落到根目录 /.cache),agent 运行时须设 PLAYWRIGHT_BROWSERS_PATH 指向它;单独钉死见 §3:CHROMIUM_HEADLESS_SHELL_REV 设为纯数字修订号即钉死,未设时只记录、不强制 |
|
||||
| 客机基础 | openssh-server(SSH 通道用)、cloud-init(qemu-guest-agent 已裁剪,CHV 不用它) | sshd 只允许 key 登录,禁止密码登录——build.sh 写入 /etc/ssh/sshd_config.d/99-chv.conf 并经 sshd -t + grep 断言(feat-chv-secfix,M8,此前只是文档声称) |
|
||||
| Gitea 同步 | gt 1.0.0(官方 install.sh + SHA256 校验)、chv-sync(启动时 clone、显式 pull/push 的同步脚本) | 凭证走 seed 盘 /etc/chv/guest-env(0600)投递,不进镜像;见 §7 |
|
||||
| 任务代码 | 仅 `/app/host.py` 一个文件(host.py 只用标准库,不依赖 cloud/;旧版把整个 cloud/ 拷进 /app/,客内可读全套 server/worker 代码,M4 收尾已收紧为最小集) | 版本见 §3(repo_commit),不挂载宿主代码目录(VM 与宿主无共享路径) |
|
||||
|
||||
中文 LaTeX 说明:与 Docker 镜像同款口径——xelatex + CJK
|
||||
字体会再增数百 MB,本清单不默认包含;需要时单独一版镜像,
|
||||
不在本轨悄悄加。字体只留最小的 fonts-wqy-microhei(中文能
|
||||
看),不要全套 fonts-noto-cjk。
|
||||
|
||||
## 3. 版本化
|
||||
|
||||
镜像 tag 格式(构建时三项缺一即失败,不生成无版本镜像):
|
||||
|
||||
```
|
||||
chv-guest-<ubuntu 版本>-<codex 版本>-<仓库 commit 短 hash>
|
||||
```
|
||||
|
||||
例如(示例格式,非已建成):`chv-guest-24.04-0.164.0-alpha.9-aebc623`。
|
||||
|
||||
构建产物清单(构建方必须输出,执行器用它做 preflight 之外的
|
||||
版本核对,v1 执行器暂不校验 tag,见就绪清单):
|
||||
|
||||
- `vmlinux`(与 Ubuntu 云镜像配套的内核)
|
||||
- `disk.raw`(根文件系统镜像)
|
||||
- `BUILDINFO.json`:`{ubuntu, codex_version, repo_commit, built_at,
|
||||
contents: [清单逐项名]}`——contents 只有名字列表;逐项版本
|
||||
在 `versions` 里(B-02:dpkg + pip freeze 从客机读回,
|
||||
chromium 以 ms-playwright 缓存目录名记,gt 以 gt_version 记),
|
||||
`generate_lockfile.py` / `check_lockfile.py` 以它为唯一来源。
|
||||
Chromium 单独钉死:构建时设 CHROMIUM_HEADLESS_SHELL_REV
|
||||
为纯数字修订号(如 1248),buildinfo.py 校验读回的缓存
|
||||
目录名须恰好是 `chromium_headless_shell-<rev>`,否则构建
|
||||
失败,BUILDINFO.chromium_pinned_rev 记该值;未设时记 null、
|
||||
只记录不强制,构建输出明说“未钉死”。真实 lockfile 产出
|
||||
须等 V-9 真机构建(本机无 /dev/kvm,版本号绝不编造)。
|
||||
|
||||
## 4. 构建脚本(补缺口轨:已从骨架改成真实步骤)
|
||||
|
||||
见 `build.sh`:参数校验 → 下载 Ubuntu Minimal 云镜像(SHA256
|
||||
必填)→ qemu-img 转 raw(轻量版不扩盘,底座根分区 2.4G 够用;
|
||||
全量版 B-16 的 +12G 扩盘已删除)→ virt-customize 装 §2 清单、
|
||||
拷入 codex 成套二进制与 host.py(仅此一个文件,不拷 cloud/)
|
||||
到 /app、装 gt 1.0.0、拷入
|
||||
chv-sync 并写 cloud-init 启动钩子、写 host key 不重生成配置 →
|
||||
virt-get-kernel 取内核 → 生成并校验 BUILDINFO.json。
|
||||
`./build.sh --dry-run out/` 只打印命令、不执行、不要求
|
||||
KVM;全量模式在无 /dev/kvm 的机器上第一步就明确退出
|
||||
(`CHV_GUEST_BUILD needs a KVM host`),绝不假装构建成功。
|
||||
全量版曾在 WSL KVM 宿主(ajpkmq6i)上跑过四轮真机构建,
|
||||
轻量版沿用同一宿主与流程,构建记录见 `~/workspace/chv-prod-audit/`。
|
||||
|
||||
## 5. Host key 首启注入硬性要求(补缺口轨新增)
|
||||
|
||||
执行器的 per-VM 预置方案(见 `cloud/chv/hostkeys.py`)能
|
||||
跑通的前提是客机镜像必须满足以下全部,否则预置在真机
|
||||
上跑不通,执行器会 fail-fast,不回落成弱方案:
|
||||
- 装 cloud-init 且支持 NoCloud seed(seed 磁盘(CIDATA ISO,feat-chv-fix 起挂进 VmConfig
|
||||
第二块只读盘)由执行器在 VM 启动前生成,含 meta-data 与 user-data);
|
||||
- 首启时先应用 seed 再启动 sshd,seed 里的 user-data
|
||||
把该 VM 的 host key 对写进 /etc/ssh/(write_files,
|
||||
0600);seed 里绝不能带 `runcmd:`——user-data 的
|
||||
runcmd 会整体替换镜像 `/etc/cloud/cloud.cfg.d/` 里
|
||||
的 runcmd(cloud-init 不跨配置类型合并),旧版 seed
|
||||
的 restart ssh 两行就这样顶掉了 `99-chv-sync.cfg`
|
||||
的 `chv-sync boot` 钩子(sync-test 首轮实测);sshd
|
||||
重启由镜像自己的 runcmd(`99-chv-sync.cfg`,cloud-final
|
||||
阶段)做,见 `cloud/chv/hostkeys.py` 根因禁令;
|
||||
- 镜像里已钉死 `ssh_deletekeys: false`(build.sh 已写),首启
|
||||
不得无条件重新生成 host key——重生成会使预置的 known_hosts
|
||||
立即失效,SSH 钉死直接拒绝,这是设计,不是 bug;无预置 key
|
||||
时首启 bootcmd 只在缺 key 时 `ssh-keygen -A` 生成(B-23:
|
||||
曾写 `ssh_genkeytypes: []` 彻底关掉首启生成,客机 sshd 起
|
||||
不来,已删,绝不再写);
|
||||
- sshd 只允许 key 登录,禁止密码登录(build.sh 写入
|
||||
/etc/ssh/sshd_config.d/99-chv.conf:PasswordAuthentication no、
|
||||
PermitRootLogin prohibit-password、ChallengeResponseAuthentication no、
|
||||
KbdInteractiveAuthentication no、PubkeyAuthentication yes,并经
|
||||
sshd -t + grep 断言;feat-chv-secfix,M8);
|
||||
- seed 磁盘是 per-VM 敏感物,执行器在 VM 关闭/
|
||||
回收时删除,镜像构建方不得把任何一把 host key 固化
|
||||
进镜像(所有 VM 同 key 是安全事故)。
|
||||
|
||||
## 6. 与执行器的对接
|
||||
|
||||
- 执行器 preflight 只校验内核/镜像文件存在且可读,不校验镜像
|
||||
内容版本——内容版本核对(BUILDINFO)是镜像发布流程的责任,
|
||||
在就绪清单里标为“需镜像构建”项,不在本轨假装已做。
|
||||
- 每任务磁盘:执行器 copy 模式下把 `disk.raw` 复制为每任务
|
||||
磁盘,任务之间文件系统独立;reuse 模式仅单租户实验,preflight 拦截(须同时设 CHV_REUSE_SINGLE_TENANT_EXPERIMENT=1 并审计放行,feat-chv-fix)。
|
||||
- 快照基准:`CHV_SNAPSHOT_URL` 指向的快照必须由本镜像的干净
|
||||
启动状态制成(pause 后 snapshot),预热池用它做干净 VM 来源。
|
||||
|
||||
## 7. Gitea 双向同步(用户工作区口径,2026-11-06 追加)
|
||||
|
||||
主角是用户,不是 agent:用户在 Gitea 上的仓库在 VM 启动时
|
||||
clone 到 `~/workspace`,用完再同步回 Gitea;VM 是用户的工作
|
||||
区,Gitea 是权威远端。同步口径沿用 Git Workspace Platform
|
||||
已定的规则:分叉或冲突时失败关闭,不强制覆盖、不自动
|
||||
rebase、不静默丢提交。
|
||||
|
||||
- 凭证:seed 盘投递 `/etc/chv/guest-env`(0600,root 所有),
|
||||
含 `GITEA_URL` / `GITEA_USER` / `GITEA_TOKEN` / `SYNC_REPO`
|
||||
/ 可选 `SYNC_BRANCH` / 可选 `GITEA_EMAIL`。token 绝不进镜像、
|
||||
代码、日志;`chv-sync boot`(root,cloud-init runcmd 调)把
|
||||
token 写进 `/home/chv/.git-credentials`(0600,git credential
|
||||
store 标准机制),再以 chv 身份 clone;`pull` / `push` /
|
||||
`status` 以 chv 身份跑,只用已落盘的 credential,不再读
|
||||
guest-env。无 guest-env 的启动静默不同步,VM 照常用。
|
||||
- 行为:`boot` 只在未 clone 时 clone,已 clone 则静默退出,
|
||||
不自动 pull(避免重启时覆盖进行中的工作);`pull` 要求
|
||||
工作树干净且只接受 fast-forward;`push` 无 upstream 时
|
||||
`push -u origin HEAD`,否则普通 push,非 fast-forward 即
|
||||
失败,绝不 `--force`。
|
||||
- gt 1.0.0 随镜像装好(官方 install.sh + SHA256 校验),登录
|
||||
不 baked:运行时用 `GITEA_TOKEN` + `GITEA_CLI_HOST` 环境变量
|
||||
即可用(`gt repo list` 等),或在 VM 里 `gt auth login`。
|
||||
- 验收(用户视角):VM 启动后仓在 `~/workspace` 且是最新
|
||||
HEAD;VM 内改文件 commit push 后 Gitea 网页可见该提交;
|
||||
Gitea 上改一次后 VM 内 pull 能同步回来。双向都通才算集成通。
|
||||
- 集成(2026-11-07,第二轮第一棒):执行器生成 seed 时自动写
|
||||
guest-env 已做(`cloud/chv/gitea_sync.py` + `wake_vm`):
|
||||
GITEA_TOKEN 进密钥集(只进 seed,绝不进 argv),其余 5 键
|
||||
也随 seed 进 guest-env;来源是宿主 `CHV_GITEA_*` 环境变量
|
||||
或任务 `gitea_sync` dict;必填 4 键缺失时起 VMM 前
|
||||
fail-closed(`gitea_sync_incomplete`),无同步变量时静默
|
||||
不同步。客机用户与静态网络由 seed 附加项供给:
|
||||
`CHV_SEED_USER_DATA_EXTRA_FILE`(含 `users:` 块的文件)与
|
||||
`CHV_SEED_NETWORK_CONFIG_FILE`(NoCloud network-config),
|
||||
路径配了但文件读不出时 wake 前 fail-closed
|
||||
(`seed_extra_missing`);附加 user-data 里禁 `runcmd:`。
|
||||
专用 token 与测试仓名须用户给,本轨不自编 token、不动
|
||||
生产 Gitea 配置。
|
||||
@@ -1,379 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chv-authd:客内凭证守护进程(feat-chv-muse-p3)。
|
||||
|
||||
对齐 Muse 官方安全文:authd 保管凭证(含 token,存于用户 VM
|
||||
而非集中基础设施),代理只见替身令牌;凭证存入运行时单元之外
|
||||
的安全凭证存储,代理不可见,仅在需要时注入。
|
||||
|
||||
本进程跑在客内、运行时单元之外(root 身份的 systemd 服务,
|
||||
见 chv-authd.service;P2 的 nspawn 单元落地后,codex 在单元内,
|
||||
本进程在单元外——单元内 root 映射为非特权用户,读不到本进程的
|
||||
socket 与 seed 文件)。它只做三件事:
|
||||
|
||||
1. 启动时读 /etc/chv/authd-seed(0600 root,seed 投递),把
|
||||
GITEA_TOKEN 持在内存里(cloud.chv.authd.CredentialVault),
|
||||
除此之外不落盘、不进日志;
|
||||
2. 在 Unix socket /run/chv-authd/authd.sock 上接受 git credential
|
||||
helper 的请求(SO_PEERCRED 只放行 root 与 chv 用户;socket
|
||||
用 group 权限——目录 0750、socket 0660、组为第一个非 root
|
||||
放行 uid 的主组,round-11 真机修复层 3,探针用的 0666
|
||||
不上生产);
|
||||
3. 每次放凭证前先过 Sentinel 评估(cloud.chv.sentinel,主机/
|
||||
方法/路径/解析后 IP/SSRF),再按次释放(单次、300 秒 TTL),
|
||||
全程审计(审计里无值)。
|
||||
|
||||
只用标准库。cloud/chv/authd.py 与 cloud/chv/sentinel.py 是叶子
|
||||
模块(只 import 标准库),镜像构建时随本文件一起拷进
|
||||
/usr/local/lib/chv-authd/(见 docs/chv-muse-p3-gap.md §2 的接入
|
||||
说明),本文件把该目录加进 sys.path 后 import。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import socketserver
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
|
||||
DEFAULT_LIB_DIR = "/usr/local/lib/chv-authd"
|
||||
DEFAULT_SEED_PATH = "/etc/chv/authd-seed"
|
||||
DEFAULT_SOCKET_PATH = "/run/chv-authd/authd.sock"
|
||||
DEFAULT_LOG_PATH = "/var/log/chv-authd.log"
|
||||
|
||||
# seed 文件里允许的键(与 cloud.chv.gitea_sync.GITEA_SYNC_KEYS
|
||||
# 同款白名单;未知键一律丢掉,调用方拼错的键绝不静默进保险库)。
|
||||
SEED_KEYS = frozenset({
|
||||
"GITEA_URL", "GITEA_USER", "GITEA_TOKEN",
|
||||
"SYNC_REPO", "SYNC_BRANCH", "GITEA_EMAIL",
|
||||
})
|
||||
|
||||
# SO_PEERCRED 放行的 uid:root + chv(git 以 chv 身份跑 helper)。
|
||||
# 可用环境变量 CHV_AUTHD_ALLOWED_UIDS 覆盖(逗号分隔)。
|
||||
DEFAULT_ALLOWED_UIDS = (0, 1000)
|
||||
|
||||
|
||||
def _socket_group_gid(allowed_uids, socket_group=None):
|
||||
"""socket 共享组的 gid:socket_group 显式给时优先(int 即 gid,
|
||||
str 即组名);否则取第一个非 root 放行 uid 的主组。解析不出
|
||||
(uid 无 passwd 条目、组名不存在)即 None——调用方回落
|
||||
root-only 权限(目录 0700、socket 0600),绝不把 socket 放给
|
||||
全员可读(round-11 真机修复层 3:探针用 0666,生产用 group
|
||||
权限——chv 在组内,目录 0750 + socket 0660 + chgrp)。"""
|
||||
import grp
|
||||
import pwd
|
||||
if socket_group is not None:
|
||||
if isinstance(socket_group, int):
|
||||
return socket_group
|
||||
try:
|
||||
return grp.getgrnam(socket_group).gr_gid
|
||||
except (KeyError, OverflowError):
|
||||
return None
|
||||
for uid in sorted(allowed_uids or ()):
|
||||
if uid == 0:
|
||||
continue
|
||||
try:
|
||||
return pwd.getpwuid(uid).pw_gid
|
||||
except (KeyError, OverflowError):
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _set_group_perms(path, gid, mode):
|
||||
"""目录/socket 的组权限:先 chgrp(失败只放过,不中断启动),
|
||||
再 chmod。gid 为 None 时只 chmod(root-only 模式)。"""
|
||||
if gid is not None:
|
||||
try:
|
||||
os.chown(path, -1, gid)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
os.chmod(path, mode)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _load_seed(path):
|
||||
"""读 seed 文件(KEY=VALUE,每行一项)→ 白名单后的 dict。
|
||||
文件不存在即 FileNotFoundError(调用方 fail-fast,绝不带着
|
||||
“无凭证”的 authd 继续跑、让 git 静默拿不到凭证)。"""
|
||||
values = {}
|
||||
with open(path) as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.split("=", 1)
|
||||
key = key.strip()
|
||||
if key in SEED_KEYS and value.strip():
|
||||
values[key] = value.strip()
|
||||
return values
|
||||
|
||||
|
||||
def _audit(log_path, record):
|
||||
"""审计落盘(JSON 行,0600):只含绑定与 verdict,不含值。
|
||||
落盘失败只记 stderr,绝不炸守护进程(审计失败不改变放行
|
||||
决策是 worker 层的既定语义,见 executor._emit)。"""
|
||||
line = json.dumps(record, sort_keys=True)
|
||||
try:
|
||||
with open(log_path, "a") as fh:
|
||||
fh.write(line + "\n")
|
||||
except OSError as exc:
|
||||
print("chv-authd: audit 落盘失败:%s" % exc, file=sys.stderr)
|
||||
print("chv-authd: %s" % line, file=sys.stderr)
|
||||
|
||||
|
||||
def _operation_for_path(path):
|
||||
"""git 智能 HTTP 路径 → 操作(helper 不知道 git 操作,守护
|
||||
进程按路径推断;Sentinel 的 L7 规则已把路径限定在这三条内,
|
||||
推断不到的一律拒绝):
|
||||
- .../git-receive-pack → push(写)
|
||||
- .../git-upload-pack → pull(读)
|
||||
- .../info/refs → pull(读,clone 也走这条)
|
||||
- 裸 .../ → pull(读;round-11 真机修复层 4:git 智能
|
||||
HTTP 的发现请求只发裸 .git 路径,裸路径一律当 pull——
|
||||
只读默认;push 只走显式的 git-receive-pack)"""
|
||||
if path.endswith(".git/git-receive-pack"):
|
||||
return "POST", "push"
|
||||
if path.endswith(".git/git-upload-pack"):
|
||||
return "POST", "pull"
|
||||
if path.endswith(".git/info/refs"):
|
||||
return "GET", "pull"
|
||||
if path.endswith(".git"):
|
||||
return "GET", "pull"
|
||||
return None, None
|
||||
|
||||
|
||||
def _repo_from_path(path):
|
||||
"""从 git 路径取仓库名:/admin/repo.git/info/refs → admin/repo。
|
||||
取不出即 None(调用方拒绝)。"""
|
||||
parts = [p for p in (path or "").split("/") if p]
|
||||
if len(parts) < 2:
|
||||
return None
|
||||
repo = parts[1]
|
||||
if not repo.endswith(".git"):
|
||||
return None
|
||||
return "%s/%s" % (parts[0], repo[:-len(".git")])
|
||||
|
||||
|
||||
class AuthdHandler(socketserver.BaseRequestHandler):
|
||||
"""单次请求处理(每连接一 JSON 行):鉴权 → Sentinel 评估 →
|
||||
按次释放 → 回包。任何一步拒绝都回 {ok:false},不含值。"""
|
||||
|
||||
def handle(self):
|
||||
server = self.server
|
||||
try:
|
||||
creds = self.request.getsockopt(
|
||||
socket.SOL_SOCKET, socket.SO_PEERCRED,
|
||||
struct.calcsize("3i"))
|
||||
_pid, uid, _gid = struct.unpack("3i", creds)
|
||||
except (OSError, struct.error):
|
||||
self._reply({"ok": False, "error": "peercred_failed"})
|
||||
return
|
||||
if uid not in server.allowed_uids:
|
||||
server.audit({"event": "authd_peer_rejected",
|
||||
"uid": uid})
|
||||
self._reply({"ok": False, "error": "uid_not_allowed"})
|
||||
return
|
||||
try:
|
||||
buf = b""
|
||||
while b"\n" not in buf:
|
||||
chunk = self.request.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
buf += chunk
|
||||
if len(buf) > 65536:
|
||||
break
|
||||
req = json.loads(buf.decode("utf-8"))
|
||||
except (OSError, ValueError):
|
||||
self._reply({"ok": False, "error": "bad_request"})
|
||||
return
|
||||
self._serve(req)
|
||||
|
||||
def _reply(self, obj):
|
||||
try:
|
||||
self.request.sendall(
|
||||
(json.dumps(obj) + "\n").encode("utf-8"))
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _serve(self, req):
|
||||
server = self.server
|
||||
connector = req.get("connector")
|
||||
host = (req.get("host") or "").lower()
|
||||
path = req.get("path") or "/"
|
||||
if connector != "gitea" or not host or not path:
|
||||
server.audit({"event": "authd_credential_refused",
|
||||
"kind": "bad_request_fields",
|
||||
"connector": connector})
|
||||
self._reply({"ok": False, "error": "bad_request_fields"})
|
||||
return
|
||||
method, operation = _operation_for_path(path)
|
||||
if method is None:
|
||||
server.audit({"event": "authd_credential_refused",
|
||||
"kind": "path_not_allowed",
|
||||
"connector": connector, "host": host,
|
||||
"path": path})
|
||||
self._reply({"ok": False, "error": "path_not_allowed"})
|
||||
return
|
||||
repo = _repo_from_path(path)
|
||||
if repo is None:
|
||||
server.audit({"event": "authd_credential_refused",
|
||||
"kind": "repo_unparseable",
|
||||
"connector": connector, "host": host})
|
||||
self._reply({"ok": False, "error": "repo_unparseable"})
|
||||
return
|
||||
# Sentinel 评估:主机/方法/路径/解析后 IP/SSRF。
|
||||
from cloud.chv.sentinel import EgressRequest
|
||||
scheme = "https"
|
||||
port = 443
|
||||
egress = EgressRequest(connector, method, scheme, host,
|
||||
port, path)
|
||||
decision = server.policy.evaluate(
|
||||
egress, resolve=getattr(server, "resolve", None))
|
||||
if decision.verdict != "allow":
|
||||
record = decision.audit(egress)
|
||||
server.audit(record)
|
||||
self._reply({"ok": False,
|
||||
"error": "sentinel_" + decision.verdict,
|
||||
"reasons": decision.reasons})
|
||||
return
|
||||
server.audit(decision.audit(egress))
|
||||
# 按次释放:绑定须与入库作用域一致(主机/仓库),单次有效。
|
||||
try:
|
||||
release = server.vault.release(
|
||||
connector, "GITEA_TOKEN", host, repo, operation)
|
||||
except Exception as exc: # AuthdRefusal:kind 在异常上
|
||||
kind = getattr(exc, "kind", "refused")
|
||||
server.audit({"event": "authd_credential_refused",
|
||||
"kind": kind, "connector": connector,
|
||||
"host": host, "repo": repo,
|
||||
"operation": operation})
|
||||
self._reply({"ok": False, "error": kind})
|
||||
return
|
||||
server.audit(server.vault.audit_release(release))
|
||||
try:
|
||||
value = server.vault.consume(release.release_id)
|
||||
except Exception as exc:
|
||||
kind = getattr(exc, "kind", "refused")
|
||||
server.audit({"event": "authd_credential_refused",
|
||||
"kind": kind, "connector": connector,
|
||||
"host": host, "repo": repo,
|
||||
"operation": operation})
|
||||
self._reply({"ok": False, "error": kind})
|
||||
return
|
||||
self._reply({"ok": True, "username": server.gitea_user,
|
||||
"password": value,
|
||||
"expires_at": release.expires_at})
|
||||
|
||||
|
||||
class AuthdServer(socketserver.ThreadingUnixStreamServer):
|
||||
"""Unix socket 服务(线程化,daemon_threads=True 避免僵尸线程)。"""
|
||||
|
||||
daemon_threads = True
|
||||
allow_reuse_address = False
|
||||
|
||||
def __init__(self, socket_path, vault, policy, gitea_user,
|
||||
allowed_uids, log_path, resolve=None,
|
||||
socket_group=None):
|
||||
self.vault = vault
|
||||
self.policy = policy
|
||||
self.gitea_user = gitea_user
|
||||
self.allowed_uids = frozenset(allowed_uids)
|
||||
self.log_path = log_path
|
||||
# 解析器可注入(测试用假解析器;不给时 Sentinel 用真实
|
||||
# getaddrinfo)。与执行器的 sentinel_resolver 同款口径。
|
||||
self.resolve = resolve
|
||||
# round-11 真机修复(层 3):socket 用 group 权限。
|
||||
# gid 取 socket_group 显式值或第一个非 root 放行 uid 的
|
||||
# 主组;解析不出即 None,回落 root-only(0700/0600)。
|
||||
self.socket_gid = _socket_group_gid(allowed_uids, socket_group)
|
||||
directory = os.path.dirname(socket_path)
|
||||
if directory:
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
if self.socket_gid is not None:
|
||||
_set_group_perms(directory, self.socket_gid, 0o750)
|
||||
else:
|
||||
_set_group_perms(directory, None, 0o700)
|
||||
if os.path.exists(socket_path):
|
||||
os.unlink(socket_path)
|
||||
super().__init__(socket_path, AuthdHandler)
|
||||
if self.socket_gid is not None:
|
||||
_set_group_perms(socket_path, self.socket_gid, 0o660)
|
||||
else:
|
||||
_set_group_perms(socket_path, None, 0o600)
|
||||
|
||||
def audit(self, record):
|
||||
_audit(self.log_path, record)
|
||||
|
||||
|
||||
def build_server(seed_path=DEFAULT_SEED_PATH,
|
||||
socket_path=DEFAULT_SOCKET_PATH,
|
||||
log_path=DEFAULT_LOG_PATH,
|
||||
lib_dir=DEFAULT_LIB_DIR,
|
||||
allowed_uids=None,
|
||||
resolve=None,
|
||||
socket_group=None):
|
||||
"""建服务(不启动):读 seed → 入库 → 建策略 → 建 socket 服务。
|
||||
|
||||
seed 缺必填键(GITEA_URL/GITEA_USER/GITEA_TOKEN/SYNC_REPO)
|
||||
即 ValueError(只含缺失键名,不含值);无 seed 文件即
|
||||
FileNotFoundError——authd 绝不带着“无凭证”启动。"""
|
||||
if lib_dir not in sys.path:
|
||||
sys.path.insert(0, lib_dir)
|
||||
from cloud.chv.authd import CredentialVault
|
||||
from cloud.chv.sentinel import SentinelPolicy
|
||||
seed = _load_seed(seed_path)
|
||||
missing = sorted({"GITEA_URL", "GITEA_USER", "GITEA_TOKEN",
|
||||
"SYNC_REPO"} - set(seed))
|
||||
if missing:
|
||||
raise ValueError(
|
||||
"authd seed 不完整,缺失:%s" % ",".join(missing))
|
||||
from urllib.parse import urlparse
|
||||
gitea_host = (urlparse(seed["GITEA_URL"]).hostname or "").lower()
|
||||
if not gitea_host:
|
||||
raise ValueError("authd seed 的 GITEA_URL 主机解析不出")
|
||||
vault = CredentialVault()
|
||||
vault.store("gitea", "GITEA_TOKEN", seed["GITEA_TOKEN"],
|
||||
host=gitea_host, repo=seed["SYNC_REPO"])
|
||||
extra = [h.strip().lower()
|
||||
for h in (os.environ.get("CHV_SENTINEL_ALLOW_HOSTS")
|
||||
or "").split(",") if h.strip()]
|
||||
policy = SentinelPolicy(
|
||||
allow_hosts={gitea_host} | set(extra))
|
||||
uids = allowed_uids
|
||||
if uids is None:
|
||||
raw = os.environ.get("CHV_AUTHD_ALLOWED_UIDS") or ""
|
||||
uids = [int(p) for p in raw.split(",") if p.strip()] \
|
||||
if raw.strip() else list(DEFAULT_ALLOWED_UIDS)
|
||||
server = AuthdServer(socket_path, vault, policy,
|
||||
seed["GITEA_USER"], uids, log_path,
|
||||
resolve=resolve, socket_group=socket_group)
|
||||
return server
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
"""前台运行(systemd Type=simple 直接跑本进程,不 daemonize)。"""
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--seed", default=DEFAULT_SEED_PATH)
|
||||
parser.add_argument("--socket", default=DEFAULT_SOCKET_PATH)
|
||||
parser.add_argument("--log", default=DEFAULT_LOG_PATH)
|
||||
parser.add_argument("--lib", default=DEFAULT_LIB_DIR)
|
||||
args = parser.parse_args(argv)
|
||||
server = build_server(seed_path=args.seed,
|
||||
socket_path=args.socket,
|
||||
log_path=args.log, lib_dir=args.lib)
|
||||
print("chv-authd: listening on %s" % args.socket,
|
||||
file=sys.stderr)
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
server.server_close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -1,14 +0,0 @@
|
||||
"""chv-authd 叶子树的 stub cloud/chv/__init__.py(round-11 真机修复,层 1)。
|
||||
|
||||
真实的 cloud/chv/__init__.py import executor/orphans(进而要整套
|
||||
cloud 包),拷进 authd 的最小叶子树(/usr/local/lib/chv-authd)
|
||||
会让 chv-authd 在 import 时 ModuleNotFoundError 而 crash loop:
|
||||
先是 cloud.chv.errors 缺失,补上 errors.py 后又会在 executor
|
||||
上失败。探针 fixprobe3 的形状:叶子树里只放 errors.py + 仅导出
|
||||
ChvFailure 的 stub __init__.py——authd 只需要这个名字。
|
||||
build.sh 拷本文件作叶子树的 cloud/chv/__init__.py,不拷真实的
|
||||
包 __init__。
|
||||
"""
|
||||
from cloud.chv.errors import ChvFailure
|
||||
|
||||
__all__ = ["ChvFailure"]
|
||||
@@ -1,18 +0,0 @@
|
||||
# CHV 浏览器静态拦截名单(对齐 Muse P4)
|
||||
#
|
||||
# 格式:一行一个条目,# 开头是注释,空行跳过。
|
||||
# example.com 拦截该域名及其全部子域
|
||||
# .example.com 同上(前导点可写可不写)
|
||||
# 匹配规则见 chv_browser_guard.py 的 check_blocklist。
|
||||
#
|
||||
# 来源与更新机制(诚实口径):
|
||||
# 本文件随镜像发布,是静态名单。首版只含 RFC 2606 保留 .test 域
|
||||
# 的格式示例与自测条目(.test 域名保证不可解析,永不误拦真实站),
|
||||
# 不编造真实恶意域名。生产名单由运维在构建时用公开情报源
|
||||
# (如 URLhaus / Google Safe Browsing 导出)的域名替换本文件后
|
||||
# 重建镜像;更新 = 换文件 + 重建镜像,无在线更新(在线更新列为
|
||||
# 后续,不冒充已做)。名单缺失时视为空名单,不静默失败。
|
||||
#
|
||||
# 自测条目(.test 保留域,仅供测试与格式示例):
|
||||
malware.test
|
||||
.phishing.test
|
||||
@@ -1,548 +0,0 @@
|
||||
#!/bin/bash
|
||||
# chv-guest 轻量镜像构建脚本(2026-11-06 用户定:停掉 5.5G 全量版,
|
||||
# 按轻量口径重开;不再对齐 Manus,不做三面板/多租户/生产级全套)。
|
||||
#
|
||||
# 底座:Ubuntu Minimal 24.04 官方 qcow2(248MB,
|
||||
# https://cloud-images.ubuntu.com/minimal/releases/noble/release/
|
||||
# ubuntu-24.04-minimal-cloudimg-amd64.img——minimal 发布在 noble
|
||||
# 代号目录下,不是 releases/24.04)。只装 codex 0.164.0-alpha.9
|
||||
# linux-x64 + Python + SSH + cloud-init + 一个薄客机 agent
|
||||
# (/app/host.py,仅此一个文件——host.py 只用标准库,不依赖
|
||||
# cloud/;旧版把整个 $REPO_DIR/cloud/ 拷进 /app/,客内 agent
|
||||
# 可读全套 server/worker 代码,M4 收尾已收紧为最小集)。
|
||||
# 浏览器:Playwright +
|
||||
# chromium-headless-shell(不要完整 Chromium);字体只要最小的
|
||||
# fonts-wqy-microhei(中文能看),不要全套 fonts-noto-cjk。
|
||||
# 目标:qcow2 实际占用 ≤1GB,raw ≤2GB(见下注)。
|
||||
# 控制面只留最小的:现有 cloud/chv/executor.py 起停 VM + SSH
|
||||
# 进去调 codex + Playwright 截图。
|
||||
#
|
||||
# Gitea 双向同步(用户工作区口径,2026-11-06 追加):镜像里加
|
||||
# git + gt 1.0.0(官方 install.sh + SHA256 校验,跟 170/172
|
||||
# 同款),配好 git user,不含真实 token,不设 credential helper
|
||||
#(round-11 真机修复层 5:store helper 已去掉,authd helper 是
|
||||
# 唯一凭证 helper,且自身绝不落盘);客机启动时 chv-sync boot
|
||||
# 按 seed 盘 /etc/chv/guest-env(0600)把用户选定的仓 clone 到
|
||||
# ~/workspace,后续 pull/push 由用户显式同步,分叉/冲突时
|
||||
# 失败关闭(不 --force、不自动 rebase)。token 绝不进镜像、
|
||||
# 代码、日志。
|
||||
#
|
||||
# 对齐 Muse P2(2026-11-07):镜像里加 systemd-nspawn 运行时单元
|
||||
#(chv-cell:codex 跑在单元内)+ 单元外 broker(chv-broker:
|
||||
#Unix socket + SO_PEERCRED 白名单代办)+ 浏览器子代理(chv-browser:
|
||||
#单元外独立进程,只返回无障碍树快照)。boot clone 仍在单元外做;
|
||||
#单元隔离口径见 docs/chv-muse-p2-gap.md。
|
||||
#
|
||||
# 用法(必须在带 /dev/kvm 的宿主上跑全量模式):
|
||||
# CODEX_BIN_DIR=bin UBUNTU_VERSION=24.04 CODEX_VERSION=0.164.0-alpha.9 \
|
||||
# REPO_COMMIT=$(git rev-parse --short HEAD) \
|
||||
# UBUNTU_IMAGE_SHA256=<minimal 镜像 sha256> ./build.sh out/
|
||||
# ./build.sh --dry-run out/ # 只打印命令,不执行、不要求 KVM
|
||||
#
|
||||
# 产物(out/ 内):
|
||||
# vmlinux # virt-get-kernel 从云镜像取出的内核(CHV --kernel 指向它)
|
||||
# disk.raw # qemu-img 转成的 raw 根文件系统镜像(CHV_ROOTFS 指向它)
|
||||
# BUILDINFO.json # {tag, ubuntu, codex_version, repo_commit, built_at, contents}
|
||||
# + versions {dpkg, pip, chromium, gt}(构建时从客机读回的实际版本)+
|
||||
# ubuntu_image_sha256_set(SHA256 必填,见下)
|
||||
# 绝不进镜像的东西:任何模型 key、系统 key、用户账号数据、
|
||||
# 生产凭证;模型 key 由客机 cloud-init 预置的运行环境变量名
|
||||
# 注入,不写进镜像层。
|
||||
# feat-chv-secfix(M4/M8):UBUNTU_IMAGE_SHA256 必填;
|
||||
# sshd 加固由构建写入 /etc/ssh/sshd_config.d/99-chv.conf
|
||||
# 并经 sshd -t + grep 断言(README 的“只允许 key 登录”
|
||||
# 从此是构建断言,不再只是文档声称)。
|
||||
# B-10:virt-customize 的 chroot 里 sshd -t 须先 ssh-keygen -A
|
||||
# 临时生成 host key 并 mkdir -p /run/sshd,否则分别报无 host
|
||||
# key / 缺 privilege separation 目录而失败;断言通过后立即删
|
||||
# 临时 key,镜像仍不带 host key(per-VM 预置才生效)。
|
||||
#
|
||||
# 客机 host key 硬性要求(见 README §5,补缺口轨新增):
|
||||
# 客机必须装 cloud-init 且支持 NoCloud seed,首启先应用
|
||||
# seed 再启动 sshd,且不得无条件重新生成 host key,否则
|
||||
# 执行器的 per-VM 预置钉死即失效。本脚本在镜像里写
|
||||
# /etc/cloud/cloud.cfg.d/99-chv-hostkey.cfg:ssh_deletekeys:
|
||||
# false(绝不删已有 key)+ 首启 bootcmd 只在缺 key 时
|
||||
# ssh-keygen -A(bootcmd 跑在 seed write_files 之前,预置
|
||||
# key 随后覆盖生成值,最终仍是预置 key)。
|
||||
# B-23:曾写 ssh_genkeytypes: [],彻底关掉首启生成;Minimal
|
||||
# 底座 cloud.cfg 又不含 ssh 模块,客机无 host key、sshd -t
|
||||
# 失败、ssh.service 连败,SSH 全程不通——[] 已删,绝不再写。
|
||||
#
|
||||
# 注:Minimal 底座虚拟盘 3.5G(根分区 2.4G,实测 1.9G 空闲),
|
||||
# qemu-img convert 出的 disk.raw 表观 3.5G、实际占用约 0.6G
|
||||
# (稀疏)。raw 表观 ≤2G 须在构建后另做一步收窄(guestfish
|
||||
# 缩根文件系统 + sgdisk 重建 1 号分区 + qemu-img resize
|
||||
# --shrink),不在本脚本内做——收窄前先确认装完后根分区实际
|
||||
# 占用,避免把镜像收到无法启动;qcow2 压缩产物是首要验收项。
|
||||
set -euo pipefail
|
||||
|
||||
DRY_RUN=0
|
||||
if [ "${1:-}" = "--dry-run" ]; then
|
||||
DRY_RUN=1
|
||||
shift
|
||||
fi
|
||||
OUT="${1:-out}"
|
||||
: "${CODEX_BIN_DIR:?CODEX_BIN_DIR 未设:需 npm 安装 @openai/codex 后的配对二进制目录}"
|
||||
: "${UBUNTU_VERSION:?UBUNTU_VERSION 未设}"
|
||||
: "${CODEX_VERSION:?CODEX_VERSION 未设}"
|
||||
: "${REPO_COMMIT:?REPO_COMMIT 未设:构建时仓库 commit 短 hash}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_DIR="${REPO_DIR:-$(cd "$SCRIPT_DIR/.." && pwd)}"
|
||||
# gt 1.0.0:官方 install.sh(gitea.com/gitea/cli,main 分支
|
||||
# scripts/install.sh),脚本本身下载 dl.gitea.com 的二进制并
|
||||
# 验 SHA256;此处再验脚本自身的 SHA256。2026-11-07 06:44
|
||||
# 上游轮换过一次(V-9 实测:WSL fresh curl 与失败镜像内下载残留
|
||||
# 逐字节一致,新 sha 49ffe090…;新脚本全文已读,只从 dl.gitea.com
|
||||
# 取 1.0.0 二进制并验 hash,行为干净)——旧值 cf82… 已失效,
|
||||
# 现钉死新值;上游再轮换时构建会再次明确失败,不静默跑新脚本。
|
||||
GT_VERSION="${GT_VERSION:-1.0.0}"
|
||||
# feat-chv-closeup(M4):Chromium 单独钉死。PLAYWRIGHT=1 时
|
||||
# ms-playwright 缓存目录名即 Chromium 版本标识(如
|
||||
# chromium_headless_shell-1248),此前只记录不钉死——playwright
|
||||
# 一升级,目录名跟着变,构建静默换浏览器版本。
|
||||
# CHROMIUM_HEADLESS_SHELL_REV 设为纯数字修订号(如 1248)即钉死:
|
||||
# buildinfo.py 校验读回的目录名须恰好是该修订号,否则构建失败。
|
||||
# 未设时只记录、不强制,并在构建输出里明说“未钉死”,绝不假装
|
||||
# 已钉死;真机 V-9 验收须用已设该变量的构建。
|
||||
CHROMIUM_HEADLESS_SHELL_REV="${CHROMIUM_HEADLESS_SHELL_REV:-}"
|
||||
if [ -n "$CHROMIUM_HEADLESS_SHELL_REV" ] && ! printf '%s' "$CHROMIUM_HEADLESS_SHELL_REV" | grep -Eq '^[0-9]+$'; then
|
||||
echo "CHROMIUM_HEADLESS_SHELL_REV 须为纯数字修订号(如 1248):$CHROMIUM_HEADLESS_SHELL_REV" >&2; exit 1
|
||||
fi
|
||||
GT_INSTALL_SH_SHA256="${GT_INSTALL_SH_SHA256:-49ffe0909be9117e36cf7c8a35b5937939639090322063cbad7667e0bde7c9e9}"
|
||||
# B-03/B-04:输入格式校验(干跑与全量都做,报错点提前,
|
||||
# 不等到 curl 404 或 sha256sum -c 才失败)。
|
||||
if ! printf '%s' "$UBUNTU_VERSION" | grep -Eq '^[0-9]+\.[0-9]+$'; then
|
||||
echo "UBUNTU_VERSION 格式须为 NN.NN:$UBUNTU_VERSION" >&2; exit 1
|
||||
fi
|
||||
if ! printf '%s' "$REPO_COMMIT" | grep -Eq '^[0-9a-f]{7,40}$'; then
|
||||
echo "REPO_COMMIT 须为 7-40 位 hex:$REPO_COMMIT" >&2; exit 1
|
||||
fi
|
||||
if ! printf '%s' "$CODEX_VERSION" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+'; then
|
||||
echo "CODEX_VERSION 格式异常:$CODEX_VERSION" >&2; exit 1
|
||||
fi
|
||||
if ! printf '%s' "$GT_VERSION" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+$'; then
|
||||
echo "GT_VERSION 格式异常:$GT_VERSION" >&2; exit 1
|
||||
fi
|
||||
if ! printf '%s' "$GT_INSTALL_SH_SHA256" | grep -Eq '^[0-9a-f]{64}$'; then
|
||||
echo "GT_INSTALL_SH_SHA256 须为 64 位 hex" >&2; exit 1
|
||||
fi
|
||||
if [ -n "${UBUNTU_IMAGE_SHA256:-}" ] && ! printf '%s' "$UBUNTU_IMAGE_SHA256" | grep -Eq '^[0-9a-f]{64}$'; then
|
||||
echo "UBUNTU_IMAGE_SHA256 须为 64 位 hex" >&2; exit 1
|
||||
fi
|
||||
# B-09:版本声称须核对,不只进 tag。codex 二进制真实版本
|
||||
# 须包含 CODEX_VERSION;REPO_COMMIT 须等于仓库 HEAD 短 hash;
|
||||
# 工作树脏即拒绝构建(tag 会谎称干净 commit)。
|
||||
CODEX_ACTUAL_VERSION="$("$CODEX_BIN_DIR/codex" --version 2>/dev/null || true)"
|
||||
case "$CODEX_ACTUAL_VERSION" in
|
||||
*"$CODEX_VERSION"*) ;;
|
||||
*) echo "codex 二进制版本与 CODEX_VERSION 不符:got [$CODEX_ACTUAL_VERSION] want 含 [$CODEX_VERSION]" >&2; exit 1 ;;
|
||||
esac
|
||||
HEAD_SHORT="$(git -C "$REPO_DIR" rev-parse --short HEAD)"
|
||||
if [ "$HEAD_SHORT" != "$REPO_COMMIT" ]; then
|
||||
echo "REPO_COMMIT 与仓库 HEAD 不符:$REPO_COMMIT != $HEAD_SHORT" >&2; exit 1
|
||||
fi
|
||||
if [ -n "$(git -C "$REPO_DIR" status --porcelain)" ]; then
|
||||
echo "仓库工作树有未提交修改,拒绝构建(tag 会谎称干净 commit)" >&2; exit 1
|
||||
fi
|
||||
TAG="chv-guest-${UBUNTU_VERSION}-${CODEX_VERSION}-${REPO_COMMIT}"
|
||||
UBUNTU_IMAGE_URL="${UBUNTU_IMAGE_URL:-https://cloud-images.ubuntu.com/minimal/releases/noble/release/ubuntu-${UBUNTU_VERSION}-minimal-cloudimg-amd64.img}"
|
||||
|
||||
run() {
|
||||
if [ "$DRY_RUN" = "1" ]; then
|
||||
echo "DRY-RUN: $*"
|
||||
else
|
||||
echo "+ $*"
|
||||
"$@"
|
||||
fi
|
||||
}
|
||||
|
||||
# 参数校验(干跑与全量都做,不依赖 KVM)。
|
||||
for bin in codex codex-code-mode-host; do
|
||||
if [ ! -x "$CODEX_BIN_DIR/$bin" ]; then
|
||||
echo "缺少配对二进制:$CODEX_BIN_DIR/$bin,构建失败" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
if [ ! -d "$CODEX_BIN_DIR/codex-resources" ]; then
|
||||
echo "缺少 codex-resources(bwrap 载体),构建失败" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -f "$REPO_DIR/host.py" ]; then
|
||||
echo "REPO_DIR 不是 codex-app-server 仓库根(缺 host.py):$REPO_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -f "$SCRIPT_DIR/chv-sync.sh" ]; then
|
||||
echo "缺少 chv-sync.sh(Gitea 双向同步脚本):$SCRIPT_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
# 对齐 Muse P2:单元启动器 / broker / 浏览器子代理三脚本缺一即失败。
|
||||
for p2_script in chv-cell.py chv-broker.py chv-browser.py; do
|
||||
if [ ! -f "$SCRIPT_DIR/$p2_script" ]; then
|
||||
echo "缺少 $p2_script(Muse P2 客机脚本):$SCRIPT_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
# 对齐 Muse P3:authd 三件套缺一即失败(与 build.sh 同目录,P2
|
||||
# 同款规矩)。云端叶子模块(cloud/chv/authd.py 与 sentinel.py)
|
||||
# 不做硬要求:M4 既定“/app 最小集、build.sh 不再要求 cloud/”
|
||||
#(test_chv_residual.BuildScriptMinimalAppTest 钉住),缺失时只
|
||||
# 警告、不失败——缺叶子模块的镜像里 chv-authd 起不来,chv-sync
|
||||
# 按 P3 差距表 §3.5 回落旧落盘路径并警告,绝不静默假装有 authd。
|
||||
# round-11 真机修复(层 1):叶子树还须 errors.py + stub
|
||||
# __init__(authd_init_stub.py)——真实的 cloud/chv/__init__.py
|
||||
# import executor/orphans,拷进最小叶子树会让 chv-authd
|
||||
# ModuleNotFoundError 而 crash loop(探针 fixprobe3 的形状)。
|
||||
for p3_file in authd.py git-credential-chv-authd chv-authd.service authd_init_stub.py; do
|
||||
if [ ! -f "$SCRIPT_DIR/$p3_file" ]; then
|
||||
echo "缺少 $p3_file(Muse P3 客机文件):$SCRIPT_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
P3_LIBS_PRESENT=0
|
||||
if [ -f "$REPO_DIR/cloud/chv/authd.py" ] && [ -f "$REPO_DIR/cloud/chv/sentinel.py" ] && [ -f "$REPO_DIR/cloud/chv/errors.py" ] && [ -f "$REPO_DIR/cloud/__init__.py" ] && [ -f "$REPO_DIR/cloud/chv/__init__.py" ]; then
|
||||
P3_LIBS_PRESENT=1
|
||||
else
|
||||
echo "警告:Muse P3 云端叶子模块不在仓库里(cloud/chv/authd.py 与 sentinel.py),镜像暂不含 authd 叶子模块" >&2
|
||||
fi
|
||||
# 对齐 Muse P4:CDP broker / guard 共享模块 / 拦截名单三件缺一即失败。
|
||||
for p4_file in chv-cdp-broker.py chv_browser_guard.py browser-blocklist.txt; do
|
||||
if [ ! -f "$SCRIPT_DIR/$p4_file" ]; then
|
||||
echo "缺少 $p4_file(Muse P4 客机文件):$SCRIPT_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
if [ ! -f "$SCRIPT_DIR/chv-memory.py" ]; then
|
||||
echo "缺少 chv-memory.py(客内记忆文件脚本):$SCRIPT_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
# 对齐 Muse P6:privsep worker / 薄客户端 / 单元 / 路径触发器四件缺一即失败。
|
||||
for p6_file in chv-gitea-worker.py chv-sync-worker-client.py chv-gitea-worker.service chv-gitea-worker.path; do
|
||||
if [ ! -f "$SCRIPT_DIR/$p6_file" ]; then
|
||||
echo "缺少 $p6_file(Muse P6 客机文件):$SCRIPT_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
# P6 云端叶子模块(privsep.py / safety.py)只警告、不失败——
|
||||
# 与 P3 同款口径(M4 既定 build.sh 不硬要求 cloud/);缺失时
|
||||
# worker 起不来、git 直连回落路径也不走 worker。
|
||||
P6_LIBS_PRESENT=0
|
||||
if [ -f "$REPO_DIR/cloud/chv/privsep.py" ] && [ -f "$REPO_DIR/cloud/chv/safety.py" ]; then
|
||||
P6_LIBS_PRESENT=1
|
||||
else
|
||||
echo "警告:Muse P6 云端叶子模块不在仓库里(cloud/chv/privsep.py 与 safety.py),镜像暂不含 worker 叶子模块" >&2
|
||||
fi
|
||||
# P10 云端检测模块(checkout_detect.py)只警告、不失败——
|
||||
# 与 P3/P6 同款口径(M4 既定 build.sh 不硬要求 cloud/);缺失时
|
||||
# broker 的结账检测降级关闭(写警告、不 crash-loop),但完整
|
||||
# 镜像必须含它(extra_contents 与 lockfile 均有登记)。
|
||||
P10_CHECKOUT_PRESENT=0
|
||||
if [ -f "$REPO_DIR/cloud/chv/checkout_detect.py" ]; then
|
||||
P10_CHECKOUT_PRESENT=1
|
||||
else
|
||||
echo "警告:Muse P10 结账检测模块不在仓库里(cloud/chv/checkout_detect.py),镜像暂不含 checkout_detect" >&2
|
||||
fi
|
||||
|
||||
if [ "$DRY_RUN" = "0" ]; then
|
||||
if [ ! -e /dev/kvm ]; then
|
||||
echo "CHV_GUEST_BUILD needs a KVM host: /dev/kvm 不存在" >&2
|
||||
exit 1
|
||||
fi
|
||||
for tool in curl qemu-img virt-customize virt-get-kernel python3 virt-cat; do
|
||||
if ! command -v "$tool" >/dev/null 2>&1; then
|
||||
echo "缺少构建工具:$tool,构建失败" >&2
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
run mkdir -p "$OUT"
|
||||
# B-07:磁盘空间预检(仅全量模式;干跑不建目录,无法 df)。
|
||||
# 轻量版实际占用约 2G(底座 248M + disk.raw 稀疏约 0.6G +
|
||||
# 内核约 46M + 构建余量),阈值从全量版 20G 降到 8G。
|
||||
if [ "$DRY_RUN" = "0" ]; then
|
||||
avail_kb="$(df -k --output=avail "$OUT" | tail -1 | tr -d ' ')"
|
||||
if [ "${avail_kb:-0}" -lt 8388608 ]; then
|
||||
echo "磁盘不足:$OUT 所在盘可用 ${avail_kb}KB,需 ≥8GB" >&2; exit 1
|
||||
fi
|
||||
fi
|
||||
IMG="$OUT/ubuntu-${UBUNTU_VERSION}-minimal-cloudimg-amd64.img"
|
||||
# feat-chv-secfix(M4):云镜像 SHA256 改为必填(干跑只验
|
||||
# “已设”,不验文件;全量模式下载后验哈希)。未设即失败,
|
||||
# 不再只依赖 HTTPS 传输层作唯一锚。
|
||||
if [ -z "${UBUNTU_IMAGE_SHA256:-}" ]; then
|
||||
echo "UBUNTU_IMAGE_SHA256 未设:云镜像须以哈希锚定,构建失败" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ "$DRY_RUN" = "1" ]; then
|
||||
echo "DRY-RUN: curl -fL --retry 3 --retry-all-errors --connect-timeout 30 $UBUNTU_IMAGE_URL -o $IMG"
|
||||
echo "DRY-RUN: [ -s $IMG ] || exit 1"
|
||||
echo "DRY-RUN: sha256sum -c (UBUNTU_IMAGE_SHA256)"
|
||||
else
|
||||
if [ ! -s "$IMG" ]; then
|
||||
curl -fL --retry 3 --retry-all-errors --connect-timeout 30 "$UBUNTU_IMAGE_URL" -o "$IMG"
|
||||
fi
|
||||
[ -s "$IMG" ] || { echo "下载的云镜像为空:$IMG" >&2; exit 1; }
|
||||
# B-06:哈希不符时删文件重下一次,不直接退出让操作员手动清理。
|
||||
if ! echo "$UBUNTU_IMAGE_SHA256 $IMG" | sha256sum -c -; then
|
||||
echo "云镜像哈希不符,删除后重下一次:$IMG" >&2
|
||||
rm -f "$IMG"
|
||||
curl -fL --retry 3 --retry-all-errors --connect-timeout 30 "$UBUNTU_IMAGE_URL" -o "$IMG"
|
||||
[ -s "$IMG" ] || { echo "重下的云镜像为空:$IMG" >&2; exit 1; }
|
||||
echo "$UBUNTU_IMAGE_SHA256 $IMG" | sha256sum -c -
|
||||
fi
|
||||
fi
|
||||
run qemu-img convert -f qcow2 -O raw "$IMG" "$OUT/disk.raw"
|
||||
# 轻量版不扩盘:Minimal 底座根分区 2.4G、实测 1.9G 空闲,
|
||||
# 轻量清单(见下)装完仍有余量;全量版 B-16 的 +12G 扩盘
|
||||
# 已删除——扩盘会把 raw 表观顶到 16G,与 raw ≤2G 目标直接冲突。
|
||||
# B-21:apt 必须 --no-install-recommends。virt-customize
|
||||
# --install 默认装 Recommends,python3-pip 会连带拉入
|
||||
# python3-dev → gcc/g++/build-essential 全套工具链,根分区
|
||||
# 2.4G 被顶到 96%(实测 2.2G/95M 空闲),Playwright 下载
|
||||
# headless-shell 到 70% 时 ENOSPC 失败(round 6)。改走
|
||||
# --run-command apt-get install --no-install-recommends,
|
||||
# 并把会被 Recommends 带进来的必需项显式列出:fontconfig
|
||||
# (字体发现,没有它 Chromium 找不到中文字体)、openssh-client
|
||||
# (git 的 Recommends,agent 任务可能要 ssh 出站)。
|
||||
# apt-get clean 只清 .deb 缓存、不动 lists(playwright
|
||||
# --with-deps 稍后还要 apt 装浏览器依赖);lists 在最后一步清。
|
||||
# B-22:virt-customize 的客机 shell 里 HOME=/,Playwright 默认
|
||||
# 缓存路径 ~/.cache/ms-playwright 会落到根目录 /.cache(round 7
|
||||
# 实测:浏览器装在 /.cache/ms-playwright/chromium_headless_shell-1248,
|
||||
# /root/.cache 根本不存在,ls 读回空文件、BUILDINFO.versions.chromium
|
||||
# 为空,check_lockfile 兜底报未覆盖)。改走系统路径
|
||||
# PLAYWRIGHT_BROWSERS_PATH=/usr/local/share/ms-playwright:构建时
|
||||
# 装这里、读回这里;运行时 agent 也须设同一变量,否则 chv 用户
|
||||
# 会在自己家目录下找不到浏览器。
|
||||
# Playwright 默认开启(PLAYWRIGHT=0 可显式关掉),浏览器只装
|
||||
# chromium-headless-shell(不要完整 Chromium)。单次
|
||||
# virt-customize:apt 安装、pip、拷入、sshd 加固、git 占位、
|
||||
# gt 安装、chv-sync、版本读回都在同一次定制里按参数顺序跑;
|
||||
# Playwright 装在第一次 freeze 之后、最终 freeze 之前(B-08:
|
||||
# 最终 freeze 须在 Playwright 安装之后,否则
|
||||
# BUILDINFO.versions.pip 里没有 playwright;Chromium 版本以
|
||||
# ms-playwright 缓存目录名记录(如 chromium_headless_shell-1248),
|
||||
# 写进 /app/guest-versions-chromium.txt)。
|
||||
GT_INSTALL_CMD="curl -fsSL -o /tmp/gt-install.sh https://gitea.com/gitea/cli/raw/branch/main/scripts/install.sh && echo \"${GT_INSTALL_SH_SHA256} /tmp/gt-install.sh\" | sha256sum -c - && GITEA_CLI_VERSION=${GT_VERSION} GITEA_CLI_INSTALL_DIR=/usr/local/bin sh /tmp/gt-install.sh && rm -f /tmp/gt-install.sh && /usr/local/bin/gt version"
|
||||
VC_ARGS=(-a "$OUT/disk.raw"
|
||||
--run-command 'apt-get update -qq && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends git openssh-server openssh-client cloud-init python3 python3-pip curl ca-certificates xz-utils fonts-wqy-microhei fontconfig systemd-container iproute2 iptables && apt-get clean'
|
||||
--copy-in "$CODEX_BIN_DIR/codex:/usr/local/bin/"
|
||||
--copy-in "$CODEX_BIN_DIR/codex-code-mode-host:/usr/local/bin/"
|
||||
--copy-in "$CODEX_BIN_DIR/codex-resources:/usr/local/bin/"
|
||||
--run-command 'chmod +x /usr/local/bin/codex /usr/local/bin/codex-code-mode-host; ln -sf /usr/local/bin/codex-resources/bwrap /usr/local/bin/bwrap'
|
||||
--mkdir /app
|
||||
--copy-in "$REPO_DIR/host.py:/app/"
|
||||
--copy-in "$SCRIPT_DIR/chv-sync.sh:/usr/local/bin/"
|
||||
--run-command 'mv /usr/local/bin/chv-sync.sh /usr/local/bin/chv-sync && chmod +x /usr/local/bin/chv-sync'
|
||||
--copy-in "$SCRIPT_DIR/chv-cell.py:/usr/local/bin/"
|
||||
--copy-in "$SCRIPT_DIR/chv-broker.py:/usr/local/bin/"
|
||||
--copy-in "$SCRIPT_DIR/chv-browser.py:/usr/local/bin/"
|
||||
--run-command 'mv /usr/local/bin/chv-cell.py /usr/local/bin/chv-cell && mv /usr/local/bin/chv-broker.py /usr/local/bin/chv-broker && mv /usr/local/bin/chv-browser.py /usr/local/bin/chv-browser && chmod +x /usr/local/bin/chv-cell /usr/local/bin/chv-broker /usr/local/bin/chv-browser'
|
||||
--mkdir /usr/local/lib/chv-authd
|
||||
--mkdir /usr/local/lib/chv-authd/cloud
|
||||
--mkdir /usr/local/lib/chv-authd/cloud/chv
|
||||
--copy-in "$SCRIPT_DIR/authd.py:/usr/local/bin/"
|
||||
--copy-in "$SCRIPT_DIR/git-credential-chv-authd:/usr/local/bin/"
|
||||
--copy-in "$SCRIPT_DIR/chv-authd.service:/etc/systemd/system/"
|
||||
--run-command 'mv /usr/local/bin/authd.py /usr/local/bin/chv-authd && chmod +x /usr/local/bin/chv-authd /usr/local/bin/git-credential-chv-authd'
|
||||
--run-command 'systemctl enable chv-authd || true'
|
||||
# round-12 修复轮第三轮(缺口 ①):runcmd 只启动 .path 单元
|
||||
#(开机时它已由 enable 激活,这里是兜底),绝不直接 start 服务
|
||||
#——cloud-final 此时工作区尚未 clone,直接 start 服务会在命名
|
||||
# 空间搭建时 226;.path 单元在 boot 的 mkdir -p 建出工作区后
|
||||
# 才触发服务启动。
|
||||
--run-command 'printf "runcmd:\n - systemctl restart ssh || service ssh restart || true\n - systemctl start chv-authd || true\n - test -f /etc/chv/guest-env && chown chv:chv /etc/chv/guest-env || true\n - systemctl start chv-gitea-worker.path || true\n - /usr/local/bin/chv-sync boot >>/var/log/chv-sync.log 2>&1 || true\n" > /etc/cloud/cloud.cfg.d/99-chv-sync.cfg'
|
||||
--copy-in "$SCRIPT_DIR/chv-cdp-broker.py:/usr/local/bin/"
|
||||
--copy-in "$SCRIPT_DIR/chv_browser_guard.py:/usr/local/bin/"
|
||||
# round-12 修复轮(项 0):c3bf9fd 认领——browser-blocklist.txt
|
||||
# 拷进 /usr/local/share/chv/ 须先建目录,否则 virt-customize
|
||||
# 报 target is not a directory,纯分支建不出镜像。
|
||||
--mkdir /usr/local/share/chv
|
||||
--copy-in "$SCRIPT_DIR/browser-blocklist.txt:/usr/local/share/chv/"
|
||||
--run-command 'mv /usr/local/bin/chv-cdp-broker.py /usr/local/bin/chv-cdp-broker && chmod +x /usr/local/bin/chv-cdp-broker && chmod 0644 /usr/local/bin/chv_browser_guard.py /usr/local/share/chv/browser-blocklist.txt'
|
||||
--copy-in "$SCRIPT_DIR/chv-memory.py:/usr/local/bin/"
|
||||
--run-command 'mv /usr/local/bin/chv-memory.py /usr/local/bin/chv-memory && chmod +x /usr/local/bin/chv-memory'
|
||||
# 对齐 Muse P6:privsep worker(gap 文档 §2 接入说明)。
|
||||
--mkdir /usr/local/lib/chv-gitea
|
||||
--mkdir /usr/local/lib/chv-gitea/cloud
|
||||
--mkdir /usr/local/lib/chv-gitea/cloud/chv
|
||||
--copy-in "$SCRIPT_DIR/chv-gitea-worker.py:/usr/local/bin/"
|
||||
--copy-in "$SCRIPT_DIR/chv-sync-worker-client.py:/usr/local/bin/"
|
||||
--copy-in "$SCRIPT_DIR/chv-gitea-worker.service:/etc/systemd/system/"
|
||||
--copy-in "$SCRIPT_DIR/chv-gitea-worker.path:/etc/systemd/system/"
|
||||
--run-command 'mv /usr/local/bin/chv-gitea-worker.py /usr/local/bin/chv-gitea-worker && mv /usr/local/bin/chv-sync-worker-client.py /usr/local/bin/chv-sync-worker-client && chmod +x /usr/local/bin/chv-gitea-worker /usr/local/bin/chv-sync-worker-client'
|
||||
--run-command 'systemctl enable chv-gitea-worker chv-gitea-worker.path || true'
|
||||
|
||||
# round-11 真机修复(层 5):镜像不再设 credential.helper store
|
||||
#(authd helper 是唯一凭证 helper);P4 的 copy-in 行保留。
|
||||
--run-command 'git config --system user.name chv && git config --system user.email chv@localhost && git config --system init.defaultBranch main'
|
||||
--run-command "$GT_INSTALL_CMD"
|
||||
--run-command '/usr/local/bin/gt version > /app/guest-versions-gt.txt'
|
||||
--run-command 'printf "ssh_deletekeys: false\nbootcmd:\n - \"[ -f /etc/ssh/ssh_host_ed25519_key ] || ssh-keygen -A\"\n" > /etc/cloud/cloud.cfg.d/99-chv-hostkey.cfg; systemctl enable ssh || true'
|
||||
--run-command 'printf "PasswordAuthentication no\nPermitRootLogin prohibit-password\nChallengeResponseAuthentication no\nKbdInteractiveAuthentication no\nPubkeyAuthentication yes\n" > /etc/ssh/sshd_config.d/99-chv.conf && mkdir -p /run/sshd && ssh-keygen -A && sshd -t && grep -q "^PasswordAuthentication no" /etc/ssh/sshd_config.d/99-chv.conf && grep -q "^PermitRootLogin prohibit-password" /etc/ssh/sshd_config.d/99-chv.conf && rm -f /etc/ssh/ssh_host_* && test -z "$(ls /etc/ssh/ssh_host_* 2>/dev/null)"'
|
||||
--run-command 'dpkg-query -W -f "\${Package} \${Version}\n" > /app/guest-versions-dpkg.txt; pip3 freeze > /app/guest-versions-pip.txt || true'
|
||||
--run-command 'useradd --system --home-dir /var/lib/chv-browser --shell /usr/sbin/nologin chv-browser && mkdir -p /var/lib/chv-browser /run/chv/shots /run/chv-browser && chown chv-browser:chv-browser /var/lib/chv-browser /run/chv/shots /run/chv-browser && chgrp chv-browser /run/chv && chmod 2775 /run/chv && chmod 0755 /run/chv/shots /run/chv-browser'
|
||||
--run-command 'printf "[Unit]\nDescription=CHV cell broker (Muse P2)\nAfter=network.target\n\n[Service]\nType=simple\nExecStart=/usr/bin/python3 /usr/local/bin/chv-broker\nRestart=on-failure\nRestartSec=2\n\n[Install]\nWantedBy=multi-user.target\n" > /etc/systemd/system/chv-broker.service'
|
||||
--run-command 'printf "[Unit]\nDescription=CHV browser sub-agent (Muse P2)\nAfter=network.target\n\n[Service]\nType=simple\nUser=chv-browser\nGroup=chv-browser\nExecStart=/usr/bin/python3 /usr/local/bin/chv-browser\nRestart=on-failure\nRestartSec=2\n\n[Install]\nWantedBy=multi-user.target\n" > /etc/systemd/system/chv-browser.service'
|
||||
--run-command 'printf "[Unit]\nDescription=CHV CDP broker (Muse P4)\nAfter=network.target\n\n[Service]\nType=simple\nUser=chv-browser\nGroup=chv-browser\nExecStart=/usr/bin/python3 /usr/local/bin/chv-cdp-broker\nRestart=on-failure\nRestartSec=2\n\n[Install]\nWantedBy=multi-user.target\n" > /etc/systemd/system/chv-cdp-broker.service'
|
||||
# round-11 真机修复(缺陷 4):tmpfiles 须登记 /run/chv-browser
|
||||
#(browser.sock 自有目录);P4 的 ask-queue 行保留。
|
||||
# round-12 修复轮(项 1,P4):/run/chv 改为组 chv-browser
|
||||
# 可写(2775,setgid 沿用手工修复的口径——R12 真机里
|
||||
# chgrp chv-browser + chmod 2775 后 broker 全过)。原 0755
|
||||
# root 所有时 chv-browser 建不了 cdp-broker.sock,broker
|
||||
# bind 报 PermissionError 而 crash-loop。绝不用 0666/0777
|
||||
# 那种放宽(与 round-11 authd socket 组权限 0660 同款口径)。
|
||||
# round-12 修复轮第二轮(项 1,P6):tmpfiles.d 再预建 worker
|
||||
# 的运行时路径。chv-gitea-worker.service 用 ProtectSystem=strict
|
||||
# + ReadWritePaths=/home/chv/workspace /run/chv-gitea
|
||||
# /var/log/chv-gitea-worker.log;systemd 建 mount namespace 时
|
||||
# 要求每个列出路径事先存在,否则 226/NAMESPACE(R12 复测钉死:
|
||||
# /run 是 tmpfs、开机不存在;log 文件要 worker 首次写审计才建,
|
||||
# 而 worker 永远跑不到那一步)。ExecStartPre 救不了——命名空间
|
||||
# 搭建在任何 ExecStartPre 之前,226 发生在它运行之前;改
|
||||
# ReadWritePaths 为已存在目录(/var/log)会把写权限放大到整个
|
||||
# 目录,也不要。tmpfiles.d 由 systemd-tmpfiles-setup 在 sysinit
|
||||
# 跑、早于 multi-user 的服务启动,且与 P4 的 /run/chv 同款写法,
|
||||
# 不新增进镜像的文件(chv.conf 本就是 build.sh 内联生成),
|
||||
# lockfile 无需登记。模式与 worker 审计 0600 同款口径。
|
||||
# round-12 修复轮第三轮(缺口 ②):/run/chv-gitea 改为组权限
|
||||
# 0750,与 worker 新方案一致(目录 0750 + socket 0660,组为
|
||||
# chv 主组;绝不用 0666)。gid 写数字 1000 而不用组名 chv——
|
||||
# tmpfiles-setup 跑在 07:23:24,chv 用户由 cloud-init 在
|
||||
# 07:23:40 才建,组名当时解析不出;1000 是 chv 的 uid/gid
|
||||
#(seed users 块未指定 uid/gid,cloud-init 取首个空闲 1000;
|
||||
# authd DEFAULT_ALLOWED_UIDS 同款假设)。worker 启动时再按
|
||||
# passwd 重新推导 gid 并重设目录/socket 权限,tmpfiles 行只是
|
||||
# 开机存在性保证。
|
||||
--run-command 'printf "d /run/chv 2775 root chv-browser -\nd /run/chv/shots 0755 chv-browser chv-browser -\nd /run/chv/ask-queue 0750 chv-browser chv-browser -\nd /run/chv-browser 0755 chv-browser chv-browser -\nd /run/chv-gitea 0750 root 1000 -\nf /var/log/chv-gitea-worker.log 0600 root root -\n" > /etc/tmpfiles.d/chv.conf && systemctl enable chv-broker chv-browser chv-cdp-broker || true')
|
||||
# P3 交接(Muse P3 的 authd/sentinel 叶子模块):文件在仓库里存在时
|
||||
# 拷入镜像(本分支已有,合入后自动生效);不存在时只警告、不失败——
|
||||
# M4 既定 build.sh 不硬要求 cloud/,见上。缺叶子模块时 chv-authd
|
||||
# 守护进程起不来,chv-sync 按差距表 §3.5 回落旧路径并警告。
|
||||
# round-11 真机修复(层 1):叶子树是 errors.py + stub __init__
|
||||
#(authd_init_stub.py),不拷真实的 cloud/chv/__init__.py(它
|
||||
# import executor/orphans,会让 authd 在最小叶子树里
|
||||
# ModuleNotFoundError 而 crash loop);cloud/__init__.py 本身只
|
||||
# 有 docstring,照常拷。
|
||||
if [ "$P3_LIBS_PRESENT" = "1" ]; then
|
||||
VC_ARGS+=(--copy-in "$REPO_DIR/cloud/chv/authd.py:/usr/local/lib/chv-authd/cloud/chv/"
|
||||
--copy-in "$REPO_DIR/cloud/chv/sentinel.py:/usr/local/lib/chv-authd/cloud/chv/"
|
||||
--copy-in "$REPO_DIR/cloud/chv/errors.py:/usr/local/lib/chv-authd/cloud/chv/"
|
||||
--copy-in "$REPO_DIR/cloud/__init__.py:/usr/local/lib/chv-authd/cloud/"
|
||||
--copy-in "$SCRIPT_DIR/authd_init_stub.py:/usr/local/lib/chv-authd/cloud/chv/"
|
||||
--run-command 'mv /usr/local/lib/chv-authd/cloud/chv/authd_init_stub.py /usr/local/lib/chv-authd/cloud/chv/__init__.py')
|
||||
fi
|
||||
# P6 交接(Muse P6 的 worker 叶子模块):文件在仓库里存在时
|
||||
# 拷入 /usr/local/lib/chv-gitea/ 叶子树;不存在时只警告、不失败——
|
||||
# 与 P3 同款口径(M4 既定 build.sh 不硬要求 cloud/)。
|
||||
# 注意:叶子树的 cloud/chv/__init__.py 用 stub(authd_init_stub.py),
|
||||
# 绝不拷真实的 cloud/chv/__init__.py(它 import executor/orphans,
|
||||
# 会让 worker 在 import 时 ModuleNotFoundError;round-11 层 1 同款
|
||||
# 教训)。stub 要从 cloud.chv.errors import ChvFailure,故
|
||||
# errors.py 也一并拷进本树。
|
||||
if [ "$P6_LIBS_PRESENT" = "1" ]; then
|
||||
VC_ARGS+=(--copy-in "$REPO_DIR/cloud/chv/privsep.py:/usr/local/lib/chv-gitea/cloud/chv/"
|
||||
--copy-in "$REPO_DIR/cloud/chv/safety.py:/usr/local/lib/chv-gitea/cloud/chv/"
|
||||
--copy-in "$REPO_DIR/cloud/chv/errors.py:/usr/local/lib/chv-gitea/cloud/chv/"
|
||||
--copy-in "$REPO_DIR/cloud/__init__.py:/usr/local/lib/chv-gitea/cloud/"
|
||||
--copy-in "$SCRIPT_DIR/authd_init_stub.py:/usr/local/lib/chv-gitea/cloud/chv/"
|
||||
--run-command 'mv /usr/local/lib/chv-gitea/cloud/chv/authd_init_stub.py /usr/local/lib/chv-gitea/cloud/chv/__init__.py')
|
||||
fi
|
||||
# P10 交接(Muse P10 的结账检测模块,round-12 修复轮项 4):
|
||||
# 文件在仓库里存在时拷入 /usr/local/bin/ 供 chv-cdp-broker 加载
|
||||
#(broker 先找同目录,再找仓库 cloud/chv/;0644 即可,broker
|
||||
# 以 chv-browser 身份读);不存在时只警告、不失败(与 P3/P6
|
||||
# 同款口径,broker 降级关闭检测、不 crash-loop)。
|
||||
if [ "$P10_CHECKOUT_PRESENT" = "1" ]; then
|
||||
VC_ARGS+=(--copy-in "$REPO_DIR/cloud/chv/checkout_detect.py:/usr/local/bin/"
|
||||
--run-command 'chmod 0644 /usr/local/bin/checkout_detect.py')
|
||||
fi
|
||||
# P1 交接(Muse P1 的 vsock_notify.py):文件在仓库里存在时拷入
|
||||
# 镜像并建 vsock-notify.service(自动启动,round-11 真机修复:
|
||||
# 此前只拷文件、不建服务,vsock 通道从未自动启动;服务形状按
|
||||
# vsock_notify.py 的 docstring:ExecStart=/usr/local/bin/
|
||||
# vsock-notify %H,Restart=no,After/WantedBy=multi-user.target;
|
||||
# printf 格式串里 %H 须写 %%H);不存在时只警告、不失败。
|
||||
VSOCK_NOTIFY_PRESENT=0
|
||||
if [ -f "$REPO_DIR/cloud/chv/guest/vsock_notify.py" ]; then
|
||||
VC_ARGS+=(--copy-in "$REPO_DIR/cloud/chv/guest/vsock_notify.py:/usr/local/bin/"
|
||||
--run-command 'mv /usr/local/bin/vsock_notify.py /usr/local/bin/vsock-notify && chmod +x /usr/local/bin/vsock-notify'
|
||||
--run-command 'printf "[Unit]\nDescription=CHV vsock notify (Muse P1)\nAfter=multi-user.target\n\n[Service]\nType=simple\nExecStart=/usr/local/bin/vsock-notify %%H\nRestart=no\n\n[Install]\nWantedBy=multi-user.target\n" > /etc/systemd/system/vsock-notify.service && systemctl enable vsock-notify || true')
|
||||
VSOCK_NOTIFY_PRESENT=1
|
||||
else
|
||||
echo "警告:cloud/chv/guest/vsock_notify.py 不在仓库里(P1 未合并),镜像暂不含 vsock notify" >&2
|
||||
fi
|
||||
if [ "${PLAYWRIGHT:-1}" = "1" ]; then
|
||||
VC_ARGS+=(
|
||||
--run-command '(pip3 install --no-cache-dir --break-system-packages playwright || pip3 install --no-cache-dir playwright) && PLAYWRIGHT_BROWSERS_PATH=/usr/local/share/ms-playwright python3 -m playwright install --with-deps chromium-headless-shell'
|
||||
--run-command 'dpkg-query -W -f "\${Package} \${Version}\n" > /app/guest-versions-dpkg.txt; pip3 freeze > /app/guest-versions-pip.txt || true'
|
||||
--run-command 'ls /usr/local/share/ms-playwright/ > /app/guest-versions-chromium.txt 2>/dev/null || true')
|
||||
fi
|
||||
# B-21 收尾:清掉两轮 apt 的缓存与 lists(--with-deps 装完后
|
||||
# 不再有 apt 活动),把空间还给镜像。
|
||||
VC_ARGS+=(
|
||||
--run-command 'apt-get clean && rm -rf /var/lib/apt/lists/*')
|
||||
run virt-customize "${VC_ARGS[@]}"
|
||||
# B-18:virt-get-kernel 的 -o 是输出目录(不是文件),它在
|
||||
# 目录里下载 vmlinuz-<ver> 与 initrd.img-<ver>;旧写法
|
||||
# -o "$OUT/vmlinux" 会建出 vmlinux 目录,后面的 test -s 文件
|
||||
# 校验必然失败。改成 -o "$OUT",再把 vmlinuz-* 移为 vmlinux
|
||||
# 文件;initrd.img-* 留在 OUT 里作配套产物。另:第三轮真机
|
||||
# 构建里 virt-get-kernel 首跑报过一次 "file receive cancelled
|
||||
# by daemon",同一盘手动重跑即成功(瞬时故障),此处失败重跑
|
||||
# 一次,与 B-05/B-06 的重试口径一致。
|
||||
if [ "$DRY_RUN" = "1" ]; then
|
||||
echo "DRY-RUN: virt-get-kernel -a $OUT/disk.raw -o $OUT"
|
||||
echo "DRY-RUN: mv $OUT/vmlinuz-* $OUT/vmlinux"
|
||||
else
|
||||
run virt-get-kernel -a "$OUT/disk.raw" -o "$OUT" \
|
||||
|| run virt-get-kernel -a "$OUT/disk.raw" -o "$OUT"
|
||||
mv "$OUT"/vmlinuz-* "$OUT/vmlinux"
|
||||
fi
|
||||
# feat-chv-secfix(M4):构建时把客机内实际装好的版本
|
||||
# 读回 BUILDINFO.versions(dpkg + pip freeze),名字列表
|
||||
# 之外再有逐项版本——可审计、可复现比对。事前 lockfile
|
||||
# 钉死(apt/pip 精确版本预先锁定)仍待构建宿主生成,
|
||||
# 见 README 与就绪清单 §11;本步先保证“建成什么版本
|
||||
# 都有记录”,不再只有名字。
|
||||
if [ "$DRY_RUN" = "1" ]; then
|
||||
echo "DRY-RUN: virt-cat -a $OUT/disk.raw /app/guest-versions-dpkg.txt > $OUT/guest-versions-dpkg.txt"
|
||||
echo "DRY-RUN: virt-cat -a $OUT/disk.raw /app/guest-versions-pip.txt > $OUT/guest-versions-pip.txt"
|
||||
echo "DRY-RUN: virt-cat -a $OUT/disk.raw /app/guest-versions-gt.txt > $OUT/guest-versions-gt.txt"
|
||||
if [ "${PLAYWRIGHT:-1}" = "1" ]; then
|
||||
echo "DRY-RUN: virt-cat -a $OUT/disk.raw /app/guest-versions-chromium.txt > $OUT/guest-versions-chromium.txt"
|
||||
fi
|
||||
else
|
||||
virt-cat -a "$OUT/disk.raw" /app/guest-versions-dpkg.txt > "$OUT/guest-versions-dpkg.txt"
|
||||
virt-cat -a "$OUT/disk.raw" /app/guest-versions-pip.txt > "$OUT/guest-versions-pip.txt"
|
||||
virt-cat -a "$OUT/disk.raw" /app/guest-versions-gt.txt > "$OUT/guest-versions-gt.txt" || true
|
||||
if [ "${PLAYWRIGHT:-1}" = "1" ]; then
|
||||
virt-cat -a "$OUT/disk.raw" /app/guest-versions-chromium.txt > "$OUT/guest-versions-chromium.txt" || true
|
||||
fi
|
||||
fi
|
||||
# 产物校验(干跑只打印,不执行校验命令的副作用部分)。
|
||||
if [ "$DRY_RUN" = "1" ]; then
|
||||
echo "DRY-RUN: test -s $OUT/vmlinux && test -s $OUT/disk.raw"
|
||||
echo "DRY-RUN: python3 $SCRIPT_DIR/buildinfo.py $OUT $TAG $UBUNTU_VERSION $CODEX_VERSION $REPO_COMMIT <sha256> ${PLAYWRIGHT:-1} ${CHROMIUM_HEADLESS_SHELL_REV:-} [vsock_notify?] # 生成 $OUT/BUILDINFO.json 并校验 tag=$TAG"
|
||||
else
|
||||
test -s "$OUT/vmlinux" || { echo "内核产物为空:$OUT/vmlinux" >&2; exit 1; }
|
||||
test -s "$OUT/disk.raw" || { echo "磁盘产物为空:$OUT/disk.raw" >&2; exit 1; }
|
||||
if [ "${PLAYWRIGHT:-1}" = "1" ] && [ -z "$CHROMIUM_HEADLESS_SHELL_REV" ]; then
|
||||
echo "警告:CHROMIUM_HEADLESS_SHELL_REV 未设,Chromium 版本只记录、不钉死(V-9 验收须设该变量重建)" >&2
|
||||
fi
|
||||
EXTRA_CONTENTS=""
|
||||
[ "$P3_LIBS_PRESENT" = "1" ] && EXTRA_CONTENTS="cloud/chv/authd.py,cloud/chv/sentinel.py,cloud/chv/errors.py,cloud/chv/__init__.py"
|
||||
[ "$P6_LIBS_PRESENT" = "1" ] && EXTRA_CONTENTS="${EXTRA_CONTENTS:+$EXTRA_CONTENTS,}cloud/chv/privsep.py,cloud/chv/safety.py"
|
||||
[ "$P10_CHECKOUT_PRESENT" = "1" ] && EXTRA_CONTENTS="${EXTRA_CONTENTS:+$EXTRA_CONTENTS,}cloud/chv/checkout_detect.py"
|
||||
[ "$VSOCK_NOTIFY_PRESENT" = "1" ] && EXTRA_CONTENTS="${EXTRA_CONTENTS:+$EXTRA_CONTENTS,}vsock_notify"
|
||||
python3 "$SCRIPT_DIR/buildinfo.py" "$OUT" "$TAG" "$UBUNTU_VERSION" "$CODEX_VERSION" "$REPO_COMMIT" "$UBUNTU_IMAGE_SHA256" "${PLAYWRIGHT:-1}" "${CHROMIUM_HEADLESS_SHELL_REV:-}" "$EXTRA_CONTENTS"
|
||||
fi
|
||||
echo "tag=$TAG"
|
||||
echo "out=$OUT"
|
||||
if [ "$DRY_RUN" = "1" ]; then
|
||||
echo "干跑结束:未执行任何构建步骤,镜像未建成"
|
||||
else
|
||||
echo "构建完成:$OUT/vmlinux $OUT/disk.raw $OUT/BUILDINFO.json"
|
||||
fi
|
||||
@@ -1,199 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""生成 CHV 客机镜像的 BUILDINFO.json(build.sh 调用)。
|
||||
|
||||
从 build.sh 抽出(B-02),使其可独立单元测试。输入全部是
|
||||
已落盘的构建产物与读回文件,不编造版本号:versions 来自
|
||||
客机读回的 dpkg-query / pip freeze(Playwright=1 时另有
|
||||
ms-playwright 缓存目录名作 chromium 标识);artifacts 的
|
||||
sha256/size 是对产物文件实测的;image_sha256 即 disk.raw
|
||||
的 sha256,供 generate_lockfile.py 沿用。
|
||||
|
||||
用法:
|
||||
python3 buildinfo.py OUT_DIR TAG UBUNTU CODEX_VERSION \\
|
||||
REPO_COMMIT UBUNTU_IMAGE_SHA256 PLAYWRIGHT(0|1)
|
||||
[CHROMIUM_HEADLESS_SHELL_REV]
|
||||
|
||||
CHROMIUM_HEADLESS_SHELL_REV(可选,纯数字):钉死的
|
||||
ms-playwright 缓存修订号(如 1248)。PLAYWRIGHT=1 且
|
||||
给了该值时,versions.chromium 里须恰好出现
|
||||
chromium_headless_shell-<rev>,否则退出 1、构建失败;
|
||||
未给时只记录、不强制,BUILDINFO.chromium_pinned_rev
|
||||
记 null,构建输出由 build.sh 明说“未钉死”。
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
|
||||
# 轻量口径(2026-11-06 用户定):Ubuntu Minimal 底座,只装
|
||||
# codex + Python + SSH + cloud-init + 薄 agent + git + gt +
|
||||
# 最小中文字体;已裁剪 pandoc/texlive/python-docx 等文档工具
|
||||
# 与 qemu-guest-agent。chv-sync 是仓库内脚本,版本随 repo_commit。
|
||||
BASE_CONTENTS = [
|
||||
"git", "openssh-server", "openssh-client", "cloud-init",
|
||||
"python3", "python3-pip", "curl", "ca-certificates",
|
||||
"xz-utils", "fonts-wqy-microhei", "fontconfig", "codex",
|
||||
"codex-code-mode-host", "codex-resources/bwrap",
|
||||
"/app/host.py", "gt", "chv-sync", "chv-memory",
|
||||
# 对齐 Muse P2:nspawn 单元启动器 + broker + 浏览器子代理。
|
||||
"systemd-container", "chv-cell", "chv-broker", "chv-browser",
|
||||
# 对齐 Muse P3(round-11 合入镜像):authd 守护进程 + git
|
||||
# credential helper + systemd 单元(三件套是无条件拷入的)。
|
||||
# 云端叶子模块(cloud/chv/authd.py、sentinel.py)不进 BASE:
|
||||
# 它们是条件拷入(缺失只警告、不失败),存在时才经
|
||||
# extra_contents 登记,绝不在缺失的构建里谎称已进镜像。
|
||||
"chv-authd", "git-credential-chv-authd", "chv-authd.service",
|
||||
# 对齐 Muse P4:单元外 CDP broker + 浏览器安全共享模块 +
|
||||
# 静态拦截名单。
|
||||
"chv-cdp-broker", "chv-browser-guard", "browser-blocklist",
|
||||
# 对齐 Muse P6:privsep worker + 薄客户端 + 单元 + 路径触发器
|
||||
#(无条件拷入)。路径触发器是 round-12 修复轮第三轮缺口 ①
|
||||
# 的拉起机制(工作区出现才启动服务),与单元同款登记。
|
||||
"chv-gitea-worker", "chv-sync-worker-client",
|
||||
"chv-gitea-worker.service", "chv-gitea-worker.path",
|
||||
|
||||
]
|
||||
|
||||
# 执行器默认内核 cmdline(config.py),构建产物须与它一起记录(B-12),
|
||||
# 否则换宿主/换执行器配置时无法复现。
|
||||
KERNEL_CMDLINE = "console=hvc0 root=/dev/vda1 rw"
|
||||
|
||||
# 浏览器装在系统路径(B-22),不是某用户的 ~/.cache:构建时
|
||||
# PLAYWRIGHT_BROWSERS_PATH 指这里,agent 运行时也须设同一变量。
|
||||
PLAYWRIGHT_BROWSERS_PATH = "/usr/local/share/ms-playwright"
|
||||
|
||||
|
||||
def _read_lines(path):
|
||||
try:
|
||||
with open(path) as fh:
|
||||
return [line.strip() for line in fh if line.strip()]
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
|
||||
def _sha256_size(path):
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
with open(path, "rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
size += len(chunk)
|
||||
return digest.hexdigest(), size
|
||||
|
||||
|
||||
def build(outdir, tag, ubuntu, codex_version, repo_commit,
|
||||
ubuntu_image_sha256, playwright, chromium_pinned_rev=None,
|
||||
extra_contents=None):
|
||||
disk_path = os.path.join(outdir, "disk.raw")
|
||||
vmlinux_path = os.path.join(outdir, "vmlinux")
|
||||
disk_sha, disk_size = _sha256_size(disk_path)
|
||||
vmlinux_sha, vmlinux_size = _sha256_size(vmlinux_path)
|
||||
contents = list(BASE_CONTENTS)
|
||||
# extra_contents:构建时才确定的附加内容(如 P1 的 vsock_notify,
|
||||
# 文件存在才拷入)。调用方传什么记什么,不编默认值。
|
||||
for item in extra_contents or []:
|
||||
if item and item not in contents:
|
||||
contents.append(item)
|
||||
versions = {
|
||||
"dpkg": _read_lines(
|
||||
os.path.join(outdir, "guest-versions-dpkg.txt")),
|
||||
"pip": _read_lines(
|
||||
os.path.join(outdir, "guest-versions-pip.txt")),
|
||||
"gt": _read_lines(
|
||||
os.path.join(outdir, "guest-versions-gt.txt")),
|
||||
}
|
||||
gt_version = ""
|
||||
for line in versions["gt"]:
|
||||
m = re.search(r"(\d+\.\d+\.\d+)", line)
|
||||
if m:
|
||||
gt_version = m.group(1)
|
||||
break
|
||||
if playwright:
|
||||
contents += ["playwright", "chromium-headless-shell"]
|
||||
versions["chromium"] = _read_lines(
|
||||
os.path.join(outdir, "guest-versions-chromium.txt"))
|
||||
info = {
|
||||
"tag": tag,
|
||||
"ubuntu": ubuntu,
|
||||
"codex_version": codex_version,
|
||||
"repo_commit": repo_commit,
|
||||
"gt_version": gt_version,
|
||||
"built_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"contents": contents,
|
||||
"versions": versions,
|
||||
"ubuntu_image_sha256": ubuntu_image_sha256,
|
||||
"ubuntu_image_sha256_set": True,
|
||||
"kernel_cmdline": KERNEL_CMDLINE,
|
||||
"playwright": bool(playwright),
|
||||
"playwright_browsers_path": PLAYWRIGHT_BROWSERS_PATH
|
||||
if playwright else "",
|
||||
# feat-chv-closeup(M4):钉死的 Chromium 修订号(未设
|
||||
# 记 null,绝不编一个默认值出来假装已钉死)。
|
||||
"chromium_pinned_rev": chromium_pinned_rev or None,
|
||||
"image_sha256": disk_sha,
|
||||
"artifacts": {
|
||||
"disk.raw": {"sha256": disk_sha, "size": disk_size},
|
||||
"vmlinux": {"sha256": vmlinux_sha, "size": vmlinux_size},
|
||||
},
|
||||
}
|
||||
path = os.path.join(outdir, "BUILDINFO.json")
|
||||
with open(path, "w") as fh:
|
||||
json.dump(info, fh, indent=2)
|
||||
fh.write("\n")
|
||||
return info
|
||||
|
||||
|
||||
def main(argv):
|
||||
if len(argv) not in (8, 9, 10):
|
||||
print("用法:buildinfo.py OUT TAG UBUNTU CODEX_VERSION "
|
||||
"REPO_COMMIT UBUNTU_IMAGE_SHA256 PLAYWRIGHT "
|
||||
"[CHROMIUM_HEADLESS_SHELL_REV] [EXTRA_CONTENTS]", file=sys.stderr)
|
||||
return 2
|
||||
_, outdir, tag, ubuntu, codex_version, repo_commit, \
|
||||
ubuntu_image_sha256, playwright = argv[:8]
|
||||
chromium_pinned_rev = argv[8] if len(argv) >= 9 else None
|
||||
# 第 10 参数:逗号分隔的附加内容(如 vsock_notify),可空。
|
||||
extra_contents = None
|
||||
if len(argv) == 10 and argv[9]:
|
||||
extra_contents = [s for s in argv[9].split(",") if s]
|
||||
if chromium_pinned_rev and not re.fullmatch(r"[0-9]+",
|
||||
chromium_pinned_rev):
|
||||
print("CHROMIUM_HEADLESS_SHELL_REV 须为纯数字:%s"
|
||||
% chromium_pinned_rev, file=sys.stderr)
|
||||
return 2
|
||||
info = build(outdir, tag, ubuntu, codex_version, repo_commit,
|
||||
ubuntu_image_sha256, playwright == "1",
|
||||
chromium_pinned_rev, extra_contents)
|
||||
if not info["versions"]["dpkg"]:
|
||||
print("BUILDINFO versions.dpkg 为空", file=sys.stderr)
|
||||
return 1
|
||||
if not info["versions"]["pip"]:
|
||||
print("BUILDINFO versions.pip 为空", file=sys.stderr)
|
||||
return 1
|
||||
if not info["gt_version"]:
|
||||
print("BUILDINFO gt_version 为空(gt 未装好)", file=sys.stderr)
|
||||
return 1
|
||||
if playwright == "1" and not info["versions"].get("chromium"):
|
||||
print("BUILDINFO versions.chromium 为空(浏览器未装好或"
|
||||
"读回路径不对)", file=sys.stderr)
|
||||
return 1
|
||||
if playwright == "1" and chromium_pinned_rev:
|
||||
# feat-chv-closeup(M4):钉死校验——读回的缓存目录名
|
||||
# 须恰好是该修订号,playwright 升级导致目录名变了就
|
||||
# 在这里失败,不静默换浏览器版本。
|
||||
expected = "chromium_headless_shell-%s" % chromium_pinned_rev
|
||||
if expected not in (info["versions"].get("chromium") or []):
|
||||
print("Chromium 修订号与钉死值不符:读回 %s,钉死 %s"
|
||||
% (info["versions"].get("chromium"), expected),
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
if info["tag"] != tag:
|
||||
print("BUILDINFO tag 不一致", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
@@ -1,181 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""校验已构建镜像的 BUILDINFO 供应链清单(事后 lockfile 检查)。
|
||||
|
||||
生成端见 generate_lockfile.py(读 BUILDINFO.versions 生成
|
||||
lockfile.json,版本唯一来源是 build.sh 从客机读回的
|
||||
dpkg/pip freeze + chromium 目录名 + codex/repo_commit/gt
|
||||
声明字段,绝不查构建宿主现场、绝不编造版本号)。
|
||||
本脚本校验 BUILDINFO:versions 非空、每条有版本、contents
|
||||
全覆盖、tag 与预期一致、钉死的 Chromium 修订号一致、
|
||||
可选的冻结清单(--frozen JSON,形如
|
||||
{"dpkg": ["pkg version", ...]})双向比对,漂移即退出非 0。
|
||||
|
||||
用法:
|
||||
python3 check_lockfile.py out/BUILDINFO.json [--tag TAG]
|
||||
[--frozen frozen.json]
|
||||
"""
|
||||
import json
|
||||
import sys
|
||||
|
||||
# 非包名的内容项:版本来自 BUILDINFO 的声明字段,不是 versions 列表。
|
||||
DECLARED_SOURCES = {
|
||||
"codex": "codex_version",
|
||||
"codex-code-mode-host": "codex_version",
|
||||
"codex-resources/bwrap": "codex_version",
|
||||
"/app/host.py": "repo_commit",
|
||||
"gt": "gt_version",
|
||||
"chv-sync": "repo_commit",
|
||||
# 对齐 Muse P2:nspawn 单元启动器 + broker + 浏览器子代理,
|
||||
# 均是仓库内脚本,版本随 repo_commit(与 chv-sync 同款)。
|
||||
"chv-cell": "repo_commit",
|
||||
"chv-broker": "repo_commit",
|
||||
"chv-browser": "repo_commit",
|
||||
# 对齐 Muse P3 + P1 vsock:authd 三件套、authd/sentinel
|
||||
# 叶子模块与 vsock 通知脚本,均是仓库内脚本,版本随
|
||||
# repo_commit。缺了它们 check 会报 contents 未覆盖(717efc4
|
||||
# 真机实测:6 项 unresolved,verify 与 check 双双失败)。
|
||||
# round-11 真机修复(层 1):叶子树新增 errors.py 与 stub
|
||||
# __init__.py(authd_init_stub.py),同款 repo_commit 登记。
|
||||
"chv-authd": "repo_commit",
|
||||
"git-credential-chv-authd": "repo_commit",
|
||||
"chv-authd.service": "repo_commit",
|
||||
"cloud/chv/authd.py": "repo_commit",
|
||||
"cloud/chv/sentinel.py": "repo_commit",
|
||||
"cloud/chv/errors.py": "repo_commit",
|
||||
"cloud/chv/__init__.py": "repo_commit",
|
||||
"vsock_notify": "repo_commit",
|
||||
# 对齐 Muse P4:CDP broker 三件在 contents 里,登记自 P4 起
|
||||
# 一直缺失;本轮补上,否则新镜像的 lockfile 仍 unresolved。
|
||||
"chv-cdp-broker": "repo_commit",
|
||||
"chv-browser-guard": "repo_commit",
|
||||
"browser-blocklist": "repo_commit",
|
||||
# 对齐 Muse P6:worker 四件(+ 路径触发器)+ 叶子模块(privsep/safety)。
|
||||
"chv-gitea-worker": "repo_commit",
|
||||
"chv-sync-worker-client": "repo_commit",
|
||||
"chv-gitea-worker.service": "repo_commit",
|
||||
"chv-gitea-worker.path": "repo_commit",
|
||||
"cloud/chv/privsep.py": "repo_commit",
|
||||
"cloud/chv/checkout_detect.py": "repo_commit",
|
||||
"cloud/chv/safety.py": "repo_commit",
|
||||
}
|
||||
|
||||
|
||||
def _names(entries, pip_style=False):
|
||||
"""versions 条目 → 小写名集合。dpkg 行是 "name version",
|
||||
pip freeze 行是 "name==version"。"""
|
||||
names = set()
|
||||
for entry in entries or []:
|
||||
text = str(entry)
|
||||
if pip_style and "==" in text:
|
||||
names.add(text.split("==", 1)[0].lower())
|
||||
else:
|
||||
parts = text.split(None, 1)
|
||||
if parts:
|
||||
names.add(parts[0].lower())
|
||||
return names
|
||||
|
||||
|
||||
def _check_format(kind, entries):
|
||||
"""每条须带非空版本:dpkg "name version",pip "name==version"。"""
|
||||
for entry in entries or []:
|
||||
text = str(entry)
|
||||
if kind == "pip" and "==" in text:
|
||||
if not text.split("==", 1)[1].strip():
|
||||
print("版本为空的条目:%s %r" % (kind, entry),
|
||||
file=sys.stderr)
|
||||
return False
|
||||
else:
|
||||
parts = text.split()
|
||||
if len(parts) < 2 or not parts[1]:
|
||||
print("版本为空的条目:%s %r" % (kind, entry),
|
||||
file=sys.stderr)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def main(argv):
|
||||
if len(argv) < 2:
|
||||
print("用法:check_lockfile.py BUILDINFO.json [--tag TAG] "
|
||||
"[--frozen frozen.json]", file=sys.stderr)
|
||||
return 2
|
||||
path = argv[1]
|
||||
tag = None
|
||||
frozen_path = None
|
||||
i = 2
|
||||
while i < len(argv):
|
||||
if argv[i] == "--tag" and i + 1 < len(argv):
|
||||
tag = argv[i + 1]
|
||||
i += 2
|
||||
elif argv[i] == "--frozen" and i + 1 < len(argv):
|
||||
frozen_path = argv[i + 1]
|
||||
i += 2
|
||||
else:
|
||||
print("未知参数:%s" % argv[i], file=sys.stderr)
|
||||
return 2
|
||||
with open(path) as fh:
|
||||
data = json.load(fh)
|
||||
if tag is not None and data.get("tag") != tag:
|
||||
print("tag 不一致:%s != %s" % (data.get("tag"), tag),
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
versions = data.get("versions") or {}
|
||||
if not versions.get("dpkg") or not versions.get("pip"):
|
||||
print("BUILDINFO.versions.dpkg/pip 为空,供应链清单缺失",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
if not _check_format("dpkg", versions.get("dpkg")):
|
||||
return 1
|
||||
if not _check_format("pip", versions.get("pip")):
|
||||
return 1
|
||||
# contents 全覆盖:每项须在 dpkg/pip 名单里,或有声明字段版本。
|
||||
covered = _names(versions.get("dpkg")) | _names(
|
||||
versions.get("pip"), pip_style=True)
|
||||
missing = []
|
||||
for name in data.get("contents") or []:
|
||||
if name.lower() in covered:
|
||||
continue
|
||||
source = DECLARED_SOURCES.get(name)
|
||||
if source and data.get(source):
|
||||
continue
|
||||
if name.lower() in ("chromium", "chromium-headless-shell") \
|
||||
and versions.get("chromium"):
|
||||
continue
|
||||
missing.append(name)
|
||||
if missing:
|
||||
print("contents 未覆盖:%s" % sorted(missing), file=sys.stderr)
|
||||
return 1
|
||||
# feat-chv-closeup(M4):钉死的 Chromium 修订号须与读回
|
||||
# 的缓存目录名一致;未钉(pinned_rev 为空)时不校验。
|
||||
pinned = data.get("chromium_pinned_rev")
|
||||
if pinned:
|
||||
expected = "chromium_headless_shell-%s" % pinned
|
||||
if expected not in (versions.get("chromium") or []):
|
||||
print("Chromium 修订号与钉死值不符:读回 %s,钉死 %s"
|
||||
% (versions.get("chromium"), expected),
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
if frozen_path:
|
||||
with open(frozen_path) as fh:
|
||||
frozen = json.load(fh)
|
||||
for kind in ("dpkg", "pip"):
|
||||
want = set(frozen.get(kind) or [])
|
||||
got = set(versions.get(kind) or [])
|
||||
absent = want - got
|
||||
if absent:
|
||||
print("%s 漂移:冻结清单里有、构建产物里无:%s"
|
||||
% (kind, sorted(absent)), file=sys.stderr)
|
||||
return 1
|
||||
extra = got - want
|
||||
if extra:
|
||||
print("%s 漂移:构建产物里有、冻结清单里无:%s"
|
||||
% (kind, sorted(extra)), file=sys.stderr)
|
||||
return 1
|
||||
print("lockfile 校验通过:tag=%s dpkg=%d pip=%d contents=%d"
|
||||
% (data.get("tag"), len(versions["dpkg"]),
|
||||
len(versions["pip"]),
|
||||
len(data.get("contents") or [])))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
@@ -1,22 +0,0 @@
|
||||
[Unit]
|
||||
Description=CHV authd (credential broker for chv-sync)
|
||||
# 注意时序:本服务须在 chv-sync boot 之前就绪。cloud-final 的
|
||||
# runcmd(99-chv-sync.cfg)在 chv-sync boot 之前显式
|
||||
# `systemctl start chv-authd`;这里不依赖 After=cloud-final
|
||||
#(cloud-final 跑完才 After 得到它,太晚)。enable 后开机自启
|
||||
# 只是兜底,首启顺序以 runcmd 的显式 start 为准。
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
# authd 跑在运行时单元之外(root 身份):它是凭证的唯一持有者,
|
||||
# 单元内的 agent 只能经 Unix socket + SO_PEERCRED 向它要凭证,
|
||||
# 读不到它的内存与 seed 文件(P2 单元隔离落地后)。
|
||||
ExecStart=/usr/bin/python3 /usr/local/bin/chv-authd
|
||||
Restart=on-failure
|
||||
RestartSec=2
|
||||
StandardOutput=journal
|
||||
StandardError=journal
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,336 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chv-broker:单元外 IPC broker(客机侧、单元外服务)。
|
||||
|
||||
对齐 Muse P2(见 docs/chv-muse-p2-gap.md):单元内外走 Unix socket +
|
||||
SO_PEERCRED 的最小权限 IPC。broker 只认白名单操作,凭证与浏览器
|
||||
都在单元外——单元看不见 token,只能请 broker 代办。
|
||||
|
||||
校验(双重,缺一不可才放行;无 cell pid 文件时只做 uid 检查并打日志):
|
||||
1. SO_PEERCRED 的 uid 须在单元映射范围 [1000, 1000+65536) 内;
|
||||
2. 对端 pid 的 pid ns 须与任一活着的单元 init 同 ns(part 4:
|
||||
并发 cell 各有 cell-<machine>.pid,旧的单文件 cell.pid 只记
|
||||
最新单元;读不到任何 pid 文件时跳过第 2 项并警告;有 pid 文
|
||||
件但对端 ns 与任一可读的单元 ns 都对不上时拒绝,不降级)。
|
||||
|
||||
操作白名单:
|
||||
ping -> {"ok": true}
|
||||
sync {action: pull|push|status, repo?} -> 单元外以 chv 身份跑
|
||||
chv-sync(list argv,不进 shell;仓目录只允许
|
||||
~/workspace 下,realpath + commonpath 校验)。
|
||||
browser {…} -> 原样转发给 chv-browser.sock(浏览器子代理),
|
||||
回包原样返回;浏览器服务不在时报错,不替它跑 playwright。
|
||||
|
||||
只用标准库。单连接一请求一响应(JSON 行,≤64KB);异常只断本
|
||||
连接,绝不杀死 broker 循环。
|
||||
"""
|
||||
import glob
|
||||
import grp
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
BROKER_SOCK = "/run/chv/broker.sock"
|
||||
# round-11 真机修复(缺陷 4):browser.sock 从 /run/chv 移到
|
||||
# chv-browser 自有目录——/run/chv 是 0755 root root,
|
||||
# chv-browser 在 bind 时 EACCES 而无限 crash loop(restart 21);
|
||||
# 放进组可写的 /run/chv 会给 chv-browser 对 broker.sock/cell.pid
|
||||
# 的 unlink 能力,所以用独立目录(0755 chv-browser 所有)。
|
||||
BROWSER_SOCK = "/run/chv-browser/browser.sock"
|
||||
CELL_PID_FILE = "/run/chv/cell.pid"
|
||||
CELL_PID_GLOB = "/run/chv/cell-*.pid"
|
||||
UID_BASE = 1000
|
||||
UID_COUNT = 65536
|
||||
WORKSPACE = "/home/chv/workspace"
|
||||
SYNC_USER = "chv"
|
||||
SYNC_BIN = "/usr/local/bin/chv-sync"
|
||||
MAX_REQUEST_BYTES = 65536
|
||||
MAX_OUTPUT_BYTES = 8192
|
||||
|
||||
SYNC_ACTIONS = ("pull", "push", "status")
|
||||
|
||||
|
||||
def _read_ns_link(pid, proc_root="/proc"):
|
||||
"""读 /proc/<pid>/ns/pid 的链接目标;读不到返回 None。"""
|
||||
try:
|
||||
return os.readlink(os.path.join(proc_root, str(pid), "ns", "pid"))
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def peer_allowed(peer_uid, peer_pid, cell_pid=None, proc_root="/proc"):
|
||||
"""对端是否允许用 broker。返回 (allowed, reason)。
|
||||
cell_pid 为 None 时只做 uid 检查(broker 打日志警告,调用方负责)。"""
|
||||
if not (UID_BASE <= peer_uid < UID_BASE + UID_COUNT):
|
||||
return False, "uid %d 不在单元映射范围 [%d, %d)" % (
|
||||
peer_uid, UID_BASE, UID_BASE + UID_COUNT)
|
||||
if cell_pid is None:
|
||||
return True, "uid 在范围内(cell.pid 缺失,未做 pid ns 对照)"
|
||||
cell_ns = _read_ns_link(cell_pid, proc_root)
|
||||
if cell_ns is None:
|
||||
return False, "cell.pid 的 pid ns 读不到,拒绝(不降级)"
|
||||
peer_ns = _read_ns_link(peer_pid, proc_root)
|
||||
if peer_ns is None:
|
||||
return False, "对端 pid ns 读不到,拒绝"
|
||||
if peer_ns != cell_ns:
|
||||
return False, "对端 pid ns 与单元不一致"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
def peer_allowed_multi(peer_uid, peer_pid, cell_pids, proc_root="/proc"):
|
||||
"""多 cell 版:对端 pid ns 与任一活 cell 的 init 同 ns 即放行
|
||||
(part 4 并发 cell 用;单 cell 时与 peer_allowed 等价)。
|
||||
cell_pids 为空即只做 uid 检查;全部 pid 的 ns 都读不到时
|
||||
拒绝(不降级);个别 pid 的 ns 读不到(已退出的残留文件)
|
||||
只跳过该项,不因此拒绝整个请求。"""
|
||||
if not (UID_BASE <= peer_uid < UID_BASE + UID_COUNT):
|
||||
return False, "uid %d 不在单元映射范围 [%d, %d)" % (
|
||||
peer_uid, UID_BASE, UID_BASE + UID_COUNT)
|
||||
if not cell_pids:
|
||||
# part 5:reason 文案与单 cell 版 peer_allowed 对齐,保留
|
||||
# “cell.pid 缺失”字样——P2-9 驱动 grep 该串认降级日志;
|
||||
# part 4 切到 multi 版时把这里改写成“无 cell pid 文件”,
|
||||
# 该串自此再也没进过 journal,是 RETEST4
|
||||
# degraded_logged=False 的根因(不是时序、不是日志路径)。
|
||||
return True, "uid 在范围内(cell.pid 缺失,未做 pid ns 对照)"
|
||||
peer_ns = _read_ns_link(peer_pid, proc_root)
|
||||
if peer_ns is None:
|
||||
return False, "对端 pid ns 读不到,拒绝"
|
||||
readable = 0
|
||||
for cell_pid in cell_pids:
|
||||
cell_ns = _read_ns_link(cell_pid, proc_root)
|
||||
if cell_ns is None:
|
||||
continue
|
||||
readable += 1
|
||||
if peer_ns == cell_ns:
|
||||
return True, "ok"
|
||||
if readable == 0:
|
||||
return False, "cell pid 的 pid ns 均读不到,拒绝(不降级)"
|
||||
return False, "对端 pid ns 与任何单元不一致"
|
||||
|
||||
|
||||
def _read_cell_pid():
|
||||
try:
|
||||
with open(CELL_PID_FILE) as fh:
|
||||
return int(fh.read().strip())
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _read_cell_pids(cell_dir="/run/chv"):
|
||||
"""读全部 cell pid:每单元一份 cell-<machine>.pid + 兼容旧
|
||||
路径 cell.pid(最新单元)。去重、跳过坏内容,返回 int 列表。"""
|
||||
pids = []
|
||||
paths = sorted(glob.glob(os.path.join(cell_dir, "cell-*.pid")))
|
||||
paths.append(os.path.join(cell_dir, "cell.pid"))
|
||||
for path in paths:
|
||||
try:
|
||||
with open(path) as fh:
|
||||
pid = int(fh.read().strip())
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if pid not in pids:
|
||||
pids.append(pid)
|
||||
return pids
|
||||
|
||||
|
||||
def _degraded_warning(reason, pid, uid):
|
||||
"""降级警告行:reason 原样进日志。serve() 在无 cell pid 文件
|
||||
时打这条;P2-9 驱动 grep 'cell.pid 缺失' 认的就是它(part 5:
|
||||
part 4 之前 serve 直接打印含该串的警告行,RETEST3 同项为
|
||||
True;part 4 切到 peer_allowed_multi 后警告行改写成
|
||||
“无 cell pid 文件”,该串再也没进过 journal——RETEST4 的
|
||||
degraded_logged=False 即由此来,不是时序、不是日志路径
|
||||
变化。reason 与警告行分开写曾让两者悄悄漂移,今后改文案
|
||||
只改 peer_allowed_multi 的 reason,警告行自动跟随)。"""
|
||||
return ("chv-broker: 警告:%s (pid=%d uid=%d)"
|
||||
% (reason, pid, uid))
|
||||
|
||||
|
||||
def _resolve_repo(repo):
|
||||
"""仓目录只允许 ~/workspace 下。repo 为空时返回 None(由
|
||||
chv-sync 自己取唯一仓)。返回解析后的绝对路径或抛 ValueError。"""
|
||||
if not repo:
|
||||
return None
|
||||
base = os.path.realpath(WORKSPACE)
|
||||
target = os.path.realpath(repo)
|
||||
if os.path.commonpath([base, target]) != base or target == base:
|
||||
raise ValueError("仓目录只允许 %s 下:%s" % (WORKSPACE, repo))
|
||||
return target
|
||||
|
||||
|
||||
def _truncate(text, limit=MAX_OUTPUT_BYTES):
|
||||
if text is None:
|
||||
return ""
|
||||
if len(text) > limit:
|
||||
return text[:limit] + "\n…(已截断)"
|
||||
return text
|
||||
|
||||
|
||||
def _run_sync(action, repo):
|
||||
if action not in SYNC_ACTIONS:
|
||||
return {"ok": False,
|
||||
"error": "sync action 只允许 %s:%s"
|
||||
% ("/".join(SYNC_ACTIONS), action)}
|
||||
try:
|
||||
repo_path = _resolve_repo(repo)
|
||||
except ValueError as e:
|
||||
return {"ok": False, "error": str(e)}
|
||||
cmd = ["runuser", "-u", SYNC_USER, "--", "env", "HOME=/home/chv",
|
||||
SYNC_BIN, action]
|
||||
if repo_path:
|
||||
cmd.append(repo_path)
|
||||
try:
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True,
|
||||
timeout=300)
|
||||
except subprocess.TimeoutExpired:
|
||||
return {"ok": False, "error": "chv-sync 超时(300 秒)"}
|
||||
except OSError as e:
|
||||
return {"ok": False, "error": "chv-sync 起不来:%s" % e}
|
||||
return {"ok": proc.returncode == 0,
|
||||
"stdout": _truncate(proc.stdout),
|
||||
"stderr": _truncate(proc.stderr),
|
||||
"returncode": proc.returncode}
|
||||
|
||||
|
||||
def _forward_browser(req, sock_path=BROWSER_SOCK, timeout=120):
|
||||
"""转发给浏览器子代理;回包须是带 ok 的 dict,否则报错。"""
|
||||
try:
|
||||
conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
conn.settimeout(timeout)
|
||||
conn.connect(sock_path)
|
||||
except OSError as e:
|
||||
return {"ok": False,
|
||||
"error": "浏览器子代理未就绪:%s" % e}
|
||||
try:
|
||||
conn.sendall((json.dumps(req, ensure_ascii=False) + "\n").encode())
|
||||
data = b""
|
||||
while len(data) < MAX_REQUEST_BYTES:
|
||||
chunk = conn.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
data += chunk
|
||||
if data.endswith(b"\n"):
|
||||
break
|
||||
except OSError as e:
|
||||
return {"ok": False, "error": "浏览器子代理通信失败:%s" % e}
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
resp = json.loads(data.decode())
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
return {"ok": False, "error": "浏览器子代理回包不是 JSON"}
|
||||
if not isinstance(resp, dict) or "ok" not in resp:
|
||||
return {"ok": False, "error": "浏览器子代理回包格式不对"}
|
||||
return resp
|
||||
|
||||
|
||||
def handle_request(req, browser_sock=BROWSER_SOCK):
|
||||
"""处理一个请求 dict,返回响应 dict(纯逻辑,可单测)。"""
|
||||
if not isinstance(req, dict):
|
||||
return {"ok": False, "error": "请求须是 JSON 对象"}
|
||||
op = req.get("op")
|
||||
if op == "ping":
|
||||
return {"ok": True, "cell": True}
|
||||
if op == "sync":
|
||||
return _run_sync(req.get("action"), req.get("repo"))
|
||||
if op == "browser":
|
||||
return _forward_browser(req, sock_path=browser_sock)
|
||||
return {"ok": False, "error": "未知操作:%r(白名单:ping/sync/browser)"
|
||||
% (op,)}
|
||||
|
||||
|
||||
def _peer_creds(conn):
|
||||
"""SO_PEERCRED:返回 (pid, uid, gid)。Linux struct ucred = 3 个 int。"""
|
||||
raw = conn.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED,
|
||||
struct.calcsize("3i"))
|
||||
pid, uid, gid = struct.unpack("3i", raw)
|
||||
return pid, uid, gid
|
||||
|
||||
|
||||
def serve(sock_path=BROKER_SOCK, browser_sock=BROWSER_SOCK):
|
||||
"""监听 Unix socket,逐连接处理。返回码 0 只能是被信号杀死;
|
||||
socket 建不起来即抛错(systemd 会重启服务,不要静默带病跑)。"""
|
||||
if os.path.exists(sock_path):
|
||||
os.remove(sock_path)
|
||||
os.makedirs(os.path.dirname(sock_path), exist_ok=True)
|
||||
srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
srv.bind(sock_path)
|
||||
srv.listen(16)
|
||||
# 0660 + 组 chv:单元内 root 在客机侧是 uid 1000(chv),
|
||||
# 组 chv 可连;其它用户连不上(再叠加 SO_PEERCRED 校验)。
|
||||
os.chmod(sock_path, 0o660)
|
||||
try:
|
||||
gid = grp.getgrnam("chv").gr_gid
|
||||
except KeyError:
|
||||
print("chv-broker: 组 chv 不存在,socket 权限置不全,拒绝启动",
|
||||
file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
os.chown(sock_path, -1, gid)
|
||||
print("chv-broker: 监听 %s" % sock_path, file=sys.stderr, flush=True)
|
||||
while True:
|
||||
try:
|
||||
conn, _ = srv.accept()
|
||||
except OSError:
|
||||
continue
|
||||
try:
|
||||
pid, uid, _gid = _peer_creds(conn)
|
||||
cell_pids = _read_cell_pids()
|
||||
allowed, reason = peer_allowed_multi(uid, pid, cell_pids)
|
||||
if not allowed:
|
||||
print("chv-broker: 拒绝连接 pid=%d uid=%d:%s"
|
||||
% (pid, uid, reason), file=sys.stderr, flush=True)
|
||||
conn.close()
|
||||
continue
|
||||
if not cell_pids:
|
||||
print(_degraded_warning(reason, pid, uid),
|
||||
file=sys.stderr, flush=True)
|
||||
conn.settimeout(310)
|
||||
data = b""
|
||||
while len(data) < MAX_REQUEST_BYTES:
|
||||
chunk = conn.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
data += chunk
|
||||
if data.endswith(b"\n"):
|
||||
break
|
||||
try:
|
||||
req = json.loads(data.decode())
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
resp = {"ok": False, "error": "请求不是合法 JSON"}
|
||||
else:
|
||||
resp = handle_request(req, browser_sock=browser_sock)
|
||||
conn.sendall((json.dumps(resp, ensure_ascii=False)
|
||||
+ "\n").encode())
|
||||
except OSError as e:
|
||||
print("chv-broker: 连接处理异常:%s" % e, file=sys.stderr,
|
||||
flush=True)
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def main(argv):
|
||||
sock_path = BROKER_SOCK
|
||||
browser_sock = BROWSER_SOCK
|
||||
i = 0
|
||||
while i < len(argv):
|
||||
if argv[i] == "--socket" and i + 1 < len(argv):
|
||||
sock_path = argv[i + 1]
|
||||
i += 2
|
||||
elif argv[i] == "--browser-socket" and i + 1 < len(argv):
|
||||
browser_sock = argv[i + 1]
|
||||
i += 2
|
||||
else:
|
||||
print("chv-broker: 未知参数:%s" % argv[i], file=sys.stderr)
|
||||
return 2
|
||||
serve(sock_path, browser_sock)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -1,316 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chv-browser:浏览器子代理(客机侧、单元外独立进程)。
|
||||
|
||||
对齐 Muse P4(见 docs/chv-muse-p4-gap.md):本文件源自 P2 的
|
||||
chv-browser.py(c61484a),P4 在此基础上重构——子代理不再直连
|
||||
Playwright/Chromium,Chromium 的所有权在单元外 CDP broker
|
||||
(chv-cdp-broker.py)手里;本进程只是 broker 的客户端。
|
||||
|
||||
子代理只向单元返回页面无障碍快照(Playwright
|
||||
Page.aria_snapshot() 的 ARIA 快照字符串,经 broker 取回),不见
|
||||
原始 DOM,不能在页面上下文跑 JS,不暴露交互操作,不开 DevTools。
|
||||
|
||||
操作(Unix socket /run/chv-browser/browser.sock,JSON 行):
|
||||
open {url} -> 经 broker 导航并返回 {title, url, a11y,
|
||||
truncated, untrusted_input, injection_flags,
|
||||
form_risk}
|
||||
snapshot {} -> 当前页同上
|
||||
screenshot {} -> 截图落盘,返回 {path}
|
||||
close {} -> 关当前页
|
||||
pause {} -> 暂停(用户接管/填表/自动暂停的统一状态)
|
||||
resume {} -> 解除暂停
|
||||
status {} -> {paused, url, form_risk}
|
||||
list_asks {} -> 待审批 ask 列表
|
||||
resolve_ask {ask_id, decision} -> 写回 allow/denied
|
||||
ping {} -> {"ok": true}
|
||||
|
||||
暂停时除 ping/resume/status/list_asks/resolve_ask 外一切操作
|
||||
拒绝——用户接管或凭证注入填表时,代理完全不能行动。
|
||||
|
||||
URL 校验、快照截断、错误消毒、禁用操作表都在
|
||||
chv_browser_guard.py(纯函数模块,两脚本共用)。本层再做一次
|
||||
validate_url 与 truncate_aria(纵深防御),回包先过泄露检查。
|
||||
|
||||
只用标准库。运行身份:系统用户 chv-browser,socket 0600
|
||||
(broker 以 root 连,不受模式限制)。
|
||||
"""
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
|
||||
# round-11 真机修复(缺陷 4):自有目录 /run/chv-browser,
|
||||
# 0755 chv-browser 所有——/run/chv 是 0755 root root,
|
||||
# 在此 bind 会 EACCES 而无限 crash loop(见 chv-broker.py)。
|
||||
# P4 的 CDP_SOCK 保留:broker 侧仍走 /run/chv/cdp-broker.sock。
|
||||
BROWSER_SOCK = "/run/chv-browser/browser.sock"
|
||||
CDP_SOCK = "/run/chv/cdp-broker.sock"
|
||||
MAX_REQUEST_BYTES = 65536
|
||||
|
||||
_GUARD = None
|
||||
|
||||
|
||||
def _load_guard():
|
||||
"""加载同目录的 chv_browser_guard(纯函数模块)。"""
|
||||
global _GUARD
|
||||
if _GUARD is not None:
|
||||
return _GUARD
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
path = os.path.join(here, "chv_browser_guard.py")
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"chv_browser_guard", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
_GUARD = module
|
||||
return module
|
||||
|
||||
|
||||
class BrowserAgent:
|
||||
"""浏览器子代理:CDP broker 的客户端。broker_call 供单测注入
|
||||
假 broker(收到 req dict 返回 resp dict);正常运行时走
|
||||
Unix socket 直连 broker。"""
|
||||
|
||||
def __init__(self, cdp_sock_path=CDP_SOCK, broker_call=None):
|
||||
self._guard = _load_guard()
|
||||
self._cdp_sock_path = cdp_sock_path
|
||||
self._broker_call = broker_call
|
||||
self._paused = False
|
||||
|
||||
def _call(self, req):
|
||||
if self._broker_call is not None:
|
||||
return self._broker_call(req)
|
||||
try:
|
||||
conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
conn.settimeout(120)
|
||||
conn.connect(self._cdp_sock_path)
|
||||
except OSError as e:
|
||||
return {"ok": False,
|
||||
"error": "CDP broker 未就绪:%s" % e}
|
||||
try:
|
||||
conn.sendall((json.dumps(req, ensure_ascii=False)
|
||||
+ "\n").encode())
|
||||
data = b""
|
||||
while len(data) < MAX_REQUEST_BYTES:
|
||||
chunk = conn.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
data += chunk
|
||||
if data.endswith(b"\n"):
|
||||
break
|
||||
except OSError as e:
|
||||
return {"ok": False,
|
||||
"error": "CDP broker 通信失败:%s" % e}
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
resp = json.loads(data.decode())
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
return {"ok": False, "error": "CDP broker 回包不是 JSON"}
|
||||
if not isinstance(resp, dict) or "ok" not in resp:
|
||||
return {"ok": False, "error": "CDP broker 回包格式不对"}
|
||||
return resp
|
||||
|
||||
def _sync_paused(self, resp):
|
||||
"""从回包同步暂停镜像状态(broker 暂停时本层同样拒绝)。"""
|
||||
if isinstance(resp, dict) and "已暂停" in str(
|
||||
resp.get("error", "")):
|
||||
self._paused = True
|
||||
return resp
|
||||
|
||||
def _blocked_while_paused(self):
|
||||
return {"ok": False,
|
||||
"error": "浏览器已暂停(用户接管/填表/自动暂停)"}
|
||||
|
||||
def open(self, url):
|
||||
if self._paused:
|
||||
return self._blocked_while_paused()
|
||||
ok, err = self._guard.validate_url(url)
|
||||
if not ok:
|
||||
return {"ok": False, "error": err}
|
||||
resp = self._call({"op": "open", "url": url})
|
||||
if resp.get("ok") and isinstance(resp.get("a11y"), str):
|
||||
aria, truncated = self._guard.truncate_aria(resp["a11y"])
|
||||
resp = dict(resp)
|
||||
resp["a11y"] = aria
|
||||
resp["truncated"] = bool(resp.get("truncated")
|
||||
or truncated)
|
||||
return self._sync_paused(resp)
|
||||
|
||||
def snapshot(self):
|
||||
if self._paused:
|
||||
return self._blocked_while_paused()
|
||||
resp = self._call({"op": "snapshot"})
|
||||
if resp.get("ok") and isinstance(resp.get("a11y"), str):
|
||||
aria, truncated = self._guard.truncate_aria(resp["a11y"])
|
||||
resp = dict(resp)
|
||||
resp["a11y"] = aria
|
||||
resp["truncated"] = bool(resp.get("truncated")
|
||||
or truncated)
|
||||
return self._sync_paused(resp)
|
||||
|
||||
def screenshot(self):
|
||||
if self._paused:
|
||||
return self._blocked_while_paused()
|
||||
return self._sync_paused(self._call({"op": "screenshot"}))
|
||||
|
||||
def close(self):
|
||||
if self._paused:
|
||||
return self._blocked_while_paused()
|
||||
resp = self._call({"op": "close"})
|
||||
return self._sync_paused(resp)
|
||||
|
||||
def pause(self):
|
||||
resp = self._call({"op": "pause"})
|
||||
if resp.get("ok"):
|
||||
self._paused = True
|
||||
return resp
|
||||
|
||||
def resume(self):
|
||||
resp = self._call({"op": "resume"})
|
||||
if resp.get("ok"):
|
||||
self._paused = False
|
||||
return resp
|
||||
|
||||
def status(self):
|
||||
resp = self._call({"op": "status"})
|
||||
if isinstance(resp, dict) and "paused" in resp:
|
||||
self._paused = bool(resp["paused"])
|
||||
return resp
|
||||
|
||||
def list_asks(self):
|
||||
return self._call({"op": "list_asks"})
|
||||
|
||||
def resolve_ask(self, ask_id, decision):
|
||||
resp = self._call({"op": "resolve_ask", "ask_id": ask_id,
|
||||
"decision": decision})
|
||||
if resp.get("ok") and resp.get("decision") == "allowed":
|
||||
self._paused = False
|
||||
return resp
|
||||
|
||||
|
||||
def handle_request(agent, req):
|
||||
"""处理一个请求 dict,返回响应 dict(纯逻辑,可单测)。
|
||||
op "browser" 是 broker 原样转发的形状(round-11 真机修复):
|
||||
内层 op 在 action 里;无 action 但带 url 时默认 open(驱动
|
||||
只发 url);action 同样受 FORBIDDEN_OPS 约束。"""
|
||||
guard = _load_guard()
|
||||
if not isinstance(req, dict):
|
||||
return {"ok": False, "error": "请求须是 JSON 对象"}
|
||||
op = req.get("op")
|
||||
if op == "browser":
|
||||
op = req.get("action")
|
||||
if op is None and req.get("url"):
|
||||
op = "open"
|
||||
if op in guard.FORBIDDEN_OPS:
|
||||
return {"ok": False,
|
||||
"error": "操作 %r 不暴露(无 JS/无原始 DOM/无交互)"
|
||||
% op}
|
||||
if op == "ping":
|
||||
return {"ok": True, "browser": True}
|
||||
if op == "open":
|
||||
return agent.open(req.get("url"))
|
||||
if op == "snapshot":
|
||||
return agent.snapshot()
|
||||
if op == "screenshot":
|
||||
return agent.screenshot()
|
||||
if op == "close":
|
||||
return agent.close()
|
||||
if op == "pause":
|
||||
return agent.pause()
|
||||
if op == "resume":
|
||||
return agent.resume()
|
||||
if op == "status":
|
||||
return agent.status()
|
||||
if op == "list_asks":
|
||||
return agent.list_asks()
|
||||
if op == "resolve_ask":
|
||||
return agent.resolve_ask(req.get("ask_id"),
|
||||
req.get("decision"))
|
||||
return {"ok": False,
|
||||
"error": "未知操作:%r(白名单:open/snapshot/screenshot/"
|
||||
"close/pause/resume/status/list_asks/resolve_ask/"
|
||||
"ping)" % (op,)}
|
||||
|
||||
|
||||
def _handle_conn(agent, conn):
|
||||
"""一个连接一请求一响应;回包先过泄露检查,不过即换通用错;
|
||||
错误信息过消毒,绝不把页面原文原样放进回包。"""
|
||||
guard = _load_guard()
|
||||
conn.settimeout(120)
|
||||
data = b""
|
||||
while len(data) < MAX_REQUEST_BYTES:
|
||||
chunk = conn.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
data += chunk
|
||||
if data.endswith(b"\n"):
|
||||
break
|
||||
try:
|
||||
req = json.loads(data.decode())
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
resp = {"ok": False, "error": "请求不是合法 JSON"}
|
||||
else:
|
||||
try:
|
||||
resp = handle_request(agent, req)
|
||||
except Exception as e:
|
||||
resp = {"ok": False,
|
||||
"error": guard.sanitize_error(e)}
|
||||
if guard.response_leaks(resp):
|
||||
resp = {"ok": False, "error": "响应泄露检查未通过(已拦截)"}
|
||||
conn.sendall((json.dumps(resp, ensure_ascii=False) + "\n").encode())
|
||||
|
||||
|
||||
def serve(sock_path=BROWSER_SOCK, cdp_sock_path=CDP_SOCK):
|
||||
"""监听 Unix socket,逐连接处理。socket 0600。建不起来即抛错,
|
||||
不要静默带病跑。"""
|
||||
if os.path.exists(sock_path):
|
||||
os.remove(sock_path)
|
||||
os.makedirs(os.path.dirname(sock_path), exist_ok=True)
|
||||
srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
srv.bind(sock_path)
|
||||
srv.listen(16)
|
||||
os.chmod(sock_path, 0o600)
|
||||
print("chv-browser: 监听 %s" % sock_path, file=sys.stderr, flush=True)
|
||||
agent = BrowserAgent(cdp_sock_path=cdp_sock_path)
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
conn, _ = srv.accept()
|
||||
except OSError:
|
||||
continue
|
||||
try:
|
||||
_handle_conn(agent, conn)
|
||||
except OSError as e:
|
||||
print("chv-browser: 连接处理异常:%s"
|
||||
% _load_guard().sanitize_error(e),
|
||||
file=sys.stderr, flush=True)
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
pass
|
||||
|
||||
|
||||
def main(argv):
|
||||
sock_path = BROWSER_SOCK
|
||||
cdp_sock_path = CDP_SOCK
|
||||
i = 0
|
||||
while i < len(argv):
|
||||
if argv[i] == "--socket" and i + 1 < len(argv):
|
||||
sock_path = argv[i + 1]
|
||||
i += 2
|
||||
elif argv[i] == "--cdp-socket" and i + 1 < len(argv):
|
||||
cdp_sock_path = argv[i + 1]
|
||||
i += 2
|
||||
else:
|
||||
print("chv-browser: 未知参数:%s" % argv[i], file=sys.stderr)
|
||||
return 2
|
||||
serve(sock_path, cdp_sock_path)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -1,559 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chv-cdp-broker:单元外 CDP broker(客机侧、单元外独立进程)。
|
||||
|
||||
对齐 Muse P4(见 docs/chv-muse-p4-gap.md):Chromium 的所有权
|
||||
在这里,不在浏览器子代理手里。broker 独家持有 Playwright,
|
||||
CDP 连接只存在于本进程内;子代理(chv-browser.py)只经本
|
||||
socket 说话,绝不直连浏览器、绝不见 CDP 端点。
|
||||
|
||||
操作(Unix socket /run/chv/cdp-broker.sock,JSON 行):
|
||||
open {url} -> 导航并返回 {title, url, a11y, truncated,
|
||||
untrusted_input, injection_flags, form_risk,
|
||||
checkout}
|
||||
snapshot {} -> 当前页同上
|
||||
screenshot {} -> 截图落盘 /run/chv/shots/<uuid>.png,返回 {path}
|
||||
close {} -> 关当前页(浏览器进程继续跑)
|
||||
pause {} -> 暂停(用户接管/填表/自动暂停的统一状态)
|
||||
resume {} -> 解除暂停
|
||||
status {} -> {paused, url, form_risk}
|
||||
list_asks {} -> 待审批 ask 列表
|
||||
resolve_ask {ask_id, decision} -> 写回 allow/denied
|
||||
ping {} -> {"ok": true}
|
||||
|
||||
暂停时除 ping/resume/status/list_asks 外一切操作拒绝——
|
||||
用户接管或凭证注入填表时,代理完全不能行动。
|
||||
|
||||
安全规则(纯函数在 chv_browser_guard.py,本文件只做接线):
|
||||
导航前 validate_url + 静态拦截名单;快照恒带
|
||||
untrusted_input=true(外部页面文本永远是数据、不是指令),
|
||||
命中注入启发式带 injection_flags;高风险表单在页时 open
|
||||
导航即返回 ask(入队、执行停止);结账页命中时 open 导航
|
||||
同样返回 ask 并暂停(round-12 修复轮项 4,P10 接线,
|
||||
detect_checkout 在快照时重算、导航前后两道门);观测到
|
||||
导航 POST 即自动暂停 + ask;下载命中可执行扩展名即取消
|
||||
+ 暂停 + ask。
|
||||
所有回包与日志过 sanitize_error;响应里绝无 ws 端点、
|
||||
DevTools 地址、原始 HTML。
|
||||
|
||||
Playwright 调用白名单:launch/goto/wait_for_load_state/title/
|
||||
url/aria_snapshot/screenshot/close + 只读监听
|
||||
on(request/download)。evaluate/add_script_tag/add_init_script/
|
||||
expose_function/set_content 等脚本注入面不在代码里出现,
|
||||
测试用 AST 扫本文件钉住。
|
||||
|
||||
只用标准库 + playwright(运行时,懒加载)。运行身份:系统用户
|
||||
chv-browser,socket 0600。
|
||||
"""
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import uuid
|
||||
|
||||
CDP_SOCK = "/run/chv/cdp-broker.sock"
|
||||
ASK_QUEUE_DIR = "/run/chv/ask-queue"
|
||||
BLOCKLIST_PATH = "/usr/local/share/chv/browser-blocklist.txt"
|
||||
SHOTS_DIR = "/run/chv/shots"
|
||||
PLAYWRIGHT_BROWSERS_PATH = "/usr/local/share/ms-playwright"
|
||||
MAX_REQUEST_BYTES = 65536
|
||||
NAV_TIMEOUT_MS = 30000
|
||||
|
||||
_GUARD = None
|
||||
_CHECKOUT = None
|
||||
|
||||
|
||||
def _load_guard():
|
||||
"""加载同目录的 chv_browser_guard(纯函数模块)。运行时
|
||||
安装在 /usr/local/bin/chv_browser_guard.py;测试时与本
|
||||
文件同在 chv-guest/。只在首次调用加载一次。"""
|
||||
global _GUARD
|
||||
if _GUARD is not None:
|
||||
return _GUARD
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
path = os.path.join(here, "chv_browser_guard.py")
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"chv_browser_guard", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
_GUARD = module
|
||||
return module
|
||||
|
||||
|
||||
def _load_checkout():
|
||||
"""加载结账检测模块 checkout_detect(纯函数模块)。
|
||||
|
||||
运行时安装在 /usr/local/bin/checkout_detect.py;测试时在
|
||||
仓库 cloud/chv/checkout_detect.py。找到即只加载一次;两处
|
||||
都找不到时返回 None(降级:结账检测关闭,broker 继续跑并
|
||||
写警告——不因缺一个检测模块把浏览器整路 crash-loop,但完
|
||||
整镜像必须含它,build.sh 有登记、BUILDINFO 有登记)。
|
||||
"""
|
||||
global _CHECKOUT
|
||||
if _CHECKOUT is not None:
|
||||
return _CHECKOUT
|
||||
here = os.path.dirname(os.path.abspath(__file__))
|
||||
candidates = [
|
||||
os.path.join(here, "checkout_detect.py"),
|
||||
os.path.join(os.path.dirname(here), "cloud", "chv",
|
||||
"checkout_detect.py"),
|
||||
]
|
||||
for path in candidates:
|
||||
if not os.path.exists(path):
|
||||
continue
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"checkout_detect", path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
_CHECKOUT = module
|
||||
return module
|
||||
sys.stderr.write(
|
||||
"chv-cdp-broker: 警告:checkout_detect 未找到(%s),"
|
||||
"结账检测关闭\n" % candidates)
|
||||
_CHECKOUT = None
|
||||
return None
|
||||
|
||||
|
||||
def _load_playwright():
|
||||
"""懒加载 playwright;缺失时报明错(不静默假装有浏览器)。
|
||||
返回调用后的上下文管理器(sync_playwright()),不是函数本身——
|
||||
round-11 复测回归(原在 chv-browser.py,P4 起 Playwright
|
||||
所有权移到本文件,同形 bug 随之移到此处;合入时移植):
|
||||
曾返回裸 sync_playwright 函数,_ensure_browser 对它调
|
||||
__enter__ 即 AttributeError,open 永远失败;单测的假工厂
|
||||
直接返回带 __enter__ 的对象,盖不住这个形状(见
|
||||
test_chv_muse_p4.py 的 test_load_playwright_returns_called_cm)。"""
|
||||
os.environ.setdefault("PLAYWRIGHT_BROWSERS_PATH",
|
||||
PLAYWRIGHT_BROWSERS_PATH)
|
||||
try:
|
||||
from playwright.sync_api import sync_playwright
|
||||
except ImportError as e:
|
||||
raise RuntimeError(
|
||||
"chv-cdp-broker: playwright 未装(%s)" % e)
|
||||
return sync_playwright()
|
||||
|
||||
|
||||
class CdpBroker:
|
||||
"""CDP broker:独家持有 Chromium。playwright_factory 供单测
|
||||
注入假实现;正常运行时用 _load_playwright()。"""
|
||||
|
||||
def __init__(self, playwright_factory=None,
|
||||
blocklist_path=BLOCKLIST_PATH,
|
||||
ask_queue_dir=ASK_QUEUE_DIR, shots_dir=SHOTS_DIR):
|
||||
self._guard = _load_guard()
|
||||
# round-12 修复轮(项 4,P10):结账检测接线。模块缺失
|
||||
# 时 _checkout_mod 为 None,检测降级关闭(见 _load_checkout)。
|
||||
self._checkout_mod = _load_checkout()
|
||||
self._checkout = {"is_checkout": False, "confidence": "low",
|
||||
"reasons": [], "form_risk": None,
|
||||
"amount_text": None}
|
||||
# 本次会话里已批过 ask 的结账页 URL:批过之后同一页
|
||||
# 再 open 不再重复 ask(与 form_risk“批准即放行”同口径)。
|
||||
self._checkout_approved = set()
|
||||
self._factory = playwright_factory or _load_playwright
|
||||
self._blocklist_path = blocklist_path
|
||||
self._ask_queue_dir = ask_queue_dir
|
||||
self._shots_dir = shots_dir
|
||||
self._pw = None
|
||||
self._browser = None
|
||||
self._page = None
|
||||
self._paused = False
|
||||
self._form_risk = {"level": "none", "fields": []}
|
||||
self._current_url = ""
|
||||
|
||||
def _ensure_browser(self):
|
||||
if self._browser is not None:
|
||||
return
|
||||
ok, err = self._guard.check_launch_args(
|
||||
self._guard.LAUNCH_ARGS)
|
||||
if not ok:
|
||||
raise RuntimeError("启动参数不合规:%s" % err)
|
||||
self._pw = self._factory()
|
||||
entered = self._pw.__enter__()
|
||||
self._browser = entered.chromium.launch(
|
||||
headless=True, args=list(self._guard.LAUNCH_ARGS))
|
||||
|
||||
def _new_page(self):
|
||||
if self._page is not None:
|
||||
try:
|
||||
self._page.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._page = self._browser.new_page()
|
||||
self._page.on("request", self._on_request)
|
||||
self._page.on("download", self._on_download)
|
||||
|
||||
def _on_request(self, request):
|
||||
"""只读监听:导航 POST(表单提交)即自动暂停 + 入队。
|
||||
非导航 POST(beacon/fetch)不在此处理——它不代表提交。"""
|
||||
try:
|
||||
method = request.method
|
||||
is_nav = request.is_navigation_request()
|
||||
except Exception:
|
||||
return
|
||||
if method != "POST" or not is_nav:
|
||||
return
|
||||
self._paused = True
|
||||
self._guard.enqueue_ask(
|
||||
self._ask_queue_dir, "form_submit_observed",
|
||||
self._current_url, "观测到导航 POST(疑似表单提交)")
|
||||
|
||||
def _on_download(self, download):
|
||||
"""只读监听:命中可执行扩展名即取消下载 + 暂停 + 入队。"""
|
||||
try:
|
||||
name = download.suggested_filename
|
||||
except Exception:
|
||||
return
|
||||
blocked, reason = self._guard.scan_download(name)
|
||||
if not blocked:
|
||||
return
|
||||
try:
|
||||
download.cancel()
|
||||
except Exception:
|
||||
pass
|
||||
self._paused = True
|
||||
self._guard.enqueue_ask(
|
||||
self._ask_queue_dir, "download_blocked",
|
||||
self._current_url, "下载命中拦截规则:%s" % reason)
|
||||
|
||||
def _paused_resp(self):
|
||||
return {"ok": False,
|
||||
"error": "浏览器已暂停(用户接管/填表/自动暂停)"}
|
||||
|
||||
def _ask_resp(self, ask_id, detail):
|
||||
return {"ok": False, "ask": True, "ask_id": ask_id,
|
||||
"error": "需审批:%s" % detail}
|
||||
|
||||
def _checkout_verdict(self, url, aria):
|
||||
"""结账检测:URL 信号 + ARIA 快照行拼成的伪节点。
|
||||
|
||||
broker 只有 aria_snapshot() 字符串,没有节点树;每行
|
||||
快照文本作一个节点(name=行文本),与 detect_checkout
|
||||
的节点口径一致(_node_text 拼 role/name/label/type)。
|
||||
模块缺失时返回“非结账页”(降级,见 _load_checkout)。
|
||||
"""
|
||||
if self._checkout_mod is None:
|
||||
return {"is_checkout": False, "confidence": "low",
|
||||
"reasons": [], "form_risk": None,
|
||||
"amount_text": None}
|
||||
nodes = [{"role": "", "name": line, "label": "", "type": ""}
|
||||
for line in (aria or "").splitlines()
|
||||
if line.strip()]
|
||||
verdict = self._checkout_mod.detect_checkout(url, nodes)
|
||||
return {"is_checkout": verdict.is_checkout,
|
||||
"confidence": verdict.confidence,
|
||||
"reasons": list(verdict.reasons),
|
||||
"form_risk": verdict.form_risk,
|
||||
"amount_text": verdict.amount_text}
|
||||
|
||||
def _checkout_ask(self, url, pause):
|
||||
"""结账页 ask:与 P4 的 form_submit 同种(P10 gap §4.1
|
||||
对齐),入队并按需暂停,返回 ask 响应。"""
|
||||
reasons = ",".join(self._checkout.get("reasons") or [])
|
||||
ask_id = self._guard.enqueue_ask(
|
||||
self._ask_queue_dir, "form_submit", url,
|
||||
"结账页命中(%s),购买前须审批" % reasons)
|
||||
if pause:
|
||||
self._paused = True
|
||||
return self._ask_resp(ask_id, "结账页命中,购买前须审批")
|
||||
|
||||
def open(self, url):
|
||||
if self._paused:
|
||||
return self._paused_resp()
|
||||
ok, err = self._guard.validate_url(url)
|
||||
if not ok:
|
||||
return {"ok": False, "error": err}
|
||||
host = self._guard.url_host(url)
|
||||
entries = self._guard.load_blocklist(self._blocklist_path)
|
||||
if self._guard.check_blocklist(host, entries):
|
||||
return {"ok": False, "error": "命中恶意站拦截名单:%s"
|
||||
% host}
|
||||
if self._form_risk.get("level") != "none":
|
||||
ask_id = self._guard.enqueue_ask(
|
||||
self._ask_queue_dir, "form_submit", url,
|
||||
"高风险表单在页(%s),导航须审批"
|
||||
% self._form_risk["level"])
|
||||
return self._ask_resp(ask_id, "高风险表单在页,导航须审批")
|
||||
# round-12 修复轮(项 4,P10):导航前门——上一页已是
|
||||
# 结账页时,下一次导航须先审批(与 _form_risk 门同形)。
|
||||
if (self._checkout.get("is_checkout")
|
||||
and url not in self._checkout_approved):
|
||||
return self._checkout_ask(url, pause=False)
|
||||
try:
|
||||
self._ensure_browser()
|
||||
self._new_page()
|
||||
self._page.goto(url, timeout=NAV_TIMEOUT_MS)
|
||||
self._page.wait_for_load_state("domcontentloaded",
|
||||
timeout=NAV_TIMEOUT_MS)
|
||||
except Exception as e:
|
||||
return {"ok": False,
|
||||
"error": self._guard.sanitize_error(e)}
|
||||
result = self.snapshot()
|
||||
# 导航后门——落点本身是结账页时,返回 ask 并暂停
|
||||
#(P10 gap §4.1:open 导航返回 ask 并暂停)。
|
||||
if (result.get("ok") and self._checkout.get("is_checkout")
|
||||
and url not in self._checkout_approved):
|
||||
return self._checkout_ask(url, pause=True)
|
||||
return result
|
||||
|
||||
def snapshot(self):
|
||||
if self._paused:
|
||||
return self._paused_resp()
|
||||
if self._page is None:
|
||||
return {"ok": False, "error": "还没有打开的页面(先 open)"}
|
||||
try:
|
||||
title = self._page.title()
|
||||
url = self._page.url
|
||||
# round-12 移植 round-11 的 179a86d 纠正:Playwright
|
||||
# 没有 page.accessibility(Puppeteer 的 API),原生
|
||||
# 快照是 Page.aria_snapshot()(字符串);a11y 字段仍
|
||||
# 叫 a11y,值改为 ARIA 快照字符串(截断到 200 节点
|
||||
# 行/64KB)。合入 round-12 时这处遗漏,仍调不存在的
|
||||
# accessibility.snapshot(),真机上 snapshot 必走异常
|
||||
# 分支返回错误。
|
||||
aria = self._page.aria_snapshot()
|
||||
except Exception as e:
|
||||
return {"ok": False,
|
||||
"error": self._guard.sanitize_error(e)}
|
||||
aria, truncated = self._guard.truncate_aria(aria or "")
|
||||
flags = self._guard.detect_injection(aria)
|
||||
self._form_risk = self._guard.detect_form_risk(aria)
|
||||
# round-12 修复轮(项 4,P10):每次快照重算结账 verdict,
|
||||
# 与 _form_risk 同形(快照缓存,open 的两道门读它)。
|
||||
self._checkout = self._checkout_verdict(url, aria)
|
||||
self._current_url = url
|
||||
return {"ok": True, "title": title, "url": url,
|
||||
"a11y": aria, "truncated": truncated,
|
||||
"untrusted_input": True, "injection_flags": flags,
|
||||
"form_risk": self._form_risk,
|
||||
"checkout": self._checkout}
|
||||
|
||||
def screenshot(self):
|
||||
if self._paused:
|
||||
return self._paused_resp()
|
||||
if self._page is None:
|
||||
return {"ok": False, "error": "还没有打开的页面(先 open)"}
|
||||
try:
|
||||
os.makedirs(self._shots_dir, exist_ok=True)
|
||||
path = os.path.join(self._shots_dir, "%s.png"
|
||||
% uuid.uuid4().hex)
|
||||
self._page.screenshot(path=path)
|
||||
except Exception as e:
|
||||
return {"ok": False,
|
||||
"error": self._guard.sanitize_error(e)}
|
||||
return {"ok": True, "path": path}
|
||||
|
||||
def close(self):
|
||||
if self._page is not None:
|
||||
try:
|
||||
self._page.close()
|
||||
except Exception:
|
||||
pass
|
||||
self._page = None
|
||||
self._form_risk = {"level": "none", "fields": []}
|
||||
self._checkout = {"is_checkout": False, "confidence": "low",
|
||||
"reasons": [], "form_risk": None,
|
||||
"amount_text": None}
|
||||
self._current_url = ""
|
||||
return {"ok": True, "closed": True}
|
||||
|
||||
def pause(self):
|
||||
self._paused = True
|
||||
return {"ok": True, "paused": True}
|
||||
|
||||
def ping(self):
|
||||
return {"ok": True, "broker": True}
|
||||
|
||||
def resume(self):
|
||||
self._paused = False
|
||||
return {"ok": True, "paused": False}
|
||||
|
||||
def status(self):
|
||||
return {"ok": True, "paused": self._paused,
|
||||
"url": self._current_url,
|
||||
"form_risk": self._form_risk,
|
||||
"checkout": self._checkout}
|
||||
|
||||
def list_asks(self):
|
||||
return {"ok": True,
|
||||
"asks": self._guard.list_asks(self._ask_queue_dir)}
|
||||
|
||||
def resolve_ask(self, ask_id, decision):
|
||||
# 批之前先读回 ask 记录:批后它不再是 pending,
|
||||
# list_asks 读不到——结账 ask 批过后要把 URL 记进
|
||||
# _checkout_approved(同页再 open 不再重复 ask)。
|
||||
record = None
|
||||
for ask in self._guard.list_asks(self._ask_queue_dir):
|
||||
if ask.get("ask_id") == ask_id:
|
||||
record = ask
|
||||
break
|
||||
ok, err = self._guard.resolve_ask(
|
||||
self._ask_queue_dir, ask_id, decision)
|
||||
if not ok:
|
||||
return {"ok": False, "error": err}
|
||||
if decision == "allowed":
|
||||
self._paused = False
|
||||
# 批准即放行这次操作:清掉快照缓存的表单风险(下一次
|
||||
# snapshot 会按新页面重算,不会丢检测)。
|
||||
self._form_risk = {"level": "none", "fields": []}
|
||||
# round-12 修复轮(项 4,P10):结账 ask 批过后,
|
||||
# 同 URL 再 open 不重复 ask;_checkout 本身也清掉,
|
||||
# 下一次 snapshot 按新页面重算。
|
||||
if record is not None and str(
|
||||
record.get("detail", "")).startswith("结账页命中"):
|
||||
self._checkout_approved.add(record.get("url"))
|
||||
self._checkout = {"is_checkout": False,
|
||||
"confidence": "low", "reasons": [],
|
||||
"form_risk": None, "amount_text": None}
|
||||
else:
|
||||
self.close()
|
||||
self._paused = False
|
||||
return {"ok": True, "ask_id": ask_id,
|
||||
"decision": decision}
|
||||
|
||||
def shutdown(self):
|
||||
try:
|
||||
if self._browser is not None:
|
||||
self._browser.close()
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
if self._pw is not None:
|
||||
self._pw.__exit__(None, None, None)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def handle_request(broker, req):
|
||||
"""处理一个请求 dict,返回响应 dict(纯逻辑,可单测)。
|
||||
未知操作与禁用操作一律拒绝,不转发、不执行。"""
|
||||
guard = _load_guard()
|
||||
if not isinstance(req, dict):
|
||||
return {"ok": False, "error": "请求须是 JSON 对象"}
|
||||
op = req.get("op")
|
||||
if op in guard.FORBIDDEN_OPS:
|
||||
return {"ok": False,
|
||||
"error": "操作 %r 不暴露(无 JS/无原始 DOM/无交互)"
|
||||
% op}
|
||||
if op == "ping":
|
||||
return {"ok": True, "broker": True}
|
||||
if op == "open":
|
||||
return broker.open(req.get("url"))
|
||||
if op == "snapshot":
|
||||
return broker.snapshot()
|
||||
if op == "screenshot":
|
||||
return broker.screenshot()
|
||||
if op == "close":
|
||||
return broker.close()
|
||||
if op == "pause":
|
||||
return broker.pause()
|
||||
if op == "resume":
|
||||
return broker.resume()
|
||||
if op == "status":
|
||||
return broker.status()
|
||||
if op == "list_asks":
|
||||
return broker.list_asks()
|
||||
if op == "resolve_ask":
|
||||
return broker.resolve_ask(req.get("ask_id"),
|
||||
req.get("decision"))
|
||||
return {"ok": False,
|
||||
"error": "未知操作:%r(白名单:open/snapshot/screenshot/"
|
||||
"close/pause/resume/status/list_asks/resolve_ask/"
|
||||
"ping)" % (op,)}
|
||||
|
||||
|
||||
def _handle_conn(broker, conn):
|
||||
"""一个连接一请求一响应;回包先过泄露检查,不过即换通用错。"""
|
||||
guard = _load_guard()
|
||||
conn.settimeout(120)
|
||||
data = b""
|
||||
while len(data) < MAX_REQUEST_BYTES:
|
||||
chunk = conn.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
data += chunk
|
||||
if data.endswith(b"\n"):
|
||||
break
|
||||
try:
|
||||
req = json.loads(data.decode())
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
resp = {"ok": False, "error": "请求不是合法 JSON"}
|
||||
else:
|
||||
try:
|
||||
resp = handle_request(broker, req)
|
||||
except Exception as e:
|
||||
resp = {"ok": False,
|
||||
"error": guard.sanitize_error(e)}
|
||||
if guard.response_leaks(resp):
|
||||
resp = {"ok": False, "error": "响应泄露检查未通过(已拦截)"}
|
||||
conn.sendall((json.dumps(resp, ensure_ascii=False) + "\n").encode())
|
||||
|
||||
|
||||
def serve(sock_path=CDP_SOCK, shots_dir=SHOTS_DIR,
|
||||
blocklist_path=BLOCKLIST_PATH, ask_queue_dir=ASK_QUEUE_DIR,
|
||||
playwright_factory=None):
|
||||
"""监听 Unix socket,逐连接处理。socket 0600。建不起来即抛错,
|
||||
不要静默带病跑。playwright_factory 只供测试注入假实现。"""
|
||||
if os.path.exists(sock_path):
|
||||
os.remove(sock_path)
|
||||
os.makedirs(os.path.dirname(sock_path), exist_ok=True)
|
||||
srv = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
srv.bind(sock_path)
|
||||
srv.listen(16)
|
||||
os.chmod(sock_path, 0o600)
|
||||
print("chv-cdp-broker: 监听 %s" % sock_path,
|
||||
file=sys.stderr, flush=True)
|
||||
broker = CdpBroker(playwright_factory=playwright_factory,
|
||||
shots_dir=shots_dir,
|
||||
blocklist_path=blocklist_path,
|
||||
ask_queue_dir=ask_queue_dir)
|
||||
try:
|
||||
while True:
|
||||
try:
|
||||
conn, _ = srv.accept()
|
||||
except OSError:
|
||||
continue
|
||||
try:
|
||||
_handle_conn(broker, conn)
|
||||
except OSError as e:
|
||||
print("chv-cdp-broker: 连接处理异常:%s"
|
||||
% _load_guard().sanitize_error(e),
|
||||
file=sys.stderr, flush=True)
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
broker.shutdown()
|
||||
|
||||
|
||||
def main(argv):
|
||||
sock_path = CDP_SOCK
|
||||
shots_dir = SHOTS_DIR
|
||||
blocklist_path = BLOCKLIST_PATH
|
||||
ask_queue_dir = ASK_QUEUE_DIR
|
||||
i = 0
|
||||
while i < len(argv):
|
||||
if argv[i] == "--socket" and i + 1 < len(argv):
|
||||
sock_path = argv[i + 1]
|
||||
i += 2
|
||||
elif argv[i] == "--shots-dir" and i + 1 < len(argv):
|
||||
shots_dir = argv[i + 1]
|
||||
i += 2
|
||||
elif argv[i] == "--blocklist" and i + 1 < len(argv):
|
||||
blocklist_path = argv[i + 1]
|
||||
i += 2
|
||||
elif argv[i] == "--ask-queue" and i + 1 < len(argv):
|
||||
ask_queue_dir = argv[i + 1]
|
||||
i += 2
|
||||
else:
|
||||
print("chv-cdp-broker: 未知参数:%s" % argv[i],
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
serve(sock_path, shots_dir, blocklist_path, ask_queue_dir)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -1,844 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chv-cell:客机内 systemd-nspawn 运行时单元(Hatch 式)启动器。
|
||||
|
||||
对齐 Muse P2(见 docs/chv-muse-p2-gap.md):codex 跑在单元内,不再
|
||||
直接跑在客机用户态。单元是单命令容器(--as-pid2,不 boot 第二个
|
||||
systemd),单元即沙箱——单元内 codex 用 danger-full-access,不再
|
||||
套 bwrap(bwrap 的 --proc 挂载在嵌套命名空间里走不通,见 AGENTS.md
|
||||
内层 proc 定位)。
|
||||
|
||||
关键映射(与 Muse“单元 root 映射为 host 侧非特权用户”同款):
|
||||
--private-users=1000:65536 —— 单元内 uid 0 → 客机侧 chv(uid 1000)。
|
||||
效果:workspace / ~/.codex(chv 所有)单元内仍可写;host-root 0600
|
||||
文件(如 /etc/chv/guest-env)在单元内显示为 overflow uid、不可读;
|
||||
再加 --inaccessible 遮蔽作纵深防御(只遮 /usr 下路径:--volatile=yes
|
||||
下容器根是基本空的 tmpfs,/etc/chv 等在容器根里根本不存在,按构造
|
||||
即不可达,列出反而让 nspawn 解析失败而起不来,见 INACCESSIBLE_BASE)。
|
||||
|
||||
网络:--network-veth。每次 run_cell 在锁下分配唯一身份
|
||||
(CellNet:machine chv-cell-<xx>、宿主 ve-chv-cell-<xx>、
|
||||
10.200.<xx>.0/24,宿主 .1、单元 .2),不再共用固定名与固定
|
||||
子网——固定名是并发 cell 撞毁的根因(part 4)。客机侧 NAT
|
||||
出站(MASQUERADE 走客机默认路由的上行口,动态发现)。单元内
|
||||
配网经 nsenter 进单元 init 的 netns 做。veth 建不起来即大声
|
||||
失败,不静默回落共享网络;CHV_CELL_NETWORK=shared 是显式调试
|
||||
逃生口,须显式设置并打日志,绝不默认。
|
||||
起步门(part 4):payload 在 Popen 的瞬间就开始跑,与配网全程
|
||||
并发是启动竞态的根因——快 payload 在配网完成前退出,nspawn 随
|
||||
之退出,包装器再去配网即撞墙。run_cell 把 payload 包一层
|
||||
sh 起步门(等 /run/chv/.cell-gate-<machine> 出现再 exec),
|
||||
配网完成后才开门;find_cell_init_pid 改成带超时轮询;nspawn
|
||||
已死时返回它的退出码而不是抛错。
|
||||
volatile 空根(part 4):--volatile=yes 下单元根是基本空的
|
||||
tmpfs,/app、/etc/resolv.conf、/etc/hosts、/etc/ssl/certs 都须
|
||||
显式绑进去(/app 与 CA 只读绑;resolv/hosts 是 run_cell 按客机
|
||||
有效 DNS 生成的单元专属文件,只读绑;不绑 nsswitch.conf,
|
||||
glibc 默认 files+dns 正是单元需要的)。part 6(RETEST5)纠正
|
||||
part 5 的设想:/app/work 以可写 bind 盖住只读 /app 的做法在真机
|
||||
上不成立——后挂的 bind 只有在挂载点已存在时才能盖住,挂载点
|
||||
不存在时 nspawn 在只读 /app 下连建都建不起来(全量 73 次同一
|
||||
错误,所有 cell 启动失败)。worktree 根改移出 /app:单元内
|
||||
make_worktree 改用 workspace 绑定下已可写的
|
||||
/home/chv/workspace/.chv-worktrees(经 CHV_WORKTREE_ROOT 环境
|
||||
变量传进单元),不再发任何指向 /app 之下的 bind。
|
||||
|
||||
环境透传:nspawn 不透传调用方环境,本脚本只转发白名单变量(模型
|
||||
网关与代理变量)。GITEA_TOKEN 等凭证变量不在白名单里,绝不进单元。
|
||||
|
||||
用法(客机侧 root 运行,执行器经 SSH 调):
|
||||
chv-cell run [--network veth|shared] -- <cmd...>
|
||||
# 例:chv-cell run -- python3 /app/host.py --state
|
||||
# /home/chv/.codex/host-state.json --codex-home /home/chv/.codex
|
||||
# --sandbox danger-full-access dispatch --name t --task ...
|
||||
chv-cell argv [--network veth|shared] -- <cmd...>
|
||||
# 只打印 nspawn 参数表(JSON),不启动
|
||||
chv-cell network-commands [--uplink IFACE] [--cell-pid PID]
|
||||
# 只打印配网命令表(JSON),不执行
|
||||
只用标准库。单元启动失败、配网失败一律非零退出,不静默降级。
|
||||
"""
|
||||
import fcntl
|
||||
import glob
|
||||
import json
|
||||
import os
|
||||
import pwd
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from typing import NamedTuple
|
||||
|
||||
MACHINE = "chv-cell"
|
||||
# 单元映射:单元内 uid 0 → 客机侧 chv(uid 1000,非特权用户)。
|
||||
UID_BASE = 1000
|
||||
UID_COUNT = 65536
|
||||
CELL_USER = "chv"
|
||||
CELL_UID = 1000
|
||||
CELL_GID = 1000
|
||||
WORKSPACE = "/home/chv/workspace"
|
||||
CODEX_HOME = "/home/chv/.codex"
|
||||
BROKER_DIR = "/run/chv"
|
||||
# part 6(RETEST5 新发现):单元内 worktree 根(RETEST4 新发现 2
|
||||
# 的旧址是 HERE/work,即单元内 /app/work——/app 是 --bind-ro,
|
||||
# 带 repo 的默认 dispatch 撞 OSError Errno 30;part 5 把该目录以
|
||||
# 可写 bind 挂到 /app/work 上,真机证明不可行:挂载点不存在时
|
||||
# nspawn 在只读 /app 下连建都建不起来,所有 cell 启动失败)。
|
||||
# 新址是 workspace 绑定下的 .chv-worktrees:--bind=/home/chv/
|
||||
# workspace 已把整个 workspace 以可写方式挂进单元(目的路径与
|
||||
# 源相同、挂载点即 workspace 自身,挂载时已存在),worktree 根
|
||||
# 落在这里无需任何额外 bind;backing 仍在客机 workspace 下、
|
||||
# 随客机磁盘持久化(跨 cell 的 run/merge 续跑仍找得到 worktree),
|
||||
# 隔离与落盘语义不变,只是单元内路径从 /app/work/host-worktrees
|
||||
# 改为 /home/chv/workspace/.chv-worktrees/host-worktrees。
|
||||
# 该值经 --setenv=CHV_WORKTREE_ROOT 传进单元,host.py 的
|
||||
# worktree_base() 读它(与 host.py 的 CELL_WORKTREE_ROOT 同值,
|
||||
# 两处须保持一致)。
|
||||
WORKTREE_DIR = "/home/chv/workspace/.chv-worktrees"
|
||||
CELL_PID_FILE = "/run/chv/cell.pid"
|
||||
|
||||
# 单元网络(与客机 tap 网 192.168.10.0/24 不重叠的独立 /24)。
|
||||
# part 4(2026-11-07):固定名/固定子网是并发 cell 撞毁的根因——
|
||||
# 每次 run_cell 都在锁下分配唯一身份(CellNet),不再共用常量。
|
||||
# machine 名须短:nspawn 把宿主侧 veth 叫 ve-<machine>,接口名
|
||||
# 上限 15 字符;"chv-cell-%02x" 为 11 字符,ve- 前缀后 14 字符。
|
||||
CELL_NET_BASE = "10.200"
|
||||
CELL_NET = "10.200.0.0/24"
|
||||
HOST_IP = "10.200.0.1"
|
||||
CELL_IP = "10.200.0.2"
|
||||
HOST_IFACE = "ve-chv-cell"
|
||||
CELL_IFACE = "host0"
|
||||
MAX_CELL_INDEX = 250
|
||||
ALLOC_LOCK_FILE = "/run/chv/.cell-alloc.lock"
|
||||
|
||||
# Muse 点名要丢的能力 + 同批丢的危险能力(nspawn 默认集已有限,
|
||||
# 这里是显式再丢,--drop-capability 逐项写,测试逐项断言)。
|
||||
DROP_CAPABILITIES = [
|
||||
"CAP_SYS_ADMIN",
|
||||
"CAP_SYS_PTRACE",
|
||||
"CAP_NET_ADMIN",
|
||||
"CAP_NET_RAW",
|
||||
"CAP_SYS_MODULE",
|
||||
"CAP_SYS_RAWIO",
|
||||
"CAP_SYS_BOOT",
|
||||
"CAP_MKNOD",
|
||||
]
|
||||
|
||||
# syscall 过滤:@system-service 集合再扣掉 io_uring 三件套
|
||||
#(Muse 点名禁用 io_uring)。
|
||||
SYSCALL_FILTER = ("@system-service ~io_uring_setup "
|
||||
"~io_uring_enter ~io_uring_register")
|
||||
|
||||
# 单元内遮掉的路径:浏览器(单元内跑不起自己的浏览器,
|
||||
# 只能走 broker 的 chv-browser)。只列 /usr 下的路径——
|
||||
# round-11 复测纠正(第二轮):--volatile=yes 下容器根是
|
||||
# 基本空的 tmpfs(只挂进只读 /usr),nspawn 按容器根解析
|
||||
# --inaccessible 路径,不存在的路径即启动失败;/etc/chv 与
|
||||
# /home/chv/.git-credentials 在容器根里根本不存在(无 /home、
|
||||
# 无 /etc/chv),凭证文件按构造就不可达,不再列出遮蔽。
|
||||
# 旧的两项若列回,cell 会报 Failed to resolve 而起不来。
|
||||
INACCESSIBLE_BASE = [
|
||||
"/usr/local/share/ms-playwright",
|
||||
]
|
||||
|
||||
# 只转发白名单环境变量进单元。凭证类变量(GITEA_TOKEN 等)
|
||||
# 不在白名单里,绝不进单元。
|
||||
FORWARD_ENV_ALLOWLIST = (
|
||||
"SUB2API_BASE_URL",
|
||||
"SUB2API_API_KEY",
|
||||
"XCM_API_KEY",
|
||||
"HTTP_PROXY",
|
||||
"HTTPS_PROXY",
|
||||
"NO_PROXY",
|
||||
"http_proxy",
|
||||
"https_proxy",
|
||||
"no_proxy",
|
||||
)
|
||||
|
||||
|
||||
class CellNet(NamedTuple):
|
||||
"""一次 cell 运行的唯一网络身份(锁下分配,见 allocate_cell_net)。
|
||||
machine 同时决定宿主侧 veth 名(ve-<machine>,nspawn 约定)与
|
||||
/run/chv 下的专属文件后缀(gate/resolv/hosts/pid)。"""
|
||||
index: int
|
||||
machine: str
|
||||
host_iface: str
|
||||
host_ip: str
|
||||
cell_ip: str
|
||||
subnet: str
|
||||
|
||||
|
||||
def legacy_cell_net():
|
||||
"""无身份时的旧常量(argv/network-commands 子命令与旧测试用;
|
||||
run_cell 一律走 allocate_cell_net,不用它)。"""
|
||||
return CellNet(index=0, machine=MACHINE, host_iface=HOST_IFACE,
|
||||
host_ip=HOST_IP, cell_ip=CELL_IP, subnet=CELL_NET)
|
||||
|
||||
# Muse 点名要丢的能力 + 同批丢的危险能力(nspawn 默认集已有限,
|
||||
# 这里是显式再丢,--drop-capability 逐项写,测试逐项断言)。
|
||||
DROP_CAPABILITIES = [
|
||||
"CAP_SYS_ADMIN",
|
||||
"CAP_SYS_PTRACE",
|
||||
"CAP_NET_ADMIN",
|
||||
"CAP_NET_RAW",
|
||||
"CAP_SYS_MODULE",
|
||||
"CAP_SYS_RAWIO",
|
||||
"CAP_SYS_BOOT",
|
||||
"CAP_MKNOD",
|
||||
]
|
||||
|
||||
# syscall 过滤:@system-service 集合再扣掉 io_uring 三件套
|
||||
#(Muse 点名禁用 io_uring)。
|
||||
SYSCALL_FILTER = ("@system-service ~io_uring_setup "
|
||||
"~io_uring_enter ~io_uring_register")
|
||||
|
||||
# 单元内遮掉的路径:浏览器(单元内跑不起自己的浏览器,
|
||||
# 只能走 broker 的 chv-browser)。只列 /usr 下的路径——
|
||||
# round-11 复测纠正(第二轮):--volatile=yes 下容器根是
|
||||
# 基本空的 tmpfs(只挂进只读 /usr),nspawn 按容器根解析
|
||||
# --inaccessible 路径,不存在的路径即启动失败;/etc/chv 与
|
||||
# /home/chv/.git-credentials 在容器根里根本不存在(无 /home、
|
||||
# 无 /etc/chv),凭证文件按构造就不可达,不再列出遮蔽。
|
||||
# 旧的两项若列回,cell 会报 Failed to resolve 而起不来。
|
||||
INACCESSIBLE_BASE = [
|
||||
"/usr/local/share/ms-playwright",
|
||||
]
|
||||
|
||||
# 只转发白名单环境变量进单元。凭证类变量(GITEA_TOKEN 等)
|
||||
# 不在白名单里,绝不进单元。
|
||||
FORWARD_ENV_ALLOWLIST = (
|
||||
"SUB2API_BASE_URL",
|
||||
"SUB2API_API_KEY",
|
||||
"XCM_API_KEY",
|
||||
"HTTP_PROXY",
|
||||
"HTTPS_PROXY",
|
||||
"NO_PROXY",
|
||||
"http_proxy",
|
||||
"https_proxy",
|
||||
"no_proxy",
|
||||
)
|
||||
|
||||
|
||||
def _playwright_package_dir():
|
||||
"""playwright pip 包目录(遮蔽用)。算不出时回落到 noble 默认路径。"""
|
||||
try:
|
||||
import sysconfig
|
||||
purelib = sysconfig.get_path("purelib")
|
||||
if purelib:
|
||||
return os.path.join(purelib, "playwright")
|
||||
except Exception:
|
||||
pass
|
||||
return "/usr/local/lib/python3.12/dist-packages/playwright"
|
||||
|
||||
|
||||
def build_nspawn_argv(command, network="veth",
|
||||
workspace=WORKSPACE, codex_home=CODEX_HOME,
|
||||
broker_dir=BROKER_DIR,
|
||||
playwright_package_dir=None,
|
||||
cell_net=None, extra_ro_binds=None):
|
||||
"""产 nspawn 参数表(纯函数,可单测)。command 是单元内要跑的命令
|
||||
(list,如 ["python3", "/app/host.py", ...])。
|
||||
cell_net 为 None 时用旧常量身份(argv 子命令与旧测试);
|
||||
run_cell 一律传 allocate_cell_net() 的唯一身份。
|
||||
extra_ro_binds 是 [(src, dest)],只读绑进单元(如单元专属
|
||||
resolv.conf/hosts;/app 与 /etc/ssl/certs 是固定只读绑定,
|
||||
直接列在下面)。worktree 根不经 bind:WORKTREE_DIR 在已可写
|
||||
绑进单元的 workspace 之下(part 6,RETEST5),经
|
||||
--setenv=CHV_WORKTREE_ROOT 传进单元供 host.py 的
|
||||
worktree_base() 用;argv 里绝不出现指向 /app 之下(只读父
|
||||
目录)的 bind——挂载点不存在时 nspawn 在只读父目录下建不
|
||||
了挂载点,单元直接起不来。"""
|
||||
if not command:
|
||||
raise ValueError("chv-cell: 单元内命令不能为空")
|
||||
if network not in ("veth", "shared"):
|
||||
raise ValueError("chv-cell: network 只能是 veth|shared:%r"
|
||||
% network)
|
||||
if playwright_package_dir is None:
|
||||
playwright_package_dir = _playwright_package_dir()
|
||||
machine = cell_net.machine if cell_net is not None else MACHINE
|
||||
argv = [
|
||||
"systemd-nspawn", "--quiet",
|
||||
"--machine=%s" % machine,
|
||||
"--directory=/",
|
||||
# round-11 真机修复:nspawn 255 拒裸 --directory=/(须
|
||||
# --ephemeral 或 --volatile),也拒裸 --no-new-privileges
|
||||
#(把它当作带必填参数的选项,吞掉下一个参数后报错)。
|
||||
# 2026-11-07 复测纠正:首轮修用的 --ephemeral 在客内
|
||||
# ext4 根盘(2.4G、已用 1.8G、可用 593M)上做整盘快照
|
||||
# 拷贝,空间不够即 ENOSPC,全部 cell 调用失败。改
|
||||
# --volatile=yes:根为基本空的 tmpfs、/usr 只读挂进,
|
||||
# 单元内除 bind 出去的目录外一律可写、用完即弃——与
|
||||
# --ephemeral 的设计意图一致,但不拷盘、不占空间。
|
||||
# --volatile=state 不行:整树只读、仅 /var 为 tmpfs,
|
||||
# codex 写 $HOME/.cache 等非 bind 路径会失败。
|
||||
"--volatile=yes",
|
||||
"--as-pid2",
|
||||
"--private-users=%d:%d" % (UID_BASE, UID_COUNT),
|
||||
"--no-new-privileges=yes",
|
||||
"--system-call-filter=%s" % SYSCALL_FILTER,
|
||||
]
|
||||
for cap in DROP_CAPABILITIES:
|
||||
argv.append("--drop-capability=%s" % cap)
|
||||
for path in INACCESSIBLE_BASE + [playwright_package_dir]:
|
||||
argv.append("--inaccessible=%s" % path)
|
||||
argv += [
|
||||
"--bind=%s" % workspace,
|
||||
"--bind=%s" % codex_home,
|
||||
"--bind=%s" % broker_dir,
|
||||
# part 4(volatile 空根缺口):--volatile=yes 下单元根是
|
||||
# 基本空的 tmpfs,执行器默认在单元内跑 python3 /app/host.py、
|
||||
# codex 要做 DNS 与 TLS——/app、CA bundle、resolv.conf、
|
||||
# hosts 都须显式绑进去,否则单元内一律 ENOENT/无 DNS。
|
||||
# /app 与 /etc/ssl/certs 是镜像/系统固定路径,只读绑;
|
||||
# resolv.conf/hosts 是 run_cell 按客机有效 DNS 生成的
|
||||
# 单元专属文件,经 extra_ro_binds 传进来(见
|
||||
# prepare_cell_files:客机 /etc/resolv.conf 可能是
|
||||
# systemd-resolved stub(127.0.0.53),单元内无 resolved,
|
||||
# 直接绑 stub 即 DNS 全灭,故须生成真实服务器列表)。
|
||||
# 不绑 /etc/nsswitch.conf:客机版可能含 resolve 模块,
|
||||
# 单元内无 resolved/dbus,绑了反而可能挂起;glibc 缺
|
||||
# nsswitch.conf 时默认 files+dns,正是单元需要的。
|
||||
"--bind-ro=/app",
|
||||
# part 6(RETEST5):此处绝不再加 --bind=...:/app/work。
|
||||
# part 5 的该绑定已证不可行:/app/work 挂载点在 --bind-ro
|
||||
# 的 /app 镜像里不存在,nspawn 在只读父目录下建不了挂
|
||||
# 载点(EROFS),单元启动即失败。worktree 根已移出 /app,
|
||||
# 见 WORKTREE_DIR 与 --setenv=CHV_WORKTREE_ROOT。
|
||||
"--bind-ro=/etc/ssl/certs",
|
||||
"--tmpfs=/tmp",
|
||||
]
|
||||
for src, dest in (extra_ro_binds or []):
|
||||
argv.append("--bind-ro=%s:%s" % (src, dest))
|
||||
if network == "veth":
|
||||
argv.append("--network-veth")
|
||||
argv += [
|
||||
"--setenv=CHV_CELL=1",
|
||||
# part 6:单元内 worktree 根(host.py worktree_base() 读
|
||||
# 它;值与 WORKTREE_DIR 一致,客机侧与单元内是同一路径,
|
||||
# 因 workspace 以同名可写 bind 进单元)。
|
||||
"--setenv=CHV_WORKTREE_ROOT=%s" % WORKTREE_DIR,
|
||||
"--setenv=HOME=/home/chv",
|
||||
]
|
||||
for name in FORWARD_ENV_ALLOWLIST:
|
||||
value = os.environ.get(name)
|
||||
if value:
|
||||
argv.append("--setenv=%s=%s" % (name, value))
|
||||
argv += ["--chdir=%s" % workspace]
|
||||
argv += list(command)
|
||||
return argv
|
||||
|
||||
|
||||
def parse_default_route_iface(ip_route_output):
|
||||
"""从 `ip route show default` 的输出里解析上行口名。
|
||||
无默认路由时返回 None(调用方大声失败)。"""
|
||||
for line in (ip_route_output or "").splitlines():
|
||||
line = line.strip()
|
||||
if not line.startswith("default "):
|
||||
continue
|
||||
m = re.search(r"\bdev\s+(\S+)", line)
|
||||
if m:
|
||||
return m.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def find_uplink_iface(run=None):
|
||||
"""发现客机上行口(默认路由的 dev)。run 供单测注入。"""
|
||||
run = run or subprocess.run
|
||||
proc = run(["ip", "route", "show", "default"],
|
||||
capture_output=True, text=True)
|
||||
if proc.returncode != 0:
|
||||
return None
|
||||
return parse_default_route_iface(proc.stdout)
|
||||
|
||||
|
||||
def host_network_commands(uplink_iface, net=None):
|
||||
"""客机侧配网命令表(纯函数):ve 接口起 IP + 转发 + NAT。
|
||||
调用方按顺序执行,任一失败即停。net 为 None 时用旧常量
|
||||
(argv/network-commands 子命令与旧测试);run_cell 传
|
||||
allocate_cell_net() 的唯一身份。"""
|
||||
if not uplink_iface:
|
||||
raise ValueError("chv-cell: 上行口名为空,配不出 NAT")
|
||||
net = net or legacy_cell_net()
|
||||
return [
|
||||
["ip", "addr", "add", "%s/24" % net.host_ip, "dev",
|
||||
net.host_iface],
|
||||
["ip", "link", "set", net.host_iface, "up"],
|
||||
["sysctl", "-w", "net.ipv4.ip_forward=1"],
|
||||
["iptables", "-t", "nat", "-A", "POSTROUTING",
|
||||
"-s", net.subnet, "-o", uplink_iface, "-j", "MASQUERADE"],
|
||||
["iptables", "-A", "FORWARD", "-i", net.host_iface,
|
||||
"-o", uplink_iface, "-j", "ACCEPT"],
|
||||
["iptables", "-A", "FORWARD", "-i", uplink_iface,
|
||||
"-o", net.host_iface, "-m", "state", "--state",
|
||||
"RELATED,ESTABLISHED", "-j", "ACCEPT"],
|
||||
]
|
||||
|
||||
|
||||
def host_network_cleanup_commands(uplink_iface, net=None):
|
||||
"""配网清理命令表(与 host_network_commands 配对,-A 改 -D)。
|
||||
清理是 best-effort:删不掉(规则已不在)不报错。"""
|
||||
if not uplink_iface:
|
||||
raise ValueError("chv-cell: 上行口名为空,清不了 NAT")
|
||||
cmds = []
|
||||
for cmd in host_network_commands(uplink_iface, net):
|
||||
if "iptables" not in cmd:
|
||||
continue
|
||||
cleaned = [c if c != "-A" else "-D" for c in cmd]
|
||||
cmds.append(cleaned)
|
||||
return cmds
|
||||
|
||||
|
||||
def find_cell_init_pid(nspawn_pid, proc_root="/proc", timeout=0.0,
|
||||
interval=0.1, proc_alive=None):
|
||||
"""找单元 init 的 pid:nspawn 进程的第一个子进程
|
||||
(--as-pid2 时它是 stub init,payload 是 PID 2;两者同 netns)。
|
||||
timeout=0(默认)即只查一次(旧行为);timeout>0 时带超时轮询
|
||||
(part 4 竞态修复:init 可能还没 fork,单次查找即误判)。
|
||||
proc_alive 供调用方传“nspawn 是否还活着”(如 proc.poll() is
|
||||
None 的 lambda):nspawn 已退出即立刻回 None,不再空等超时。
|
||||
找不到返回 None。"""
|
||||
deadline = time.time() + timeout
|
||||
while True:
|
||||
found = None
|
||||
try:
|
||||
task_dir = os.path.join(proc_root, str(nspawn_pid), "task")
|
||||
for tid in os.listdir(task_dir):
|
||||
children_path = os.path.join(task_dir, tid, "children")
|
||||
try:
|
||||
with open(children_path) as fh:
|
||||
children = fh.read().split()
|
||||
except OSError:
|
||||
continue
|
||||
if children:
|
||||
try:
|
||||
found = int(children[0])
|
||||
except ValueError:
|
||||
continue
|
||||
break
|
||||
except OSError:
|
||||
# 任务目录还没出现(init 未 fork)即当作“这次没找到”,
|
||||
# 继续轮询,不直接回 None(旧代码这里直接 return None,
|
||||
# 正是单次查找误判的根因之一)。
|
||||
found = None
|
||||
if found is not None:
|
||||
return found
|
||||
if proc_alive is not None and not proc_alive():
|
||||
return None
|
||||
if time.time() >= deadline:
|
||||
return None
|
||||
time.sleep(min(interval, max(0.0, deadline - time.time())))
|
||||
|
||||
|
||||
def cell_network_commands(cell_pid, net=None):
|
||||
"""单元内配网命令表(纯函数):nsenter 进单元 init 的 netns,
|
||||
给 host0 配 IP、起接口、加默认路由。net 为 None 时用旧常量。"""
|
||||
if not cell_pid:
|
||||
raise ValueError("chv-cell: 单元 init pid 为空,配不了单元内网络")
|
||||
net = net or legacy_cell_net()
|
||||
prefix = ["nsenter", "-t", str(cell_pid), "-n"]
|
||||
return [
|
||||
prefix + ["ip", "addr", "add", "%s/24" % net.cell_ip,
|
||||
"dev", CELL_IFACE],
|
||||
prefix + ["ip", "link", "set", CELL_IFACE, "up"],
|
||||
prefix + ["ip", "route", "add", "default", "via", net.host_ip],
|
||||
]
|
||||
|
||||
|
||||
def _ensure_bind_source(path, uid=CELL_UID, gid=CELL_GID):
|
||||
"""bind 源目录须存在且属 chv(单元内 root 即客机 chv,属主不对
|
||||
就写不进 workspace)。root 建的目录要 chown 回 chv。"""
|
||||
os.makedirs(path, exist_ok=True)
|
||||
try:
|
||||
st = os.stat(path)
|
||||
if st.st_uid != uid or st.st_gid != gid:
|
||||
os.chown(path, uid, gid)
|
||||
except OSError as e:
|
||||
print("chv-cell: chown %s 失败:%s" % (path, e), file=sys.stderr)
|
||||
raise
|
||||
|
||||
|
||||
def gate_wrapper_command(command, gate_path):
|
||||
"""把单元内命令包一层起步门(纯函数):payload 先在单元内
|
||||
等 gate 文件出现再 exec 原命令。gate 文件由 run_cell 在配网
|
||||
完成后才创建(见 run_cell)——payload 与配网零同步是 RETEST3
|
||||
新缺陷 1 的根因(快 payload 在配网完成前退出,nspawn 随之
|
||||
退出,包装器再去配网即撞墙)。/usr/bin/sh 用绝对路径:
|
||||
volatile 单元根里 /bin 未必有,/usr 是只读挂进来的,必有。
|
||||
注意:$1 是 gate 路径,shift 后 "$@" 即原命令。"""
|
||||
script = ("while [ ! -e \"$1\" ]; do sleep 0.05; done; "
|
||||
"shift; exec \"$@\"")
|
||||
return ["/usr/bin/sh", "-c", script, "chv-cell-gate",
|
||||
gate_path] + list(command)
|
||||
|
||||
|
||||
def _parse_nameservers(path):
|
||||
"""读一个 resolv.conf 风格文件,返回 nameserver 列表
|
||||
(去 127.0.0.0/8 与 ::1——单元内无 resolved,stub 指过去
|
||||
即 DNS 全灭)。文件不存在或读不出即 []。"""
|
||||
servers = []
|
||||
try:
|
||||
with open(path) as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line.startswith("nameserver"):
|
||||
continue
|
||||
parts = line.split()
|
||||
if len(parts) < 2:
|
||||
continue
|
||||
ip = parts[1]
|
||||
if ip.startswith("127.") or ip == "::1":
|
||||
continue
|
||||
if ip not in servers:
|
||||
servers.append(ip)
|
||||
except OSError:
|
||||
return []
|
||||
return servers
|
||||
|
||||
|
||||
def _effective_nameservers(resolv_conf="/etc/resolv.conf",
|
||||
resolved_conf="/run/systemd/resolve/resolv.conf"):
|
||||
"""客机有效 DNS:先读 /etc/resolv.conf;只剩 stub(127.0.0.53)
|
||||
或为空时读 resolved 的 resolv.conf;仍为空即大声失败。"""
|
||||
servers = _parse_nameservers(resolv_conf)
|
||||
if not servers:
|
||||
servers = _parse_nameservers(resolved_conf)
|
||||
if not servers:
|
||||
raise RuntimeError(
|
||||
"chv-cell: 客机无有效 DNS,配不出单元 resolv.conf")
|
||||
return servers
|
||||
|
||||
|
||||
def prepare_cell_files(machine, run_dir=BROKER_DIR,
|
||||
resolv_conf="/etc/resolv.conf",
|
||||
resolved_conf="/run/systemd/resolve/resolv.conf",
|
||||
hosts_file="/etc/hosts"):
|
||||
"""准备单元专属文件(run_cell 在 Popen 之前调),返回 dict:
|
||||
gate(起步门,run_cell 配网完成后才创建——注意 gate 文件本身
|
||||
在这里不创建,只给路径)、resolv(单元专属 resolv.conf,已写
|
||||
好)、hosts(单元专属 hosts,已写好)。文件都在 /run/chv 下
|
||||
(bind 进单元),0644(单元内 root 在客机侧是 chv,0600 的
|
||||
root 文件在单元内读不到;DNS/hosts 不含秘密,0644 安全)。
|
||||
/app 与 /etc/ssl/certs 是固定只读绑定,不走这里。"""
|
||||
gate = os.path.join(run_dir, ".cell-gate-%s" % machine)
|
||||
resolv = os.path.join(run_dir, "resolv.conf.%s" % machine)
|
||||
hosts = os.path.join(run_dir, "hosts.%s" % machine)
|
||||
servers = _effective_nameservers(resolv_conf, resolved_conf)
|
||||
with open(resolv, "w") as fh:
|
||||
for server in servers:
|
||||
fh.write("nameserver %s\n" % server)
|
||||
os.chmod(resolv, 0o644)
|
||||
try:
|
||||
with open(hosts_file) as src:
|
||||
content = src.read()
|
||||
except OSError:
|
||||
content = ""
|
||||
if "localhost" not in content:
|
||||
content = "127.0.0.1 localhost\n" + content
|
||||
with open(hosts, "w") as fh:
|
||||
fh.write(content)
|
||||
os.chmod(hosts, 0o644)
|
||||
return {"gate": gate, "resolv": resolv, "hosts": hosts}
|
||||
|
||||
|
||||
def _wait_for_iface(iface, timeout=10.0):
|
||||
deadline = time.time() + timeout
|
||||
while time.time() < deadline:
|
||||
proc = subprocess.run(["ip", "link", "show", iface],
|
||||
capture_output=True)
|
||||
if proc.returncode == 0:
|
||||
return True
|
||||
time.sleep(0.2)
|
||||
return False
|
||||
|
||||
|
||||
def _alloc_lock(lock_path, timeout=30.0):
|
||||
"""拿分配锁(fcntl 排他锁,阻塞拿不到即超时大声失败)。
|
||||
返回锁 fd(调用方负责 close 即解锁)。"""
|
||||
fd = os.open(lock_path, os.O_CREAT | os.O_RDWR, 0o600)
|
||||
deadline = time.time() + timeout
|
||||
while True:
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
return fd
|
||||
except BlockingIOError:
|
||||
if time.time() >= deadline:
|
||||
os.close(fd)
|
||||
raise RuntimeError(
|
||||
"chv-cell: 分配锁 %s 拿不到(超时 %.0f 秒)"
|
||||
% (lock_path, timeout))
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
def _used_cell_indices(run):
|
||||
"""扫描已占用的 cell 下标:活着的 ve-chv-cell-<xx> 接口名、
|
||||
10.200.<N>.1/24 地址、machinectl 里仍在册的同名 machine
|
||||
(注销滞后时避让,见 RETEST3 新缺陷 1 的 veth 超时分析)。
|
||||
run 供单测注入(subprocess.run 兼容)。machinectl 查不到
|
||||
(无此命令/失败)即跳过该项,不因此失败。"""
|
||||
used = set()
|
||||
proc = run(["ip", "-o", "link", "show"], capture_output=True,
|
||||
text=True)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError("chv-cell: 扫描已占用 cell 下标失败:"
|
||||
"ip -o link show")
|
||||
for m in re.finditer(r"\bve-chv-cell-([0-9a-f]{2})\b",
|
||||
proc.stdout or ""):
|
||||
try:
|
||||
used.add(int(m.group(1), 16))
|
||||
except ValueError:
|
||||
continue
|
||||
proc = run(["ip", "-o", "addr", "show"], capture_output=True,
|
||||
text=True)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError("chv-cell: 扫描已占用 cell 下标失败:"
|
||||
"ip -o addr show")
|
||||
for m in re.finditer(r"\b10\.200\.(\d+)\.1/24\b",
|
||||
proc.stdout or ""):
|
||||
try:
|
||||
used.add(int(m.group(1)))
|
||||
except ValueError:
|
||||
continue
|
||||
try:
|
||||
proc = run(["machinectl", "list", "--no-legend"],
|
||||
capture_output=True, text=True)
|
||||
if proc.returncode == 0:
|
||||
for m in re.finditer(r"\bchv-cell-([0-9a-f]{2})\b",
|
||||
proc.stdout or ""):
|
||||
try:
|
||||
used.add(int(m.group(1), 16))
|
||||
except ValueError:
|
||||
continue
|
||||
except OSError:
|
||||
pass
|
||||
return used
|
||||
|
||||
|
||||
def allocate_cell_net(run=None, lock_path=ALLOC_LOCK_FILE):
|
||||
"""锁下分配唯一 cell 身份(run_cell 每调一次即分配一个)。
|
||||
取最小空闲下标 0..250;machine 名 chv-cell-<xx>(11 字符,
|
||||
ve- 前缀后 14 字符,符合接口名 15 字符上限);子网
|
||||
10.200.<idx>.0/24、宿主 10.200.<idx>.1、单元 10.200.<idx>.2。
|
||||
全部下标占满即大声失败,不静默回落固定身份。"""
|
||||
run = run or subprocess.run
|
||||
fd = _alloc_lock(lock_path)
|
||||
try:
|
||||
used = _used_cell_indices(run)
|
||||
for index in range(MAX_CELL_INDEX + 1):
|
||||
if index not in used:
|
||||
break
|
||||
else:
|
||||
raise RuntimeError(
|
||||
"chv-cell: cell 下标已占满(0..%d)" % MAX_CELL_INDEX)
|
||||
machine = "chv-cell-%02x" % index
|
||||
return CellNet(
|
||||
index=index,
|
||||
machine=machine,
|
||||
host_iface="ve-%s" % machine,
|
||||
host_ip="%s.%d.1" % (CELL_NET_BASE, index),
|
||||
cell_ip="%s.%d.2" % (CELL_NET_BASE, index),
|
||||
subnet="%s.%d.0/24" % (CELL_NET_BASE, index))
|
||||
finally:
|
||||
try:
|
||||
fcntl.flock(fd, fcntl.LOCK_UN)
|
||||
except OSError:
|
||||
pass
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _write_cell_pid_files(machine, cell_pid):
|
||||
"""写 pid 文件:每单元一份 cell-<machine>.pid + 兼容旧路径
|
||||
/run/chv/cell.pid(最新单元;P2-9 驱动与 broker 降级路径
|
||||
仍读它)。"""
|
||||
per_cell = os.path.join(BROKER_DIR, "cell-%s.pid" % machine)
|
||||
with open(per_cell, "w") as fh:
|
||||
fh.write(str(cell_pid))
|
||||
with open(CELL_PID_FILE, "w") as fh:
|
||||
fh.write(str(cell_pid))
|
||||
|
||||
|
||||
def _remove_cell_pid_files(machine, cell_pid):
|
||||
"""清 pid 文件(best-effort):每单元份必删;旧路径只在
|
||||
内容仍指向本单元时才删(并发 cell 时别删别人的指针)。"""
|
||||
for path in (os.path.join(BROKER_DIR, "cell-%s.pid" % machine),):
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
with open(CELL_PID_FILE) as fh:
|
||||
if fh.read().strip() == str(cell_pid):
|
||||
os.remove(CELL_PID_FILE)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _run_cmds(cmds, what):
|
||||
for cmd in cmds:
|
||||
print("chv-cell: + %s" % " ".join(cmd), file=sys.stderr)
|
||||
proc = subprocess.run(cmd)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
"chv-cell: %s 失败(rc=%d):%s"
|
||||
% (what, proc.returncode, " ".join(cmd)))
|
||||
|
||||
|
||||
def run_cell(command, network="veth"):
|
||||
"""起单元、配网、等 payload 退出、清 NAT,返回 payload 退出码。
|
||||
part 4 顺序(RETEST3 新缺陷 1):分配唯一身份 → 准备单元专属
|
||||
文件 → Popen 带起步门的 payload(payload 在门后等着,不与配网
|
||||
并发)→ 等 veth → 客机侧配网 → 轮询找 init pid → 写 pid 文件 →
|
||||
单元内配网 → 开门放行 payload → proc.wait()。payload 已退出
|
||||
(nspawn 起不来等)时返回它的退出码而不是抛错;其余配网失败
|
||||
仍大声抛错,不静默回落共享网络。"""
|
||||
if os.geteuid() != 0:
|
||||
print("chv-cell: 须以 root 运行(nspawn + 配网都要 root)",
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
_ensure_bind_source(WORKSPACE)
|
||||
_ensure_bind_source(CODEX_HOME)
|
||||
_ensure_bind_source(BROKER_DIR, uid=0, gid=0)
|
||||
# part 6:worktree 根目录须在 Popen 前存在,且属 chv(单元内
|
||||
# root 即客机 chv,属主不对就写不进 worktree)。它在 workspace
|
||||
# 绑定之下,进单元后 host.py 的 makedirs 也能建,这里预建只
|
||||
# 是把属主一次定对;不再是任何 bind 的源(无 /app/work 绑定)。
|
||||
_ensure_bind_source(WORKTREE_DIR)
|
||||
net = None
|
||||
files = None
|
||||
uplink = None
|
||||
cell_pid = None
|
||||
proc = None
|
||||
if network == "veth":
|
||||
net = allocate_cell_net()
|
||||
files = prepare_cell_files(net.machine)
|
||||
wrapped = gate_wrapper_command(command, files["gate"])
|
||||
else:
|
||||
wrapped = list(command)
|
||||
argv = build_nspawn_argv(
|
||||
wrapped, network=network, cell_net=net,
|
||||
extra_ro_binds=[(files["resolv"], "/etc/resolv.conf"),
|
||||
(files["hosts"], "/etc/hosts")]
|
||||
if files else None)
|
||||
print("chv-cell: 启动单元:%s" % " ".join(argv), file=sys.stderr)
|
||||
proc = subprocess.Popen(argv)
|
||||
try:
|
||||
if network == "veth":
|
||||
# veth 等待放宽到 20 秒:RETEST3 里连续调用在 10.2/10.3
|
||||
# 秒处超时(疑似 machine 注销滞后);唯一身份 + 起步门
|
||||
# 后配网不再与 payload 赛跑,等久一点无害。
|
||||
if not _wait_for_iface(net.host_iface, timeout=20.0):
|
||||
raise RuntimeError(
|
||||
"chv-cell: veth 接口 %s 未出现,拒绝回落共享网络"
|
||||
% net.host_iface)
|
||||
uplink = find_uplink_iface()
|
||||
if not uplink:
|
||||
raise RuntimeError(
|
||||
"chv-cell: 客机无默认路由,配不出 NAT")
|
||||
_run_cmds(host_network_commands(uplink, net), "客机侧配网")
|
||||
cell_pid = find_cell_init_pid(
|
||||
proc.pid, timeout=10.0,
|
||||
proc_alive=lambda: proc.poll() is None)
|
||||
if not cell_pid:
|
||||
raise RuntimeError("chv-cell: 找不到单元 init pid")
|
||||
_write_cell_pid_files(net.machine, cell_pid)
|
||||
_run_cmds(cell_network_commands(cell_pid, net), "单元内配网")
|
||||
# 配网完成,开门放行 payload。
|
||||
with open(files["gate"], "w") as fh:
|
||||
fh.write("")
|
||||
return proc.wait()
|
||||
except RuntimeError:
|
||||
# nspawn 已死(如 machine 名冲突、argv 错)即返回它的退出
|
||||
# 码,不把“起不来”报成配网失败;还活着才是真配网失败,大声抛。
|
||||
if proc is not None and proc.poll() is not None:
|
||||
return proc.returncode
|
||||
raise
|
||||
finally:
|
||||
if uplink:
|
||||
for cmd in host_network_cleanup_commands(uplink, net):
|
||||
subprocess.run(cmd, capture_output=True)
|
||||
if cell_pid:
|
||||
_remove_cell_pid_files(net.machine, cell_pid)
|
||||
if files:
|
||||
for key in ("gate", "resolv", "hosts"):
|
||||
try:
|
||||
os.remove(files[key])
|
||||
except OSError:
|
||||
pass
|
||||
if proc is not None and proc.poll() is None:
|
||||
proc.terminate()
|
||||
try:
|
||||
proc.wait(timeout=10)
|
||||
except subprocess.TimeoutExpired:
|
||||
proc.kill()
|
||||
|
||||
|
||||
def main(argv):
|
||||
if len(argv) < 1:
|
||||
print("用法:chv-cell run|argv|network-commands …",
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
verb = argv[0]
|
||||
rest = argv[1:]
|
||||
network = os.environ.get("CHV_CELL_NETWORK", "veth")
|
||||
# --network 只在 run/argv 前接受,其它位置一律不认。
|
||||
if rest and rest[0] == "--network":
|
||||
if len(rest) < 2:
|
||||
print("chv-cell: --network 缺值", file=sys.stderr)
|
||||
return 2
|
||||
network = rest[1]
|
||||
rest = rest[2:]
|
||||
if network not in ("veth", "shared"):
|
||||
print("chv-cell: network 只能是 veth|shared:%r" % network,
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
if network == "shared":
|
||||
print("chv-cell: 警告:共享网络模式(调试逃生口),单元与"
|
||||
"客机共用网络命名空间", file=sys.stderr)
|
||||
if verb == "run":
|
||||
if not rest or rest[0] != "--":
|
||||
print("chv-cell: run 用法:chv-cell run -- <cmd...>",
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
command = rest[1:]
|
||||
if not command:
|
||||
print("chv-cell: 单元内命令不能为空", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
return run_cell(command, network=network)
|
||||
except RuntimeError as e:
|
||||
print(str(e), file=sys.stderr)
|
||||
return 1
|
||||
if verb == "argv":
|
||||
if not rest or rest[0] != "--":
|
||||
print("chv-cell: argv 用法:chv-cell argv -- <cmd...>",
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
print(json.dumps(build_nspawn_argv(rest[1:], network=network),
|
||||
ensure_ascii=False))
|
||||
return 0
|
||||
if verb == "network-commands":
|
||||
uplink = None
|
||||
cell_pid = None
|
||||
i = 0
|
||||
while i < len(rest):
|
||||
if rest[i] == "--uplink" and i + 1 < len(rest):
|
||||
uplink = rest[i + 1]
|
||||
i += 2
|
||||
elif rest[i] == "--cell-pid" and i + 1 < len(rest):
|
||||
cell_pid = int(rest[i + 1])
|
||||
i += 2
|
||||
else:
|
||||
print("chv-cell: network-commands 未知参数:%s"
|
||||
% rest[i], file=sys.stderr)
|
||||
return 2
|
||||
out = {}
|
||||
if uplink:
|
||||
out["host"] = host_network_commands(uplink)
|
||||
out["host_cleanup"] = host_network_cleanup_commands(uplink)
|
||||
if cell_pid:
|
||||
out["cell"] = cell_network_commands(cell_pid)
|
||||
print(json.dumps(out, ensure_ascii=False))
|
||||
return 0
|
||||
print("chv-cell: 未知子命令:%s" % verb, file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv[1:]))
|
||||
@@ -1,20 +0,0 @@
|
||||
[Unit]
|
||||
Description=Start chv-gitea-worker once the sync workspace exists (feat-chv-muse-p6)
|
||||
# round-12 修复轮第三轮(缺口 ①):worker 的 ReadWritePaths 含
|
||||
# /home/chv/workspace,systemd 建 mount namespace 时该路径必须已
|
||||
# 存在;工作区由 cloud-final 的 chv-sync boot 克隆出来,晚于
|
||||
# multi-user.target 约 9 秒(R12 复测 2:worker 07:23:31 首启
|
||||
# 226,工作区 07:23:40.40 才出现,5 次启动预算同一秒内耗尽)。
|
||||
# 本单元只在工作区出现时触发服务启动——服务绝不在工作区出现前
|
||||
# 启动,命名空间搭建前即有保证。须与服务的 ConditionPathExists
|
||||
# 配对:Condition 挡住开机那次过早启动(干净跳过、不 226),本
|
||||
# 单元提供 Condition 本身没有的“出现后再试一次”。
|
||||
# 不用 StartLimit 放宽(仍是时序赌博,wake 时间在负载下已翻倍,
|
||||
# 预算仍可能耗尽)与 tmpfiles 补行(/home/chv 本身晚于
|
||||
# tmpfiles-setup,补了也没用),见服务单元 [Unit] 注释。
|
||||
|
||||
[Path]
|
||||
PathExists=/home/chv/workspace
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,397 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chv-gitea-worker:Gitea 同步 privsep worker(feat-chv-muse-p6)。
|
||||
|
||||
对齐 Muse 官方安全文:内置连接器的业务逻辑由 systemd 沙箱化的
|
||||
privsep worker 执行;每个 worker 由 cgroup 标识、有明确凭证白名单。
|
||||
本进程跑在客内、运行时单元之外(root 身份的 systemd 服务,见
|
||||
chv-gitea-worker.service;P2 的 nspawn 单元落地后,codex 在单元内,
|
||||
本进程在单元外——单元内 root 映射为非特权用户,读不到本进程的
|
||||
socket)。它只做 Gitea 同步的 git 业务逻辑:
|
||||
|
||||
- 在 Unix socket /run/chv-gitea/worker.sock 上接受 broker/薄 CLI
|
||||
的类型化请求(SO_PEERCRED 只放行 root 与 chv;socket 用 group
|
||||
权限——目录 0750、socket 0660、组为第一个非 root 放行 uid 的主
|
||||
组,round-12 修复轮第三轮缺口 ②,round-11 authd 同款口径,拒绝
|
||||
0666;chv-sync 薄客户端以 chv 身份连,单元内 agent 同样映射为
|
||||
chv uid 1000,故也只能走类型化协议——操作白名单、仓库
|
||||
confinement、安全扫描三处仍在,worker 不持凭证);
|
||||
- 操作白名单只有 pull/push/status/ping;仓库只限 ~/workspace 下;
|
||||
- git 以 chv 身份跑(runuser -u chv,文件归属不变),argv 里只配
|
||||
credential helper(P3 的 git-credential-chv-authd,见下),绝无
|
||||
token 值;凭证由 git 经 helper 向 authd 按次取,本进程绝不持有
|
||||
token;
|
||||
- git 输出先过 safety 扫描 + 打上来源标记再返回;block 即阻断
|
||||
(内容不往回流),flag 即标记复核。
|
||||
|
||||
只用标准库。cloud/chv/privsep.py 与 cloud/chv/safety.py 是叶子
|
||||
模块(只 import 标准库),镜像构建时随本文件一起拷进
|
||||
/usr/local/lib/chv-gitea/(见 docs/chv-muse-p6-gap.md §2 的接入
|
||||
说明),本文件把该目录加进 sys.path 后 import。
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import socketserver
|
||||
import struct
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
|
||||
DEFAULT_LIB_DIR = "/usr/local/lib/chv-gitea"
|
||||
DEFAULT_SOCKET_PATH = "/run/chv-gitea/worker.sock"
|
||||
DEFAULT_LOG_PATH = "/var/log/chv-gitea-worker.log"
|
||||
WORKSPACE = "/home/chv/workspace"
|
||||
SYNC_USER = "chv"
|
||||
# SO_PEERCRED 放行的 uid:root + chv(chv-sync 以 chv 身份跑薄客户
|
||||
# 端;round-11 authd 同款 DEFAULT_ALLOWED_UIDS)。可用环境变量
|
||||
# CHV_GITEA_WORKER_ALLOWED_UIDS 覆盖(逗号分隔)。
|
||||
DEFAULT_ALLOWED_UIDS = (0, 1000)
|
||||
# P3 的 credential helper(git 经它向 authd 按次取凭证)与 path
|
||||
# 配置(P3 修过的坑:git 默认不把 path 发给 helper)。
|
||||
HELPER_PATH = "/usr/local/bin/git-credential-chv-authd"
|
||||
GIT_TIMEOUT_SECONDS = 300
|
||||
MAX_REQUEST_BYTES = 65536
|
||||
|
||||
|
||||
def _audit(log_path, record):
|
||||
"""审计落盘(JSON 行,0600):只含绑定与 verdict,不含值。
|
||||
落盘失败只记 stderr,绝不炸 worker。"""
|
||||
line = json.dumps(record, sort_keys=True)
|
||||
try:
|
||||
with open(log_path, "a") as fh:
|
||||
fh.write(line + "\n")
|
||||
except OSError as exc:
|
||||
print("chv-gitea-worker: audit 落盘失败:%s" % exc,
|
||||
file=sys.stderr)
|
||||
print("chv-gitea-worker: %s" % line, file=sys.stderr)
|
||||
|
||||
|
||||
def _git_base_argv():
|
||||
"""git 基础 argv:只配 credential helper,不含任何凭证值。
|
||||
helper 缺失时由调用方先拒绝(fail-closed),不回落读 token。"""
|
||||
return ["git", "-c",
|
||||
"credential.helper=%s" % HELPER_PATH,
|
||||
"-c", "credential.useHttpPath=true"]
|
||||
|
||||
|
||||
def _resolve_repo(repo, workspace):
|
||||
"""解析仓目录:显式给则用它(须在 workspace 下,调用方已
|
||||
校验);省略时取 workspace 下唯一的 git 仓库;0 个或多个
|
||||
即 ValueError(由调用方转成类型化失败)。"""
|
||||
if repo:
|
||||
return repo
|
||||
found = []
|
||||
try:
|
||||
names = sorted(os.listdir(workspace))
|
||||
except OSError as exc:
|
||||
raise ValueError("工作区读不到:%s" % exc)
|
||||
for name in names:
|
||||
path = os.path.join(workspace, name)
|
||||
if os.path.isdir(os.path.join(path, ".git")):
|
||||
found.append(path)
|
||||
if len(found) == 1:
|
||||
return found[0]
|
||||
raise ValueError(
|
||||
"workspace 下 git 仓库不是唯一(%d 个),请显式给仓目录"
|
||||
% len(found))
|
||||
|
||||
|
||||
def _run_git(action, repo, run_git=None):
|
||||
"""跑一次 git(pull/push/status),返回 (returncode, stdout,
|
||||
stderr)。run_git 可注入(测试用假 git,不跑真 git)。
|
||||
git 以 chv 身份跑,文件归属不变;argv 是 list,不进 shell。"""
|
||||
if run_git is not None:
|
||||
return run_git(action, repo)
|
||||
argv = ["runuser", "-u", SYNC_USER, "--", "env",
|
||||
"HOME=/home/chv"] + _git_base_argv()
|
||||
if action == "pull":
|
||||
argv += ["-C", repo, "pull", "--ff-only"]
|
||||
elif action == "push":
|
||||
argv += ["-C", repo, "push"]
|
||||
elif action == "status":
|
||||
argv += ["-C", repo, "status", "-sb"]
|
||||
else:
|
||||
return 2, "", "未知 git 操作:%s" % action
|
||||
try:
|
||||
proc = subprocess.run(argv, capture_output=True, text=True,
|
||||
timeout=GIT_TIMEOUT_SECONDS)
|
||||
except subprocess.TimeoutExpired:
|
||||
return 124, "", "git 超时(%d 秒)" % GIT_TIMEOUT_SECONDS
|
||||
except OSError as exc:
|
||||
return 2, "", "git 起不来:%s" % exc
|
||||
return proc.returncode, proc.stdout, proc.stderr
|
||||
|
||||
|
||||
def _truncate(text, limit=8192):
|
||||
if text is None:
|
||||
return ""
|
||||
if len(text) > limit:
|
||||
return text[:limit] + "\n…(已截断)"
|
||||
return text
|
||||
|
||||
|
||||
def _socket_group_gid(allowed_uids, socket_group=None):
|
||||
"""socket 共享组的 gid:socket_group 显式给时优先(int 即 gid,
|
||||
str 即组名);否则取第一个非 root 放行 uid 的主组。解析不出
|
||||
(uid 无 passwd 条目、组名不存在)即 None——调用方回落
|
||||
root-only 权限(目录 0700、socket 0600),绝不把 socket 放给
|
||||
全员可读(round-11 authd 同款口径;0666 探针口径不上生产)。"""
|
||||
import grp
|
||||
import pwd
|
||||
if socket_group is not None:
|
||||
if isinstance(socket_group, int):
|
||||
return socket_group
|
||||
try:
|
||||
return grp.getgrnam(socket_group).gr_gid
|
||||
except (KeyError, OverflowError):
|
||||
return None
|
||||
for uid in sorted(allowed_uids or ()):
|
||||
if uid == 0:
|
||||
continue
|
||||
try:
|
||||
return pwd.getpwuid(uid).pw_gid
|
||||
except (KeyError, OverflowError):
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def _set_group_perms(path, gid, mode):
|
||||
"""目录/socket 的组权限:先 chgrp(失败只放过,不中断启动),
|
||||
再 chmod。gid 为 None 时只 chmod(root-only 模式)。"""
|
||||
if gid is not None:
|
||||
try:
|
||||
os.chown(path, -1, gid)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
os.chmod(path, mode)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
class WorkerHandler(socketserver.BaseRequestHandler):
|
||||
"""单次请求处理(每连接一 JSON 行):鉴权 → 校验 → 跑 git →
|
||||
扫描输出 → 回包。任何一步拒绝都回 {ok:false},不含值。"""
|
||||
|
||||
def handle(self):
|
||||
server = self.server
|
||||
try:
|
||||
creds = self.request.getsockopt(
|
||||
socket.SOL_SOCKET, socket.SO_PEERCRED,
|
||||
struct.calcsize("3i"))
|
||||
_pid, uid, _gid = struct.unpack("3i", creds)
|
||||
except (OSError, struct.error):
|
||||
self._reply({"ok": False, "error": "peercred_failed",
|
||||
"stdout": "", "stderr": "",
|
||||
"returncode": 2, "safety": {},
|
||||
"request_id": ""})
|
||||
return
|
||||
if uid not in server.allowed_uids:
|
||||
server.audit({"event": "privsep_peer_rejected",
|
||||
"uid": uid})
|
||||
self._reply({"ok": False, "error": "uid_not_allowed",
|
||||
"stdout": "", "stderr": "",
|
||||
"returncode": 2, "safety": {},
|
||||
"request_id": ""})
|
||||
return
|
||||
try:
|
||||
buf = b""
|
||||
while b"\n" not in buf:
|
||||
chunk = self.request.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
buf += chunk
|
||||
if len(buf) > MAX_REQUEST_BYTES:
|
||||
break
|
||||
req = json.loads(buf.decode("utf-8"))
|
||||
except (OSError, ValueError):
|
||||
self._reply({"ok": False, "error": "bad_request",
|
||||
"stdout": "", "stderr": "",
|
||||
"returncode": 2, "safety": {},
|
||||
"request_id": ""})
|
||||
return
|
||||
self._serve(req)
|
||||
|
||||
def _reply(self, obj):
|
||||
try:
|
||||
self.request.sendall(
|
||||
(json.dumps(obj) + "\n").encode("utf-8"))
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _serve(self, req):
|
||||
server = self.server
|
||||
try:
|
||||
from cloud.chv.privsep import (
|
||||
WorkerRequest, WorkerResponse)
|
||||
request = WorkerRequest.from_dict(
|
||||
req, workspace=server.workspace)
|
||||
except Exception as exc:
|
||||
kind = getattr(exc, "kind", "bad_request")
|
||||
server.audit({"event": "privsep_request_refused",
|
||||
"kind": kind})
|
||||
self._reply({"ok": False, "error": kind,
|
||||
"stdout": "", "stderr": "",
|
||||
"returncode": 2, "safety": {},
|
||||
"request_id": ""})
|
||||
return
|
||||
if request.op == "ping":
|
||||
resp = WorkerResponse(True, "", "", 0,
|
||||
{"verdict": "allow", "rules": []},
|
||||
request.request_id)
|
||||
self._reply(resp.to_dict())
|
||||
return
|
||||
if not os.path.exists(server.helper_path):
|
||||
server.audit({"event": "privsep_request_refused",
|
||||
"kind": "credential_helper_missing",
|
||||
"op": request.op})
|
||||
self._reply({"ok": False,
|
||||
"error": "credential_helper_missing",
|
||||
"stdout": "", "stderr": "",
|
||||
"returncode": 2, "safety": {},
|
||||
"request_id": request.request_id})
|
||||
return
|
||||
try:
|
||||
repo_path = _resolve_repo(request.repo, server.workspace)
|
||||
except ValueError as exc:
|
||||
server.audit({"event": "privsep_request_refused",
|
||||
"kind": "repo_not_unique",
|
||||
"op": request.op})
|
||||
self._reply({"ok": False, "error": "repo_not_unique",
|
||||
"stdout": "", "stderr": str(exc),
|
||||
"returncode": 2, "safety": {},
|
||||
"request_id": request.request_id})
|
||||
return
|
||||
self._run_and_reply(request, repo_path)
|
||||
|
||||
def _run_and_reply(self, request, repo_path):
|
||||
server = self.server
|
||||
from cloud.chv.privsep import WorkerResponse
|
||||
from cloud.chv.safety import (
|
||||
check_context_input, mark_untrusted, redact_secrets)
|
||||
code, out, err = _run_git(request.op, repo_path,
|
||||
run_git=server.run_git)
|
||||
source = "gitea:%s" % (request.repo or "unknown")
|
||||
checked = check_context_input(
|
||||
mark_untrusted(out + "\n" + err, source))
|
||||
safety = checked.describe()
|
||||
# 脱敏后再返回:git 输出万一含 token,不原样回流。
|
||||
out = redact_secrets(out)
|
||||
err = redact_secrets(err)
|
||||
server.audit({"event": "privsep_worker_call",
|
||||
"op": request.op, "repo": request.repo,
|
||||
"returncode": code,
|
||||
"safety": safety,
|
||||
"cgroup": server.cgroup_id})
|
||||
if checked.verdict == "block":
|
||||
self._reply({"ok": False,
|
||||
"error": "safety_blocked:%s" % ",".join(
|
||||
f.rule for f in checked.findings),
|
||||
"stdout": "", "stderr": "",
|
||||
"returncode": code, "safety": safety,
|
||||
"request_id": request.request_id})
|
||||
return
|
||||
resp = WorkerResponse(code == 0, _truncate(out),
|
||||
_truncate(err), code, safety,
|
||||
request.request_id)
|
||||
self._reply(resp.to_dict())
|
||||
|
||||
|
||||
class WorkerServer(socketserver.ThreadingUnixStreamServer):
|
||||
"""Unix socket 服务(线程化,daemon_threads=True 避免僵尸线程)。"""
|
||||
|
||||
daemon_threads = True
|
||||
allow_reuse_address = False
|
||||
|
||||
def __init__(self, socket_path, workspace, log_path,
|
||||
allowed_uids=None, run_git=None,
|
||||
helper_path=HELPER_PATH, socket_group=None):
|
||||
self.workspace = workspace
|
||||
self.log_path = log_path
|
||||
self.allowed_uids = frozenset(
|
||||
allowed_uids if allowed_uids is not None
|
||||
else DEFAULT_ALLOWED_UIDS)
|
||||
self.run_git = run_git
|
||||
self.helper_path = helper_path
|
||||
try:
|
||||
with open("/proc/self/cgroup") as fh:
|
||||
self.cgroup_id = fh.readline().strip()
|
||||
except OSError:
|
||||
self.cgroup_id = ""
|
||||
# round-12 修复轮第三轮(缺口 ②):socket 用 group 权限。
|
||||
# gid 取 socket_group 显式值或第一个非 root 放行 uid 的
|
||||
# 主组;解析不出即 None,回落 root-only(0700/0600)。
|
||||
# tmpfiles.d 的 0750 行只是开机存在性保证,worker 启动时
|
||||
# 按 passwd 重新推导并重设权限,才是最终口径。
|
||||
self.socket_gid = _socket_group_gid(
|
||||
self.allowed_uids, socket_group)
|
||||
directory = os.path.dirname(socket_path)
|
||||
if directory:
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
if self.socket_gid is not None:
|
||||
_set_group_perms(directory, self.socket_gid, 0o750)
|
||||
else:
|
||||
_set_group_perms(directory, None, 0o700)
|
||||
if os.path.exists(socket_path):
|
||||
os.unlink(socket_path)
|
||||
super().__init__(socket_path, WorkerHandler)
|
||||
if self.socket_gid is not None:
|
||||
_set_group_perms(socket_path, self.socket_gid, 0o660)
|
||||
else:
|
||||
_set_group_perms(socket_path, None, 0o600)
|
||||
|
||||
def audit(self, record):
|
||||
_audit(self.log_path, record)
|
||||
|
||||
|
||||
def build_server(socket_path=DEFAULT_SOCKET_PATH,
|
||||
workspace=WORKSPACE,
|
||||
log_path=DEFAULT_LOG_PATH,
|
||||
lib_dir=DEFAULT_LIB_DIR,
|
||||
allowed_uids=None,
|
||||
run_git=None,
|
||||
helper_path=HELPER_PATH,
|
||||
socket_group=None):
|
||||
"""建服务(不启动):socket + 工作区 + 审计路径就绪即返回。
|
||||
helper 缺失不在建服务时报错——建服务时只备好 socket,git
|
||||
调用时再拒绝(fail-closed,见 _serve)。
|
||||
allowed_uids 不给时取环境变量 CHV_GITEA_WORKER_ALLOWED_UIDS
|
||||
(逗号分隔),再不给即默认 (0, 1000)(root + chv)。"""
|
||||
if lib_dir not in sys.path:
|
||||
sys.path.insert(0, lib_dir)
|
||||
if allowed_uids is None:
|
||||
raw = os.environ.get("CHV_GITEA_WORKER_ALLOWED_UIDS") or ""
|
||||
allowed_uids = ([int(p) for p in raw.split(",") if p.strip()]
|
||||
if raw.strip() else list(DEFAULT_ALLOWED_UIDS))
|
||||
return WorkerServer(socket_path, workspace, log_path,
|
||||
allowed_uids, run_git=run_git,
|
||||
helper_path=helper_path,
|
||||
socket_group=socket_group)
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
"""前台运行(systemd Type=simple 直接跑本进程,不 daemonize)。"""
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--socket", default=DEFAULT_SOCKET_PATH)
|
||||
parser.add_argument("--workspace", default=WORKSPACE)
|
||||
parser.add_argument("--log", default=DEFAULT_LOG_PATH)
|
||||
parser.add_argument("--lib", default=DEFAULT_LIB_DIR)
|
||||
args = parser.parse_args(argv)
|
||||
server = build_server(socket_path=args.socket,
|
||||
workspace=args.workspace,
|
||||
log_path=args.log, lib_dir=args.lib)
|
||||
print("chv-gitea-worker: listening on %s (cgroup %s)"
|
||||
% (args.socket, server.cgroup_id), file=sys.stderr)
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
finally:
|
||||
server.server_close()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,55 +0,0 @@
|
||||
[Unit]
|
||||
Description=CHV Gitea sync privsep worker (feat-chv-muse-p6)
|
||||
After=network-online.target chv-authd.service
|
||||
Wants=network-online.target
|
||||
# authd 是可选依赖:worker 的 git 经 P3 helper 向 authd 取凭证;
|
||||
# authd 没起来时 worker 的 git 调用直接拒绝,不静默用旧凭证。
|
||||
# 开机时序(round-12 修复轮第三轮,缺口 ①):工作区
|
||||
# /home/chv/workspace 由 cloud-final 的 chv-sync boot 克隆出来,
|
||||
# 晚于 multi-user.target 约 9 秒;而 ReadWritePaths 的路径必须在
|
||||
# systemd 建 mount namespace 时已存在,否则 226/NAMESPACE。
|
||||
# ConditionPathExists 让开机那次启动在工作区出现前干净跳过
|
||||
# (不耗重启预算、不 226);真正拉起靠 chv-gitea-worker.path
|
||||
# (工作区一出现即触发启动)。Condition 本身绝不重试——没有
|
||||
# .path 单元它就是一次跳过、服务永远不起;反过来只有 .path 而
|
||||
# 无 Condition,开机那次启动仍会在命名空间搭建时 226 撞上限速。
|
||||
# 两者缺一不可。StartLimit 不放宽:放宽只是把竞态窗口拉大,
|
||||
# wake 时间在负载下已翻倍(R12 复测 2),预算仍可能耗尽;
|
||||
# tmpfiles.d 也不补工作区行:/home/chv 本身 07:23:40 才出现,
|
||||
# 晚于 tmpfiles-setup(07:23:24),补了也没用。
|
||||
ConditionPathExists=/home/chv/workspace
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=root
|
||||
Group=root
|
||||
ExecStart=/usr/local/bin/chv-gitea-worker
|
||||
Restart=on-failure
|
||||
RestartSec=2
|
||||
# systemd 沙箱(与 P3 authd 同款 root 单元外服务口径,再收紧):
|
||||
# 只读系统、独立 /tmp、不许提权、不留 capability、只许 Unix 与
|
||||
# inet 协议族、只许 @system-service 系统调用、读写路径只限工作区
|
||||
# 与 worker 运行时目录。网络层强制(eBPF)是 P5 的范围,不在这里
|
||||
# 冒充——出口收紧靠 Sentinel 评估(authd 放凭证前)+ 只配 helper
|
||||
# 的 git(见 chv-gitea-worker.py)。
|
||||
ProtectSystem=strict
|
||||
PrivateTmp=yes
|
||||
NoNewPrivileges=yes
|
||||
CapabilityBoundingSet=
|
||||
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
|
||||
SystemCallFilter=@system-service
|
||||
# ReadWritePaths 下的路径必须在开机时已存在(ProtectSystem=strict
|
||||
# 下 systemd 建 mount namespace 的硬要求,缺一个即 226/NAMESPACE):
|
||||
# /run/chv-gitea 与 /var/log/chv-gitea-worker.log 由
|
||||
# /etc/tmpfiles.d/chv.conf 预建(build.sh 内联写,round-12 修复轮
|
||||
# 第二轮项 1);/home/chv/workspace 由 ConditionPathExists +
|
||||
# chv-gitea-worker.path 守门(round-12 修复轮第三轮缺口 ①,见
|
||||
# [Unit];R12 复测 2 钉死它才是剩下的 226——开机时序竞态,不要
|
||||
# 再写“开机时已在”)。ExecStartPre 不能替代 tmpfiles.d——命名空间
|
||||
# 搭建在它之前,226 发生在它运行之前。
|
||||
ReadWritePaths=/home/chv/workspace /run/chv-gitea /var/log/chv-gitea-worker.log
|
||||
# worker 崩溃不拖垮 agent:broker/薄 CLI 收到类型化失败即返回,
|
||||
# 不重试、不挂起;systemd 只负责拉起 worker 本身。
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,182 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""客内记忆文件操作(对齐 Muse P7,纯标准库)。
|
||||
|
||||
记忆家目录 /home/chv/memory(workspace 之外,不同步到 Gitea),
|
||||
布局与记忆系统同款:MEMORY.md + memory/*.md。由 executor 经
|
||||
SSH 调用(与 host.py 同级,跑在 nspawn cell 之外);绝不含凭证
|
||||
值——写操作过本地守卫,命中疑似凭证模式即拒绝。
|
||||
|
||||
用法:
|
||||
chv-memory.py list [--home DIR]
|
||||
chv-memory.py read <相对路径> [--home DIR]
|
||||
chv-memory.py append <相对路径> <行...> [--home DIR]
|
||||
chv-memory.py forget <needle> [--home DIR]
|
||||
chv-memory.py export <目标目录> [--home DIR]
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
|
||||
DEFAULT_HOME = "/home/chv/memory"
|
||||
|
||||
_SECRET_VALUE_PATTERNS = (
|
||||
re.compile(r"ghp_[A-Za-z0-9]{20,}"),
|
||||
re.compile(r"gho_[A-Za-z0-9]{20,}"),
|
||||
re.compile(r"github_pat_[A-Za-z0-9_]{20,}"),
|
||||
re.compile(r"sk-[A-Za-z0-9]{20,}"),
|
||||
re.compile(r"xox[baprs]-[A-Za-z0-9-]{10,}"),
|
||||
re.compile(r"AKIA[0-9A-Z]{16}"),
|
||||
re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"),
|
||||
)
|
||||
|
||||
|
||||
def _fail(message: str) -> "NoReturn":
|
||||
"""打印错误到 stderr 并以 1 退出。"""
|
||||
print("chv-memory: %s" % message, file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def _check_no_secret(text: str) -> None:
|
||||
"""文本命中凭证模式即拒(不打印命中内容)。"""
|
||||
for pattern in _SECRET_VALUE_PATTERNS:
|
||||
if pattern.search(text):
|
||||
_fail("refused: text looks like a credential")
|
||||
|
||||
|
||||
def _resolve(home: str, rel: str) -> str:
|
||||
"""相对路径 → 家目录内绝对路径,穿越即拒。"""
|
||||
base = os.path.abspath(home)
|
||||
candidate = os.path.abspath(os.path.join(base, rel))
|
||||
if candidate != base and not candidate.startswith(base + os.sep):
|
||||
_fail("path escapes memory home: %s" % rel)
|
||||
return candidate
|
||||
|
||||
|
||||
def _ensure(home: str) -> None:
|
||||
"""建家目录结构(MEMORY.md + memory/)。"""
|
||||
os.makedirs(os.path.join(home, "memory"), exist_ok=True)
|
||||
memory_md = os.path.join(home, "MEMORY.md")
|
||||
if not os.path.exists(memory_md):
|
||||
with open(memory_md, "w", encoding="utf-8") as handle:
|
||||
handle.write("# MEMORY.md\n")
|
||||
|
||||
|
||||
def cmd_list(home: str) -> dict:
|
||||
"""列家目录下全部文件(相对路径,排序)。"""
|
||||
_ensure(home)
|
||||
found = []
|
||||
for root, _dirs, names in os.walk(home):
|
||||
for name in names:
|
||||
found.append(os.path.relpath(os.path.join(root, name), home))
|
||||
return {"files": sorted(found)}
|
||||
|
||||
|
||||
def cmd_read(home: str, rel: str) -> dict:
|
||||
"""读一个记忆文件。"""
|
||||
_ensure(home)
|
||||
path = _resolve(home, rel)
|
||||
if not os.path.isfile(path):
|
||||
_fail("memory file missing: %s" % rel)
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
return {"path": rel, "text": handle.read()}
|
||||
|
||||
|
||||
def cmd_append(home: str, rel: str, lines: list) -> dict:
|
||||
"""向一个记忆文件追加行。"""
|
||||
_ensure(home)
|
||||
for line in lines:
|
||||
_check_no_secret(line)
|
||||
path = _resolve(home, rel)
|
||||
parent = os.path.dirname(path)
|
||||
if parent:
|
||||
os.makedirs(parent, exist_ok=True)
|
||||
with open(path, "a", encoding="utf-8") as handle:
|
||||
for line in lines:
|
||||
handle.write(line if line.endswith("\n") else line + "\n")
|
||||
return {"path": rel, "appended": len(lines)}
|
||||
|
||||
|
||||
def cmd_forget(home: str, needle: str) -> dict:
|
||||
"""删掉全部 Markdown 里含 needle 的行(大小写不敏感)。"""
|
||||
_ensure(home)
|
||||
removed = 0
|
||||
touched = []
|
||||
for root, _dirs, names in os.walk(home):
|
||||
for name in names:
|
||||
if not name.endswith(".md"):
|
||||
continue
|
||||
path = os.path.join(root, name)
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
lines = handle.readlines()
|
||||
kept = [ln for ln in lines
|
||||
if needle.lower() not in ln.lower()]
|
||||
cut = len(lines) - len(kept)
|
||||
if cut:
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
handle.writelines(kept)
|
||||
removed += cut
|
||||
touched.append(os.path.relpath(path, home))
|
||||
return {"needle": needle, "removed": removed, "files": touched}
|
||||
|
||||
|
||||
def cmd_export(home: str, dest: str) -> dict:
|
||||
"""导出全部记忆文件到目标目录(覆盖式,不含审计日志)。"""
|
||||
_ensure(home)
|
||||
os.makedirs(dest, exist_ok=True)
|
||||
count = 0
|
||||
for root, _dirs, names in os.walk(home):
|
||||
for name in names:
|
||||
if name == "audit.jsonl":
|
||||
continue
|
||||
src = os.path.join(root, name)
|
||||
rel = os.path.relpath(src, home)
|
||||
target = os.path.join(dest, rel)
|
||||
os.makedirs(os.path.dirname(target) or dest, exist_ok=True)
|
||||
shutil.copyfile(src, target)
|
||||
count += 1
|
||||
return {"dest": dest, "files": count}
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
"""命令行解析器。"""
|
||||
parser = argparse.ArgumentParser(prog="chv-memory.py")
|
||||
parser.add_argument("--home", default=DEFAULT_HOME)
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
sub.add_parser("list")
|
||||
read_p = sub.add_parser("read")
|
||||
read_p.add_argument("path")
|
||||
append_p = sub.add_parser("append")
|
||||
append_p.add_argument("path")
|
||||
append_p.add_argument("lines", nargs="+")
|
||||
forget_p = sub.add_parser("forget")
|
||||
forget_p.add_argument("needle")
|
||||
export_p = sub.add_parser("export")
|
||||
export_p.add_argument("dest")
|
||||
return parser
|
||||
|
||||
|
||||
def main(argv: list | None = None) -> int:
|
||||
"""入口:按子命令分发,打印 JSON 结果。"""
|
||||
args = build_parser().parse_args(argv)
|
||||
if args.cmd == "list":
|
||||
result = cmd_list(args.home)
|
||||
elif args.cmd == "read":
|
||||
result = cmd_read(args.home, args.path)
|
||||
elif args.cmd == "append":
|
||||
result = cmd_append(args.home, args.path, args.lines)
|
||||
elif args.cmd == "forget":
|
||||
result = cmd_forget(args.home, args.needle)
|
||||
elif args.cmd == "export":
|
||||
result = cmd_export(args.home, args.dest)
|
||||
else: # pragma: no cover - argparse required=True 已挡住
|
||||
_fail("unknown command: %s" % args.cmd)
|
||||
print(json.dumps(result, sort_keys=True))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,87 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chv-sync-worker-client:Gitea worker 薄客户端(feat-chv-muse-p6)。
|
||||
|
||||
agent 侧(broker / chv-sync)只剩薄 CLI:解析参数、发类型化 JSON
|
||||
请求到 worker socket、原样回显 worker 的 stdout/stderr、按 worker
|
||||
的 returncode 退出。本进程不跑 git、不碰凭证、不做业务逻辑。
|
||||
|
||||
用法:chv-sync-worker-client <pull|push|status|ping> [仓目录]
|
||||
仓目录省略时由 worker 侧取唯一仓(与 chv-sync 同款语义,由 worker
|
||||
的 repo=None 处理——本客户端只透传,不自己解析)。
|
||||
只用标准库。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
|
||||
DEFAULT_SOCKET_PATH = "/run/chv-gitea/worker.sock"
|
||||
|
||||
|
||||
def _exchange(sock_path, request):
|
||||
"""连 worker、发请求、收一帧回包。
|
||||
返回 (resp_dict, error_str),成功时 error_str 为 None。"""
|
||||
try:
|
||||
conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
conn.settimeout(310)
|
||||
conn.connect(sock_path)
|
||||
except OSError as exc:
|
||||
return None, "worker 连不上:%s" % exc
|
||||
try:
|
||||
conn.sendall((json.dumps(request) + "\n").encode("utf-8"))
|
||||
data = b""
|
||||
while len(data) < 65536:
|
||||
chunk = conn.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
data += chunk
|
||||
if data.endswith(b"\n"):
|
||||
break
|
||||
except OSError as exc:
|
||||
return None, "worker 通信失败:%s" % exc
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
resp = json.loads(data.decode("utf-8"))
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
return None, "worker 回包不是 JSON"
|
||||
if not isinstance(resp, dict) or "ok" not in resp:
|
||||
return None, "worker 回包格式不对"
|
||||
return resp, None
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
"""发一次类型化请求并回显。返回 worker 的 returncode;
|
||||
通信失败即 exit 2(调用方按失败处理,不静默成功)。"""
|
||||
args = list(argv if argv is not None else sys.argv[1:])
|
||||
if not args or args[0] not in ("pull", "push", "status", "ping"):
|
||||
print("用法:chv-sync-worker-client "
|
||||
"pull|push|status|ping [仓目录]", file=sys.stderr)
|
||||
return 2
|
||||
op = args[0]
|
||||
repo = args[1] if len(args) > 1 and args[1] else None
|
||||
sock_path = os.environ.get("CHV_GITEA_WORKER_SOCK",
|
||||
DEFAULT_SOCKET_PATH)
|
||||
request = {"op": op, "connector": "gitea", "repo": repo,
|
||||
"request_id": "cli-%d" % int(time.time() * 1000)}
|
||||
resp, err = _exchange(sock_path, request)
|
||||
if err is not None:
|
||||
print("chv-sync-worker-client: %s" % err, file=sys.stderr)
|
||||
return 2
|
||||
if resp.get("stdout"):
|
||||
sys.stdout.write(resp["stdout"])
|
||||
if resp.get("stderr"):
|
||||
sys.stderr.write(resp["stderr"])
|
||||
if resp.get("error"):
|
||||
print("chv-sync-worker-client: %s" % resp["error"],
|
||||
file=sys.stderr)
|
||||
if resp.get("safety", {}).get("verdict") == "flag":
|
||||
print("chv-sync-worker-client: 输出已标记复核:%s"
|
||||
% resp["safety"], file=sys.stderr)
|
||||
return int(resp.get("returncode") or (0 if resp["ok"] else 1))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,203 +0,0 @@
|
||||
#!/bin/bash
|
||||
# chv-sync:轻量 VM 里的 Gitea 双向同步(用户工作区口径)。
|
||||
#
|
||||
# 主角是用户,不是 agent:用户在 Gitea 上的仓库在 VM 启动时
|
||||
# clone 到 ~/workspace,用完再同步回 Gitea;VM 是用户的工作区,
|
||||
# Gitea 是权威远端。同步口径沿用 Git Workspace Platform 已定
|
||||
# 的规则:分叉或冲突时失败关闭,不强制覆盖、不自动 rebase、
|
||||
# 不静默丢提交。
|
||||
#
|
||||
# 凭证绝不进镜像。启动时由 seed 盘投递两份文件(0600,root 所有):
|
||||
# /etc/chv/guest-env(非密钥同步变量:GITEA_URL/GITEA_USER/
|
||||
# SYNC_REPO/SYNC_BRANCH/GITEA_EMAIL)与
|
||||
# /etc/chv/authd-seed(feat-chv-muse-p3:只装 GITEA_TOKEN,
|
||||
# 只给客内 authd 守护进程读)。
|
||||
# boot(root,cloud-init runcmd 调)读 guest-env,authd 在时配
|
||||
# git credential helper 走 authd 按次取凭证(只进 git 子进程
|
||||
# 内存,不落盘);authd 不在时直接失败(round-11 真机修复层 5:
|
||||
# 旧镜像的落盘回落已废止——它在真机测试里把 authd 释放的长效
|
||||
# token 写进了客内 ~/.git-credentials;authd helper 是唯一凭证
|
||||
# helper,且自身绝不落盘)。pull/push/status 以 chv 身份跑,用
|
||||
# helper,不再读 guest-env;token 不会出现在 git remote URL 或
|
||||
# 镜像里。
|
||||
#
|
||||
# 用法:
|
||||
# chv-sync boot # 启动时 clone(已 clone 则静默退出,不自动 pull)
|
||||
# chv-sync pull [仓目录] # 脏树拒绝;只接受 fast-forward,分叉即失败
|
||||
# chv-sync push [仓目录] # 普通 push;无 upstream 时 push -u origin HEAD;
|
||||
# # 非 fast-forward 即失败,绝不 --force
|
||||
# chv-sync status [仓目录] # 本地状态 + 最近提交
|
||||
# 仓目录省略时取 $WORKSPACE 下唯一的 git 仓库;多个时必须显式给。
|
||||
set -euo pipefail
|
||||
|
||||
GUEST_ENV="${CHV_GUEST_ENV:-/etc/chv/guest-env}"
|
||||
WORKSPACE="${CHV_WORKSPACE:-/home/chv/workspace}"
|
||||
USER_HOME="${CHV_USER_HOME:-/home/chv}"
|
||||
SYNC_USER="${CHV_USER:-chv}"
|
||||
# privsep worker(feat-chv-muse-p6):socket 在时 pull/push/status
|
||||
# 改走 worker(类型化请求),不在时走旧的直接路径(过渡兼容)。
|
||||
# round-12 修复轮第二轮(项 2):回落仍是 fail-open,不改
|
||||
# fail-closed——旧镜像(无 worker)须仍能同步,改 fail-closed 会把
|
||||
# 所有旧镜像的同步当场 brick,且无镜像外恢复手段;本轮项 1 已让新
|
||||
# 镜像的 worker 开机即起,回落只在 worker 真挂 / 旧镜像时发生,不
|
||||
# 再是常态路径。但回落不再静默:每次回落都在 stderr 警告(保留
|
||||
# “worker 不在”标记供驱动识别)+ 追加一行 JSON 审计
|
||||
# ($USER_HOME/.chv-sync-fallback-audit.log,可用
|
||||
# CHV_SYNC_FALLBACK_LOG 改位置;写失败只记 stderr,绝不中断同步,
|
||||
# 与 worker 的 _audit 同款口径)。事后 grep 该审计文件即能数出回落
|
||||
# 次数、动作与仓。旧镜像退役后改为无 worker 即 fail-closed。
|
||||
WORKER_SOCK="${CHV_GITEA_WORKER_SOCK:-/run/chv-gitea/worker.sock}"
|
||||
WORKER_CLIENT="${CHV_GITEA_WORKER_CLIENT:-/usr/local/bin/chv-sync-worker-client}"
|
||||
|
||||
die() { echo "chv-sync: $*" >&2; exit 1; }
|
||||
info() { echo "chv-sync: $*"; }
|
||||
|
||||
# 试走 worker:返回 42 = worker 不在(调用方走旧路径);其它返回
|
||||
# 值即 worker 的退出码(调用方直接退出,不再回落重跑)。
|
||||
try_worker() {
|
||||
local action="$1"; shift
|
||||
[ -S "$WORKER_SOCK" ] || return 42
|
||||
[ -x "$WORKER_CLIENT" ] || return 42
|
||||
"$WORKER_CLIENT" "$action" "$@"
|
||||
}
|
||||
|
||||
# 回落显性化(round-12 修复轮第二轮,项 2):一次回落留两处痕——
|
||||
# stderr 警告 + 一行 JSON 审计(事件/动作/仓/时间戳,明确记
|
||||
# delegated_to_worker=false、privsep_active=false)。审计写失败
|
||||
# 只记 stderr,绝不中断同步。
|
||||
fallback_warn() {
|
||||
local action="$1" repo="${2:-}"
|
||||
echo "chv-sync: worker 不在,走旧的直接路径(过渡兼容):未走 worker,privsep 未生效" >&2
|
||||
local log="${CHV_SYNC_FALLBACK_LOG:-$USER_HOME/.chv-sync-fallback-audit.log}"
|
||||
local ts
|
||||
ts="$(date -u +%Y-%m-%dT%H:%M:%SZ 2>/dev/null || echo unknown)"
|
||||
repo="${repo//\"/}"
|
||||
printf '%s\n' "{\"event\":\"chv_sync_worker_fallback\",\"action\":\"$action\",\"repo\":\"$repo\",\"delegated_to_worker\":false,\"privsep_active\":false,\"ts\":\"$ts\"}" >> "$log" 2>/dev/null \
|
||||
|| echo "chv-sync: 回落审计落盘失败:$log" >&2
|
||||
}
|
||||
|
||||
# 解析仓目录:显式给则用它;否则取 WORKSPACE 下唯一的 git 仓库。
|
||||
resolve_repo() {
|
||||
if [ -n "${1:-}" ]; then
|
||||
[ -d "$1/.git" ] || die "不是 git 仓库:$1"
|
||||
printf '%s' "$1"
|
||||
return 0
|
||||
fi
|
||||
local found=()
|
||||
local d
|
||||
for d in "$WORKSPACE"/*/; do
|
||||
[ -d "${d}.git" ] && found+=("${d%/}")
|
||||
done
|
||||
if [ "${#found[@]}" -eq 1 ]; then
|
||||
printf '%s' "${found[0]}"
|
||||
return 0
|
||||
fi
|
||||
die "WORKSPACE 下 git 仓库不是唯一(${#found[@]} 个),请显式给仓目录"
|
||||
}
|
||||
|
||||
cmd_boot() {
|
||||
# 无 guest-env = 本次启动不同步任何仓,静默成功(VM 照常用)。
|
||||
[ -f "$GUEST_ENV" ] || { info "无 $GUEST_ENV,本次启动不同步"; exit 0; }
|
||||
# shellcheck disable=SC1090
|
||||
set -a; . "$GUEST_ENV"; set +a
|
||||
: "${GITEA_URL:?guest-env 缺 GITEA_URL}"
|
||||
: "${GITEA_USER:?guest-env 缺 GITEA_USER}"
|
||||
: "${SYNC_REPO:?guest-env 缺 SYNC_REPO}"
|
||||
[ "$(id -u)" = "0" ] || die "boot 须以 root 运行(cloud-init runcmd)"
|
||||
local name="${SYNC_REPO##*/}"
|
||||
local dir="$WORKSPACE/$name"
|
||||
mkdir -p "$WORKSPACE"
|
||||
chown "$SYNC_USER:$SYNC_USER" "$WORKSPACE"
|
||||
# feat-chv-muse-p3:authd 路径——socket 存在时凭证走 helper,
|
||||
# 不再把真 token 写进 .git-credentials 落盘;helper 按次向
|
||||
# authd 取凭证,只进 git 子进程内存。socket 不存在时直接失败
|
||||
#(round-11 真机修复层 5:落盘回落已废止,见下)。
|
||||
# 注意:authd 路径下 GITEA_TOKEN 不在 guest-env 里(它只进
|
||||
# authd-seed 给守护进程),所以这里不要求它。
|
||||
local authd_socket="${CHV_AUTHD_SOCKET:-/run/chv-authd/authd.sock}"
|
||||
local helper="/usr/local/bin/git-credential-chv-authd"
|
||||
if [ -S "$authd_socket" ] && [ -x "$helper" ]; then
|
||||
runuser -u "$SYNC_USER" -- env HOME="$USER_HOME" git config --global credential.helper "$helper"
|
||||
# helper 按路径推断仓库与操作(Sentinel L7 绑定),git
|
||||
# 默认不把 path 发给 helper(须 credential.useHttpPath),
|
||||
# 不开它每次都会因“请求不完整”拒绝——新测试抓到的真问题。
|
||||
runuser -u "$SYNC_USER" -- env HOME="$USER_HOME" git config --global credential.useHttpPath true
|
||||
info "凭证走 chv-authd(不落盘)"
|
||||
else
|
||||
# round-11 真机修复(层 5):authd 不在时直接失败,不再
|
||||
# 把真 token 写进 ~/.git-credentials 落盘——旧路径在
|
||||
# round-11 真机测试里把 authd 释放的长效 token 写进了
|
||||
# 客内磁盘(/home/chv/.git-credentials,0600)。authd
|
||||
# helper 是唯一凭证 helper,且自身绝不落盘;旧镜像退役
|
||||
# 后过渡兼容即废止,不再留落盘逃生口。
|
||||
die "chv-authd 未运行(socket 或 helper 缺失),拒绝同步:凭证绝不落盘,请用含 authd 的镜像"
|
||||
fi
|
||||
if [ -d "$dir/.git" ]; then
|
||||
info "已 clone:$dir(不自动 pull,改动由用户显式同步)"
|
||||
exit 0
|
||||
fi
|
||||
[ -e "$dir" ] && die "$dir 已存在但不是 git 仓库,请人工处理"
|
||||
local clone_args=(clone)
|
||||
[ -n "${SYNC_BRANCH:-}" ] && clone_args+=(--branch "$SYNC_BRANCH")
|
||||
clone_args+=("$GITEA_URL/$SYNC_REPO.git" "$dir")
|
||||
runuser -u "$SYNC_USER" -- env HOME="$USER_HOME" git "${clone_args[@]}"
|
||||
runuser -u "$SYNC_USER" -- env HOME="$USER_HOME" git -C "$dir" config user.name "$GITEA_USER"
|
||||
if [ -n "${GITEA_EMAIL:-}" ]; then
|
||||
runuser -u "$SYNC_USER" -- env HOME="$USER_HOME" git -C "$dir" config user.email "$GITEA_EMAIL"
|
||||
fi
|
||||
info "已 clone $SYNC_REPO 到 $dir,HEAD=$(runuser -u "$SYNC_USER" -- env HOME="$USER_HOME" git -C "$dir" rev-parse --short HEAD)"
|
||||
}
|
||||
|
||||
cmd_pull() {
|
||||
local rc=0
|
||||
try_worker pull "${1:-}" || rc=$?
|
||||
[ "$rc" != "42" ] && exit "$rc"
|
||||
# round-12 修复轮第三轮(小项 A):先解析仓目录再记回落审计,
|
||||
# repo 字段记 resolve_repo 解析后的仓目录(无参调用时不再是
|
||||
# 空串,多仓审计才能分清是哪一仓回落)。解析失败即 die,
|
||||
# 不留回落审计——同步本身已失败,没有“回落”可记。
|
||||
local dir
|
||||
dir="$(resolve_repo "${1:-}")"
|
||||
fallback_warn pull "$dir"
|
||||
[ -z "$(git -C "$dir" status --porcelain)" ] \
|
||||
|| die "工作树有未提交改动,拒绝 pull(先 commit 或 stash)"
|
||||
git -C "$dir" pull --ff-only \
|
||||
|| die "pull 不是 fast-forward(本地与远端已分叉),请人工处理,不自动 rebase/merge"
|
||||
info "已同步:$dir HEAD=$(git -C "$dir" rev-parse --short HEAD)"
|
||||
}
|
||||
|
||||
cmd_push() {
|
||||
local rc=0
|
||||
try_worker push "${1:-}" || rc=$?
|
||||
[ "$rc" != "42" ] && exit "$rc"
|
||||
local dir
|
||||
dir="$(resolve_repo "${1:-}")"
|
||||
fallback_warn push "$dir"
|
||||
if git -C "$dir" rev-parse --abbrev-ref --symbolic-full-name '@{u}' >/dev/null 2>&1; then
|
||||
git -C "$dir" push \
|
||||
|| die "push 被拒绝(远端已有新提交,先 pull 再 push,不 --force)"
|
||||
else
|
||||
git -C "$dir" push -u origin HEAD \
|
||||
|| die "push 被拒绝(远端已有新提交,先 pull 再 push,不 --force)"
|
||||
fi
|
||||
info "已 push:$dir HEAD=$(git -C "$dir" rev-parse --short HEAD)"
|
||||
}
|
||||
|
||||
cmd_status() {
|
||||
local rc=0
|
||||
try_worker status "${1:-}" || rc=$?
|
||||
[ "$rc" != "42" ] && exit "$rc"
|
||||
local dir
|
||||
dir="$(resolve_repo "${1:-}")"
|
||||
fallback_warn status "$dir"
|
||||
git -C "$dir" status -sb
|
||||
git -C "$dir" log --oneline -5
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
boot) shift; cmd_boot "$@" ;;
|
||||
pull) shift; cmd_pull "$@" ;;
|
||||
push) shift; cmd_push "$@" ;;
|
||||
status) shift; cmd_status "$@" ;;
|
||||
*) echo "用法:chv-sync boot|pull|push|status [仓目录]" >&2; exit 2 ;;
|
||||
esac
|
||||
@@ -1,408 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""chv_browser_guard:浏览器安全共享纯函数(客机侧)。
|
||||
|
||||
对齐 Muse P4(见 docs/chv-muse-p4-gap.md):本模块只放纯函数,
|
||||
不 import playwright、不起进程、不连网络,供 chv-cdp-broker.py
|
||||
与 chv-browser.py 两脚本共用,避免三份拷贝漂移。
|
||||
|
||||
内容:
|
||||
validate_url P2 的 URL 规则:只 http/https,拒 localhost/
|
||||
私有地址字面量(DNS 重绑定是 Sentinel 的事)。
|
||||
check_blocklist 静态恶意站名单(browser-blocklist.txt)后缀匹配。
|
||||
truncate_aria ARIA 快照字符串截断(200 节点行/64KB)。
|
||||
sanitize_error 错误信息消毒:去 HTML 标签、截断 200 字符、
|
||||
仍含尖括号即只留异常类型名——快照路径上不许有
|
||||
原始 DOM/HTML 外泄口子。
|
||||
detect_injection DOM 内提示注入启发式(中英文模式表,命中返标签;
|
||||
是规则,不是模型分类器)。
|
||||
detect_form_risk ARIA 快照字符串(或旧 dict 树)扫密码/支付字段。
|
||||
scan_download 下载扫描钩子:一条可执行扩展名规则。
|
||||
enqueue/list/resolve_ask 文件型待审批队列(与 P3 ApprovalQueue
|
||||
同语义:pending/allowed/denied)。
|
||||
LAUNCH_ARGS / FORBIDDEN_PAGE_ATTRS Chromium 启动参数白名单与
|
||||
禁用的 Playwright 页面调用名表。
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import uuid
|
||||
|
||||
MAX_A11Y_NODES = 200
|
||||
MAX_A11Y_CHARS = 65536
|
||||
MAX_ERROR_CHARS = 200
|
||||
|
||||
# 拒绝的 URL 主机:localhost 与私有地址字面量(P2 原样)。
|
||||
_PRIVATE_HOST_RE = re.compile(
|
||||
r"^(localhost|127\.\d+\.\d+\.\d+|10\.\d+\.\d+\.\d+|"
|
||||
r"172\.(1[6-9]|2\d|3[01])\.\d+\.\d+|192\.168\.\d+\.\d+|"
|
||||
r"\[::1\]|\[fe8[0-9a-f]*:.*\])$",
|
||||
re.IGNORECASE)
|
||||
|
||||
# Chromium 启动参数白名单:均为真实开关;刻意没有
|
||||
# --remote-debugging-port(DevTools 不暴露)与 --no-sandbox
|
||||
#(浏览器以非 root 的 chv-browser 身份跑,不降级沙箱)。
|
||||
LAUNCH_ARGS = [
|
||||
"--disable-gpu",
|
||||
"--no-first-run",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-extensions",
|
||||
"--disable-background-networking",
|
||||
"--disable-sync",
|
||||
"--disable-translate",
|
||||
"--mute-audio",
|
||||
]
|
||||
|
||||
# 禁用的 Playwright 页面调用名:页面上下文 JS 与进程内 exec
|
||||
# 的全部入口。broker 与子代理的代码里不得出现这些属性名,
|
||||
# 测试用 AST 扫两个脚本钉住(字符串常量里的出现不算)。
|
||||
FORBIDDEN_PAGE_ATTRS = (
|
||||
"evaluate", "evaluate_handle", "add_script_tag",
|
||||
"add_init_script", "expose_function", "set_content",
|
||||
"keyboard", "mouse", "touchscreen",
|
||||
)
|
||||
|
||||
# 浏览器 socket 白名单之外的操作名(子代理层拒绝)。
|
||||
FORBIDDEN_OPS = (
|
||||
"evaluate", "content", "devtools", "js", "click", "fill",
|
||||
"type", "press", "hover", "select", "check", "route",
|
||||
"expose", "add_init_script", "add_script_tag", "set_content",
|
||||
"eval",
|
||||
)
|
||||
|
||||
# 下载拦截:可执行扩展名(一条规则;后续规则只加条目)。
|
||||
BLOCKED_DOWNLOAD_EXTS = frozenset({
|
||||
".exe", ".msi", ".bat", ".cmd", ".ps1", ".sh", ".dmg",
|
||||
".pkg", ".apk", ".scr", ".vbs", ".com", ".gadget",
|
||||
})
|
||||
|
||||
# 提示注入启发式模式表(中英文各 6 条):字面匹配,不是语义判断。
|
||||
# 命中只做标注(untrusted + injection_flags),不做拦截决定。
|
||||
INJECTION_PATTERNS = (
|
||||
("en_ignore_instructions", r"ignore (all )?(previous|prior|above) instructions"),
|
||||
("en_you_are_now", r"you are now (a|an|the)"),
|
||||
("en_system_prompt", r"(reveal|show|print|disregard).{0,40}system prompt"),
|
||||
("en_disregard", r"disregard (all )?(previous|prior|above)"),
|
||||
("en_new_instructions", r"new instructions:? (you|to|do)"),
|
||||
("en_jailbreak", r"(jailbreak|DAN mode|do anything now)"),
|
||||
("zh_ignore", r"忽略(以上|之前|所有|先前)的?(指令|指示|提示)"),
|
||||
("zh_pretend", r"假装你是"),
|
||||
("zh_now_you_are", r"你现在是"),
|
||||
("zh_leak_prompt", r"(泄露|泄漏|输出|打印).{0,20}系统提示"),
|
||||
("zh_admin", r"以管理员(身份|权限)"),
|
||||
("zh_no_limit", r"(无视|绕过).{0,20}(限制|规则|审查)"),
|
||||
)
|
||||
|
||||
# 表单风险:ARIA 快照行里的字段名/autocomplete 里的敏感词(小写匹配)。
|
||||
PASSWORD_HINTS = ("password", "密码", "current-password",
|
||||
"new-password", "passwd", "pass word")
|
||||
PAYMENT_HINTS = ("payment", "支付", "信用卡", "银行卡", "卡号",
|
||||
"cc-number", "cc-exp", "cc-csc", "cc-name",
|
||||
"card number", "cvv", "expiry", "有效期")
|
||||
|
||||
|
||||
def validate_url(url):
|
||||
"""URL 校验:只允许 http/https,且主机不是 localhost/私有地址
|
||||
字面量。返回 (ok, error)。"""
|
||||
if not isinstance(url, str) or not url:
|
||||
return False, "url 不能为空"
|
||||
m = re.match(r"^(https?)://([^/]+)", url, re.IGNORECASE)
|
||||
if not m:
|
||||
return False, "只允许 http/https URL:%s" % url[:80]
|
||||
host = m.group(2).split("@")[-1].split(":")[0].lower()
|
||||
if _PRIVATE_HOST_RE.match(host):
|
||||
return False, "拒绝 localhost/私有地址:%s" % host
|
||||
return True, ""
|
||||
|
||||
|
||||
def url_host(url):
|
||||
"""取 URL 的主机小写形式;取不到返回空串。"""
|
||||
m = re.match(r"^(https?)://([^/]+)", url or "", re.IGNORECASE)
|
||||
if not m:
|
||||
return ""
|
||||
return m.group(2).split("@")[-1].split(":")[0].lower()
|
||||
|
||||
|
||||
def load_blocklist(path):
|
||||
"""读静态拦截名单:每行一个条目(域名或 .后缀),# 开头是注释,
|
||||
空行跳过。文件缺失即空名单(不静默编造条目)。"""
|
||||
entries = set()
|
||||
try:
|
||||
with open(path) as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
entries.add(line.lower())
|
||||
except OSError:
|
||||
return set()
|
||||
return entries
|
||||
|
||||
|
||||
def check_blocklist(host, entries):
|
||||
"""主机是否命中拦截名单:精确匹配或后缀匹配(条目 .example.com
|
||||
命中 a.example.com;条目 example.com 同样命中子域)。"""
|
||||
host = (host or "").lower()
|
||||
if not host or not entries:
|
||||
return False
|
||||
for entry in entries:
|
||||
entry = entry.lstrip(".")
|
||||
if host == entry or host.endswith("." + entry):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def truncate_aria(text, max_nodes=MAX_A11Y_NODES,
|
||||
max_chars=MAX_A11Y_CHARS):
|
||||
"""截断 ARIA 快照字符串:节点行(行首 "- ")超上限时剪掉多余
|
||||
行,字符超上限时截断。返回 (text, truncated)。纯函数,可单测。
|
||||
round-11 复测纠正(第二轮,179a86d):Playwright 根本没有
|
||||
page.accessibility(那是 Puppeteer 的 API),原生快照是
|
||||
Page.aria_snapshot() 返回的字符串;旧的 dict 版 truncate_a11y
|
||||
随之废止。round-12 合入时从 chv-browser.py 移植到共享 guard
|
||||
(P4 架构里快照走 chv-cdp-broker.py)。a11y 字段仍叫 a11y,
|
||||
值改为 ARIA 快照字符串。"""
|
||||
text = text or ""
|
||||
lines = text.splitlines()
|
||||
kept = []
|
||||
nodes = 0
|
||||
truncated = False
|
||||
for line in lines:
|
||||
if line.lstrip().startswith("- "):
|
||||
nodes += 1
|
||||
if nodes > max_nodes:
|
||||
truncated = True
|
||||
break
|
||||
kept.append(line)
|
||||
out = "\n".join(kept)
|
||||
if len(out) > max_chars:
|
||||
out = out[:max_chars]
|
||||
truncated = True
|
||||
return out, truncated
|
||||
|
||||
|
||||
def sanitize_error(exc):
|
||||
"""错误信息消毒:只留异常类型名 + 去 HTML 标签、截断后的首行;
|
||||
凭据形内容(token=/password= 等 key=value 与 sk- 形 token)打码;
|
||||
清洗后仍含尖括号或 javascript: 即丢弃原文。绝不把页面原文
|
||||
或凭据原样放进回包或日志。"""
|
||||
name = type(exc).__name__ if not isinstance(exc, str) else "Error"
|
||||
text = exc if isinstance(exc, str) else str(exc)
|
||||
text = re.sub(r"<[^>]{0,500}>", "", text)
|
||||
text = re.sub(r"(?i)(token|password|passwd|secret|api[-_]?key)"
|
||||
r"\s*=\s*[^\s&;\"']+", r"\1=<redacted>", text)
|
||||
text = re.sub(r"sk-[A-Za-z0-9_\-]{8,}", "<redacted>", text)
|
||||
text = " ".join(text.split())[:MAX_ERROR_CHARS]
|
||||
if "<" in text or ">" in text or "javascript:" in text.lower():
|
||||
return "浏览器操作失败(%s)" % name
|
||||
if not text:
|
||||
return "浏览器操作失败(%s)" % name
|
||||
return "浏览器操作失败(%s):%s" % (name, text)
|
||||
|
||||
|
||||
def _collect_text(node, out):
|
||||
"""收集无障碍树里全部字符串值(name/value/description 等),
|
||||
供注入启发式扫描。只收字符串,不收 role 名以外的结构。"""
|
||||
if isinstance(node, dict):
|
||||
for value in node.values():
|
||||
_collect_text(value, out)
|
||||
elif isinstance(node, list):
|
||||
for item in node:
|
||||
_collect_text(item, out)
|
||||
elif isinstance(node, str):
|
||||
out.append(node)
|
||||
|
||||
|
||||
def detect_injection(tree):
|
||||
"""DOM 内提示注入启发式:扫描快照全部文本,返回命中的模式标签
|
||||
列表(去重、按模式表顺序)。无命中即空列表。规则而已,不是
|
||||
模型分类器——命中只标注,不拦截。输入是 ARIA 快照字符串
|
||||
(round-12 移植后 broker 传的就是它);旧 dict 树仍接受,
|
||||
收集全部字符串值后同样扫描。"""
|
||||
if isinstance(tree, str):
|
||||
blob = tree
|
||||
else:
|
||||
texts = []
|
||||
_collect_text(tree or {}, texts)
|
||||
blob = "\n".join(texts)
|
||||
hits = []
|
||||
for label, pattern in INJECTION_PATTERNS:
|
||||
if re.search(pattern, blob, re.IGNORECASE):
|
||||
hits.append(label)
|
||||
return hits
|
||||
|
||||
|
||||
def _node_hints(node, hints):
|
||||
"""一个无障碍树节点的 name/autocomplete/type 里是否含敏感词。"""
|
||||
blob = " ".join(str(node.get(k, "")) for k in
|
||||
("name", "autocomplete", "type", "description"))
|
||||
blob = blob.lower()
|
||||
return any(h in blob for h in hints)
|
||||
|
||||
|
||||
def _walk_form_risk(node, found):
|
||||
if not isinstance(node, dict):
|
||||
return
|
||||
role = str(node.get("role", "")).lower()
|
||||
if role in ("textbox", "textfield", "combobox"):
|
||||
if _node_hints(node, PASSWORD_HINTS):
|
||||
found.setdefault("password", []).append(
|
||||
str(node.get("name", ""))[:40])
|
||||
elif _node_hints(node, PAYMENT_HINTS):
|
||||
found.setdefault("payment", []).append(
|
||||
str(node.get("name", ""))[:40])
|
||||
for child in node.get("children") or []:
|
||||
_walk_form_risk(child, found)
|
||||
|
||||
|
||||
_ARIA_FIELD_RE = re.compile(
|
||||
r"^\s*-\s+(textbox|textfield|combobox)\b", re.IGNORECASE)
|
||||
_ARIA_NAME_RE = re.compile(r'"([^"]{0,40})"')
|
||||
|
||||
|
||||
def _form_risk_from_aria(text, found):
|
||||
"""ARIA 快照字符串版表单风险扫描:每行行首 "- textbox/
|
||||
textfield/combobox" 的字段行,整行小写匹配敏感词(字段名与
|
||||
[autocomplete=...] 都在行内);字段名取行内第一个引号串。"""
|
||||
for line in (text or "").splitlines():
|
||||
if not _ARIA_FIELD_RE.match(line):
|
||||
continue
|
||||
blob = line.lower()
|
||||
m = _ARIA_NAME_RE.search(line)
|
||||
name = m.group(1) if m else ""
|
||||
if any(h in blob for h in PASSWORD_HINTS):
|
||||
found.setdefault("password", []).append(name)
|
||||
elif any(h in blob for h in PAYMENT_HINTS):
|
||||
found.setdefault("payment", []).append(name)
|
||||
|
||||
|
||||
def detect_form_risk(tree):
|
||||
"""扫快照里的密码/支付字段。返回 {level, fields}:level 是
|
||||
none/password/payment(支付优先于密码),fields 是命中的字段名
|
||||
(最多 5 个,供审批记录看,不进模型上下文原样只作标注)。
|
||||
输入是 ARIA 快照字符串(round-12 移植后 broker 传的就是它);
|
||||
旧 dict 树仍接受,走原来的节点遍历。"""
|
||||
found = {}
|
||||
if isinstance(tree, str):
|
||||
_form_risk_from_aria(tree, found)
|
||||
else:
|
||||
_walk_form_risk(tree or {}, found)
|
||||
if found.get("payment"):
|
||||
return {"level": "payment",
|
||||
"fields": found["payment"][:5]}
|
||||
if found.get("password"):
|
||||
return {"level": "password",
|
||||
"fields": found["password"][:5]}
|
||||
return {"level": "none", "fields": []}
|
||||
|
||||
|
||||
def scan_download(filename):
|
||||
"""下载扫描钩子(一条规则):可执行扩展名、或中间段含可执行
|
||||
扩展名(双扩展名把戏,如 report.pdf.exe / archive.exe.pdf),
|
||||
即拦。返回 (blocked, reason)。"""
|
||||
name = (filename or "").strip().lower()
|
||||
if not name:
|
||||
return False, ""
|
||||
parts = [p for p in name.split(".") if p]
|
||||
if len(parts) < 2:
|
||||
return False, ""
|
||||
final_ext = "." + parts[-1]
|
||||
if final_ext in BLOCKED_DOWNLOAD_EXTS:
|
||||
return True, "可执行扩展名:%s" % final_ext
|
||||
for middle in parts[1:-1]:
|
||||
if ("." + middle) in BLOCKED_DOWNLOAD_EXTS:
|
||||
return True, "双扩展名可疑:%s" % filename[:60]
|
||||
return False, ""
|
||||
|
||||
|
||||
def check_launch_args(args):
|
||||
"""Chromium 启动参数校验:不得含 --remote-debugging-*(DevTools
|
||||
不暴露),不得含 devtools 字串。返回 (ok, error)。"""
|
||||
for arg in args or []:
|
||||
if arg.startswith("--remote-debugging"):
|
||||
return False, "启动参数不得含 %s" % arg
|
||||
if "devtools" in arg.lower():
|
||||
return False, "启动参数不得含 DevTools:%s" % arg
|
||||
return True, ""
|
||||
|
||||
|
||||
def _ask_path(queue_dir, ask_id):
|
||||
return os.path.join(queue_dir, "ask-%s.json" % ask_id)
|
||||
|
||||
|
||||
def enqueue_ask(queue_dir, kind, url, detail):
|
||||
"""待审批入队:一条 ask 一个 JSON 文件(原子写:先写 tmp 再
|
||||
rename)。返回 ask_id。队列目录不存在即建。"""
|
||||
os.makedirs(queue_dir, exist_ok=True)
|
||||
ask_id = uuid.uuid4().hex[:12]
|
||||
record = {"ask_id": ask_id, "kind": kind, "url": url or "",
|
||||
"detail": detail or "", "status": "pending",
|
||||
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ",
|
||||
time.gmtime())}
|
||||
tmp_path = _ask_path(queue_dir, ask_id) + ".tmp"
|
||||
with open(tmp_path, "w") as fh:
|
||||
json.dump(record, fh, ensure_ascii=False)
|
||||
fh.write("\n")
|
||||
os.replace(tmp_path, _ask_path(queue_dir, ask_id))
|
||||
return ask_id
|
||||
|
||||
|
||||
def list_asks(queue_dir):
|
||||
"""列出待审批(status pending)的 ask,按创建时间排序。"""
|
||||
out = []
|
||||
try:
|
||||
names = sorted(os.listdir(queue_dir))
|
||||
except OSError:
|
||||
return []
|
||||
for name in names:
|
||||
if not (name.startswith("ask-") and name.endswith(".json")):
|
||||
continue
|
||||
try:
|
||||
with open(os.path.join(queue_dir, name)) as fh:
|
||||
record = json.load(fh)
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
if record.get("status") == "pending":
|
||||
out.append(record)
|
||||
out.sort(key=lambda r: r.get("created_at", ""))
|
||||
return out
|
||||
|
||||
|
||||
def resolve_ask(queue_dir, ask_id, decision):
|
||||
"""写回审批决定:decision 只认 allowed/denied。返回 (ok, error)。
|
||||
不存在的 ask_id 即报错,不静默成功。"""
|
||||
if decision not in ("allowed", "denied"):
|
||||
return False, "decision 只认 allowed/denied:%s" % decision
|
||||
path = _ask_path(queue_dir, ask_id)
|
||||
try:
|
||||
with open(path) as fh:
|
||||
record = json.load(fh)
|
||||
except (OSError, ValueError):
|
||||
return False, "ask 不存在:%s" % ask_id
|
||||
if record.get("status") != "pending":
|
||||
return False, "ask 已处理:%s" % ask_id
|
||||
record["status"] = decision
|
||||
tmp_path = path + ".tmp"
|
||||
with open(tmp_path, "w") as fh:
|
||||
json.dump(record, fh, ensure_ascii=False)
|
||||
fh.write("\n")
|
||||
os.replace(tmp_path, path)
|
||||
return True, ""
|
||||
|
||||
|
||||
def response_leaks(resp):
|
||||
"""回包泄露检查(测试与 broker 自检共用):递归扫响应里的字符串,
|
||||
命中 ws://、DevTools 字串或 HTML 标签即返回问题列表。空列表即干净。"""
|
||||
problems = []
|
||||
text = json.dumps(resp, ensure_ascii=False)
|
||||
if "ws://" in text or "wss://" in text:
|
||||
problems.append("响应含 ws 端点")
|
||||
# 只查 CDP 形态(devtools/ 路径、remote-debugging),不查裸词——
|
||||
# 正常 URL 里也可能含 devtools(如 devtools.example.com)。
|
||||
if re.search(r"devtools/", text, re.IGNORECASE):
|
||||
problems.append("响应含 DevTools 路径")
|
||||
if "remote-debugging" in text.lower():
|
||||
problems.append("响应含 remote-debugging")
|
||||
if re.search(r"<[a-zA-Z][^>]{0,200}>", text):
|
||||
problems.append("响应含 HTML 标签")
|
||||
return problems
|
||||
@@ -1,243 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""在构建宿主上生成 CHV 客机镜像的供应链 lockfile。
|
||||
|
||||
事后校验见 check_lockfile.py(校验 BUILDINFO.versions);
|
||||
本脚本是生成端:读 BUILDINFO.json,以其中的 versions
|
||||
(build.sh 已从客机读回的 dpkg-query / pip freeze,
|
||||
Playwright=1 时另有 chromium 标识)为唯一版本来源,
|
||||
逐项给 contents 清单配版本,写出 lockfile.json:
|
||||
|
||||
{"tag": ..., "image_sha256": ... | null,
|
||||
"generated_at": UTC ISO 时间,
|
||||
"versions": {"dpkg": ["name version", ...],
|
||||
"pip": ["name version", ...],
|
||||
"declared": ["name version", ...]},
|
||||
"unresolved": ["查不到版本的名", ...]}
|
||||
|
||||
版本来源映射(绝不编造版本号,查不到的进 unresolved):
|
||||
- dpkg 包名 → versions.dpkg("name version" 行)
|
||||
- pip 包名 → versions.pip("name==version" 行)
|
||||
- codex / codex-code-mode-host / codex-resources/bwrap
|
||||
→ BUILDINFO.codex_version(随 codex 一起发布)
|
||||
- /app/host.py → BUILDINFO.repo_commit
|
||||
- chromium / chromium-headless-shell → versions.chromium
|
||||
(ms-playwright 缓存目录名)
|
||||
|
||||
注意:旧版逐项查构建宿主的 dpkg/pip,方法不成立——宿主
|
||||
版本与客机版本没有必然关系,且 /app/host.py、codex 三件
|
||||
套在宿主上永远查不到,--verify 永远过不了。本版只读
|
||||
BUILDINFO.versions,不再现场查宿主。
|
||||
|
||||
用法(均在构建宿主上跑):
|
||||
python3 generate_lockfile.py BUILDINFO.json --output lockfile.json
|
||||
python3 generate_lockfile.py --verify lockfile.json \
|
||||
--buildinfo BUILDINFO.json
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
# 非包名的内容项:版本来自 BUILDINFO 的声明字段,不是 dpkg/pip。
|
||||
DECLARED_SOURCES = {
|
||||
"codex": "codex_version",
|
||||
"codex-code-mode-host": "codex_version",
|
||||
"codex-resources/bwrap": "codex_version",
|
||||
"/app/host.py": "repo_commit",
|
||||
"gt": "gt_version",
|
||||
"chv-sync": "repo_commit",
|
||||
# 对齐 Muse P2:nspawn 单元启动器 + broker + 浏览器子代理,
|
||||
# 均是仓库内脚本,版本随 repo_commit(与 chv-sync 同款)。
|
||||
"chv-cell": "repo_commit",
|
||||
"chv-broker": "repo_commit",
|
||||
"chv-browser": "repo_commit",
|
||||
# 对齐 Muse P3 + P1 vsock:authd 三件套、authd/sentinel
|
||||
# 叶子模块与 vsock 通知脚本,均是仓库内脚本,版本随
|
||||
# repo_commit。缺了它们 generate 会记 6 项 unresolved,
|
||||
# verify 与 check 双双失败(717efc4 真机实测)。
|
||||
# round-11 真机修复(层 1):叶子树新增 errors.py 与 stub
|
||||
# __init__.py(authd_init_stub.py),同款 repo_commit 登记,
|
||||
# 缺了它们新镜像的 lockfile 同样会 unresolved。
|
||||
"chv-authd": "repo_commit",
|
||||
"git-credential-chv-authd": "repo_commit",
|
||||
"chv-authd.service": "repo_commit",
|
||||
"cloud/chv/authd.py": "repo_commit",
|
||||
"cloud/chv/sentinel.py": "repo_commit",
|
||||
"cloud/chv/errors.py": "repo_commit",
|
||||
"cloud/chv/__init__.py": "repo_commit",
|
||||
"vsock_notify": "repo_commit",
|
||||
# 对齐 Muse P4:CDP broker 三件在 contents 里,登记自 P4 起
|
||||
# 一直缺失;本轮补上,否则新镜像的 lockfile 仍 unresolved。
|
||||
"chv-cdp-broker": "repo_commit",
|
||||
"chv-browser-guard": "repo_commit",
|
||||
"browser-blocklist": "repo_commit",
|
||||
# 对齐 Muse P6:worker 四件(+ 路径触发器)+ 叶子模块(privsep/safety)。
|
||||
"chv-gitea-worker": "repo_commit",
|
||||
"chv-sync-worker-client": "repo_commit",
|
||||
"chv-gitea-worker.service": "repo_commit",
|
||||
"chv-gitea-worker.path": "repo_commit",
|
||||
"cloud/chv/privsep.py": "repo_commit",
|
||||
"cloud/chv/checkout_detect.py": "repo_commit",
|
||||
"cloud/chv/safety.py": "repo_commit",
|
||||
}
|
||||
|
||||
|
||||
def _index_dpkg(lines):
|
||||
"""dpkg 行 "name version" → {小写名: "name version"}。"""
|
||||
index = {}
|
||||
for line in lines or []:
|
||||
parts = str(line).split(None, 1)
|
||||
if len(parts) == 2:
|
||||
index[parts[0].lower()] = "%s %s" % (parts[0], parts[1])
|
||||
return index
|
||||
|
||||
|
||||
def _index_pip(lines):
|
||||
"""pip freeze 行 "name==version" → {小写名: "name version"}。"""
|
||||
index = {}
|
||||
for line in lines or []:
|
||||
if "==" in str(line):
|
||||
name, version = str(line).split("==", 1)
|
||||
index[name.lower()] = "%s %s" % (name, version.strip())
|
||||
return index
|
||||
|
||||
|
||||
def _index_chromium(lines):
|
||||
"""chromium 标识行 → 键。完整 Chromium 的目录名是
|
||||
chromium-<rev>,headless-shell 的是 chromium_headless_shell-<rev>,
|
||||
各自映射到同名 contents 项。"""
|
||||
index = {}
|
||||
for line in lines or []:
|
||||
text = str(line).strip()
|
||||
if text.startswith("chromium_headless_shell"):
|
||||
index["chromium-headless-shell"] = \
|
||||
"chromium-headless-shell %s" % text
|
||||
elif text:
|
||||
index["chromium"] = "chromium %s" % text
|
||||
return index
|
||||
|
||||
|
||||
def generate(buildinfo_path, output_path):
|
||||
with open(buildinfo_path) as fh:
|
||||
buildinfo = json.load(fh)
|
||||
names = buildinfo.get("contents") or []
|
||||
if not isinstance(names, list):
|
||||
print("BUILDINFO.contents 不是列表", file=sys.stderr)
|
||||
return 2
|
||||
versions = buildinfo.get("versions") or {}
|
||||
dpkg_index = _index_dpkg(versions.get("dpkg"))
|
||||
pip_index = _index_pip(versions.get("pip"))
|
||||
chromium_index = _index_chromium(versions.get("chromium"))
|
||||
dpkg, pip, declared, unresolved = [], [], [], []
|
||||
for name in names:
|
||||
key = name.lower()
|
||||
if name in DECLARED_SOURCES:
|
||||
version = buildinfo.get(DECLARED_SOURCES[name])
|
||||
if version:
|
||||
declared.append("%s %s" % (name, version))
|
||||
else:
|
||||
unresolved.append(name)
|
||||
continue
|
||||
if key in chromium_index:
|
||||
declared.append(chromium_index[key])
|
||||
continue
|
||||
if key in dpkg_index:
|
||||
dpkg.append(dpkg_index[key])
|
||||
continue
|
||||
if key in pip_index:
|
||||
pip.append(pip_index[key])
|
||||
continue
|
||||
unresolved.append(name)
|
||||
lockfile = {
|
||||
"tag": buildinfo.get("tag"),
|
||||
"image_sha256": buildinfo.get("image_sha256")
|
||||
or buildinfo.get("sha256"),
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"versions": {"dpkg": sorted(dpkg), "pip": sorted(pip),
|
||||
"declared": sorted(declared)},
|
||||
"unresolved": sorted(unresolved),
|
||||
}
|
||||
with open(output_path, "w") as fh:
|
||||
json.dump(lockfile, fh, indent=2, sort_keys=True)
|
||||
fh.write("\n")
|
||||
if unresolved:
|
||||
print("警告:%d 项查不到版本,已列入 unresolved:%s"
|
||||
% (len(unresolved), ", ".join(sorted(unresolved))),
|
||||
file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
def verify(lockfile_path, buildinfo_path):
|
||||
with open(lockfile_path) as fh:
|
||||
lockfile = json.load(fh)
|
||||
if buildinfo_path:
|
||||
with open(buildinfo_path) as fh:
|
||||
buildinfo = json.load(fh)
|
||||
if lockfile.get("tag") != buildinfo.get("tag"):
|
||||
print("tag 不一致:lockfile %s != BUILDINFO %s"
|
||||
% (lockfile.get("tag"), buildinfo.get("tag")),
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
# feat-chv-closeup(M4):钉死的 Chromium 修订号须与
|
||||
# lockfile 里记的 chromium-headless-shell 条目一致;
|
||||
# BUILDINFO 没钉(pinned_rev 为空)时不校验。
|
||||
pinned = buildinfo.get("chromium_pinned_rev")
|
||||
if pinned:
|
||||
expected = "chromium_headless_shell-%s" % pinned
|
||||
declared = (lockfile.get("versions") or {}).get(
|
||||
"declared") or []
|
||||
hit = [e for e in declared
|
||||
if str(e).startswith("chromium-headless-shell ")]
|
||||
if not hit or expected not in str(hit[0]):
|
||||
print("Chromium 修订号与钉死值不符:lockfile %s,"
|
||||
"钉死 chromium_headless_shell-%s"
|
||||
% (hit, pinned), file=sys.stderr)
|
||||
return 1
|
||||
versions = lockfile.get("versions") or {}
|
||||
for kind in ("dpkg", "pip"):
|
||||
entries = versions.get(kind)
|
||||
if not entries:
|
||||
print("versions.%s 为空,供应链清单缺失" % kind,
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
for entry in entries:
|
||||
parts = str(entry).split()
|
||||
if len(parts) < 2 or not parts[1]:
|
||||
print("版本为空的条目:%s %r" % (kind, entry),
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
unresolved = lockfile.get("unresolved") or []
|
||||
if unresolved:
|
||||
print("清单不完整,unresolved 非空:%s" % sorted(unresolved),
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
print("lockfile 校验通过:tag=%s dpkg=%d pip=%d declared=%d"
|
||||
% (lockfile.get("tag"), len(versions["dpkg"]),
|
||||
len(versions["pip"]),
|
||||
len(versions.get("declared") or [])))
|
||||
return 0
|
||||
|
||||
|
||||
def main(argv):
|
||||
if "--verify" in argv:
|
||||
lockfile_path = argv[argv.index("--verify") + 1]
|
||||
buildinfo_path = None
|
||||
if "--buildinfo" in argv:
|
||||
buildinfo_path = argv[argv.index("--buildinfo") + 1]
|
||||
return verify(lockfile_path, buildinfo_path)
|
||||
if len(argv) < 2:
|
||||
print("用法:generate_lockfile.py BUILDINFO.json "
|
||||
"--output lockfile.json | --verify lockfile.json "
|
||||
"[--buildinfo BUILDINFO.json]", file=sys.stderr)
|
||||
return 2
|
||||
output_path = None
|
||||
if "--output" in argv:
|
||||
output_path = argv[argv.index("--output") + 1]
|
||||
if not output_path:
|
||||
print("缺少 --output", file=sys.stderr)
|
||||
return 2
|
||||
return generate(argv[1], output_path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
@@ -1,107 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""git credential helper:向 chv-authd 按次取凭证(feat-chv-muse-p3)。
|
||||
|
||||
git credential 协议:git 把 `protocol/host/path`(每行 `key=value`,
|
||||
空行结束)写进本进程 stdin;本进程向 authd 的 Unix socket 发
|
||||
一次 JSON 请求,拿到用户名/口令后按协议回给 git(属性 + 空行)。
|
||||
|
||||
口令只进 git 子进程内存:本进程不写文件、不打日志(拒绝时只
|
||||
报 kind,不含值);`password_expiry_utc` 按释放的 expires_at
|
||||
回给 git,git 在该时间后重新问 helper(不缓存过期凭证)。
|
||||
authd 拒绝 / socket 不通时本进程 exit 1——git 操作随之失败
|
||||
(fail-closed,绝不回落成“无凭证也照推”)。
|
||||
|
||||
用法(chv-sync boot 配好后 git 自动调,不用手工跑):
|
||||
git config --global credential.helper /usr/local/bin/git-credential-chv-authd
|
||||
git config --global credential.useHttpPath true
|
||||
(后者不可少:git 默认不把 path 发给 helper,没有 path 本进程
|
||||
无法推断仓库与操作,会直接拒绝。)
|
||||
socket 路径可用 CHV_AUTHD_SOCKET 覆盖(测试用 /tmp 下的假 socket)。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
|
||||
DEFAULT_SOCKET_PATH = "/run/chv-authd/authd.sock"
|
||||
SOCKET_TIMEOUT_SECONDS = 5
|
||||
|
||||
|
||||
def _read_credential_request():
|
||||
"""读 git credential 协议的请求属性(空行结束)→ dict。"""
|
||||
attrs = {}
|
||||
for line in sys.stdin:
|
||||
line = line.rstrip("\n")
|
||||
if not line:
|
||||
break
|
||||
if "=" in line:
|
||||
key, value = line.split("=", 1)
|
||||
attrs[key.strip()] = value.strip()
|
||||
return attrs
|
||||
|
||||
|
||||
def _ask_authd(socket_path, host, path):
|
||||
"""向 authd 要一次凭证:返回 (username, password, expires_at)。
|
||||
拒绝/通信失败即 RuntimeError(只含 kind,不含值)。"""
|
||||
req = {"connector": "gitea", "host": host, "path": path}
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
sock.settimeout(SOCKET_TIMEOUT_SECONDS)
|
||||
try:
|
||||
sock.connect(socket_path)
|
||||
sock.sendall((json.dumps(req) + "\n").encode("utf-8"))
|
||||
chunks = []
|
||||
while True:
|
||||
data = sock.recv(65536)
|
||||
if not data:
|
||||
break
|
||||
chunks.append(data)
|
||||
if b"\n" in data:
|
||||
break
|
||||
except OSError as exc:
|
||||
raise RuntimeError("authd_unreachable:%s" % exc)
|
||||
finally:
|
||||
sock.close()
|
||||
try:
|
||||
resp = json.loads(b"".join(chunks).decode("utf-8"))
|
||||
except ValueError:
|
||||
raise RuntimeError("authd_bad_response")
|
||||
if not resp.get("ok"):
|
||||
raise RuntimeError("authd_refused:%s"
|
||||
% resp.get("error", "refused"))
|
||||
return (resp["username"], resp["password"],
|
||||
resp.get("expires_at"))
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
"""helper 入口:读协议 → 问 authd → 回属性。拒绝即 exit 1。"""
|
||||
attrs = _read_credential_request()
|
||||
host = attrs.get("host", "")
|
||||
path = attrs.get("path", "")
|
||||
if attrs.get("protocol") != "https" or not host or not path:
|
||||
print("git-credential-chv-authd: 请求不完整(须 protocol=https "
|
||||
"+ host + path),拒绝", file=sys.stderr)
|
||||
return 1
|
||||
if not path.startswith("/"):
|
||||
path = "/" + path
|
||||
# git 的 info/refs 路径带 ?service=... 查询串:Sentinel 的
|
||||
# L7 规则按干净路径匹配,这里先去掉查询串再问 authd。
|
||||
path = path.split("?", 1)[0]
|
||||
socket_path = os.environ.get("CHV_AUTHD_SOCKET") \
|
||||
or DEFAULT_SOCKET_PATH
|
||||
try:
|
||||
username, password, expires_at = _ask_authd(
|
||||
socket_path, host, path)
|
||||
except RuntimeError as exc:
|
||||
print("git-credential-chv-authd: %s" % exc, file=sys.stderr)
|
||||
return 1
|
||||
print("username=%s" % username)
|
||||
print("password=%s" % password)
|
||||
if expires_at:
|
||||
print("password_expiry_utc=%d" % int(expires_at))
|
||||
print()
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
+49
-69
@@ -1252,6 +1252,55 @@ interrupted 且终态任务 cancel 返回 400、心跳后 sweep 不扫刚心跳
|
||||
仍是串行一轮一任务)。两个端点都要 X-API-Key(与其它已认证
|
||||
端点一致);worker 直接写库注册时不需要 key,HTTP 心跳是给有
|
||||
key 的外部 worker 用的另一条路。
|
||||
- 沙箱规模(本轮新增,见 `cloud/tests/test_sandbox_scale.py`):
|
||||
- /tmp 隔离分层表(唯一口径 `sandbox.tmp_isolation_status()`):
|
||||
Docker 任务容器 `--tmpfs /tmp` + `--tmpfs /var/tmp`(容器随
|
||||
`--rm` 消失,/tmp 也随之消失,绝不落宿主 /tmp);且 tmpfs
|
||||
挂载在全部 `-v` 之前,附件 seed 的 `/tmp/codex-seed:ro` 子挂载
|
||||
压在 tmpfs 之上(旧顺序把 tmpfs 放最后,本 VM 无 docker 未
|
||||
证实旧顺序确实遮挡,故写为顺序加严而非已证实 bug)。
|
||||
宿主级 codex workspace-write(bwrap)仍不隔离:挂载由 codex
|
||||
二进制内部决定,host.py 只传 sandboxPolicy 类型、无挂载旋钮,
|
||||
本仓库改不了它;需要 /tmp 隔离的任务请走 docker 执行器,
|
||||
不要拿 Docker 层的隔离给宿主级任务背书。
|
||||
- workers 注册表寻址:workers 表新增 host(worker 的
|
||||
`--host-name`,容器内 worker 绝不能用容器名冒充宿主名)、
|
||||
pid(worker 进程号)、address(CLI `--worker-address`,显式给
|
||||
才有,不含任何凭证);老库幂等补列,老行三列为 NULL 按未知
|
||||
处理。`POST /workers/heartbeat` 同样接受三字段(host 非字符串、
|
||||
pid 非整数、address 非字符串时 400);心跳未给时保留旧值,
|
||||
worker 每次心跳都显式刷新本机 host 与本进程 pid,重启复用
|
||||
同 worker_id 时不残留旧 pid。
|
||||
- 失联容器精确清理(`workers.orphaned_containers()` +
|
||||
`Worker.cleanup_orphaned_containers()`,只写识别与杀容器,
|
||||
不改任务状态):容器名必须精确匹配 `codex-<32 位 hex 任务 id>`,
|
||||
且任务行存在、状态为 running/interrupted、认领记录
|
||||
`tasks.claimed_by_worker`(`_claim` 在 CAS 认领时写)指向
|
||||
心跳过期的 worker,才算失联;基础设施容器(如 codex-postgres)、
|
||||
非任务名、已完成任务、无认领记录、owner 仍存活的一律排除,
|
||||
绝不按 `codex-` 前缀删。默认只杀 pid_verified 的容器(本机 host
|
||||
+ 已知 pid 且 pid 明确已死);远端 host / pid 未知 / 未验证的
|
||||
只列不杀,显式 `verified_only=False` 才杀(仅单机语义已确认
|
||||
时开,跨主机场景绝不能默认开)。旧的
|
||||
`cleanup_stale_containers()`(任务正则兜底、不区分 owner)保留
|
||||
不改行为,供认领记录缺失的老任务容器用;任务状态本身由
|
||||
`sweep_stale_tasks`(心跳过期 -> interrupted)负责,两者分工。
|
||||
本轮只写代码与测试,绝不在本轮动生产容器、不杀任何生产任务。
|
||||
- 长任务天花板与并行验证的代码支撑(不宣称已大规模实测):
|
||||
超时边界抽成纯函数 `worker.docker_run_timeout()`(取较小值、
|
||||
None 语义、no_time_limit 语义、非法值回落,行为与原内联一致);
|
||||
取消 -> interrupted(执行结果绝不覆盖)、超时 -> failed、
|
||||
恢复靠断点续跑(goal-resume / run 续跑),三者状态语义不变。
|
||||
天花板验证方法(待真实大规模跑时按此执行,非本轮已做):
|
||||
睡眠循环任务(每轮 sleep + 一行回复)测轮次/墙钟上限、
|
||||
取消与超时各走一遍终态、owner 崩溃后 sweep + 精确清理各走
|
||||
一遍;多端并行只做 web 自适应跨视口:console 已含 viewport
|
||||
meta 与小屏 media 查询(单测只守基线),手工验证矩阵为
|
||||
320 / 375 / 768 / 1024 / 1440 宽度(无横向滚动、导航可收起、
|
||||
任务列表与详情可读),iOS/Android 原生明确不在本轮范围。
|
||||
未验证项(诚实清单):真实 40 人并行、真实大规模长任务天花板、
|
||||
真实 docker 下 tmpfs 顺序是否曾遮挡 seed、宿主级 bwrap /tmp
|
||||
隔离(本仓库无修复手段,见上)。
|
||||
- 任务按执行器池派发:tasks 表新增 `executor` 列(可空,老库幂等
|
||||
补列);`POST /tasks` 接受可选 executor(local/docker/cloud,
|
||||
非法值 400),未传时为 None 表示任意 worker 可认领;worker
|
||||
@@ -1288,75 +1337,6 @@ interrupted 且终态任务 cancel 返回 400、心跳后 sweep 不扫刚心跳
|
||||
错误文本或返回结果。worker CLI 的 `--executor` 新增 cloud 选项,
|
||||
构造 CloudExecutor;worker 的 run_one 走 dispatch_task,
|
||||
CloudExecutor 的 dispatch_task/run_task 直接委托给 execute。
|
||||
- CloudHypervisorExecutor(生产级 v1,`cloud/chv/`,旧路径
|
||||
`cloud/sandbox.py` 只留兼容 re-export,执行器名
|
||||
`cloud-hypervisor`,绝不用 `cloud` 缩写):
|
||||
单任务单 VM:一个任务一个 VM、独立内核/文件系统/网络,
|
||||
任务结束销毁,租户之间无共享路径、无东西向互访;VM 内
|
||||
agent 可有 root,但只能影响本 VM。密钥按现有惯例走运行
|
||||
时环境变量名注入,不进镜像、不进代码、不进任务 prompt
|
||||
落库(SSH argv 里只有 key 文件路径,绝无 key 值)。
|
||||
唤醒是 API 模式:preflight(见下)→ 容量门 → 磁盘配额门
|
||||
→ 每任务磁盘供给(copy 模式复制基础镜像)→ 只以
|
||||
`--api-socket` 起 VMM → 等 `vmm.ping` 通 → `PUT vm.create`
|
||||
(VmConfig)→ `PUT vm.boot` → 等 guest SSH 就绪(默认
|
||||
120 秒)才跑 host.py;超时关 VM 并报分类失败。直接启动
|
||||
命令(`build_vm_command`)只作手动等价写法,执行器不走它。
|
||||
暂停/快照/休眠:`pause_vm`/`resume_vm` 走 ch-remote;
|
||||
`restore_vm` 先 restore 再显式 resume(恢复后默认 paused,
|
||||
不 resume 不算醒);CHV 无原生 hibernate 端点,`hibernate_vm`
|
||||
用 pause + snapshot + shutdown 等价实现,恢复走 restore。
|
||||
预热池 `WarmPool`:预热条目是一次性干净快照 VM,acquire
|
||||
消耗、release 一律销毁,绝不跨任务复用脏 VM;池空回落
|
||||
冷启动,绝不降级 Docker/Local;池状态另落
|
||||
`state_dir/warm-pool.json` registry,worker 重启后载入的
|
||||
条目一律当孤儿、只能 `reconcile_orphans()` 清理,绝不
|
||||
重新发给任务,registry 损坏时 quarantine 成 `.corrupt` 并
|
||||
从空池开始。SSH host key 按 VM 钉死(#216 的
|
||||
`StrictHostKeyChecking=no` + `/dev/null` 已废除):每 VM
|
||||
在 `state_dir/<vm>/known_hosts` 有独立文件,首次启动经
|
||||
ssh-keyscan 在本机回环端口捕获 key 写入该文件(首次信任
|
||||
TOFU,只防首次之后的替换,不验证首次本身),此后 SSH
|
||||
一律 `StrictHostKeyChecking=yes` + 该文件校验;客机用户
|
||||
(`CHV_SSH_USER`,默认 root,显式空串直接拒绝)与端口
|
||||
(`CHV_SSH_PORT`,默认 2222,1-65535)为显式配置。
|
||||
`execute()` 只读已运行 VM:未运行报 `vm_not_running`
|
||||
分类失败,不为一次 status 起 VM。长任务中途死亡
|
||||
(传输异常、或结果已 failed 且登记 VMM 进程已死)判
|
||||
`interrupted` 而非笼统 failed;中途心跳经评估未做
|
||||
(host.py 单次 SSH 长回合无客内进度信号可采,另起轮询
|
||||
只能证明 VM 活着)。关闭与清理:任务结束/
|
||||
失败/取消时 `shutdown_vm`(ch-remote shutdown + 杀进程 +
|
||||
删 socket),`cleanup_stale_vms` 处理已死/已关/socket 残留
|
||||
三种情况。审计:VM 的 create/boot/ready/pause/resume/
|
||||
snapshot/restore/shutdown/destroy/failed/recycled 均经
|
||||
audit_callback 发出,worker 接线后转 AuthStore 审计;失败
|
||||
带 kind 分类(kvm_missing/image_missing/guest_not_ready/
|
||||
api_timeout/host_capacity_exceeded/disk_quota_exceeded 等),
|
||||
不笼统 failed。资源与成本:每 VM 的 CPU/内存/磁盘配额、
|
||||
超时、最大轮次(goal 上限 100)复用 worker 现有 quota 语义,
|
||||
不另起一套计费;空闲休眠省成本;autoscaler 按 executor 过
|
||||
滤队列深度并透传 `--executor` 给拉起的 worker。网络与治理:
|
||||
VM 出口默认必须过 CHV_EGRESS_PROXY 白名单代理(未设且未
|
||||
显式允许直连时 preflight 直接失败);审批门仍走任务的
|
||||
approval 与 server 的 merge_gate,VM 内 agent 不能自批。
|
||||
配置(环境变量,见该类 docstring 全表):CHV_BINARY、
|
||||
CHV_KERNEL、CHV_ROOTFS(或 CHV_IMAGE 别名)、CHV_CPU、
|
||||
CHV_MEM、CHV_STATE_DIR、CHV_SSH_KEY/USER/PORT、CHV_CMDLINE、
|
||||
CHV_NET(直接命令用)/CHV_NET_JSON(API 用)、CHV_REMOTE、
|
||||
CHV_MAX_VMS、CHV_DISK_MODE(copy 默认/reuse 仅实验)、
|
||||
CHV_DISK_GB、CHV_EGRESS_PROXY、CHV_TASK_TIMEOUT、
|
||||
CHV_SNAPSHOT_URL、CHV_WARM_POOL_SIZE、CHV_API_TIMEOUT、
|
||||
CHV_READY_TIMEOUT/INTERVAL。默认关闭:server 默认执行器仍
|
||||
是 docker,worker 默认 local,CHV 只在任务创建时显式选
|
||||
`executor=cloud-hypervisor` 且有 CHV worker 认领时才跑;
|
||||
任务内 LLM 绝不能发现/调用它(调度层选择,只在创建层可见:
|
||||
POST /tasks 可选值、本文档、控制台选项,不新增任务内 MCP/
|
||||
工具)。真实验证边界:未做真实 VM 启动验证——生产主机
|
||||
本身是 CVM、无 /dev/kvm、无嵌套虚拟化,本 VM 容器内同样
|
||||
无 /dev/kvm;真 VM、快照恢复、压测、安全审计均在
|
||||
`docs/chv-production-readiness.md` 里标为待办,绝不声称
|
||||
已生产可用。客机镜像内容清单与构建骨架见 `chv-guest/`。
|
||||
- 自动 PR(默认关闭,显式触发,`POST /tasks/{id}/pr`,需项目
|
||||
maintainer 以上权限 approve_merge):仅当任务状态为 merged 时
|
||||
可调(其它状态 409;任务不存在 404;权限不足 403)。逻辑按顺序
|
||||
|
||||
@@ -0,0 +1,145 @@
|
||||
"""审批超时率自动调阈(G7 运维指标,v1.1 契约口径)。
|
||||
|
||||
只依赖标准库与 cloud.approvals 的建表函数,不 import server /
|
||||
worker,避免循环 import。
|
||||
|
||||
口径(与 v1.1 冻结条一致):
|
||||
- 默认审批超时仍是 10 分钟(600 秒)。注意:这是本调阈口径
|
||||
的默认值,不是 approvals.py elicitation 队列的 3600 秒
|
||||
默认值——两条审批线各有默认值,本模块绝不改写后者。
|
||||
- 超时率 = 已终态审批项中 status='expired' 的占比;已终态 =
|
||||
approved + denied + expired。pending 不计入分母(还没结果
|
||||
的项不能算超时也不能算准时);未扫荡的过期 pending 同样
|
||||
不计入——扫荡发生在审批列表与运维端点,统计前应先扫荡。
|
||||
- 超时率 > 20% 时建议调到 30 分钟(1800 秒);< 5% 时不动;
|
||||
5%~20% 之间同样不动(只升不降,降阈须人工决定,本模块
|
||||
绝不自动把超时缩回去)。边界值本身不动:>0.20 才升,
|
||||
==0.20 与 ==0.05 都按不动处理。
|
||||
- 本模块只算建议并记日志,不自动改写任何默认值或已建审批
|
||||
项的 expires_at:调阈生效须运维按建议改配置后重启/重载,
|
||||
建议值进 /metrics 与运维端点,审计靠日志行(见
|
||||
evaluate_approval_timeout 的 info 日志)。
|
||||
"""
|
||||
|
||||
import logging
|
||||
|
||||
from cloud.approvals import ensure_approvals_table
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_TIMEOUT_SECONDS = 600
|
||||
RAISED_TIMEOUT_SECONDS = 1800
|
||||
RAISE_THRESHOLD = 0.20
|
||||
KEEP_THRESHOLD = 0.05
|
||||
|
||||
|
||||
def timeout_rate(expired_count, decided_count):
|
||||
"""超时率:已终态为 0 时回 None(不是 0%,是算不出来)。"""
|
||||
try:
|
||||
decided = int(decided_count)
|
||||
expired = int(expired_count)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
if decided <= 0:
|
||||
return None
|
||||
if expired < 0:
|
||||
expired = 0
|
||||
if expired > decided:
|
||||
expired = decided
|
||||
return expired / decided
|
||||
|
||||
|
||||
def recommend_approval_timeout(expired_count, decided_count,
|
||||
current_seconds=DEFAULT_TIMEOUT_SECONDS):
|
||||
"""纯计算函数:输入计数,输出建议(无 I/O,可单测三档)。
|
||||
|
||||
返回键:expired_count、decided_count、timeout_rate、
|
||||
current_seconds、recommended_seconds、action(raise_to_30m
|
||||
或 keep)、reason。current_seconds 非法时按默认值计,不抛。
|
||||
"""
|
||||
try:
|
||||
current = int(current_seconds)
|
||||
except (TypeError, ValueError):
|
||||
current = DEFAULT_TIMEOUT_SECONDS
|
||||
if current <= 0:
|
||||
current = DEFAULT_TIMEOUT_SECONDS
|
||||
rate = timeout_rate(expired_count, decided_count)
|
||||
if rate is not None and rate > RAISE_THRESHOLD:
|
||||
action = "raise_to_30m"
|
||||
recommended = RAISED_TIMEOUT_SECONDS
|
||||
reason = ("approval timeout rate %.4f > %.2f, "
|
||||
"recommend %d seconds" % (rate, RAISE_THRESHOLD,
|
||||
recommended))
|
||||
else:
|
||||
action = "keep"
|
||||
recommended = current
|
||||
if rate is None:
|
||||
reason = "no decided approvals, keep %d seconds" % current
|
||||
else:
|
||||
reason = ("approval timeout rate %.4f <= %.2f, "
|
||||
"keep %d seconds" % (rate, RAISE_THRESHOLD,
|
||||
current))
|
||||
return {
|
||||
"expired_count": int(expired_count or 0),
|
||||
"decided_count": int(decided_count or 0),
|
||||
"timeout_rate": rate,
|
||||
"current_seconds": current,
|
||||
"recommended_seconds": recommended,
|
||||
"action": action,
|
||||
"reason": reason,
|
||||
}
|
||||
|
||||
|
||||
def approval_timeout_stats(conn, project_ids=None):
|
||||
"""从 task_approvals 统计已终态与超时数(纯读,不扫荡)。
|
||||
|
||||
project_ids 为 None 时全库统计;否则只计这些项目(/metrics
|
||||
的成员项目口径)。表不存在时先幂等建表(空表即零统计)。
|
||||
"""
|
||||
ensure_approvals_table(conn)
|
||||
query = ("SELECT COUNT(*), COALESCE(SUM(CASE WHEN status = 'expired'"
|
||||
" THEN 1 ELSE 0 END), 0), COALESCE(SUM(CASE WHEN"
|
||||
" status = 'pending' THEN 1 ELSE 0 END), 0)"
|
||||
" FROM task_approvals WHERE status IN"
|
||||
" ('approved', 'denied', 'expired', 'pending')")
|
||||
args = []
|
||||
if project_ids is not None:
|
||||
ids = sorted(set(project_ids))
|
||||
if not ids:
|
||||
return {"decided_count": 0, "expired_count": 0,
|
||||
"pending_count": 0}
|
||||
query += " AND project_id IN (%s)" % ", ".join(
|
||||
"?" for _ in ids)
|
||||
args.extend(ids)
|
||||
row = conn.execute(query, args).fetchone()
|
||||
decided = int(row[0] or 0) - int(row[2] or 0)
|
||||
return {
|
||||
"decided_count": decided,
|
||||
"expired_count": int(row[1] or 0),
|
||||
"pending_count": int(row[2] or 0),
|
||||
}
|
||||
|
||||
|
||||
def evaluate_approval_timeout(conn, current_seconds=DEFAULT_TIMEOUT_SECONDS,
|
||||
project_ids=None, log=None):
|
||||
"""统计 + 建议 + 日志的一站式运维入口。
|
||||
|
||||
返回 approval_timeout_stats 与 recommend_approval_timeout
|
||||
的合并 dict,并写一条 info 日志(log 未给时用模块 logger)。
|
||||
调用方在需要“过期 pending 也算超时”时须先调 approvals
|
||||
的 expire_overdue 扫荡——本函数自己不写库。
|
||||
"""
|
||||
stats = approval_timeout_stats(conn, project_ids=project_ids)
|
||||
result = recommend_approval_timeout(
|
||||
stats["expired_count"], stats["decided_count"],
|
||||
current_seconds=current_seconds)
|
||||
result.update(stats)
|
||||
(log or logger).info(
|
||||
"approval_timeout metric: decided=%d expired=%d pending=%d "
|
||||
"rate=%s action=%s recommended_seconds=%d",
|
||||
result["decided_count"], result["expired_count"],
|
||||
result["pending_count"],
|
||||
("%.4f" % result["timeout_rate"]
|
||||
if result["timeout_rate"] is not None else "unknown"),
|
||||
result["action"], result["recommended_seconds"])
|
||||
return result
|
||||
+19
-26
@@ -64,33 +64,29 @@
|
||||
- item/tool/call 与 account/chatgptAuthTokens/refresh:前者是
|
||||
动态工具执行,后者是凭证刷新,都不是用户审批,排除。
|
||||
|
||||
本模块只依赖标准库,不 import cloud 其他模块,避免循环
|
||||
import;服务端与未来 worker/hook 共用。占位符用 ?,建表与
|
||||
列补齐写法与 peer_messages 同款(db 适配层负责翻译)。
|
||||
本模块只依赖标准库与同为标准库叶子的 cloud.timeouts,
|
||||
不 import 其他 cloud 模块,避免循环 import;服务端与未来
|
||||
worker/hook 共用。占位符用 ?,建表与列补齐写法与
|
||||
peer_messages 同款(db 适配层负责翻译)。
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from cloud.peer_messages import row_get
|
||||
from cloud.timeouts import APPROVAL_DEFAULT_SECONDS as _APPROVAL_DEFAULT
|
||||
from cloud.timeouts import expires_at_for as _expires_at_for
|
||||
from cloud.timeouts import resolve_module_timeout as _resolve_timeout
|
||||
|
||||
KINDS = ("elicitation", "command", "file_change", "permissions",
|
||||
"user_input")
|
||||
DECISIONS = ("approved", "denied")
|
||||
DEFAULT_TIMEOUT_SECONDS = 3600
|
||||
DEFAULT_TIMEOUT_SECONDS = _APPROVAL_DEFAULT
|
||||
MAX_SUMMARY_CHARS = 2000
|
||||
MAX_DETAIL_CHARS = 20000
|
||||
|
||||
|
||||
def row_get(row, name):
|
||||
"""行取值容错:sqlite3.Row 缺列报 IndexError、dict 缺键报
|
||||
KeyError、空行报 TypeError,一律回 None(与 peer_messages
|
||||
的同名函数同款,模块独立所以这里自带一份)。"""
|
||||
try:
|
||||
return row[name]
|
||||
except (KeyError, IndexError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def ensure_approvals_table(conn):
|
||||
"""幂等建审批队列表。调用方负责 commit(服务端入口在写
|
||||
操作后统一 commit;本函数自己不 commit,避免半截事务)。"""
|
||||
@@ -167,12 +163,15 @@ def approval_to_dict(row):
|
||||
|
||||
def create_approval(conn, task_id, project_id, kind="elicitation",
|
||||
summary="", detail=None, requested_by=None,
|
||||
timeout_seconds=DEFAULT_TIMEOUT_SECONDS, now=None):
|
||||
timeout_seconds=None, now=None):
|
||||
"""创建一条待审批项,返回 dict。非法输入抛 ValueError,
|
||||
调用方据此回 400;成功时 commit。
|
||||
|
||||
timeout_seconds:None 用默认值;0 表示不过期(只允许
|
||||
模块级调用,服务端入口另有限制);负数非法。"""
|
||||
timeout_seconds:None 用默认值(cloud/timeouts.py 的
|
||||
环境变量感知默认值,未设环境变量时为 3600;签名默认
|
||||
故意写 None 而不是 3600,避免默认值在 import 时绑定死、
|
||||
环境覆盖层传不进去);0 表示不过期(只允许模块级调用,
|
||||
服务端入口另有限制);负数非法。"""
|
||||
if kind not in KINDS:
|
||||
raise ValueError("kind must be one of: " + ", ".join(KINDS))
|
||||
if not isinstance(summary, str) or not summary.strip():
|
||||
@@ -180,17 +179,11 @@ def create_approval(conn, task_id, project_id, kind="elicitation",
|
||||
summary = summary.strip()
|
||||
if len(summary) > MAX_SUMMARY_CHARS:
|
||||
raise ValueError("summary too long")
|
||||
if timeout_seconds is None:
|
||||
timeout_seconds = DEFAULT_TIMEOUT_SECONDS
|
||||
if not isinstance(timeout_seconds, (int, float)) \
|
||||
or isinstance(timeout_seconds, bool) \
|
||||
or timeout_seconds < 0:
|
||||
raise ValueError("timeout_seconds invalid")
|
||||
# 不传 default:None 时走 timeouts 的环境变量感知默认值。
|
||||
timeout_seconds = _resolve_timeout(timeout_seconds)
|
||||
detail_str = _detail_to_str(detail)
|
||||
now = time.time() if now is None else now
|
||||
expires_at = None
|
||||
if timeout_seconds:
|
||||
expires_at = now + timeout_seconds
|
||||
expires_at = _expires_at_for(now, timeout_seconds)
|
||||
item_id = uuid.uuid4().hex
|
||||
ensure_approvals_table(conn)
|
||||
conn.execute(
|
||||
|
||||
@@ -0,0 +1,298 @@
|
||||
"""Architecture-conformance checks for the control plane and for
|
||||
skill-driven projects (software-engineering skill, G2 frozen artefacts).
|
||||
|
||||
Zero third-party dependencies; standard library only. This module
|
||||
imports no ``cloud.*`` module, so it can never violate the layer bans
|
||||
it enforces, and it can be reused verbatim against a skill project's
|
||||
own directory tree (see ``check_skill_project``).
|
||||
|
||||
The four checks, each with a stated rule and a stated reason:
|
||||
|
||||
1. Contract version (``check_contract_version``). Rule mirrors the
|
||||
skill's contract-freeze change protocol: after freeze, a version
|
||||
change is allowed only with a coordinator-approved change record
|
||||
that names backward compatibility, the affected clients, and (for
|
||||
a breaking change) a regression task per affected client; the
|
||||
version must advance by exactly one. Reason: a silent contract
|
||||
drift is how parallel client threads diverge; the protocol is the
|
||||
only sanctioned path, so the check enforces the protocol instead
|
||||
of merely documenting it.
|
||||
|
||||
2. Forbidden directories (``is_within`` / ``check_task_writes``).
|
||||
Rule reuses the two directory definitions that already exist and
|
||||
invents no third one: on the control plane a task's own directory
|
||||
is its tenant work directory (``worker._tenant_dirs`` ``work``);
|
||||
in a skill project a client thread's own directory is its
|
||||
``apps/<client>/`` tree. Any write outside the own directory --
|
||||
including ``..`` escapes and symlink escapes, both resolved before
|
||||
comparison -- is a violation. Reason: cross-directory writes are
|
||||
how one tenant/client corrupts another's worktree.
|
||||
|
||||
3. Forbidden imports (``check_layer_imports``). Bans were chosen by
|
||||
auditing the current tree, not by wishful layering; every ban
|
||||
below passes on the baseline, and the repo-level test pins that:
|
||||
|
||||
- Foundation modules (``db``, ``models``, ``queue``,
|
||||
``logging_config``) must not import orchestration/service
|
||||
modules (``server``, ``worker``, ``pr_exec``, ``webhooks``,
|
||||
``autoscaler``, ``cost``). Reason: the foundation is imported
|
||||
by everything; an upward import is a cycle waiting to happen
|
||||
and drags service state into pure helpers.
|
||||
- The execution module (``sandbox``) must not import persistence
|
||||
or orchestration (``db``, ``server``, ``worker``, ``cost``,
|
||||
``auth``, ``pr``, ``pr_exec``, ``webhooks``). Reason: executors
|
||||
must stay swappable and testable with fake runners; decisions
|
||||
about mounts and networks arrive as arguments, not by reading
|
||||
the database.
|
||||
- No cloud module except the driver-adjacent ones (``db``,
|
||||
``backup``, ``migrations``) may open a database connection
|
||||
directly (``sqlite3.connect`` or any ``psycopg*`` import).
|
||||
Service modules must obtain connections through
|
||||
``cloud.db.connect`` or a Store. Reason: the backend switch
|
||||
(sqlite/postgres) lives in exactly one place; a direct driver
|
||||
call bypasses it. Deliberately NOT banned: ``server.py``'s use
|
||||
of ``sqlite3.Row`` / ``sqlite3.IntegrityError`` as type adapters
|
||||
on connections obtained from ``cloud.db`` -- that is adaptation,
|
||||
not a bypass, and banning it would fail the baseline for no
|
||||
safety gain. The test asserts the narrow form (no ``connect``
|
||||
call, no driver import) rather than the broad form.
|
||||
|
||||
Only static ``ast`` imports are seen; dynamic imports
|
||||
(``importlib``) are out of scope and are noted, not claimed.
|
||||
|
||||
4. Frozen stack (``check_frozen_stack``). Rule mirrors the skill's
|
||||
G2 rule "frozen stack must not be swapped mid-flight": a client
|
||||
whose stack entry changes, or disappears, after freeze fails
|
||||
unless an approved change record covers it; adding a brand-new
|
||||
client (no frozen entry) is an addition, not a swap, and passes.
|
||||
Reason: the shared core is built against the frozen table; a
|
||||
mid-flight swap silently invalidates it.
|
||||
|
||||
All checkers return plain data (dicts/lists), never raise for a
|
||||
violation, so CI can report every violation in one run instead of
|
||||
stopping at the first.
|
||||
"""
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
FOUNDATION_MODULES = frozenset({"db", "models", "queue", "logging_config"})
|
||||
ORCHESTRATION_MODULES = frozenset(
|
||||
{"server", "worker", "pr_exec", "webhooks", "autoscaler", "cost"})
|
||||
EXECUTION_MODULES = frozenset({"sandbox"})
|
||||
EXECUTION_BANNED = frozenset(
|
||||
{"db", "server", "worker", "cost", "auth", "pr", "pr_exec", "webhooks"})
|
||||
DRIVER_ADJACENT_MODULES = frozenset({"db", "backup", "migrations"})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- contract
|
||||
|
||||
def parse_version(text):
|
||||
"""Parse ``v1`` / ``1`` to int; anything else is None (unparseable
|
||||
is a failure, never silently treated as the frozen version)."""
|
||||
if text is None:
|
||||
return None
|
||||
text = str(text).strip()
|
||||
if text.startswith("v"):
|
||||
text = text[1:]
|
||||
if text.isdigit():
|
||||
return int(text)
|
||||
return None
|
||||
|
||||
|
||||
def _change_record_problems(change, breaking):
|
||||
"""Missing pieces of a contract-change record, per the protocol."""
|
||||
problems = []
|
||||
if not isinstance(change, dict):
|
||||
return ["change record missing (frozen contract changed without "
|
||||
"a coordinator-approved change record)"]
|
||||
if not change.get("approved"):
|
||||
problems.append("change not coordinator-approved")
|
||||
if change.get("backward_compatible") is None:
|
||||
problems.append("backward compatibility not declared")
|
||||
affected = change.get("affected_clients")
|
||||
if not affected or not isinstance(affected, list):
|
||||
problems.append("affected clients not listed")
|
||||
if breaking and not change.get("regression_task"):
|
||||
problems.append("breaking change without a regression task")
|
||||
return problems
|
||||
|
||||
|
||||
def check_contract_version(frozen, current, change=None):
|
||||
"""Return ``{"ok": bool, "reasons": [...]}``.
|
||||
|
||||
Equal versions pass regardless of ``change``. A different version
|
||||
passes only when it is exactly frozen + 1 and ``change`` satisfies
|
||||
the protocol; a breaking change is one where
|
||||
``backward_compatible`` is False.
|
||||
"""
|
||||
frozen_n = parse_version(frozen)
|
||||
current_n = parse_version(current)
|
||||
if frozen_n is None or current_n is None:
|
||||
return {"ok": False,
|
||||
"reasons": ["unparseable contract version: frozen=%r "
|
||||
"current=%r" % (frozen, current)]}
|
||||
if current_n == frozen_n:
|
||||
return {"ok": True, "reasons": []}
|
||||
reasons = []
|
||||
if current_n != frozen_n + 1:
|
||||
reasons.append("contract version must advance by exactly one "
|
||||
"(frozen v%d -> v%d, not v%d)"
|
||||
% (frozen_n, frozen_n + 1, current_n))
|
||||
breaking = isinstance(change, dict) and \
|
||||
change.get("backward_compatible") is False
|
||||
reasons.extend(_change_record_problems(change, breaking))
|
||||
return {"ok": not reasons, "reasons": reasons}
|
||||
|
||||
|
||||
# ------------------------------------------------------------- frozen stack
|
||||
|
||||
def check_frozen_stack(frozen, current, change=None):
|
||||
"""``frozen``/``current`` map client -> stack string.
|
||||
|
||||
Changed or removed entries fail without an approved change record
|
||||
(same shape as the contract-change record, minus the version
|
||||
rules); added clients pass as additions.
|
||||
"""
|
||||
frozen = frozen or {}
|
||||
current = current or {}
|
||||
reasons = []
|
||||
swapped = sorted(c for c in frozen
|
||||
if c not in current or current[c] != frozen[c])
|
||||
if not swapped:
|
||||
return {"ok": True, "reasons": [], "swapped": []}
|
||||
problems = _change_record_problems(change, breaking=False)
|
||||
if problems:
|
||||
reasons.append("frozen stack changed for %s without an approved "
|
||||
"change record: %s" % (swapped, "; ".join(problems)))
|
||||
return {"ok": not reasons, "reasons": reasons, "swapped": swapped}
|
||||
|
||||
|
||||
# ------------------------------------------------------------ directories
|
||||
|
||||
def is_within(root, candidate):
|
||||
"""True iff ``candidate`` resolves inside ``root`` (symlinks and
|
||||
``..`` resolved first, so both escape routes are caught)."""
|
||||
try:
|
||||
root_p = Path(root).resolve()
|
||||
cand_p = Path(candidate).resolve()
|
||||
except (OSError, RuntimeError):
|
||||
return False
|
||||
return cand_p == root_p or root_p in cand_p.parents
|
||||
|
||||
|
||||
def check_task_writes(own_dir, paths):
|
||||
"""Return the subset of ``paths`` written outside ``own_dir``."""
|
||||
return [p for p in paths if not is_within(own_dir, p)]
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- imports
|
||||
|
||||
def scan_imports(source):
|
||||
"""Static import scan of one module's source.
|
||||
|
||||
Returns ``{"cloud": set, "driver_connect": bool, "driver_import": bool}``:
|
||||
``cloud`` holds the ``cloud.<name>`` top-level names imported;
|
||||
``driver_connect`` is True on any ``sqlite3.connect`` call;
|
||||
``driver_import`` is True on any ``psycopg*`` import. A bare
|
||||
``import sqlite3`` (Row/exception adapters) sets neither flag.
|
||||
"""
|
||||
cloud_imports = set()
|
||||
driver_connect = False
|
||||
driver_import = False
|
||||
try:
|
||||
tree = ast.parse(source)
|
||||
except SyntaxError:
|
||||
return {"cloud": cloud_imports,
|
||||
"driver_connect": driver_connect,
|
||||
"driver_import": driver_import,
|
||||
"parse_error": True}
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
if alias.name == "sqlite3":
|
||||
continue
|
||||
if alias.name.startswith("psycopg"):
|
||||
driver_import = True
|
||||
if alias.name == "cloud" or alias.name.startswith("cloud."):
|
||||
parts = alias.name.split(".")
|
||||
if len(parts) > 1:
|
||||
cloud_imports.add(parts[1])
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
if node.module and node.module.startswith("psycopg"):
|
||||
driver_import = True
|
||||
if node.module == "cloud" or (node.module or "").startswith("cloud."):
|
||||
if node.module == "cloud":
|
||||
for alias in node.names:
|
||||
cloud_imports.add(alias.name)
|
||||
else:
|
||||
cloud_imports.add(node.module.split(".")[1])
|
||||
elif isinstance(node, ast.Call):
|
||||
func = node.func
|
||||
if isinstance(func, ast.Attribute) and func.attr == "connect":
|
||||
value = func.value
|
||||
if isinstance(value, ast.Name) and value.id == "sqlite3":
|
||||
driver_connect = True
|
||||
return {"cloud": cloud_imports,
|
||||
"driver_connect": driver_connect,
|
||||
"driver_import": driver_import}
|
||||
|
||||
|
||||
def check_layer_imports(sources):
|
||||
"""``sources`` maps module name -> source text. Returns a list of
|
||||
violation dicts; empty means the layer rules hold."""
|
||||
violations = []
|
||||
for module, source in sorted(sources.items()):
|
||||
scanned = scan_imports(source)
|
||||
if scanned.get("parse_error"):
|
||||
violations.append({"module": module, "rule": "parse",
|
||||
"detail": "source does not parse; "
|
||||
"imports cannot be verified"})
|
||||
continue
|
||||
if module in FOUNDATION_MODULES:
|
||||
bad = scanned["cloud"] & ORCHESTRATION_MODULES
|
||||
if bad:
|
||||
violations.append({"module": module,
|
||||
"rule": "foundation_must_not_import_orchestration",
|
||||
"detail": sorted(bad)})
|
||||
if module in EXECUTION_MODULES:
|
||||
bad = scanned["cloud"] & EXECUTION_BANNED
|
||||
if bad:
|
||||
violations.append({"module": module,
|
||||
"rule": "execution_must_not_import_persistence_or_orchestration",
|
||||
"detail": sorted(bad)})
|
||||
if module not in DRIVER_ADJACENT_MODULES:
|
||||
if scanned["driver_connect"] or scanned["driver_import"]:
|
||||
violations.append({"module": module,
|
||||
"rule": "service_must_not_open_driver_connection_directly",
|
||||
"detail": "use cloud.db.connect or a Store"})
|
||||
return violations
|
||||
|
||||
|
||||
def scan_cloud_package(repo_root):
|
||||
"""Run ``check_layer_imports`` over every ``cloud/*.py`` in a repo
|
||||
checkout (tests and caches excluded). This is the repo-level test
|
||||
that makes a layer violation fail CI."""
|
||||
cloud_dir = Path(repo_root) / "cloud"
|
||||
sources = {}
|
||||
if cloud_dir.is_dir():
|
||||
for path in sorted(cloud_dir.glob("*.py")):
|
||||
sources[path.stem] = path.read_text(encoding="utf-8")
|
||||
return check_layer_imports(sources)
|
||||
|
||||
|
||||
def check_skill_project(project_root, frozen_version, current_version,
|
||||
frozen_stack, current_stack, own_dir, writes,
|
||||
change=None):
|
||||
"""One-call conformance for a skill project: contract version,
|
||||
frozen stack, and directory writes together. Returns a dict with
|
||||
one entry per check plus an overall ``ok``."""
|
||||
contract = check_contract_version(frozen_version, current_version,
|
||||
change=change)
|
||||
stack = check_frozen_stack(frozen_stack, current_stack, change=change)
|
||||
outside = check_task_writes(own_dir, writes)
|
||||
directory = {"ok": not outside,
|
||||
"reasons": (["writes outside own directory: %s" % outside]
|
||||
if outside else [])}
|
||||
return {"ok": contract["ok"] and stack["ok"] and directory["ok"],
|
||||
"contract": contract, "stack": stack, "directory": directory}
|
||||
@@ -0,0 +1,366 @@
|
||||
"""Lifecycle hook policy for the task-attempt ledger (audit gap 3
|
||||
follow-up: "which dispatch counts as an attempt, and who judges
|
||||
``is_defect``").
|
||||
|
||||
This module is the CANONICAL policy text. The same policy is
|
||||
restated (shorter) in ``docs/arch-conformance-metrics.md`` and
|
||||
pinned by ``cloud/tests/test_attempt_hook.py``; if the three ever
|
||||
disagree, this module wins and the other two must be fixed.
|
||||
|
||||
The problem it solves: audit gap 3 built the ledger but
|
||||
deliberately left the writer unconnected, because a naive writer
|
||||
-- one row per worker execution -- misclassifies exactly twice:
|
||||
it counts a platform auto-resume as an attempt, and it counts a
|
||||
transient retry as a defect. This module fixes the writer's
|
||||
trigger, not the ledger's math: ``aggregate_attempts`` is
|
||||
untouched by the hook (its only change in this track is the
|
||||
correction-row rule in ``attempt_ledger.py``), and the cost
|
||||
dual-column code (``cloud.cost``) is untouched entirely.
|
||||
|
||||
POLICY (each rule is enforced in code below, not just prose):
|
||||
|
||||
1. One counted dispatch = one row, at most. A dispatch that the
|
||||
hook counts writes EXACTLY ONE ledger row, when its outcome is
|
||||
known (terminal ``completed`` / ``failed`` / ``interrupted``,
|
||||
or the goal-conversion case in ``worker.run_one``), with
|
||||
``is_defect=False`` always (rule 6). A dispatch that the hook
|
||||
does not count writes NO row at all -- not even an
|
||||
``interrupted`` row -- so an uncounted dispatch can never
|
||||
create the ledger table as a side effect.
|
||||
|
||||
2. What counts as one attempt -- a logical agent round, not a
|
||||
worker execution:
|
||||
- First dispatch of a task: counts (``first_or_human_retry``).
|
||||
It is the task's first agent round.
|
||||
- Human ``/retry`` of a failed/interrupted/merge-failed task:
|
||||
counts (same kind). A retry is a human-initiated new round:
|
||||
new model work, new result, new cost. The server endpoint
|
||||
leaves no marker behind, so the worker cannot distinguish a
|
||||
retry from a first dispatch -- and does not need to, because
|
||||
both count.
|
||||
- Human ``/continue`` with a new message: counts
|
||||
(``human_continue``). The message starts a new round on the
|
||||
same session thread. Non-goal tasks and goal tasks opening a
|
||||
NEW goal loop (``_resume_kind == "new-goal"``) are both here.
|
||||
- Human breakpoint resume (``/goal-resume``, or ``/continue``
|
||||
on an old-verify goal task, i.e. ``_resume_kind ==
|
||||
"breakpoint"``): counts (``breakpoint_resume``) -- but as
|
||||
the SAME logical round's row, not as an extra attempt. The
|
||||
fragment that was interrupted wrote only an excluded
|
||||
``interrupted`` row (gap 3 rule), so the resume's terminal
|
||||
row is the round's single counted row. A resume that ends
|
||||
interrupted again writes only another excluded row: the
|
||||
round is still pending, and still contributes zero counted
|
||||
attempts. Rationale: one goal loop = one attempt, however
|
||||
many executions the loop took; the interruption history
|
||||
stays visible in the ``interrupted`` rows and in
|
||||
``auto_resume_count`` / audit / task_steps, not in the
|
||||
attempt count. EXCEPTION (ordering, rule 4): when the
|
||||
resume was itself platform-triggered, the platform marker
|
||||
wins and the dispatch does NOT count -- a platform goal
|
||||
auto-resume and a human goal-resume are the same loop
|
||||
continuation, but only the human one is attributed to the
|
||||
agent. That asymmetry is deliberate and uniform with the
|
||||
platform rule below; flipping it is the listed user
|
||||
decision, not a silent default.
|
||||
- Coordinator parent dispatch: does NOT count
|
||||
(``coordinator_orchestration``). The parent runs no model
|
||||
round (``cost_record_id`` is None) and its terminal status
|
||||
is an aggregate of its children; each child records its own
|
||||
row through its own dispatch. Counting the parent would
|
||||
double-count orchestration as an agent attempt.
|
||||
- Platform auto-resume dispatch (worker
|
||||
``auto_resume_interrupted`` requeue, goal or non-goal):
|
||||
does NOT count (``platform_auto_resume``). This is the
|
||||
default the gap-3 track demanded, and it is UNIFORM: every
|
||||
platform requeue is excluded, whether it continues a goal
|
||||
loop or fresh-runs a non-goal task. Rationale: the platform
|
||||
killed the previous execution and re-ran it without any
|
||||
human or coordinator decision; attributing that execution
|
||||
to the agent would punish platform instability in an
|
||||
agent-quality rate. TRADE-OFF, STATED NOT HIDDEN: a task
|
||||
whose terminal outcome came ONLY from platform dispatches
|
||||
stays unknown in the ledger (no counted row), so it is
|
||||
excluded from rate denominators -- the rates describe
|
||||
tasks whose terminal outcome came from a human/first
|
||||
round. The platform reruns themselves are NOT lost: they
|
||||
are recorded in ``tasks.auto_resume_count``, the
|
||||
``worker_auto_resumed`` audit events, and the
|
||||
``task_steps`` ``auto_resume`` rows. The rejected
|
||||
alternative was a separately marked ledger row that is
|
||||
excluded from denominators: same denominator outcome, but
|
||||
a second row kind, a new column, and aggregate exclusion
|
||||
logic, for co-location only. If a future policy wants
|
||||
platform completions counted as loop completions, the flip
|
||||
is one line in ``classify_dispatch`` (platform + breakpoint
|
||||
-> counted) plus tests -- it is listed as a user decision
|
||||
in the track summary, not silently decided here.
|
||||
- Pre-execution gate failures: do NOT count. A dispatch that
|
||||
never reached the model -- missing key, repository
|
||||
resolution failure, quota gate, goal-rounds cap, gateway
|
||||
preparation failure -- is an admission rejection, not an
|
||||
attempt. It writes no row (and no cost record either, which
|
||||
is the existing behaviour); it stays visible in audit
|
||||
(``worker_missing_key`` / ``worker_repo_failed`` /
|
||||
``worker_quota_stop`` / ``worker_goal_cap_rejected`` /
|
||||
``worker_gateway_failed``).
|
||||
- Dispatches that never ran (budget gate, no ready task,
|
||||
claim lost to another worker): do NOT count and are not
|
||||
even classified -- ``run_one`` returns before the hook
|
||||
runs, and no marker is consumed (see rule 4).
|
||||
|
||||
3. Transient retries inside one dispatch are NOT new attempts.
|
||||
The worker's ``max_retries`` loop, its one real fallback
|
||||
re-run, its goal "already exists" breakpoint fallback, and
|
||||
its follow-up drain all run inside a single ``run_one``
|
||||
dispatch, share one cost aggregate, and produce ONE ledger
|
||||
row: the dispatch's terminal outcome. An inner failure that
|
||||
is later retried successfully leaves no failure row behind;
|
||||
an inner failure that exhausts retries leaves one failure
|
||||
row, not N. This is enforced by placement (the hook is
|
||||
called once per dispatch, at ``run_one``'s terminal points)
|
||||
rather than by counting executions.
|
||||
|
||||
4. Platform identification -- the marker mechanism. The worker
|
||||
that requeues a task is not necessarily the worker that later
|
||||
claims it, so the origin must survive in the database, not in
|
||||
memory. ``auto_resume_interrupted`` writes one marker row
|
||||
(``task_attempt_origin_markers``) in the SAME transaction as
|
||||
the requeue, with ``queued_at`` equal to the ``updated_at``
|
||||
the requeue writes. ``run_one`` captures the queued task's
|
||||
``updated_at`` BEFORE claiming (claim overwrites it), and
|
||||
after a successful claim consumes the marker ONLY if
|
||||
``queued_at`` still matches. Any intervening human action
|
||||
(cancel, retry, continue) rewrites ``updated_at``, so a stale
|
||||
marker can never misclassify a later human round: on
|
||||
mismatch the marker is deleted as stale and the dispatch is
|
||||
classified by its own fields. A claim that loses the race
|
||||
consumes nothing, so the winning worker still sees its
|
||||
marker. Marker writes and reads happen ONLY when the hook is
|
||||
enabled; with the hook off, ``auto_resume_interrupted``
|
||||
writes no marker and ``run_one`` reads none, so off behaviour
|
||||
is byte-identical to gap 3 (no new table, no new row).
|
||||
Marker-table creation is a write-path-only step, like the
|
||||
ledger table itself; the read path (``GET /metrics-report``)
|
||||
never touches it.
|
||||
|
||||
5. Switch. ``Worker(attempt_ledger_hook=False)`` (CLI
|
||||
``--attempt-ledger-hook``), default OFF. OFF means: no ledger
|
||||
row, no marker row, no marker-table creation, no ledger-table
|
||||
creation from the worker -- exactly gap 3's "explicit calls
|
||||
only" behaviour, including old tasks still returning ``None``
|
||||
(unknown, not 0) and the read endpoint never creating a
|
||||
table. ON means rules 1-4 apply. Enabling the hook
|
||||
mid-flight has one documented edge: a task requeued by
|
||||
auto-resume WHILE the hook was off carries no marker, so its
|
||||
next dispatch counts as a human/first round (one extra
|
||||
counted row for that in-flight task, once). That is the
|
||||
fail-safe direction (a human round is the common case), and
|
||||
it self-heals: every later requeue writes a marker.
|
||||
|
||||
6. ``is_defect`` -- who judges, and how a wrong judgement is
|
||||
fixed. The worker hook NEVER judges a defect: it always
|
||||
records ``is_defect=False``. A failed dispatch is not a
|
||||
defect (transient failures, quota stops, and cancelled rounds
|
||||
all fail without being defects), and the worker cannot tell
|
||||
a transient failure from a defect round at runtime -- so it
|
||||
must not guess. The ONLY defect judgements in the ledger are
|
||||
explicit ones, made at record time by an explicit recorder:
|
||||
a human maintainer (or a coordinator review) calling
|
||||
``record_attempt`` / ``record_correction`` directly with
|
||||
``is_defect=True`` and a category. The bar for that authority
|
||||
is maintainer-level (``approve_merge``, the same bar as
|
||||
``/retry``: both spend model money and move decision-grade
|
||||
rates). A defect judgement made in error is corrected ONLY by
|
||||
appending a correction row (``record_correction``): the old
|
||||
row is never updated or deleted, corrections apply in
|
||||
attempt order with last-wins, and a correction can only
|
||||
change the defect judgement, never the outcome. There is NO
|
||||
HTTP endpoint for marking or correcting defects in this
|
||||
track -- the function is the interface, and an endpoint (with
|
||||
the ``approve_merge`` permission check) is follow-up work,
|
||||
listed as unconnected in the track summary. Until it exists,
|
||||
defect marks come only from explicit, reviewed calls, never
|
||||
from automation.
|
||||
|
||||
7. Failure discipline. Every hook write is best-effort: a ledger
|
||||
or marker failure is logged (``worker_attempt_ledger_failed``
|
||||
/ ``worker_attempt_marker_failed``) and NEVER fails the task,
|
||||
blocks a requeue, or changes a task's status. The one
|
||||
exception is validation inside ``record_attempt`` /
|
||||
``record_correction`` (bad outcome, defect on an interrupted
|
||||
row, correcting a nonexistent attempt): those raise
|
||||
``ValueError`` BEFORE writing, by design, so ambiguous data
|
||||
is never stored -- but the worker hook only ever passes
|
||||
validated values (a terminal outcome, ``is_defect=False``),
|
||||
so it cannot hit that path in normal operation.
|
||||
|
||||
8. Postgres. The marker table follows the ledger table's manual
|
||||
discipline: it is NOT created by ``init_postgres_schema``,
|
||||
and ``PG_MARKER_DDL`` below is the exact statement to run by
|
||||
hand on production Postgres (and verify) before enabling the
|
||||
hook against a Postgres-backed control plane. Until that
|
||||
step runs, marker writes fail (best-effort, rule 7) and
|
||||
platform dispatches degrade to counted -- the same
|
||||
fail-safe direction as rule 5's mid-flight edge, and the
|
||||
reason the PG manual step is a precondition for enabling
|
||||
the hook in production, not an afterthought. Real Postgres
|
||||
verification was NOT done on this track (no local Postgres):
|
||||
PG evidence here is adapter-level only (``?`` placeholders
|
||||
translate to ``%s`` through ``cloud.db``, DDL uses only
|
||||
Postgres-valid types, no autoincrement/identity column),
|
||||
exactly like gap 3's PG evidence, and labelled as such.
|
||||
Integration v2 out-of-box closure (see
|
||||
``cloud/ledger_onboarding.py``): the DDL checklist, the
|
||||
explicit enable path and the read-only
|
||||
``verify_ledger_ready`` / ``ledger_setup_report`` probe
|
||||
close the "hook off by default + manual PG DDL" gap
|
||||
WITHOUT changing the default (still OFF) or auto-running
|
||||
DDL on production Postgres -- the manual PG step remains
|
||||
an operator action, now checkable instead of assumed.
|
||||
|
||||
This module imports ``cloud.attempt_ledger`` only, opens no
|
||||
database connection itself (callers pass a connection in), and
|
||||
contains no ``sqlite3.connect`` / ``psycopg*`` import, so the
|
||||
``arch_conformance`` layer rules are untouched.
|
||||
"""
|
||||
|
||||
from cloud.attempt_ledger import record_attempt
|
||||
|
||||
MARKER_TABLE = "task_attempt_origin_markers"
|
||||
|
||||
# One DDL serves both backends: TEXT / DOUBLE PRECISION / composite
|
||||
# primary key, no AUTOINCREMENT or identity column -- the same
|
||||
# discipline as the ledger DDL. ``queued_at`` is the ``updated_at``
|
||||
# the requeue wrote; it is the generation guard, not a timestamp
|
||||
# for display.
|
||||
MARKER_DDL = (
|
||||
"CREATE TABLE IF NOT EXISTS task_attempt_origin_markers("
|
||||
"task_id TEXT NOT NULL PRIMARY KEY, "
|
||||
"origin TEXT NOT NULL, "
|
||||
"queued_at DOUBLE PRECISION NOT NULL)"
|
||||
)
|
||||
|
||||
# Production Postgres manual step (see rule 8). Intentionally NOT
|
||||
# part of POSTGRES_SCHEMA, same reason as the ledger PG_DDL.
|
||||
PG_MARKER_DDL = MARKER_DDL
|
||||
|
||||
PLATFORM_ORIGIN = "platform_auto_resume"
|
||||
|
||||
# Decision kinds returned by classify_dispatch. ``counted`` is the
|
||||
# only field the worker acts on; ``kind`` is the audit label.
|
||||
KINDS = frozenset({
|
||||
"coordinator_orchestration",
|
||||
"platform_auto_resume",
|
||||
"breakpoint_resume",
|
||||
"human_continue",
|
||||
"first_or_human_retry",
|
||||
})
|
||||
|
||||
|
||||
def classify_dispatch(task, platform_marker):
|
||||
"""Pure policy: task fields + marker -> decision dict.
|
||||
|
||||
``task`` is the dispatch's task dict (post-claim, post
|
||||
breakpoint resolution: ``_goal_resume_breakpoint`` is True
|
||||
only when a breakpoint resume actually applies, False when
|
||||
resolution fell back to a fresh goal run). ``platform_marker``
|
||||
is True when this dispatch consumed a platform auto-resume
|
||||
marker for the queued generation (rule 4), else False.
|
||||
|
||||
Returns ``{"counted": bool, "kind": str}``. Order matters
|
||||
and is part of the policy: coordinator fan-out never counts
|
||||
even with a marker; a platform marker beats every other
|
||||
signal (uniform platform exclusion, rule 2); a breakpoint
|
||||
resume beats a continue message (a stale resume_message on
|
||||
a breakpoint task is not a new round); anything else with a
|
||||
resume message is a human continue; the rest is a first
|
||||
dispatch or a human retry, which count identically.
|
||||
"""
|
||||
task = task or {}
|
||||
if task.get("coordinator_parts"):
|
||||
return {"counted": False,
|
||||
"kind": "coordinator_orchestration"}
|
||||
if platform_marker:
|
||||
return {"counted": False, "kind": "platform_auto_resume"}
|
||||
if task.get("_goal_resume_breakpoint") is True or (
|
||||
task.get("_resume_kind") == "breakpoint"):
|
||||
return {"counted": True, "kind": "breakpoint_resume"}
|
||||
if task.get("resume_message"):
|
||||
return {"counted": True, "kind": "human_continue"}
|
||||
return {"counted": True, "kind": "first_or_human_retry"}
|
||||
|
||||
|
||||
def ensure_marker_table(conn):
|
||||
"""Create the marker table if absent. Idempotent; the caller
|
||||
owns the commit (``mark_platform_rerun`` commits via the
|
||||
caller's requeue transaction in the worker; a bare ensure
|
||||
inside a larger transaction does not commit)."""
|
||||
conn.execute(MARKER_DDL)
|
||||
|
||||
|
||||
def mark_platform_rerun(conn, task_id, queued_at):
|
||||
"""Record that ``task_id`` was requeued by platform auto-resume
|
||||
at ``queued_at`` (the requeue's ``updated_at``). Overwrites
|
||||
any marker for the task: a requeue supersedes any older
|
||||
generation. Callers must call this inside the requeue
|
||||
transaction (the worker does: same connection, same commit
|
||||
as the status change), so a marker can never exist without
|
||||
its queued state, or vice versa. Best-effort at the call
|
||||
site: on failure the caller logs and proceeds with the
|
||||
requeue (rule 7) -- a missing marker degrades to counted,
|
||||
never to a blocked requeue."""
|
||||
if not task_id:
|
||||
raise ValueError("task_id is required")
|
||||
ensure_marker_table(conn)
|
||||
conn.execute(
|
||||
"DELETE FROM task_attempt_origin_markers WHERE task_id = ?",
|
||||
(task_id,),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO task_attempt_origin_markers(task_id, origin, "
|
||||
"queued_at) VALUES(?, ?, ?)",
|
||||
(task_id, PLATFORM_ORIGIN, queued_at),
|
||||
)
|
||||
|
||||
|
||||
def consume_platform_marker(conn, task_id, queued_at):
|
||||
"""Consume the platform marker for ``task_id`` iff its
|
||||
``queued_at`` still matches this dispatch's queued
|
||||
``updated_at``. Returns True on a match (this dispatch IS
|
||||
the platform rerun). On mismatch the marker belongs to an
|
||||
older generation (a human action rewrote ``updated_at``
|
||||
since) and is deleted as stale; returns False. When
|
||||
``queued_at`` is None the generation cannot be verified, so
|
||||
nothing is consumed or deleted and False is returned (the
|
||||
dispatch then counts -- the rule-5 fail-safe direction).
|
||||
A missing marker table raises the backend's OperationalError
|
||||
(loud, not silent): the worker call site treats that as
|
||||
"no marker" and logs it, it never fails the dispatch."""
|
||||
if not task_id or queued_at is None:
|
||||
return False
|
||||
cur = conn.execute(
|
||||
"DELETE FROM task_attempt_origin_markers "
|
||||
"WHERE task_id = ? AND queued_at = ?",
|
||||
(task_id, queued_at),
|
||||
)
|
||||
if cur.rowcount == 1:
|
||||
return True
|
||||
conn.execute(
|
||||
"DELETE FROM task_attempt_origin_markers WHERE task_id = ?",
|
||||
(task_id,),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def record_dispatch_result(conn, task_id, outcome):
|
||||
"""Write the ledger row for one COUNTED dispatch. The worker
|
||||
calls this only when ``classify_dispatch`` said counted, and
|
||||
only with a terminal outcome (``success`` / ``failure`` /
|
||||
``interrupted``). ``is_defect`` is always False here (rule
|
||||
6): the worker never judges a defect. Returns the new
|
||||
``attempt_no``. The counted filter lives in the caller
|
||||
(``classify_dispatch``); this function takes the outcome
|
||||
only, so a miscounted call site cannot smuggle a defect
|
||||
judgement in through here either."""
|
||||
return record_attempt(conn, task_id, outcome, is_defect=False)
|
||||
@@ -0,0 +1,395 @@
|
||||
"""Task-attempt ledger for the step-5 metrics loop (audit gap 3).
|
||||
|
||||
Choice made here, with the trade-off stated rather than hidden:
|
||||
|
||||
- This module adds a SEPARATE ``task_attempts`` table; it does NOT
|
||||
add ``attempts``/``defects`` columns to ``tasks``. Columns would be
|
||||
simpler to query (one row per task, no join), but they collapse
|
||||
history into two counters: a task's per-attempt story (which
|
||||
attempt failed, with what defect category, in what order) would be
|
||||
unrecoverable, and widening the hot ``tasks`` table on production
|
||||
Postgres is a heavier migration than adding a new, empty table.
|
||||
The ledger costs one aggregate query at report time, and that
|
||||
query is exactly what ``aggregate_attempts`` exists to pin down.
|
||||
- The ledger is append-only in intent: ``record_attempt`` only ever
|
||||
inserts a new ``(task_id, attempt_no)`` row (``attempt_no`` is
|
||||
``MAX + 1`` per task, never reused). There is no update path, so a
|
||||
recorded attempt cannot be silently rewritten into a first-pass
|
||||
success after the fact.
|
||||
|
||||
Semantics (pinned by tests, not left to each caller):
|
||||
|
||||
- One row = one real attempt at the task. ``outcome`` is one of
|
||||
``success`` / ``failure`` / ``interrupted``.
|
||||
- ``interrupted`` rows are kept for audit but NEVER count toward
|
||||
``attempts`` or ``defects``: a platform interruption that
|
||||
auto-resumes as a fresh run is a platform event, not an agent
|
||||
failure, and the step-5 rule that ``interrupted`` is not a
|
||||
terminal task status would be meaningless if the interruption
|
||||
still inflated the attempt count. A task whose ledger holds only
|
||||
interrupted rows has NO counted attempt, so it is omitted from
|
||||
the aggregate entirely -- the endpoint then still reports its
|
||||
rates as unknown (``None``), never as 0.
|
||||
- ``is_defect`` is set explicitly by the recorder, never inferred
|
||||
from ``outcome``: a failed attempt is not automatically a defect
|
||||
(a retry can fail for a transient reason without being a
|
||||
defect round), and a defect round is a judgement the recording
|
||||
caller makes. ``defect_category`` requires ``is_defect``; an
|
||||
interrupted row can never be a defect. Both contradictions raise
|
||||
``ValueError`` instead of being stored as ambiguous data.
|
||||
- ``defect_category`` for a task (the single category that
|
||||
``compute_metrics`` carries per task) is the category of its
|
||||
FIRST defect row in attempt order. Later defect rows still count
|
||||
toward ``defects``; only the category label is first-wins, so it
|
||||
is deterministic. Corrections (below) re-evaluate this label:
|
||||
the first defect row AFTER corrections is applied wins.
|
||||
- A defect judgement can be CORRECTED, never rewritten:
|
||||
``record_correction`` appends a new row whose
|
||||
``corrects_attempt_no`` points at the attempt it corrects; the
|
||||
old row stays untouched, and aggregation applies corrections in
|
||||
attempt order with last-wins (so a wrongly marked defect can be
|
||||
unmarked, and a defect missed at record time can be added after
|
||||
review). A correction row is never itself counted as an
|
||||
attempt, cannot change an attempt's outcome (only its defect
|
||||
judgement), and cannot target an interrupted row, another
|
||||
correction row, or a nonexistent attempt -- all raise
|
||||
``ValueError`` and write nothing. Who may correct is a policy
|
||||
question answered in ``cloud/attempt_hook.py`` (explicit
|
||||
recorder only, maintainer bar); the worker hook never writes
|
||||
corrections and never marks a defect.
|
||||
- Old tasks have no ledger rows, and that is NOT backfilled: the
|
||||
aggregate simply has no entry for them, the endpoint leaves their
|
||||
``attempts``/``defects`` as ``None``, and ``compute_metrics``
|
||||
excludes them from rate denominators (they are neither counted
|
||||
as first-pass nor as defect-free). Fabricating a row for an old
|
||||
task (e.g. ``attempts=1, defects=0`` from "it completed") would
|
||||
be inventing history.
|
||||
|
||||
Persistence notes:
|
||||
|
||||
- SQLite: ``ensure_task_attempts_table`` runs
|
||||
``CREATE TABLE IF NOT EXISTS`` -- idempotent, safe to call on
|
||||
every write, and a no-op on a database that already has the
|
||||
table (including one that already has rows).
|
||||
- Postgres: the production table is NOT created by
|
||||
``init_postgres_schema`` (deliberately: that schema is covered by
|
||||
the migrations test asserting ``MIGRATION_TABLES`` equals the
|
||||
schema set, and this ledger must not silently widen that set).
|
||||
``PG_DDL`` below is the exact statement to run manually on
|
||||
production Postgres and verify (``\\d task_attempts`` or the
|
||||
``information_schema`` equivalent) before any Postgres-backed
|
||||
endpoint is expected to return real rates. The DDL uses only
|
||||
Postgres-valid types (``TEXT``/``INTEGER``/``DOUBLE PRECISION``),
|
||||
a composite primary key, and no ``AUTOINCREMENT``/identity
|
||||
column, so it needs no sequence handling; the parameterized
|
||||
``INSERT`` uses ``?`` placeholders, which the ``cloud.db``
|
||||
Postgres adapter translates to ``%s`` (see the adapter test in
|
||||
``test_attempt_ledger.py``). Until that manual step runs, the
|
||||
endpoint treats a missing table exactly like "no ledger rows":
|
||||
unknown rates, not an error -- the same fail-soft shape already
|
||||
used for missing ``cost_records``.
|
||||
- The read path (``GET /metrics-report``) never creates the table:
|
||||
a read-only endpoint must not mutate schema. Table creation
|
||||
happens only on the write path, inside ``record_attempt`` /
|
||||
``record_correction``.
|
||||
- Lifecycle hook (this track, see ``cloud/attempt_hook.py``):
|
||||
the worker writes exactly one row per COUNTED dispatch (see
|
||||
that module's policy for which dispatches count: human/first
|
||||
rounds and human breakpoint resumes count; platform auto-resume
|
||||
reruns, coordinator fan-out, and pre-execution gate failures do
|
||||
not). The hook is OFF by default (``attempt_ledger_hook=False``):
|
||||
with it off, this module's behaviour is exactly audit gap 3 --
|
||||
the ledger fills only through explicit ``record_attempt`` /
|
||||
``record_correction`` calls, and production rates stay honestly
|
||||
unknown until the hook is deliberately enabled. The worker hook
|
||||
always records ``is_defect=False``: it never judges a defect,
|
||||
so enabling it can never turn a transient retry into a defect.
|
||||
- Schema evolution note (pre-production, this track):
|
||||
``corrects_attempt_no`` was added to the DDL after audit gap 3
|
||||
but BEFORE either DDL reached production (gap 3 is still an
|
||||
OPEN, unmerged PR), so no production table needs migration:
|
||||
fresh tables get the column from the DDL below. A database
|
||||
created from gap 3's DDL alone (temporary/test databases only)
|
||||
gains the column with one statement --
|
||||
``ALTER TABLE task_attempts ADD COLUMN corrects_attempt_no
|
||||
INTEGER`` -- which is also the documented step if gap 3's DDL
|
||||
was ever run on production Postgres before this track lands.
|
||||
The read path uses ``SELECT *`` deliberately so a table without
|
||||
the column still reads (its rows aggregate as plain attempts).
|
||||
|
||||
This module imports no ``cloud.*`` module and opens no database
|
||||
connection itself: callers pass a connection in, so the layer rules
|
||||
(``arch_conformance``: no direct driver connection outside
|
||||
``cloud.db``) are untouched -- there is no ``sqlite3.connect`` and
|
||||
no ``psycopg*`` import here at all.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
||||
TABLE = "task_attempts"
|
||||
|
||||
OUTCOMES = frozenset({"success", "failure", "interrupted"})
|
||||
|
||||
# One DDL serves both backends: every type and the composite key are
|
||||
# valid in SQLite and in Postgres, and there is no AUTOINCREMENT or
|
||||
# identity column for either backend's sequence handling to chase.
|
||||
# ``corrects_attempt_no`` (NULL = a real attempt row; non-NULL = a
|
||||
# correction row for that attempt_no, never counted as an attempt)
|
||||
# was added by the lifecycle-hook track; it SUPERSEDES gap 3's DDL
|
||||
# for any production manual step (gap 3 never merged, so no
|
||||
# production table exists yet -- see the module docstring).
|
||||
SQLITE_DDL = (
|
||||
"CREATE TABLE IF NOT EXISTS task_attempts("
|
||||
"task_id TEXT NOT NULL, "
|
||||
"attempt_no INTEGER NOT NULL, "
|
||||
"outcome TEXT NOT NULL, "
|
||||
"is_defect INTEGER NOT NULL DEFAULT 0, "
|
||||
"defect_category TEXT, "
|
||||
"created_at DOUBLE PRECISION, "
|
||||
"corrects_attempt_no INTEGER, "
|
||||
"PRIMARY KEY (task_id, attempt_no))"
|
||||
)
|
||||
|
||||
# Production Postgres: run this verbatim by hand and verify the
|
||||
# table exists before expecting real rates from a Postgres-backed
|
||||
# endpoint. It is intentionally NOT part of POSTGRES_SCHEMA (see
|
||||
# the module docstring: migrations coverage must not widen silently).
|
||||
# This is the NEW DDL (with ``corrects_attempt_no``): if gap 3's
|
||||
# older DDL was ever run first, follow it with
|
||||
# ``ALTER TABLE task_attempts ADD COLUMN corrects_attempt_no
|
||||
# INTEGER`` and verify before expecting corrections to work.
|
||||
PG_DDL = SQLITE_DDL
|
||||
|
||||
# One-line upgrade for a table created from gap 3's DDL (no
|
||||
# ``corrects_attempt_no`` column yet). Valid in both backends.
|
||||
OLD_TABLE_ALTER = (
|
||||
"ALTER TABLE task_attempts "
|
||||
"ADD COLUMN corrects_attempt_no INTEGER"
|
||||
)
|
||||
|
||||
|
||||
def ensure_task_attempts_table(conn):
|
||||
"""Create the ledger table if absent. Idempotent; does not
|
||||
touch existing rows. The caller owns the commit (``record_attempt``
|
||||
commits; a bare ensure inside a larger transaction does not)."""
|
||||
conn.execute(SQLITE_DDL)
|
||||
|
||||
|
||||
def record_attempt(conn, task_id, outcome, is_defect=False,
|
||||
defect_category=None, created_at=None):
|
||||
"""Append one attempt row for ``task_id``. Returns ``attempt_no``.
|
||||
|
||||
Validates before writing: unknown outcome, a defect flag on an
|
||||
interrupted row, and a defect category without a defect flag
|
||||
all raise ``ValueError`` and write nothing.
|
||||
"""
|
||||
if not task_id:
|
||||
raise ValueError("task_id is required")
|
||||
if outcome not in OUTCOMES:
|
||||
raise ValueError("outcome must be one of %s, got %r"
|
||||
% (sorted(OUTCOMES), outcome))
|
||||
if outcome == "interrupted" and is_defect:
|
||||
raise ValueError("an interrupted attempt is never a defect")
|
||||
if defect_category is not None and not is_defect:
|
||||
raise ValueError("defect_category requires is_defect=True")
|
||||
ensure_task_attempts_table(conn)
|
||||
row = conn.execute(
|
||||
"SELECT COALESCE(MAX(attempt_no), 0) FROM task_attempts "
|
||||
"WHERE task_id = ?",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
attempt_no = int(row[0]) + 1
|
||||
conn.execute(
|
||||
"INSERT INTO task_attempts(task_id, attempt_no, outcome, "
|
||||
"is_defect, defect_category, created_at, "
|
||||
"corrects_attempt_no) "
|
||||
"VALUES(?, ?, ?, ?, ?, ?, NULL)",
|
||||
(task_id, attempt_no, outcome, 1 if is_defect else 0,
|
||||
defect_category,
|
||||
time.time() if created_at is None else created_at),
|
||||
)
|
||||
conn.commit()
|
||||
return attempt_no
|
||||
|
||||
|
||||
def record_correction(conn, task_id, attempt_no, is_defect,
|
||||
defect_category=None, created_at=None):
|
||||
"""Append a CORRECTION row for an already-recorded attempt.
|
||||
|
||||
The correction does not rewrite history: the target row stays
|
||||
exactly as recorded, and this new row (its own ``MAX + 1``
|
||||
``attempt_no``) carries the corrected defect judgement in
|
||||
``is_defect`` / ``defect_category`` plus
|
||||
``corrects_attempt_no = attempt_no``. Aggregation applies
|
||||
corrections in attempt order, last-wins, and never counts a
|
||||
correction row as an attempt.
|
||||
|
||||
Only the defect judgement is correctable: the correction row
|
||||
copies the target's outcome, so it cannot turn a success into
|
||||
a failure or vice versa. Validates before writing: missing
|
||||
``task_id``, a nonexistent target, a target that is itself a
|
||||
correction row, a target whose outcome is ``interrupted``
|
||||
(interrupted rows are never defects, so there is no defect
|
||||
judgement to correct), and a defect category without a defect
|
||||
flag all raise ``ValueError`` and write nothing. On a table
|
||||
created from gap 3's DDL (no ``corrects_attempt_no`` column),
|
||||
the INSERT raises the backend's OperationalError loudly --
|
||||
run ``OLD_TABLE_ALTER`` first (see the module docstring);
|
||||
a correction must never silently degrade into a plain row.
|
||||
Returns the correction row's own ``attempt_no``.
|
||||
"""
|
||||
if not task_id:
|
||||
raise ValueError("task_id is required")
|
||||
if defect_category is not None and not is_defect:
|
||||
raise ValueError("defect_category requires is_defect=True")
|
||||
ensure_task_attempts_table(conn)
|
||||
target = conn.execute(
|
||||
"SELECT outcome, corrects_attempt_no FROM task_attempts "
|
||||
"WHERE task_id = ? AND attempt_no = ?",
|
||||
(task_id, attempt_no),
|
||||
).fetchone()
|
||||
if target is None:
|
||||
raise ValueError(
|
||||
"no such attempt to correct: %s #%s" % (task_id, attempt_no))
|
||||
if target["corrects_attempt_no"] is not None:
|
||||
raise ValueError(
|
||||
"cannot correct a correction row: %s #%s"
|
||||
% (task_id, attempt_no))
|
||||
if target["outcome"] == "interrupted":
|
||||
raise ValueError(
|
||||
"an interrupted attempt has no defect judgement to "
|
||||
"correct: %s #%s" % (task_id, attempt_no))
|
||||
row = conn.execute(
|
||||
"SELECT COALESCE(MAX(attempt_no), 0) FROM task_attempts "
|
||||
"WHERE task_id = ?",
|
||||
(task_id,),
|
||||
).fetchone()
|
||||
correction_no = int(row[0]) + 1
|
||||
conn.execute(
|
||||
"INSERT INTO task_attempts(task_id, attempt_no, outcome, "
|
||||
"is_defect, defect_category, created_at, "
|
||||
"corrects_attempt_no) "
|
||||
"VALUES(?, ?, ?, ?, ?, ?, ?)",
|
||||
(task_id, correction_no, target["outcome"],
|
||||
1 if is_defect else 0, defect_category,
|
||||
time.time() if created_at is None else created_at,
|
||||
attempt_no),
|
||||
)
|
||||
conn.commit()
|
||||
return correction_no
|
||||
|
||||
|
||||
def read_attempt_rows(conn):
|
||||
"""All ledger rows in attempt order. Raises the backend's
|
||||
OperationalError when the table does not exist yet; the
|
||||
endpoint caller treats that as "no ledger", not as a failure.
|
||||
|
||||
``SELECT *`` is deliberate: a table created from gap 3's DDL
|
||||
has no ``corrects_attempt_no`` column, and naming that column
|
||||
here would turn those readable rows into an endpoint error.
|
||||
With ``*`` the rows still read, and ``aggregate_attempts``
|
||||
treats a missing ``corrects_attempt_no`` as a plain attempt
|
||||
row (see ``_field_optional``).
|
||||
"""
|
||||
return conn.execute(
|
||||
"SELECT * FROM task_attempts "
|
||||
"ORDER BY task_id, attempt_no").fetchall()
|
||||
|
||||
|
||||
def _field(row, name):
|
||||
# sqlite3.Row, dict, and the Postgres adapter's rows all support
|
||||
# key access; a plain tuple row is not accepted (callers pass
|
||||
# rows, not raw tuples, so a wrong shape fails loudly here).
|
||||
return row[name]
|
||||
|
||||
|
||||
def _field_optional(row, name):
|
||||
"""Like ``_field`` but missing keys mean "absent", not an
|
||||
error. Used ONLY for ``corrects_attempt_no``: rows built by
|
||||
gap-3-era readers (plain dicts, or tables without the column)
|
||||
carry no such key, and they are plain attempt rows by
|
||||
definition -- failing loudly there would break every gap-3
|
||||
aggregate test for no benefit."""
|
||||
try:
|
||||
return row[name]
|
||||
except (KeyError, IndexError, TypeError):
|
||||
return None
|
||||
|
||||
|
||||
def aggregate_attempts(rows):
|
||||
"""Pure aggregation: ledger rows -> ``{task_id: record}`` where
|
||||
``record`` is the ``attempts``/``defects``/``defect_category``
|
||||
dict that ``compute_metrics`` consumes.
|
||||
|
||||
Interrupted rows are excluded from both counts, and a task
|
||||
with only interrupted rows is omitted entirely (no counted
|
||||
attempt => its rates stay unknown, never 0). Correction rows
|
||||
(``corrects_attempt_no`` set) are excluded from the attempt
|
||||
count and instead OVERWRITE the defect judgement of their
|
||||
target attempt, in attempt order, last-wins: a task's
|
||||
``defects`` is the number of its counted attempts whose
|
||||
judgement AFTER corrections is a defect, and its
|
||||
``defect_category`` is the category of its FIRST defect
|
||||
attempt in attempt order AFTER corrections.
|
||||
"""
|
||||
per_task = {}
|
||||
ordered = sorted(
|
||||
rows,
|
||||
key=lambda r: (_field(r, "task_id"), _field(r, "attempt_no")))
|
||||
# Per task: attempt_no -> [is_defect, defect_category], in
|
||||
# attempt order (insertion order = attempt_no order, since the
|
||||
# input was sorted). Corrections mutate the target entry; they
|
||||
# never create an entry of their own.
|
||||
judged = {}
|
||||
for row in ordered:
|
||||
task_id = _field(row, "task_id")
|
||||
corrects = _field_optional(row, "corrects_attempt_no")
|
||||
if corrects is not None:
|
||||
entries = judged.get(task_id)
|
||||
if entries is not None and corrects in entries:
|
||||
entries[corrects] = [
|
||||
bool(_field(row, "is_defect")),
|
||||
_field(row, "defect_category")]
|
||||
continue
|
||||
if _field(row, "outcome") == "interrupted":
|
||||
continue
|
||||
entries = judged.setdefault(task_id, {})
|
||||
entries[_field(row, "attempt_no")] = [
|
||||
bool(_field(row, "is_defect")),
|
||||
_field(row, "defect_category")]
|
||||
for task_id, entries in judged.items():
|
||||
attempts = len(entries)
|
||||
if attempts == 0:
|
||||
continue
|
||||
defects = 0
|
||||
defect_category = None
|
||||
for _no in sorted(entries):
|
||||
is_defect, category = entries[_no]
|
||||
if is_defect:
|
||||
defects += 1
|
||||
if defect_category is None:
|
||||
defect_category = category
|
||||
per_task[task_id] = {
|
||||
"attempts": attempts, "defects": defects,
|
||||
"defect_category": defect_category}
|
||||
return per_task
|
||||
|
||||
|
||||
def apply_to_records(records, aggregates):
|
||||
"""Return a new record list with ledger values merged in.
|
||||
|
||||
Records for tasks with no aggregate entry keep whatever
|
||||
``attempts``/``defects`` they already carried (the endpoint
|
||||
passes ``None`` there, i.e. old tasks stay unknown). Input
|
||||
records are not modified.
|
||||
"""
|
||||
merged = []
|
||||
for record in records:
|
||||
out = dict(record)
|
||||
agg = aggregates.get(record.get("id"))
|
||||
if agg is not None:
|
||||
out["attempts"] = agg["attempts"]
|
||||
out["defects"] = agg["defects"]
|
||||
out["defect_category"] = agg["defect_category"]
|
||||
merged.append(out)
|
||||
return merged
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,385 @@
|
||||
"""Audit-of-audit sampling: human review of audit verdicts.
|
||||
|
||||
Zero third-party dependencies; standard library only, no ``cloud.*``
|
||||
imports. All functions are pure: they return plans, verdicts and
|
||||
records, never auto-apply a correction -- applying a correction is a
|
||||
separate, recorded step (see ``handle_sampling_result``).
|
||||
|
||||
This module implements the sampling half of the "audit itself must
|
||||
be audited" rule. It does NOT change the audit verdicts, the
|
||||
pushback/rebuild counters, or the golden-set approval flow; it only
|
||||
defines which audit cases a human must re-examine, and what happens
|
||||
when that re-examination finds the audit wrong.
|
||||
|
||||
UNIFIED SAMPLING RULE (single rule; this module is canonical for
|
||||
selection, ``cloud/audit_pushback.py::select_audit_sample`` is only
|
||||
its record-shaped adapter -- there is no second rule):
|
||||
|
||||
The scope of "mandatory" comes from track 3 (#244): golden-task
|
||||
records and rebuild / refreeze / escalation records. The drawing
|
||||
mechanism comes from track 5 (#238): seeded random draws with ceil
|
||||
rounding. Track 3's old deterministic "every 10th record" rule is
|
||||
superseded and removed: a batch of fewer than 10 ordinary records
|
||||
drew zero under it, and a position-known draw is predictable. The
|
||||
rates below are TEST DEFAULTS (``DEFAULT_*``), not approved
|
||||
production values; the production rates are still a user decision.
|
||||
Mandatory 1.0 is a rule, not a tunable rate.
|
||||
|
||||
Sampling rules (all testable, none left to "regular review"):
|
||||
|
||||
1. Mandatory sampling (ratio 1.0, never skipped). A case is
|
||||
mandatory when ANY of these triggers holds (trigger order for
|
||||
display and for the legacy single-trigger helper is golden,
|
||||
rebuild, third-failure, refreeze, escalate):
|
||||
- ``golden_task``: the case's task is in the human-approved
|
||||
golden set (``task_id`` when present, else ``id``). A golden
|
||||
task's records are mandatory whatever their kind -- an
|
||||
ordinary pass verdict on a golden task is sampled, not just
|
||||
its failures.
|
||||
- ``rebuild``: any case whose audit decision was rebuild-from-
|
||||
scratch (推倒重做). A rebuild destroys the old implementation
|
||||
in the workspace, so its verdict must always be re-examined.
|
||||
- ``third_failure_near_complete_still_failed``: a case whose
|
||||
third failure was root-caused as near-complete (接近完成),
|
||||
was given one targeted continuation round, and still failed.
|
||||
This is the exact boundary where the "one more round vs
|
||||
rebuild" judgement is most error-prone, so it is always sampled.
|
||||
- ``refreeze``: a contract/design re-freeze record.
|
||||
- ``escalate``: an escalation record.
|
||||
A near-complete third failure whose continuation round SUCCEEDED
|
||||
is not mandatory (the judgement proved itself), and a
|
||||
near-complete first/second failure is not mandatory either (the
|
||||
threshold is the third failure, per the rebuild rule). A
|
||||
``blocked_quota`` record is NOT mandatory under this rule (it
|
||||
is not in track 3's mandatory kind list); it is sampled only
|
||||
through the random tiers below.
|
||||
|
||||
2. Random sampling of the rest, with the seed recorded:
|
||||
- normal audit cases: default ratio 0.10;
|
||||
- critical-path cases (``critical_path=True`` and NOT already
|
||||
mandatory): default ratio 0.50 -- critical paths get a higher
|
||||
rate, not a free pass. (Track 3's old ``critical_path``
|
||||
reason label for mandatory kinds is retired: mandatory kinds
|
||||
now carry their own trigger names, and ``critical_path``
|
||||
means only this 0.50 random tier.)
|
||||
Mandatory cases are removed from the random pools BEFORE
|
||||
drawing, so they consume no random quota. The seed is part of
|
||||
the plan: a sample selected without a recorded seed is not
|
||||
reproducible and therefore not auditable. Selection is
|
||||
deterministic given (cases, ratios, seed).
|
||||
|
||||
4. One pass, no double counting. Selection partitions every case
|
||||
exactly once: mandatory (all its triggers listed, one sample,
|
||||
one sampling entry) or random-critical or random-normal. A
|
||||
record that is both a golden task AND a rebuild is sampled
|
||||
ONCE, with ``triggers == ["golden_task", "rebuild"]``; its
|
||||
correction, if any, is a single record whose retrigger follows
|
||||
the (original, corrected) verdict pair, not the trigger list --
|
||||
triggers decide selection only, never the correction action.
|
||||
Two different records (e.g. a triage record and the later
|
||||
rebuild record for the same task) are two samples, not a
|
||||
double count: deduplication is per record, not per task.
|
||||
|
||||
5. Golden-set construction. The caller-supplied golden id set
|
||||
must be built from human-approved, non-sample tasks only:
|
||||
``build_golden_task_ids`` applies both vocabularies -- a task
|
||||
counts only with ``human_approved_by`` set AND none of
|
||||
``is_example``, ``sample=True``, or an id starting with
|
||||
``sample-``. Track 3's per-task approver rule and track 5's
|
||||
sample-exclusion rule are both preserved; neither alone is
|
||||
sufficient.
|
||||
|
||||
6. Correction gate effect (shared with track 3 / wiring #247).
|
||||
A sampling correction NEVER unblocks a merge by itself: a
|
||||
correction to ``pass`` (a wrong block) schedules a re-audit
|
||||
and the gate stays blocked until a new passing re-audit
|
||||
conclusion exists; a correction to ``fail`` (a wrong pass)
|
||||
re-triggers pushback and re-blocks at once. Vocabulary
|
||||
synonyms for the same pair actions: track 3
|
||||
``retrigger_pushback_and_reblock`` == track 5
|
||||
``reopen_audit_and_pushback`` (wrong pass), and track 3
|
||||
``schedule_reaudit`` == track 5
|
||||
``cancel_pushback_and_rerun_audit`` (wrong block, where
|
||||
"cancel" means cancel-and-rerun, never cancel-and-merge).
|
||||
|
||||
3. Golden-set approval is separate from sampling and is NOT
|
||||
satisfied by sample files: ``filter_golden_tasks`` drops any task
|
||||
flagged as a sample (``sample=True`` or an id starting with
|
||||
``sample-``) and reports how many were dropped; a golden set
|
||||
counts only its remaining, human-approved tasks. The approval
|
||||
checklist (``golden_approval_checklist``) lists every item that
|
||||
must be human-completed before a golden set is approved; until
|
||||
then every item reads ``pending_human``.
|
||||
|
||||
Handling a sampling result (``handle_sampling_result``):
|
||||
|
||||
- Sampling says the audit was CORRECT: no correction, only a record
|
||||
entry (the audit stands, with evidence it was re-examined).
|
||||
- Sampling says the audit was WRONG: the original conclusion is
|
||||
corrected in the record, and the corresponding flow is
|
||||
re-triggered:
|
||||
- wrongly issued pushback -> ``cancel_pushback_and_rerun_audit``
|
||||
(cancel the pushback, restore the pre-pushback state, rerun);
|
||||
- wrongly issued rebuild -> ``halt_rebuild_and_restore``
|
||||
(halt the rebuild, restore the original implementation from
|
||||
git history/tag, rerun -- the old implementation is preserved
|
||||
by the rebuild rule itself, so restoration is possible);
|
||||
- wrongly issued pass (a failure the audit missed) ->
|
||||
``reopen_audit_and_pushback`` (reopen with the corrected
|
||||
verdict and run the pushback flow the audit should have run);
|
||||
- unknown original decision -> ``escalate_to_human`` (never
|
||||
invent a retrigger).
|
||||
Every outcome, correct or corrected, is appended to the
|
||||
audit-of-audit record (``append_audit_record``); a sampling result
|
||||
that is not recorded is treated as not having happened.
|
||||
"""
|
||||
|
||||
import math
|
||||
import random
|
||||
|
||||
DEFAULT_NORMAL_RATIO = 0.10
|
||||
DEFAULT_CRITICAL_PATH_RATIO = 0.50
|
||||
|
||||
# Original audit decisions -> retrigger when sampling proves them wrong.
|
||||
_RETRIGGER = {
|
||||
"fail_pushback": "cancel_pushback_and_rerun_audit",
|
||||
"rebuild": "halt_rebuild_and_restore",
|
||||
"pass": "reopen_audit_and_pushback",
|
||||
}
|
||||
|
||||
|
||||
def mandatory_triggers(case, golden_task_ids=None):
|
||||
"""Return ALL mandatory trigger names for ``case`` (possibly []).
|
||||
|
||||
Trigger order: golden_task, rebuild,
|
||||
third_failure_near_complete_still_failed, refreeze, escalate.
|
||||
``case`` fields used: ``task_id`` (else ``id``) for the golden
|
||||
check, ``decision`` / ``kind`` for rebuild / refreeze /
|
||||
escalate, ``failure_count``, ``root_cause``
|
||||
(``"structural"`` / ``"near_complete"`` / other),
|
||||
``continuation_failed`` (bool).
|
||||
"""
|
||||
triggers = []
|
||||
golden = set(golden_task_ids or [])
|
||||
task_id = case.get("task_id") if case.get("task_id") is not None \
|
||||
else case.get("id")
|
||||
if task_id is not None and task_id in golden:
|
||||
triggers.append("golden_task")
|
||||
if case.get("decision") == "rebuild" or case.get("kind") == "rebuild":
|
||||
triggers.append("rebuild")
|
||||
if (case.get("failure_count", 0) >= 3
|
||||
and case.get("root_cause") == "near_complete"
|
||||
and case.get("continuation_failed")):
|
||||
triggers.append("third_failure_near_complete_still_failed")
|
||||
if case.get("decision") == "refreeze" or case.get("kind") == "refreeze":
|
||||
triggers.append("refreeze")
|
||||
if case.get("decision") == "escalate" or case.get("kind") == "escalate":
|
||||
triggers.append("escalate")
|
||||
return triggers
|
||||
|
||||
|
||||
def mandatory_trigger(case, golden_task_ids=None):
|
||||
"""Return the first mandatory trigger name for ``case``, or None.
|
||||
|
||||
Legacy single-trigger view of ``mandatory_triggers``; new code
|
||||
must use ``mandatory_triggers`` so a multi-trigger case is not
|
||||
silently reported under one reason only.
|
||||
"""
|
||||
triggers = mandatory_triggers(case, golden_task_ids)
|
||||
return triggers[0] if triggers else None
|
||||
|
||||
|
||||
def build_golden_task_ids(tasks):
|
||||
"""Build the golden task id list from candidate tasks.
|
||||
|
||||
A task counts only when it has a human approver
|
||||
(``human_approved_by``) AND is not a sample/example under any
|
||||
vocabulary: ``is_example``, ``sample=True``, or an id starting
|
||||
with ``sample-``. Returns ids in input order, samples dropped
|
||||
silently here (count them with ``filter_golden_tasks`` when a
|
||||
dropped count must be reported).
|
||||
"""
|
||||
ids = []
|
||||
for task in tasks or []:
|
||||
if not isinstance(task, dict):
|
||||
continue
|
||||
task_id = task.get("id", "")
|
||||
if task.get("is_example") or task.get("sample"):
|
||||
continue
|
||||
if str(task_id).startswith("sample-"):
|
||||
continue
|
||||
if not task.get("human_approved_by"):
|
||||
continue
|
||||
ids.append(task_id)
|
||||
return ids
|
||||
|
||||
|
||||
def select_sample(cases, seed, normal_ratio=DEFAULT_NORMAL_RATIO,
|
||||
critical_ratio=DEFAULT_CRITICAL_PATH_RATIO,
|
||||
golden_task_ids=None):
|
||||
"""Select the audit sample. Returns a plan dict.
|
||||
|
||||
Single pass, no double counting: mandatory cases (all their
|
||||
triggers listed) are always included and removed from the
|
||||
random pools before drawing. The remaining cases are grouped
|
||||
into critical-path and normal, each shuffled with
|
||||
``random.Random(seed + ":" + group)`` and truncated to
|
||||
``ceil(ratio * group_size)`` (a group of 1 with any positive
|
||||
ratio therefore contributes 1 -- a small group is never rounded
|
||||
down to zero and silently unsampled). ``seed`` is required: an
|
||||
unseeded selection cannot be reproduced on review.
|
||||
``golden_task_ids`` is the human-approved golden set (see
|
||||
``build_golden_task_ids``); without it no case is golden.
|
||||
"""
|
||||
if seed is None or seed == "":
|
||||
raise ValueError("sampling seed is required: an unseeded "
|
||||
"sample cannot be reproduced on review")
|
||||
mandatory = []
|
||||
critical = []
|
||||
normal = []
|
||||
for case in cases:
|
||||
triggers = mandatory_triggers(case, golden_task_ids)
|
||||
if triggers:
|
||||
mandatory.append((case, triggers))
|
||||
elif case.get("critical_path"):
|
||||
critical.append(case)
|
||||
else:
|
||||
normal.append(case)
|
||||
|
||||
def _draw(group, ratio, label):
|
||||
shuffled = list(group)
|
||||
random.Random("%s:%s" % (seed, label)).shuffle(shuffled)
|
||||
if not shuffled or ratio <= 0:
|
||||
return []
|
||||
return shuffled[:math.ceil(ratio * len(shuffled))]
|
||||
|
||||
sampled_critical = _draw(critical, critical_ratio, "critical")
|
||||
sampled_normal = _draw(normal, normal_ratio, "normal")
|
||||
return {
|
||||
"seed": seed,
|
||||
"ratios": {"normal": normal_ratio,
|
||||
"critical_path": critical_ratio,
|
||||
"mandatory": 1.0},
|
||||
"golden_task_ids": sorted(golden_task_ids or []),
|
||||
"mandatory": [{"case_id": c.get("id"), "trigger": t[0],
|
||||
"triggers": list(t)}
|
||||
for c, t in mandatory],
|
||||
"random_critical_path": [c.get("id") for c in sampled_critical],
|
||||
"random_normal": [c.get("id") for c in sampled_normal],
|
||||
"sampled_ids": ([c.get("id") for c, _ in mandatory]
|
||||
+ [c.get("id") for c in sampled_critical]
|
||||
+ [c.get("id") for c in sampled_normal]),
|
||||
}
|
||||
|
||||
|
||||
def filter_golden_tasks(tasks):
|
||||
"""Drop sample entries from a golden-set candidate list.
|
||||
|
||||
A task is a sample (and never counts as golden) when it is
|
||||
flagged ``sample=True`` or its id starts with ``sample-``.
|
||||
Returns ``{"golden": [...], "dropped_samples": int}``.
|
||||
"""
|
||||
golden = []
|
||||
dropped = 0
|
||||
for task in tasks:
|
||||
task_id = task.get("id", "") if isinstance(task, dict) else ""
|
||||
is_sample = (isinstance(task, dict) and task.get("sample")) \
|
||||
or str(task_id).startswith("sample-")
|
||||
if is_sample:
|
||||
dropped += 1
|
||||
else:
|
||||
golden.append(task)
|
||||
return {"golden": golden, "dropped_samples": dropped}
|
||||
|
||||
|
||||
def golden_approval_checklist(tasks, approver=None, approved_at=None,
|
||||
list_reviewed=False,
|
||||
criteria_defined=False):
|
||||
"""Approval checklist for a golden set. Every item is
|
||||
``pending_human`` until a human completes it (via the
|
||||
``list_reviewed`` / ``criteria_defined`` confirmations, plus
|
||||
approver + timestamp); the set is approved only when every item
|
||||
is done AND at least one non-sample golden task remains. Sample
|
||||
files never satisfy an item (see ``filter_golden_tasks``)."""
|
||||
filtered = filter_golden_tasks(tasks)
|
||||
items = [
|
||||
{"item": "task_list_reviewed",
|
||||
"status": "done" if list_reviewed else "pending_human",
|
||||
"note": "a human reviewed the golden task list"},
|
||||
{"item": "acceptance_criteria_defined",
|
||||
"status": "done" if criteria_defined else "pending_human",
|
||||
"note": "each golden task has human-defined acceptance criteria"},
|
||||
{"item": "sample_files_excluded",
|
||||
"status": "done",
|
||||
"note": "%d sample entries excluded from the golden set "
|
||||
"(exclusion is applied by filter_golden_tasks, never "
|
||||
"by trusting the candidate list)"
|
||||
% filtered["dropped_samples"]},
|
||||
{"item": "approver_and_timestamp_recorded",
|
||||
"status": "done" if approver and approved_at else "pending_human",
|
||||
"note": "approver and approval timestamp recorded"},
|
||||
]
|
||||
approved = all(i["status"] == "done" for i in items) \
|
||||
and bool(filtered["golden"]) and bool(approver) \
|
||||
and bool(approved_at)
|
||||
return {"approved": approved, "items": items,
|
||||
"golden_tasks": filtered["golden"],
|
||||
"dropped_samples": filtered["dropped_samples"]}
|
||||
|
||||
|
||||
def handle_sampling_result(case, sampling_verdict, corrected_verdict=None,
|
||||
corrector=None, original_signer=None):
|
||||
"""Turn one human sampling result into a record entry + actions.
|
||||
|
||||
``sampling_verdict`` is ``"correct"`` or ``"incorrect"``.
|
||||
When incorrect, ``corrected_verdict`` (the audit decision that
|
||||
should have been issued) must be supplied; the retrigger follows
|
||||
the ORIGINAL decision via ``_RETRIGGER``, or escalates when the
|
||||
original decision is unknown. Returns the entry dict; the caller
|
||||
appends it with ``append_audit_record``. This function never
|
||||
mutates ``case``: the original conclusion is preserved and the
|
||||
correction lives only in the returned entry.
|
||||
|
||||
Corrector separation (track 3 rule, enforced here when both
|
||||
names are supplied): ``corrector`` must differ from
|
||||
``original_signer``. Gate effect (shared with track 3 / wiring
|
||||
#247, stated here so the retrigger names cannot be misread):
|
||||
a correction never unblocks a merge by itself -- a correction
|
||||
to ``pass`` schedules a re-audit (``cancel_pushback_and_rerun_audit``
|
||||
== track 3 ``schedule_reaudit``) and the gate stays blocked
|
||||
until a new passing re-audit exists; a correction to ``fail``
|
||||
re-triggers pushback and re-blocks at once
|
||||
(``reopen_audit_and_pushback`` == track 3
|
||||
``retrigger_pushback_and_reblock``).
|
||||
"""
|
||||
case_id = case.get("id")
|
||||
original = case.get("decision")
|
||||
if sampling_verdict not in ("correct", "incorrect"):
|
||||
raise ValueError("sampling_verdict must be 'correct' or "
|
||||
"'incorrect', got %r" % (sampling_verdict,))
|
||||
if (corrector is not None and original_signer is not None
|
||||
and corrector == original_signer):
|
||||
raise ValueError("corrector must differ from original signer")
|
||||
if sampling_verdict == "correct":
|
||||
return {"case_id": case_id, "original_decision": original,
|
||||
"sampling_verdict": "correct",
|
||||
"corrected_decision": None, "retrigger": None,
|
||||
"recorded_in": "audit_of_audit"}
|
||||
if corrected_verdict is None:
|
||||
raise ValueError("an incorrect sampling verdict requires a "
|
||||
"corrected_verdict")
|
||||
retrigger = _RETRIGGER.get(original, "escalate_to_human")
|
||||
return {"case_id": case_id, "original_decision": original,
|
||||
"sampling_verdict": "incorrect",
|
||||
"corrected_decision": corrected_verdict,
|
||||
"retrigger": retrigger,
|
||||
"recorded_in": "audit_of_audit"}
|
||||
|
||||
|
||||
def append_audit_record(log, entry):
|
||||
"""Append one sampling entry to the audit-of-audit record.
|
||||
|
||||
Returns a new list; the caller persists it. A sampling result
|
||||
that is not appended here is treated as not having happened.
|
||||
"""
|
||||
return list(log) + [entry]
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,746 @@
|
||||
"""Wiring between three already-landed tracks (no new semantics here):
|
||||
|
||||
- track 1 (``cloud/merge_gate_hard.py``, PR #240): the hard merge
|
||||
gate, five checks + branch protection, machine-readable ``blocks``
|
||||
codes on its 409 response;
|
||||
- track 3 (``cloud/audit_pushback.py``, PR #244): the independent
|
||||
audit + auto-pushback decision, the pushback package, and the QA
|
||||
re-run rule;
|
||||
- track 4 (``cloud/ledger.py``, PR #241): the append-only hash-chain
|
||||
ledger, the only durable store in this wiring.
|
||||
|
||||
What this module owns, and nothing else:
|
||||
|
||||
1. Translation at the boundary. Track-3 field names and enums are
|
||||
authoritative for callers; track-4 storage field names are
|
||||
authoritative on the chain. This module translates, it does not
|
||||
invent a second ledger schema:
|
||||
|
||||
============================ ===================================
|
||||
track-3 event chain record (track 4 action)
|
||||
============================ ===================================
|
||||
audit verdict ``audit_verdict``
|
||||
pushback round (pushed) ``audit_pushback_round`` (pushed)
|
||||
fix (landed, re-audit open) ``audit_pushback_round`` with
|
||||
``fix_diff`` + package marker
|
||||
``wiring_event="fix"`` (evidence
|
||||
row only; see the gate reader)
|
||||
fix + re-audit ``audit_pushback_round``
|
||||
(re_reviewed, ``fix_diff`` +
|
||||
``re_review_result``)
|
||||
rebuild decision / done ``audit_rebuild``
|
||||
third-failure triage ``audit_third_failure_assessment``
|
||||
test-change single audit ``audit_test_change``
|
||||
audit-of-audit correction ``audit_meta_review``
|
||||
rebuild notice / confirmation ``audit_rebuild_notice``
|
||||
escalation / blocked quota / ``audit_wiring_event`` (the one
|
||||
audit sample compat action added to track 4
|
||||
by this wiring; see below)
|
||||
============================ ===================================
|
||||
|
||||
2. CONDITIONAL mapping, done here and only here. Track 4 stores
|
||||
verdicts as ``pass`` / ``fail`` only, and track 3 says
|
||||
CONDITIONAL is not a pass. Every verdict (audit verdict and
|
||||
re-audit verdict) that arrives as ``"conditional"`` -- or as
|
||||
anything that is not exactly ``"pass"`` -- is stored as
|
||||
``"fail"``, with the original value preserved in
|
||||
``extra.original_verdict`` (verdict rows) or in
|
||||
``re_review_result.original_verdict`` (pushback rows). The
|
||||
original conclusion is never overwritten: the chain keeps the
|
||||
mapped value AND the original, and corrections are separate
|
||||
``audit_meta_review`` rows, never edits.
|
||||
|
||||
3. The gate reader (``pushback_block_for_task``). It replays a
|
||||
task's chain rows in id order and returns a ``gate_block()``-
|
||||
shaped dict (``merge_blocked`` / ``release_blocked`` /
|
||||
``blocked_by="audit_pushback"`` / ``round`` / ``failed_checks``,
|
||||
plus a ``modules`` detail map). ``cloud/server.py`` ANDs this
|
||||
with the hard gate's own five checks: either side blocks and
|
||||
the merge returns 409 with the task status unchanged. The
|
||||
pushback block applies in every gate mode -- an open audit
|
||||
pushback is not something a ``record``-mode caller can confirm
|
||||
away, and ``merge_gate_confirm`` never touches it.
|
||||
|
||||
4. The gate-blocks-to-package bridge
|
||||
(``pushback_package_from_gate``). A hard-gate 409's ``blocks``
|
||||
codes are machine input to the pushback package: each code maps
|
||||
to a track-3 gate check (``ci_*`` -> ``ci_green``,
|
||||
``directory_*`` -> ``directory_isolation``,
|
||||
``contract_*`` -> ``contract_version``,
|
||||
``coordinator_*`` -> ``coordinator_approval``,
|
||||
``review_*`` / ``unresolved_review_comments`` ->
|
||||
``unresolved_review_comments``). Branch-protection codes have
|
||||
no track-3 gate-check member, so they travel as the extra check
|
||||
name ``"branch_protection"`` in the package's failed items --
|
||||
they are never dropped, and they are never disguised as one of
|
||||
the ten checks. Callers must take the codes from the 409
|
||||
``merge_gate`` object, never by parsing audit detail text.
|
||||
|
||||
Known translation losses (stated, not hidden):
|
||||
|
||||
- Track 3 distinguishes root causes ``"contract"`` vs ``"design"``;
|
||||
track 4 stores only ``"implementation"`` / ``"contract_design"``.
|
||||
Both contract and design map to ``"contract_design"`` on rebuild
|
||||
rows (track 3's rebuild branch treats them identically: refreeze,
|
||||
no code rebuild). The finer distinction survives on escalation
|
||||
rows, which carry the original ``root_cause`` verbatim.
|
||||
- Track 3's third-failure triage verdict ``"early_refreeze"`` has
|
||||
no track-4 assessment determination (only ``early_rebuild`` /
|
||||
``continuation_round`` exist there). It is recorded as an
|
||||
``audit_wiring_event`` of kind ``escalate_refreeze``, never
|
||||
forced into an assessment row with the wrong determination.
|
||||
- Track 3's ``"unknown"`` root cause has no track-4 value and is
|
||||
refused (``ValueError``) rather than guessed: a rebuild row must
|
||||
say which side the root cause is on.
|
||||
- A fix-only row is counted by track 4's timeline summary inside
|
||||
``pushback_rounds`` (that summary counts rows, and a round
|
||||
already spans two rows there). The gate reader skips fix rows
|
||||
for state transitions, so the count wart cannot unblock or block
|
||||
a merge by itself.
|
||||
|
||||
What this module does NOT do:
|
||||
|
||||
- It does not change any track-1 / track-3 / track-4 semantic:
|
||||
hard stays fail-closed by default, protection unknown /
|
||||
unprotected still block in hard mode, confirmation still never
|
||||
covers a directory violation, the chain stays fail-closed and
|
||||
append-only, and the cost dual-column rule is untouched.
|
||||
- It does not create pushback subtasks, send notices, or run
|
||||
fixes. ``record_*`` only appends chain rows; deriving a fix
|
||||
subtask from a package is the caller's job (track 3's
|
||||
``build_pushback_package`` already describes its shape).
|
||||
- It does not gate ``deploy.py``. There is no server-side release
|
||||
endpoint in this codebase: the merge endpoint is where
|
||||
``merge_blocked`` is enforced, and ``release_blocked`` is
|
||||
returned alongside it so a future release endpoint can enforce
|
||||
it with this same reader. Deploy remains an operator action
|
||||
outside this wiring.
|
||||
"""
|
||||
|
||||
from cloud.audit_pushback import GATE_CHECKS
|
||||
from cloud.audit_pushback import build_pushback_package as _build_package
|
||||
from cloud.audit_pushback import gate_block as _gate_block
|
||||
from cloud.ledger import WIRING_EVENT_ACTION
|
||||
from cloud.ledger import audit_timeline as _audit_timeline
|
||||
from cloud.merge_gate_hard import block_descriptions as _block_descriptions
|
||||
from cloud.qa_matrix import (
|
||||
check_reaudit_reran_links as _check_reran_links,
|
||||
)
|
||||
|
||||
# Hard-gate block code -> track-3 gate check. Branch-protection
|
||||
# codes have no track-3 member; see BRANCH_PROTECTION_CHECK.
|
||||
_GATE_BLOCK_TO_CHECK = {}
|
||||
for _code in (
|
||||
"ci_evidence_missing", "ci_not_green", "ci_check_failed",
|
||||
"ci_required_checks_unconfigured", "ci_required_check_missing"):
|
||||
_GATE_BLOCK_TO_CHECK[_code] = "ci_green"
|
||||
for _code in (
|
||||
"directory_evidence_missing", "directory_owned_dir_unknown",
|
||||
"directory_isolation_violation"):
|
||||
_GATE_BLOCK_TO_CHECK[_code] = "directory_isolation"
|
||||
for _code in (
|
||||
"contract_version_missing", "contract_not_frozen",
|
||||
"contract_version_mismatch"):
|
||||
_GATE_BLOCK_TO_CHECK[_code] = "contract_version"
|
||||
for _code in (
|
||||
"coordinator_approval_missing", "coordinator_not_approved"):
|
||||
_GATE_BLOCK_TO_CHECK[_code] = "coordinator_approval"
|
||||
for _code in ("review_evidence_missing", "unresolved_review_comments"):
|
||||
_GATE_BLOCK_TO_CHECK[_code] = "unresolved_review_comments"
|
||||
# Branch-protection codes (hard-mode codes from track 1's #240
|
||||
# layer plus record-mode codes from the #209 layer).
|
||||
for _code in ("branch_protection_unknown",
|
||||
"branch_protection_unprotected",
|
||||
"branch_protection_enforce",
|
||||
"branch_protection_risk_unconfirmed"):
|
||||
_GATE_BLOCK_TO_CHECK[_code] = "branch_protection"
|
||||
|
||||
BRANCH_PROTECTION_CHECK = "branch_protection"
|
||||
|
||||
# Track-3 decision actions that block merge/release (mirrors
|
||||
# track 3's gate_block() membership; kept here so the package
|
||||
# builder does not have to re-derive it).
|
||||
BLOCKING_ACTIONS = (
|
||||
"pushback", "rebuild", "escalate", "escalate_refreeze",
|
||||
"blocked_quota", "rebuild_pending_user_confirmation",
|
||||
)
|
||||
|
||||
# Wiring-event kinds that block (compat action rows).
|
||||
_BLOCKING_WIRING_KINDS = ("escalate", "escalate_refreeze",
|
||||
"blocked_quota")
|
||||
|
||||
# Triage verdict -> track-4 assessment determination. "early_refreeze"
|
||||
# is deliberately absent: it goes to record_escalation instead.
|
||||
_TRIAGE_TO_DETERMINATION = {
|
||||
"early_rebuild": "early_rebuild",
|
||||
"targeted_fix": "continuation_round",
|
||||
}
|
||||
|
||||
# Track-3 root cause -> track-4 root cause. "unknown" is absent
|
||||
# on purpose (refused, not guessed).
|
||||
_ROOT_CAUSE_MAP = {
|
||||
"implementation": "implementation",
|
||||
"contract": "contract_design",
|
||||
"design": "contract_design",
|
||||
}
|
||||
|
||||
|
||||
def normalize_verdict(verdict):
|
||||
"""Track-3 verdict -> (track-4 verdict, original).
|
||||
|
||||
Exactly ``"pass"`` stays a pass; anything else -- including
|
||||
``"conditional"`` and missing/garbage values -- maps to
|
||||
``"fail"`` (fail-closed), with the original preserved for the
|
||||
chain. The caller stores the original alongside; this function
|
||||
never drops it silently.
|
||||
"""
|
||||
if verdict == "pass":
|
||||
return "pass", "pass"
|
||||
original = verdict if isinstance(verdict, str) else None
|
||||
return "fail", original
|
||||
|
||||
|
||||
def normalize_re_review_result(result):
|
||||
"""Copy of a re-review result with its verdict normalized.
|
||||
|
||||
``CONDITIONAL`` re-audits do not pass (track 3); the stored
|
||||
verdict becomes ``"fail"`` and the original is kept under
|
||||
``original_verdict``. A result without a verdict dict is
|
||||
returned unchanged (the ledger validates the shape).
|
||||
"""
|
||||
if not isinstance(result, dict):
|
||||
return result
|
||||
copied = dict(result)
|
||||
normalized, original = normalize_verdict(result.get("verdict"))
|
||||
copied["verdict"] = normalized
|
||||
if normalized == "fail" and original != "fail":
|
||||
copied["original_verdict"] = original
|
||||
return copied
|
||||
|
||||
|
||||
def gate_failed_checks(gate):
|
||||
"""Hard-gate 409 -> (track-3 checks, extra checks), in order.
|
||||
|
||||
Track-3 checks come first in GATE_CHECKS order, then the
|
||||
branch-protection extra (if any). Unknown codes pass through
|
||||
as extras rather than being dropped: a code the wiring does
|
||||
not recognise must still show up in the package.
|
||||
"""
|
||||
checks = []
|
||||
extras = []
|
||||
for code in (gate or {}).get("blocks") or []:
|
||||
mapped = _GATE_BLOCK_TO_CHECK.get(code)
|
||||
if mapped == BRANCH_PROTECTION_CHECK:
|
||||
if BRANCH_PROTECTION_CHECK not in extras:
|
||||
extras.append(BRANCH_PROTECTION_CHECK)
|
||||
elif mapped is not None:
|
||||
if mapped not in checks:
|
||||
checks.append(mapped)
|
||||
elif code not in extras:
|
||||
extras.append(code)
|
||||
order = {name: index for index, name in enumerate(GATE_CHECKS)}
|
||||
checks.sort(key=lambda name: order.get(name, len(order)))
|
||||
return checks, extras
|
||||
|
||||
|
||||
def gate_block_items(gate):
|
||||
"""One pushback-package failed item per hard-gate block code.
|
||||
|
||||
The package builder also appends decision ``failed_checks``
|
||||
that are not already items, so the item ``check`` values here
|
||||
use the mapped track-3 names (branch protection keeps its
|
||||
extra name) to avoid duplicate generic entries.
|
||||
"""
|
||||
descriptions = _block_descriptions((gate or {}).get("blocks") or [])
|
||||
items = []
|
||||
for code, description in zip((gate or {}).get("blocks") or [],
|
||||
descriptions):
|
||||
check = _GATE_BLOCK_TO_CHECK.get(code, code)
|
||||
items.append({
|
||||
"check": check,
|
||||
"detail": description,
|
||||
"evidence": {
|
||||
"test_name": None,
|
||||
"file_path": None,
|
||||
"line_no": None,
|
||||
"contract_clause": None,
|
||||
"diff_range": None,
|
||||
},
|
||||
"reproduction_steps": [
|
||||
"re-run the merge gate with full evidence; "
|
||||
"the failing code is %s" % code,
|
||||
],
|
||||
"reaudit_criterion": "re-run the full re-audit checklist "
|
||||
"for this check",
|
||||
})
|
||||
return items
|
||||
|
||||
|
||||
def pushback_package_from_gate(gate, task_id, round_no,
|
||||
decision_action="pushback",
|
||||
previous_artifacts=None,
|
||||
qa_failed_links=None):
|
||||
"""Build a track-3 pushback package from a hard-gate 409.
|
||||
|
||||
``gate`` is the ``merge_gate`` object from the 409 response
|
||||
(not audit text). ``round_no`` is the track-3 decision round
|
||||
(0 for the first audit); the ledger round number is derived
|
||||
by the caller (see ``record_pushback``'s docstring). The
|
||||
package carries the QA failed links verbatim when given, so
|
||||
the re-audit must re-run exactly those links (see
|
||||
``record_reaudit``'s QA guard).
|
||||
"""
|
||||
if decision_action not in BLOCKING_ACTIONS and \
|
||||
decision_action != "allow":
|
||||
raise ValueError("unknown decision action: %r" % (decision_action,))
|
||||
checks, extras = gate_failed_checks(gate)
|
||||
audit = {
|
||||
"task_id": task_id,
|
||||
"verdict": "fail",
|
||||
"verdict_text": "merge gate blocked: %s" % (
|
||||
", ".join((gate or {}).get("blocks") or ["unknown"])),
|
||||
"failed_items": gate_block_items(gate),
|
||||
}
|
||||
decision = {
|
||||
"round": round_no,
|
||||
"next_round": round_no + 1,
|
||||
"action": decision_action,
|
||||
"failed_checks": checks + extras,
|
||||
}
|
||||
return _build_package(audit, decision, previous_artifacts,
|
||||
qa_failed_links=qa_failed_links)
|
||||
|
||||
|
||||
def _require_module_contract(module_id, contract_version):
|
||||
"""Wiring strictness: pushback rows must be attributable.
|
||||
|
||||
Track 4 allows module/contract to be None on pushback rows,
|
||||
but unattributed rows cannot be scoped by the gate reader, so
|
||||
this wiring refuses them instead of writing rows the gate
|
||||
cannot reason about. Verdict/rebuild rows already require
|
||||
both in track 4 itself.
|
||||
"""
|
||||
if not isinstance(module_id, str) or not module_id:
|
||||
raise ValueError("module_id is required for wiring records")
|
||||
if not isinstance(contract_version, str) or not contract_version:
|
||||
raise ValueError("contract_version is required for wiring records")
|
||||
|
||||
|
||||
def record_audit_verdict(store, task_id, project_id, module_id,
|
||||
contract_version, verdict, source,
|
||||
failures=None, user_id="system", extra=None):
|
||||
"""Append one audit/acceptance verdict to the chain.
|
||||
|
||||
``verdict`` uses track-3 vocabulary (``pass`` / ``fail`` /
|
||||
``conditional``); ``"conditional"`` is stored as ``"fail"``
|
||||
with the original in ``extra.original_verdict``.
|
||||
"""
|
||||
normalized, original = normalize_verdict(verdict)
|
||||
merged_extra = dict(extra or {})
|
||||
if normalized == "fail" and original != "fail":
|
||||
merged_extra.setdefault("original_verdict", original)
|
||||
return store.append_audit_verdict(
|
||||
task_id, project_id, module_id, contract_version,
|
||||
normalized, source, failures=failures, user_id=user_id,
|
||||
extra=merged_extra)
|
||||
|
||||
|
||||
def record_pushback(store, task_id, project_id, module_id,
|
||||
contract_version, round_no, trigger_items,
|
||||
package, failure_source=None, failure_counts=None,
|
||||
user_id="system"):
|
||||
"""Append one pushed pushback round to the chain.
|
||||
|
||||
``round_no`` is the LEDGER round number, starting at 1 for
|
||||
the first pushback. Track 3's ``decide_pushback`` counts
|
||||
rounds already used (0 for the first audit) and returns
|
||||
``next_round``; for a pushback action that value IS the
|
||||
ledger round number (first failure: decision round 0,
|
||||
next_round 1, ledger round_no 1). Trigger items are the
|
||||
package's failed checks/items (machine codes, not prose).
|
||||
"""
|
||||
_require_module_contract(module_id, contract_version)
|
||||
return store.append_pushback_round(
|
||||
task_id, project_id, round_no, trigger_items, package,
|
||||
user_id=user_id, module_id=module_id,
|
||||
contract_version=contract_version,
|
||||
failure_source=failure_source, failure_counts=failure_counts)
|
||||
|
||||
|
||||
def record_fix(store, task_id, project_id, module_id,
|
||||
contract_version, round_no, fix_diff, package,
|
||||
user_id="system"):
|
||||
"""Append fix evidence for a round whose re-audit is still open.
|
||||
|
||||
The row carries ``fix_diff`` (summary + sha, never the full
|
||||
diff) and a package copy marked ``wiring_event="fix"``. The
|
||||
gate reader skips such rows for state transitions: landing a
|
||||
fix does not unblock anything -- only a passing re-audit does.
|
||||
When fix and re-audit arrive together, use ``record_reaudit``
|
||||
with ``fix_diff`` instead of writing both rows.
|
||||
"""
|
||||
_require_module_contract(module_id, contract_version)
|
||||
marked = dict(package or {})
|
||||
marked["wiring_event"] = "fix"
|
||||
return store.append_pushback_round(
|
||||
task_id, project_id, round_no, _package_trigger_items(package),
|
||||
marked, fix_diff=fix_diff, user_id=user_id,
|
||||
module_id=module_id, contract_version=contract_version)
|
||||
|
||||
|
||||
def _package_trigger_items(package):
|
||||
"""Trigger items for a fix row: the package's failed checks."""
|
||||
if isinstance(package, dict):
|
||||
checks = package.get("failed_checks")
|
||||
if isinstance(checks, list) and checks:
|
||||
return list(checks)
|
||||
return ["audit_fix"]
|
||||
|
||||
|
||||
def check_reaudit_qa_guard(package, re_review_result, rerun_results):
|
||||
"""Fail-closed QA guard for a claimed passing re-audit.
|
||||
|
||||
If the package lists ``reaudit_must_rerun_links`` and the
|
||||
re-audit claims a pass, every listed link must have a passing
|
||||
re-run result; a missing or failed re-run raises ValueError
|
||||
instead of letting a false pass reach the chain. A failing
|
||||
re-audit needs no guard (it stays blocked either way).
|
||||
Returns the normalized re-review result.
|
||||
"""
|
||||
normalized_result = normalize_re_review_result(re_review_result)
|
||||
if normalized_result.get("verdict") != "pass":
|
||||
return normalized_result
|
||||
required = (package or {}).get("reaudit_must_rerun_links") or []
|
||||
if not required:
|
||||
return normalized_result
|
||||
checked = _check_reran_links(package, rerun_results or {})
|
||||
if not checked["reran_all_failed_links"]:
|
||||
raise ValueError(
|
||||
"re-audit claims pass but QA failed links were not "
|
||||
"all re-run passing: missing=%s failed=%s"
|
||||
% (checked["missing"], checked["failed"]))
|
||||
return normalized_result
|
||||
|
||||
|
||||
def record_reaudit(store, task_id, project_id, module_id,
|
||||
contract_version, round_no, re_review_result,
|
||||
package=None, fix_diff=None, rerun_results=None,
|
||||
failure_source=None, failure_counts=None,
|
||||
user_id="system"):
|
||||
"""Append one re-audited pushback round to the chain.
|
||||
|
||||
This is the row that can unblock: a stored ``pass`` verdict
|
||||
here clears the module's block in the gate reader, a stored
|
||||
``fail`` (including a mapped ``conditional``) keeps it
|
||||
blocked. The QA guard in ``check_reaudit_qa_guard`` runs
|
||||
before anything is written, so a claimed pass with un-rerun
|
||||
or failed QA links never reaches the chain.
|
||||
"""
|
||||
_require_module_contract(module_id, contract_version)
|
||||
guarded = check_reaudit_qa_guard(package, re_review_result,
|
||||
rerun_results)
|
||||
return store.append_pushback_round(
|
||||
task_id, project_id, round_no,
|
||||
_package_trigger_items(package), package or {},
|
||||
fix_diff=fix_diff, re_review_result=guarded,
|
||||
user_id=user_id, module_id=module_id,
|
||||
contract_version=contract_version,
|
||||
failure_source=failure_source, failure_counts=failure_counts)
|
||||
|
||||
|
||||
def record_rebuild(store, task_id, project_id, module_id,
|
||||
contract_version, rebuild_no, old_impl_ref,
|
||||
new_impl_ref=None, root_cause="implementation",
|
||||
status="decided", prior_failure_total=None,
|
||||
retained_assets=None, trigger_source=None,
|
||||
assessment_entry_id=None, user_id="system"):
|
||||
"""Append one rebuild decision / completion to the chain.
|
||||
|
||||
``root_cause`` uses track-3 vocabulary; ``"contract"`` and
|
||||
``"design"`` both store as track-4 ``"contract_design"``
|
||||
(see the module docstring). ``"unknown"`` is refused.
|
||||
A ``rebuilt`` row does NOT unblock the gate on its own: the
|
||||
new implementation still needs a passing re-audit.
|
||||
"""
|
||||
mapped = _ROOT_CAUSE_MAP.get(root_cause)
|
||||
if mapped is None:
|
||||
raise ValueError(
|
||||
"root_cause must be implementation|contract|design, "
|
||||
"not %r (unknown is refused, not guessed)" % (root_cause,))
|
||||
return store.append_rebuild_decision(
|
||||
task_id, project_id, module_id, contract_version,
|
||||
rebuild_no, old_impl_ref, new_impl_ref=new_impl_ref,
|
||||
root_cause=mapped, status=status,
|
||||
prior_failure_total=prior_failure_total,
|
||||
user_id=user_id, retained_assets=retained_assets,
|
||||
trigger_source=trigger_source,
|
||||
assessment_entry_id=assessment_entry_id)
|
||||
|
||||
|
||||
def record_third_failure_assessment(store, task_id, project_id,
|
||||
module_id, contract_version,
|
||||
triage_verdict, basis,
|
||||
failure_total=3,
|
||||
max_extra_rounds=None,
|
||||
user_id="system"):
|
||||
"""Append one third-failure triage determination to the chain.
|
||||
|
||||
Track-3 triage verdicts map as ``early_rebuild`` ->
|
||||
``early_rebuild`` and ``targeted_fix`` -> ``continuation_round``.
|
||||
``early_refreeze`` has no assessment determination in track 4
|
||||
and is refused here: record it with ``record_escalation``
|
||||
(kind ``escalate_refreeze``) instead.
|
||||
"""
|
||||
determination = _TRIAGE_TO_DETERMINATION.get(triage_verdict)
|
||||
if determination is None:
|
||||
raise ValueError(
|
||||
"triage_verdict %r has no assessment determination; "
|
||||
"use record_escalation(kind='escalate_refreeze') for "
|
||||
"early_refreeze" % (triage_verdict,))
|
||||
return store.append_third_failure_assessment(
|
||||
task_id, project_id, module_id, contract_version,
|
||||
determination, basis, failure_total=failure_total,
|
||||
max_extra_rounds=max_extra_rounds, user_id=user_id)
|
||||
|
||||
|
||||
def record_test_change(store, task_id, project_id, module_id,
|
||||
contract_version, test_changes, auditor,
|
||||
contract_refrozen, refreeze_ref=None,
|
||||
reason="", user_id="system"):
|
||||
"""Append one test-change single audit (track-4 native)."""
|
||||
return store.append_test_change_audit(
|
||||
task_id, project_id, module_id, contract_version,
|
||||
test_changes, auditor, contract_refrozen,
|
||||
refreeze_ref=refreeze_ref, reason=reason, user_id=user_id)
|
||||
|
||||
|
||||
def record_meta_review(store, task_id, project_id, module_id,
|
||||
contract_version, original_entry_id,
|
||||
original_verdict, corrected_verdict,
|
||||
misjudgment_found, reviewer, findings=None,
|
||||
triggered_process=None, user_id="system"):
|
||||
"""Append one audit-of-audit correction (history is kept).
|
||||
|
||||
The correction never rewrites the original row. How the gate
|
||||
reader treats it is a resolved semantic tension, stated here
|
||||
rather than buried (see ``pushback_block_for_task``): a
|
||||
correction to ``fail`` (a wrong pass) re-blocks immediately,
|
||||
but a correction to ``pass`` (a wrong block) does NOT unblock
|
||||
by itself -- track 3's restored action for that case is
|
||||
"schedule a re-audit", not "release the gate", and a
|
||||
meta-review is a review of an audit, not a full re-audit.
|
||||
Unblocking still needs a fresh passing verdict or a passing
|
||||
re-reviewed pushback row. Track 4's failure counts are
|
||||
unaffected by this choice: counts use effective (corrected)
|
||||
verdicts to decide rebuild triggers, while the gate asks a
|
||||
different question ("has a passing re-audit been recorded
|
||||
since the block?"), so the two may legitimately disagree
|
||||
while a scheduled re-audit is still pending.
|
||||
"""
|
||||
return store.append_audit_of_audit(
|
||||
task_id, project_id, module_id, contract_version,
|
||||
original_entry_id, original_verdict, corrected_verdict,
|
||||
misjudgment_found, reviewer, findings=findings,
|
||||
triggered_process=triggered_process, user_id=user_id)
|
||||
|
||||
|
||||
def record_rebuild_notice(store, task_id, project_id, module_id,
|
||||
contract_version, rebuild_no,
|
||||
notification_content, notified_party,
|
||||
confirmation_status, confirmed_by=None,
|
||||
user_id="system"):
|
||||
"""Append one rebuild-notice row (pending -> confirmed are two rows)."""
|
||||
return store.append_rebuild_notice(
|
||||
task_id, project_id, module_id, contract_version,
|
||||
rebuild_no, notification_content, notified_party,
|
||||
confirmation_status, confirmed_by=confirmed_by,
|
||||
user_id=user_id)
|
||||
|
||||
|
||||
def record_escalation(store, task_id, project_id, module_id,
|
||||
contract_version, kind, target=None, reason="",
|
||||
round_no=None, root_cause=None,
|
||||
failed_checks=None, user_id="system"):
|
||||
"""Append one escalation / quota / audit-sample wiring event.
|
||||
|
||||
``kind`` is track-3 vocabulary: ``escalate``,
|
||||
``escalate_refreeze``, ``blocked_quota``, ``audit_sample``.
|
||||
Escalation kinds block the gate; ``audit_sample`` (a sampling
|
||||
selection, not a verdict) does not change gate state. The
|
||||
original ``root_cause`` (including the contract/design
|
||||
distinction) is stored verbatim here.
|
||||
"""
|
||||
_require_module_contract(module_id, contract_version)
|
||||
return store.append_wiring_event(
|
||||
task_id, project_id, module_id, contract_version, kind,
|
||||
target=target, reason=reason, round_no=round_no,
|
||||
root_cause=root_cause, failed_checks=failed_checks,
|
||||
user_id=user_id)
|
||||
|
||||
|
||||
def _trigger_check_names(trigger_items):
|
||||
"""Trigger items (str or dict) -> check-name strings."""
|
||||
names = []
|
||||
for item in trigger_items or []:
|
||||
if isinstance(item, str):
|
||||
names.append(item)
|
||||
elif isinstance(item, dict) and item.get("check"):
|
||||
names.append(item["check"])
|
||||
else:
|
||||
names.append(str(item))
|
||||
return names
|
||||
|
||||
|
||||
def _module_key(parsed):
|
||||
"""(module_id, contract_version) or None for unattributed rows."""
|
||||
module_id = parsed.get("module_id") if isinstance(parsed, dict) else None
|
||||
contract_version = parsed.get("contract_version") \
|
||||
if isinstance(parsed, dict) else None
|
||||
if module_id and contract_version:
|
||||
return (module_id, contract_version)
|
||||
return None
|
||||
|
||||
|
||||
def pushback_block_for_task(conn, task_id):
|
||||
"""Replay a task's chain into a gate_block()-shaped verdict.
|
||||
|
||||
State machine, in chain-id order, per (module, contract
|
||||
version) plus one global slot for unattributed rows:
|
||||
|
||||
- verdict row: its stored verdict sets blocked (pass clears,
|
||||
fail blocks).
|
||||
- meta-review row: a correction to ``fail`` blocks at once
|
||||
(a wrong pass is re-blocked); a correction to ``pass``
|
||||
changes nothing by itself -- the wrong block stays blocked
|
||||
until a fresh passing verdict or re-reviewed row arrives
|
||||
(track 3's "schedule a re-audit", not "release the gate").
|
||||
This is deliberately NOT track 4's effective-verdict
|
||||
counting: counts and gate state answer different
|
||||
questions, and conflating them would let a review of an
|
||||
audit substitute for a re-audit.
|
||||
- pushback pushed: blocked. pushback re_reviewed: blocked
|
||||
unless the stored re-audit verdict is a pass (a stored
|
||||
``conditional`` was already mapped to fail at write time;
|
||||
anything non-pass here still blocks, fail-closed).
|
||||
- pushback fix-evidence row (package ``wiring_event="fix"``):
|
||||
no state change.
|
||||
- rebuild row (decided or rebuilt): blocked -- a rebuilt
|
||||
implementation still needs a passing re-audit.
|
||||
- wiring event escalate / escalate_refreeze / blocked_quota:
|
||||
blocked. audit_sample: no change.
|
||||
- rebuild notice pending: blocked. confirmed / rejected: no
|
||||
change (confirmation alone never unblocks).
|
||||
- assessment / test-change / meta rows: no direct change
|
||||
(meta rows act only through corrections).
|
||||
|
||||
A task is blocked if the global slot or any module slot is
|
||||
blocked. A missing ``audit_ledger`` table means no chain has
|
||||
ever been written for this database, which is unblocked, not
|
||||
an error; any other read failure propagates so the server can
|
||||
fail closed (see server.py).
|
||||
"""
|
||||
try:
|
||||
timeline = _audit_timeline(conn, task_id=task_id, limit=10000)
|
||||
except Exception as exc:
|
||||
if "no such table" in str(exc):
|
||||
return _gate_block("allow", [], 0)
|
||||
raise
|
||||
entries = timeline.get("entries") or []
|
||||
modules = {}
|
||||
global_state = {"blocked": False, "round": None, "checks": []}
|
||||
last_blocked_round = None
|
||||
|
||||
def slot_for(parsed):
|
||||
key = _module_key(parsed)
|
||||
if key is None:
|
||||
return None, global_state
|
||||
if key not in modules:
|
||||
modules[key] = {"blocked": False, "round": None,
|
||||
"checks": []}
|
||||
return key, modules[key]
|
||||
|
||||
for entry in entries:
|
||||
parsed = entry.get("parsed")
|
||||
if not isinstance(parsed, dict):
|
||||
continue
|
||||
action = entry.get("action")
|
||||
_key, slot = slot_for(parsed)
|
||||
if action == "audit_verdict":
|
||||
blocked = parsed.get("verdict") != "pass"
|
||||
slot["blocked"] = blocked
|
||||
slot["checks"] = _trigger_check_names(
|
||||
parsed.get("failures")) if blocked else []
|
||||
elif action == "audit_meta_review":
|
||||
# Re-block on a corrected fail; a corrected pass
|
||||
# schedules a re-audit and does not unblock (see the
|
||||
# module docstring: correction != re-audit).
|
||||
if parsed.get("misjudgment_found") and \
|
||||
parsed.get("corrected_verdict") == "fail":
|
||||
slot["blocked"] = True
|
||||
if not slot["checks"]:
|
||||
slot["checks"] = ["audit_meta_review"]
|
||||
elif action == "audit_pushback_round":
|
||||
package = parsed.get("pushback_package") or {}
|
||||
if package.get("wiring_event") == "fix":
|
||||
continue
|
||||
round_no = parsed.get("round_no")
|
||||
if parsed.get("round_status") == "pushed":
|
||||
slot["blocked"] = True
|
||||
slot["round"] = round_no
|
||||
slot["checks"] = _trigger_check_names(
|
||||
parsed.get("trigger_items"))
|
||||
last_blocked_round = round_no
|
||||
elif parsed.get("round_status") == "re_reviewed":
|
||||
result = parsed.get("re_review_result") or {}
|
||||
blocked = result.get("verdict") != "pass"
|
||||
slot["blocked"] = blocked
|
||||
slot["round"] = round_no
|
||||
slot["checks"] = _trigger_check_names(
|
||||
parsed.get("trigger_items")) if blocked else []
|
||||
if blocked:
|
||||
last_blocked_round = round_no
|
||||
elif action == "audit_rebuild":
|
||||
slot["blocked"] = True
|
||||
if not slot["checks"]:
|
||||
slot["checks"] = ["audit_rebuild"]
|
||||
last_blocked_round = slot["round"]
|
||||
elif action == WIRING_EVENT_ACTION:
|
||||
if parsed.get("wiring_kind") in _BLOCKING_WIRING_KINDS:
|
||||
slot["blocked"] = True
|
||||
checks = parsed.get("failed_checks")
|
||||
if checks:
|
||||
slot["checks"] = list(checks)
|
||||
if parsed.get("round_no") is not None:
|
||||
slot["round"] = parsed.get("round_no")
|
||||
last_blocked_round = parsed.get("round_no")
|
||||
elif action == "audit_rebuild_notice":
|
||||
if parsed.get("confirmation_status") == "pending":
|
||||
slot["blocked"] = True
|
||||
blocked = global_state["blocked"] or any(
|
||||
slot["blocked"] for slot in modules.values())
|
||||
checks = []
|
||||
if global_state["blocked"]:
|
||||
checks.extend(global_state["checks"])
|
||||
for slot in modules.values():
|
||||
if slot["blocked"]:
|
||||
for check in slot["checks"]:
|
||||
if check not in checks:
|
||||
checks.append(check)
|
||||
block = _gate_block(
|
||||
"pushback" if blocked else "allow", checks,
|
||||
last_blocked_round if last_blocked_round is not None else 0)
|
||||
block["modules"] = {
|
||||
"%s@%s" % key: {
|
||||
"blocked": slot["blocked"],
|
||||
"round": slot["round"],
|
||||
"failed_checks": list(slot["checks"]),
|
||||
}
|
||||
for key, slot in modules.items()
|
||||
}
|
||||
if global_state["blocked"]:
|
||||
block["modules"]["<global>"] = {
|
||||
"blocked": True,
|
||||
"round": global_state["round"],
|
||||
"failed_checks": list(global_state["checks"]),
|
||||
}
|
||||
return block
|
||||
+85
-16
@@ -25,6 +25,8 @@ import hashlib
|
||||
import secrets
|
||||
import sqlite3
|
||||
from cloud.db import connect as _db_connect
|
||||
from cloud.ledger import append_entry as _ledger_append
|
||||
from cloud.ledger import ensure_schema as _ledger_ensure
|
||||
import time
|
||||
import uuid
|
||||
|
||||
@@ -39,6 +41,55 @@ _ACTION_MIN_ROLE = {
|
||||
"manage_project": "owner",
|
||||
}
|
||||
|
||||
# audit_log 建表 SQL(AuthStore.__init__ 与 audit_on_conn 共用同一
|
||||
# 份,绝不能各写一份后漂移)。PG 适配层把其中的
|
||||
# INTEGER PRIMARY KEY AUTOINCREMENT 翻译成 identity 列,见
|
||||
# cloud/db.py;SQLite 下原样执行。
|
||||
_AUDIT_LOG_TABLE_SQL = (
|
||||
"CREATE TABLE IF NOT EXISTS audit_log("
|
||||
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
|
||||
"user_id TEXT, "
|
||||
"project_id TEXT, "
|
||||
"action TEXT, "
|
||||
"detail TEXT, "
|
||||
"created_at REAL)"
|
||||
)
|
||||
|
||||
|
||||
def audit_on_conn(conn, user_id, project_id, action, detail="",
|
||||
created_at=None):
|
||||
"""在调用方给定的连接上写一对审计记录(单次尝试)。
|
||||
|
||||
与 AuthStore.audit() 是同一对写:audit_log 行 + audit_ledger
|
||||
链条目同 created_at(调用方不给时取当前时间),先建两张表
|
||||
(幂等),再写两行。本函数不 commit、不 rollback、不重试:
|
||||
事务边界属于调用方——调用方须在自己的事务里 commit,失败
|
||||
时 rollback 整段业务写、链头唯一冲突时 rollback 后重试整段
|
||||
(只重试本对写会丢掉调用方同事务里已写但未提交的业务行;
|
||||
见 cloud/ledger.py append_entry 的警告)。AuthStore.audit()
|
||||
即本函数 + “自有连接上 commit、唯一冲突重试 5 次”的包装。
|
||||
|
||||
存在理由:automations.run_due_automation 拿到的是调用方连接
|
||||
(任务 INSERT 与产出记录都在该连接的事务里),不能像服务端
|
||||
handler 那样另开 AuthStore(自有连接、自有事务)去写审计——
|
||||
那样审计与任务不在同一事务,审计失败时任务已落盘,所谓
|
||||
“整对回滚”只保审计对、不保业务对。本函数让自动化执行器
|
||||
的审计行与任务行、产出记录行同事务提交、同事务回滚。
|
||||
"""
|
||||
if created_at is None:
|
||||
created_at = time.time()
|
||||
created_at = float(created_at)
|
||||
conn.execute(_AUDIT_LOG_TABLE_SQL)
|
||||
_ledger_ensure(conn)
|
||||
conn.execute(
|
||||
"INSERT INTO audit_log(user_id, project_id, action,"
|
||||
" detail, created_at)"
|
||||
" VALUES(?, ?, ?, ?, ?)",
|
||||
(user_id, project_id, action, detail, created_at),
|
||||
)
|
||||
_ledger_append(conn, user_id, project_id, action,
|
||||
detail=detail, created_at=created_at)
|
||||
|
||||
|
||||
class AuthStore:
|
||||
"""基于 SQLite 的用户 / 项目成员 / 审计存储。"""
|
||||
@@ -60,15 +111,10 @@ class AuthStore:
|
||||
"role TEXT, "
|
||||
"PRIMARY KEY(project_id, user_id))"
|
||||
)
|
||||
self.conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS audit_log("
|
||||
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
|
||||
"user_id TEXT, "
|
||||
"project_id TEXT, "
|
||||
"action TEXT, "
|
||||
"detail TEXT, "
|
||||
"created_at REAL)"
|
||||
)
|
||||
self.conn.execute(_AUDIT_LOG_TABLE_SQL)
|
||||
# 防篡改账本(哈希链):audit_log 本身不动,链写进
|
||||
# audit_ledger;建表失败即建库失败,不静默丢链。
|
||||
_ledger_ensure(self.conn)
|
||||
self.conn.commit()
|
||||
|
||||
def create_user(self, username):
|
||||
@@ -350,13 +396,36 @@ class AuthStore:
|
||||
)
|
||||
|
||||
def audit(self, user_id, project_id, action, detail=""):
|
||||
"""只追加一条审计记录。"""
|
||||
self.conn.execute(
|
||||
"INSERT INTO audit_log(user_id, project_id, action, detail, created_at)"
|
||||
" VALUES(?, ?, ?, ?, ?)",
|
||||
(user_id, project_id, action, detail, time.time()),
|
||||
)
|
||||
self.conn.commit()
|
||||
"""只追加一条审计记录,并同步写哈希链(同事务)。
|
||||
|
||||
audit_log 行与 audit_ledger 链条目同 created_at、同事务
|
||||
提交:链写失败时整对回滚并抛错(fail-closed,绝不留
|
||||
“audit_log 有行、链上无行”的静默缺口)。并发抢链头
|
||||
导致唯一冲突时在事务边界重试整对写入,最多 5 次;
|
||||
其它错误直接回滚抛错,不重试。
|
||||
|
||||
实现即 audit_on_conn() + 本连接上的 commit/重试包装,
|
||||
对写本身与 #241 完全一致(同一函数、同一 created_at
|
||||
语义),不是第二套审计写路径。
|
||||
"""
|
||||
now = time.time()
|
||||
last_exc = None
|
||||
for _attempt in range(5):
|
||||
try:
|
||||
audit_on_conn(self.conn, user_id, project_id, action,
|
||||
detail=detail, created_at=now)
|
||||
self.conn.commit()
|
||||
return
|
||||
except Exception as exc:
|
||||
last_exc = exc
|
||||
try:
|
||||
self.conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
if not isinstance(exc, sqlite3.IntegrityError) \
|
||||
and type(exc).__name__ != "PostgresIntegrityError":
|
||||
raise
|
||||
raise last_exc
|
||||
|
||||
def list_audit(self, project_id, limit=100):
|
||||
"""按 created_at 倒序返回该项目的审计列表(dict)。"""
|
||||
|
||||
+15
-29
@@ -53,12 +53,18 @@ cloud/migrations.py 的对齐说明;PG DDL 同步进
|
||||
cloud/db.py POSTGRES_SCHEMA 与 MIGRATION_TABLES。
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
from cloud.db import connect as _db_connect
|
||||
from cloud.identity import IdentityStore
|
||||
# 行取值统一走 peer_messages.row_get(A7):本模块旧私有版与之
|
||||
# 同款语义(缺列/缺键/空行回 default,默认 None),调用处仍
|
||||
# 用 _row_get 别名,行为不变。
|
||||
from cloud.peer_messages import row_get as _row_get
|
||||
from cloud.quota import default_quota_usd_micros as _quota_default
|
||||
from cloud.quota import monthly_usage as _quota_monthly_usage
|
||||
from cloud.quota import quota_for_owner as _quota_for_owner
|
||||
|
||||
DEFAULT_SETTINGS = {
|
||||
"generate_memories": 0,
|
||||
@@ -113,12 +119,9 @@ def redact_text(text):
|
||||
|
||||
|
||||
def _default_quota_usd_micros():
|
||||
"""与 worker/server 同款默认配额解析,避免循环 import。"""
|
||||
try:
|
||||
return max(0, int(os.environ.get(
|
||||
"CLOUD_DEFAULT_QUOTA_USD_MICROS", "10000000")))
|
||||
except (TypeError, ValueError):
|
||||
return 10000000
|
||||
"""默认配额解析:收敛到 cloud/quota.py(quota 模块是
|
||||
标准库叶子,不存在旧注释担心的循环 import)。"""
|
||||
return _quota_default()
|
||||
|
||||
|
||||
def quota_remaining_percent(db_path, owner):
|
||||
@@ -131,27 +134,17 @@ def quota_remaining_percent(db_path, owner):
|
||||
month = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m")
|
||||
conn = _db_connect(db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS user_quotas("
|
||||
"user_id TEXT PRIMARY KEY, quota_usd_micros BIGINT, "
|
||||
"updated_at DOUBLE PRECISION)")
|
||||
# 配额行与当月 USD 用量取数收敛到 cloud/quota.py
|
||||
#(无价部分本函数不用,只取 USD 一项;缺 cost_records
|
||||
# 表时仍按旧行为计 0,其它查库异常仍抛给调用方)。
|
||||
quota = _quota_for_owner(conn, owner)
|
||||
conn.commit()
|
||||
row = conn.execute(
|
||||
"SELECT quota_usd_micros FROM user_quotas WHERE user_id = ?",
|
||||
(owner,)).fetchone()
|
||||
quota = _default_quota_usd_micros() if row is None else row[0]
|
||||
if quota is None:
|
||||
return None
|
||||
if quota <= 0:
|
||||
return 0.0
|
||||
try:
|
||||
used_row = conn.execute(
|
||||
"SELECT COALESCE(SUM(c.usd_micros), 0)"
|
||||
" FROM cost_records c JOIN tasks t"
|
||||
" ON c.task_id = t.id"
|
||||
" WHERE t.created_by = ? AND c.day LIKE ?",
|
||||
(owner, month + "%")).fetchone()
|
||||
used = used_row[0] if used_row else 0
|
||||
used, _unpriced = _quota_monthly_usage(conn, owner, month)
|
||||
except Exception as exc:
|
||||
if "no such table" in str(exc):
|
||||
used = 0
|
||||
@@ -162,13 +155,6 @@ def quota_remaining_percent(db_path, owner):
|
||||
conn.close()
|
||||
|
||||
|
||||
def _row_get(row, name, default=None):
|
||||
try:
|
||||
return row[name]
|
||||
except (KeyError, IndexError, TypeError):
|
||||
return default
|
||||
|
||||
|
||||
def _first_line(text, max_len=200):
|
||||
for line in (text or "").splitlines():
|
||||
line = " ".join(line.split())
|
||||
|
||||
+154
-253
@@ -38,55 +38,52 @@
|
||||
id 记在 automation_runs.task_id 里,派发与任务绑定,不用
|
||||
为自动化另起一套通知系统)。
|
||||
|
||||
本模块只依赖标准库,不 import cloud 其它模块,避免循环
|
||||
import;表迁移写法仿 peer_messages(CREATE TABLE IF NOT
|
||||
EXISTS + PRAGMA 查列补齐,SQLite 与 Postgres 适配层同款)。
|
||||
本模块的 cloud 依赖只有两处叶子引用,避免循环 import:
|
||||
配额判定走同为标准库叶子的 cloud.quota;审计行上链
|
||||
(方案 A,见 run_due_automation)只走 cloud.auth.audit_on_conn——
|
||||
与 AuthStore.audit() 同一对写(audit_log 行 + audit_ledger
|
||||
链条目,同 created_at),不再有 audit_log 直写旁路。
|
||||
auth 只依赖 db / ledger 两个叶模块,三者都不 import
|
||||
automations / server / worker,不构成循环 import;其余 SQL
|
||||
(配额口径、表迁移)仍就地实现,不另 import 业务模块。
|
||||
表迁移写法仿 peer_messages(CREATE TABLE IF NOT
|
||||
EXISTS + PRAGMA 查列补齐,SQLite 与 Postgres 适配层同款)。
|
||||
"""
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
import time
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from cloud.peer_messages import row_get
|
||||
from cloud.quota import decide as _quota_decide
|
||||
from cloud.quota import default_quota_usd_micros as _quota_default
|
||||
from cloud.quota import monthly_usage as _quota_monthly_usage
|
||||
from cloud.quota import quota_for_owner as _quota_for_owner
|
||||
from cloud.auth import audit_on_conn as _audit_on_conn
|
||||
|
||||
MIN_INTERVAL_SECONDS = 60
|
||||
MAX_INTERVAL_SECONDS = 30 * 86400
|
||||
MAX_NAME_LEN = 100
|
||||
MAX_PROMPT_LEN = 20000
|
||||
MAX_REPO_URL_LEN = 1000
|
||||
TRIAGE_LIMIT = 50
|
||||
# 单项目自动化定义上限(本项目自定,非 Meta 官方值:Meta 未公开
|
||||
# cron 的数量上限;超限建定义即 ValueError)。
|
||||
MAX_AUTOMATIONS_PER_PROJECT = 100
|
||||
# 失败重试默认值(本项目自定,非 Meta 官方值:Meta 未公开失败
|
||||
# 重试策略;退避 = 基数 × 2^(n-1),上限见 crons 模块)。
|
||||
DEFAULT_MAX_RETRIES = 3
|
||||
DEFAULT_RETRY_BACKOFF_SECONDS = 60
|
||||
MAX_RETRIES_LIMIT = 10
|
||||
MIN_RETRY_BACKOFF_SECONDS = 10
|
||||
MAX_RETRY_BACKOFF_SECONDS = 3600
|
||||
|
||||
ACTIVE = "active"
|
||||
PAUSED = "paused"
|
||||
CANCELLED = "cancelled"
|
||||
|
||||
|
||||
def row_get(row, name):
|
||||
"""行取值容错:sqlite3.Row 缺列报 IndexError、dict 缺键报
|
||||
KeyError、空行报 TypeError,一律回 None(老库行缺列时
|
||||
读自动化列表不能因此炸掉整页)。"""
|
||||
try:
|
||||
return row[name]
|
||||
except (KeyError, IndexError, TypeError):
|
||||
return None
|
||||
# 行取值走 cloud.peer_messages.row_get(A7 全仓统一):旧私有版
|
||||
# 与统一版在真实行类型(None / dict / sqlite3.Row)下语义完全
|
||||
# 相同(缺键/缺列/空行一律回 None),故直接复用,不再自带一份。
|
||||
|
||||
|
||||
AUTOMATION_COLUMNS = (
|
||||
"id", "project_id", "name", "prompt", "repo_url",
|
||||
"interval_seconds", "status", "last_run_at", "last_claimed_at",
|
||||
"created_by", "created_at", "cancelled_at",
|
||||
"first_run_at", "run_once",
|
||||
"max_retries", "retry_backoff_seconds",
|
||||
"consecutive_failures", "next_retry_at")
|
||||
"first_run_at", "run_once")
|
||||
RUN_COLUMNS = (
|
||||
"id", "automation_id", "project_id", "task_id", "status",
|
||||
"detail", "created_at")
|
||||
@@ -117,11 +114,8 @@ def _existing_columns(conn, table):
|
||||
def ensure_automations_tables(conn):
|
||||
"""幂等建自动化表与产出记录表,并补齐新增列。
|
||||
|
||||
automations:一条定义;status 只取 active / paused /
|
||||
cancelled,取消是软删除(行保留、审计可查,名字可重用);
|
||||
paused 是停用(可恢复,不到期、不抢单,见 crons 模块)。
|
||||
max_retries / retry_backoff_seconds / consecutive_failures /
|
||||
next_retry_at 是失败重试列(P8 crons,见该模块)。
|
||||
automations:一条定义;status 只取 active / cancelled,
|
||||
取消是软删除(行保留、审计可查,名字可重用)。
|
||||
automation_runs:一次执行产出;task_id 为空表示执行器
|
||||
抢单后尚未建任务(只应短暂存在,执行器建任务后补记)。
|
||||
调用方在写成功后 commit(server 的 handler 与 peer 模式
|
||||
@@ -141,11 +135,7 @@ def ensure_automations_tables(conn):
|
||||
"created_at DOUBLE PRECISION, "
|
||||
"cancelled_at DOUBLE PRECISION, "
|
||||
"first_run_at DOUBLE PRECISION, "
|
||||
"run_once INTEGER, "
|
||||
"max_retries INTEGER, "
|
||||
"retry_backoff_seconds INTEGER, "
|
||||
"consecutive_failures INTEGER, "
|
||||
"next_retry_at DOUBLE PRECISION)"
|
||||
"run_once INTEGER)"
|
||||
)
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS automation_runs("
|
||||
@@ -163,11 +153,7 @@ def ensure_automations_tables(conn):
|
||||
("last_claimed_at", "DOUBLE PRECISION"),
|
||||
("cancelled_at", "DOUBLE PRECISION"),
|
||||
("first_run_at", "DOUBLE PRECISION"),
|
||||
("run_once", "INTEGER"),
|
||||
("max_retries", "INTEGER"),
|
||||
("retry_backoff_seconds", "INTEGER"),
|
||||
("consecutive_failures", "INTEGER"),
|
||||
("next_retry_at", "DOUBLE PRECISION")):
|
||||
("run_once", "INTEGER")):
|
||||
if col not in existing:
|
||||
try:
|
||||
conn.execute(
|
||||
@@ -256,30 +242,11 @@ def automation_to_dict(row, now=None):
|
||||
"cancelled_at": row_get(row, "cancelled_at"),
|
||||
"first_run_at": row_get(row, "first_run_at"),
|
||||
"run_once": bool(row_get(row, "run_once")),
|
||||
"max_retries": _retry_int(row, "max_retries",
|
||||
DEFAULT_MAX_RETRIES),
|
||||
"retry_backoff_seconds": _retry_int(
|
||||
row, "retry_backoff_seconds",
|
||||
DEFAULT_RETRY_BACKOFF_SECONDS),
|
||||
"consecutive_failures": _retry_int(
|
||||
row, "consecutive_failures", 0),
|
||||
"next_retry_at": row_get(row, "next_retry_at"),
|
||||
"next_run_at": nxt,
|
||||
"due": is_due(row, now),
|
||||
}
|
||||
|
||||
|
||||
def _retry_int(row, name, default):
|
||||
"""重试列取值:老库行缺列回 None 时用默认值;脏值(非
|
||||
int/负数)同样回默认值,绝不因此炸掉整页。"""
|
||||
value = row_get(row, name)
|
||||
if isinstance(value, bool) or not isinstance(value, int):
|
||||
return default
|
||||
if value < 0:
|
||||
return default
|
||||
return value
|
||||
|
||||
|
||||
def _validate_fields(name, prompt, interval_seconds, repo_url,
|
||||
first_run_at=None, run_once=False):
|
||||
if not name or not isinstance(name, str) or not name.strip():
|
||||
@@ -319,45 +286,18 @@ def _validate_fields(name, prompt, interval_seconds, repo_url,
|
||||
first_run_at, run_once
|
||||
|
||||
|
||||
def _validate_retry(max_retries, retry_backoff_seconds):
|
||||
"""重试参数校验(本项目自定范围):max_retries 0–10,
|
||||
retry_backoff_seconds 10–3600 秒;非法即 ValueError。"""
|
||||
if isinstance(max_retries, bool) \
|
||||
or not isinstance(max_retries, int):
|
||||
raise ValueError("max_retries must be an integer")
|
||||
if not 0 <= max_retries <= MAX_RETRIES_LIMIT:
|
||||
raise ValueError("max_retries must be between 0 and %d"
|
||||
% MAX_RETRIES_LIMIT)
|
||||
if isinstance(retry_backoff_seconds, bool) \
|
||||
or not isinstance(retry_backoff_seconds, int):
|
||||
raise ValueError("retry_backoff_seconds must be an integer")
|
||||
if not MIN_RETRY_BACKOFF_SECONDS <= retry_backoff_seconds \
|
||||
<= MAX_RETRY_BACKOFF_SECONDS:
|
||||
raise ValueError("retry_backoff_seconds must be between %d"
|
||||
" and %d" % (MIN_RETRY_BACKOFF_SECONDS,
|
||||
MAX_RETRY_BACKOFF_SECONDS))
|
||||
return max_retries, retry_backoff_seconds
|
||||
|
||||
|
||||
def create_automation(conn, project_id, name, prompt, created_by,
|
||||
interval_seconds=86400, repo_url=None, now=None,
|
||||
first_run_at=None, run_once=False,
|
||||
max_retries=DEFAULT_MAX_RETRIES,
|
||||
retry_backoff_seconds=DEFAULT_RETRY_BACKOFF_SECONDS):
|
||||
first_run_at=None, run_once=False):
|
||||
"""创建自动化定义:同项目内 active 的名字唯一(取消过的
|
||||
名字可重用,与环境名唯一规则同款语义)。成功时 commit
|
||||
并返回 dict;校验失败抛 ValueError(调用方转 400)。
|
||||
first_run_at 为首次运行时刻(epoch 秒,过去则建完即
|
||||
到期);run_once 为 True 时跑过一次后不再到期。
|
||||
max_retries / retry_backoff_seconds 是失败重试策略
|
||||
(P8 crons,本项目自定);单项目定义上限
|
||||
MAX_AUTOMATIONS_PER_PROJECT,超限即 ValueError。"""
|
||||
到期);run_once 为 True 时跑过一次后不再到期。"""
|
||||
(name, prompt, interval_seconds, repo_url,
|
||||
first_run_at, run_once) = _validate_fields(
|
||||
name, prompt, interval_seconds, repo_url,
|
||||
first_run_at, run_once)
|
||||
(max_retries, retry_backoff_seconds) = _validate_retry(
|
||||
max_retries, retry_backoff_seconds)
|
||||
if now is None:
|
||||
now = time.time()
|
||||
ensure_automations_tables(conn)
|
||||
@@ -367,30 +307,16 @@ def create_automation(conn, project_id, name, prompt, created_by,
|
||||
(project_id, name, ACTIVE)).fetchone()
|
||||
if existing is not None:
|
||||
raise ValueError("automation name already exists in this project")
|
||||
count_row = conn.execute(
|
||||
"SELECT COUNT(*) FROM automations WHERE project_id = ?"
|
||||
" AND status IN (?, ?)",
|
||||
(project_id, ACTIVE, PAUSED)).fetchone()
|
||||
try:
|
||||
count = int(count_row[0])
|
||||
except (TypeError, ValueError, IndexError):
|
||||
count = 0
|
||||
if count >= MAX_AUTOMATIONS_PER_PROJECT:
|
||||
raise ValueError("too many automations in this project")
|
||||
auto_id = uuid.uuid4().hex
|
||||
conn.execute(
|
||||
"INSERT INTO automations(id, project_id, name, prompt,"
|
||||
" repo_url, interval_seconds, status, last_run_at,"
|
||||
" last_claimed_at, created_by, created_at, cancelled_at,"
|
||||
" first_run_at, run_once, max_retries,"
|
||||
" retry_backoff_seconds, consecutive_failures,"
|
||||
" next_retry_at)"
|
||||
" VALUES(?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?, NULL, ?, ?,"
|
||||
" ?, ?, 0, NULL)",
|
||||
" first_run_at, run_once)"
|
||||
" VALUES(?, ?, ?, ?, ?, ?, ?, NULL, NULL, ?, ?, NULL, ?, ?)",
|
||||
(auto_id, project_id, name, prompt, repo_url,
|
||||
interval_seconds, ACTIVE, created_by, now,
|
||||
first_run_at, 1 if run_once else 0,
|
||||
max_retries, retry_backoff_seconds))
|
||||
first_run_at, 1 if run_once else 0))
|
||||
conn.commit()
|
||||
row = conn.execute(
|
||||
"SELECT * FROM automations WHERE id = ?", (auto_id,)).fetchone()
|
||||
@@ -422,7 +348,7 @@ def get_automation(conn, automation_id):
|
||||
|
||||
|
||||
def cancel_automation(conn, automation_id, now=None):
|
||||
"""取消自动化(软删除):仅 active / paused 的行能取消。
|
||||
"""取消自动化(软删除):仅 active 的行能取消。
|
||||
|
||||
返回 (ok, auto):ok 为 True 时 auto 是取消后的 dict;
|
||||
行不存在时 (False, None);已取消时 (False, 旧 dict),
|
||||
@@ -436,12 +362,12 @@ def cancel_automation(conn, automation_id, now=None):
|
||||
if row is None:
|
||||
return False, None
|
||||
row = _row_dict(row, AUTOMATION_COLUMNS)
|
||||
if row["status"] not in (ACTIVE, PAUSED):
|
||||
if row["status"] != ACTIVE:
|
||||
return False, automation_to_dict(row, now)
|
||||
cur = conn.execute(
|
||||
"UPDATE automations SET status = ?, cancelled_at = ?"
|
||||
" WHERE id = ? AND status IN (?, ?)",
|
||||
(CANCELLED, now, automation_id, ACTIVE, PAUSED))
|
||||
" WHERE id = ? AND status = ?",
|
||||
(CANCELLED, now, automation_id, ACTIVE))
|
||||
conn.commit()
|
||||
if cur.rowcount != 1:
|
||||
return False, automation_to_dict(row, now)
|
||||
@@ -454,16 +380,16 @@ def cancel_automation(conn, automation_id, now=None):
|
||||
def update_automation(conn, automation_id, fields, now=None):
|
||||
"""更新自动化定义(automation_update):fields 只认
|
||||
name / prompt / interval_seconds / repo_url / first_run_at /
|
||||
run_once / max_retries / retry_backoff_seconds 八个键,其它键
|
||||
一律忽略(不报错、不存入,避免调用方拼错字段名后以为已
|
||||
改);八个键全没给时抛 ValueError(空更新不是成功)。校验与创建同款
|
||||
run_once 六个键,其它键一律忽略(不报错、不存入,避免
|
||||
调用方拼错字段名后以为已改);六个键全没给时抛
|
||||
ValueError(空更新不是成功)。校验与创建同款
|
||||
(_validate_fields,名字仍按“同项目 active 名字唯一”
|
||||
查重,改名撞名时 400)。
|
||||
|
||||
返回 (ok, auto):行不存在时 (False, None);已取消时
|
||||
(False, 旧 dict)——取消是终态,改已取消的定义必须
|
||||
重建,不能靠 update 复活(与 cancel 的软删除语义
|
||||
一致)。paused 的行可改(改完仍是 paused,不自动恢复)。成功时 commit。注意:更新不重置 last_run_at /
|
||||
一致)。成功时 commit。注意:更新不重置 last_run_at /
|
||||
last_claimed_at——改间隔只影响下一次到期判断(next_run
|
||||
由 last_run_at + 新间隔算),不立即触发一轮;想立即
|
||||
跑一轮应走执行器的 claim 路径,不是改定义。"""
|
||||
@@ -476,7 +402,7 @@ def update_automation(conn, automation_id, fields, now=None):
|
||||
if row is None:
|
||||
return False, None
|
||||
row = _row_dict(row, AUTOMATION_COLUMNS)
|
||||
if row["status"] not in (ACTIVE, PAUSED):
|
||||
if row["status"] != ACTIVE:
|
||||
return False, automation_to_dict(row, now)
|
||||
if not isinstance(fields, dict):
|
||||
raise ValueError("fields must be an object")
|
||||
@@ -491,16 +417,11 @@ def update_automation(conn, automation_id, fields, now=None):
|
||||
run_once = fields.get("run_once", current["run_once"])
|
||||
if not any(k in fields for k in (
|
||||
"name", "prompt", "interval_seconds", "repo_url",
|
||||
"first_run_at", "run_once",
|
||||
"max_retries", "retry_backoff_seconds")):
|
||||
"first_run_at", "run_once")):
|
||||
raise ValueError("no updatable field given")
|
||||
(name, prompt, interval, repo_url,
|
||||
first_run_at, run_once) = _validate_fields(
|
||||
name, prompt, interval, repo_url, first_run_at, run_once)
|
||||
(max_retries, retry_backoff_seconds) = _validate_retry(
|
||||
fields.get("max_retries", current["max_retries"]),
|
||||
fields.get("retry_backoff_seconds",
|
||||
current["retry_backoff_seconds"]))
|
||||
if name != current["name"]:
|
||||
clash = conn.execute(
|
||||
"SELECT id FROM automations WHERE project_id = ?"
|
||||
@@ -513,11 +434,9 @@ def update_automation(conn, automation_id, fields, now=None):
|
||||
conn.execute(
|
||||
"UPDATE automations SET name = ?, prompt = ?,"
|
||||
" interval_seconds = ?, repo_url = ?, first_run_at = ?,"
|
||||
" run_once = ?, max_retries = ?,"
|
||||
" retry_backoff_seconds = ? WHERE id = ?",
|
||||
" run_once = ? WHERE id = ?",
|
||||
(name, prompt, interval, repo_url, first_run_at,
|
||||
1 if run_once else 0, max_retries, retry_backoff_seconds,
|
||||
automation_id))
|
||||
1 if run_once else 0, automation_id))
|
||||
conn.commit()
|
||||
row = conn.execute(
|
||||
"SELECT * FROM automations WHERE id = ?", (automation_id,)
|
||||
@@ -611,13 +530,9 @@ def record_automation_run(conn, automation_id, project_id, task_id=None,
|
||||
|
||||
|
||||
def _default_quota_usd_micros():
|
||||
"""默认月配额(微美元):与服务端 _default_quota_usd_micros
|
||||
同值同环境变量(CLOUD_DEFAULT_QUOTA_USD_MICROS,默认 $10)。"""
|
||||
try:
|
||||
return max(0, int(os.environ.get(
|
||||
"CLOUD_DEFAULT_QUOTA_USD_MICROS", "10000000")))
|
||||
except (TypeError, ValueError):
|
||||
return 10000000
|
||||
"""默认月配额(微美元):解析收敛到 cloud/quota.py
|
||||
(同值同环境变量 CLOUD_DEFAULT_QUOTA_USD_MICROS,默认 $10)。"""
|
||||
return _quota_default()
|
||||
|
||||
|
||||
def _unpriced_token_cap():
|
||||
@@ -640,29 +555,17 @@ def _automation_quota_exceeded(conn, owner):
|
||||
过处理,绝不建任务)。cost_records 表不存在时视为零花费
|
||||
(CostStore 首次记账时才建表,不存在即从未记过账),其
|
||||
它查库异常才 fail-closed——全新库上不能因此误拒全部运行。
|
||||
本模块只依赖标准库,不 import cloud 其它模块,故此处就
|
||||
地实现 SQL 版口径,注释里标明三处同口径点,改口径时三
|
||||
处要一起改(server._quota_status、worker._quota_exceeded、
|
||||
本函数)。
|
||||
取数与比较判定收敛到 cloud/quota.py(本函数只保留
|
||||
表存在探针与 PG 事务回滚处理,因其与自动化执行器的
|
||||
事务形态绑定,不宜塞进通用取数层)。
|
||||
"""
|
||||
if not owner:
|
||||
return ("quota_exceeded: 自动化缺少创建人,无法查配额,"
|
||||
"拒绝建任务")
|
||||
try:
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS user_quotas("
|
||||
"user_id TEXT PRIMARY KEY, quota_usd_micros BIGINT, "
|
||||
" updated_at DOUBLE PRECISION)")
|
||||
row = conn.execute(
|
||||
"SELECT quota_usd_micros FROM user_quotas"
|
||||
" WHERE user_id = ?",
|
||||
(owner,)).fetchone()
|
||||
if row is None:
|
||||
quota = _default_quota_usd_micros()
|
||||
else:
|
||||
quota = row_get(row, "quota_usd_micros")
|
||||
if quota is None:
|
||||
return None
|
||||
quota = _quota_for_owner(conn, owner)
|
||||
if quota is None:
|
||||
return None
|
||||
month = datetime.now(timezone.utc).strftime("%Y-%m")
|
||||
try:
|
||||
conn.execute("SELECT COUNT(*) FROM cost_records").fetchone()
|
||||
@@ -676,22 +579,8 @@ def _automation_quota_exceeded(conn, owner):
|
||||
pass
|
||||
table_exists = False
|
||||
if table_exists:
|
||||
used_row = conn.execute(
|
||||
"SELECT COALESCE(SUM(c.usd_micros), 0) AS total"
|
||||
" FROM cost_records c JOIN tasks t"
|
||||
" ON c.task_id = t.id"
|
||||
" WHERE t.created_by = ? AND c.day LIKE ?",
|
||||
(owner, month + "%")).fetchone()
|
||||
used = row_get(used_row, "total") or 0
|
||||
unpriced_row = conn.execute(
|
||||
"SELECT COALESCE(SUM(c.input_tokens + c.output_tokens), 0)"
|
||||
" AS total"
|
||||
" FROM cost_records c JOIN tasks t"
|
||||
" ON c.task_id = t.id"
|
||||
" WHERE t.created_by = ? AND c.day LIKE ?"
|
||||
" AND c.usd_micros IS NULL AND c.source = 'actual'",
|
||||
(owner, month + "%")).fetchone()
|
||||
unpriced_used = row_get(unpriced_row, "total") or 0
|
||||
used, unpriced_used = _quota_monthly_usage(
|
||||
conn, owner, month)
|
||||
else:
|
||||
used = 0
|
||||
unpriced_used = 0
|
||||
@@ -703,85 +592,81 @@ def _automation_quota_exceeded(conn, owner):
|
||||
except Exception:
|
||||
pass
|
||||
return "quota_check_failed: %s" % exc
|
||||
if used >= quota:
|
||||
return ("quota_exceeded: 本月配额已用完(已用 %d / "
|
||||
"配额 %d 微美元,%s)" % (used, quota, month))
|
||||
if unpriced_used >= _unpriced_token_cap():
|
||||
return ("quota_exceeded: 本月无价模型 token 已用完"
|
||||
"(已用 %d / 上限 %d,%s)"
|
||||
% (unpriced_used, _unpriced_token_cap(), month))
|
||||
decision = _quota_decide(quota, used, unpriced_used,
|
||||
_unpriced_token_cap(), month)
|
||||
if decision is not None:
|
||||
return decision["message"]
|
||||
return None
|
||||
|
||||
|
||||
def _audit_row(conn, project_id, action, detail, now):
|
||||
"""best-effort 写审计行:失败不影响自动化运行本身。
|
||||
def _is_integrity_error(exc):
|
||||
"""链头唯一冲突判定:与 cloud/ledger.py _is_integrity_error
|
||||
同规则(sqlite3.IntegrityError,或 PG 适配层映射后的
|
||||
PostgresIntegrityError,按名判定,驱动类不跨模块 import)。
|
||||
|
||||
旧实现显式插 TEXT uuid 当 id:SQLite 下能跑,但生产 PG 的
|
||||
audit_log.id 是 BIGINT GENERATED BY DEFAULT AS IDENTITY,
|
||||
插 TEXT 直接报错、审计行全丢(best-effort 吞掉后无声无息)。
|
||||
修法:INSERT 一律不带 id,让两端自增/identity 生成;
|
||||
表不存在时先按 PG 口径建、再按 SQLite 口径建、逐个重试,
|
||||
哪边建表成功就用哪边——automations 按设计不 import 其它
|
||||
cloud 模块,所以用“先插、失败再按口径重试”做方言探测,
|
||||
不靠 import 判断后端。
|
||||
只有这类错误值得在事务边界重试整段;其它错误重试无用,
|
||||
必须立即回滚抛错,不能被重试掩盖成“锁/冲突问题”。
|
||||
本模块不直写 audit_log:审计只走 _audit_on_conn(见模块文档);
|
||||
下面的 _audit_row 仅为兼容包装(供 main #275 resident.py),
|
||||
本身仍走 _audit_on_conn,不含 audit_log INSERT。
|
||||
"""
|
||||
if isinstance(exc, sqlite3.IntegrityError):
|
||||
return True
|
||||
return type(exc).__name__ == "PostgresIntegrityError"
|
||||
|
||||
|
||||
def _audit_row(conn, project_id, action, detail, now):
|
||||
"""兼容包装:供 main #275 resident.py 的审计调用。
|
||||
|
||||
本函数不直写 audit_log(#250 断言只允许 auth.py 出现
|
||||
audit_log INSERT):经 cloud.auth.audit_on_conn 写
|
||||
audit_log 行 + audit_ledger 链条目同一对,user_id 仍为
|
||||
"system"、created_at 用调用方给的 now。保留 main 旧版的
|
||||
best-effort 自提交语义:写对失败时回滚后吞掉,不影响
|
||||
resident 轮询本身;调用方已 commit 的业务行不受影响
|
||||
(审计对写是独立的后续写)。
|
||||
"""
|
||||
params = ("system", project_id, action, detail, now)
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO audit_log(user_id, project_id, action,"
|
||||
" detail, created_at) VALUES(?, ?, ?, ?, ?)", params)
|
||||
_audit_on_conn(conn, "system", project_id, action, detail,
|
||||
created_at=now)
|
||||
conn.commit()
|
||||
return
|
||||
except Exception:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
for ddl in (
|
||||
"CREATE TABLE IF NOT EXISTS audit_log("
|
||||
"id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, "
|
||||
"user_id TEXT, project_id TEXT, action TEXT, detail TEXT, "
|
||||
"created_at DOUBLE PRECISION)",
|
||||
"CREATE TABLE IF NOT EXISTS audit_log("
|
||||
"id INTEGER PRIMARY KEY AUTOINCREMENT, "
|
||||
"user_id TEXT, project_id TEXT, action TEXT, detail TEXT, "
|
||||
"created_at REAL)"):
|
||||
try:
|
||||
conn.execute(ddl)
|
||||
conn.execute(
|
||||
"INSERT INTO audit_log(user_id, project_id, action,"
|
||||
" detail, created_at) VALUES(?, ?, ?, ?, ?)", params)
|
||||
conn.commit()
|
||||
return
|
||||
except Exception:
|
||||
try:
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def run_due_automation(conn, automation_id, now=None, claim_fn=None):
|
||||
def run_due_automation(conn, automation_id, now=None):
|
||||
"""云端自动化执行器的唯一入口:抢单 -> 配额门 -> 建任务 -> 记产出。
|
||||
|
||||
claim_fn 可注入抢单函数(默认 claim_due_automation):
|
||||
crons 模块的重试入口传自己的重试抢单,两条路共用配额门
|
||||
与建任务逻辑,抢单语义各守各的(间隔抢单改 last_run_at,
|
||||
重试抢单只清 next_retry_at、不动间隔时钟)。
|
||||
抢不到(不存在、已取消、未到期、已被别的执行器抢走)时
|
||||
返回 None,不记产出、不建任务。抢到后先过配额门:创建人
|
||||
超配额时不建任务,产出记 skipped_quota(不是失败,不进
|
||||
失败重试循环)并记审计,返回该产出;配额内才 INSERT 云
|
||||
端任务(queued,executor/model 留 NULL 走 worker 默认,
|
||||
与服务端建任务缺省行为一致),产出记 created 并记审计。
|
||||
任务 INSERT 与产出记录由 record_automation_run 的 commit
|
||||
一并落盘。调用方(未来的定时执行器)只调本函数,绝不
|
||||
调用方(未来的定时执行器)只调本函数,绝不
|
||||
自己拼 claim + INSERT,否则配额门可被绕过。
|
||||
|
||||
审计失败语义(方案 A,旧 best-effort _audit_row 已删除):
|
||||
审计对写(audit_log 行 + audit_ledger 链条目,同 created_at
|
||||
= now,走 cloud.auth.audit_on_conn)先于产出记录的 commit
|
||||
执行,任务行、产出记录行、审计对行同事务提交:审计失败
|
||||
时三者一起回滚,异常抛给执行器——不记“任务已建、审计缺
|
||||
失”的半截状态,也不把审计失败记成 failed 产出、不进失败
|
||||
重试循环。抢单已单独提交(last_run_at 已写),按 claim 的
|
||||
文档,这次执行等下个间隔再跑,与“抢单后崩掉等下个间隔
|
||||
补跑”一致。链头唯一冲突时整段回滚重试,最多 5 次(与
|
||||
AuthStore.audit 同数);其它错误立即回滚抛错,不重试。
|
||||
审计行 user_id 仍为 "system",detail 格式不变
|
||||
(created:"<自动化id> -> <任务id>";skipped:
|
||||
"<自动化id>: <配额门文本>")。
|
||||
"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
ensure_automations_tables(conn)
|
||||
claim = claim_fn if claim_fn is not None else claim_due_automation
|
||||
if not claim(conn, automation_id, now):
|
||||
if not claim_due_automation(conn, automation_id, now):
|
||||
return None
|
||||
auto = conn.execute(
|
||||
"SELECT id, project_id, prompt, created_by FROM automations"
|
||||
@@ -793,46 +678,62 @@ def run_due_automation(conn, automation_id, now=None, claim_fn=None):
|
||||
prompt = row_get(auto, "prompt")
|
||||
owner = row_get(auto, "created_by")
|
||||
gate = _automation_quota_exceeded(conn, owner)
|
||||
if gate is not None:
|
||||
run = record_automation_run(
|
||||
conn, automation_id, project_id, task_id=None,
|
||||
status="skipped_quota", detail=gate, now=now)
|
||||
_audit_row(conn, project_id, "automation_run_quota_skipped",
|
||||
"%s: %s" % (automation_id, gate), now)
|
||||
return run
|
||||
task_id = uuid.uuid4().hex
|
||||
# 任务列补齐(与 ensure_automations_tables 的补列惯例同款):
|
||||
# 裸库/老库的 tasks 表可能没有 executor 等列,INSERT 前补齐,
|
||||
# 否则建任务直接报错;PG 适配层会翻译 PRAGMA 查询。
|
||||
existing_tasks = _existing_columns(conn, "tasks")
|
||||
for col, ddl in (
|
||||
("executor", "TEXT"),
|
||||
("model", "TEXT"),
|
||||
("effort", "TEXT"),
|
||||
("environment_id", "TEXT"),
|
||||
("parent_task_id", "TEXT"),
|
||||
("coordinator_part", "TEXT")):
|
||||
if col not in existing_tasks:
|
||||
last_exc = None
|
||||
for _attempt in range(5):
|
||||
try:
|
||||
if gate is not None:
|
||||
_audit_on_conn(
|
||||
conn, "system", project_id,
|
||||
"automation_run_quota_skipped",
|
||||
"%s: %s" % (automation_id, gate), created_at=now)
|
||||
return record_automation_run(
|
||||
conn, automation_id, project_id, task_id=None,
|
||||
status="skipped_quota", detail=gate, now=now)
|
||||
# 任务列补齐(与 ensure_automations_tables 的补列惯例
|
||||
# 同款):裸库/老库的 tasks 表可能没有 executor 等列,
|
||||
# INSERT 前补齐,否则建任务直接报错;PG 适配层会翻译
|
||||
# PRAGMA 查询。放在重试循环内:回滚会撤销未提交的
|
||||
# 补列,重试时须重新查列补齐,不能只补一次。
|
||||
existing_tasks = _existing_columns(conn, "tasks")
|
||||
for col, ddl in (
|
||||
("executor", "TEXT"),
|
||||
("model", "TEXT"),
|
||||
("effort", "TEXT"),
|
||||
("environment_id", "TEXT"),
|
||||
("parent_task_id", "TEXT"),
|
||||
("coordinator_part", "TEXT")):
|
||||
if col not in existing_tasks:
|
||||
try:
|
||||
conn.execute(
|
||||
"ALTER TABLE tasks ADD COLUMN %s %s"
|
||||
% (col, ddl))
|
||||
except Exception:
|
||||
pass
|
||||
conn.execute(
|
||||
"INSERT INTO tasks(id, project_id, prompt, status,"
|
||||
" idempotency_key, depends_on, priority, created_by,"
|
||||
" created_at, updated_at, last_heartbeat, executor,"
|
||||
" model, effort, environment_id, parent_task_id,"
|
||||
" coordinator_part)"
|
||||
" VALUES(?, ?, ?, 'queued', NULL, '[]', 0, ?, ?, ?,"
|
||||
" NULL, NULL, NULL, NULL, NULL, NULL, NULL)",
|
||||
(task_id, project_id, prompt, owner, now, now))
|
||||
_audit_on_conn(
|
||||
conn, "system", project_id, "automation_run_created",
|
||||
"%s -> %s" % (automation_id, task_id), created_at=now)
|
||||
return record_automation_run(
|
||||
conn, automation_id, project_id, task_id=task_id,
|
||||
status="created", detail="", now=now)
|
||||
except Exception as exc:
|
||||
last_exc = exc
|
||||
try:
|
||||
conn.execute(
|
||||
"ALTER TABLE tasks ADD COLUMN %s %s" % (col, ddl))
|
||||
conn.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
conn.execute(
|
||||
"INSERT INTO tasks(id, project_id, prompt, status,"
|
||||
" idempotency_key, depends_on, priority, created_by,"
|
||||
" created_at, updated_at, last_heartbeat, executor,"
|
||||
" model, effort, environment_id, parent_task_id,"
|
||||
" coordinator_part)"
|
||||
" VALUES(?, ?, ?, 'queued', NULL, '[]', 0, ?, ?, ?,"
|
||||
" NULL, NULL, NULL, NULL, NULL, NULL, NULL)",
|
||||
(task_id, project_id, prompt, owner, now, now))
|
||||
run = record_automation_run(
|
||||
conn, automation_id, project_id, task_id=task_id,
|
||||
status="created", detail="", now=now)
|
||||
_audit_row(conn, project_id, "automation_run_created",
|
||||
"%s -> %s" % (automation_id, task_id), now)
|
||||
return run
|
||||
if not _is_integrity_error(exc):
|
||||
raise
|
||||
raise last_exc
|
||||
|
||||
|
||||
def list_automation_runs(conn, automation_id, limit=TRIAGE_LIMIT):
|
||||
|
||||
+9
-13
@@ -66,9 +66,15 @@ from cloud.workers import DEFAULT_ALIVE_SECONDS as _WORKER_ALIVE_SECONDS
|
||||
def queue_depth(db_path, executor=None):
|
||||
"""当前就绪任务数:依赖已满足且状态为 queued 的任务。
|
||||
|
||||
依赖成环时 ready_tasks 抛 ValueError,视为 0(成环任务
|
||||
本就派不出去,不应为它扩容)。executor 给定时只数该
|
||||
执行器能认领的任务(未指定执行器、或指定为该执行器)。
|
||||
依赖成环的任务不贡献深度,但不是因为 ready_tasks 抛
|
||||
ValueError 后整体归 0:ready_tasks 先按“依赖全部已完成”
|
||||
过滤(见 cloud/queue.py),成环任务的依赖从未完成、根本
|
||||
进不了拓扑排序,所以既不抛错、也不拖累其它就绪任务——
|
||||
5 个独立就绪任务加 1 对成环任务时深度仍为 5,只有纯成环
|
||||
时深度才为 0(此时就绪集本身为空)。若 ready_tasks 因
|
||||
其它原因抛 ValueError,本函数仍按原有兜底返回 0。
|
||||
executor 给定时只数该执行器能认领的任务(未指定执行器、
|
||||
或指定为该执行器)。
|
||||
"""
|
||||
conn = _db_connect(db_path)
|
||||
try:
|
||||
@@ -304,14 +310,6 @@ class Autoscaler:
|
||||
if not any(a == "--host-name" or a.startswith("--host-name=")
|
||||
for a in self.worker_args):
|
||||
cmd.extend(["--host-name", socket.gethostname()])
|
||||
# 执行器透传:scaler 的 --executor 只算该类型队列深度,
|
||||
# 不透传时拉起的 worker 默认 local,永远认领不到该类型
|
||||
# 任务(cloud-hypervisor 池就是这样被饿死的)。worker_args
|
||||
# 里已显式给 --executor 时以显式为准,不覆盖。
|
||||
if self.executor and not any(
|
||||
a == "--executor" or a.startswith("--executor=")
|
||||
for a in self.worker_args):
|
||||
cmd.extend(["--executor", self.executor])
|
||||
return cmd
|
||||
|
||||
def _start_worker(self, worker_id):
|
||||
@@ -561,8 +559,6 @@ def main(argv=None):
|
||||
help="各 worker 子进程的日志目录")
|
||||
parser.add_argument("--executor", default=None,
|
||||
help="只按该执行器类型的就绪任务算深度;"
|
||||
"同时透传给拉起的 worker(worker_args 里"
|
||||
"已显式给 --executor 时以显式为准);"
|
||||
"不给时按全部任务算(可能轻微多扩,"
|
||||
"受 max-workers 封顶)")
|
||||
parser.add_argument("worker_args", nargs=argparse.REMAINDER,
|
||||
|
||||
@@ -1,33 +0,0 @@
|
||||
"""Cloud Hypervisor 执行器包(从 cloud/sandbox.py 巨类拆出)。
|
||||
|
||||
公开名与 #216 一致,cloud/sandbox.py 会 re-export 生产
|
||||
接口,保证旧导入路径可用——但 WarmPool 除外(本轨破环
|
||||
性清理,见下):
|
||||
- CloudHypervisorExecutor:执行器本体(executor.py)
|
||||
- ChvFailure:分类失败(errors.py)
|
||||
- reconcile_orphans_for_executor:生产残留清理唯一入口
|
||||
(orphans.py,只读 registry 文件做清理,不构造池)
|
||||
|
||||
破环性清理(feat-chv-arch-maint-fe,未合并前即生效,
|
||||
合并时按 PR 描述的清单检查调用方):WarmPool 已从
|
||||
cloud/chv/pool.py 移到 cloud/chv/experimental_pool.py,
|
||||
本包与 cloud/sandbox.py 不再导出它。旧路径
|
||||
`from cloud.chv import WarmPool`、
|
||||
`from cloud.chv.pool import WarmPool`、
|
||||
`from cloud.sandbox import WarmPool` 一律 ImportError;
|
||||
实验用法只走
|
||||
`from cloud.chv.experimental_pool import WarmPool`。
|
||||
清理理由:CHV 尚未合并、尚无生产用户,兼容保留只是
|
||||
“怕断 import”,留着反而让生产代码有路可走进实验接口。
|
||||
|
||||
子模块分工见 executor.py 头注释与 docs/chv-architecture.md
|
||||
ADR-12。本包顶层 import 不触 cloud.sandbox(执行器方法
|
||||
内按需 import 共享 helper),所以任何导入顺序都不会循环。
|
||||
"""
|
||||
|
||||
from cloud.chv.errors import ChvFailure
|
||||
from cloud.chv.executor import CloudHypervisorExecutor
|
||||
from cloud.chv.orphans import reconcile_orphans_for_executor
|
||||
|
||||
__all__ = ["ChvFailure", "CloudHypervisorExecutor",
|
||||
"reconcile_orphans_for_executor"]
|
||||
@@ -1,274 +0,0 @@
|
||||
"""活动视图后端(feat-chv-muse-p12,纯标准库 + atomic_file)。
|
||||
|
||||
接 Muse 官方要求(报告 §2.2 第 7 步、§2.6):完整审计轨迹展示
|
||||
已做与计划做的事;头像下方显示正在做什么的摘要,点头像可见
|
||||
完整活动日志与已批准权限;审批是严格能力,绑定连接器/目的地
|
||||
与用例。
|
||||
|
||||
本模块只给后端 + 数据接口,前端视图(客户端 UI)是另一侧的
|
||||
工作,契约见各函数 docstring。
|
||||
|
||||
三件东西:
|
||||
|
||||
- ``PlanStore``:计划登记——已做 vs 计划做。状态只认
|
||||
pending/running/done/cancelled,流转 pending→running/
|
||||
cancelled、running→done/cancelled,终态不可再动,非法流转
|
||||
抛 ValueError。落 JSON 文件,原子写(0600)。
|
||||
- ``PermissionRegistry``:已批准权限登记。接 P5
|
||||
``ScopedCapability.describe()`` 的 dict 形状(capability_id /
|
||||
tier / connector / host / use_case / session_id / task_id /
|
||||
methods / max_uses / uses_left / expires_at / revoked),不
|
||||
import P5 模块——P5 不在本基线里,集成时由 round-11 合并。
|
||||
``list_active()`` 只返回未撤销、未过期、次数未用完的授权。
|
||||
- ``activity_summary()``:纯函数,把计划列表 + 近期审计记录 +
|
||||
有效授权拼成活动摘要:正在做 / 计划中 / 已做 / 近期记录 /
|
||||
已批准权限数。正在做来自计划登记与审计记录,不是实时进程
|
||||
探针;VM 内真实执行状态以执行器为准。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
|
||||
from cloud.chv.atomic_file import atomic_write_text
|
||||
|
||||
# 计划状态与合法流转(终态 done/cancelled 不可再动)。
|
||||
PLAN_STATUSES = ("pending", "running", "done", "cancelled")
|
||||
_PLAN_TRANSITIONS: dict[str, tuple[str, ...]] = {
|
||||
"pending": ("running", "cancelled"),
|
||||
"running": ("done", "cancelled"),
|
||||
"done": (),
|
||||
"cancelled": (),
|
||||
}
|
||||
|
||||
|
||||
def _load_json(path: str) -> dict:
|
||||
"""读 JSON 文件:不存在或坏掉返回空 dict(调用方重建)。"""
|
||||
if not os.path.exists(path):
|
||||
return {}
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
except (ValueError, OSError):
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
def _save_json(path: str, data: dict) -> None:
|
||||
"""原子写 JSON 文件(0600,写失败原样抛错)。"""
|
||||
text = json.dumps(data, ensure_ascii=False, indent=2)
|
||||
atomic_write_text(path, text, mode=0o600)
|
||||
|
||||
|
||||
class PlanStore:
|
||||
"""计划登记:一个项目一串计划,状态流转受控。"""
|
||||
|
||||
def __init__(self, path: str) -> None:
|
||||
self.path = path
|
||||
self._data = _load_json(path)
|
||||
|
||||
def add_plan(self, project_id: str, title: str,
|
||||
task_id: str = "",
|
||||
ts: float | None = None) -> dict:
|
||||
"""登记一条计划(初始 pending),返回该计划。"""
|
||||
if not project_id:
|
||||
raise ValueError("project_id 不能为空")
|
||||
if not title:
|
||||
raise ValueError("title 不能为空")
|
||||
now = ts if ts is not None else time.time()
|
||||
plan = {
|
||||
"plan_id": "%d-%s" % (int(now * 1000),
|
||||
str(len(self._plans(project_id)))),
|
||||
"project_id": project_id,
|
||||
"title": title,
|
||||
"task_id": task_id,
|
||||
"status": "pending",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
self._data.setdefault(project_id, []).append(plan)
|
||||
_save_json(self.path, self._data)
|
||||
return dict(plan)
|
||||
|
||||
def set_status(self, project_id: str, plan_id: str,
|
||||
status: str, ts: float | None = None) -> dict:
|
||||
"""改计划状态:非法流转或未知 plan_id 抛 ValueError。"""
|
||||
if status not in PLAN_STATUSES:
|
||||
raise ValueError("未知计划状态:%s" % status)
|
||||
for plan in self._plans(project_id):
|
||||
if plan["plan_id"] != plan_id:
|
||||
continue
|
||||
allowed = _PLAN_TRANSITIONS[plan["status"]]
|
||||
if status not in allowed:
|
||||
raise ValueError(
|
||||
"计划 %s 不能从 %s 到 %s"
|
||||
% (plan_id, plan["status"], status))
|
||||
plan["status"] = status
|
||||
plan["updated_at"] = ts if ts is not None else time.time()
|
||||
_save_json(self.path, self._data)
|
||||
return dict(plan)
|
||||
raise ValueError("未知计划:%s" % plan_id)
|
||||
|
||||
def list_plans(self, project_id: str, status: str = "") -> list:
|
||||
"""该项目的计划(副本,按创建时间排序,可按状态过滤)。"""
|
||||
plans = [dict(p) for p in self._plans(project_id)]
|
||||
plans.sort(key=lambda p: p.get("created_at", 0.0))
|
||||
if status:
|
||||
plans = [p for p in plans if p.get("status") == status]
|
||||
return plans
|
||||
|
||||
def _plans(self, project_id: str) -> list:
|
||||
"""该项目的计划列表(内部引用,调用方不得外泄)。"""
|
||||
items = self._data.get(project_id, [])
|
||||
return items if isinstance(items, list) else []
|
||||
|
||||
|
||||
class PermissionRegistry:
|
||||
"""已批准权限登记:接 P5 能力描述,不存任何凭证值。
|
||||
|
||||
授权只含绑定(连接器/主机/用例/档位/有效期/次数余量),
|
||||
与 P5 ``ScopedCapability.describe()`` 的字段一一对应。
|
||||
"""
|
||||
|
||||
def __init__(self, path: str) -> None:
|
||||
self.path = path
|
||||
self._data = _load_json(path)
|
||||
|
||||
def record_grant(self, project_id: str, capability: dict,
|
||||
ts: float | None = None) -> dict:
|
||||
"""登记一条已签发的授权,返回该授权(副本)。
|
||||
|
||||
capability 是 P5 describe() 形状的 dict;缺
|
||||
capability_id/tier/connector/host/use_case 任一即
|
||||
抛 ValueError。同 capability_id 重复登记视为更新。
|
||||
"""
|
||||
if not project_id:
|
||||
raise ValueError("project_id 不能为空")
|
||||
required = ("capability_id", "tier", "connector", "host",
|
||||
"use_case")
|
||||
for key in required:
|
||||
if not capability.get(key):
|
||||
raise ValueError("能力描述缺字段:%s" % key)
|
||||
now = ts if ts is not None else time.time()
|
||||
grant = {
|
||||
"capability_id": str(capability["capability_id"]),
|
||||
"project_id": project_id,
|
||||
"tier": str(capability["tier"]),
|
||||
"connector": str(capability["connector"]),
|
||||
"host": str(capability["host"]),
|
||||
"use_case": str(capability["use_case"]),
|
||||
"session_id": capability.get("session_id") or "",
|
||||
"task_id": capability.get("task_id") or "",
|
||||
"methods": sorted(capability.get("methods") or []),
|
||||
"max_uses": capability.get("max_uses"),
|
||||
"uses_left": capability.get("uses_left"),
|
||||
"expires_at": capability.get("expires_at"),
|
||||
"revoked": bool(capability.get("revoked", False)),
|
||||
"granted_at": now,
|
||||
}
|
||||
grants = self._grants(project_id)
|
||||
for index, old in enumerate(grants):
|
||||
if old["capability_id"] == grant["capability_id"]:
|
||||
grants[index] = grant
|
||||
break
|
||||
else:
|
||||
grants.append(grant)
|
||||
self._data.setdefault(project_id, grants)
|
||||
_save_json(self.path, self._data)
|
||||
return dict(grant)
|
||||
|
||||
def note_use(self, project_id: str, capability_id: str) -> bool:
|
||||
"""记一次使用:次数余量减一;未知 id 或已无余量返回 False。"""
|
||||
for grant in self._grants(project_id):
|
||||
if grant["capability_id"] != capability_id:
|
||||
continue
|
||||
if grant.get("revoked"):
|
||||
return False
|
||||
left = grant.get("uses_left")
|
||||
if left is None:
|
||||
return True
|
||||
if left <= 0:
|
||||
return False
|
||||
grant["uses_left"] = left - 1
|
||||
_save_json(self.path, self._data)
|
||||
return True
|
||||
return False
|
||||
|
||||
def revoke(self, project_id: str, capability_id: str) -> bool:
|
||||
"""撤销一条授权:未知 id 返回 False,已撤销也返回 False。"""
|
||||
for grant in self._grants(project_id):
|
||||
if grant["capability_id"] != capability_id:
|
||||
continue
|
||||
if grant.get("revoked"):
|
||||
return False
|
||||
grant["revoked"] = True
|
||||
_save_json(self.path, self._data)
|
||||
return True
|
||||
return False
|
||||
|
||||
def list_active(self, project_id: str,
|
||||
now: float | None = None) -> list:
|
||||
"""该项目当前有效的授权:未撤销、未过期、次数未用完。"""
|
||||
at = now if now is not None else time.time()
|
||||
active = [dict(g) for g in self._grants(project_id)
|
||||
if _grant_active(g, at)]
|
||||
active.sort(key=lambda g: g.get("granted_at", 0.0))
|
||||
return active
|
||||
|
||||
def list_all(self, project_id: str) -> list:
|
||||
"""该项目全部授权(含已撤销/已过期,副本,按签发排序)。"""
|
||||
grants = [dict(g) for g in self._grants(project_id)]
|
||||
grants.sort(key=lambda g: g.get("granted_at", 0.0))
|
||||
return grants
|
||||
|
||||
def _grants(self, project_id: str) -> list:
|
||||
"""该项目的授权列表(内部引用,调用方不得外泄)。"""
|
||||
items = self._data.get(project_id, [])
|
||||
return items if isinstance(items, list) else []
|
||||
|
||||
|
||||
def _grant_active(grant: dict, now: float) -> bool:
|
||||
"""单条授权是否当前有效(未撤销 + 未过期 + 有余量)。"""
|
||||
if grant.get("revoked"):
|
||||
return False
|
||||
expires_at = grant.get("expires_at")
|
||||
if expires_at is not None and float(expires_at) <= now:
|
||||
return False
|
||||
left = grant.get("uses_left")
|
||||
if left is not None and int(left) <= 0:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def activity_summary(plans: list, recent_records: list,
|
||||
grants: list) -> dict:
|
||||
"""拼活动摘要(纯函数):正在做 / 计划中 / 已做 / 近期 / 授权。
|
||||
|
||||
入参是调用方已按项目过滤好的列表:plans 为计划 dict 列表,
|
||||
recent_records 为近期审计记录(倒序),grants 为有效授权。
|
||||
返回只含描述性字段,不含任何密钥与对话原文。
|
||||
"""
|
||||
running = [p for p in plans if p.get("status") == "running"]
|
||||
pending = [p for p in plans if p.get("status") == "pending"]
|
||||
done = [p for p in plans if p.get("status") == "done"]
|
||||
current = running[0]["title"] if running else ""
|
||||
recent = [{"ts": r.get("ts"), "action": r.get("action"),
|
||||
"category": r.get("category"),
|
||||
"result": r.get("result"),
|
||||
"object_type": r.get("object_type"),
|
||||
"object_id": r.get("object_id")}
|
||||
for r in recent_records]
|
||||
return {
|
||||
"current": current,
|
||||
"running": running,
|
||||
"planned": pending,
|
||||
"done": done,
|
||||
"recent": recent,
|
||||
"granted_permissions": grants,
|
||||
"counts": {
|
||||
"running": len(running),
|
||||
"planned": len(pending),
|
||||
"done": len(done),
|
||||
"granted": len(grants),
|
||||
},
|
||||
}
|
||||
@@ -1,162 +0,0 @@
|
||||
"""Cloud Hypervisor REST API 传输(叶子模块,不依赖
|
||||
cloud.sandbox,避免循环导入)。
|
||||
|
||||
只用标准库 raw HTTP over Unix socket,无第三方依赖。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
|
||||
|
||||
def _read_header(sock):
|
||||
"""读满响应头:返回 (header_text, 已收体部前缀)。
|
||||
|
||||
超时或提前关流即 RuntimeError,绝不把半截头当结束
|
||||
(feat-chv-fix P1-6 的语义,纯移动)。"""
|
||||
buf = b""
|
||||
header_end = None
|
||||
timed_out = False
|
||||
while header_end is None:
|
||||
try:
|
||||
data = sock.recv(65536)
|
||||
except socket.timeout:
|
||||
timed_out = True
|
||||
break
|
||||
if not data:
|
||||
break
|
||||
buf += data
|
||||
header_end = buf.find(b"\r\n\r\n")
|
||||
if timed_out or header_end is None:
|
||||
raise RuntimeError("CHV API 响应头不完整(超时或提前关流)")
|
||||
header = buf[:header_end].decode("utf-8", errors="replace")
|
||||
return header, buf[header_end + 4:]
|
||||
|
||||
|
||||
def _parse_status(header: str) -> int:
|
||||
"""从响应状态行取状态码(解析失败即 0,与
|
||||
_parse_response 的口径一致)。"""
|
||||
try:
|
||||
return int(header.split()[1])
|
||||
except (IndexError, ValueError):
|
||||
return 0
|
||||
|
||||
|
||||
def _parse_content_length(header):
|
||||
"""从响应头取 Content-Length:chunked 或无长度即拒绝。
|
||||
|
||||
chunked 明确拒绝(CHV REST 不应发 chunked);无长度则
|
||||
无法证完整性,同样拒绝。"""
|
||||
content_length = None
|
||||
is_chunked = False
|
||||
for line in header.split("\r\n")[1:]:
|
||||
if ":" not in line:
|
||||
continue
|
||||
name, value = line.split(":", 1)
|
||||
name = name.strip().lower()
|
||||
value = value.strip()
|
||||
if name == "content-length":
|
||||
try:
|
||||
content_length = int(value)
|
||||
except ValueError:
|
||||
raise RuntimeError("CHV API Content-Length 非法")
|
||||
elif name == "transfer-encoding" \
|
||||
and "chunked" in value.lower():
|
||||
is_chunked = True
|
||||
if is_chunked:
|
||||
raise RuntimeError("CHV API 响应为 chunked,拒绝解析")
|
||||
if content_length is None:
|
||||
raise RuntimeError("CHV API 响应无 Content-Length,拒绝解析")
|
||||
return content_length
|
||||
|
||||
|
||||
def _read_body(sock, body, content_length):
|
||||
"""按 Content-Length 读满体部:超时或提前关流即截断错误。"""
|
||||
while len(body) < content_length:
|
||||
try:
|
||||
data = sock.recv(65536)
|
||||
except socket.timeout:
|
||||
raise RuntimeError(
|
||||
"CHV API 响应体截断:已收 %d / 需 %d 字节"
|
||||
% (len(body), content_length))
|
||||
if not data:
|
||||
raise RuntimeError(
|
||||
"CHV API 响应体截断:已收 %d / 需 %d 字节"
|
||||
% (len(body), content_length))
|
||||
body += data
|
||||
return body[:content_length]
|
||||
|
||||
|
||||
def _parse_response(header, body):
|
||||
"""从已读满的头与体部解析 (status, body):能解析 JSON
|
||||
时为 dict,否则为原文本;状态行解析失败时 status 为 0。"""
|
||||
text = header + "\r\n\r\n" + body.decode("utf-8", errors="replace")
|
||||
status = 0
|
||||
if text.startswith("HTTP/"):
|
||||
try:
|
||||
status = int(text.split()[1])
|
||||
except (IndexError, ValueError):
|
||||
status = 0
|
||||
body_text = body.decode("utf-8", errors="replace")
|
||||
try:
|
||||
parsed = json.loads(body_text) if body_text.strip() else {}
|
||||
except ValueError:
|
||||
parsed = body_text
|
||||
return status, parsed
|
||||
|
||||
|
||||
def _unix_socket_api(socket_path, method, path, payload=None):
|
||||
"""Cloud Hypervisor REST API 的默认传输:标准库 raw HTTP
|
||||
over Unix socket(无第三方依赖)。返回 (status, body),
|
||||
body 能解析 JSON 时为 dict,否则为原文本。socket 不存在
|
||||
或连接失败时 raise RuntimeError(调用方据此判 api 超时
|
||||
/失败,不编造 200)。读满语义见 _read_header /
|
||||
_parse_content_length / _read_body(P1-6,纯移动)。
|
||||
204/304 例外:按 HTTP 语义体长恒为 0,不要求
|
||||
Content-Length、也不读体——CHV 的 vm.create/vm.boot
|
||||
成功即回 204 且不带该头(2026-11-07 WSL 真机实测:
|
||||
vmm.ping 200 带长度、vm.create 204 无长度),严格
|
||||
校验会把合法成功当失败拒掉;其它状态仍严格(无长度
|
||||
/截断/chunked 均拒绝)。"""
|
||||
if not os.path.exists(socket_path):
|
||||
raise RuntimeError("api socket 不存在:%s" % socket_path)
|
||||
body = b""
|
||||
if payload is not None:
|
||||
body = json.dumps(payload).encode("utf-8")
|
||||
req_lines = ["%s /api/v1%s HTTP/1.1" % (method, path),
|
||||
"Host: localhost",
|
||||
"Accept: application/json",
|
||||
"Content-Type: application/json",
|
||||
"Content-Length: %d" % len(body),
|
||||
"Connection: close", "", ""]
|
||||
raw = "\r\n".join(req_lines).encode("utf-8") + body
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
sock.settimeout(5)
|
||||
try:
|
||||
sock.connect(socket_path)
|
||||
sock.sendall(raw)
|
||||
header, body = _read_header(sock)
|
||||
if _parse_status(header) in (204, 304):
|
||||
# 体长恒为 0:不读体(keep-alive 下读会等到超时)。
|
||||
body = b""
|
||||
else:
|
||||
content_length = _parse_content_length(header)
|
||||
body = _read_body(sock, body, content_length)
|
||||
finally:
|
||||
sock.close()
|
||||
return _parse_response(header, body)
|
||||
|
||||
|
||||
def normalize_api_result(result: dict) -> tuple:
|
||||
"""API 调用归一化:fake api_client 可返回 (status, body)
|
||||
或 {"status":..., "body":...} 或带 .status 的对象,本
|
||||
函数统一成 (status:int, body)。"""
|
||||
if isinstance(result, tuple) and len(result) == 2:
|
||||
return int(result[0]), result[1]
|
||||
if isinstance(result, dict):
|
||||
status = result.get("status", result.get("status_code"))
|
||||
body = result.get("body", result)
|
||||
return (int(status) if status is not None else 0), body
|
||||
status = getattr(result, "status", None) \
|
||||
or getattr(result, "status_code", None)
|
||||
return (int(status) if status is not None else 0), result
|
||||
@@ -1,641 +0,0 @@
|
||||
"""每 VM 结构化应用状态库(对齐 Muse P7,叶子模块)。
|
||||
|
||||
Muse 官方安全文(security.muse.ai 2026-09-08,见
|
||||
docs/chv-muse-p7-gap.md §1):Postgres 存持久应用状态,与运行时
|
||||
单元、凭证存储分离。本轨的选型结论写在差距表 §4:轻量线用
|
||||
SQLite(每 VM 一个 0600 文件,WAL),不是 Postgres——镜像
|
||||
+150–250MB 与每 VM 一个 server 进程对 2GB 轻量 VM 太重;宿主侧
|
||||
共享 Postgres 会把多用户状态集中到宿主,破坏每用户 VM 隔离。
|
||||
|
||||
与 Postgres 的差距(诚实写明):无多写者服务器、无网络访问、
|
||||
无行级安全、无内置复制(复制 = 备份/恢复路径)。DDL 用两方
|
||||
共通的受限子集(TEXT/INTEGER/REAL 列、TEXT id、无
|
||||
AUTOINCREMENT、无 SQLite 专有语法、参数化查询),换 Postgres =
|
||||
换连接工厂 + 同一份 DDL。
|
||||
|
||||
分离保证:
|
||||
- 库文件只活在宿主 `state_dir/<vm>/app-state.sqlite`,不在客机
|
||||
磁盘镜像里,nspawn cell 物理上够不着(测试钉住)。
|
||||
- 凭证绝不进库:所有写操作过无凭证守卫(键名 + 值模式双重
|
||||
检查);守卫拒绝时只报键名/模式,不含值。唯一例外是
|
||||
round-12 新增的 ``kv_put``(P8 登记文档含用户 prompt,
|
||||
prompt 里合法出现密钥样文本是调用方的事,旧 JSON 后端
|
||||
同样不过守卫;见 kv_put 的 docstring)。
|
||||
- audit 表只追加:本模块不提供任何删除/修改审计的方法。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import sqlite3
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
|
||||
from cloud.chv.atomic_file import atomic_write, atomic_write_text
|
||||
|
||||
SCHEMA_VERSION = "1"
|
||||
|
||||
# DDL:SQLite 与 Postgres 共通的受限子集。TEXT id 由调用方生成
|
||||
#(uuid hex),不用 AUTOINCREMENT;时间戳用 ISO 文本。
|
||||
DDL = """
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS tasks (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
kind TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
payload_json TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_tasks_user ON tasks(user_id, status);
|
||||
CREATE TABLE IF NOT EXISTS goals (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
plan_json TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_goals_user ON goals(user_id, status);
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL,
|
||||
chat_id TEXT NOT NULL,
|
||||
summary TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL,
|
||||
updated_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_user ON sessions(user_id, chat_id);
|
||||
CREATE TABLE IF NOT EXISTS audit (
|
||||
id TEXT PRIMARY KEY,
|
||||
actor TEXT NOT NULL,
|
||||
action TEXT NOT NULL,
|
||||
target TEXT NOT NULL,
|
||||
detail_json TEXT NOT NULL,
|
||||
created_at TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_target ON audit(target, created_at);
|
||||
-- P12 统一审计轨迹(round-12 迁移目标):与 JSONL 版同一记录
|
||||
-- 形状(seq/hash/prev_hash 链、project_id 隔离),seq 由写入方
|
||||
-- 显式给(链是全局的,不按项目分链),不靠自增。
|
||||
CREATE TABLE IF NOT EXISTS audit_trail (
|
||||
seq INTEGER PRIMARY KEY,
|
||||
project_id TEXT NOT NULL,
|
||||
actor TEXT NOT NULL,
|
||||
category TEXT NOT NULL,
|
||||
action TEXT NOT NULL,
|
||||
object_type TEXT NOT NULL,
|
||||
object_id TEXT NOT NULL,
|
||||
result TEXT NOT NULL,
|
||||
reason TEXT NOT NULL,
|
||||
task_id TEXT NOT NULL,
|
||||
approval_id TEXT NOT NULL,
|
||||
detail_json TEXT NOT NULL,
|
||||
ts REAL NOT NULL,
|
||||
prev_hash TEXT NOT NULL,
|
||||
hash TEXT NOT NULL,
|
||||
legacy INTEGER NOT NULL
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_trail_project
|
||||
ON audit_trail(project_id, ts, seq);
|
||||
"""
|
||||
|
||||
_BACKUP_FILENAME = "app-state.sqlite"
|
||||
_MANIFEST_FILENAME = "manifest.json"
|
||||
|
||||
# 无凭证守卫:键名含这些子串即拒;值匹配这些模式即拒。
|
||||
_SECRET_KEY_SUBSTRINGS = ("TOKEN", "SECRET", "PASSWORD", "PRIVATE_KEY",
|
||||
"API_KEY", "ACCESS_KEY", "CLIENT_SECRET")
|
||||
_SECRET_VALUE_PATTERNS = (
|
||||
re.compile(r"ghp_[A-Za-z0-9]{20,}"),
|
||||
re.compile(r"gho_[A-Za-z0-9]{20,}"),
|
||||
re.compile(r"github_pat_[A-Za-z0-9_]{20,}"),
|
||||
re.compile(r"sk-[A-Za-z0-9]{20,}"),
|
||||
re.compile(r"xox[baprs]-[A-Za-z0-9-]{10,}"),
|
||||
re.compile(r"AKIA[0-9A-Z]{16}"),
|
||||
re.compile(r"-----BEGIN [A-Z ]*PRIVATE KEY-----"),
|
||||
)
|
||||
|
||||
|
||||
class SecretRejectedError(ValueError):
|
||||
"""写操作含疑似凭证:只报键名/模式,绝不含值。"""
|
||||
|
||||
|
||||
class AppStateError(RuntimeError):
|
||||
"""状态库操作失败(未打开、备份损坏等)。"""
|
||||
|
||||
|
||||
def state_db_path(state_dir: str, vm_name: str) -> str:
|
||||
"""某 VM 状态库路径:宿主 state_dir 下,不在客机磁盘镜像里。"""
|
||||
return os.path.join(state_dir, vm_name, "app-state.sqlite")
|
||||
|
||||
|
||||
def host_state_db_path(state_dir: str) -> str:
|
||||
"""宿主级状态库路径(round-12):与 per-VM 库同库不同文件。
|
||||
|
||||
P8 的三处登记与 P12 的审计轨迹是宿主级状态(横跨多个 VM、
|
||||
按 project_id 隔离),不属于任何一个 VM,所以不进
|
||||
``state_db_path(state_dir, vm_name)`` 的 per-VM 库,另落在
|
||||
state_dir 根下的 app-state.sqlite。同一个 AppStateStore 类、
|
||||
同一套备份/恢复与 0600 权限口径,只是路径不同。"""
|
||||
return os.path.join(state_dir, "app-state.sqlite")
|
||||
|
||||
|
||||
def _utcnow() -> str:
|
||||
"""ISO 时间戳(UTC,秒级)。"""
|
||||
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
|
||||
|
||||
def _new_id() -> str:
|
||||
"""TEXT 主键:uuid hex。"""
|
||||
return uuid.uuid4().hex
|
||||
|
||||
|
||||
def _scan_value(value: object, path: str) -> None:
|
||||
"""递归扫一个值:键名与字符串值过无凭证守卫。"""
|
||||
if isinstance(value, Mapping):
|
||||
for key, item in value.items():
|
||||
_check_key_name("%s.%s" % (path, key))
|
||||
_scan_value(item, "%s.%s" % (path, key))
|
||||
elif isinstance(value, (list, tuple)):
|
||||
for item in value:
|
||||
_scan_value(item, path)
|
||||
elif isinstance(value, str):
|
||||
_check_secret_value(value, path)
|
||||
|
||||
|
||||
def _check_key_name(path: str) -> None:
|
||||
"""键路径含凭证子串即拒(只报路径,不含值)。"""
|
||||
upper = path.upper()
|
||||
for sub in _SECRET_KEY_SUBSTRINGS:
|
||||
if sub in upper:
|
||||
raise SecretRejectedError(
|
||||
"secret_key_name:%s contains %s" % (path, sub))
|
||||
|
||||
|
||||
def _check_secret_value(value: str, path: str) -> None:
|
||||
"""字符串值命中凭证模式即拒(只报路径与模式序号,不含值)。"""
|
||||
for index, pattern in enumerate(_SECRET_VALUE_PATTERNS):
|
||||
if pattern.search(value):
|
||||
raise SecretRejectedError(
|
||||
"secret_value_pattern:%s pattern#%d" % (path, index))
|
||||
|
||||
|
||||
def assert_no_secrets(mapping: Mapping[str, object]) -> None:
|
||||
"""断言一个映射可入库:键名与值都不过凭证守卫即拒。"""
|
||||
_scan_value(dict(mapping), "payload")
|
||||
|
||||
|
||||
def _sha256_file(path: str) -> str:
|
||||
"""文件 sha256(流式读,不整文件进内存)。"""
|
||||
digest = hashlib.sha256()
|
||||
with open(path, "rb") as handle:
|
||||
while True:
|
||||
chunk = handle.read(65536)
|
||||
if not chunk:
|
||||
break
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _read_manifest(snapshot_dir: str) -> dict[str, str]:
|
||||
"""读并校验备份 manifest:文件在 + sha256 对得上才返回。"""
|
||||
manifest_path = os.path.join(snapshot_dir, _MANIFEST_FILENAME)
|
||||
if not os.path.isfile(manifest_path):
|
||||
raise AppStateError("backup manifest missing: %s" % manifest_path)
|
||||
with open(manifest_path, "r", encoding="utf-8") as handle:
|
||||
manifest = json.load(handle)
|
||||
backup_path = os.path.join(snapshot_dir, _BACKUP_FILENAME)
|
||||
if not os.path.isfile(backup_path):
|
||||
raise AppStateError("backup file missing: %s" % backup_path)
|
||||
actual = _sha256_file(backup_path)
|
||||
if actual != manifest.get("sha256"):
|
||||
raise AppStateError("backup sha256 mismatch: %s" % backup_path)
|
||||
return manifest
|
||||
|
||||
|
||||
def verify_backup(snapshot_dir: str) -> dict[str, str]:
|
||||
"""校验一次备份(manifest + sha256),返回 manifest。"""
|
||||
return _read_manifest(snapshot_dir)
|
||||
|
||||
|
||||
def restore_app_state(snapshot_dir: str, dest_path: str) -> dict[str, str]:
|
||||
"""从备份恢复:先验 manifest,成功后先留 pre-restore 副本再替换。"""
|
||||
manifest = _read_manifest(snapshot_dir)
|
||||
backup_path = os.path.join(snapshot_dir, _BACKUP_FILENAME)
|
||||
if os.path.exists(dest_path):
|
||||
stamp = time.strftime("%Y%m%dT%H%M%S", time.gmtime())
|
||||
preserved = "%s.pre-restore-%s" % (dest_path, stamp)
|
||||
shutil.copyfile(dest_path, preserved)
|
||||
os.chmod(preserved, 0o600)
|
||||
os.makedirs(os.path.dirname(dest_path) or ".", exist_ok=True)
|
||||
shutil.copyfile(backup_path, dest_path)
|
||||
os.chmod(dest_path, 0o600)
|
||||
return manifest
|
||||
|
||||
|
||||
class AppStateStore:
|
||||
"""每 VM 应用状态库:单写者(进程内锁 + SQLite busy 超时)。
|
||||
|
||||
用法:store = AppStateStore(path); store.open(); ...;
|
||||
store.close()。也可用作上下文管理器。库文件 0600,目录 0700。
|
||||
"""
|
||||
|
||||
def __init__(self, path: str) -> None:
|
||||
self._path = path
|
||||
self._conn: sqlite3.Connection | None = None
|
||||
self._lock = threading.RLock()
|
||||
|
||||
def __enter__(self) -> "AppStateStore":
|
||||
self.open()
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc: object) -> None:
|
||||
self.close()
|
||||
|
||||
@property
|
||||
def path(self) -> str:
|
||||
"""库文件路径。"""
|
||||
return self._path
|
||||
|
||||
def open(self) -> None:
|
||||
"""打开(建库 + 建表 + 记 schema 版本),幂等。"""
|
||||
with self._lock:
|
||||
if self._conn is not None:
|
||||
return
|
||||
parent = os.path.dirname(self._path)
|
||||
if parent:
|
||||
os.makedirs(parent, mode=0o700, exist_ok=True)
|
||||
conn = sqlite3.connect(self._path, timeout=30.0)
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA synchronous=FULL")
|
||||
conn.execute("PRAGMA busy_timeout=30000")
|
||||
conn.executescript(DDL)
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)",
|
||||
("schema_version", SCHEMA_VERSION))
|
||||
conn.commit()
|
||||
os.chmod(self._path, 0o600)
|
||||
self._conn = conn
|
||||
|
||||
def close(self) -> None:
|
||||
"""关闭连接(幂等)。"""
|
||||
with self._lock:
|
||||
if self._conn is None:
|
||||
return
|
||||
self._conn.close()
|
||||
self._conn = None
|
||||
|
||||
def _require_conn(self) -> sqlite3.Connection:
|
||||
"""取已打开的连接,未打开即报错。"""
|
||||
if self._conn is None:
|
||||
raise AppStateError("app state store not open: %s" % self._path)
|
||||
return self._conn
|
||||
|
||||
def schema_version(self) -> str:
|
||||
"""库的 schema 版本。"""
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
row = conn.execute(
|
||||
"SELECT value FROM meta WHERE key = ?",
|
||||
("schema_version",)).fetchone()
|
||||
return row[0] if row else ""
|
||||
|
||||
def kv_put(self, namespace: str, key: str, value: str) -> None:
|
||||
"""写一条命名空间 KV(round-12:P8 三处登记的迁移后端)。
|
||||
|
||||
键存为 ``kv:<namespace>:<key>``,值原样存 TEXT(调用方
|
||||
自己 JSON 序列化)。命名空间与键不得含 ``:``(分隔符),
|
||||
不得为空。
|
||||
|
||||
注意:本方法**不过无凭证守卫**——P8 登记文档里含用户
|
||||
写的子代理 prompt,prompt 里合法出现密钥样文本是调用
|
||||
方的事(旧 JSON 文件后端同样不过守卫);守卫仍覆盖
|
||||
put_task/put_goal/put_session/append_audit 这些写操作。
|
||||
KV 的键是代码里写死的固定名,不含用户输入。"""
|
||||
if not namespace or ":" in namespace \
|
||||
or not isinstance(namespace, str):
|
||||
raise ValueError("kv namespace must be a non-empty string"
|
||||
" without ':'")
|
||||
if not key or ":" in key or not isinstance(key, str):
|
||||
raise ValueError("kv key must be a non-empty string"
|
||||
" without ':'")
|
||||
if not isinstance(value, str):
|
||||
raise ValueError("kv value must be str")
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
conn.execute(
|
||||
"INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)",
|
||||
("kv:%s:%s" % (namespace, key), value))
|
||||
conn.commit()
|
||||
|
||||
def kv_get(self, namespace: str, key: str) -> str | None:
|
||||
"""读一条命名空间 KV;不存在回 None。"""
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
row = conn.execute(
|
||||
"SELECT value FROM meta WHERE key = ?",
|
||||
("kv:%s:%s" % (namespace, key),)).fetchone()
|
||||
return row[0] if row else None
|
||||
|
||||
def kv_delete(self, namespace: str, key: str) -> None:
|
||||
"""删一条命名空间 KV(不存在也静默成功)。"""
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
conn.execute("DELETE FROM meta WHERE key = ?",
|
||||
("kv:%s:%s" % (namespace, key),))
|
||||
conn.commit()
|
||||
|
||||
def insert_audit_trail_row(self, record: dict) -> None:
|
||||
"""插一条 P12 审计轨迹行(round-12):原样存入,不重算
|
||||
链——seq/hash/prev_hash 由调用方(StoreAuditTrail)算好,
|
||||
迁移时也原样搬旧行。seq 重复即抛 sqlite3.IntegrityError。"""
|
||||
detail = record.get("detail")
|
||||
if not isinstance(detail, dict):
|
||||
detail = {}
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
conn.execute(
|
||||
"INSERT INTO audit_trail(seq, project_id, actor,"
|
||||
" category, action, object_type, object_id, result,"
|
||||
" reason, task_id, approval_id, detail_json, ts,"
|
||||
" prev_hash, hash, legacy) VALUES (?, ?, ?, ?, ?, ?,"
|
||||
" ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(int(record["seq"]), str(record.get("project_id", "")),
|
||||
str(record.get("actor", "")),
|
||||
str(record.get("category", "")),
|
||||
str(record.get("action", "")),
|
||||
str(record.get("object_type", "")),
|
||||
str(record.get("object_id", "")),
|
||||
str(record.get("result", "ok")),
|
||||
str(record.get("reason", "")),
|
||||
str(record.get("task_id", "")),
|
||||
str(record.get("approval_id", "")),
|
||||
json.dumps(detail, sort_keys=True,
|
||||
ensure_ascii=False),
|
||||
float(record.get("ts", 0.0)),
|
||||
str(record.get("prev_hash", "")),
|
||||
str(record.get("hash", "")),
|
||||
1 if record.get("legacy") else 0))
|
||||
conn.commit()
|
||||
|
||||
def list_audit_trail_rows(self) -> list[dict]:
|
||||
"""全部审计轨迹行,按 seq 升序(链顺序),与 JSONL 版
|
||||
同款记录形状;legacy 只在为真时出现。"""
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
rows = conn.execute(
|
||||
"SELECT seq, project_id, actor, category, action,"
|
||||
" object_type, object_id, result, reason, task_id,"
|
||||
" approval_id, detail_json, ts, prev_hash, hash,"
|
||||
" legacy FROM audit_trail ORDER BY seq ASC").fetchall()
|
||||
out = []
|
||||
for row in rows:
|
||||
rec = {"seq": row[0], "ts": row[12],
|
||||
"project_id": row[1], "actor": row[2],
|
||||
"category": row[3], "action": row[4],
|
||||
"object_type": row[5], "object_id": row[6],
|
||||
"result": row[7], "reason": row[8],
|
||||
"task_id": row[9], "approval_id": row[10],
|
||||
"detail": json.loads(row[11]),
|
||||
"prev_hash": row[13], "hash": row[14]}
|
||||
if row[15]:
|
||||
rec["legacy"] = True
|
||||
out.append(rec)
|
||||
return out
|
||||
|
||||
def counts(self) -> dict[str, int]:
|
||||
"""各表行数(备份/恢复校验用)。"""
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
out: dict[str, int] = {}
|
||||
for table in ("tasks", "goals", "sessions", "audit",
|
||||
"audit_trail"):
|
||||
row = conn.execute(
|
||||
"SELECT COUNT(*) FROM %s" % table).fetchone()
|
||||
out[table] = int(row[0])
|
||||
return out
|
||||
|
||||
def put_task(self, user_id: str, task_id: str, kind: str,
|
||||
status: str, payload: Mapping[str, object]) -> None:
|
||||
"""写/覆盖一个任务(payload 过无凭证守卫)。"""
|
||||
assert_no_secrets(payload)
|
||||
now = _utcnow()
|
||||
blob = json.dumps(dict(payload), sort_keys=True)
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
conn.execute(
|
||||
"INSERT INTO tasks(id, user_id, kind, status,"
|
||||
" payload_json, created_at, updated_at)"
|
||||
" VALUES (?, ?, ?, ?, ?, ?, ?)"
|
||||
" ON CONFLICT(id) DO UPDATE SET"
|
||||
" status = excluded.status,"
|
||||
" payload_json = excluded.payload_json,"
|
||||
" updated_at = excluded.updated_at",
|
||||
(task_id, user_id, kind, status, blob, now, now))
|
||||
conn.commit()
|
||||
|
||||
def get_task(self, task_id: str) -> dict[str, object] | None:
|
||||
"""取一个任务,不存在返回 None。"""
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
row = conn.execute(
|
||||
"SELECT id, user_id, kind, status, payload_json,"
|
||||
" created_at, updated_at FROM tasks WHERE id = ?",
|
||||
(task_id,)).fetchone()
|
||||
return _row_to_task(row) if row else None
|
||||
|
||||
def list_tasks(self, user_id: str,
|
||||
status: str | None = None) -> list[dict[str, object]]:
|
||||
"""列某用户的任务(可按状态过滤,按创建时间升序)。"""
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
if status is None:
|
||||
rows = conn.execute(
|
||||
"SELECT id, user_id, kind, status, payload_json,"
|
||||
" created_at, updated_at FROM tasks"
|
||||
" WHERE user_id = ? ORDER BY created_at",
|
||||
(user_id,)).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"SELECT id, user_id, kind, status, payload_json,"
|
||||
" created_at, updated_at FROM tasks"
|
||||
" WHERE user_id = ? AND status = ? ORDER BY created_at",
|
||||
(user_id, status)).fetchall()
|
||||
return [_row_to_task(row) for row in rows]
|
||||
|
||||
def update_task_status(self, task_id: str, status: str) -> bool:
|
||||
"""只改任务状态(不碰 payload),改到返回 True。"""
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
cur = conn.execute(
|
||||
"UPDATE tasks SET status = ?, updated_at = ?"
|
||||
" WHERE id = ?",
|
||||
(status, _utcnow(), task_id))
|
||||
conn.commit()
|
||||
return cur.rowcount == 1
|
||||
|
||||
def put_goal(self, user_id: str, goal_id: str, title: str,
|
||||
plan: Mapping[str, object], status: str) -> None:
|
||||
"""写/覆盖一个目标(plan 过无凭证守卫)。"""
|
||||
assert_no_secrets(plan)
|
||||
now = _utcnow()
|
||||
blob = json.dumps(dict(plan), sort_keys=True)
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
conn.execute(
|
||||
"INSERT INTO goals(id, user_id, title, plan_json,"
|
||||
" status, created_at, updated_at)"
|
||||
" VALUES (?, ?, ?, ?, ?, ?, ?)"
|
||||
" ON CONFLICT(id) DO UPDATE SET"
|
||||
" title = excluded.title,"
|
||||
" plan_json = excluded.plan_json,"
|
||||
" status = excluded.status,"
|
||||
" updated_at = excluded.updated_at",
|
||||
(goal_id, user_id, title, blob, status, now, now))
|
||||
conn.commit()
|
||||
|
||||
def get_goal(self, goal_id: str) -> dict[str, object] | None:
|
||||
"""取一个目标,不存在返回 None。"""
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
row = conn.execute(
|
||||
"SELECT id, user_id, title, plan_json, status,"
|
||||
" created_at, updated_at FROM goals WHERE id = ?",
|
||||
(goal_id,)).fetchone()
|
||||
return _row_to_goal(row) if row else None
|
||||
|
||||
def list_goals(self, user_id: str) -> list[dict[str, object]]:
|
||||
"""列某用户的目标(按创建时间升序)。"""
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
rows = conn.execute(
|
||||
"SELECT id, user_id, title, plan_json, status,"
|
||||
" created_at, updated_at FROM goals"
|
||||
" WHERE user_id = ? ORDER BY created_at",
|
||||
(user_id,)).fetchall()
|
||||
return [_row_to_goal(row) for row in rows]
|
||||
|
||||
def put_session(self, user_id: str, session_id: str, chat_id: str,
|
||||
summary: str) -> None:
|
||||
"""写/覆盖一个会话摘要(summary 过无凭证守卫)。"""
|
||||
assert_no_secrets({"summary": summary})
|
||||
now = _utcnow()
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
conn.execute(
|
||||
"INSERT INTO sessions(id, user_id, chat_id, summary,"
|
||||
" created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?)"
|
||||
" ON CONFLICT(id) DO UPDATE SET"
|
||||
" summary = excluded.summary,"
|
||||
" updated_at = excluded.updated_at",
|
||||
(session_id, user_id, chat_id, summary, now, now))
|
||||
conn.commit()
|
||||
|
||||
def get_session(self, session_id: str) -> dict[str, object] | None:
|
||||
"""取一个会话,不存在返回 None。"""
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
row = conn.execute(
|
||||
"SELECT id, user_id, chat_id, summary, created_at,"
|
||||
" updated_at FROM sessions WHERE id = ?",
|
||||
(session_id,)).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return {"id": row[0], "user_id": row[1], "chat_id": row[2],
|
||||
"summary": row[3], "created_at": row[4],
|
||||
"updated_at": row[5]}
|
||||
|
||||
def append_audit(self, actor: str, action: str, target: str,
|
||||
detail: Mapping[str, object]) -> str:
|
||||
"""追加一条审计记录(只追加,无删除/修改接口)。"""
|
||||
assert_no_secrets(detail)
|
||||
record_id = _new_id()
|
||||
blob = json.dumps(dict(detail), sort_keys=True)
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
conn.execute(
|
||||
"INSERT INTO audit(id, actor, action, target,"
|
||||
" detail_json, created_at) VALUES (?, ?, ?, ?, ?, ?)",
|
||||
(record_id, actor, action, target, blob, _utcnow()))
|
||||
conn.commit()
|
||||
return record_id
|
||||
|
||||
def list_audit(self, target: str | None = None,
|
||||
limit: int = 100) -> list[dict[str, object]]:
|
||||
"""列审计记录(最新在前;target 可过滤)。"""
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
if target is None:
|
||||
rows = conn.execute(
|
||||
"SELECT id, actor, action, target, detail_json,"
|
||||
" created_at FROM audit ORDER BY created_at DESC"
|
||||
" LIMIT ?", (limit,)).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"SELECT id, actor, action, target, detail_json,"
|
||||
" created_at FROM audit WHERE target = ?"
|
||||
" ORDER BY created_at DESC LIMIT ?",
|
||||
(target, limit)).fetchall()
|
||||
return [_row_to_audit(row) for row in rows]
|
||||
|
||||
def backup(self, dest_dir: str) -> dict[str, str]:
|
||||
"""在线备份到目录:库文件 + manifest(sha256 + 行数 + 版本)。"""
|
||||
with self._lock:
|
||||
conn = self._require_conn()
|
||||
counts = self.counts()
|
||||
os.makedirs(dest_dir, mode=0o700, exist_ok=True)
|
||||
backup_path = os.path.join(dest_dir, _BACKUP_FILENAME)
|
||||
if os.path.exists(backup_path):
|
||||
os.remove(backup_path)
|
||||
target = sqlite3.connect(backup_path)
|
||||
try:
|
||||
conn.backup(target)
|
||||
finally:
|
||||
target.close()
|
||||
os.chmod(backup_path, 0o600)
|
||||
manifest = {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"sha256": _sha256_file(backup_path),
|
||||
"tasks": str(counts["tasks"]),
|
||||
"goals": str(counts["goals"]),
|
||||
"sessions": str(counts["sessions"]),
|
||||
"audit": str(counts["audit"]),
|
||||
"audit_trail": str(counts["audit_trail"]),
|
||||
"created_at": _utcnow(),
|
||||
}
|
||||
atomic_write_text(
|
||||
os.path.join(dest_dir, _MANIFEST_FILENAME),
|
||||
json.dumps(manifest, sort_keys=True, indent=2) + "\n")
|
||||
return manifest
|
||||
|
||||
|
||||
def _row_to_task(row: tuple) -> dict[str, object]:
|
||||
"""tasks 行 → dict(payload_json 解析回 dict)。"""
|
||||
return {"id": row[0], "user_id": row[1], "kind": row[2],
|
||||
"status": row[3],
|
||||
"payload": json.loads(row[4]),
|
||||
"created_at": row[5], "updated_at": row[6]}
|
||||
|
||||
|
||||
def _row_to_goal(row: tuple) -> dict[str, object]:
|
||||
"""goals 行 → dict(plan_json 解析回 dict)。"""
|
||||
return {"id": row[0], "user_id": row[1], "title": row[2],
|
||||
"plan": json.loads(row[3]), "status": row[4],
|
||||
"created_at": row[5], "updated_at": row[6]}
|
||||
|
||||
|
||||
def _row_to_audit(row: tuple) -> dict[str, object]:
|
||||
"""audit 行 → dict(detail_json 解析回 dict)。"""
|
||||
return {"id": row[0], "actor": row[1], "action": row[2],
|
||||
"target": row[3], "detail": json.loads(row[4]),
|
||||
"created_at": row[5]}
|
||||
@@ -1,172 +0,0 @@
|
||||
"""结构化审批卡数据(feat-chv-muse-p12,叶子模块,纯标准库)。
|
||||
|
||||
接 Muse 官方要求(报告 §2.6):确定性 UI——结构化审批卡
|
||||
(明确接受/拒绝动作);审批是严格能力,绑定特定连接器/
|
||||
目的地与用例,支持一次性、会话级、任务级、时限级、永久级
|
||||
授权,Sentinel 决定向用户呈现哪几种授权类型;审批 UI 不在
|
||||
对话内,Sentinel 直接向客户端发审批请求,用户答复直接回
|
||||
Sentinel(防止对话内容伪造审批)。
|
||||
|
||||
本模块只做卡的**数据**:把 P3 ``ApprovalQueue`` 的 pending
|
||||
记录 dict 转成客户端可呈现的结构化卡。不 import P3/P5 模块
|
||||
——它们不在本基线里,集成时由 round-11 合并;只按 dict
|
||||
契约接线。
|
||||
|
||||
卡数据(纯数据 dict,可直接 JSON 序列化)::
|
||||
{"pending_id": str, "action": str, "target": {...},
|
||||
"reasons": [str, ...], "risk": "low|medium|high",
|
||||
"actions": [{"id": "accept", "label": "接受"},
|
||||
{"id": "deny", "label": "拒绝"}],
|
||||
"grant_options": [{"tier": str, "label": str,
|
||||
"requires_explicit_opt_in": bool}, ...],
|
||||
"expires_at": float | None}
|
||||
|
||||
- ``risk`` 分级(确定性规则,不冒充模型分类器):写方法
|
||||
(POST/PUT/PATCH/DELETE)+ 敏感路径(/admin、/settings、
|
||||
/login、token 相关)或私有主机 → high;写方法但目标不
|
||||
敏感 → medium;其余只读 → low。
|
||||
- ``grant_options`` 的档位列表由调用方传入(Sentinel 决定
|
||||
呈现哪几种);不传默认五档全列,永久档永远标注
|
||||
``requires_explicit_opt_in: true``(P5 口径:永久默认拒绝,
|
||||
须显式 opt-in)。
|
||||
- ``card_matches_channel()``:校验卡数据与 P5 对话外通道载荷
|
||||
(``ApprovalRequest.payload()``)一致——pending_id 相同且
|
||||
目标字段(连接器/方法/scheme/主机/端口/路径)全等;不一
|
||||
致即客户端不该呈现该卡。
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
|
||||
# 五档授权类型(P3/P5 口径):一次/会话/任务/时限/永久。
|
||||
TIERS = ("one-shot", "session", "task", "timed", "permanent")
|
||||
|
||||
TIER_LABELS = {
|
||||
"one-shot": "仅这一次",
|
||||
"session": "本次会话",
|
||||
"task": "本任务期间",
|
||||
"timed": "限时授权",
|
||||
"permanent": "永久授权",
|
||||
}
|
||||
|
||||
# 目标描述字段:卡与通道载荷比对只认这些键。
|
||||
TARGET_KEYS = ("connector", "method", "scheme", "host", "port",
|
||||
"path")
|
||||
|
||||
_WRITE_METHODS = ("POST", "PUT", "PATCH", "DELETE")
|
||||
|
||||
_SENSITIVE_PATHS = ("/admin", "/settings", "/login", "/token",
|
||||
"/oauth", "/credential")
|
||||
|
||||
# 通道审批默认有效期(秒):15 分钟,与 P5 通道一致。
|
||||
DEFAULT_TTL_SECONDS = 900
|
||||
|
||||
|
||||
def _is_private_host(host: str) -> bool:
|
||||
"""主机是否私有/本地:IP 段判断,localhost 视为私有。"""
|
||||
if not host:
|
||||
return False
|
||||
lowered = host.lower()
|
||||
if lowered in ("localhost", "localhost.localdomain"):
|
||||
return True
|
||||
try:
|
||||
return ipaddress.ip_address(host).is_private
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
|
||||
def _is_sensitive_path(path: str) -> bool:
|
||||
"""路径是否敏感:命中敏感前缀即敏感。"""
|
||||
lowered = (path or "").lower()
|
||||
return any(lowered.startswith(p) for p in _SENSITIVE_PATHS)
|
||||
|
||||
|
||||
def risk_level(queue_record: dict) -> str:
|
||||
"""审批风险分级:high/medium/low(确定性规则)。
|
||||
|
||||
写方法 +(敏感路径或私有主机)→ high;写方法 → medium;
|
||||
其余只读 → low。
|
||||
"""
|
||||
method = str(queue_record.get("method", "")).upper()
|
||||
if method not in _WRITE_METHODS:
|
||||
return "low"
|
||||
if _is_sensitive_path(str(queue_record.get("path", ""))):
|
||||
return "high"
|
||||
if _is_private_host(str(queue_record.get("host", ""))):
|
||||
return "high"
|
||||
return "medium"
|
||||
|
||||
|
||||
def _target_from_record(queue_record: dict) -> dict:
|
||||
"""队列记录里只取目标描述字段,别的什么都不带出卡。"""
|
||||
return {k: queue_record[k] for k in TARGET_KEYS
|
||||
if k in queue_record}
|
||||
|
||||
|
||||
def _grant_options(tiers: tuple | list | None) -> list:
|
||||
"""档位选项:未知档位丢弃,永久档永远标注须显式 opt-in。"""
|
||||
wanted = list(tiers) if tiers is not None else list(TIERS)
|
||||
options = []
|
||||
for tier in wanted:
|
||||
if tier not in TIER_LABELS:
|
||||
continue
|
||||
options.append({
|
||||
"tier": tier,
|
||||
"label": TIER_LABELS[tier],
|
||||
"requires_explicit_opt_in": tier == "permanent",
|
||||
})
|
||||
return options
|
||||
|
||||
|
||||
def build_approval_card(queue_record: dict,
|
||||
tiers: tuple | list | None = None,
|
||||
ttl_seconds: int = DEFAULT_TTL_SECONDS
|
||||
) -> dict:
|
||||
"""P3 队列记录转结构化审批卡数据。
|
||||
|
||||
queue_record 是 ``ApprovalQueue.record()`` 记入的 dict:
|
||||
目标描述(connector/method/scheme/host/port/path)+
|
||||
pending_id + reasons + ts。缺 pending_id 抛 ValueError。
|
||||
expires_at 由记录 ts + 有效期算出;无 ts 则为 None。
|
||||
"""
|
||||
pending_id = queue_record.get("pending_id")
|
||||
if not pending_id:
|
||||
raise ValueError("队列记录缺 pending_id")
|
||||
target = _target_from_record(queue_record)
|
||||
connector = str(target.get("connector", ""))
|
||||
method = str(target.get("method", "")).upper()
|
||||
action = "%s %s" % (connector, method) if connector else method
|
||||
ts = queue_record.get("ts")
|
||||
expires_at = None
|
||||
if isinstance(ts, (int, float)):
|
||||
expires_at = float(ts) + ttl_seconds
|
||||
return {
|
||||
"pending_id": str(pending_id),
|
||||
"action": action,
|
||||
"target": target,
|
||||
"reasons": list(queue_record.get("reasons") or []),
|
||||
"risk": risk_level(queue_record),
|
||||
"actions": [
|
||||
{"id": "accept", "label": "接受"},
|
||||
{"id": "deny", "label": "拒绝"},
|
||||
],
|
||||
"grant_options": _grant_options(tiers),
|
||||
"expires_at": expires_at,
|
||||
}
|
||||
|
||||
|
||||
def card_matches_channel(card: dict, channel_payload: dict) -> bool:
|
||||
"""卡数据与 P5 通道载荷是否一致(pending_id + 目标全等)。
|
||||
|
||||
任一目标字段不一致、或 pending_id 不同,即返回 False。
|
||||
客户端只应呈现校验通过的卡。
|
||||
"""
|
||||
if card.get("pending_id") != channel_payload.get("pending_id"):
|
||||
return False
|
||||
card_target = card.get("target") or {}
|
||||
channel_target = channel_payload.get("target") or {}
|
||||
for key in TARGET_KEYS:
|
||||
if key not in card_target and key not in channel_target:
|
||||
continue
|
||||
if card_target.get(key) != channel_target.get(key):
|
||||
return False
|
||||
return True
|
||||
@@ -1,211 +0,0 @@
|
||||
"""对话外审批通道(feat-chv-muse-p5-C,叶子模块,纯标准库)。
|
||||
|
||||
接在 P3 的 ``ApprovalQueue`` 之上:队列只记 pending 待审批,
|
||||
答复只走本通道,绝不走 agent 对话内容。对话里出现的“批准/
|
||||
同意/allow”字样没有任何效力——没有合法 nonce 与 HMAC 签名,
|
||||
校验一律失败。
|
||||
|
||||
通道契约(客户端 UI 呈现是另一侧的事,本模块只给协议):
|
||||
- 请求载荷(``ApprovalRequest.payload()``,可外发给客户端)::
|
||||
{"pending_id": str, "nonce": str, "target": {...},
|
||||
"reasons": [str, ...], "expires_at": float}
|
||||
``target`` 只有目标描述(连接器/方法/scheme/主机/端口/路径),
|
||||
``reasons`` 只有原因名;载荷里绝无凭证值与用户数据内容。
|
||||
P10 扩展:可选 ``details`` 字段(支付审批的确切购买详情,
|
||||
商家/商品/金额/货币,供客户端 UI 展示;不传时 payload 与
|
||||
P5 完全一致)。
|
||||
``nonce`` 是随机 16 字节 hex(32 字符),每次请求唯一。
|
||||
``expires_at`` 默认请求后 15 分钟(900 秒),可注入时钟。
|
||||
- 答复(``Answer``,客户端答完直接回 Sentinel)::
|
||||
{"pending_id": str, "nonce": str, "verdict": "allow"|"deny",
|
||||
"expires_at": float, "signature": str}
|
||||
``signature`` 是 HMAC-SHA256,key 为通道密钥,签名覆盖
|
||||
``pending_id + nonce + verdict + expires_at``(见 ``_sign``)。
|
||||
- 校验(``verify_answer``):pending 存在(本通道签发过且未
|
||||
消费)、nonce 对上、未过期、签名对上、verdict 在 allow/deny
|
||||
内——五项缺一即 False。答复只能用一次:``resolve_via_channel``
|
||||
成功后该 pending 的通道记录即失效,重放同一答复被拒。
|
||||
- 密钥由调用方保管:通道本身不保存密钥、不把密钥写进载荷、
|
||||
绝不进日志;签发请求时传 ``secret`` 只是调用方证明其持有
|
||||
通道密钥(本模块不存它),签名由持有密钥的答复方计算。
|
||||
- 队列接线:校验通过才调 ``ApprovalQueue.resolve``;校验失败
|
||||
返回 False 且队列原样不动(pending 仍在,须重新答复)。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import secrets
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
|
||||
from cloud.chv.sentinel import ApprovalQueue
|
||||
|
||||
# 默认审批有效期(秒):15 分钟。
|
||||
DEFAULT_TTL_SECONDS = 900
|
||||
|
||||
_VERDICTS = ("allow", "deny")
|
||||
|
||||
# 本通道签发过、尚未消费的请求:pending_id -> {"nonce", "expires_at"}。
|
||||
# 只记 nonce 与过期,不记任何凭证值与用户数据内容。
|
||||
_REQUESTS: dict[str, dict] = {}
|
||||
|
||||
|
||||
def _sign(pending_id: str, nonce: str, verdict: str,
|
||||
expires_at: float, secret: str) -> str:
|
||||
"""HMAC-SHA256 签名:覆盖 pending_id + nonce + verdict +
|
||||
expires_at。expires_at 统一按 float 格式化,签发与校验
|
||||
两侧口径一致。"""
|
||||
message = "\n".join((pending_id, nonce, verdict,
|
||||
repr(float(expires_at)))).encode("utf-8")
|
||||
return hmac.new(secret.encode("utf-8"), message,
|
||||
hashlib.sha256).hexdigest()
|
||||
|
||||
|
||||
def _target_from_record(queue_record: dict) -> dict:
|
||||
"""队列记录里只取目标描述字段,别的什么都不带出通道。"""
|
||||
keys = ("connector", "method", "scheme", "host", "port", "path")
|
||||
return {k: queue_record[k] for k in keys if k in queue_record}
|
||||
|
||||
|
||||
class ApprovalRequest:
|
||||
"""一次通道审批请求:pending_id + 目标描述 + 原因 + nonce
|
||||
+ 过期时间。只含描述性字段,无凭证值、无用户数据内容。"""
|
||||
|
||||
def __init__(self, pending_id: str, target: dict,
|
||||
reasons: Iterable[str] | None,
|
||||
nonce: str | None = None,
|
||||
now: float | None = None,
|
||||
ttl_seconds: int = DEFAULT_TTL_SECONDS,
|
||||
details: dict | None = None) -> None:
|
||||
self.pending_id = pending_id
|
||||
self.target = dict(target)
|
||||
self.reasons = list(reasons or [])
|
||||
self.nonce = nonce or secrets.token_hex(16)
|
||||
at = now if now is not None else time.time()
|
||||
self.expires_at = float(at) + ttl_seconds
|
||||
# 可选的确切详情(P10 支付审批用:商家、商品、金额、
|
||||
# 货币——供客户端 UI 展示;无 details 时 payload 与
|
||||
# P5 完全一致)。只含描述字段,无凭证值。
|
||||
self.details = dict(details) if details is not None else None
|
||||
|
||||
def payload(self) -> dict:
|
||||
"""可外发给客户端的请求载荷(只含描述字段 + nonce +
|
||||
过期,无密钥、无凭证值、无用户数据内容)。有 details
|
||||
时带上(支付审批的确切购买详情)。"""
|
||||
out: dict = {"pending_id": self.pending_id, "nonce": self.nonce,
|
||||
"target": dict(self.target),
|
||||
"reasons": list(self.reasons),
|
||||
"expires_at": self.expires_at}
|
||||
if self.details is not None:
|
||||
out["details"] = dict(self.details)
|
||||
return out
|
||||
|
||||
|
||||
def request_approval(queue_record: dict, secret: str,
|
||||
now: float | None = None,
|
||||
ttl_seconds: int = DEFAULT_TTL_SECONDS,
|
||||
details: dict | None = None
|
||||
) -> ApprovalRequest:
|
||||
"""为一条队列记录生成通道请求并登记(nonce + 过期)。
|
||||
|
||||
``secret`` 是调用方持有的通道密钥:本模块不保存它、不进
|
||||
载荷、不进日志;答复方用同一密钥签名,校验时对上才算数。
|
||||
``details`` 可选(P10 支付审批的确切购买详情,进 payload
|
||||
供客户端 UI 展示;不传时行为与 P5 完全一致)。
|
||||
返回的 ``ApprovalRequest.payload()`` 即发给客户端的东西。
|
||||
"""
|
||||
pending_id = queue_record["pending_id"]
|
||||
req = ApprovalRequest(
|
||||
pending_id, _target_from_record(queue_record),
|
||||
queue_record.get("reasons", []),
|
||||
now=now, ttl_seconds=ttl_seconds, details=details)
|
||||
_REQUESTS[pending_id] = {"nonce": req.nonce,
|
||||
"expires_at": req.expires_at}
|
||||
return req
|
||||
|
||||
|
||||
class Answer:
|
||||
"""一次通道答复:pending_id + nonce + verdict + 过期 +
|
||||
HMAC 签名。verdict 只认 allow/deny,其它值校验即拒。"""
|
||||
|
||||
def __init__(self, pending_id: str, nonce: str, verdict: str,
|
||||
expires_at: float, signature: str) -> None:
|
||||
self.pending_id = pending_id
|
||||
self.nonce = nonce
|
||||
self.verdict = verdict
|
||||
self.expires_at = float(expires_at)
|
||||
self.signature = signature
|
||||
|
||||
def as_dict(self) -> dict:
|
||||
"""答复的线上传输格式(与请求载荷同款纯数据 dict)。"""
|
||||
return {"pending_id": self.pending_id, "nonce": self.nonce,
|
||||
"verdict": self.verdict,
|
||||
"expires_at": self.expires_at,
|
||||
"signature": self.signature}
|
||||
|
||||
|
||||
def make_answer(request: ApprovalRequest, verdict: str,
|
||||
secret: str) -> Answer:
|
||||
"""答复方(持有通道密钥的客户端)构造已签名的答复。
|
||||
verdict 只认 allow/deny,其它值直接抛错、不签发。"""
|
||||
if verdict not in _VERDICTS:
|
||||
raise ValueError("verdict must be allow or deny")
|
||||
signature = _sign(request.pending_id, request.nonce, verdict,
|
||||
request.expires_at, secret)
|
||||
return Answer(request.pending_id, request.nonce, verdict,
|
||||
request.expires_at, signature)
|
||||
|
||||
|
||||
def _as_answer(answer) -> Answer | None:
|
||||
"""Answer 对象或同款 dict 统一成 Answer;字段不全即 None
|
||||
(缺字段的答复直接判无效,不抛错)。"""
|
||||
if isinstance(answer, Answer):
|
||||
return answer
|
||||
if isinstance(answer, dict):
|
||||
try:
|
||||
return Answer(answer["pending_id"], answer["nonce"],
|
||||
answer["verdict"], answer["expires_at"],
|
||||
answer["signature"])
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def verify_answer(answer: Answer | dict, secret: str,
|
||||
now: float | None = None) -> bool:
|
||||
"""校验一次答复:pending 存在(本通道签发过且未消费)、
|
||||
nonce 对上、未过期、签名对上、verdict 合法——五项缺一
|
||||
即 False。本函数是纯校验,不消费答复(单次性由
|
||||
``resolve_via_channel`` 在成功时消费登记来保证)。"""
|
||||
ans = _as_answer(answer)
|
||||
if ans is None or ans.verdict not in _VERDICTS:
|
||||
return False
|
||||
issued = _REQUESTS.get(ans.pending_id)
|
||||
if issued is None:
|
||||
return False
|
||||
if ans.nonce != issued["nonce"]:
|
||||
return False
|
||||
at = now if now is not None else time.time()
|
||||
if at >= issued["expires_at"] or at >= ans.expires_at:
|
||||
return False
|
||||
expected = _sign(ans.pending_id, ans.nonce, ans.verdict,
|
||||
ans.expires_at, secret)
|
||||
return hmac.compare_digest(ans.signature, expected)
|
||||
|
||||
|
||||
def resolve_via_channel(queue: ApprovalQueue,
|
||||
answer: Answer | dict, secret: str,
|
||||
now: float | None = None) -> bool:
|
||||
"""校验通过才调队列的 resolve;校验失败返回 False 且
|
||||
队列不动。成功时消费该 pending 的通道登记(答复只能用
|
||||
一次),队列 resolve 失败(pending 已不在队列)则不消费、
|
||||
返回 False。"""
|
||||
if not verify_answer(answer, secret, now=now):
|
||||
return False
|
||||
ans = _as_answer(answer)
|
||||
if ans is None:
|
||||
return False
|
||||
if not queue.resolve(ans.pending_id, ans.verdict):
|
||||
return False
|
||||
_REQUESTS.pop(ans.pending_id, None)
|
||||
return True
|
||||
@@ -1,71 +0,0 @@
|
||||
"""CHV 宿主侧小文件的原子写(P2-18 收尾)。
|
||||
|
||||
此前 net.json / snapshots.json / pool registry /
|
||||
known_hosts 各自用固定名 `<path>.tmp` + 直接 os.replace,
|
||||
存在两个问题:
|
||||
|
||||
1. 固定 tmp 名:同一进程内两个写者(或一次崩溃残留的
|
||||
.tmp)会互相覆盖/误读;不同 VM 的文件虽在不同目录,
|
||||
但同一目录内的快照登记与 net 登记仍共用可预测的
|
||||
tmp 名,崩溃后残留的 .tmp 会被下一次写静默覆盖,
|
||||
丢了“上次写了一半”的信号。
|
||||
2. 无 fsync:os.replace 只保证目录项原子,不保证内容
|
||||
已落盘;宿主断电/崩溃后新文件可能回到旧内容或
|
||||
0 字节,而调用方已认为登记成功。
|
||||
|
||||
本模块是唯一写路径:mkstemp 随机名(0600)→ 写 →
|
||||
flush + fsync 文件 → os.replace → fsync 父目录。
|
||||
写失败时删临时文件再原样抛错,调用方按各自语义
|
||||
处理(审计/拒绝/脏标记),本模块不吞错、不记日志。
|
||||
"""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
|
||||
def atomic_write(path: str, data: bytes, mode: int = 0o600) -> None:
|
||||
"""原子写 bytes 到 path(随机 tmp 名 + 双 fsync)。
|
||||
|
||||
mode 是最终文件的权限(默认 0600:这些文件都在
|
||||
0700 的 state_dir 内、且含登记/密钥类内容,按敏感
|
||||
文件对待;调用方要放宽时显式传)。
|
||||
"""
|
||||
directory = os.path.dirname(path) or "."
|
||||
fd, tmp_path = tempfile.mkstemp(prefix=".tmp-", dir=directory)
|
||||
try:
|
||||
os.fchmod(fd, mode)
|
||||
with os.fdopen(fd, "wb") as fh:
|
||||
fh.write(data)
|
||||
fh.flush()
|
||||
os.fsync(fh.fileno())
|
||||
os.replace(tmp_path, path)
|
||||
_fsync_dir(directory)
|
||||
except BaseException:
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
except OSError:
|
||||
pass
|
||||
raise
|
||||
|
||||
|
||||
def atomic_write_text(path: str, text: str, mode: int = 0o600) -> None:
|
||||
"""atomic_write 的 str 版(UTF-8 编码)。"""
|
||||
atomic_write(path, text.encode("utf-8"), mode=mode)
|
||||
|
||||
|
||||
def _fsync_dir(directory: str) -> None:
|
||||
"""fsync 父目录,让 rename 本身落盘(best-effort:
|
||||
某些文件系统不支持目录 fsync,失败即跳过——文件
|
||||
内容的 fsync 已在 atomic_write 里做完,目录 fsync
|
||||
丢了只影响“rename 是否在崩溃后可见”,不影响内容)。
|
||||
"""
|
||||
try:
|
||||
dir_fd = os.open(directory, os.O_RDONLY)
|
||||
except OSError:
|
||||
return
|
||||
try:
|
||||
os.fsync(dir_fd)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
os.close(dir_fd)
|
||||
@@ -1,507 +0,0 @@
|
||||
"""统一审计轨迹(feat-chv-muse-p12,叶子模块,纯标准库)。
|
||||
|
||||
接 Muse 官方要求(报告 §2.2 第 7 步、§2.6):完整审计轨迹展示
|
||||
已做与计划做的事;审批走确定性 UI;用户可检查、编辑、下载
|
||||
VM 内所有文件。
|
||||
|
||||
现状收敛:基线里审计是散的——``AuthStore.audit()`` 只追加
|
||||
``(user_id, project_id, action, detail, created_at)`` 到 SQLite
|
||||
``audit_log``,``Worker._audit()`` 是 best-effort 包装,
|
||||
``worker_adapter.make_chv_audit_callback`` 把 CHV 事件转成
|
||||
``"chv_vm_" + event``。本模块给它们一个统一记录模型与一条
|
||||
追加写 JSONL 轨迹,不改现有调用方。
|
||||
|
||||
记录模型(每条都是纯数据 dict,可直接 JSON 序列化)::
|
||||
{"seq": int, "ts": float, "project_id": str, "actor": str,
|
||||
"category": str, "action": str, "object_type": str,
|
||||
"object_id": str, "result": str, "reason": str,
|
||||
"task_id": str, "approval_id": str, "detail": dict,
|
||||
"prev_hash": str, "hash": str}
|
||||
|
||||
- ``category`` 只认 ``AUDIT_CATEGORIES``:任务派发、VM 生命周期、
|
||||
同步、审批、支付(P10)、配额(P11)、记忆操作(P7 的忘记)、
|
||||
后台任务(P8)、推理/遥测(P9)、安全事件。
|
||||
- ``detail`` 只含描述性字段:``sanitize_detail()`` 按字段级白名
|
||||
单清洗——密钥、token 值、对话原文绝不进审计。清洗是字段名
|
||||
黑名单 + ``key=value`` 模式脱敏,不是内容理解:密钥若藏在
|
||||
自由文本里且不带标记,清洗器看不出来,调用方仍须只传描述
|
||||
性字段(与 P3/P5 的口径一致)。
|
||||
- 哈希链:每条记录的 ``hash`` = SHA-256(prev_hash + 规范 JSON),
|
||||
首条 prev_hash 为 ``GENESIS_HASH``。``verify()`` 能发现改写/
|
||||
删除/调序。诚实边界:这是**篡改可发现**,不是防篡改——能写
|
||||
文件的人重算整条链就能掩盖改动;真防篡改要外部锚定(远端
|
||||
日志/签名时间戳),不在本轨。
|
||||
- project_id 强制:``list_records()``/``export_json()`` 必须传 project_id,
|
||||
只返回该项目记录;跨项目读取返回空,不抛错(与现有
|
||||
``list_audit`` 行为一致,不泄露“存在别的项目记录”)。
|
||||
- 持久化:round-12 已迁移到 P7 状态库(``StoreAuditTrail``,
|
||||
宿主级 AppStateStore 的 ``audit_trail`` 表,同一记录形状与
|
||||
哈希链);``AuditTrail``(JSONL 0600)保留,用于读旧文件与
|
||||
迁移源,``migrate_jsonl_to_store()`` 原样搬链(坏链拒绝、
|
||||
分叉拒绝、重跑幂等);``from_legacy_row()`` 把现有
|
||||
``audit_log`` 行转成统一记录(标 ``legacy: true``,本身无
|
||||
哈希链,append 时重新入链)。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
# 审计事件分类:P1~P11 各轨的事件都落进这里,不另起表。
|
||||
AUDIT_CATEGORIES = frozenset((
|
||||
"task_dispatch", "vm_lifecycle", "gitea_sync", "approval",
|
||||
"payment", "quota", "memory", "background", "inference",
|
||||
"telemetry", "security",
|
||||
))
|
||||
|
||||
# 创世前哈希:首条记录的 prev_hash。
|
||||
GENESIS_HASH = "0" * 64
|
||||
|
||||
# 字段名里出现这些子串(小写比对)的键一律丢弃:值疑似密钥。
|
||||
_SECRET_KEY_HINTS = (
|
||||
"token", "secret", "password", "passwd", "api_key", "apikey",
|
||||
"credential", "private_key", "bearer", "cookie", "auth",
|
||||
"session",
|
||||
)
|
||||
|
||||
# 自由文本里的 ``key=value`` / ``key: value`` 模式:值脱敏。
|
||||
_SECRET_VALUE_RE = re.compile(
|
||||
r"(?i)(token|secret|password|passwd|api[-_]?key|bearer"
|
||||
r"|authorization)\s*[:=]\s*[^,\s;\"']+")
|
||||
|
||||
_MAX_STRING_LEN = 4096
|
||||
|
||||
|
||||
def _is_secret_key(key: str) -> bool:
|
||||
"""字段名是否疑似密钥(小写子串比对)。"""
|
||||
lowered = key.lower()
|
||||
return any(hint in lowered for hint in _SECRET_KEY_HINTS)
|
||||
|
||||
|
||||
def _sanitize_string(value: str) -> str:
|
||||
"""自由文本脱敏:密钥模式的值打码,超长截断。"""
|
||||
redacted = _SECRET_VALUE_RE.sub(r"\1=***", value)
|
||||
if len(redacted) > _MAX_STRING_LEN:
|
||||
return redacted[:_MAX_STRING_LEN] + "...(truncated)"
|
||||
return redacted
|
||||
|
||||
|
||||
def sanitize_detail(detail: dict | None) -> dict:
|
||||
"""递归清洗 detail:密钥字段丢弃,自由文本脱敏。
|
||||
|
||||
dict 键疑似密钥即整键丢弃;list 逐项清洗;str 脱敏;
|
||||
其它标量原样保留。返回新 dict,不改入参。
|
||||
"""
|
||||
if not detail:
|
||||
return {}
|
||||
cleaned: dict = {}
|
||||
for key, value in detail.items():
|
||||
if _is_secret_key(str(key)):
|
||||
continue
|
||||
cleaned[str(key)] = _sanitize_value(value)
|
||||
return cleaned
|
||||
|
||||
|
||||
def _sanitize_value(value: object) -> object:
|
||||
"""清洗单个值:dict/list/str 递归,其它原样。"""
|
||||
if isinstance(value, dict):
|
||||
return {str(k): _sanitize_value(v)
|
||||
for k, v in value.items() if not _is_secret_key(str(k))}
|
||||
if isinstance(value, list):
|
||||
return [_sanitize_value(item) for item in value]
|
||||
if isinstance(value, str):
|
||||
return _sanitize_string(value)
|
||||
return value
|
||||
|
||||
|
||||
def _canonical(record: dict) -> str:
|
||||
"""记录的规范 JSON(排序键、紧凑分隔、原文编码)。"""
|
||||
return json.dumps(record, sort_keys=True, separators=(",", ":"),
|
||||
ensure_ascii=False)
|
||||
|
||||
|
||||
def compute_hash(prev_hash: str, record: dict) -> str:
|
||||
"""一条记录的链哈希:SHA-256(prev_hash + 规范 JSON)。"""
|
||||
body = prev_hash + "\n" + _canonical(record)
|
||||
return hashlib.sha256(body.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def from_legacy_row(row: dict) -> dict:
|
||||
"""现有 audit_log 行转统一记录(标 legacy,无哈希链)。
|
||||
|
||||
入参是 ``AuthStore.list_audit()`` 的行 dict:
|
||||
id/user_id/project_id/action/detail/created_at。
|
||||
detail 若是 str 则原样包进 ``{"detail": ...}`` 再清洗;
|
||||
若是 dict 直接清洗。返回的记录由调用方经 ``append()``
|
||||
重新入链,seq/hash 由 ``append()`` 补。
|
||||
"""
|
||||
detail = row.get("detail")
|
||||
if isinstance(detail, str):
|
||||
detail = {"detail": detail}
|
||||
if not isinstance(detail, dict):
|
||||
detail = {}
|
||||
return {
|
||||
"project_id": str(row.get("project_id", "")),
|
||||
"actor": str(row.get("user_id", "")),
|
||||
"category": "security",
|
||||
"action": str(row.get("action", "")),
|
||||
"object_type": "",
|
||||
"object_id": str(row.get("id", "")),
|
||||
"result": "ok",
|
||||
"reason": "legacy audit_log row",
|
||||
"task_id": "",
|
||||
"approval_id": "",
|
||||
"detail": sanitize_detail(detail),
|
||||
"ts": float(row.get("created_at", 0.0) or 0.0),
|
||||
"legacy": True,
|
||||
}
|
||||
|
||||
|
||||
class AuditTrail:
|
||||
"""追加写 JSONL 审计轨迹:哈希链 + 项目隔离 + 导出。
|
||||
|
||||
文件不存在即创建(0600);每条 append 都 O_APPEND 写 +
|
||||
fsync,写失败原样抛错(审计失败不静默丢,由调用方决定
|
||||
是否 best-effort)。读路径全部只读,不改文件。
|
||||
"""
|
||||
|
||||
def __init__(self, path: str) -> None:
|
||||
self.path = path
|
||||
self._last_hash = GENESIS_HASH
|
||||
self._seq = 0
|
||||
self._load_tail()
|
||||
|
||||
def _load_tail(self) -> None:
|
||||
"""启动时扫一遍文件,恢复末尾 seq 与链哈希。"""
|
||||
if not os.path.exists(self.path):
|
||||
return
|
||||
with open(self.path, "r", encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except ValueError:
|
||||
continue
|
||||
seq = record.get("seq")
|
||||
if isinstance(seq, int):
|
||||
self._seq = seq
|
||||
chain_hash = record.get("hash")
|
||||
if isinstance(chain_hash, str):
|
||||
self._last_hash = chain_hash
|
||||
|
||||
def append(self, project_id: str, actor: str, category: str,
|
||||
action: str, object_type: str = "",
|
||||
object_id: str = "", result: str = "ok",
|
||||
reason: str = "", task_id: str = "",
|
||||
approval_id: str = "",
|
||||
detail: dict | None = None,
|
||||
ts: float | None = None) -> dict:
|
||||
"""追加一条审计记录,返回该记录(含 seq 与链哈希)。"""
|
||||
if not project_id:
|
||||
raise ValueError("project_id 不能为空")
|
||||
if category not in AUDIT_CATEGORIES:
|
||||
raise ValueError("未知审计分类:%s" % category)
|
||||
seq = self._seq + 1
|
||||
record = {
|
||||
"seq": seq,
|
||||
"ts": ts if ts is not None else time.time(),
|
||||
"project_id": project_id,
|
||||
"actor": actor,
|
||||
"category": category,
|
||||
"action": action,
|
||||
"object_type": object_type,
|
||||
"object_id": object_id,
|
||||
"result": result,
|
||||
"reason": reason,
|
||||
"task_id": task_id,
|
||||
"approval_id": approval_id,
|
||||
"detail": sanitize_detail(detail),
|
||||
"prev_hash": self._last_hash,
|
||||
}
|
||||
chain_hash = compute_hash(self._last_hash, record)
|
||||
record["hash"] = chain_hash
|
||||
line = _canonical(record) + "\n"
|
||||
fd = os.open(self.path, os.O_WRONLY | os.O_CREAT | os.O_APPEND,
|
||||
0o600)
|
||||
try:
|
||||
os.write(fd, line.encode("utf-8"))
|
||||
os.fsync(fd)
|
||||
finally:
|
||||
os.close(fd)
|
||||
self._seq = seq
|
||||
self._last_hash = chain_hash
|
||||
return dict(record)
|
||||
|
||||
def list_records(self, project_id: str, category: str = "",
|
||||
action: str = "", task_id: str = "",
|
||||
since: float = 0.0, until: float = 0.0,
|
||||
limit: int = 100, offset: int = 0) -> list:
|
||||
"""按时间倒序返回该项目的审计记录(副本,可分页过滤)。
|
||||
|
||||
方法名不用 ``list``:类内同名方法会遮住内置 ``list``
|
||||
类型,导致返回注解被 mypy 认成方法本身。
|
||||
project_id 为空或不匹配一律返回空列表。limit <= 0
|
||||
表示不限制条数。
|
||||
"""
|
||||
if not project_id:
|
||||
return []
|
||||
rows = self._read_all()
|
||||
kept = [r for r in rows if r.get("project_id") == project_id]
|
||||
kept = _filter_rows(kept, category, action, task_id,
|
||||
since, until)
|
||||
kept.sort(key=lambda r: (r.get("ts", 0.0), r.get("seq", 0)),
|
||||
reverse=True)
|
||||
if offset:
|
||||
kept = kept[offset:]
|
||||
if limit > 0:
|
||||
kept = kept[:limit]
|
||||
return [dict(r) for r in kept]
|
||||
|
||||
def verify(self) -> tuple[bool, int]:
|
||||
"""校验整条哈希链:返回 (是否通过, 首个坏行下标)。
|
||||
|
||||
通过时下标为 -1。坏行包括:解析失败、seq 不连续、
|
||||
prev_hash 对不上、hash 重算不符。
|
||||
"""
|
||||
rows = self._read_raw()
|
||||
expected_prev = GENESIS_HASH
|
||||
expected_seq = 1
|
||||
for index, raw in enumerate(rows):
|
||||
try:
|
||||
record = json.loads(raw)
|
||||
except ValueError:
|
||||
return False, index
|
||||
if not _chain_ok(record, expected_prev, expected_seq):
|
||||
return False, index
|
||||
expected_prev = record["hash"]
|
||||
expected_seq += 1
|
||||
return True, -1
|
||||
|
||||
def export_json(self, project_id: str, dest: str) -> int:
|
||||
"""导出该项目全部审计记录为 JSON 数组,返回导出条数。
|
||||
|
||||
导出内容与 ``list_records()`` 同款清洗后记录(含哈希链
|
||||
字段,不含任何原始 detail 密钥——清洗在 append 时已做)。
|
||||
"""
|
||||
rows = self.list_records(project_id, limit=0)
|
||||
rows.sort(key=lambda r: (r.get("ts", 0.0), r.get("seq", 0)))
|
||||
with open(dest, "w", encoding="utf-8") as f:
|
||||
json.dump(rows, f, ensure_ascii=False, indent=2)
|
||||
os.chmod(dest, 0o600)
|
||||
return len(rows)
|
||||
|
||||
def _read_all(self) -> list:
|
||||
"""读全部可解析记录(跳过坏行,坏行由 verify 发现)。"""
|
||||
rows = []
|
||||
for raw in self._read_raw():
|
||||
try:
|
||||
record = json.loads(raw)
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(record, dict):
|
||||
rows.append(record)
|
||||
return rows
|
||||
|
||||
def _read_raw(self) -> list:
|
||||
"""读全部非空行原文(verify 用)。"""
|
||||
if not os.path.exists(self.path):
|
||||
return []
|
||||
with open(self.path, "r", encoding="utf-8") as f:
|
||||
return [line.strip() for line in f if line.strip()]
|
||||
|
||||
|
||||
class StoreAuditTrail:
|
||||
"""AppStateStore 后端的统一审计轨迹(round-12 迁移)。
|
||||
|
||||
与 ``AuditTrail``(JSONL)同一记录形状、同一哈希链算法、
|
||||
同一过滤/导出语义;持久化改走宿主级 AppStateStore 的
|
||||
``audit_trail`` 表(``state_dir/app-state.sqlite``)。
|
||||
JSONL 版保留,用于读旧文件与迁移源;新写一律走本类。
|
||||
|
||||
写失败不静默丢(与 JSONL 版同款:append 抛错由调用方
|
||||
决定是否 best-effort);库打不开时 __init__ 即抛错,不
|
||||
回空链——审计链丢了不能假装没事(与 P8 登记的 fail-safe
|
||||
不同,见差距表)。
|
||||
"""
|
||||
|
||||
def __init__(self, state_dir: str) -> None:
|
||||
if not state_dir or not isinstance(state_dir, str):
|
||||
raise ValueError("state_dir required")
|
||||
self.state_dir = state_dir
|
||||
self._last_hash = GENESIS_HASH
|
||||
self._seq = 0
|
||||
self._load_tail()
|
||||
|
||||
def _open_store(self):
|
||||
"""打开宿主级状态库(每次调用新建连接,用完即关)。"""
|
||||
from cloud.chv.app_state import AppStateStore, host_state_db_path
|
||||
store = AppStateStore(host_state_db_path(self.state_dir))
|
||||
store.open()
|
||||
return store
|
||||
|
||||
def _load_tail(self) -> None:
|
||||
"""启动时读库尾,恢复末尾 seq 与链哈希。"""
|
||||
with self._open_store() as store:
|
||||
rows = store.list_audit_trail_rows()
|
||||
if not rows:
|
||||
return
|
||||
last = rows[-1]
|
||||
self._seq = int(last["seq"])
|
||||
self._last_hash = str(last["hash"])
|
||||
|
||||
def append(self, project_id: str, actor: str, category: str,
|
||||
action: str, object_type: str = "",
|
||||
object_id: str = "", result: str = "ok",
|
||||
reason: str = "", task_id: str = "",
|
||||
approval_id: str = "",
|
||||
detail: dict | None = None,
|
||||
ts: float | None = None) -> dict:
|
||||
"""追加一条审计记录,返回该记录(含 seq 与链哈希)。"""
|
||||
if not project_id:
|
||||
raise ValueError("project_id 不能为空")
|
||||
if category not in AUDIT_CATEGORIES:
|
||||
raise ValueError("未知审计分类:%s" % category)
|
||||
seq = self._seq + 1
|
||||
record = {
|
||||
"seq": seq,
|
||||
"ts": ts if ts is not None else time.time(),
|
||||
"project_id": project_id,
|
||||
"actor": actor,
|
||||
"category": category,
|
||||
"action": action,
|
||||
"object_type": object_type,
|
||||
"object_id": object_id,
|
||||
"result": result,
|
||||
"reason": reason,
|
||||
"task_id": task_id,
|
||||
"approval_id": approval_id,
|
||||
"detail": sanitize_detail(detail),
|
||||
"prev_hash": self._last_hash,
|
||||
}
|
||||
chain_hash = compute_hash(self._last_hash, record)
|
||||
record["hash"] = chain_hash
|
||||
with self._open_store() as store:
|
||||
store.insert_audit_trail_row(record)
|
||||
self._seq = seq
|
||||
self._last_hash = chain_hash
|
||||
return dict(record)
|
||||
|
||||
def list_records(self, project_id: str, category: str = "",
|
||||
action: str = "", task_id: str = "",
|
||||
since: float = 0.0, until: float = 0.0,
|
||||
limit: int = 100, offset: int = 0) -> list:
|
||||
"""按时间倒序返回该项目的审计记录(副本,可分页过滤)。
|
||||
project_id 为空或不匹配一律返回空列表。limit <= 0
|
||||
表示不限制条数。"""
|
||||
if not project_id:
|
||||
return []
|
||||
with self._open_store() as store:
|
||||
rows = store.list_audit_trail_rows()
|
||||
kept = [r for r in rows if r.get("project_id") == project_id]
|
||||
kept = _filter_rows(kept, category, action, task_id,
|
||||
since, until)
|
||||
kept.sort(key=lambda r: (r.get("ts", 0.0), r.get("seq", 0)),
|
||||
reverse=True)
|
||||
if offset:
|
||||
kept = kept[offset:]
|
||||
if limit > 0:
|
||||
kept = kept[:limit]
|
||||
return [dict(r) for r in kept]
|
||||
|
||||
def verify(self) -> tuple[bool, int]:
|
||||
"""校验整条哈希链:返回 (是否通过, 首个坏行下标)。
|
||||
通过时下标为 -1。"""
|
||||
with self._open_store() as store:
|
||||
rows = store.list_audit_trail_rows()
|
||||
expected_prev = GENESIS_HASH
|
||||
expected_seq = 1
|
||||
for index, record in enumerate(rows):
|
||||
if not _chain_ok(record, expected_prev, expected_seq):
|
||||
return False, index
|
||||
expected_prev = record["hash"]
|
||||
expected_seq += 1
|
||||
return True, -1
|
||||
|
||||
def export_json(self, project_id: str, dest: str) -> int:
|
||||
"""导出该项目全部审计记录为 JSON 数组,返回导出条数。"""
|
||||
rows = self.list_records(project_id, limit=0)
|
||||
rows.sort(key=lambda r: (r.get("ts", 0.0), r.get("seq", 0)))
|
||||
with open(dest, "w", encoding="utf-8") as f:
|
||||
json.dump(rows, f, ensure_ascii=False, indent=2)
|
||||
os.chmod(dest, 0o600)
|
||||
return len(rows)
|
||||
|
||||
|
||||
def migrate_jsonl_to_store(path: str, state_dir: str) -> int:
|
||||
"""把 JSONL 轨迹原样搬进 StoreAuditTrail,返回新插入条数。
|
||||
|
||||
先校验 JSONL 整条链(坏链即 ValueError,拒绝迁移,不静
|
||||
默丢行);行按原文 seq/hash/prev_hash 插入,不重算链;
|
||||
库里已有且内容相同的 seq 跳过(幂等,可重跑);已有但
|
||||
内容不同的 seq 即 ValueError(分叉,fail-closed)。
|
||||
文件不存在回 0。"""
|
||||
if not os.path.exists(path):
|
||||
return 0
|
||||
trail = AuditTrail(path)
|
||||
ok, bad = trail.verify()
|
||||
if not ok:
|
||||
raise ValueError("JSONL 审计链校验失败,拒绝迁移:行 %d"
|
||||
% bad)
|
||||
rows = trail._read_all()
|
||||
from cloud.chv.app_state import AppStateStore, host_state_db_path
|
||||
store = AppStateStore(host_state_db_path(state_dir))
|
||||
store.open()
|
||||
try:
|
||||
existing = {int(r["seq"]): r
|
||||
for r in store.list_audit_trail_rows()}
|
||||
inserted = 0
|
||||
for row in sorted(rows, key=lambda r: int(r.get("seq", 0))):
|
||||
seq = int(row["seq"])
|
||||
if seq in existing:
|
||||
if existing[seq] != row:
|
||||
raise ValueError(
|
||||
"审计 seq %d 在库里已有不同内容,拒绝迁移"
|
||||
% seq)
|
||||
continue
|
||||
store.insert_audit_trail_row(row)
|
||||
inserted += 1
|
||||
return inserted
|
||||
finally:
|
||||
store.close()
|
||||
|
||||
|
||||
def _filter_rows(rows: list, category: str, action: str,
|
||||
task_id: str, since: float, until: float) -> list:
|
||||
"""list() 的过滤:分类/动作/任务/时间窗口,全空即不过滤。"""
|
||||
kept = rows
|
||||
if category:
|
||||
kept = [r for r in kept if r.get("category") == category]
|
||||
if action:
|
||||
kept = [r for r in kept if r.get("action") == action]
|
||||
if task_id:
|
||||
kept = [r for r in kept if r.get("task_id") == task_id]
|
||||
if since:
|
||||
kept = [r for r in kept if r.get("ts", 0.0) >= since]
|
||||
if until:
|
||||
kept = [r for r in kept if r.get("ts", 0.0) <= until]
|
||||
return kept
|
||||
|
||||
|
||||
def _chain_ok(record: dict, expected_prev: str,
|
||||
expected_seq: int) -> bool:
|
||||
"""单条记录的链校验:seq 连续 + prev 对上 + hash 重算符。"""
|
||||
if not isinstance(record, dict):
|
||||
return False
|
||||
if record.get("seq") != expected_seq:
|
||||
return False
|
||||
if record.get("prev_hash") != expected_prev:
|
||||
return False
|
||||
claimed = record.get("hash")
|
||||
if not isinstance(claimed, str):
|
||||
return False
|
||||
body = {k: v for k, v in record.items() if k != "hash"}
|
||||
return compute_hash(expected_prev, body) == claimed
|
||||
@@ -1,311 +0,0 @@
|
||||
"""authd 式替身凭证服务(feat-chv-muse-p3,叶子模块)。
|
||||
|
||||
对齐 Muse 官方安全文(security.muse.ai 2026-09-08,见
|
||||
docs/chv-muse-p3-gap.md):凭证由 VM 内独立服务 authd 保管
|
||||
(含 OAuth token,存于用户 VM 而非集中基础设施),代理只见
|
||||
替身令牌;内置连接器的执行分离——每个 privsep worker 有明确
|
||||
凭证白名单(日历 worker 不能仅改参数就索取邮件凭证);登录
|
||||
凭证存入运行时单元之外的安全凭证存储,代理不可见,仅在需要
|
||||
时注入。
|
||||
|
||||
本模块是纯逻辑(只用标准库,不依赖 cloud.*),宿主侧与客内
|
||||
authd 守护进程共用同一份代码(见 chv-guest/authd.py):
|
||||
- `CredentialVault`:只活内存的保险库;`store()` 只接受白名单
|
||||
内的(连接器,凭证键);`release()` 按次发放,绑定
|
||||
{连接器, 主机, 仓库路径, 操作},单次有效 + 300 秒 TTL,
|
||||
过期即废;`__repr__` 与审计记录里绝无凭证值。
|
||||
- 替身的两档(见差距表 §1):本期默认是“按次释放”——释放给
|
||||
git 子进程的值仍是真 token,但只进 git 进程内存、不进 agent
|
||||
环境/argv/日志/落盘;真铸 token(Gitea API 铸短期专用 token)
|
||||
由 `mint_gitea_token()` 实现,默认关闭(须真 token 有建 token
|
||||
权限,现有 write:repository 测试 token 铸 token 会失败)。
|
||||
- 位置决策:authd 跑在客内、单元外(root 身份的 systemd 服务),
|
||||
不在宿主侧——Muse 的 authd 就在用户 VM 里;宿主侧方案要走
|
||||
vsock(P1 范围、未就绪),且会把全部 VM 的 token 集中到宿主。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import secrets
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
# 连接器白名单(privsep 口径):本期只认 gitea 一个连接器。
|
||||
# 未知连接器一律拒绝,不得以“默认放行”扩展。
|
||||
ALLOWED_CONNECTORS = frozenset({"gitea"})
|
||||
|
||||
# 每连接器的凭证白名单:连接器 → 允许的凭证键。gitea 只允许
|
||||
# GITEA_TOKEN;其它键(哪怕调用方传进来)一律拒绝入库。
|
||||
CONNECTOR_CREDENTIALS = {
|
||||
"gitea": frozenset({"GITEA_TOKEN"}),
|
||||
}
|
||||
|
||||
# 按次释放的默认有效期(秒)与次数:单次、300 秒,过期即废。
|
||||
DEFAULT_RELEASE_TTL_SECONDS = 300
|
||||
DEFAULT_RELEASE_MAX_USES = 1
|
||||
|
||||
|
||||
class AuthdRefusal(Exception):
|
||||
"""authd 拒绝:kind + 原因,不含任何凭证值。"""
|
||||
|
||||
def __init__(self, kind: str, message: str) -> None:
|
||||
super().__init__("authd_refused:%s: %s" % (kind, message))
|
||||
self.kind = kind
|
||||
|
||||
|
||||
class SurrogateRelease:
|
||||
"""一次按次释放(替身):绑定到一次具体的凭证使用。
|
||||
|
||||
release_id 是不透明 id(审计与吊销用,不含值);value 只在
|
||||
释放对象里出现一次,由调用方(客内守护进程)经 Unix socket
|
||||
交给 git credential helper、再由 helper 交给 git 子进程——
|
||||
除此之外不进任何环境、argv、日志、审计。`consume()` 校验
|
||||
绑定与有效期,单次有效:用过一次即废,过期即废。"""
|
||||
|
||||
def __init__(self, release_id: str, connector: str, key: str,
|
||||
host: str, repo: str, operation: str, value: str,
|
||||
expires_at: float, max_uses: int = 1) -> None:
|
||||
self.release_id = release_id
|
||||
self.connector = connector
|
||||
self.key = key
|
||||
self.host = host
|
||||
self.repo = repo
|
||||
self.operation = operation
|
||||
self.value = value
|
||||
self.expires_at = expires_at
|
||||
self.uses_left = max_uses
|
||||
|
||||
def describe(self) -> dict:
|
||||
"""审计用描述(只含绑定,不含值)。"""
|
||||
return {"release_id": self.release_id,
|
||||
"connector": self.connector, "key": self.key,
|
||||
"host": self.host, "repo": self.repo,
|
||||
"operation": self.operation,
|
||||
"expires_at": self.expires_at,
|
||||
"uses_left": self.uses_left}
|
||||
|
||||
def __repr__(self) -> str: # 绝不含 value
|
||||
return ("SurrogateRelease(id=%s connector=%s host=%s "
|
||||
"repo=%s operation=%s)"
|
||||
% (self.release_id, self.connector, self.host,
|
||||
self.repo, self.operation))
|
||||
|
||||
|
||||
class CredentialVault:
|
||||
"""只活内存的凭证保险库(一个 VM 一个实例,客内 authd 持有)。
|
||||
|
||||
- 值只存在 `self._store` 的内存 dict 里,不落盘、不进日志;
|
||||
- `store()` 校验白名单:未知连接器 / 该连接器不允许的键 /
|
||||
空值一律拒绝入库(AuthdRefusal),绝不静默收下;
|
||||
- `release()` 按次发放:调用方须给出完整绑定(主机、仓库、
|
||||
操作),绑定与入库时的作用域不一致即拒绝;
|
||||
- `consume()` 单次兑现:release 兑现一次即从库里删除,过期
|
||||
也兑现不了;兑现失败不留可重试的残值。
|
||||
clock 可注入(测试用假时钟测过期),默认 time.time。"""
|
||||
|
||||
def __init__(self, clock: Callable[[], float] | None = None) -> None:
|
||||
self._store: dict[tuple[str, str], dict] = {}
|
||||
self._releases: dict[str, SurrogateRelease] = {}
|
||||
self._clock = clock or time.time
|
||||
|
||||
def __repr__(self) -> str: # 绝不含任何凭证值
|
||||
return "CredentialVault(connectors=%s)" % sorted(
|
||||
{c for (c, _k) in self._store})
|
||||
|
||||
def store(self, connector: str, key: str, value: str,
|
||||
host: str | None = None, repo: str | None = None) -> None:
|
||||
"""入库:白名单校验 + 作用域记录(host/repo 是该凭证
|
||||
允许使用的范围,release 时须一致)。"""
|
||||
if connector not in ALLOWED_CONNECTORS:
|
||||
raise AuthdRefusal(
|
||||
"unknown_connector",
|
||||
"连接器 %s 不在 authd 白名单内" % connector)
|
||||
allowed = CONNECTOR_CREDENTIALS.get(connector, frozenset())
|
||||
if key not in allowed:
|
||||
raise AuthdRefusal(
|
||||
"credential_not_allowlisted",
|
||||
"连接器 %s 不允许凭证键 %s" % (connector, key))
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise AuthdRefusal(
|
||||
"empty_credential", "凭证值为空,拒绝入库")
|
||||
self._store[(connector, key)] = {
|
||||
"value": value, "host": host, "repo": repo}
|
||||
|
||||
def has(self, connector: str, key: str) -> bool:
|
||||
"""该(连接器,键)是否已入库(不返回值)。"""
|
||||
return (connector, key) in self._store
|
||||
|
||||
def forget(self, connector: str, key: str) -> bool:
|
||||
"""删一条入库凭证(VM 停止时调用);同时作废其全部未兑现
|
||||
释放。未知键返回 False,不抛错。"""
|
||||
gone = self._store.pop((connector, key), None) is not None
|
||||
if gone:
|
||||
doomed = [rid for rid, rel in self._releases.items()
|
||||
if rel.connector == connector
|
||||
and rel.key == key]
|
||||
for rid in doomed:
|
||||
self._releases.pop(rid, None)
|
||||
return gone
|
||||
|
||||
def release(self, connector: str, key: str, host: str, repo: str,
|
||||
operation: str,
|
||||
ttl_seconds: int = DEFAULT_RELEASE_TTL_SECONDS,
|
||||
max_uses: int = DEFAULT_RELEASE_MAX_USES
|
||||
) -> SurrogateRelease:
|
||||
"""按次发放:绑定须与入库作用域一致(host/repo 有记录时
|
||||
须一致,无记录时本次绑定即定作用域),否则拒绝。发放的
|
||||
释放单次有效、ttl 后过期;发放本身记入库内待兑现表,
|
||||
须经 consume() 兑现一次才交出值。"""
|
||||
entry = self._store.get((connector, key))
|
||||
if entry is None:
|
||||
raise AuthdRefusal(
|
||||
"credential_not_stored",
|
||||
"连接器 %s 的凭证 %s 未入库" % (connector, key))
|
||||
if entry["host"] is not None and entry["host"] != host:
|
||||
raise AuthdRefusal(
|
||||
"scope_mismatch",
|
||||
"凭证作用域主机不一致,拒绝释放")
|
||||
if entry["repo"] is not None and entry["repo"] != repo:
|
||||
raise AuthdRefusal(
|
||||
"scope_mismatch",
|
||||
"凭证作用域仓库不一致,拒绝释放")
|
||||
if operation not in ("clone", "pull", "push"):
|
||||
raise AuthdRefusal(
|
||||
"operation_not_allowed",
|
||||
"操作 %s 不在允许集内" % operation)
|
||||
release = SurrogateRelease(
|
||||
release_id=secrets.token_hex(8),
|
||||
connector=connector, key=key, host=host, repo=repo,
|
||||
operation=operation, value=entry["value"],
|
||||
expires_at=self._clock() + ttl_seconds,
|
||||
max_uses=max_uses)
|
||||
self._releases[release.release_id] = release
|
||||
return release
|
||||
|
||||
def consume(self, release_id: str) -> str:
|
||||
"""兑现一次释放:过期/用完/未知 id 一律拒绝(AuthdRefusal,
|
||||
不含值)。兑现成功返回凭证值,并把该释放从库里删除
|
||||
(单次有效,用过即废)。"""
|
||||
release = self._releases.get(release_id)
|
||||
if release is None:
|
||||
raise AuthdRefusal(
|
||||
"release_unknown", "释放记录不存在或已兑现")
|
||||
if self._clock() >= release.expires_at:
|
||||
self._releases.pop(release_id, None)
|
||||
raise AuthdRefusal("release_expired", "释放已过期")
|
||||
if release.uses_left <= 0:
|
||||
self._releases.pop(release_id, None)
|
||||
raise AuthdRefusal("release_exhausted", "释放已用完")
|
||||
release.uses_left -= 1
|
||||
value = release.value
|
||||
self._releases.pop(release_id, None)
|
||||
return value
|
||||
|
||||
def audit_release(self, release: SurrogateRelease) -> dict:
|
||||
"""发放审计记录(只含绑定,不含值)。"""
|
||||
record = release.describe()
|
||||
record["event"] = "authd_credential_released"
|
||||
return record
|
||||
|
||||
def audit_refusal(self, kind: str, connector: str,
|
||||
key: str | None = None) -> dict:
|
||||
"""拒绝审计记录(只含 kind 与名,不含值)。"""
|
||||
return {"event": "authd_credential_refused", "kind": kind,
|
||||
"connector": connector, "key": key}
|
||||
|
||||
|
||||
def _default_post(url: str, headers: dict, payload: dict):
|
||||
"""默认 HTTP 客户端(urllib):返回 (status, body_dict)。
|
||||
只用于 mint_gitea_token,测试注入假 post,不联网。"""
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
req = urllib.request.Request(url, data=data, headers=headers,
|
||||
method="POST")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
return resp.status, json.loads(resp.read() or b"{}")
|
||||
except urllib.error.HTTPError as exc:
|
||||
try:
|
||||
body = json.loads(exc.read() or b"{}")
|
||||
except ValueError:
|
||||
body = {}
|
||||
return exc.code, body
|
||||
|
||||
|
||||
def mint_gitea_token(api_url: str, user: str, master_token: str,
|
||||
name: str, scopes: Iterable[str] = ("write:repository",),
|
||||
post: Callable[[str, dict, dict], tuple[int, dict]]
|
||||
| None = None) -> str:
|
||||
"""经 Gitea API 铸短期专用 token(真替身路径,默认关闭)。
|
||||
|
||||
调 `POST {api_url}/users/{user}/tokens`(Gitea 官方 API),
|
||||
新 token 与 master token 是不同的值、可独立吊销(DELETE 同
|
||||
路径),到期由调用方吊销。返回新 token 值;失败即
|
||||
AuthdRefusal(不含任何 token 值)。
|
||||
前提(见差距表 §3.2):master token 须有建 token 的权限;
|
||||
现有 write:repository 测试 token 铸 token 会失败(403),
|
||||
开这条路须另配有铸 token 权限的 token 并做真机验证。
|
||||
post 可注入(测试用假 post),默认走 urllib。"""
|
||||
if not api_url or not user or not master_token or not name:
|
||||
raise AuthdRefusal(
|
||||
"mint_args_missing", "铸 token 参数不全")
|
||||
base = api_url.rstrip("/")
|
||||
url = "%s/users/%s/tokens" % (base, user)
|
||||
headers = {"Authorization": "token %s" % master_token,
|
||||
"Content-Type": "application/json"}
|
||||
payload = {"name": name, "scopes": list(scopes)}
|
||||
post_fn = post or _default_post
|
||||
status, body = post_fn(url, headers, payload)
|
||||
if status not in (200, 201):
|
||||
raise AuthdRefusal(
|
||||
"mint_failed",
|
||||
"Gitea 铸 token 失败:status=%s" % status)
|
||||
token = (body or {}).get("sha1") or (body or {}).get("token")
|
||||
if not token:
|
||||
raise AuthdRefusal(
|
||||
"mint_empty", "Gitea 铸 token 未返回 token 值")
|
||||
return token
|
||||
|
||||
|
||||
def revoke_gitea_token(api_url: str, user: str, master_token: str,
|
||||
token_id: object,
|
||||
delete: Callable[[str, dict], int] | None = None
|
||||
) -> bool:
|
||||
"""吊销一个铸出的 token(`DELETE {api}/users/{user}/tokens/{id}`)。
|
||||
成功返回 True;404 视为已不在(也返回 True,不报错);
|
||||
其它失败返回 False(调用方审计,不抛错——吊销失败不能炸任务,
|
||||
但须留审计)。delete 可注入,默认走 urllib。"""
|
||||
if not api_url or not user or not master_token \
|
||||
or token_id is None:
|
||||
return False
|
||||
base = api_url.rstrip("/")
|
||||
url = "%s/users/%s/tokens/%s" % (base, user, token_id)
|
||||
headers = {"Authorization": "token %s" % master_token}
|
||||
if delete is None:
|
||||
def _do_delete(url: str, headers: dict) -> int:
|
||||
req = urllib.request.Request(url, headers=headers,
|
||||
method="DELETE")
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=15) as resp:
|
||||
return resp.status
|
||||
except urllib.error.HTTPError as exc:
|
||||
return exc.code
|
||||
delete = _do_delete
|
||||
status = delete(url, headers)
|
||||
return status in (200, 204, 404)
|
||||
|
||||
|
||||
def surrogate_fingerprint(value: str) -> str:
|
||||
"""凭证值的指纹(审计/日志里唯一允许出现的凭证派生形式):
|
||||
SHA-256 前 8 位 hex。指纹不可逆推出原值,审计里只记指纹、
|
||||
不记值。"""
|
||||
return hashlib.sha256(value.encode("utf-8")).hexdigest()[:8]
|
||||
|
||||
|
||||
def hmac_sign(key: str, message: str) -> str:
|
||||
"""HMAC-SHA256 签名(释放记录的防篡改绑定,客内守护进程
|
||||
用它签发 release_id 的校验码;与 Gitea 无关)。"""
|
||||
return hmac.new(key.encode("utf-8"), message.encode("utf-8"),
|
||||
hashlib.sha256).hexdigest()
|
||||
@@ -1,264 +0,0 @@
|
||||
"""P8 browser_tasks:每浏览器任务一个实例——建/绑/毁生命周期。
|
||||
|
||||
对齐 Muse B4(官方安全文):每个浏览器任务由专门浏览器子代理
|
||||
完成。P2 已把浏览器子代理独立成进程(chv-browser.py,常驻、
|
||||
close 只关页不杀进程);本轨改为每任务一个实例:建实例、绑
|
||||
定一个浏览器任务、用完即毁,状态绝不跨任务复用。
|
||||
|
||||
transport 接口:真实 transport 走 P2 的 broker 协议
|
||||
(open/snapshot/screenshot/close/ping,经 /run/chv/browser.sock);
|
||||
客机侧“每任务起一个全新 chv-browser.py、close 后杀进程”
|
||||
是差距表 §3/§6 的跟进项(P2 的 chv-broker.py 里改,本轨不
|
||||
复制 P2 的文件)。本层只管生命周期、单实例单任务与状态隔离,
|
||||
transport 注入(测试用 fake)。
|
||||
|
||||
存储:宿主级 AppStateStore(round-12 已迁移,见
|
||||
docs/chv-muse-p8-gap.md §6):state_dir/app-state.sqlite 的
|
||||
``p8-browser-tasks`` 命名空间,一行存整份登记;旧
|
||||
browser-tasks.json 首次加载时自动导入并改名 ``.migrated``;
|
||||
行损坏/库打不开时回空结构(fail-safe,orphans 同款口径),
|
||||
下次成功保存即覆盖自愈。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from cloud.chv.app_state import AppStateStore, host_state_db_path
|
||||
from cloud.chv.compat215 import resolve_inside
|
||||
|
||||
_LOGGER = logging.getLogger("cloud.cloud_hypervisor")
|
||||
|
||||
_REGISTRY_FILENAME = "browser-tasks.json"
|
||||
_KV_NAMESPACE = "p8-browser-tasks"
|
||||
_KV_KEY = "registry"
|
||||
|
||||
|
||||
class BrowserTransport:
|
||||
"""浏览器实例 transport 接口:真实实现走 P2 的 broker 协议。
|
||||
|
||||
spawn 建实例、close 毁实例、ping 探活。子类须实现三者;
|
||||
基类方法一律 NotImplementedError(绝不静默假装成功)。"""
|
||||
|
||||
def spawn(self, instance: dict) -> None:
|
||||
"""建一个浏览器实例(instance 含 instance_id/task_id)。"""
|
||||
raise NotImplementedError
|
||||
|
||||
def close(self, instance: dict) -> None:
|
||||
"""毁一个浏览器实例(进程退出、状态清理)。"""
|
||||
raise NotImplementedError
|
||||
|
||||
def ping(self, instance: dict) -> bool:
|
||||
"""实例是否还活着。"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
def _open_store(state_dir: str) -> AppStateStore:
|
||||
"""打开宿主级状态库(每次调用新建连接,用完即关)。"""
|
||||
store = AppStateStore(host_state_db_path(state_dir))
|
||||
store.open()
|
||||
return store
|
||||
|
||||
|
||||
def _registry_path(state_dir: str) -> str:
|
||||
"""旧版实例登记文件路径(只用于一次性迁移导入)。"""
|
||||
return os.path.join(state_dir, _REGISTRY_FILENAME)
|
||||
|
||||
|
||||
def _quarantine(path: str) -> None:
|
||||
"""把损坏的登记文件挪到 <path>.corrupt 旁(尽力而为)。"""
|
||||
try:
|
||||
os.replace(path, path + ".corrupt")
|
||||
except OSError:
|
||||
_LOGGER.warning("chv_browser_tasks_quarantine_failed "
|
||||
"path=%s", path)
|
||||
|
||||
|
||||
def _load_registry(state_dir: str) -> dict:
|
||||
"""读登记:KV 行缺失时走一次性旧文件导入;行损坏/库打不
|
||||
开时回空结构(fail-safe,orphans 同款口径;下次成功保存
|
||||
即覆盖自愈)。"""
|
||||
try:
|
||||
with _open_store(state_dir) as store:
|
||||
raw = store.kv_get(_KV_NAMESPACE, _KV_KEY)
|
||||
except (sqlite3.Error, OSError):
|
||||
_LOGGER.warning("chv_browser_tasks_store_unreadable "
|
||||
"state_dir=%s", state_dir)
|
||||
return {"instances": {}}
|
||||
if raw is None:
|
||||
return _import_legacy(state_dir)
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except ValueError:
|
||||
_LOGGER.warning("chv_browser_tasks_corrupt state_dir=%s",
|
||||
state_dir)
|
||||
return {"instances": {}}
|
||||
if not isinstance(data, dict) \
|
||||
or not isinstance(data.get("instances"), dict):
|
||||
return {"instances": {}}
|
||||
return data
|
||||
|
||||
|
||||
def _import_legacy(state_dir: str) -> dict:
|
||||
"""一次性导入旧 browser-tasks.json:合法即进 KV 并把文件
|
||||
改名 ``.migrated``;无文件回空;损坏则 quarantine 后回空
|
||||
(旧口径)。导入失败时返回读到的数据、文件保留,下次加
|
||||
载重试。"""
|
||||
path = _registry_path(state_dir)
|
||||
try:
|
||||
with open(path, "r") as fh:
|
||||
data = json.load(fh)
|
||||
except FileNotFoundError:
|
||||
return {"instances": {}}
|
||||
except (OSError, ValueError):
|
||||
_quarantine(path)
|
||||
return {"instances": {}}
|
||||
if not isinstance(data, dict) \
|
||||
or not isinstance(data.get("instances"), dict):
|
||||
_quarantine(path)
|
||||
return {"instances": {}}
|
||||
try:
|
||||
with _open_store(state_dir) as store:
|
||||
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
|
||||
except (sqlite3.Error, OSError):
|
||||
_LOGGER.warning("chv_browser_tasks_import_failed "
|
||||
"state_dir=%s", state_dir)
|
||||
return data
|
||||
try:
|
||||
os.replace(path, path + ".migrated")
|
||||
except OSError:
|
||||
pass
|
||||
return data
|
||||
|
||||
|
||||
def _save_registry(state_dir: str, data: dict) -> None:
|
||||
"""落盘登记到 KV 行;写失败只记日志。"""
|
||||
try:
|
||||
with _open_store(state_dir) as store:
|
||||
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
|
||||
except (sqlite3.Error, OSError):
|
||||
_LOGGER.warning("chv_browser_tasks_save_failed "
|
||||
"state_dir=%s", state_dir)
|
||||
|
||||
|
||||
def _get_or_raise(data: dict, task_id: str) -> dict:
|
||||
"""取某任务的实例;不存在即 ValueError。"""
|
||||
instance = data.get("instances", {}).get(task_id)
|
||||
if not isinstance(instance, dict):
|
||||
raise ValueError("browser instance not found")
|
||||
return instance
|
||||
|
||||
|
||||
def create_instance(state_dir: str, task_id: str,
|
||||
transport: BrowserTransport,
|
||||
now: float | None = None) -> dict:
|
||||
"""为一个浏览器任务建一个实例:单任务单实例(该任务已有
|
||||
任何实例记录——active、destroyed 乃至 failed——即
|
||||
ValueError;一个任务一生只有一个实例,毁后不复建,审计
|
||||
轨迹留在登记里);先落盘 creating、再调 transport.spawn、
|
||||
成功后置 active;spawn 抛错时记 failed 并原样抛出
|
||||
(不留半个实例)。返回实例 dict。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
if not task_id or not isinstance(task_id, str):
|
||||
raise ValueError("task_id required")
|
||||
data = _load_registry(state_dir)
|
||||
existing = data["instances"].get(task_id)
|
||||
if isinstance(existing, dict):
|
||||
raise ValueError("browser instance already exists")
|
||||
instance_id = uuid.uuid4().hex
|
||||
instance = {
|
||||
"instance_id": instance_id,
|
||||
"task_id": task_id,
|
||||
"status": "creating",
|
||||
"shots_dir": "browser-tasks/%s/shots" % instance_id,
|
||||
"created_at": now,
|
||||
}
|
||||
data["instances"][task_id] = instance
|
||||
_save_registry(state_dir, data)
|
||||
try:
|
||||
transport.spawn(instance)
|
||||
except Exception:
|
||||
instance["status"] = "failed"
|
||||
_save_registry(state_dir, data)
|
||||
raise
|
||||
instance["status"] = "active"
|
||||
_save_registry(state_dir, data)
|
||||
return dict(instance)
|
||||
|
||||
|
||||
def get_instance(state_dir: str, task_id: str) -> dict | None:
|
||||
"""取某任务的实例;不存在回 None。"""
|
||||
data = _load_registry(state_dir)
|
||||
instance = data.get("instances", {}).get(task_id)
|
||||
if not isinstance(instance, dict):
|
||||
return None
|
||||
return dict(instance)
|
||||
|
||||
|
||||
def list_instances(state_dir: str) -> list[dict]:
|
||||
"""列出全部实例(按创建时间排序)。"""
|
||||
data = _load_registry(state_dir)
|
||||
out = [dict(v) for v in data["instances"].values()
|
||||
if isinstance(v, dict)]
|
||||
out.sort(key=lambda i: (i.get("created_at") or 0,
|
||||
i.get("task_id") or ""))
|
||||
return out
|
||||
|
||||
|
||||
def destroy_instance(state_dir: str, task_id: str,
|
||||
transport: BrowserTransport,
|
||||
now: float | None = None) -> dict:
|
||||
"""毁一个实例:须是 active(不存在/已毁即 ValueError);
|
||||
调 transport.close,close 抛错也记下并照常置 destroyed
|
||||
(清理不能因一次 close 失败卡住),返回实例 dict。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
data = _load_registry(state_dir)
|
||||
instance = _get_or_raise(data, task_id)
|
||||
if instance.get("status") != "active":
|
||||
raise ValueError("browser instance is not active")
|
||||
close_error = ""
|
||||
try:
|
||||
transport.close(instance)
|
||||
except Exception as exc:
|
||||
close_error = str(exc)[:300]
|
||||
instance["status"] = "destroyed"
|
||||
instance["destroyed_at"] = now
|
||||
instance["close_error"] = close_error
|
||||
_save_registry(state_dir, data)
|
||||
return dict(instance)
|
||||
|
||||
|
||||
def ping_instance(state_dir: str, task_id: str,
|
||||
transport: BrowserTransport) -> bool:
|
||||
"""实例是否还活着:非 active 一律 False;active 时问
|
||||
transport。"""
|
||||
data = _load_registry(state_dir)
|
||||
instance = data.get("instances", {}).get(task_id)
|
||||
if not isinstance(instance, dict):
|
||||
return False
|
||||
if instance.get("status") != "active":
|
||||
return False
|
||||
try:
|
||||
return bool(transport.ping(instance))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def instance_shots_dir(state_dir: str, task_id: str) -> str | None:
|
||||
"""取某任务实例的截图目录(state_dir 内、resolve_inside
|
||||
校验,越界回 None);实例不存在回 None。"""
|
||||
data = _load_registry(state_dir)
|
||||
instance = data.get("instances", {}).get(task_id)
|
||||
if not isinstance(instance, dict):
|
||||
return None
|
||||
rel = instance.get("shots_dir")
|
||||
if not isinstance(rel, str) or not rel:
|
||||
return None
|
||||
return resolve_inside(state_dir, rel)
|
||||
@@ -1,276 +0,0 @@
|
||||
"""五档严格能力(feat-chv-muse-p5-B,叶子模块,纯标准库)。
|
||||
|
||||
建在 P3 的 ``Capability`` 之上(继承,不重写绑定语义):P3 只给
|
||||
单次 + 300 秒一档;本模块加五档与用例维度,签发/使用/过期/
|
||||
超次/撤销全审计(事件名 + 能力 id + 原因,绝无凭证值与用户
|
||||
数据内容)。
|
||||
|
||||
档位(``TIERS``)与签发默认值(``CapabilityIssuer.issue``):
|
||||
- one-shot:单次、300 秒(P3 默认);
|
||||
- session:会话内多次、无固定次数,会话 id 绑定,默认 3600 秒;
|
||||
- task:任务内多次、无固定次数,任务 id 绑定,默认 3600 秒;
|
||||
- timed:时限内多次、无固定次数,须显式 ttl;
|
||||
- permanent:永久、无过期无次数上限——默认不签发,须显式
|
||||
``allow_permanent=True`` + 审计,否则 ``ValueError``。
|
||||
|
||||
能力绑定 = {连接器,主机,用例} 三元组 + 方法/路径规则 +
|
||||
次数/过期。``matches(request, now, scope)`` 精确匹配(``scope``
|
||||
须按关键字传,``now`` 仍占第二位置参数,与 P3 旧口径一致):
|
||||
用例、方法、路径、会话/任务绑定、有效期、剩余次数任一项越界
|
||||
即 False(并记审计)。``scope`` 是 dict 或带属性的对象,含
|
||||
``use_case`` / ``session_id`` / ``task_id``;没给 scope(None)
|
||||
按空 scope 计,即用例/会话/任务绑定一律不匹配(fail-closed)。
|
||||
"""
|
||||
|
||||
import time
|
||||
from collections.abc import Callable, Iterable
|
||||
|
||||
from cloud.chv.sentinel import (
|
||||
DEFAULT_CAPABILITY_TTL_SECONDS,
|
||||
Capability,
|
||||
EgressRequest,
|
||||
_path_allowed,
|
||||
)
|
||||
|
||||
TIERS = ("one-shot", "session", "task", "timed", "permanent")
|
||||
DEFAULT_SESSION_TTL_SECONDS = 3600
|
||||
DEFAULT_TASK_TTL_SECONDS = 3600
|
||||
|
||||
_EVENT_ISSUED = "sentinel_capability_issued"
|
||||
_EVENT_USED = "sentinel_capability_used"
|
||||
_EVENT_DENIED = "sentinel_capability_denied"
|
||||
_EVENT_EXPIRED = "sentinel_capability_expired"
|
||||
_EVENT_REVOKED = "sentinel_capability_revoked"
|
||||
|
||||
|
||||
def _scope_get(scope: object | None, key: str) -> object | None:
|
||||
"""从 scope(dict 或带属性的对象)取一键,缺失即 None。"""
|
||||
if scope is None:
|
||||
return None
|
||||
if isinstance(scope, dict):
|
||||
return scope.get(key)
|
||||
return getattr(scope, key, None)
|
||||
|
||||
|
||||
def _audit_record(cap: "ScopedCapability", event: str,
|
||||
reason: str) -> dict:
|
||||
"""审计记录:事件名 + 能力 id + 原因,无值。"""
|
||||
return {"event": event, "capability_id": cap.id,
|
||||
"tier": cap.tier, "reason": reason,
|
||||
"connector": cap.connector, "host": cap.host,
|
||||
"use_case": cap.use_case}
|
||||
|
||||
|
||||
class ScopedCapability(Capability):
|
||||
"""五档严格能力:P3 绑定 + 用例三元组 + 会话/任务绑定。
|
||||
|
||||
``max_uses=None`` 表示无固定次数(session/task/timed/
|
||||
permanent 的默认);``expires_at=None`` 表示永不过期
|
||||
(仅 permanent)。``revoked`` 后 ``matches`` 恒 False。
|
||||
``audit_log`` 与签发方共享:签发/使用/过期/超次/撤销
|
||||
全记进同一份审计(事件名 + 能力 id + 原因,无值)。
|
||||
"""
|
||||
|
||||
def __init__(self, connector: str, host: str,
|
||||
methods: Iterable[str],
|
||||
path_rules: Iterable[tuple[str, str, str]],
|
||||
tier: str, use_case: str,
|
||||
session_id: str | None = None,
|
||||
task_id: str | None = None,
|
||||
ttl_seconds: int | None = None,
|
||||
max_uses: int | None = None,
|
||||
now: float | None = None,
|
||||
audit: list | None = None) -> None:
|
||||
super().__init__(
|
||||
connector, host, methods, path_rules,
|
||||
ttl_seconds=(ttl_seconds if ttl_seconds is not None
|
||||
else 0),
|
||||
max_uses=(max_uses if max_uses is not None else 1),
|
||||
now=now)
|
||||
self.tier = tier
|
||||
self.use_case = use_case
|
||||
self.session_id = session_id
|
||||
self.task_id = task_id
|
||||
self.revoked = False
|
||||
self.max_uses = max_uses
|
||||
self.uses_left = max_uses
|
||||
if ttl_seconds is None:
|
||||
self.expires_at = None
|
||||
self.audit_log = audit if audit is not None else []
|
||||
|
||||
def _mismatch_reason(self, request: EgressRequest,
|
||||
scope: object | None,
|
||||
now: float | None) -> str | None:
|
||||
"""越界原因(None 表示在绑定内)。检查顺序固定:
|
||||
撤销 → 用例 → 会话/任务 → 连接器/主机/方法/路径 →
|
||||
过期 → 次数。"""
|
||||
if self.revoked:
|
||||
return "revoked"
|
||||
if _scope_get(scope, "use_case") != self.use_case:
|
||||
return "use_case_mismatch"
|
||||
if self.tier == "session" and (
|
||||
_scope_get(scope, "session_id") != self.session_id):
|
||||
return "session_mismatch"
|
||||
if self.tier == "task" and (
|
||||
_scope_get(scope, "task_id") != self.task_id):
|
||||
return "task_mismatch"
|
||||
if request.connector != self.connector:
|
||||
return "connector_mismatch"
|
||||
if request.host != self.host:
|
||||
return "host_mismatch"
|
||||
if request.method not in self.methods:
|
||||
return "method_not_allowed"
|
||||
if not _path_allowed(request, self.path_rules):
|
||||
return "path_not_allowed"
|
||||
at = now if now is not None else time.time()
|
||||
if self.expires_at is not None and at >= self.expires_at:
|
||||
return "expired"
|
||||
if self.uses_left is not None and self.uses_left <= 0:
|
||||
return "uses_exhausted"
|
||||
return None
|
||||
|
||||
def _audit(self, event: str, reason: str) -> None:
|
||||
"""记一条审计(事件名 + 能力 id + 原因,无值)。"""
|
||||
self.audit_log.append(_audit_record(self, event, reason))
|
||||
|
||||
def matches(self, request: EgressRequest,
|
||||
now: float | None = None,
|
||||
scope: object | None = None) -> bool:
|
||||
"""该请求是否在能力绑定内(不消耗次数,只判定)。
|
||||
|
||||
``scope`` 须按关键字传(``now`` 仍占第二位置参数,
|
||||
与 P3 旧口径一致)。越界即 False 并记审计:过期记
|
||||
expired,其余记 denied。
|
||||
"""
|
||||
reason = self._mismatch_reason(request, scope, now)
|
||||
if reason is None:
|
||||
return True
|
||||
event = (_EVENT_EXPIRED if reason == "expired"
|
||||
else _EVENT_DENIED)
|
||||
self._audit(event, reason)
|
||||
return False
|
||||
|
||||
def use(self, request: EgressRequest,
|
||||
now: float | None = None,
|
||||
scope: object | None = None) -> bool:
|
||||
"""消耗一次能力:绑定内才扣次数并返回 True,否则 False
|
||||
(失败原因已由 ``matches`` 记审计,不再重复记)。"""
|
||||
if not self.matches(request, now=now, scope=scope):
|
||||
return False
|
||||
if self.uses_left is not None:
|
||||
self.uses_left -= 1
|
||||
self._audit(_EVENT_USED, "used")
|
||||
return True
|
||||
|
||||
def revoke(self) -> bool:
|
||||
"""撤销:首次返回 True 并记审计;已撤销再调返回 False。
|
||||
撤销后 ``matches`` 恒 False(原因 revoked)。"""
|
||||
if self.revoked:
|
||||
self._audit(_EVENT_DENIED, "already_revoked")
|
||||
return False
|
||||
self.revoked = True
|
||||
self._audit(_EVENT_REVOKED, "revoked")
|
||||
return True
|
||||
|
||||
def describe(self) -> dict:
|
||||
"""审计/展示用绑定描述(只含绑定,不含值)。"""
|
||||
return {"capability_id": self.id, "tier": self.tier,
|
||||
"connector": self.connector, "host": self.host,
|
||||
"use_case": self.use_case,
|
||||
"session_id": self.session_id,
|
||||
"task_id": self.task_id,
|
||||
"methods": sorted(self.methods),
|
||||
"max_uses": self.max_uses,
|
||||
"uses_left": self.uses_left,
|
||||
"expires_at": self.expires_at,
|
||||
"revoked": self.revoked}
|
||||
|
||||
|
||||
class CapabilityIssuer:
|
||||
"""五档能力签发入口:按档位设默认值,审计与签出的能力共享。
|
||||
|
||||
permanent 默认拒绝:须调用时显式 ``allow_permanent=True``,
|
||||
拒绝本身也记审计(denied,原因
|
||||
``permanent_requires_explicit_opt_in``)。
|
||||
"""
|
||||
|
||||
def __init__(self, audit: list | None = None,
|
||||
clock: Callable[[], float] | None = None) -> None:
|
||||
self.audit_log = audit if audit is not None else []
|
||||
self._clock = clock or time.time
|
||||
|
||||
def issue(self, tier: str, connector: str, host: str,
|
||||
methods: Iterable[str],
|
||||
path_rules: Iterable[tuple[str, str, str]],
|
||||
use_case: str, session_id: str | None = None,
|
||||
task_id: str | None = None,
|
||||
ttl_seconds: int | None = None,
|
||||
max_uses: int | None = None,
|
||||
now: float | None = None,
|
||||
allow_permanent: bool = False) -> ScopedCapability:
|
||||
"""签发一档能力(按档位设默认值,见模块 docstring)。
|
||||
|
||||
未知档位、空用例、session 缺会话 id、task 缺任务 id、
|
||||
timed 缺 ttl、permanent 未显式开启一律 ValueError;
|
||||
校验失败先记审计(无能力 id)再抛错。
|
||||
"""
|
||||
at = now if now is not None else self._clock()
|
||||
if tier not in TIERS:
|
||||
raise ValueError("unknown_tier:%s" % tier)
|
||||
if not use_case:
|
||||
raise ValueError("use_case_required")
|
||||
try:
|
||||
ttl, uses = self._defaults(
|
||||
tier, session_id, task_id, ttl_seconds, max_uses,
|
||||
allow_permanent)
|
||||
except ValueError as exc:
|
||||
self.audit_log.append(
|
||||
{"event": _EVENT_DENIED, "capability_id": None,
|
||||
"tier": tier, "reason": str(exc),
|
||||
"connector": connector, "host": host,
|
||||
"use_case": use_case})
|
||||
raise
|
||||
cap = ScopedCapability(
|
||||
connector, host, methods, path_rules, tier, use_case,
|
||||
session_id=session_id, task_id=task_id,
|
||||
ttl_seconds=ttl, max_uses=uses, now=at,
|
||||
audit=self.audit_log)
|
||||
cap._audit(_EVENT_ISSUED, "issued")
|
||||
return cap
|
||||
|
||||
def _defaults(self, tier: str, session_id: str | None,
|
||||
task_id: str | None, ttl_seconds: int | None,
|
||||
max_uses: int | None,
|
||||
allow_permanent: bool
|
||||
) -> tuple[int | None, int | None]:
|
||||
"""各档默认值(非法组合抛 ValueError,原因即消息)。
|
||||
|
||||
one-shot:1 次/300 秒;session/task:无固定次数,默认
|
||||
3600 秒;timed:须显式 ttl;permanent:无过期无次数,
|
||||
忽略传入的 ttl/max_uses。"""
|
||||
if tier == "one-shot":
|
||||
return (ttl_seconds if ttl_seconds is not None
|
||||
else DEFAULT_CAPABILITY_TTL_SECONDS,
|
||||
max_uses if max_uses is not None else 1)
|
||||
if tier == "session":
|
||||
if not session_id:
|
||||
raise ValueError("session_id_required")
|
||||
return (ttl_seconds if ttl_seconds is not None
|
||||
else DEFAULT_SESSION_TTL_SECONDS, max_uses)
|
||||
if tier == "task":
|
||||
if not task_id:
|
||||
raise ValueError("task_id_required")
|
||||
return (ttl_seconds if ttl_seconds is not None
|
||||
else DEFAULT_TASK_TTL_SECONDS, max_uses)
|
||||
if tier == "timed":
|
||||
if ttl_seconds is None:
|
||||
raise ValueError("ttl_required")
|
||||
return (ttl_seconds, max_uses)
|
||||
if not allow_permanent:
|
||||
raise ValueError("permanent_requires_explicit_opt_in")
|
||||
return (None, None)
|
||||
|
||||
|
||||
__all__ = ["TIERS", "DEFAULT_SESSION_TTL_SECONDS",
|
||||
"DEFAULT_TASK_TTL_SECONDS", "ScopedCapability",
|
||||
"CapabilityIssuer"]
|
||||
@@ -1,143 +0,0 @@
|
||||
"""结账页检测(feat-chv-muse-p10,叶子模块,纯标准库)。
|
||||
|
||||
对齐 Muse 官方安全文(security.muse.ai 2026-09-08,见
|
||||
docs/chv-muse-p10-gap.md):已保存支付凭证的网站,浏览器内置
|
||||
结账页检测,每次都弹出人工审批并展示确切购买详情。
|
||||
|
||||
本模块是纯函数,不启动浏览器、不联网、不读原始 DOM:
|
||||
- 输入只有 URL + 无障碍树节点(与 P4 的 ``detect_form_risk``
|
||||
同输入形状——P2 的浏览器子代理只返回无障碍树快照,节点
|
||||
只有 role/name/label/type 等描述字段);
|
||||
- 输出 ``CheckoutVerdict``:是否结账页、置信度、命中原因、
|
||||
表单风险、页面金额文本;
|
||||
- 确定性规则,不冒充模型分类器(见 P6 口径)。
|
||||
|
||||
诚实边界:启发式检测,不是 100% 检出——商户自定义结账页、
|
||||
单页应用内嵌结账可能漏检。漏检时的安全底线在调用方
|
||||
(``PaymentGate`` 拒绝任何未经审批的扣款),不在本模块
|
||||
宣称“所有结账页都会被拦下”。
|
||||
|
||||
页面金额文本(``amount_text``)只作展示信号:扣款金额永远
|
||||
以 ``PurchaseDetails``(订单数据)为准,页面文本是不受信任
|
||||
输入(P6 口径),不得直接拿去扣款。
|
||||
"""
|
||||
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from urllib.parse import urlparse
|
||||
|
||||
# URL 路径段信号:结账步骤(高)与购物车(中,结账前一步)。
|
||||
_CHECKOUT_SEGMENTS = ("checkout", "check-out", "payment", "pay",
|
||||
"billing", "order-review", "place-order",
|
||||
"confirm-order")
|
||||
_CART_SEGMENTS = ("cart", "basket", "bag")
|
||||
|
||||
# 无障碍树节点文本信号(小写子串匹配)。
|
||||
_PAYMENT_HINTS = ("card number", "credit card", "debit card",
|
||||
"cardholder", "expiry", "expiration", "exp date",
|
||||
"cvc", "cvv", "security code", "卡号", "信用卡",
|
||||
"有效期", "安全码")
|
||||
_ORDER_HINTS = ("place order", "pay now", "confirm purchase",
|
||||
"complete purchase", "checkout", "下单", "立即支付",
|
||||
"确认购买", "去支付", "提交订单")
|
||||
_PASSWORD_HINTS = ("password", "密码")
|
||||
|
||||
_AMOUNT_RE = re.compile(
|
||||
r"[$€£¥]\s?\d[\d,]*(?:\.\d{1,2})?"
|
||||
r"|\b\d[\d,]*(?:\.\d{1,2})?\s?(?:USD|EUR|GBP|JPY|CNY)\b")
|
||||
|
||||
|
||||
@dataclass
|
||||
class CheckoutVerdict:
|
||||
"""一次结账检测的结果(只含描述,不含任何凭证值)。"""
|
||||
|
||||
is_checkout: bool
|
||||
confidence: str # high / medium / low
|
||||
reasons: list = field(default_factory=list)
|
||||
form_risk: str | None = None # payment / password / None
|
||||
amount_text: str | None = None
|
||||
|
||||
|
||||
def _node_text(node: dict) -> str:
|
||||
"""一个无障碍树节点的描述文本(role/name/label/type
|
||||
拼起来,小写;原始 DOM 从不到这里)。"""
|
||||
parts = [str(node.get(k, "")) for k in
|
||||
("role", "name", "label", "type")]
|
||||
return " ".join(parts).lower()
|
||||
|
||||
|
||||
def _url_level(url: str) -> str:
|
||||
"""URL 信号档位:high(结账路径)/ medium(购物车)/
|
||||
none。只看 path,不看 query(query 里可能有订单号等
|
||||
敏感值,不进原因)。"""
|
||||
try:
|
||||
path = urlparse(url).path.lower()
|
||||
except ValueError:
|
||||
return "none"
|
||||
segments = [s for s in path.split("/") if s]
|
||||
if any(s in _CHECKOUT_SEGMENTS for s in segments):
|
||||
return "high"
|
||||
if any(s in _CART_SEGMENTS for s in segments):
|
||||
return "medium"
|
||||
return "none"
|
||||
|
||||
|
||||
def _node_signals(nodes: list) -> tuple:
|
||||
"""无障碍树信号:(支付字段, 密码字段, 下单按钮, 金额文本)。
|
||||
节点文本只作特征匹配,不存原文。"""
|
||||
payment = False
|
||||
password = False
|
||||
order = False
|
||||
amount: str | None = None
|
||||
for node in nodes or []:
|
||||
text = _node_text(node)
|
||||
if not text.strip():
|
||||
continue
|
||||
if any(h in text for h in _PAYMENT_HINTS):
|
||||
payment = True
|
||||
if any(h in text for h in _PASSWORD_HINTS):
|
||||
password = True
|
||||
if any(h in text for h in _ORDER_HINTS):
|
||||
order = True
|
||||
if amount is None:
|
||||
hit = _AMOUNT_RE.search(text)
|
||||
if hit:
|
||||
amount = hit.group(0)
|
||||
return payment, password, order, amount
|
||||
|
||||
|
||||
def detect_checkout(url: str, nodes: list | None = None
|
||||
) -> CheckoutVerdict:
|
||||
"""检测一个页面是否为结账页。
|
||||
|
||||
判定(确定性规则):支付字段在页 → 结账页(高置信);
|
||||
结账路径 URL → 结账页(有下单按钮/金额则高,否则中);
|
||||
下单按钮 + 金额文本 → 结账页(中置信);只命中购物车
|
||||
路径 → 不是结账页(购物车是结账前一步);其余 → 不是。
|
||||
``form_risk``:支付字段在页为 ``payment``,否则密码字
|
||||
段在页为 ``password``,与 P4 的 ``detect_form_risk``
|
||||
口径一致。"""
|
||||
level = _url_level(url)
|
||||
has_payment, has_password, has_order, amount = _node_signals(
|
||||
nodes or [])
|
||||
reasons: list = []
|
||||
if level == "high":
|
||||
reasons.append("checkout_url")
|
||||
elif level == "medium":
|
||||
reasons.append("cart_url")
|
||||
if has_payment:
|
||||
reasons.append("payment_fields")
|
||||
if has_order:
|
||||
reasons.append("order_button")
|
||||
if amount is not None:
|
||||
reasons.append("amount_text")
|
||||
risk = "payment" if has_payment else (
|
||||
"password" if has_password else None)
|
||||
if has_payment:
|
||||
return CheckoutVerdict(True, "high", reasons, risk, amount)
|
||||
if level == "high":
|
||||
conf = "high" if has_order or amount else "medium"
|
||||
return CheckoutVerdict(True, conf, reasons, risk, amount)
|
||||
if has_order and amount is not None:
|
||||
return CheckoutVerdict(True, "medium", reasons, risk, amount)
|
||||
return CheckoutVerdict(False, "low", reasons, risk, amount)
|
||||
@@ -1,113 +0,0 @@
|
||||
"""#215(feat-arch-converge)接口对齐层,CHV 本地实现。
|
||||
|
||||
背景:#215 未合并,本基线里没有 cloud/timeouts.py、
|
||||
cloud/gates.py、cloud/quota.py、cloud/sandbox_policy.py。
|
||||
本模块绝不复制 #215 的文件,只把 CHV 真正能映射的
|
||||
接口写成与 #215 相同的签名,#215 合并后替换为 import
|
||||
即是机械的(见 docs/chv-architecture.md ADR-8)。
|
||||
|
||||
映射表(逐个核对过 #215 远程分支的真实函数,不是凭空编的):
|
||||
- gates.resolve_inside(base, rel):目录门,CHV 快照/磁盘
|
||||
路径白名单正好需要它——本模块实现同语义并已在
|
||||
hardening.validate_snapshot_url 里使用,#215 合并后
|
||||
改为 from cloud.gates import resolve_inside 即可。
|
||||
- sandbox_policy.tmpfs_isolated(executor_kind, path):CHV
|
||||
客机 /tmp 落在客机磁盘上,不是宿主 tmpfs,故对
|
||||
cloud-hypervisor 一律返回 (False, ...)——fail-closed,
|
||||
绝不宣称隔离。未知执行器同样 fail-closed。
|
||||
- timeouts.resolve_module_timeout / quota.decide:无映射。
|
||||
前者是审批/队列超时(None=不过期、0=不过期),CHV
|
||||
的 api/ready 超时是秒数且 0 无效,语义不同,硬套会
|
||||
改行为;后者是按月美元配额,CHV 的配额是 VM 数/磁盘
|
||||
GB/快照数,单位不同。两者在本模块里 deliberately
|
||||
不提供,测试钉住“CHV 不 import 它们”。
|
||||
|
||||
本模块只依赖标准库,是叶子模块,不 import cloud.sandbox。
|
||||
|
||||
合并后替换清单(REPLACEMENT_POINTS,本轨从“接口形状”
|
||||
推进到可机械执行的清单;#215 未合并前一律不 import
|
||||
cloud.gates / cloud.sandbox_policy / cloud.timeouts /
|
||||
cloud.quota,测试钉住这些模块在本基线里不存在):
|
||||
- R1 gates.resolve_inside:唯一生产调用方是
|
||||
hardening._is_within(快照 URL 白名单)。合并后两步:
|
||||
① hardening 改为 `from cloud.gates import resolve_inside`;
|
||||
② 删本模块 resolve_inside。语义必须逐字一致,改完
|
||||
重跑快照白名单测试(逃逸/符号链接/绝对路径三例)。
|
||||
- R2 sandbox_policy.tmpfs_isolated:形状占位,当前无
|
||||
生产调用方(只有测试)。合并后一步:删本模块
|
||||
tmpfs_isolated;将来有调用方时直接 import #215 版,
|
||||
不得再本地复刻。CHV 语义是“一律不隔离(fail-closed)”,
|
||||
若 #215 版对 cloud-hypervisor 返回 True,替换即改行为,
|
||||
必须先停下来核对,不得机械替换。
|
||||
- R3 timeouts / quota:永不替换。#215 的 timeouts 是
|
||||
审批/队列超时(None/0 = 不过期),quota 是按月美元
|
||||
配额;CHV 的 api/ready 超时是秒数(0 无效)、配额是
|
||||
VM 数/磁盘 GB/快照数,单位与语义都不同。清单里明写
|
||||
“不替换”,防止合并时有人按名字硬套。
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
# 合并后替换清单:每项的 local/replacement/test 字段由测试
|
||||
# 逐项核对(local 函数真实存在、#215 模块当前不存在、
|
||||
# test 命名的测试类真实存在),清单本身不得漂移。
|
||||
REPLACEMENT_POINTS = (
|
||||
{
|
||||
"id": "R1",
|
||||
"upstream": "cloud.gates.resolve_inside",
|
||||
"local": "cloud.chv.compat215.resolve_inside",
|
||||
"caller": "cloud.chv.hardening._is_within",
|
||||
"replacement": "hardening 改为 from cloud.gates import "
|
||||
"resolve_inside,并删除本地 resolve_inside",
|
||||
"test": "Compat215Test + 快照白名单测试",
|
||||
},
|
||||
{
|
||||
"id": "R2",
|
||||
"upstream": "cloud.sandbox_policy.tmpfs_isolated",
|
||||
"local": "cloud.chv.compat215.tmpfs_isolated",
|
||||
"caller": None,
|
||||
"replacement": "删除本地 tmpfs_isolated;有调用方时"
|
||||
"直接 import #215 版(先核对 CHV 一律 False 语义)",
|
||||
"test": "Compat215Test.test_tmpfs_isolated_chv_and_unknown_"
|
||||
"fail_closed",
|
||||
},
|
||||
{
|
||||
"id": "R3",
|
||||
"upstream": "cloud.timeouts / cloud.quota",
|
||||
"local": None,
|
||||
"caller": None,
|
||||
"replacement": None,
|
||||
"test": "Compat215Test.test_no_timeout_or_quota_map",
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def resolve_inside(base: str, rel: str) -> str | None:
|
||||
"""目录门:base 内的相对路径安全解析,越界回 None。
|
||||
|
||||
空串回 base 本身;绝对路径、.. 逃逸、符号链接逃逸、
|
||||
commonpath 跨盘符 ValueError 一律 None。签名与 #215
|
||||
的 gates.resolve_inside 一致,合并后可直接替换为 import。"""
|
||||
if not rel:
|
||||
return base
|
||||
if os.path.isabs(rel):
|
||||
return None
|
||||
real_base = os.path.realpath(base)
|
||||
target = os.path.realpath(os.path.join(real_base, rel))
|
||||
try:
|
||||
if os.path.commonpath([real_base, target]) != real_base:
|
||||
return None
|
||||
except ValueError:
|
||||
return None
|
||||
return target
|
||||
|
||||
|
||||
def tmpfs_isolated(executor_kind: str, path: str) -> tuple[bool, str]:
|
||||
"""某执行器下某路径是否走 tmpfs 隔离:(isolated, reason)。
|
||||
|
||||
cloud-hypervisor:客机 /tmp 落在客机磁盘上,不走宿主
|
||||
tmpfs,一律 (False, ...);未知执行器 fail-closed 同样
|
||||
False。签名与 #215 的 sandbox_policy.tmpfs_isolated 一致。"""
|
||||
if executor_kind == "cloud-hypervisor":
|
||||
return False, "CHV 客机 /tmp 落在客机磁盘上,不走宿主 tmpfs"
|
||||
return False, "未知执行器,按不隔离处理(fail-closed 口径)"
|
||||
@@ -1,376 +0,0 @@
|
||||
"""Cloud Hypervisor 执行器配置解析(叶子模块,不依赖
|
||||
cloud.sandbox,避免循环导入)。
|
||||
|
||||
职责只有两件:内存字符串解析,以及把构造参数 + 环境变量
|
||||
解析成一份显式配置。参数优先于环境变量、环境变量优先于
|
||||
默认值——与 #216 原 __init__ 的 `or` 链完全一致,只收紧了
|
||||
两处原来会静默 fallback 的 nonsense 输入(见下文),收紧点
|
||||
在 PR 描述里明写、单测钉住:
|
||||
|
||||
- CHV_SSH_USER 显式传空字符串:原来 `or` 链会静默 fallback
|
||||
成 root(调用方以为没给用户、实际跑了 root),现在直接
|
||||
ValueError。
|
||||
- CHV_SSH_PORT 显式传 0、或解析后超出 1-65535:原来 0 会
|
||||
静默 fallback 成 2222、其它越界值会原样进 ssh 命令,现在
|
||||
直接 ValueError。
|
||||
|
||||
与 #215 的关系(只列清单,不复制模块):#215 未合并,
|
||||
本轨不假装复用它。上一轨 ADR-7 的泛化映射已被证伪
|
||||
(#215 的 timeouts 是审批/队列超时、quota 是按月美元
|
||||
配额,与 CHV 的秒数超时、VM 数/磁盘 GB 单位语义不同,
|
||||
硬套会改行为),真实映射只有两处,见
|
||||
cloud/chv/compat215.py 的 REPLACEMENT_POINTS 清单
|
||||
(合并时按清单机械替换,不在本轨硬 import 未合并模块)。
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
|
||||
def _to_int(value: object) -> int:
|
||||
"""动态配置值转 int(env/参数均未定型):bool 拒绝,
|
||||
int/float 按 int(),str 按 strip 后 int(),其它 ValueError。
|
||||
旧行为是 int(value) 抛 TypeError/ValueError 后由调用方
|
||||
转 ValueError;这里直接统一成 ValueError,文案由调用方定。"""
|
||||
if isinstance(value, bool):
|
||||
raise ValueError("布尔值不是整数")
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, float):
|
||||
return int(value)
|
||||
if isinstance(value, str):
|
||||
return int(value.strip())
|
||||
raise ValueError("不是整数:%r" % (value,))
|
||||
|
||||
|
||||
def _to_float(value: object) -> float:
|
||||
"""动态配置值转 float,与 _to_int 同款口径。"""
|
||||
if isinstance(value, bool):
|
||||
raise ValueError("布尔值不是数字")
|
||||
if isinstance(value, (int, float)):
|
||||
return float(value)
|
||||
if isinstance(value, str):
|
||||
return float(value.strip())
|
||||
raise ValueError("不是数字:%r" % (value,))
|
||||
|
||||
|
||||
def _parse_memory_bytes(value: str) -> int:
|
||||
"""内存字符串转字节:2G/512M/1024K/2048(无单位=字节)。
|
||||
非法值 ValueError,绝不静默按默认算。"""
|
||||
m = re.match(r"^\s*(\d+)\s*([KMGkmg]?)\s*[Bb]?\s*$",
|
||||
str(value))
|
||||
if not m:
|
||||
raise ValueError("CHV_MEM 格式非法:%r" % (value,))
|
||||
mult = {"": 1, "K": 1024, "M": 1024 ** 2, "G": 1024 ** 3}
|
||||
return int(m.group(1)) * mult[m.group(2).upper()]
|
||||
|
||||
|
||||
class ChvConfig:
|
||||
"""解析后的 CHV 配置:纯数据对象,字段名与
|
||||
CloudHypervisorExecutor 原 __init__ 的属性名一一对应,
|
||||
执行器 __init__ 直接把字段抄到 self 上,旧测试读属性
|
||||
的方式全部不变。"""
|
||||
|
||||
def __init__(self, **fields: object) -> None:
|
||||
for key, value in fields.items():
|
||||
setattr(self, key, value)
|
||||
|
||||
|
||||
|
||||
def _base_fragment(params: dict, env: dict) -> dict:
|
||||
"""身份/资源基底片段:二进制、内核、根盘、CPU/内存、
|
||||
state_dir、KVM/cpuinfo 路径、ch-remote、模型。无校验,
|
||||
取值口径与旧 build_chv_config 内联 base 逐字一致。"""
|
||||
return {
|
||||
"binary": params.get("binary") or env.get("CHV_BINARY")
|
||||
or "cloud-hypervisor",
|
||||
"kernel": params.get("kernel") or env.get("CHV_KERNEL"),
|
||||
"rootfs": params.get("rootfs") or params.get("image")
|
||||
or env.get("CHV_ROOTFS") or env.get("CHV_IMAGE"),
|
||||
"cpus": str(params.get("cpus") or env.get("CHV_CPU") or "2"),
|
||||
"memory": params.get("memory") or env.get("CHV_MEM") or "2G",
|
||||
"state_dir": params.get("state_dir")
|
||||
or env.get("CHV_STATE_DIR")
|
||||
or os.path.join(os.path.expanduser("~"),
|
||||
".local", "share", "codex-app-server", "chv-vms"),
|
||||
"kvm_path": params.get("kvm_path")
|
||||
if params.get("kvm_path") is not None else "/dev/kvm",
|
||||
"cpuinfo_path": params.get("cpuinfo_path")
|
||||
if params.get("cpuinfo_path") is not None else "/proc/cpuinfo",
|
||||
"ch_remote": params.get("ch_remote") or env.get("CHV_REMOTE")
|
||||
or "ch-remote",
|
||||
"model": params.get("model"),
|
||||
}
|
||||
|
||||
|
||||
def _ssh_fragment(params: dict, env: dict) -> dict:
|
||||
"""SSH/命令面片段:密钥、用户、端口、host key 模式、
|
||||
客内 host.py 路径、内核命令行。错误文案与原内联代码
|
||||
逐字一致,校验时机不变(构造时)。网络字段(net、
|
||||
ssh_host、egress)已移到 _network_fragment,不在本
|
||||
片段,避免“SSH”片段名不副实。"""
|
||||
ssh_key = params.get("ssh_key") or env.get("CHV_SSH_KEY")
|
||||
if params.get("ssh_user") is not None \
|
||||
and params.get("ssh_user") == "":
|
||||
raise ValueError("CHV_SSH_USER 不能为空字符串")
|
||||
ssh_user = params.get("ssh_user") or env.get("CHV_SSH_USER") \
|
||||
or "root"
|
||||
if params.get("ssh_port") is not None:
|
||||
raw_port = params.get("ssh_port")
|
||||
else:
|
||||
raw_port = env.get("CHV_SSH_PORT") or 2222
|
||||
try:
|
||||
ssh_port = _to_int(raw_port)
|
||||
except (TypeError, ValueError):
|
||||
raise ValueError("CHV_SSH_PORT 必须是整数")
|
||||
if not 1 <= ssh_port <= 65535:
|
||||
raise ValueError("CHV_SSH_PORT 必须在 1-65535 之间")
|
||||
host_key_mode = params.get("host_key_mode") \
|
||||
or env.get("CHV_HOST_KEY_MODE") or "provisioned"
|
||||
if host_key_mode not in ("provisioned", "keyscan"):
|
||||
raise ValueError(
|
||||
"CHV_HOST_KEY_MODE 只能是 provisioned 或 keyscan:%s"
|
||||
% host_key_mode)
|
||||
return {
|
||||
"ssh_key": ssh_key, "ssh_user": ssh_user,
|
||||
"ssh_port": ssh_port, "host_key_mode": host_key_mode,
|
||||
"guest_host_py": params.get("guest_host_py") or "/app/host.py",
|
||||
"cmdline": params.get("cmdline") or env.get("CHV_CMDLINE")
|
||||
or "console=hvc0 root=/dev/vda1 rw",
|
||||
}
|
||||
|
||||
|
||||
def _network_fragment(params: dict, env: dict) -> dict:
|
||||
"""网络片段:tap/桥接描述、SSH 目标、端口转发开关、
|
||||
egress 代理与直连开关。无数值校验(校验在 preflight
|
||||
的 net/egress 门里,不在配置解析层)。egress 从旧
|
||||
quota 片段移到这里:它是网络出口策略,不是 VM 配额。"""
|
||||
return {
|
||||
"net": params.get("net") or env.get("CHV_NET"),
|
||||
"net_json": params.get("net_json") or env.get("CHV_NET_JSON"),
|
||||
"ssh_host": params.get("ssh_host") or env.get("CHV_SSH_HOST"),
|
||||
"ssh_forward": env.get("CHV_SSH_FORWARD") == "1",
|
||||
"egress_proxy": env.get("CHV_EGRESS_PROXY"),
|
||||
"allow_direct_egress": env.get("CHV_ALLOW_DIRECT_EGRESS")
|
||||
== "1",
|
||||
# feat-chv-muse-p3:Sentinel 白名单主机(逗号分隔),
|
||||
# 默认只放行配置的 Gitea 主机(执行器 wake 预审时追加)。
|
||||
"sentinel_allow_hosts": tuple(
|
||||
h.strip().lower()
|
||||
for h in (env.get("CHV_SENTINEL_ALLOW_HOSTS") or "")
|
||||
.split(",") if h.strip()),
|
||||
}
|
||||
|
||||
|
||||
def _quota_fragment(env: dict) -> dict:
|
||||
"""配额片段:仅容量数——最大并发 VM 数与预热池大小。
|
||||
磁盘/快照数上限不在这里,它们是磁盘行为,见
|
||||
_disk_fragment。校验文案与旧代码逐字一致。"""
|
||||
try:
|
||||
max_vms = _to_int(env.get("CHV_MAX_VMS") or 4)
|
||||
except ValueError:
|
||||
raise ValueError("CHV_MAX_VMS 必须是整数")
|
||||
try:
|
||||
warm_pool_size = _to_int(env.get("CHV_WARM_POOL_SIZE") or 0)
|
||||
except ValueError:
|
||||
raise ValueError("CHV_WARM_POOL_SIZE 必须是整数")
|
||||
return {"max_vms": max_vms, "warm_pool_size": warm_pool_size}
|
||||
|
||||
|
||||
def _disk_fragment(env: dict) -> dict:
|
||||
"""磁盘片段:磁盘模式/大小、单租户 reuse 实验开关、
|
||||
快照数/总量/保留期/目录/远端开关与快照 URL。校验
|
||||
文案与旧 quota 片段逐字一致(disk_mode 白名单、
|
||||
disk_gb > 0、快照数非负)。"""
|
||||
disk_mode = env.get("CHV_DISK_MODE") or "copy"
|
||||
if disk_mode not in ("copy", "reuse"):
|
||||
raise ValueError(
|
||||
"CHV_DISK_MODE 只能是 copy 或 reuse:%s" % disk_mode)
|
||||
try:
|
||||
disk_gb = _to_float(env.get("CHV_DISK_GB")) \
|
||||
if env.get("CHV_DISK_GB") else 50.0
|
||||
except ValueError:
|
||||
raise ValueError("CHV_DISK_GB 必须是数字")
|
||||
if disk_gb <= 0:
|
||||
raise ValueError("CHV_DISK_GB 必须大于 0")
|
||||
try:
|
||||
snapshot_max_count = _to_int(env.get("CHV_SNAPSHOT_MAX_COUNT")
|
||||
or 10)
|
||||
snapshot_max_total_gb = _to_float(
|
||||
env.get("CHV_SNAPSHOT_MAX_TOTAL_GB") or 20.0)
|
||||
snapshot_retention_days = _to_float(
|
||||
env.get("CHV_SNAPSHOT_RETENTION_DAYS") or 30.0)
|
||||
except ValueError:
|
||||
raise ValueError("CHV_SNAPSHOT_* 必须是数字")
|
||||
if snapshot_max_count < 0 or snapshot_max_total_gb < 0 \
|
||||
or snapshot_retention_days < 0:
|
||||
raise ValueError("CHV_SNAPSHOT_* 不能为负数")
|
||||
return {
|
||||
"disk_mode": disk_mode, "disk_gb": disk_gb,
|
||||
"reuse_experiment": env.get(
|
||||
"CHV_REUSE_SINGLE_TENANT_EXPERIMENT") == "1",
|
||||
"single_tenant_project": env.get("CHV_SINGLE_TENANT_PROJECT"),
|
||||
"snapshot_max_count": snapshot_max_count,
|
||||
"snapshot_max_total_gb": snapshot_max_total_gb,
|
||||
"snapshot_retention_days": snapshot_retention_days,
|
||||
"snapshot_dir": env.get("CHV_SNAPSHOT_DIR"),
|
||||
"allow_remote_snapshot": env.get("CHV_ALLOW_REMOTE_SNAPSHOT")
|
||||
== "1",
|
||||
"snapshot_url": env.get("CHV_SNAPSHOT_URL"),
|
||||
}
|
||||
|
||||
|
||||
def _timeout_fragment(params: dict, env: dict) -> dict:
|
||||
"""超时片段:任务超时(None = 不限,旧语义)、VMM API
|
||||
超时、客机就绪超时/间隔。ready_interval <= 0 时回落
|
||||
5 秒(旧语义,非校验失败)。校验文案逐字一致。"""
|
||||
task_timeout_raw = env.get("CHV_TASK_TIMEOUT")
|
||||
task_timeout = _to_float(task_timeout_raw) \
|
||||
if task_timeout_raw else None
|
||||
try:
|
||||
api_timeout = _to_float(
|
||||
params.get("api_timeout")
|
||||
if params.get("api_timeout") is not None
|
||||
else env.get("CHV_API_TIMEOUT") or 30)
|
||||
ready_timeout = _to_float(
|
||||
params.get("ready_timeout")
|
||||
if params.get("ready_timeout") is not None
|
||||
else env.get("CHV_READY_TIMEOUT") or 120)
|
||||
ready_interval = _to_float(
|
||||
params.get("ready_interval")
|
||||
if params.get("ready_interval") is not None
|
||||
else env.get("CHV_READY_INTERVAL") or 5)
|
||||
except ValueError:
|
||||
raise ValueError("CHV_*_TIMEOUT/INTERVAL 必须是数字")
|
||||
if ready_interval <= 0:
|
||||
ready_interval = 5
|
||||
return {
|
||||
"task_timeout": task_timeout, "api_timeout": api_timeout,
|
||||
"ready_timeout": ready_timeout,
|
||||
"ready_interval": ready_interval,
|
||||
}
|
||||
|
||||
|
||||
def _gitea_fragment(env: dict) -> dict:
|
||||
"""Gitea 同步片段:宿主配置环境变量 CHV_GITEA_*(见
|
||||
gitea_sync.CHV_GITEA_ENV_MAP)映射成的 guest-env 键值
|
||||
dict。无配置即空 dict(本次启动不同步);值本身不校验
|
||||
完整性——完整性由 wake_vm 在起 VMM 前 fail-closed
|
||||
(validate_gitea_sync_env),构造时不拦,免得一个配
|
||||
错的宿主连执行器对象都建不出来、排障信息全丢。
|
||||
import 在函数内:本模块是叶子模块,顶层只许 import
|
||||
标准库(见 test_leaf_modules_only_import_stdlib),
|
||||
与执行器的方法内 import 同款惯例。"""
|
||||
from cloud.chv.gitea_sync import config_gitea_sync_env
|
||||
return {"gitea_sync_env": config_gitea_sync_env(env)}
|
||||
|
||||
|
||||
def _to_switch(raw: object, name: str) -> bool | None:
|
||||
"""CHV_* 开关解析(P1 气球/vsock 用):未设 → None(调用方
|
||||
再定默认值);"1" 开、"0" 关;其它值一律 ValueError,绝不
|
||||
静默当关(与 CHV_SSH_FORWARD 的 `== "1"` 旧口径不同,本轨
|
||||
新开关按差距表要求非法值即报错)。"""
|
||||
if raw is None:
|
||||
return None
|
||||
if raw == "1":
|
||||
return True
|
||||
if raw == "0":
|
||||
return False
|
||||
raise ValueError("%s 只能是 0 或 1:%r" % (name, raw))
|
||||
|
||||
|
||||
def _balloon_vsock_fragment(env: dict) -> dict:
|
||||
"""气球/vsock 片段(P1,feat-chv-muse-p1):virtio-balloon
|
||||
+ free-page-reporting 开关、vsock notify 通道开关与客 CID。
|
||||
|
||||
- CHV_BALLOON:默认 1(开)。只认 "1"/"0",其它值
|
||||
ValueError。
|
||||
- CHV_VSOCK:默认 1(开)。只认 "1"/"0",其它值
|
||||
ValueError。
|
||||
- CHV_VSOCK_CID:客机 vsock CID,可选,默认 3。必须是
|
||||
整数且在 3-4294967295 之间:0/1/2 是 vsock 保留 CID
|
||||
(Hypervisor/Loopback/Host,CHV 的 VmConfig 校验会
|
||||
以 VsockSpecialCid 拒绝),-1(随机 CID)CHV 的
|
||||
vm.create 同样 400 拒绝(上游 discussion #7531),
|
||||
所以这里不接受保留值/负值;显式设了非法 CID 即
|
||||
ValueError,哪怕 CHV_VSOCK=0(配错早炸,不留到起 VM
|
||||
才暴露)。CID 由运维指定、不是 CHV 分配的:同一宿主
|
||||
上各 VM 的 vsock socket 文件不同,CID 相同(默认 3)
|
||||
并不冲突。"""
|
||||
balloon = _to_switch(env.get("CHV_BALLOON"), "CHV_BALLOON")
|
||||
if balloon is None:
|
||||
balloon = True
|
||||
vsock = _to_switch(env.get("CHV_VSOCK"), "CHV_VSOCK")
|
||||
if vsock is None:
|
||||
vsock = True
|
||||
raw_cid = env.get("CHV_VSOCK_CID")
|
||||
if raw_cid is None:
|
||||
vsock_cid = 3
|
||||
else:
|
||||
try:
|
||||
vsock_cid = _to_int(raw_cid)
|
||||
except ValueError:
|
||||
raise ValueError("CHV_VSOCK_CID 必须是整数")
|
||||
if not 3 <= vsock_cid <= 0xFFFFFFFF:
|
||||
raise ValueError(
|
||||
"CHV_VSOCK_CID 必须在 3-4294967295 之间:%s"
|
||||
% vsock_cid)
|
||||
return {"balloon": balloon, "vsock": vsock,
|
||||
"vsock_cid": vsock_cid}
|
||||
|
||||
|
||||
def _seed_fragment(params: dict, env: dict) -> dict:
|
||||
"""seed 附加项片段:只存宿主文件路径,不读文件内容
|
||||
(读文件在 wake 时 fail-closed,见执行器 _seed_extras;
|
||||
构造时不读,免得路径配错时连执行器对象都建不出来)。
|
||||
CHV_SEED_USER_DATA_EXTRA_FILE 指向含 `users:` 块的文件,
|
||||
CHV_SEED_NETWORK_CONFIG_FILE 指向 NoCloud network-config
|
||||
文件;皆空即不附加(行为与旧 seed 一致)。"""
|
||||
return {
|
||||
"seed_user_data_extra_file":
|
||||
params.get("seed_user_data_extra_file")
|
||||
or env.get("CHV_SEED_USER_DATA_EXTRA_FILE"),
|
||||
"seed_network_config_file":
|
||||
params.get("seed_network_config_file")
|
||||
or env.get("CHV_SEED_NETWORK_CONFIG_FILE"),
|
||||
}
|
||||
|
||||
|
||||
def _assemble_chv_config(fragments: list) -> "ChvConfig":
|
||||
"""把各片段 dict 合并成 ChvConfig(纯合并,不改值)。
|
||||
|
||||
片段键集必须互不相交——键冲突是编程错误,直接
|
||||
ValueError,绝不让后一片段静默覆盖前一片段的值。"""
|
||||
fields: dict = {}
|
||||
for fragment in fragments:
|
||||
for key, value in fragment.items():
|
||||
if key in fields:
|
||||
raise ValueError(
|
||||
"CHV 配置片段键冲突:%s" % key)
|
||||
fields[key] = value
|
||||
return ChvConfig(**fields)
|
||||
|
||||
|
||||
def build_chv_config(params: dict, env: dict) -> "ChvConfig":
|
||||
"""把显式构造参数与环境变量解析成 ChvConfig。
|
||||
|
||||
片段调用顺序即旧校验顺序(SSH → 配额 → 磁盘 →
|
||||
气球/vsock → 超时),只有两处可观测的顺序变化:
|
||||
warm_pool_size 从旧超时片段移到配额片段,因此
|
||||
“CHV_WARM_POOL_SIZE 非法 + 磁盘非法同时出现”时先报
|
||||
前者;单项非法时报错与旧代码逐字一致。字段集合 =
|
||||
旧代码字段 + P1 气球/vsock 三字段(balloon/vsock/
|
||||
vsock_cid)+ P3 Sentinel 字段(sentinel_allow_hosts,
|
||||
在网络片段内),由测试的字段集断言钉住。"""
|
||||
env = env if env is not None else {}
|
||||
return _assemble_chv_config([
|
||||
_base_fragment(params, env),
|
||||
_ssh_fragment(params, env),
|
||||
_quota_fragment(env),
|
||||
_disk_fragment(env),
|
||||
_balloon_vsock_fragment(env),
|
||||
_network_fragment(params, env),
|
||||
_timeout_fragment(params, env),
|
||||
_gitea_fragment(env),
|
||||
_seed_fragment(params, env),
|
||||
])
|
||||
@@ -1,281 +0,0 @@
|
||||
"""P8 crons:定时任务的启用/停用、失败重试、tick 驱动。
|
||||
|
||||
建在 cloud.automations 之上(不改 host.py run-due:它是 host
|
||||
侧唯一的执行器,见 automations 模块 docstring)。本模块只依赖
|
||||
标准库 + cloud.automations,不 import 其它 cloud 模块。
|
||||
|
||||
语义(数值全部是本项目自定,见 docs/chv-muse-p8-gap.md §4;
|
||||
Meta 未公开 cron 的调度粒度、数量上限与失败重试策略):
|
||||
- 停用(paused):不到期、不抢单、可恢复;恢复时若同项目已有
|
||||
同名 active 定义则恢复失败(调用方据 ok=False 区分)。
|
||||
- 失败重试:执行终态为 failed/interrupted 时记一次失败;
|
||||
失败次数 <= max_retries 时按下次重试时刻 next_retry_at =
|
||||
now + min(backoff × 2^(n-1), 3600) 安排重试;超限后不再
|
||||
重试,间隔时钟照常走(下个间隔仍会正常触发)。
|
||||
skipped_quota 不是失败,不进重试。
|
||||
- 抢单:间隔抢单与重试抢单各守各的门(claim_due_automation
|
||||
改 last_run_at;claim_retry_automation 只清 next_retry_at、
|
||||
不动间隔时钟),同一执行同一时刻只一个执行器拿到。
|
||||
- tick:只列出本轮到期的定义(间隔到期 + 重试到期,单项目
|
||||
单轮上限 5 个),不抢单;执行器对每条分别调
|
||||
run_due_automation / run_retry_automation(抢单在入口内)。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlite3
|
||||
import time
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from cloud.automations import ACTIVE
|
||||
from cloud.automations import AUTOMATION_COLUMNS
|
||||
from cloud.automations import DEFAULT_MAX_RETRIES
|
||||
from cloud.automations import PAUSED
|
||||
from cloud.automations import _audit_row
|
||||
from cloud.automations import _row_dict
|
||||
from cloud.automations import automation_to_dict
|
||||
from cloud.automations import due_automations
|
||||
from cloud.automations import ensure_automations_tables
|
||||
from cloud.automations import get_automation
|
||||
from cloud.automations import row_get
|
||||
from cloud.automations import run_due_automation
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from cloud.db import PgConnection
|
||||
|
||||
# 单项目单轮 tick 最多认领的执行数(本项目自定)。
|
||||
MAX_CONCURRENT_CRON_RUNS_PER_PROJECT = 5
|
||||
# 重试退避上限(秒,本项目自定)。
|
||||
RETRY_BACKOFF_CAP_SECONDS = 3600
|
||||
# 计入失败重试的执行终态(skipped_quota 不是失败,不进重试)。
|
||||
RETRYABLE_STATUSES = ("failed", "interrupted")
|
||||
|
||||
|
||||
def retry_delay_seconds(failures: int, backoff_seconds: int) -> int:
|
||||
"""第 failures 次失败后的重试等待秒数:基数 × 2^(n-1),
|
||||
上限 3600 秒。failures < 1 时按 1 算。"""
|
||||
n = failures if failures >= 1 else 1
|
||||
delay = backoff_seconds * (2 ** (n - 1))
|
||||
return min(delay, RETRY_BACKOFF_CAP_SECONDS)
|
||||
|
||||
|
||||
def _fetch(conn: sqlite3.Connection | PgConnection,
|
||||
automation_id: str) -> dict | None:
|
||||
"""取单条定义的原始行 dict;不存在回 None。"""
|
||||
ensure_automations_tables(conn)
|
||||
row = conn.execute(
|
||||
"SELECT * FROM automations WHERE id = ?",
|
||||
(automation_id,)).fetchone()
|
||||
if row is None:
|
||||
return None
|
||||
return _row_dict(row, AUTOMATION_COLUMNS)
|
||||
|
||||
|
||||
def pause_automation(conn: sqlite3.Connection | PgConnection,
|
||||
automation_id: str,
|
||||
now: float | None = None
|
||||
) -> tuple[bool, dict | None]:
|
||||
"""停用自动化:仅 active 的行能停用。
|
||||
|
||||
返回 (ok, auto):行不存在时 (False, None);已停用/已取消
|
||||
时 (False, 旧 dict),调用方据此区分 404 与 409。成功时
|
||||
commit 并记审计。停用后不到期、不抢单,last_run_at 不动
|
||||
(恢复后按原间隔时钟继续,不立即补跑一轮)。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
row = _fetch(conn, automation_id)
|
||||
if row is None:
|
||||
return False, None
|
||||
if row_get(row, "status") != ACTIVE:
|
||||
return False, get_automation(conn, automation_id)
|
||||
cur = conn.execute(
|
||||
"UPDATE automations SET status = ?"
|
||||
" WHERE id = ? AND status = ?",
|
||||
(PAUSED, automation_id, ACTIVE))
|
||||
conn.commit()
|
||||
if cur.rowcount != 1:
|
||||
return False, get_automation(conn, automation_id)
|
||||
_audit_row(conn, row_get(row, "project_id"),
|
||||
"automation_paused", automation_id, now)
|
||||
return True, get_automation(conn, automation_id)
|
||||
|
||||
|
||||
def resume_automation(conn: sqlite3.Connection | PgConnection,
|
||||
automation_id: str,
|
||||
now: float | None = None
|
||||
) -> tuple[bool, dict | None]:
|
||||
"""恢复已停用的自动化:仅 paused 的行能恢复。
|
||||
|
||||
同项目已有同名 active 定义时恢复失败(False, 旧 dict):
|
||||
名字唯一规则只认 active,停用期间建了同名新定义是允许
|
||||
的,恢复撞名时必须先改名或取消那一个。成功时 commit 并
|
||||
记审计;last_run_at 不动(恢复后若间隔已过,下一轮 tick
|
||||
即到期,不立即触发)。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
row = _fetch(conn, automation_id)
|
||||
if row is None:
|
||||
return False, None
|
||||
if row_get(row, "status") != PAUSED:
|
||||
return False, get_automation(conn, automation_id)
|
||||
clash = conn.execute(
|
||||
"SELECT id FROM automations WHERE project_id = ?"
|
||||
" AND name = ? AND status = ? AND id != ?",
|
||||
(row_get(row, "project_id"), row_get(row, "name"),
|
||||
ACTIVE, automation_id)).fetchone()
|
||||
if clash is not None:
|
||||
return False, get_automation(conn, automation_id)
|
||||
cur = conn.execute(
|
||||
"UPDATE automations SET status = ?"
|
||||
" WHERE id = ? AND status = ?",
|
||||
(ACTIVE, automation_id, PAUSED))
|
||||
conn.commit()
|
||||
if cur.rowcount != 1:
|
||||
return False, get_automation(conn, automation_id)
|
||||
_audit_row(conn, row_get(row, "project_id"),
|
||||
"automation_resumed", automation_id, now)
|
||||
return True, get_automation(conn, automation_id)
|
||||
|
||||
|
||||
def retry_due_automations(conn: sqlite3.Connection | PgConnection,
|
||||
project_id: str,
|
||||
now: float | None = None) -> list[dict]:
|
||||
"""列出某项目当前重试到期的定义(纯读,不抢单):
|
||||
active、next_retry_at 已到、失败次数 > 0 且未超 max_retries
|
||||
(老库 NULL 列按默认值 3 算)。按创建时间排序。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
ensure_automations_tables(conn)
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM automations WHERE project_id = ?"
|
||||
" AND status = ? AND next_retry_at IS NOT NULL"
|
||||
" AND next_retry_at <= ?"
|
||||
" AND COALESCE(consecutive_failures, 0) > 0"
|
||||
" AND COALESCE(consecutive_failures, 0)"
|
||||
" <= COALESCE(max_retries, ?)"
|
||||
" ORDER BY created_at, id",
|
||||
(project_id, ACTIVE, now, DEFAULT_MAX_RETRIES)).fetchall()
|
||||
return [automation_to_dict(r, now) for r in rows]
|
||||
|
||||
|
||||
def claim_retry_automation(conn: sqlite3.Connection | PgConnection,
|
||||
automation_id: str,
|
||||
now: float | None = None) -> bool:
|
||||
"""重试抢单:重试执行的唯一入口(与间隔抢单分开)。
|
||||
|
||||
条件 UPDATE 只有一行能成功;抢到即清 next_retry_at 并写
|
||||
last_claimed_at,**不动 last_run_at**(间隔时钟不受重试
|
||||
影响)。返回 True 表示抢到,False 表示不存在、非 active、
|
||||
未到重试时刻、失败次数已超限或已被别的执行器抢走。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
ensure_automations_tables(conn)
|
||||
cur = conn.execute(
|
||||
"UPDATE automations SET last_claimed_at = ?,"
|
||||
" next_retry_at = NULL"
|
||||
" WHERE id = ? AND status = ?"
|
||||
" AND next_retry_at IS NOT NULL AND next_retry_at <= ?"
|
||||
" AND COALESCE(consecutive_failures, 0) > 0"
|
||||
" AND COALESCE(consecutive_failures, 0)"
|
||||
" <= COALESCE(max_retries, ?)",
|
||||
(now, automation_id, ACTIVE, now, DEFAULT_MAX_RETRIES))
|
||||
conn.commit()
|
||||
return cur.rowcount == 1
|
||||
|
||||
|
||||
def record_automation_outcome(conn: sqlite3.Connection | PgConnection,
|
||||
automation_id: str, status: str,
|
||||
now: float | None = None
|
||||
) -> dict | None:
|
||||
"""记一次自动化执行的终态,更新失败计数与下次重试时刻。
|
||||
|
||||
- failed/interrupted:失败次数 +1;未超 max_retries 时
|
||||
next_retry_at = now + 退避,超限后清 next_retry_at
|
||||
(不再重试,间隔时钟照常走);记审计。
|
||||
- completed:失败次数清零、next_retry_at 清空。
|
||||
- 其它终态(created/skipped_quota 等):不改计数。
|
||||
定义不存在时回 None。成功时 commit 并返回最新 dict。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
auto = get_automation(conn, automation_id)
|
||||
if auto is None:
|
||||
return None
|
||||
if status in RETRYABLE_STATUSES:
|
||||
return _record_failure(conn, auto, now)
|
||||
if status == "completed":
|
||||
return _record_success(conn, auto, now)
|
||||
return auto
|
||||
|
||||
|
||||
def _record_failure(conn: sqlite3.Connection | PgConnection,
|
||||
auto: dict, now: float) -> dict | None:
|
||||
"""失败分支:失败次数 +1,未超限则安排下次重试时刻。"""
|
||||
failures = int(auto["consecutive_failures"]) + 1
|
||||
max_retries = int(auto["max_retries"])
|
||||
if failures <= max_retries:
|
||||
delay = retry_delay_seconds(
|
||||
failures, int(auto["retry_backoff_seconds"]))
|
||||
next_retry = now + delay
|
||||
else:
|
||||
next_retry = None
|
||||
conn.execute(
|
||||
"UPDATE automations SET consecutive_failures = ?,"
|
||||
" next_retry_at = ? WHERE id = ?",
|
||||
(failures, next_retry, auto["id"]))
|
||||
conn.commit()
|
||||
_audit_row(conn, auto["project_id"], "automation_run_failed",
|
||||
"%s: failures=%d next_retry_at=%s"
|
||||
% (auto["id"], failures, next_retry), now)
|
||||
return get_automation(conn, auto["id"])
|
||||
|
||||
|
||||
def _record_success(conn: sqlite3.Connection | PgConnection,
|
||||
auto: dict, now: float) -> dict | None:
|
||||
"""成功分支:失败次数清零、下次重试时刻清空。"""
|
||||
conn.execute(
|
||||
"UPDATE automations SET consecutive_failures = 0,"
|
||||
" next_retry_at = NULL WHERE id = ?",
|
||||
(auto["id"],))
|
||||
conn.commit()
|
||||
return get_automation(conn, auto["id"])
|
||||
|
||||
|
||||
def run_retry_automation(conn: sqlite3.Connection | PgConnection,
|
||||
automation_id: str,
|
||||
now: float | None = None) -> dict | None:
|
||||
"""重试执行的唯一入口:重试抢单 -> 配额门 -> 建任务 -> 记产出。
|
||||
|
||||
与 run_due_automation 共用配额门与建任务逻辑,只把抢单换
|
||||
成重试抢单;抢不到时回 None。调用方(定时执行器)只调本
|
||||
函数与 run_due_automation,绝不自己拼 claim + INSERT。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
return run_due_automation(conn, automation_id, now,
|
||||
claim_fn=claim_retry_automation)
|
||||
|
||||
|
||||
def run_cron_tick(conn: sqlite3.Connection | PgConnection,
|
||||
project_id: str,
|
||||
now: float | None = None) -> dict:
|
||||
"""一轮 tick 列出本项目当前到期的定义(纯读,不抢单):
|
||||
间隔到期在前、重试到期在后,单轮总数上限
|
||||
MAX_CONCURRENT_CRON_RUNS_PER_PROJECT(本项目自定)。
|
||||
|
||||
返回 {"project_id", "due", "retry_due", "capped"};执行器
|
||||
对 due 的每条调 run_due_automation、对 retry_due 的每条调
|
||||
run_retry_automation(抢单在入口内,两个执行器同时 tick
|
||||
也只一个能抢到)。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
due = due_automations(conn, project_id, now)
|
||||
retry = retry_due_automations(conn, project_id, now)
|
||||
cap = MAX_CONCURRENT_CRON_RUNS_PER_PROJECT
|
||||
kept_due = due[:cap]
|
||||
kept_retry = retry[:max(0, cap - len(kept_due))]
|
||||
capped = len(due) + len(retry) > cap
|
||||
return {
|
||||
"project_id": project_id,
|
||||
"due": kept_due,
|
||||
"retry_due": kept_retry,
|
||||
"capped": capped,
|
||||
}
|
||||
@@ -1,256 +0,0 @@
|
||||
"""Cloud Hypervisor 每任务磁盘供给(叶子模块)。
|
||||
|
||||
单任务单 VM 的文件系统独立性就靠这里:copy 模式每任务
|
||||
复制一份基础镜像,两个任务绝不写同一文件;reuse 模式
|
||||
直接返回基础镜像路径(仅单租户实验,生产拦截见执行器
|
||||
preflight,须同时设 CHV_REUSE_SINGLE_TENANT_EXPERIMENT=1
|
||||
并审计放行)。
|
||||
|
||||
feat-chv-fix(P1-5)配额口径:不再只比基础镜像大小。
|
||||
check_disk_quota 改为三项:state_dir 已用总量 + 本次
|
||||
拷贝(基础镜像大小)+ 宿主剩余空间必须容纳本次拷贝;
|
||||
镜像不可读时不再静默跳过,报 disk_quota_unverified
|
||||
(preflight 在 wake 路径上会先报 image_missing,这里
|
||||
是直接调用者的第二道门)。provision_task_disk 对已存在
|
||||
磁盘校验大小:与源不一致即重拷;源不可读即报失败,
|
||||
绝不复用脏磁盘。
|
||||
"""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
|
||||
from cloud.chv.errors import ChvFailure
|
||||
|
||||
|
||||
def _dir_total_bytes(path):
|
||||
total = 0
|
||||
for root, _dirs, files in os.walk(path):
|
||||
for name in files:
|
||||
try:
|
||||
total += os.path.getsize(os.path.join(root, name))
|
||||
except OSError:
|
||||
pass
|
||||
return total
|
||||
|
||||
|
||||
def check_disk_quota(rootfs: str, disk_gb: float, state_dir: str | None = None) -> None:
|
||||
"""磁盘配额门(feat-chv-fix 口径):
|
||||
|
||||
- disk_gb 未设时不检查;
|
||||
- 镜像不可读时报 disk_quota_unverified,不静默跳过;
|
||||
- 镜像大小超过 disk_gb 即 disk_quota_exceeded;
|
||||
- state_dir 已给时:已用总量 + 本次拷贝超过 disk_gb
|
||||
即 disk_quota_exceeded;宿主剩余空间不足以容纳本次
|
||||
拷贝即 disk_quota_exceeded(宿主余量校验)。
|
||||
"""
|
||||
if disk_gb is None or not rootfs:
|
||||
return
|
||||
try:
|
||||
size = os.path.getsize(rootfs)
|
||||
except OSError:
|
||||
raise ChvFailure(
|
||||
"disk_quota_unverified",
|
||||
"磁盘镜像不可读,配额无法校验:%s" % rootfs)
|
||||
if size > disk_gb * 1024 ** 3:
|
||||
raise ChvFailure(
|
||||
"disk_quota_exceeded",
|
||||
"磁盘镜像 %d 字节超过配额 %.1f GB" % (size, disk_gb))
|
||||
if state_dir and os.path.isdir(state_dir):
|
||||
used = _dir_total_bytes(state_dir)
|
||||
if used + size > disk_gb * 1024 ** 3:
|
||||
raise ChvFailure(
|
||||
"disk_quota_exceeded",
|
||||
"磁盘总量 %d 字节 + 本次 %d 字节超过配额 %.1f GB"
|
||||
% (used, size, disk_gb))
|
||||
try:
|
||||
free = shutil.disk_usage(state_dir).free
|
||||
except OSError:
|
||||
free = None
|
||||
if free is not None and free < size:
|
||||
raise ChvFailure(
|
||||
"disk_quota_exceeded",
|
||||
"宿主剩余空间 %d 字节不足以容纳本次磁盘拷贝 %d 字节"
|
||||
% (free, size))
|
||||
|
||||
|
||||
def provision_task_disk(state_dir: str, vm_name: str, rootfs: str, disk_mode: str) -> str:
|
||||
"""每任务磁盘路径:copy 模式复制到
|
||||
state_dir/<vm>/disk.raw;reuse 模式直接返回基础镜像路径。
|
||||
|
||||
feat-chv-fix:已存在磁盘须校验——大小与源不一致即
|
||||
重拷(脏磁盘不复用);源不可读即 ChvFailure,绝不
|
||||
静默返回旧磁盘。"""
|
||||
if disk_mode == "reuse":
|
||||
return rootfs
|
||||
task_dir = os.path.join(state_dir, vm_name)
|
||||
os.makedirs(task_dir, exist_ok=True)
|
||||
task_disk = os.path.join(task_dir, "disk.raw")
|
||||
# feat-chv-secfix(L1):目标是符号链接时先删链接本身,
|
||||
# 绝不跟随链接写入目标文件(copyfile 跟随链接会截断并
|
||||
# 写进链接目标)。删链接与重建之间的 TOCTOU 窗口已在
|
||||
# 文档里明写:state_dir 须 0700 且仅 worker 用户可写,
|
||||
# 宿主本地攻击者本就写不进该目录,L1 修的是“链接已
|
||||
# 预置”情形下的最后一道门。
|
||||
if os.path.islink(task_disk):
|
||||
os.remove(task_disk)
|
||||
if os.path.exists(task_disk):
|
||||
try:
|
||||
same = os.path.getsize(task_disk) == os.path.getsize(rootfs)
|
||||
except OSError:
|
||||
raise ChvFailure(
|
||||
"disk_quota_unverified",
|
||||
"任务磁盘或源镜像不可读,无法校验复用:%s" % task_disk)
|
||||
if same:
|
||||
return task_disk
|
||||
os.remove(task_disk)
|
||||
try:
|
||||
shutil.copyfile(rootfs, task_disk)
|
||||
except OSError as exc:
|
||||
raise ChvFailure(
|
||||
"disk_quota_unverified",
|
||||
"任务磁盘拷贝失败:%s" % exc)
|
||||
return task_disk
|
||||
|
||||
|
||||
def task_disk_path(state_dir: str, vm_name: str) -> str:
|
||||
"""该 VM 的任务磁盘路径:state_dir/<vm>/disk.raw。"""
|
||||
return os.path.join(state_dir, vm_name, "disk.raw")
|
||||
|
||||
|
||||
def delete_task_disk(state_dir: str, vm_name: str) -> bool:
|
||||
"""删任务磁盘(feat-chv-secfix,H3):stop/recycle/
|
||||
清理路径都调它。返回 True 表示确实删掉了一个文件,
|
||||
False 表示本来就没有;删失败抛 OSError(调用方记审计,
|
||||
绝不静默——残留盘由 gc_stale_disks 下次再清)。
|
||||
符号链接只删链接本身,不跟随。"""
|
||||
path = task_disk_path(state_dir, vm_name)
|
||||
if os.path.islink(path) or os.path.exists(path):
|
||||
os.remove(path)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# -- 快照登记与上限(feat-chv-secfix,M3)-------------------------
|
||||
# 登记文件:state_dir/<vm>/snapshots.json,内容为
|
||||
# [{url, path(file:// 才有), size, created_at}]。登记是
|
||||
# 执行器在宿主侧的账本,不是 VMM 的快照元数据;VMM 不认
|
||||
# 它,删登记不等于删快照文件(反之亦然),两者在
|
||||
# prune/record 里成对处理并审计。
|
||||
|
||||
def _snapshots_path(state_dir, vm_name):
|
||||
return os.path.join(state_dir, vm_name, "snapshots.json")
|
||||
|
||||
|
||||
def _load_snapshots(state_dir, vm_name):
|
||||
import json
|
||||
path = _snapshots_path(state_dir, vm_name)
|
||||
try:
|
||||
with open(path, "r") as fh:
|
||||
data = json.load(fh)
|
||||
except (OSError, ValueError):
|
||||
return []
|
||||
return data if isinstance(data, list) else []
|
||||
|
||||
|
||||
def _save_snapshots(state_dir, vm_name, entries):
|
||||
import json
|
||||
from cloud.chv.atomic_file import atomic_write_text
|
||||
path = _snapshots_path(state_dir, vm_name)
|
||||
os.makedirs(os.path.dirname(path), exist_ok=True)
|
||||
# feat-chv-closeup(P2-18):随机 tmp 名 + 双 fsync。
|
||||
atomic_write_text(path, json.dumps(entries))
|
||||
|
||||
|
||||
def prune_snapshots(state_dir: str, vm_name: str,
|
||||
retention_days: float | None,
|
||||
now: float | None = None,
|
||||
delete_files: bool = True) -> tuple[list, list]:
|
||||
"""按保留期剪枝:删过期条目;delete_files 时同时删
|
||||
对应的 file:// 快照文件(只删登记里记的 path,不猜
|
||||
文件名)。返回(保留条目,剪掉条目)。"""
|
||||
import time
|
||||
now = time.time() if now is None else now
|
||||
kept, pruned = [], []
|
||||
for entry in _load_snapshots(state_dir, vm_name):
|
||||
created = entry.get("created_at") or 0
|
||||
if retention_days is not None \
|
||||
and now - created > retention_days * 86400:
|
||||
pruned.append(entry)
|
||||
if delete_files and entry.get("path"):
|
||||
try:
|
||||
if os.path.exists(entry["path"]):
|
||||
os.remove(entry["path"])
|
||||
except OSError:
|
||||
pass
|
||||
else:
|
||||
kept.append(entry)
|
||||
if pruned:
|
||||
_save_snapshots(state_dir, vm_name, kept)
|
||||
return kept, pruned
|
||||
|
||||
|
||||
def check_snapshot_limits(state_dir: str, vm_name: str, max_count: int,
|
||||
max_total_gb: float,
|
||||
retention_days: float | None) -> list:
|
||||
"""快照上限预检(snapshot_vm 调 ch-remote 之前调):
|
||||
先按保留期剪枝,再查数量(已有数 + 本次 > 上限即
|
||||
snapshot_quota_exceeded)与总量(已有总量已达上限即
|
||||
拒绝;本次大小事前未知,总量在 record 时二次收紧,
|
||||
见 record_snapshot)。返回剪枝后条目。"""
|
||||
kept, _ = prune_snapshots(state_dir, vm_name, retention_days)
|
||||
if len(kept) + 1 > max_count:
|
||||
raise ChvFailure(
|
||||
"snapshot_quota_exceeded",
|
||||
"快照数量已达上限 %d(VM %s)" % (max_count, vm_name))
|
||||
total: float = 0.0
|
||||
for entry in kept:
|
||||
size = entry.get("size")
|
||||
if isinstance(size, (int, float)):
|
||||
total += size
|
||||
elif entry.get("path") and os.path.exists(entry["path"]):
|
||||
try:
|
||||
total += os.path.getsize(entry["path"])
|
||||
except OSError:
|
||||
pass
|
||||
if total >= max_total_gb * 1024 ** 3:
|
||||
raise ChvFailure(
|
||||
"snapshot_quota_exceeded",
|
||||
"快照总量 %d 字节已达上限 %.1f GB(VM %s)"
|
||||
% (total, max_total_gb, vm_name))
|
||||
return kept
|
||||
|
||||
|
||||
def record_snapshot(state_dir: str, vm_name: str, snapshot_url: str, max_total_gb: float) -> tuple[list, list]:
|
||||
"""快照成功后登记;file:// 目标记实际大小。登记后若
|
||||
总量超上限,按最旧优先删登记外的旧快照文件(总量硬
|
||||
帽,删动作由调用方审计——本函数返回被删的旧条目)。
|
||||
返回(全部条目,被总量帽删掉的旧条目)。"""
|
||||
import time
|
||||
path = None
|
||||
size = None
|
||||
if snapshot_url.startswith("file://"):
|
||||
path = snapshot_url[len("file://"):]
|
||||
try:
|
||||
size = os.path.getsize(path)
|
||||
except OSError:
|
||||
size = None
|
||||
entries = _load_snapshots(state_dir, vm_name)
|
||||
entries.append({"url": snapshot_url, "path": path,
|
||||
"size": size, "created_at": time.time()})
|
||||
_save_snapshots(state_dir, vm_name, entries)
|
||||
removed = []
|
||||
if max_total_gb is not None:
|
||||
cap = max_total_gb * 1024 ** 3
|
||||
total = sum(e.get("size") or 0 for e in entries)
|
||||
while len(entries) > 1 and total > cap:
|
||||
oldest = entries.pop(0)
|
||||
if oldest.get("path") and os.path.exists(oldest["path"]):
|
||||
try:
|
||||
os.remove(oldest["path"])
|
||||
except OSError:
|
||||
pass
|
||||
total -= oldest.get("size") or 0
|
||||
removed.append(oldest)
|
||||
if removed:
|
||||
_save_snapshots(state_dir, vm_name, entries)
|
||||
return entries, removed
|
||||
@@ -1,325 +0,0 @@
|
||||
"""eBPF 污点传播加载层(对齐 Muse P5-A,纯标准库 + 可选工具链)。
|
||||
|
||||
Muse 口径(见 docs/chv-muse-p5-gap.md §1 污点出口行):用 eBPF
|
||||
cgroup 程序做网络拦截与进程归因、及挂在 LSM hook 上的 eBPF
|
||||
程序做污点传播。
|
||||
|
||||
本模块只含两样东西,绝不冒充已加载:
|
||||
1. eBPF 程序源码(字符串常量,见 `EBPF_CGROUP_EGRESS_C` 与
|
||||
`EBPF_LSM_TAINT_C`):cgroup egress 拦截 + 进程归因 map +
|
||||
LSM hook 污点传播骨架。源码本身不是“已在拦截流量”。
|
||||
2. 加载器 `EbpfTaintLoader`:能力探测(内核版本、bpffs、
|
||||
CAP_BPF/CAP_SYS_ADMIN、bpftool/clang 是否在位)→ 状态
|
||||
loaded/degraded/blocked + 原因;加载失败一律明确降级为
|
||||
“策略引擎 + 代理层钩子”模式(`FALLBACK_MODE`),绝不把
|
||||
degraded 写成 loaded。
|
||||
|
||||
加载成功的硬条件(缺一即 blocked/degraded,不重试蒙混):
|
||||
- 内核 >= 5.8(cgroup_skb egress 挂载点;LSM hook 另需
|
||||
内核开 CONFIG_BPF_LSM,见 LSM 源码头注释);
|
||||
- /sys/fs/bpf 存在且可写(pin 落点);
|
||||
- 进程有效能力集含 CAP_BPF(bit 39)或 CAP_SYS_ADMIN
|
||||
(bit 21)——且 bpf() 系统调用本身不被 seccomp 挡;
|
||||
- bpftool + clang 在位(编译与加载工具链)。
|
||||
|
||||
本机实测结论(2026-11-07,Muse VM):不行。内核
|
||||
7.0.0-43-generic、CapEff 含 CAP_BPF 与 CAP_SYS_ADMIN,但
|
||||
bpf() 与 BPF_MAP_CREATE 均报 EPERM(Seccomp filter 在位,
|
||||
4 条过滤器),且无 /sys/fs/bpf、无 bpftool/clang。能力位
|
||||
在位不等于系统调用放行——探测只看 CapEff 会误判,所以
|
||||
`probe()` 把“缺 bpffs/缺工具链”也记为 blocker,不以
|
||||
CapEff 单项定生死。
|
||||
"""
|
||||
|
||||
import os
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
from collections.abc import Callable
|
||||
|
||||
# 加载失败后的降级模式:策略引擎(taint.py + sentinel.py 决策)
|
||||
# + 代理层钩子(Tinyproxy 白名单等现有强制),不走 eBPF。
|
||||
FALLBACK_MODE = "policy_engine_proxy_hooks"
|
||||
|
||||
# cgroup egress 拦截的最低内核版本。
|
||||
MIN_KERNEL = (5, 8)
|
||||
|
||||
# CAP 位号(/proc/self/status 的 CapEff,按 Linux capability.h)。
|
||||
_CAP_BPF_BIT = 39
|
||||
_CAP_SYS_ADMIN_BIT = 21
|
||||
|
||||
# eBPF cgroup egress 拦截程序源码。
|
||||
#
|
||||
# 作用(按 section):
|
||||
# - `SEC("cgroup_skb/egress")`:挂在 cgroup 的 egress 路径上,
|
||||
# 每个出站 skb 都过这里;返回 0 放行、返回非 0 丢弃。污点
|
||||
# 进程的直连在这里被拦下,而不是等代理层才发现。
|
||||
# - `taint_map`(BPF_MAP_TYPE_HASH,key=pid u32,value=污点
|
||||
# 标志 u8):进程归因 map。用户态追踪器(taint.py)与 LSM
|
||||
# 程序共用这张表:LSM 侧写污点,cgroup 侧读污点。
|
||||
# - `events`(BPF_MAP_TYPE_RINGBUF):拦截/放行事件上报给
|
||||
# 用户态审计,不含包内容,只含 pid 与 verdict。
|
||||
# - 归因键:`bpf_get_current_pid_tgid() >> 32` 取当前进程 pid;
|
||||
# cgroup 归属由挂载时的 cgroup 决定,不靠包里的地址猜进程。
|
||||
# 内核要求:>= 5.8,CAP_BPF(或 CAP_SYS_ADMIN),cgroup v2,
|
||||
# bpffs 可写(pin 落点),clang -target bpf + bpftool 加载。
|
||||
EBPF_CGROUP_EGRESS_C = r"""
|
||||
#include <linux/bpf.h>
|
||||
#include <bpf/bpf_helpers.h>
|
||||
|
||||
/* 进程归因 map:pid -> 污点标志(1=污点,0=干净)。LSM 程序写,
|
||||
本程序读;用户态也可经 map fd 直接查。 */
|
||||
struct {
|
||||
__uint(type, BPF_MAP_TYPE_HASH);
|
||||
__uint(max_entries, 65536);
|
||||
__type(key, __u32);
|
||||
__type(value, __u8);
|
||||
} taint_map SEC(".maps");
|
||||
|
||||
/* 事件上报:只含 pid 与 verdict,不含包内容。 */
|
||||
struct {
|
||||
__uint(type, BPF_MAP_TYPE_RINGBUF);
|
||||
__uint(max_entries, 1 << 16);
|
||||
} events SEC(".maps");
|
||||
|
||||
struct event {
|
||||
__u32 pid;
|
||||
__u32 verdict; /* 0=放行,1=丢弃 */
|
||||
};
|
||||
|
||||
/* cgroup egress 拦截:污点进程的出站包一律丢弃并上报,干净
|
||||
进程放行(放行不等于“自动放行通过”——窄范围自动放行仍由
|
||||
Sentinel 策略引擎在 L7 判定,这里只做强制层拦截)。 */
|
||||
SEC("cgroup_skb/egress")
|
||||
int egress_filter(struct __sk_buff *skb)
|
||||
{
|
||||
__u32 pid = (__u32)(bpf_get_current_pid_tgid() >> 32);
|
||||
__u8 *tainted = bpf_map_lookup_elem(&taint_map, &pid);
|
||||
struct event *e;
|
||||
if (tainted && *tainted) {
|
||||
e = bpf_ringbuf_reserve(&events, sizeof(*e), 0);
|
||||
if (e) {
|
||||
e->pid = pid;
|
||||
e->verdict = 1;
|
||||
bpf_ringbuf_submit(e, 0);
|
||||
}
|
||||
return 0; /* 丢弃:skb 不出 cgroup */
|
||||
}
|
||||
return 1; /* 放行 */
|
||||
}
|
||||
|
||||
char _license[] SEC("license") = "GPL";
|
||||
"""
|
||||
|
||||
# LSM hook 污点传播程序源码(骨架)。
|
||||
#
|
||||
# 作用(按 section):
|
||||
# - `SEC("lsm/file_open")`:进程打开文件时触发。hook 内只判
|
||||
# “打开的是不是用户数据路径”(路径表由用户态经 map 下发,
|
||||
# 这里不硬编码路径),是则把当前 pid 写进 `taint_map`
|
||||
# (与 cgroup 程序共用同一张 pin 住的 map)。
|
||||
# - 污点传播方向:读用户数据的进程变污点;子进程的污点由
|
||||
# 用户态 `fork_child` 继承(本骨架不替 fork 背书——fork
|
||||
# 时子进程尚未读文件,LSM 看不到它,必须由追踪器继承)。
|
||||
# 内核要求:>= 5.7 且内核开 CONFIG_BPF_LSM(否则 lsm/* 的
|
||||
# section 挂不上);CAP_BPF;LSM BPF 程序挂载还受系统 LSM
|
||||
# 策略约束(与 SELinux/AppArmor 共存时以后者为准)。本骨架
|
||||
# 只做“读事件 → 写 map”,不做文件内容读取、不记路径内容。
|
||||
EBPF_LSM_TAINT_C = r"""
|
||||
#include <linux/bpf.h>
|
||||
#include <bpf/bpf_helpers.h>
|
||||
|
||||
/* 与 cgroup 程序共用同一张 pin 住的 map(pin 路径
|
||||
/sys/fs/bpf/chv_taint_map,用户态建表后两程序同引)。 */
|
||||
struct {
|
||||
__uint(type, BPF_MAP_TYPE_HASH);
|
||||
__uint(max_entries, 65536);
|
||||
__type(key, __u32);
|
||||
__type(value, __u8);
|
||||
} taint_map SEC(".maps");
|
||||
|
||||
/* LSM hook 骨架:文件打开时触发。是否“用户数据路径”由用户
|
||||
态下发的路径表判定(本骨架只留判断位,不硬编码路径、不读
|
||||
文件内容);命中则把当前 pid 记为污点。 */
|
||||
SEC("lsm/file_open")
|
||||
int taint_on_file_open(struct file *file)
|
||||
{
|
||||
__u32 pid = (__u32)(bpf_get_current_pid_tgid() >> 32);
|
||||
__u8 flag = 1;
|
||||
/* TODO(真机接线时补):查用户态下发的路径表,命中才写;
|
||||
未命中直接 return 0。骨架阶段不写 map,避免误标。 */
|
||||
(void)pid;
|
||||
(void)flag;
|
||||
return 0;
|
||||
}
|
||||
|
||||
char _license[] SEC("license") = "GPL";
|
||||
"""
|
||||
|
||||
|
||||
def _parse_kernel(release: str) -> tuple[int, int, int]:
|
||||
"""`7.0.0-43-generic` 这类 release 串解为三元组。"""
|
||||
parts = release.split("-")[0].split(".")
|
||||
nums = []
|
||||
for part in parts[:3]:
|
||||
nums.append(int(part) if part.isdigit() else 0)
|
||||
while len(nums) < 3:
|
||||
nums.append(0)
|
||||
return (nums[0], nums[1], nums[2])
|
||||
|
||||
|
||||
def _cap_bit(cap_eff_hex: str, bit: int) -> bool:
|
||||
"""CapEff 十六进制串里某一位是否在位。"""
|
||||
try:
|
||||
value = int(cap_eff_hex.strip(), 16)
|
||||
except ValueError:
|
||||
return False
|
||||
return bool((value >> bit) & 1)
|
||||
|
||||
|
||||
class EbpfTaintLoader:
|
||||
"""eBPF 加载器:探测 → 加载(或明确降级)→ 状态上报。
|
||||
|
||||
状态机(`load_status()` 读到的 `status`):
|
||||
- `blocked`:探测到硬 blocker(内核太旧 / 无 bpffs /
|
||||
无能力位 / 缺工具链),加载不会尝试;
|
||||
- `degraded`:未加载成功,工作在 `FALLBACK_MODE`(策略
|
||||
引擎 + 代理层钩子),这是失败时的唯一出口;
|
||||
- `loaded`:bpftool 加载 + pin 成功,且 pin 文件在位。
|
||||
只有 `loaded` 才表示 eBPF 在拦截;其余一律不冒充。
|
||||
"""
|
||||
|
||||
def __init__(self, bpffs_path: str = "/sys/fs/bpf",
|
||||
status_path: str = "/proc/self/status",
|
||||
which: Callable[[str], str | None] | None = None
|
||||
) -> None:
|
||||
self._bpffs_path = bpffs_path
|
||||
self._status_path = status_path
|
||||
self._which = which or shutil.which
|
||||
self._status: dict = {
|
||||
"status": "degraded",
|
||||
"mode": FALLBACK_MODE,
|
||||
"reason": "not_probed_yet",
|
||||
"blockers": [],
|
||||
}
|
||||
|
||||
def _read_caps(self) -> tuple[bool, bool]:
|
||||
"""读 CapEff:返回 (CAP_BPF 在位, CAP_SYS_ADMIN 在位)。"""
|
||||
try:
|
||||
with open(self._status_path) as fh:
|
||||
for line in fh:
|
||||
if line.startswith("CapEff:"):
|
||||
hex_value = line.split(":", 1)[1]
|
||||
return (_cap_bit(hex_value, _CAP_BPF_BIT),
|
||||
_cap_bit(hex_value, _CAP_SYS_ADMIN_BIT))
|
||||
except OSError:
|
||||
return (False, False)
|
||||
return (False, False)
|
||||
|
||||
def _check_tools(self) -> dict:
|
||||
"""bpftool / clang / llvm-objdump 是否在位。"""
|
||||
found = {}
|
||||
for tool in ("bpftool", "clang", "llvm-objdump"):
|
||||
found[tool] = self._which(tool) is not None
|
||||
return found
|
||||
|
||||
def _blockers(self, kernel: tuple[int, int, int],
|
||||
cap_bpf: bool, cap_sys_admin: bool,
|
||||
tools: dict) -> list:
|
||||
"""按硬条件列 blocker(空表 = 探测关全过,可试加载)。"""
|
||||
blockers = []
|
||||
if kernel < MIN_KERNEL:
|
||||
blockers.append("kernel_too_old:%d.%d.%d" % kernel)
|
||||
if not os.path.isdir(self._bpffs_path):
|
||||
blockers.append("bpffs_missing:" + self._bpffs_path)
|
||||
elif not os.access(self._bpffs_path, os.W_OK):
|
||||
blockers.append("bpffs_not_writable:" + self._bpffs_path)
|
||||
if not cap_bpf and not cap_sys_admin:
|
||||
blockers.append("cap_bpf_and_cap_sys_admin_missing")
|
||||
for tool, present in sorted(tools.items()):
|
||||
if not present:
|
||||
blockers.append("tool_missing:" + tool)
|
||||
return blockers
|
||||
|
||||
def probe(self) -> dict:
|
||||
"""能力探测:只读检查,不加载、不改系统状态。
|
||||
|
||||
返回探测结果副本;`status` 为 blocked(有 blocker)
|
||||
或 degraded(探测关全过、但尚未加载——仍在降级模式)。
|
||||
注意:CapEff 在位不等于 bpf() 放行(本机实测:能力位
|
||||
全在,bpf() 仍 EPERM,见模块头注释),所以探测结果
|
||||
从不写“eBPF 可用”,只写“探测关是否全过”。"""
|
||||
kernel = _parse_kernel(platform.uname().release)
|
||||
cap_bpf, cap_sys_admin = self._read_caps()
|
||||
tools = self._check_tools()
|
||||
bpffs_exists = os.path.isdir(self._bpffs_path)
|
||||
blockers = self._blockers(kernel, cap_bpf, cap_sys_admin,
|
||||
tools)
|
||||
status = "blocked" if blockers else "degraded"
|
||||
reason = ("; ".join(blockers) if blockers
|
||||
else "probe_passed_not_loaded")
|
||||
self._status = {
|
||||
"status": status, "mode": FALLBACK_MODE,
|
||||
"reason": reason, "blockers": list(blockers),
|
||||
"kernel": "%d.%d.%d" % kernel,
|
||||
"kernel_ok": kernel >= MIN_KERNEL,
|
||||
"bpffs_exists": bpffs_exists,
|
||||
"bpffs_writable": (bpffs_exists and os.access(
|
||||
self._bpffs_path, os.W_OK)),
|
||||
"cap_bpf": cap_bpf, "cap_sys_admin": cap_sys_admin,
|
||||
"tools": dict(tools),
|
||||
}
|
||||
return dict(self._status)
|
||||
|
||||
def load_status(self) -> dict:
|
||||
"""当前加载状态上报(未探测过时先跑一次探测)。"""
|
||||
if self._status.get("reason") == "not_probed_yet":
|
||||
return self.probe()
|
||||
return dict(self._status)
|
||||
|
||||
def try_load(self) -> dict:
|
||||
"""尝试加载:探测关不过即降级,不尝试;过了才编译加载。
|
||||
|
||||
编译/加载任一步失败也降级,`status` stays degraded,
|
||||
`reason` 写清失败步。成功须 bpftool 加载 + pin 文件
|
||||
在位双重确认,缺一律不报 loaded。"""
|
||||
result = self.probe()
|
||||
if result["blockers"]:
|
||||
return dict(self._status)
|
||||
loaded = self._compile_and_load()
|
||||
if loaded:
|
||||
self._status = dict(result)
|
||||
self._status["status"] = "loaded"
|
||||
self._status["mode"] = "ebpf"
|
||||
self._status["reason"] = "loaded_and_pinned"
|
||||
self._status["blockers"] = []
|
||||
else:
|
||||
self._status["reason"] = "load_failed_fallback"
|
||||
return dict(self._status)
|
||||
|
||||
def _compile_and_load(self) -> bool:
|
||||
"""clang 编译 + bpftool 加载 + pin(只在探测关全过时跑)。
|
||||
|
||||
任一步失败返回 False(调用方降级)。本机永远走不到
|
||||
这里(探测关不过);真机上失败原因由 bpftool 的
|
||||
stderr 决定,这里只记成败、不猜原因。"""
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
src = os.path.join(tmp, "egress.c")
|
||||
obj = os.path.join(tmp, "egress.o")
|
||||
pin = os.path.join(self._bpffs_path,
|
||||
"chv_egress_filter")
|
||||
with open(src, "w") as fh:
|
||||
fh.write(EBPF_CGROUP_EGRESS_C)
|
||||
compiled = subprocess.run(
|
||||
["clang", "-target", "bpf", "-O2", "-c", src,
|
||||
"-o", obj],
|
||||
capture_output=True, timeout=120)
|
||||
if compiled.returncode != 0:
|
||||
return False
|
||||
loaded = subprocess.run(
|
||||
["bpftool", "prog", "load", obj, pin],
|
||||
capture_output=True, timeout=120)
|
||||
if loaded.returncode != 0:
|
||||
return False
|
||||
return os.path.exists(pin)
|
||||
@@ -1,12 +0,0 @@
|
||||
"""Cloud Hypervisor 执行器的分类失败(叶子模块,不依赖
|
||||
cloud.sandbox,避免循环导入)。"""
|
||||
|
||||
class ChvFailure(RuntimeError):
|
||||
"""Cloud Hypervisor 分类失败:RuntimeError 的子类(worker
|
||||
现有 except Exception 路径照常捕获),另带 kind 字段供
|
||||
审计与失败分类用,绝不笼统只写 failed。kind 取值见
|
||||
CloudHypervisorExecutor.preflight / wake_vm 的注释。"""
|
||||
|
||||
def __init__(self, kind: str, message: str) -> None:
|
||||
super().__init__("chv_failure:%s: %s" % (kind, message))
|
||||
self.kind = kind
|
||||
File diff suppressed because it is too large.
Load diff
@@ -1,293 +0,0 @@
|
||||
"""快照预热池 WarmPool(纯实验模块,非生产路径)。
|
||||
|
||||
本轨(feat-chv-arch-maint-fe)破环性清理:WarmPool 从
|
||||
cloud/chv/pool.py 移到本文件,cloud/chv/__init__ 与
|
||||
cloud/sandbox.py 不再 re-export 它,生产代码 import 本
|
||||
模块即违规(见 docs/chv-architecture.md ADR-12 的禁
|
||||
|
||||
import 清单与测试钉住)。实验用法只走
|
||||
`from cloud.chv.experimental_pool import WarmPool`。
|
||||
生产残留清理已彻底分文件:见 cloud/chv/orphans.py 的
|
||||
reconcile_orphans_for_executor(),它不 import 本模块。
|
||||
|
||||
定案(架构决策,见 docs/chv-architecture.md ADR-5):WarmPool
|
||||
不再宣称为生产快速启动特性。原因:预热 VM 的磁盘是干净快照
|
||||
盘,不是任务盘;dispatch 前查池若要真正接线,必须在派任务
|
||||
前给已暂停 VM 重新供给任务盘(热换盘/二盘挂载/host.py 磁盘
|
||||
语义三者都未实现),把预热盘直接当任务盘用会击穿“任务盘
|
||||
隔离”不变量。acquire 内 resume 只是实验性接口,worker
|
||||
dispatch 永远走冷启动(wake_vm),_run_with_vm 绝不查池。
|
||||
本模块的 reconcile_orphans() 只留给实验/旧测试直接调用;
|
||||
生产启动/关闭走 orphans facade,不再经本类。
|
||||
warm_pool_wired_to_dispatch 恒为 False,任何把它改成
|
||||
True 的改动都必须先实现任务盘重供给并补不变量测试,
|
||||
否则不得合入。
|
||||
|
||||
|
||||
隔离红线(与 #216 一致,绝不妥协):预热条目是一次性的
|
||||
干净快照 VM,acquire 消耗掉它,release 一律销毁,绝不把
|
||||
一个任务用过的 VM(带任务文件、带 agent 痕迹)翻出来给
|
||||
下一个任务/租户复用;池子空了就回落冷启动(wake_vm),
|
||||
绝不因为池空就降级 Docker/Local。
|
||||
|
||||
Registry(本轨新增):
|
||||
- 文件:state_dir/warm-pool.json(registry_path 可覆盖),
|
||||
内容为 {<合成名>: {"state", "vm", "task_id", "snapshot_url"}}。
|
||||
每次 prewarm/acquire/release 后立即落盘(临时文件 + 原子
|
||||
rename),worker 崩溃后重启仍能发现上一世的条目。
|
||||
- 载入时绝不复用:新池实例从 registry 读到的条目一律标为
|
||||
orphan(不计入 warm_count,acquire 绝不发给任务),因为
|
||||
旧 VMM 进程已随旧 worker 消失,无法验证它还是当初那个
|
||||
干净快照 VM。它们只能走 reconcile_orphans() 清理。
|
||||
- reconcile_orphans():worker 启动/清理时的显式入口,对
|
||||
每个 orphan/assigned 条目尽力调 executor.shutdown_vm
|
||||
(清 socket 与残留登记),条目清掉并落盘,返回已清理的
|
||||
VM 名列表。registry 里没有的残留 socket 仍由执行器的
|
||||
cleanup_stale_vms() 扫 state_dir 处理,两者互补,不互相
|
||||
替代;reconcile 绝不在构造时自动执行(同一 state_dir 只
|
||||
能有一个活池,构造时自动清理会误杀另一个活池的 VM)。
|
||||
- 损坏 fail-safe:registry 不是合法 JSON / 顶层不是 dict 时,
|
||||
整个文件 quarantine 成 <path>.corrupt(尽力而为),池子
|
||||
从空开始,绝不用损坏数据复用 VM;单条目结构非法时只
|
||||
丢该条(记 registry_corrupt_entries),不连坐其它条目。
|
||||
registry 写失败时只记日志警告、不炸任务流(池子内存态
|
||||
仍可用,重启后孤儿改由 socket 扫描兜底)。
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from cloud.chv.executor import CloudHypervisorExecutor
|
||||
|
||||
import os
|
||||
|
||||
from cloud.chv.orphans import _REGISTRY_FILENAME, _read_registry, _write_registry
|
||||
|
||||
|
||||
class WarmPool:
|
||||
"""快照预热池(实验性,见模块头定案)。
|
||||
|
||||
wired_to_dispatch = False:生产 dispatch 不查池。
|
||||
|
||||
|
||||
目标:冷启动(preflight + 起 VMM + create + boot + SSH
|
||||
就绪)是慢路径;预热池提前把干净 VM 起好并暂停,任务
|
||||
来时 acquire 直接 resume,省掉冷启动的大头。
|
||||
状态机:cold(未预热)→ prewarming(wake 中)→ warm
|
||||
(已暂停待分配)→ assigned(已分配给某任务)→
|
||||
running(已 resume)→ recycled/destroyed(release 后)。
|
||||
prewarm 的实现:对每个条目调 executor.wake_vm(冷启动)
|
||||
再 pause_vm,得到 paused 的干净 VM;mock 单测里 wake/
|
||||
pause 均可注入假实现,真机上这一步需要 KVM 宿主(见
|
||||
就绪清单“需 KVM 宿主实测”)。snapshot_url 未设时
|
||||
prewarm 返回 [](无基准快照可预热,不是静默造快照)。
|
||||
"""
|
||||
|
||||
wired_to_dispatch = False
|
||||
|
||||
def __init__(self, executor: "CloudHypervisorExecutor", size: int | None = None,
|
||||
snapshot_url: str | None = None,
|
||||
registry_path: str | None = None) -> None:
|
||||
self.executor = executor
|
||||
self.size = int(size if size is not None
|
||||
else getattr(executor, "warm_pool_size", 0))
|
||||
self.snapshot_url = snapshot_url \
|
||||
if snapshot_url is not None \
|
||||
else getattr(executor, "snapshot_url", None)
|
||||
self._entries: dict = {} # 合成名 -> {"state", "vm", "task", ...}
|
||||
self.registry_path = registry_path
|
||||
if self.registry_path is None:
|
||||
state_dir = getattr(executor, "state_dir", None)
|
||||
if state_dir:
|
||||
self.registry_path = os.path.join(
|
||||
state_dir, _REGISTRY_FILENAME)
|
||||
self.registry_corrupt = False
|
||||
self.registry_corrupt_entries = 0
|
||||
self._load_registry()
|
||||
|
||||
# -- registry ------------------------------------------------------
|
||||
def _load_registry(self) -> None:
|
||||
"""载入 registry:读到的条目一律标 orphan(见模块
|
||||
docstring,绝不直接当 warm 发给任务)。无文件时为空
|
||||
池;损坏时 quarantine 并从空开始。文件 IO 与损坏
|
||||
语义唯一来源是 cloud.chv.orphans(生产模块),本
|
||||
方法只做“条目 → 内存 orphan 态”的映射。"""
|
||||
if not self.registry_path:
|
||||
return
|
||||
data, corrupt = _read_registry(self.registry_path)
|
||||
if corrupt:
|
||||
self.registry_corrupt = True
|
||||
return
|
||||
if data is None:
|
||||
return
|
||||
for name, entry in data.items():
|
||||
if not isinstance(entry, dict) \
|
||||
or not entry.get("vm") \
|
||||
or entry.get("state") not in ("warm", "assigned"):
|
||||
self.registry_corrupt_entries += 1
|
||||
continue
|
||||
self._entries[name] = {
|
||||
"state": "orphan",
|
||||
"vm": entry["vm"],
|
||||
"task": {"id": entry.get("task_id") or name},
|
||||
"snapshot_url": entry.get("snapshot_url"),
|
||||
}
|
||||
|
||||
def _save_registry(self) -> None:
|
||||
"""落盘当前条目(只写 warm/assigned 态;orphan 是
|
||||
待清理的上一世条目,reconcile 后即删,不写回)。
|
||||
写失败只记日志,不炸任务流。落盘 IO 委托 orphans。"""
|
||||
if not self.registry_path:
|
||||
return
|
||||
data = {}
|
||||
for name, entry in self._entries.items():
|
||||
if entry["state"] not in ("warm", "assigned"):
|
||||
continue
|
||||
task = entry.get("task") or {}
|
||||
task_id = task.get("id") if isinstance(task, dict) \
|
||||
else None
|
||||
data[name] = {
|
||||
"state": entry["state"],
|
||||
"vm": entry.get("vm"),
|
||||
"task_id": task_id,
|
||||
"snapshot_url": entry.get("snapshot_url"),
|
||||
}
|
||||
_write_registry(self.registry_path, data)
|
||||
|
||||
def reconcile_orphans(self) -> list:
|
||||
"""清理本池内存里的 orphan/assigned 条目(实验入口)。
|
||||
|
||||
对每个条目尽力 shutdown_vm,条目清掉并落盘,返回
|
||||
已清理的 VM 名列表。生产启动/关闭不走本方法,走
|
||||
cloud.chv.orphans.reconcile_orphans_for_executor()。"""
|
||||
cleaned = []
|
||||
for name, entry in list(self._entries.items()):
|
||||
if entry["state"] not in ("orphan", "assigned"):
|
||||
continue
|
||||
vm = entry.get("vm") or name
|
||||
try:
|
||||
self.executor.shutdown_vm(vm)
|
||||
except Exception:
|
||||
pass
|
||||
self._entries.pop(name, None)
|
||||
cleaned.append(vm)
|
||||
if cleaned:
|
||||
self._save_registry()
|
||||
try:
|
||||
self.executor._emit(
|
||||
"warm_pool_reconcile", {"id": None},
|
||||
"cleaned=%s" % ",".join(cleaned))
|
||||
except Exception:
|
||||
pass
|
||||
return cleaned
|
||||
|
||||
# -- 池操作 ----------------------------------------------------------
|
||||
def warm_count(self) -> int:
|
||||
return sum(1 for e in self._entries.values()
|
||||
if e["state"] == "warm")
|
||||
|
||||
def orphan_count(self) -> int:
|
||||
"""上一世残留条目数(待 reconcile,不可分配)。"""
|
||||
return sum(1 for e in self._entries.values()
|
||||
if e["state"] in ("orphan", "assigned"))
|
||||
|
||||
def prewarm(self, count: int | None = None) -> list:
|
||||
"""预热到目标数:已有 warm 条目不重复建;每个新条目
|
||||
用一个合成任务名 wake + pause(合成任务 id 为
|
||||
warm-<序号>,绝不与真实任务 id 撞名)。返回新增的
|
||||
warm VM 名列表。"""
|
||||
if self.snapshot_url is None:
|
||||
return []
|
||||
target = int(count) if count is not None else self.size
|
||||
added = []
|
||||
i = 0
|
||||
while self.warm_count() < target:
|
||||
i += 1
|
||||
name = "warm-%d" % i
|
||||
if name in self._entries:
|
||||
continue
|
||||
task = {"id": name}
|
||||
self._entries[name] = {"state": "prewarming",
|
||||
"task": task}
|
||||
try:
|
||||
self.executor.wake_vm(task)
|
||||
self.executor.pause_vm(task)
|
||||
except Exception:
|
||||
self._entries.pop(name, None)
|
||||
raise
|
||||
vm = self.executor.vm_name(task)
|
||||
self._entries[name] = {
|
||||
"state": "warm", "task": task, "vm": vm,
|
||||
"snapshot_url": self.snapshot_url}
|
||||
added.append(name)
|
||||
if added:
|
||||
self._save_registry()
|
||||
return added
|
||||
|
||||
def acquire(self, task: dict) -> dict:
|
||||
"""分配一个预热 VM 给任务(feat-chv-fix,P1-3 接线):
|
||||
|
||||
有 warm 条目时先调 executor.resume_vm() 把暂停的 VM
|
||||
真正唤醒(resume 失败即条目作废、继续找下一个,
|
||||
绝不把 paused 的 VM 当 running 发给任务),再标
|
||||
assigned 并返回 mode=warm;resume 全部失败或无 warm
|
||||
条目时返回 mode=cold(调用方回落 wake_vm 冷启动)。
|
||||
注意:本方法只解决“acquire 不再悬空 resume 责任”
|
||||
这一半;worker dispatch 前查池的另一半仍未接线
|
||||
(见执行器/就绪文档的“未接线缺口”标注),预热收益
|
||||
在生产路径上仍不是已交付特性。
|
||||
feat-chv-secfix(L5,接线时的盘对应关系定义——接线
|
||||
本身仍未做,不假装已接):预热 VM 的磁盘是干净快照
|
||||
盘,不是任务盘。将来 dispatch 前查池接线时必须做到:
|
||||
acquire 返回的 VM 在派任务前仍须按任务重新供给磁盘
|
||||
(provision_task_disk 的 copy 盘,或校验快照来源与该
|
||||
任务的对应关系后才能用快照盘),绝不能把预热盘直接
|
||||
当任务盘用;接线时须补“任务盘来源”不变量的测试,
|
||||
否则 acquire 的 warm 路径不得合入生产路径。"""
|
||||
for name, entry in list(self._entries.items()):
|
||||
if entry["state"] != "warm":
|
||||
continue
|
||||
resume = getattr(self.executor, "resume_vm", None)
|
||||
if resume is not None:
|
||||
try:
|
||||
resume(entry.get("task") or task)
|
||||
except Exception:
|
||||
self._entries.pop(name, None)
|
||||
self._save_registry()
|
||||
continue
|
||||
# 无 resume_vm 的鸭子型执行器(旧测试替身):
|
||||
# 保持旧 assigned 语义,不因此丢弃条目;真实
|
||||
# CloudHypervisorExecutor 一律有 resume_vm,走
|
||||
# 上面分支,责任不悬空。
|
||||
entry["state"] = "assigned"
|
||||
entry["task"] = task
|
||||
self._save_registry()
|
||||
try:
|
||||
self.executor._emit(
|
||||
"warm_pool_acquire", task,
|
||||
"vm=%s" % name)
|
||||
except Exception:
|
||||
pass
|
||||
return {"mode": "warm", "vm": name,
|
||||
"pool_task": entry["task"]}
|
||||
return {"mode": "cold", "vm": None, "pool_task": None}
|
||||
|
||||
def release(self, task: dict, pool_vm: str | None = None) -> bool:
|
||||
"""释放:无论 warm 还是 cold 路径,VM 一律销毁
|
||||
(executor.shutdown_vm),池条目清掉;绝不把用过
|
||||
的 VM 放回 warm 池。返回 True 表示确实销毁了一个
|
||||
条目或 VM。"""
|
||||
destroyed = False
|
||||
if pool_vm and pool_vm in self._entries:
|
||||
self._entries.pop(pool_vm, None)
|
||||
destroyed = True
|
||||
try:
|
||||
if self.executor.shutdown_vm(task):
|
||||
destroyed = True
|
||||
except Exception:
|
||||
pass
|
||||
self._save_registry()
|
||||
self.executor._emit(
|
||||
"warm_pool_release", task,
|
||||
"vm=%s destroyed=%s" % (pool_vm, destroyed))
|
||||
return destroyed
|
||||
@@ -1,146 +0,0 @@
|
||||
"""Gitea 同步变量供给(CHV guest-env 集成,叶子模块)。
|
||||
|
||||
背景:客内 chv-sync boot 在 cloud-init 时运行、SSH 之前,
|
||||
只能读 seed 投递的 /etc/chv/guest-env,看不到 SSH env 前缀。
|
||||
因此同步所需的 6 个变量必须随 seed 进 guest-env,不能只靠
|
||||
extra_env 的 SSH 前缀;其中 GITEA_TOKEN 是密钥,只进 seed,
|
||||
绝不进 argv/日志/镜像。
|
||||
|
||||
变量与来源(优先级低→高,后者覆盖前者):
|
||||
1. 宿主配置环境变量(主来源,生产口径):CHV_GITEA_URL、
|
||||
CHV_GITEA_USER、CHV_GITEA_TOKEN、CHV_SYNC_REPO、
|
||||
CHV_SYNC_BRANCH、CHV_GITEA_EMAIL。由部署该 worker/执行器
|
||||
的运维在宿主环境里设置,与 CHV_SSH_KEY、CHV_EGRESS_PROXY
|
||||
同款口径;CHV 是单租户单 VM 模型,一台宿主对应一个
|
||||
Gitea 身份。密钥来源绝不硬编码进代码。
|
||||
2. 任务级覆盖(内存态):任务 dict 的 `gitea_sync` 子 dict,
|
||||
键即 6 个变量名。仅在调用方以内存态构造任务 dict 时生效
|
||||
(直接调 wake_vm、测试、手工驱动);tasks 表目前没有
|
||||
JSON/元数据列,DB 往返会丢掉它——这是已知边界,不做
|
||||
静默声称,需要每任务不同仓库时再做任务表迁移。
|
||||
3. 显式 extra_env:调用方直接传给 wake_vm/dispatch 的
|
||||
extra_env 里的同名键,优先级最高。
|
||||
|
||||
缺失语义(fail-closed,不跳过同步):
|
||||
- 6 个键一个都没有 → 本次启动不同步(chv-sync boot 见无
|
||||
guest-env 静默成功,VM 照常用)。
|
||||
- 任一键出现、但必填 4 键(GITEA_URL、GITEA_USER、
|
||||
GITEA_TOKEN、SYNC_REPO)缺失 → 校验失败,wake 在起 VMM
|
||||
之前直接拒绝;半份同步配置绝不能让 VM 带着“我以为同步
|
||||
了”的状态启动。SYNC_BRANCH、GITEA_EMAIL 可选。
|
||||
- 校验错误只含缺失的键名,绝不含任何变量值。
|
||||
|
||||
本模块是纯函数,不读文件、不跑 VM、不含任何密钥值。
|
||||
"""
|
||||
|
||||
# 同步变量全集(guest-env 里应有的键)。
|
||||
GITEA_SYNC_KEYS = frozenset({
|
||||
"GITEA_URL", "GITEA_USER", "GITEA_TOKEN",
|
||||
"SYNC_REPO", "SYNC_BRANCH", "GITEA_EMAIL",
|
||||
})
|
||||
|
||||
# 其中密钥:只进 seed,绝不进 argv(见 ssh_transport 的
|
||||
# _CHV_SECRET_ENV_NAMES)。
|
||||
GITEA_SYNC_SECRET_KEYS = frozenset({"GITEA_TOKEN"})
|
||||
|
||||
# 其中 authd 专用密钥(feat-chv-muse-p3):只进客内 authd 的
|
||||
# 独立 seed 文件 /etc/chv/authd-seed(见 hostkeys.AUTHD_SEED_PATH),
|
||||
# 不进 guest-env、不进 codex 进程环境。本期只有 GITEA_TOKEN。
|
||||
AUTHD_SECRET_KEYS = frozenset({"GITEA_TOKEN"})
|
||||
|
||||
|
||||
def split_authd_env(secrets: dict | None) -> tuple[dict, dict]:
|
||||
"""把密钥集拆成(走 authd 通道的子集,仍走 guest-env 的子集)。
|
||||
|
||||
authd 子集 = AUTHD_SECRET_KEYS 命中的键(本期只有 GITEA_TOKEN);
|
||||
其余密钥(如 SUB2API_API_KEY)仍走 guest-env(见差距表 §3.3,
|
||||
网关 key 的 privsep 改造是后续工作)。"""
|
||||
authd, rest = {}, {}
|
||||
for key, value in dict(secrets or {}).items():
|
||||
if key in AUTHD_SECRET_KEYS:
|
||||
authd[key] = value
|
||||
else:
|
||||
rest[key] = value
|
||||
return authd, rest
|
||||
|
||||
# 其中非密钥、但 boot 时就要用、也必须进 guest-env 的键
|
||||
#(SSH env 前缀到不了 cloud-init 时运行的 chv-sync boot)。
|
||||
GITEA_SYNC_PLAIN_KEYS = GITEA_SYNC_KEYS - GITEA_SYNC_SECRET_KEYS
|
||||
|
||||
# 必填键:缺一即 fail-closed(见模块头缺失语义)。
|
||||
GITEA_SYNC_REQUIRED_KEYS = frozenset({
|
||||
"GITEA_URL", "GITEA_USER", "GITEA_TOKEN", "SYNC_REPO",
|
||||
})
|
||||
|
||||
# 宿主配置环境变量名 → guest-env 键名。
|
||||
CHV_GITEA_ENV_MAP = {
|
||||
"CHV_GITEA_URL": "GITEA_URL",
|
||||
"CHV_GITEA_USER": "GITEA_USER",
|
||||
"CHV_GITEA_TOKEN": "GITEA_TOKEN",
|
||||
"CHV_SYNC_REPO": "SYNC_REPO",
|
||||
"CHV_SYNC_BRANCH": "SYNC_BRANCH",
|
||||
"CHV_GITEA_EMAIL": "GITEA_EMAIL",
|
||||
}
|
||||
|
||||
|
||||
def _clean(mapping: dict | None) -> dict:
|
||||
"""只留白名单键:值须为非空字符串(strip 后),其它
|
||||
类型/空值/未知键一律丢掉——调用方拼错的键名绝不静默
|
||||
进 seed 变成客内无人读的脏变量。"""
|
||||
cleaned = {}
|
||||
for key, value in dict(mapping or {}).items():
|
||||
if key not in GITEA_SYNC_KEYS:
|
||||
continue
|
||||
if not isinstance(value, str):
|
||||
continue
|
||||
value = value.strip()
|
||||
if not value:
|
||||
continue
|
||||
cleaned[key] = value
|
||||
return cleaned
|
||||
|
||||
|
||||
def config_gitea_sync_env(env: dict | None) -> dict:
|
||||
"""宿主配置环境变量(CHV_GITEA_*)→ guest-env 键名。"""
|
||||
mapped = {}
|
||||
for env_name, key in CHV_GITEA_ENV_MAP.items():
|
||||
value = (env or {}).get(env_name)
|
||||
if value is not None:
|
||||
mapped[key] = value
|
||||
return _clean(mapped)
|
||||
|
||||
|
||||
def task_gitea_sync_env(task: dict | None) -> dict:
|
||||
"""任务 dict 的 `gitea_sync` 子 dict → 白名单后的键值。
|
||||
不是 dict(或缺失)即空——绝不把任务其它字段误读成
|
||||
同步变量。"""
|
||||
if not isinstance(task, dict):
|
||||
return {}
|
||||
return _clean(task.get("gitea_sync"))
|
||||
|
||||
|
||||
def extract_gitea_sync_env(task: dict | None = None,
|
||||
extra_env: dict | None = None,
|
||||
config_sync: dict | None = None) -> dict:
|
||||
"""三源合并(优先级:config_sync < 任务 gitea_sync <
|
||||
extra_env 同名键)。返回白名单后的 dict,可能为空
|
||||
(空 = 本次启动不同步)。"""
|
||||
merged = {}
|
||||
merged.update(_clean(config_sync))
|
||||
merged.update(task_gitea_sync_env(task))
|
||||
merged.update(_clean(extra_env))
|
||||
return merged
|
||||
|
||||
|
||||
def validate_gitea_sync_env(env: dict | None) -> None:
|
||||
"""校验同步变量完整性:空即不同步(通过);非空但必填
|
||||
键缺失即 ValueError(只含缺失键名,不含值)。调用方
|
||||
(执行器 wake_vm)在起 VMM 前调本函数做 fail-closed。"""
|
||||
env = dict(env or {})
|
||||
if not env:
|
||||
return
|
||||
missing = sorted(GITEA_SYNC_REQUIRED_KEYS - set(env))
|
||||
if missing:
|
||||
raise ValueError(
|
||||
"Gitea 同步变量不完整,缺失:%s(同步已请求但必填"
|
||||
"键不全,拒绝启动)" % ",".join(missing))
|
||||
@@ -1,136 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""客内 vsock notify 脚本(P1,feat-chv-muse-p1)。
|
||||
|
||||
启动后连宿主 CID 2 的 1024 端口发 ready,之后每 30 秒发一次
|
||||
heartbeat。消息是 JSON 行:
|
||||
|
||||
{"type": "ready"|"heartbeat", "vm": "<vm 名>", "ts": <unix 秒>}
|
||||
|
||||
宿主在 <vsock socket>_1024 这个 Unix socket 上监听接收
|
||||
(cloud/chv/vsock.py 的 wait_ready/listen_notify)。
|
||||
|
||||
失败一律静默退出(exit 0,不打日志、不留 traceback):vsock
|
||||
只是就绪辅助信号,SSH 仍是唯一命令通道与就绪判定——本脚本
|
||||
绝不能因为连不上、发失败而影响 sshd 或任务。调试时设
|
||||
VSOCK_NOTIFY_DEBUG=1 可在 stderr 看到异常。
|
||||
|
||||
================ P2 镜像集成接入说明(本轨只写脚本,不改构建) ================
|
||||
本文件不由 chv-guest/build.sh 安装,镜像集成是 P2 的范围。
|
||||
P2 接入时做这三件:
|
||||
|
||||
1. 把本文件拷进镜像,例如 /app/vsock_notify.py(须 python3,
|
||||
客机镜像已有 Python;若最小镜像无 python3,P2 须先补);
|
||||
2. 加 systemd 单元,例如 /etc/systemd/system/vsock-notify.service:
|
||||
|
||||
[Unit]
|
||||
Description=vsock notify to host (ready/heartbeat)
|
||||
After=multi-user.target
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/bin/python3 /app/vsock_notify.py %H
|
||||
Restart=no
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
(%H 是客机 hostname;也可直接写 VM 名,或由 cloud-init
|
||||
写进环境变量 CHV_VM_NAME。Restart=no:本脚本失败即静默
|
||||
退出,不由 systemd 反复拉起打扰宿主监听。)
|
||||
3. 前置条件(缺一则本脚本静默退出,任务仍走 SSH,不报错):
|
||||
- 客机内核须含 CONFIG_VIRTIO_VSOCKETS;
|
||||
- 起 VM 时 CHV_VSOCK=1(默认开),VmConfig 带 vsock 设备
|
||||
(cloud/chv/executor.py 的 _balloon_vsock_payload);
|
||||
- 宿主侧须在 VM boot 前后监听 <socket>_1024(否则客机
|
||||
首次 connect 失败,本脚本在 30 秒内重试,仍失败即退出)。
|
||||
|
||||
端口 1024 与宿主侧 cloud/chv/vsock.py 的 VSOCK_NOTIFY_PORT
|
||||
须一致;改一处必须改另一处。
|
||||
===============================================================================
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
|
||||
# 宿主 CID(vsock 众所周知值,CHV 官方 docs/vsock.md)。
|
||||
HOST_CID = 2
|
||||
# notify 端口(与宿主侧 VSOCK_NOTIFY_PORT 一致)。
|
||||
NOTIFY_PORT = 1024
|
||||
# 首次 connect 重试窗口(秒):宿主监听可能比客机启动晚到。
|
||||
CONNECT_RETRY_SECONDS = 30
|
||||
# 心跳间隔(秒)。
|
||||
HEARTBEAT_INTERVAL = 30
|
||||
|
||||
|
||||
def _vm_name(argv) -> str:
|
||||
"""VM 名:argv[1] > CHV_VM_NAME > hostname。"""
|
||||
if len(argv) > 1 and argv[1]:
|
||||
return argv[1]
|
||||
env_name = os.environ.get("CHV_VM_NAME") or os.environ.get(
|
||||
"VSOCK_VM_NAME")
|
||||
if env_name:
|
||||
return env_name
|
||||
try:
|
||||
return socket.gethostname()
|
||||
except OSError:
|
||||
return "unknown"
|
||||
|
||||
|
||||
def _send(sock, message_type: str, vm: str) -> None:
|
||||
line = json.dumps({"type": message_type, "vm": vm,
|
||||
"ts": time.time()},
|
||||
separators=(",", ":"))
|
||||
sock.sendall((line + "\n").encode("utf-8"))
|
||||
|
||||
|
||||
def _connect(deadline: float):
|
||||
"""在 deadline 前反复连 (HOST_CID, NOTIFY_PORT),成功返回
|
||||
socket,超时返回 None。"""
|
||||
while True:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
return None
|
||||
sock = socket.socket(socket.AF_VSOCK, socket.SOCK_STREAM)
|
||||
try:
|
||||
sock.settimeout(min(remaining, 5))
|
||||
sock.connect((HOST_CID, NOTIFY_PORT))
|
||||
sock.settimeout(None)
|
||||
return sock
|
||||
except OSError as exc:
|
||||
try:
|
||||
sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
if os.environ.get("VSOCK_NOTIFY_DEBUG") == "1":
|
||||
print("vsock_notify connect 失败:%r" % (exc,),
|
||||
file=sys.stderr, flush=True)
|
||||
time.sleep(1)
|
||||
|
||||
|
||||
def main(argv) -> int:
|
||||
vm = _vm_name(argv)
|
||||
deadline = time.monotonic() + CONNECT_RETRY_SECONDS
|
||||
sock = _connect(deadline)
|
||||
if sock is None:
|
||||
return 0
|
||||
try:
|
||||
_send(sock, "ready", vm)
|
||||
while True:
|
||||
time.sleep(HEARTBEAT_INTERVAL)
|
||||
_send(sock, "heartbeat", vm)
|
||||
except (OSError, ValueError):
|
||||
# 任何发送失败都静默退出,不影响 sshd/任务。
|
||||
return 0
|
||||
except KeyboardInterrupt:
|
||||
return 0
|
||||
finally:
|
||||
try:
|
||||
sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv))
|
||||
@@ -1,213 +0,0 @@
|
||||
"""CHV 安全加固 helper(feat-chv-secfix,叶子模块)。
|
||||
|
||||
这里只放不依赖执行器类的纯校验/文件操作:state_dir 安全
|
||||
目录、VMM socket 活性、net 条目隔离校验、快照 URL 白名单、
|
||||
egress 代理地址校验。宿主防火墙规则、tap 桥接本身是运维
|
||||
外部状态,代码建不出来也不校验——那部分只写进部署清单
|
||||
(docs/chv-production-readiness.md §11),绝不在代码里
|
||||
假装已隔离。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
|
||||
from cloud.chv.errors import ChvFailure
|
||||
|
||||
# NET_JSON 条目字段白名单(L4):tap/mac/ip/mask 之外的字段
|
||||
# 一律拒绝——未知字段原样送进 VMM API 的行为不确定,静默
|
||||
# 透传等于把 VMM 的字段语义赌在运维手写上。
|
||||
_NET_FIELDS = frozenset({"tap", "mac", "ip", "mask"})
|
||||
|
||||
|
||||
def ensure_secure_state_dir(state_dir: str) -> str:
|
||||
"""建 state_dir 并保证 0700(H6):目录不存在即按 0700
|
||||
建;已存在但组/其它位可访时 chmod 0700;失败即
|
||||
ChvFailure(state_dir_insecure),绝不带着 /tmp 式
|
||||
宽松目录启动 VM。umask 影响 mkdir 的默认权限,故建
|
||||
后一律显式 chmod 校验一次。"""
|
||||
try:
|
||||
os.makedirs(state_dir, exist_ok=True)
|
||||
os.chmod(state_dir, 0o700)
|
||||
except OSError as exc:
|
||||
raise ChvFailure(
|
||||
"state_dir_insecure",
|
||||
"state_dir 无法建成安全目录(0700):%s:%s"
|
||||
% (state_dir, exc))
|
||||
try:
|
||||
mode = os.stat(state_dir).st_mode & 0o777
|
||||
except OSError as exc:
|
||||
raise ChvFailure(
|
||||
"state_dir_insecure",
|
||||
"state_dir 无法校验权限:%s:%s" % (state_dir, exc))
|
||||
if mode & 0o077:
|
||||
raise ChvFailure(
|
||||
"state_dir_insecure",
|
||||
"state_dir 权限仍宽松(%o),拒绝启动:%s"
|
||||
% (mode, state_dir))
|
||||
return state_dir
|
||||
|
||||
|
||||
def socket_is_live(socket_path: str) -> bool:
|
||||
"""VMM api-socket 是否真有监听者(M7):能连通才算活。
|
||||
普通文件、空文件、已死 VMM 的残留 socket 一律算死——
|
||||
容量计数不再被残留文件占满。连接失败的任何异常都算
|
||||
死(不区分 refused/超时,计数是尽力而为的读操作)。
|
||||
注意竞态:刚启动、尚未 listen 的 VMM 会被判死,容量
|
||||
门在并发唤醒时可能短暂低估——文档明写该窗口,容量门
|
||||
是“拒绝超限”的护栏,不是精确的分布式锁。"""
|
||||
try:
|
||||
if not socket_path or not os.path.exists(socket_path):
|
||||
return False
|
||||
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
try:
|
||||
sock.settimeout(0.5)
|
||||
return sock.connect_ex(socket_path) == 0
|
||||
finally:
|
||||
sock.close()
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def parse_net_entries(net_json: str) -> list:
|
||||
"""解析 CHV_NET_JSON 为条目 list(非法 JSON / 非
|
||||
list/dict 即 net_config_invalid)。"""
|
||||
if not net_json:
|
||||
return []
|
||||
try:
|
||||
net = json.loads(net_json)
|
||||
except ValueError:
|
||||
raise ChvFailure("net_config_invalid",
|
||||
"CHV_NET_JSON 不是合法 JSON")
|
||||
entries = net if isinstance(net, list) else [net]
|
||||
if not all(isinstance(e, dict) for e in entries):
|
||||
raise ChvFailure("net_config_invalid",
|
||||
"CHV_NET_JSON 条目须为 dict")
|
||||
return entries
|
||||
|
||||
|
||||
def validate_net_entries(entries: list, existing_entries: list) -> list:
|
||||
"""net 条目隔离校验(H7 + L4):
|
||||
|
||||
- 字段白名单:未知字段即 net_config_invalid;
|
||||
- 条目内 tap/mac/ip 归一化(mac 小写、空 tap 视为未
|
||||
指定,不参与 tap 冲突比较——空 tap 是“VMM 默认网卡”
|
||||
的运维写法,代码不编造 tap 名,但空 tap 多 VM 共用
|
||||
默认网卡的风险仍在部署清单里明写);
|
||||
- 本次条目内 tap/mac/ip 重复即 net_config_invalid;
|
||||
- 与已登记 VM(existing_entries,每项为 {tap, mac, ip,
|
||||
vm})的 tap/mac/ip 任一重复即 net_tap_conflict,
|
||||
拒绝启动并由调用方审计。
|
||||
返回归一化后的条目 list。"""
|
||||
normalized = [_normalize_net_entry(e) for e in entries]
|
||||
_check_net_duplicates(normalized)
|
||||
_check_net_conflicts(normalized, existing_entries)
|
||||
return normalized
|
||||
|
||||
|
||||
def _normalize_net_entry(entry: dict) -> dict:
|
||||
"""单条 net 条目归一化:字段白名单 + tap/mac/ip 归一化。
|
||||
|
||||
空 tap 视为未指定,不参与 tap 冲突比较(见
|
||||
validate_net_entries 的说明,纯移动)。"""
|
||||
unknown = set(entry) - _NET_FIELDS
|
||||
if unknown:
|
||||
raise ChvFailure(
|
||||
"net_config_invalid",
|
||||
"CHV_NET_JSON 条目含未知字段:%s(白名单:%s)"
|
||||
% (",".join(sorted(unknown)),
|
||||
",".join(sorted(_NET_FIELDS))))
|
||||
return {"tap": str(entry.get("tap") or "").strip(),
|
||||
"mac": str(entry.get("mac") or "").strip().lower(),
|
||||
"ip": str(entry.get("ip") or "").strip(),
|
||||
"mask": str(entry.get("mask") or "")}
|
||||
|
||||
|
||||
def _check_net_duplicates(normalized: list) -> None:
|
||||
"""本次条目内 tap/mac/ip 重复即 net_config_invalid。"""
|
||||
seen: dict[str, set[str]] = {"tap": set(), "mac": set(), "ip": set()}
|
||||
for entry in normalized:
|
||||
for kind in ("tap", "mac", "ip"):
|
||||
value = entry.get(kind)
|
||||
if not value:
|
||||
continue
|
||||
if value in seen[kind]:
|
||||
raise ChvFailure(
|
||||
"net_config_invalid",
|
||||
"CHV_NET_JSON 内 %s 重复:%s" % (kind, value))
|
||||
seen[kind].add(value)
|
||||
|
||||
|
||||
def _check_net_conflicts(normalized: list, existing_entries: list) -> None:
|
||||
"""与已登记 VM 的 tap/mac/ip 任一重复即 net_tap_conflict。"""
|
||||
for entry in normalized:
|
||||
for other in existing_entries:
|
||||
for kind in ("tap", "mac", "ip"):
|
||||
value = entry.get(kind)
|
||||
if value and value == other.get(kind):
|
||||
raise ChvFailure(
|
||||
"net_tap_conflict",
|
||||
"net %s %s 已被 VM %s 使用,拒绝共享"
|
||||
" tap/mac/ip:每 VM 必须独立"
|
||||
% (kind, value, other.get("vm")))
|
||||
|
||||
|
||||
def validate_egress_proxy(egress_proxy: str) -> str | None:
|
||||
"""egress 代理地址校验(H1 代码门的一部分):须为
|
||||
http/https URL 且带主机名;否则 net_config_invalid…
|
||||
实际 kind 用 config_invalid(preflight 的配置类失败)。
|
||||
空值由调用方另行处理(未设 + 未显式允许直连时
|
||||
preflight 已有 egress_not_configured)。"""
|
||||
if not egress_proxy:
|
||||
return None
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
parsed = urlparse(egress_proxy)
|
||||
except ValueError:
|
||||
parsed = None
|
||||
if parsed is None or parsed.scheme not in ("http", "https") \
|
||||
or not parsed.hostname:
|
||||
raise ChvFailure(
|
||||
"config_invalid",
|
||||
"CHV_EGRESS_PROXY 必须是 http/https 代理地址:%r"
|
||||
% (egress_proxy,))
|
||||
return egress_proxy
|
||||
|
||||
|
||||
def _is_within(directory, path):
|
||||
"""目录内判定:经 compat215.resolve_inside(#215 目录门
|
||||
同签名),越界回 None 即不在内。绝对路径先转相对路径
|
||||
再过门,.. 逃逸与符号链接逃逸一律挡。"""
|
||||
from cloud.chv.compat215 import resolve_inside
|
||||
rel = os.path.relpath(path, directory)
|
||||
return resolve_inside(directory, rel) is not None
|
||||
|
||||
|
||||
def validate_snapshot_url(snapshot_url: str, state_dir: str,
|
||||
snapshot_dir: str | None = None,
|
||||
allow_remote: bool = False) -> str | None:
|
||||
"""快照 URL 白名单(L2):
|
||||
|
||||
- file://:目标须在 state_dir 或 CHV_SNAPSHOT_DIR 内,
|
||||
否则 snapshot_path_not_allowed;
|
||||
- 其它协议(含无 :// 的裸路径由调用方先补 file://):
|
||||
须显式 CHV_ALLOW_REMOTE_SNAPSHOT=1,否则
|
||||
snapshot_remote_not_allowed。
|
||||
返回 file:// 目标的本地路径(非 file:// 时返回 None)。"""
|
||||
if snapshot_url.startswith("file://"):
|
||||
path = snapshot_url[len("file://"):]
|
||||
allowed = [state_dir]
|
||||
if snapshot_dir:
|
||||
allowed.append(snapshot_dir)
|
||||
if not any(_is_within(d, path) for d in allowed):
|
||||
raise ChvFailure(
|
||||
"snapshot_path_not_allowed",
|
||||
"file:// 快照须在 state_dir 或 CHV_SNAPSHOT_DIR "
|
||||
"内:%s" % snapshot_url)
|
||||
return path
|
||||
if not allow_remote:
|
||||
raise ChvFailure(
|
||||
"snapshot_remote_not_allowed",
|
||||
"非 file:// 快照须显式设 CHV_ALLOW_REMOTE_SNAPSHOT=1:%s"
|
||||
% snapshot_url)
|
||||
return None
|
||||
@@ -1,415 +0,0 @@
|
||||
"""按 VM 生成的 SSH host key 预置(替代 TOFU 的强方案)。
|
||||
|
||||
安全模型(补缺口轨,替代 #221 的 ssh-keyscan 首次捕获):
|
||||
- 绝不用镜像固定同一把 host key:所有 VM 共用一把 key 时,
|
||||
一台 VM 泄漏即全网可冒充,属安全事故。
|
||||
- 每 VM 一次性生成:宿主在 VMM 启动前为该 VM 生成一对
|
||||
ed25519 host key(私钥只活在该 VM 的 seed 里,随 VM
|
||||
销毁而删,绝不跨 VM 复用、绝不写进镜像、绝不进已知
|
||||
主机文件之外的共享位置)。
|
||||
- 公钥预置钉死:生成后立即把公钥写进该 VM 的
|
||||
state_dir/<vm>/known_hosts(格式 [127.0.0.1]:<port> ...),
|
||||
执行器在 VM 启动前就已钉死,启动后 SSH 直接用该文件
|
||||
校验,换 key 即拒绝——不再靠启动后 ssh-keyscan 抓到的
|
||||
key 来“事后钉死”。
|
||||
- 私钥注入通道(选 cloud-init NoCloud,写清唯一通道):
|
||||
seed 目录 state_dir/<vm>/seed/ 含 meta-data(instance-id
|
||||
即 VM 名)、user-data(write_files 写 /etc/ssh/ 的 host
|
||||
key 对并重启 sshd)。客机镜像硬性要求(见
|
||||
chv-guest/README.md §5):必须装 cloud-init 且支持
|
||||
NoCloud seed、首启时先应用 seed 再启动 sshd、且不得在
|
||||
seed 应用前/后无条件重新生成 host key(否则预置即失效)。
|
||||
seed 磁盘/目录是 per-VM 敏感物,VM 关闭/回收时必须删。
|
||||
- 回落边界:ssh-keyscan 首次捕获只作弱回落
|
||||
(CHV_HOST_KEY_MODE=keyscan,且文档标为弱方案):
|
||||
它只能防首次之后的替换,不能验证首次本身。生产部署
|
||||
必须用 provisioned 模式;provisioned 模式下 seed 工具
|
||||
或 keygen 不可用时直接 fail-fast,绝不静默回落 keyscan。
|
||||
|
||||
本模块只做纯函数与文件操作,不跑 VM、不联网、不含
|
||||
任何私钥值(测试用假 runner 时私钥由测试自己写文件)。
|
||||
"""
|
||||
|
||||
import os
|
||||
import shlex
|
||||
from collections.abc import Callable
|
||||
|
||||
from cloud.chv.atomic_file import atomic_write_text
|
||||
|
||||
# seed 投递的客内密钥环境文件路径(feat-chv-secfix,H2):
|
||||
# 只在 provisioned 模式的 seed user-data 里写出,0600 root,
|
||||
# 客内 host.py 命令执行前由远程命令 source 进环境;宿主侧
|
||||
# 除 seed 目录/user-data(随 VM 销毁而删)外不另存该文件。
|
||||
GUEST_ENV_PATH = "/etc/chv/guest-env"
|
||||
|
||||
# seed 投递的客内 authd 专用密钥文件路径(feat-chv-muse-p3):
|
||||
# 只写 AUTHD_SECRET_KEYS 命中的密钥(本期只有 GITEA_TOKEN),
|
||||
# 0600 root,只给客内 authd 守护进程读;codex 进程环境里永不
|
||||
# 出现这些键(见执行器 _authd_delivered)。与 guest-env 分离
|
||||
# 的理由:guest-env 会被远程命令 source 进任务进程环境,任何
|
||||
# 进 guest-env 的密钥都会进 codex 环境;authd 密钥绝不能走
|
||||
# 那条路。文件本身仍是 seed 写出的落盘文件(cloud-init 的唯一
|
||||
# 投递通道就是写文件),真正让它对 agent 不可见的是 P2 的
|
||||
# nspawn 单元隔离,见差距表 §3.1。
|
||||
AUTHD_SEED_PATH = "/etc/chv/authd-seed"
|
||||
|
||||
|
||||
def host_key_paths(state_dir: str, vm_name: str) -> dict:
|
||||
"""该 VM 的 host key 文件路径(私钥/公钥/known_hosts)。"""
|
||||
base = os.path.join(state_dir, vm_name)
|
||||
return {
|
||||
"private": os.path.join(base, "ssh_host_ed25519_key"),
|
||||
"public": os.path.join(base, "ssh_host_ed25519_key.pub"),
|
||||
"known_hosts": os.path.join(base, "known_hosts"),
|
||||
"seed_dir": os.path.join(base, "seed"),
|
||||
}
|
||||
|
||||
|
||||
def dirty_marker_path(state_dir: str, vm_name: str) -> str:
|
||||
"""清理失败脏标记路径(feat-chv-secfix,M2):stop_vm
|
||||
清理 seed/私钥/磁盘任一步失败时写下该标记;标记存在
|
||||
时同名 VM 拒绝启动(清理失败即拒起),人工清理并删
|
||||
标记后才能再起——“已审计即可”不再当作根治。"""
|
||||
return os.path.join(state_dir, vm_name, ".cleanup-dirty")
|
||||
|
||||
|
||||
def delete_host_private_key(state_dir: str, vm_name: str) -> str:
|
||||
"""删宿主侧 host 私钥文件(feat-chv-secfix,H5):seed
|
||||
盘建好后宿主私钥已无存在必要(客机已有私钥、宿主只需
|
||||
公钥 + known_hosts 做校验)。删失败即 OSError,由调用
|
||||
方 fail-fast(VM 尚未启动,此时拒绝是最安全的)。"""
|
||||
path = host_key_paths(state_dir, vm_name)["private"]
|
||||
if os.path.exists(path):
|
||||
os.remove(path)
|
||||
return path
|
||||
|
||||
|
||||
def known_hosts_line(port: int, public_key_text: str, host: str = "127.0.0.1") -> str:
|
||||
"""公钥文件内容转 known_hosts 行:只取前两列(类型+数据)。
|
||||
|
||||
公钥文件形如 'ssh-ed25519 AAAA... [comment]'。22 端口
|
||||
(OpenSSH 默认端口)时行形如 '<host> ssh-ed25519
|
||||
AAAA...'(裸 host 名);非 22 端口时才用
|
||||
'[<host>]:<port> ssh-ed25519 AAAA...'。OpenSSH 查
|
||||
known_hosts 时,默认端口只查裸 host 名、非默认端口
|
||||
只查括号形式,写错即永远匹配不上(2026-11-07 WSL
|
||||
真机实测:provisioned 模式钉死 key 写成
|
||||
'[192.168.10.10]:22',22 端口 ssh 报 Host key
|
||||
verification failed;keyscan TOFU 输出裸行所以
|
||||
lite9 没暴露)。host 缺省回环,仅作兼容;客机 IP
|
||||
方案下 host 应为解析出的客机 IP。非法内容
|
||||
ValueError,绝不把陌生行原样写进文件。"""
|
||||
parts = (public_key_text or "").strip().split()
|
||||
if len(parts) < 2 or not parts[0].startswith("ssh-"):
|
||||
raise ValueError("host 公钥格式非法")
|
||||
if int(port) == 22:
|
||||
return "%s %s %s" % (host, parts[0], parts[1])
|
||||
return "[%s]:%s %s %s" % (host, port, parts[0], parts[1])
|
||||
|
||||
|
||||
def seed_disk_path(state_dir: str, vm_name: str) -> str:
|
||||
"""该 VM 的 NoCloud seed 磁盘镜像路径(CIDATA 卷标 ISO)。"""
|
||||
return os.path.join(state_dir, vm_name, "seed.iso")
|
||||
|
||||
|
||||
def build_seed_disk(seed_dir: str, seed_disk: str,
|
||||
runner: Callable[[list[str]], object] | None = None) -> str:
|
||||
"""把 seed 目录做成 CIDATA 卷标 ISO(feat-chv-fix,P0-1 的
|
||||
注入载体):genisoimage -volid CIDATA -joliet -rock。
|
||||
|
||||
runner 不给时用 subprocess;测试可注入假 runner(假
|
||||
runner 负责把 seed_disk 文件写出来)。工具失败、返回码
|
||||
非 0、或产物不存在/为空时一律 RuntimeError,调用方
|
||||
(wake_vm provisioned 路径)fail-fast,绝不带着“seed
|
||||
目录已写但未挂进 VmConfig”的 VM 继续启动。seed 目录
|
||||
本身只是中间物,不再被当成注入通道。"""
|
||||
cmd = ["genisoimage", "-output", seed_disk, "-volid", "CIDATA",
|
||||
"-joliet", "-rock", seed_dir]
|
||||
if runner is not None:
|
||||
result = runner(cmd)
|
||||
rc = getattr(result, "returncode", 0)
|
||||
if rc not in (0, None):
|
||||
raise RuntimeError("genisoimage 失败:returncode=%s" % rc)
|
||||
else:
|
||||
import subprocess
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError("genisoimage 失败:%s" % result.stderr)
|
||||
if not os.path.isfile(seed_disk) or os.path.getsize(seed_disk) == 0:
|
||||
raise RuntimeError("seed 磁盘未生成:%s" % seed_disk)
|
||||
# feat-chv-secfix(H5):seed.iso 内含 host 私钥明文
|
||||
# user-data,生成即 0600,不再取 umask 默认 0644。
|
||||
try:
|
||||
os.chmod(seed_disk, 0o600)
|
||||
except OSError as exc:
|
||||
raise RuntimeError("seed 磁盘 chmod 0600 失败:%s" % exc)
|
||||
return seed_disk
|
||||
|
||||
|
||||
def build_seed_user_data(private_key_text: str, public_key_text: str) -> str:
|
||||
"""cloud-init user-data:把 host key 对写进客机 /etc/ssh。
|
||||
|
||||
注意:user-data 本身含私钥,只写进 per-VM seed 目录
|
||||
(0600),随 VM 销毁而删,绝不进镜像、绝不进日志、
|
||||
绝不进任务 prompt 落库。
|
||||
|
||||
根因禁令(sync-test 残留,2026-11-06 实测):seed 的
|
||||
user-data 里绝不能带 `runcmd:`。cloud-init 不跨配置
|
||||
类型合并——user-data 的 `runcmd` 会整体替换掉镜像
|
||||
`/etc/cloud/cloud.cfg.d/` 里的 `runcmd`(`merge_how`
|
||||
只合并多个 user-data 段,救不了 conf.d,见 cloud-init
|
||||
合并文档);旧版 seed 自带的 chmod + restart ssh 两行
|
||||
就这样顶掉了镜像 `99-chv-sync.cfg` 的 `chv-sync boot`
|
||||
钩子,首轮启动 clone 静默不跑。去掉的两行不需要
|
||||
补在 seed 里:chmod 600 已由 write_files 的
|
||||
permissions '0600' 保证;sshd 重启已搬到镜像自己的
|
||||
runcmd(`99-chv-sync.cfg`,cloud-final 阶段,sshd
|
||||
首次启动之后),那条是存活的列表,不会被 seed 覆盖。
|
||||
任何人再给 seed 加 runcmd 即重犯此 bug,回归测试
|
||||
`test_seed_user_data_has_no_runcmd` 会拦住。"""
|
||||
if not private_key_text or not public_key_text:
|
||||
raise ValueError("host key 对缺失,无法生成 seed")
|
||||
lines = [
|
||||
"#cloud-config",
|
||||
"ssh_deletekeys: false",
|
||||
"write_files:",
|
||||
" - path: /etc/ssh/ssh_host_ed25519_key",
|
||||
" permissions: '0600'",
|
||||
" owner: root:root",
|
||||
" content: |",
|
||||
]
|
||||
for line in private_key_text.rstrip("\n").split("\n"):
|
||||
lines.append(" " + line)
|
||||
lines += [
|
||||
" - path: /etc/ssh/ssh_host_ed25519_key.pub",
|
||||
" permissions: '0644'",
|
||||
" owner: root:root",
|
||||
" content: |",
|
||||
]
|
||||
for line in public_key_text.rstrip("\n").split("\n"):
|
||||
lines.append(" " + line)
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def _authd_env_entry(authd_env: dict) -> list:
|
||||
"""authd-seed 的 write_files 条目行(0600 root,值经
|
||||
shlex.quote 引用):装 AUTHD_SECRET_KEYS 命中的密钥
|
||||
(本期只有 GITEA_TOKEN),外加与 guest-env 同源抄进来的
|
||||
三个非密钥同步变量(GITEA_URL/GITEA_USER/SYNC_REPO——
|
||||
round-11 真机修复层 2:authd 的 build_server 要求 seed
|
||||
里同时有这三键,只 GITEA_TOKEN 会 ValueError;探针
|
||||
fixprobe3 即从 guest-env 补这三键)。三键只在调用方
|
||||
guest-env 里有才抄,没有不补、不编默认值;与 guest-env
|
||||
条目并列写进同一 write_files 列表(绝不另写一个
|
||||
write_files: 键,见 _guest_env_entry 的 YAML 重复键禁令)。
|
||||
"""
|
||||
lines = [
|
||||
" - path: %s" % AUTHD_SEED_PATH,
|
||||
" permissions: '0600'",
|
||||
" owner: root:root",
|
||||
" content: |",
|
||||
]
|
||||
for key in sorted(authd_env):
|
||||
lines.append(" %s=%s"
|
||||
% (key, shlex.quote(str(authd_env[key]))))
|
||||
return lines
|
||||
|
||||
|
||||
def _guest_env_entry(merged_env: dict) -> list:
|
||||
"""guest-env 的 write_files 条目行(0600 root,值经
|
||||
shlex.quote 引用):只能并进已有的 write_files 列表,
|
||||
绝不能再写一个 write_files: 键(YAML 重复键会覆盖掉
|
||||
host key 的两条写文件条目,预置即失效)。
|
||||
owner 恒为 root:root(V-9 真机根因:cloud-init 的 config
|
||||
阶段里 write_files 在 users-groups 之前跑,owner 写
|
||||
chv:chv 时 getpwnam('chv') 尚不存在,整个 write_files
|
||||
模块失败、guest-env 文件根本不产生)。文件改归 SSH 用户
|
||||
由客内 final 阶段 99-chv-sync.cfg runcmd 在 chv-sync
|
||||
boot 之前 chown(用户那时已建好),见 chv-guest/build.sh。"""
|
||||
lines = [
|
||||
" - path: %s" % GUEST_ENV_PATH,
|
||||
" permissions: '0600'",
|
||||
" owner: root:root",
|
||||
" content: |",
|
||||
]
|
||||
for key in sorted(merged_env):
|
||||
lines.append(" %s=%s"
|
||||
% (key, shlex.quote(str(merged_env[key]))))
|
||||
return lines
|
||||
|
||||
|
||||
def _apply_seed_extras(seed_dir: str, user_data: str,
|
||||
user_data_extra: str | None,
|
||||
network_config: str | None) -> str:
|
||||
"""seed 可选附加项(执行器 wake 路径的客机身份/网络供给):
|
||||
user_data_extra 原样追加进 user-data(调用方用它供
|
||||
`users:` 块——chv 用户 + authorized_keys,镜像本身不
|
||||
建用户);network_config 写成 seed 目录的 network-config
|
||||
文件(NoCloud 静态 IP,tap 无 DHCP 时客机才有地址)。
|
||||
两者皆空即原样返回。user_data_extra 里含 runcmd: 行即
|
||||
ValueError——seed 禁 runcmd 是 2026-11-06 根因禁令,
|
||||
附加项也不得绕过(镜像的 99-chv-sync.cfg 靠它活着)。"""
|
||||
if user_data_extra:
|
||||
for line in user_data_extra.splitlines():
|
||||
if line.strip().startswith("runcmd:"):
|
||||
raise ValueError(
|
||||
"seed 附加 user-data 里不得带 runcmd:")
|
||||
if not user_data.endswith("\n"):
|
||||
user_data += "\n"
|
||||
user_data += user_data_extra
|
||||
if not user_data.endswith("\n"):
|
||||
user_data += "\n"
|
||||
if network_config:
|
||||
net_path = os.path.join(seed_dir, "network-config")
|
||||
# feat-chv-cover100:走 atomic_file(与 user-data 同款
|
||||
# 理由,见 write_seed)——直接写 + 事后 chmod 会在
|
||||
# chmod 前以 umask 权限短暂可读。
|
||||
atomic_write_text(net_path, network_config)
|
||||
return user_data
|
||||
|
||||
|
||||
def _validate_authd_env(authd_env: dict | None) -> None:
|
||||
"""authd_env 的键必须全在 AUTHD_SECRET_KEYS 内(本期只有
|
||||
GITEA_TOKEN),否则 ValueError——authd 通道只装 authd
|
||||
密钥,别的密钥误进这里会绕过 guest-env 的 source 语义。"""
|
||||
if not authd_env:
|
||||
return
|
||||
from cloud.chv.gitea_sync import AUTHD_SECRET_KEYS
|
||||
unknown = sorted(set(authd_env) - AUTHD_SECRET_KEYS)
|
||||
if unknown:
|
||||
raise ValueError(
|
||||
"authd 通道只接受 %s,拒绝:%s"
|
||||
% (",".join(sorted(AUTHD_SECRET_KEYS)),
|
||||
",".join(unknown)))
|
||||
|
||||
|
||||
def _write_seed_meta(seed_dir: str, vm_name: str) -> str:
|
||||
"""建 seed 目录并写 meta-data(0600),返回 user-data 路径。
|
||||
feat-chv-cover100:seed 暂存文件走 atomic_file(mkstemp
|
||||
0600 → 写 → fsync → rename):user-data 含 host 私钥
|
||||
明文,直接写 + 事后 chmod 会在 chmod 前以 umask 权限
|
||||
短暂可读;半截文件到不了 ISO(见 H5),原子写同时堵
|
||||
住权限窗口与半截残留。"""
|
||||
os.makedirs(seed_dir, exist_ok=True)
|
||||
meta_path = os.path.join(seed_dir, "meta-data")
|
||||
user_path = os.path.join(seed_dir, "user-data")
|
||||
atomic_write_text(
|
||||
meta_path,
|
||||
"instance-id: %s\nlocal-hostname: %s\n" % (vm_name, vm_name))
|
||||
return user_path
|
||||
|
||||
|
||||
def _seed_env_entries(merged_env: dict,
|
||||
authd_env: dict | None) -> list:
|
||||
"""guest-env 与 authd-seed 两路的 write_files 条目行
|
||||
(皆空即空列表,不写条目)。authd-seed 除了密钥还抄
|
||||
guest-env 里的 GITEA_URL/GITEA_USER/SYNC_REPO(层 2,
|
||||
见 _authd_env_entry)——build_server 要求四键齐全。"""
|
||||
extra_lines = []
|
||||
if merged_env:
|
||||
extra_lines += _guest_env_entry(merged_env)
|
||||
if authd_env:
|
||||
seed_env = dict(authd_env)
|
||||
for key in ("GITEA_URL", "GITEA_USER", "SYNC_REPO"):
|
||||
if key in merged_env:
|
||||
seed_env[key] = merged_env[key]
|
||||
extra_lines += _authd_env_entry(seed_env)
|
||||
return extra_lines
|
||||
|
||||
|
||||
def _insert_seed_entries(user_data: str, lines: list) -> str:
|
||||
"""条目插在 runcmd 段之前、仍属同一 write_files 列表
|
||||
(seed 自 2026-11-07 起不再带 runcmd,见
|
||||
build_seed_user_data 的根因禁令;marker 分支只作
|
||||
防御,将来模板若恢复 runcmd 仍插在其前)。"""
|
||||
marker = "runcmd:\n"
|
||||
if marker in user_data:
|
||||
head, tail = user_data.split(marker, 1)
|
||||
return head + "\n".join(lines) + "\n" + marker + tail
|
||||
return user_data + "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def write_seed(seed_dir: str, vm_name: str, private_key_text: str,
|
||||
public_key_text: str,
|
||||
secret_env: dict | None = None,
|
||||
guest_env: dict | None = None,
|
||||
user_data_extra: str | None = None,
|
||||
network_config: str | None = None,
|
||||
authd_env: dict | None = None) -> str:
|
||||
"""写 seed 目录:meta-data + user-data(均为 0600),
|
||||
返回 seed 目录路径。调用方负责随 VM 销毁删掉它。
|
||||
|
||||
secret_env 给定时写进客内 GUEST_ENV_PATH(0600 root,
|
||||
远程命令 source 它得密钥环境,值不进宿主 argv/环境/
|
||||
审计明文)。guest_env 给定时与 secret_env 合并写同一
|
||||
个条目(secret 优先于同名键,防两行同名);guest_env
|
||||
装 boot 时要用的非密钥同步变量(GITEA_URL 等,见
|
||||
gitea_sync):只走 SSH 前缀到不了 cloud-init 时运行
|
||||
的 chv-sync boot。两者皆空时不写该条目。
|
||||
authd_env(feat-chv-muse-p3)给定时写进另一个独立条目
|
||||
AUTHD_SEED_PATH(0600 root,只给客内 authd 守护进程读):
|
||||
键必须全在 gitea_sync.AUTHD_SECRET_KEYS 内(本期只有
|
||||
GITEA_TOKEN),否则 ValueError——authd 通道只装 authd
|
||||
密钥,别的密钥误进这里会绕过 guest-env 的 source 语义。
|
||||
user_data_extra / network_config 见 _apply_seed_extras
|
||||
(客机用户与静态网络的供给,皆空即不写)。"""
|
||||
_validate_authd_env(authd_env)
|
||||
user_path = _write_seed_meta(seed_dir, vm_name)
|
||||
user_data = build_seed_user_data(private_key_text, public_key_text)
|
||||
merged_env = dict(guest_env or {})
|
||||
merged_env.update(secret_env or {})
|
||||
# feat-chv-muse-p3:GITEA_TOKEN 绝不进 guest-env——它只能走
|
||||
# authd_env(独立的 authd-seed 条目)。调用方(执行器)须先
|
||||
# 用 gitea_sync.split_authd_env() 拆出来;这里再拦一次,
|
||||
# 免得别的调用点把旧写法带进来、悄悄恢复泄露面。
|
||||
if "GITEA_TOKEN" in merged_env:
|
||||
raise ValueError(
|
||||
"GITEA_TOKEN 须走 authd_env,不进 guest-env")
|
||||
extra_lines = _seed_env_entries(merged_env, authd_env)
|
||||
if extra_lines:
|
||||
user_data = _insert_seed_entries(user_data, extra_lines)
|
||||
user_data = _apply_seed_extras(seed_dir, user_data,
|
||||
user_data_extra, network_config)
|
||||
atomic_write_text(user_path, user_data)
|
||||
return seed_dir
|
||||
|
||||
|
||||
def generate_host_keypair(state_dir: str, vm_name: str,
|
||||
runner: Callable[[list[str]], object] | None = None,
|
||||
keygen: str = "ssh-keygen") -> dict:
|
||||
"""为该 VM 生成(或复用本 VM 已有的)host key 对。
|
||||
|
||||
同一 VM 重启/恢复时复用已有对(否则钉死的 known_hosts
|
||||
立即失效);不同 VM 的 state_dir/<vm> 不同,绝不共用。
|
||||
runner 不给时用 subprocess,测试可注入假 runner(假
|
||||
runner 负责把私钥/公钥文件写出来)。keygen 失败即
|
||||
RuntimeError,调用方不得回落 keyscan(provisioned
|
||||
模式)或必须显式标弱回落(keyscan 模式,见执行器)。"""
|
||||
paths = host_key_paths(state_dir, vm_name)
|
||||
if (os.path.exists(paths["private"])
|
||||
and os.path.exists(paths["public"])):
|
||||
return paths
|
||||
base = os.path.dirname(paths["private"])
|
||||
os.makedirs(base, exist_ok=True)
|
||||
cmd = [keygen, "-t", "ed25519", "-N", "",
|
||||
"-f", paths["private"]]
|
||||
if runner is not None:
|
||||
result = runner(cmd)
|
||||
rc = getattr(result, "returncode", 0)
|
||||
if rc not in (0, None):
|
||||
raise RuntimeError("ssh-keygen 失败:returncode=%s" % rc)
|
||||
else:
|
||||
import subprocess
|
||||
result = subprocess.run(cmd, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
raise RuntimeError("ssh-keygen 失败:%s" % result.stderr)
|
||||
if not (os.path.exists(paths["private"])
|
||||
and os.path.exists(paths["public"])):
|
||||
raise RuntimeError("ssh-keygen 未生成 host key 对")
|
||||
try:
|
||||
os.chmod(paths["private"], 0o600)
|
||||
except OSError as exc:
|
||||
# feat-chv-secfix(H5):私钥 chmod 失败不再静默
|
||||
# pass——权限不对的私钥文件绝不能留着继续用。
|
||||
raise RuntimeError("host 私钥 chmod 0600 失败:%s" % exc)
|
||||
return paths
|
||||
@@ -1,247 +0,0 @@
|
||||
"""受限推理通道(feat-chv-muse-p9,叶子模块)。
|
||||
|
||||
对齐 Muse 官方安全文(security.muse.ai 2026-09-08,见
|
||||
docs/chv-muse-p9-gap.md):推理与遥测代理是通向外部基础设施的
|
||||
受限通道;Sentinel 是连接器动作与全部网络出口的唯一权限机构。
|
||||
|
||||
本模块是**决策层**,不是网络层强制:
|
||||
- 它不拦截客内流量。客内 codex 直连的强制仍靠 Tinyproxy 白名单 +
|
||||
宿主防火墙(第三棒落地中);不要把本模块写成“出口已全部经
|
||||
Sentinel”——与 P3 的 sentinel 模块同款声明。
|
||||
- 唯一出口点:模型调用只允许走这里决定的 origin。执行器在
|
||||
`_run_with_vm` 起 VM 前调 `check_inference_call`:allow 才继续;
|
||||
deny/ask 一律审计 + 停止,不起 VM。
|
||||
- provider=`sub2api` 时 origin 取 `SUB2API_BASE_URL`;provider 为空
|
||||
(xcmapi 路径,codex 用自身 config.toml 默认网关)时执行器看不
|
||||
到 origin,改取 operator 显式配置的 `CHV_INFERENCE_ORIGIN`,未配
|
||||
即 ask(fail-closed,不静默放行)。部署迁移见差距表 §2.5。
|
||||
- 能力说明:allow 时带出的严格能力(单次、300 秒)是**起 VM 前的
|
||||
预审授权**,不是逐次 HTTP 请求的令牌;客内 codex 的逐次调用仍由
|
||||
代理白名单 + 防火墙约束。本模块不伪装成逐请求强制。
|
||||
|
||||
本模块只用标准库,不依赖 cloud.*(除 vendored 的 sentinel),
|
||||
解析器可注入(`resolve` 参数),默认 `socket.getaddrinfo`;测试
|
||||
用假解析器,绝不联网。审计记录里绝不含凭证值。
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from typing import NamedTuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from cloud.chv.sentinel import (
|
||||
Decision,
|
||||
EgressRequest,
|
||||
SentinelPolicy,
|
||||
_one_shot_capability,
|
||||
)
|
||||
|
||||
INFERENCE_CONNECTOR = "inference"
|
||||
|
||||
# 推理连接器的 L7 路径规则:codex 以 wire_api="responses" 调模型,
|
||||
# 只放 POST /v1/responses;GET /v1/models 是否必需待真机核实
|
||||
# (codex 启动时是否拉模型列表),若非必需则收紧为只放前者。
|
||||
INFERENCE_PATH_RULES = (
|
||||
("POST", "prefix", "/v1/responses"),
|
||||
("GET", "prefix", "/v1/models"),
|
||||
)
|
||||
|
||||
# 上限默认值(本项目自定,非 Meta 官方值):超时 120 秒与
|
||||
# host.py 的 Server.request 默认超时一致;重试 1 次即保持
|
||||
# 现有行为不变(现为 range(2))。非法值一律 ValueError。
|
||||
DEFAULT_INFERENCE_TIMEOUT_SECONDS = 120
|
||||
DEFAULT_INFERENCE_MAX_RETRIES = 1
|
||||
MIN_INFERENCE_TIMEOUT_SECONDS = 10
|
||||
MAX_INFERENCE_TIMEOUT_SECONDS = 600
|
||||
MAX_INFERENCE_RETRIES = 5
|
||||
|
||||
|
||||
class InferenceLimits(NamedTuple):
|
||||
"""推理通道上限:单次请求超时(秒)与重连重试上限(次)。"""
|
||||
|
||||
timeout_seconds: int
|
||||
max_retries: int
|
||||
|
||||
|
||||
class InferenceCheck(NamedTuple):
|
||||
"""一次预审的结果:决策 + 被评估的请求(origin 缺失/非法时
|
||||
request 为 None,审计方用原因兜底)。"""
|
||||
|
||||
decision: Decision
|
||||
request: EgressRequest | None
|
||||
|
||||
|
||||
def _env_of(env: dict | None) -> dict:
|
||||
"""测试传 dict,生产默认读 os.environ。"""
|
||||
return env if env is not None else os.environ # type: ignore[return-value]
|
||||
|
||||
|
||||
def _is_local_relay_host(host: str) -> bool:
|
||||
"""字面环回/私网 IP 才是本地中继主机(无 DNS,无重绑定;
|
||||
主机名一律不是——DNS 解析结果不可信)。"""
|
||||
try:
|
||||
addr = ipaddress.ip_address(host)
|
||||
except ValueError:
|
||||
return False
|
||||
return addr.is_loopback or addr.is_private
|
||||
|
||||
|
||||
def resolve_inference_origin(provider: str | None,
|
||||
env: dict | None = None) -> str:
|
||||
"""解析推理出口 origin(不带尾斜杠,端口非默认时保留)。
|
||||
|
||||
- provider="sub2api":取 `SUB2API_BASE_URL`;
|
||||
- provider 为空或未知标签:取 `CHV_INFERENCE_ORIGIN`(与
|
||||
host.py `_server_command` 一致走默认网关;标签本身不决定
|
||||
origin,origin 永远只取 operator 配置的 env);
|
||||
变量未配/空白返回 ""(调用方按 ask 处理,不在这里抛错)。
|
||||
scheme 规则(见 cloud/README.md 的 sub2api 网关一节):
|
||||
- https:公网网关(如 xgapi.lol),端口只接受 443(或缺省);
|
||||
- http:只接受本地中继——字面环回/私网 IP 主机(如文档形状
|
||||
http://172.17.0.1:28084,Docker 桥接 IP),端口保留;
|
||||
- 其余(http 公网主机名、https 非 443 端口、带 userinfo、
|
||||
无 host)一律 ValueError——配置错了要 loud,不要静默跑向别处。
|
||||
报错不带 origin 原值(http URL 里可能含 userinfo)。
|
||||
"""
|
||||
values = _env_of(env)
|
||||
if provider is None:
|
||||
raw = values.get("CHV_INFERENCE_ORIGIN") or ""
|
||||
elif provider == "sub2api":
|
||||
raw = values.get("SUB2API_BASE_URL") or ""
|
||||
else:
|
||||
# 未知 provider 标签:与 host.py `_server_command` 一致走
|
||||
# 默认网关(`CHV_INFERENCE_ORIGIN`)——标签本身不决定
|
||||
# origin,origin 永远只取 operator 配置的 env。
|
||||
raw = values.get("CHV_INFERENCE_ORIGIN") or ""
|
||||
raw = raw.strip().rstrip("/")
|
||||
if not raw:
|
||||
return ""
|
||||
parsed = urlparse(raw)
|
||||
scheme = parsed.scheme.lower()
|
||||
if "@" in parsed.netloc:
|
||||
raise ValueError("推理 origin 不得带 userinfo(凭证禁进 URL)")
|
||||
host = (parsed.hostname or "").lower()
|
||||
if not host:
|
||||
raise ValueError("推理 origin 无主机名")
|
||||
if scheme == "https":
|
||||
if parsed.port is not None and parsed.port != 443:
|
||||
raise ValueError("推理 origin 端口只接受 443")
|
||||
return "https://" + host
|
||||
if scheme == "http":
|
||||
if not _is_local_relay_host(host):
|
||||
raise ValueError("http 推理 origin 只接受本地中继"
|
||||
"(环回/私网 IP)")
|
||||
port = parsed.port
|
||||
if port is not None:
|
||||
return "http://%s:%d" % (host, port)
|
||||
return "http://" + host
|
||||
raise ValueError("推理 origin 必须 https(或本地中继 http)")
|
||||
|
||||
|
||||
def inference_allow_hosts(provider: str | None,
|
||||
env: dict | None = None) -> frozenset:
|
||||
"""推理出口 allowlist(小写主机名集合)。
|
||||
|
||||
`CHV_INFERENCE_ALLOW_HOSTS`(逗号分隔)显式覆盖;未配时默认
|
||||
即 origin 自身的主机——allowlist 永远是“显式配置的 origin”,
|
||||
不是“解析后是什么都行”。origin 缺失时返回空集(预审必 ask)。
|
||||
"""
|
||||
values = _env_of(env)
|
||||
raw = values.get("CHV_INFERENCE_ALLOW_HOSTS") or ""
|
||||
hosts = frozenset(h.strip().lower()
|
||||
for h in raw.split(",") if h.strip())
|
||||
if hosts:
|
||||
return hosts
|
||||
try:
|
||||
origin = resolve_inference_origin(provider, env)
|
||||
except ValueError:
|
||||
return frozenset()
|
||||
if not origin:
|
||||
return frozenset()
|
||||
host = urlparse(origin).hostname or ""
|
||||
return frozenset({host}) if host else frozenset()
|
||||
|
||||
|
||||
def _parse_int_env(env: dict, name: str, default: int,
|
||||
low: int, high: int) -> int:
|
||||
"""严格整数 env 解析:未配即默认;非整数/越界即 ValueError,
|
||||
绝不静默回落(配错早炸,不留到起 VM 才暴露)。"""
|
||||
raw = env.get(name)
|
||||
if raw is None or str(raw).strip() == "":
|
||||
return default
|
||||
try:
|
||||
value = int(str(raw).strip())
|
||||
except ValueError:
|
||||
raise ValueError("%s 必须是整数: %r" % (name, raw))
|
||||
if not low <= value <= high:
|
||||
raise ValueError("%s 必须在 %d-%d 之间: %s"
|
||||
% (name, low, high, value))
|
||||
return value
|
||||
|
||||
|
||||
def inference_limits(env: dict | None = None) -> InferenceLimits:
|
||||
"""推理通道上限(见模块头默认值说明)。"""
|
||||
values = _env_of(env)
|
||||
return InferenceLimits(
|
||||
timeout_seconds=_parse_int_env(
|
||||
values, "CHV_INFERENCE_TIMEOUT",
|
||||
DEFAULT_INFERENCE_TIMEOUT_SECONDS,
|
||||
MIN_INFERENCE_TIMEOUT_SECONDS,
|
||||
MAX_INFERENCE_TIMEOUT_SECONDS),
|
||||
max_retries=_parse_int_env(
|
||||
values, "CHV_INFERENCE_MAX_RETRIES",
|
||||
DEFAULT_INFERENCE_MAX_RETRIES, 0, MAX_INFERENCE_RETRIES),
|
||||
)
|
||||
|
||||
|
||||
def check_inference_call(provider: str | None,
|
||||
env: dict | None = None,
|
||||
resolve: Callable[[str, int], list] | None = None,
|
||||
policy: SentinelPolicy | None = None,
|
||||
now: float | None = None) -> InferenceCheck:
|
||||
"""预审一次推理出口调用(起 VM 前,fail-closed)。
|
||||
|
||||
- origin 缺失 → ask(`inference_origin_not_configured`),
|
||||
调用方停止执行,不得自动放行;
|
||||
- origin 非法 → deny;
|
||||
- http 本地中继(字面环回/私网 IP):无 DNS、无重绑定,跳过
|
||||
Sentinel 的 DNS/SSRF 步;allowlist 与单次能力照常执行;
|
||||
- 其余走 Sentinel 评估:SSRF、私网 IP、scheme/端口/路径越界
|
||||
即 deny;主机不在白名单即 ask;全过即 allow + 严格能力
|
||||
(单次、300 秒,见模块头的能力说明)。
|
||||
`resolve` / `policy` 可注入(假解析器 / P5 纵深策略),默认按 allowlist 现建。
|
||||
"""
|
||||
try:
|
||||
origin = resolve_inference_origin(provider, env)
|
||||
except ValueError:
|
||||
# 原因不带原值:非法 origin 的原串里可能含 userinfo,
|
||||
# 审计原因只记“非法”本身,不记值。
|
||||
return InferenceCheck(
|
||||
Decision("deny", ["inference_origin_invalid"]), None)
|
||||
if not origin:
|
||||
return InferenceCheck(
|
||||
Decision("ask", ["inference_origin_not_configured"]), None)
|
||||
parsed = urlparse(origin)
|
||||
scheme = parsed.scheme
|
||||
host = parsed.hostname or ""
|
||||
port = parsed.port or (443 if scheme == "https" else 80)
|
||||
if policy is None:
|
||||
policy = SentinelPolicy(
|
||||
allow_hosts=inference_allow_hosts(provider, env),
|
||||
connector_paths={INFERENCE_CONNECTOR: INFERENCE_PATH_RULES})
|
||||
request = EgressRequest(INFERENCE_CONNECTOR, "POST", scheme,
|
||||
host, port, "/v1/responses")
|
||||
if scheme == "http":
|
||||
# 本地中继分支:见上。allowlist 不含该主机 → ask;
|
||||
# 含 → allow + 与公网路径同款单次能力(P3 默认档)。
|
||||
if host not in policy.allow_hosts:
|
||||
return InferenceCheck(
|
||||
Decision("ask", ["host_not_allowlisted"]), request)
|
||||
rules = policy.connector_paths.get(INFERENCE_CONNECTOR, ())
|
||||
capability = _one_shot_capability(request, rules, now)
|
||||
return InferenceCheck(
|
||||
Decision("allow", ["policy_allow", "local_relay"],
|
||||
capability=capability), request)
|
||||
decision = policy.evaluate(request, resolve=resolve, now=now)
|
||||
return InferenceCheck(decision, request)
|
||||
@@ -1,349 +0,0 @@
|
||||
"""客机记忆文件接线(对齐 Muse P7,叶子模块)。
|
||||
|
||||
Muse 官方要求(security.muse.ai / 设计文 2026-09-08,见
|
||||
docs/chv-muse-p7-gap.md §1):记忆跨对话持续存在;记忆文件可
|
||||
直接阅读与编辑;用户可随时让 Muse“忘记”指定内容;VM 数据持续
|
||||
备份以便恢复;VM 是用户放入 Muse 的一切内容的系统记录。
|
||||
|
||||
本模块只做接线,不重造记忆引擎:
|
||||
- 记忆引擎 = 记忆系统 v13(~/workspace/memory-system),经真实
|
||||
CLI 调用(`node dist/cli.js search|get|explain|forget-plan|
|
||||
forget-confirm|backup|restore|export`)。CLI 不可用时降级为本地
|
||||
Markdown 模式,bank 忘记明确记为待办,不谎称已忘。
|
||||
- 记忆文件布局与记忆系统同款:家目录下 MEMORY.md + memory/*.md;
|
||||
宿主侧镜像在 `state_dir/<vm>/memory-home/`,客内家目录在
|
||||
`/home/chv/memory`(workspace 之外,不同步到 Gitea)。
|
||||
- “忘记”是尽力删除 + 验证查不到,不是密码学擦除:已备份的快照
|
||||
里仍有旧内容(与记忆系统 CLI 的 backup 语义一致),差距表 §5
|
||||
写清。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
from collections.abc import Mapping
|
||||
|
||||
from cloud.chv.app_state import assert_no_secrets
|
||||
|
||||
GUEST_MEMORY_HOME = "/home/chv/memory"
|
||||
_MEMORY_MD = "MEMORY.md"
|
||||
|
||||
|
||||
def memory_home_path(state_dir: str, vm_name: str) -> str:
|
||||
"""某 VM 记忆家目录镜像路径:宿主 state_dir 下,不在客机磁盘里。"""
|
||||
return os.path.join(state_dir, vm_name, "memory-home")
|
||||
_MEMORY_SUBDIR = "memory"
|
||||
_AUDIT_FILENAME = "audit.jsonl"
|
||||
_CLI_TIMEOUT_SECONDS = 120
|
||||
|
||||
|
||||
class MemoryError(RuntimeError):
|
||||
"""记忆文件操作失败(路径穿越、CLI 失败等)。"""
|
||||
|
||||
|
||||
def _is_within(base: str, path: str) -> bool:
|
||||
"""path 是否在家目录内(防路径穿越)。"""
|
||||
base_abs = os.path.abspath(base)
|
||||
path_abs = os.path.abspath(path)
|
||||
return path_abs == base_abs or path_abs.startswith(base_abs + os.sep)
|
||||
|
||||
|
||||
def _remove_lines_containing(path: str, needle: str) -> int:
|
||||
"""删掉文件里含 needle 的行(大小写不敏感),返回删掉的行数。"""
|
||||
if not os.path.isfile(path):
|
||||
return 0
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
lines = handle.readlines()
|
||||
kept = [line for line in lines if needle.lower() not in line.lower()]
|
||||
removed = len(lines) - len(kept)
|
||||
if removed:
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
handle.writelines(kept)
|
||||
return removed
|
||||
|
||||
|
||||
def _audit_line(actor: str, action: str, target: str,
|
||||
detail: Mapping[str, object]) -> str:
|
||||
"""一条审计 JSON 行(detail 过无凭证守卫)。"""
|
||||
assert_no_secrets(detail)
|
||||
record = {"actor": actor, "action": action, "target": target,
|
||||
"detail": dict(detail)}
|
||||
return json.dumps(record, sort_keys=True) + "\n"
|
||||
|
||||
|
||||
class MemoryFiles:
|
||||
"""宿主侧记忆家目录镜像:读/列/写/追加,写操作记审计。"""
|
||||
|
||||
def __init__(self, home: str) -> None:
|
||||
self._home = home
|
||||
self._lock = threading.RLock()
|
||||
|
||||
@property
|
||||
def home(self) -> str:
|
||||
"""镜像家目录路径。"""
|
||||
return self._home
|
||||
|
||||
def ensure(self) -> None:
|
||||
"""建家目录结构(MEMORY.md + memory/,0700)。"""
|
||||
os.makedirs(os.path.join(self._home, _MEMORY_SUBDIR),
|
||||
mode=0o700, exist_ok=True)
|
||||
memory_md = os.path.join(self._home, _MEMORY_MD)
|
||||
if not os.path.exists(memory_md):
|
||||
with open(memory_md, "w", encoding="utf-8") as handle:
|
||||
handle.write("# MEMORY.md\n")
|
||||
os.chmod(memory_md, 0o600)
|
||||
os.chmod(self._home, 0o700)
|
||||
|
||||
def resolve(self, rel: str) -> str:
|
||||
"""相对路径 → 家目录内绝对路径,穿越即拒。"""
|
||||
candidate = os.path.abspath(os.path.join(self._home, rel))
|
||||
if not _is_within(self._home, candidate):
|
||||
raise MemoryError("path escapes memory home: %s" % rel)
|
||||
return candidate
|
||||
|
||||
def list_files(self) -> list[str]:
|
||||
"""列家目录下全部文件(相对路径,排序)。"""
|
||||
found: list[str] = []
|
||||
for root, _dirs, names in os.walk(self._home):
|
||||
for name in names:
|
||||
full = os.path.join(root, name)
|
||||
found.append(os.path.relpath(full, self._home))
|
||||
return sorted(found)
|
||||
|
||||
def read_file(self, rel: str) -> str:
|
||||
"""读一个记忆文件(相对路径),不存在即报错。"""
|
||||
path = self.resolve(rel)
|
||||
if not os.path.isfile(path):
|
||||
raise MemoryError("memory file missing: %s" % rel)
|
||||
with open(path, "r", encoding="utf-8") as handle:
|
||||
return handle.read()
|
||||
|
||||
def write_file(self, rel: str, text: str, actor: str = "host") -> None:
|
||||
"""写一个记忆文件(相对路径,0600),记审计。"""
|
||||
assert_no_secrets({"text": text})
|
||||
path = self.resolve(rel)
|
||||
parent = os.path.dirname(path)
|
||||
if parent:
|
||||
os.makedirs(parent, mode=0o700, exist_ok=True)
|
||||
with self._lock:
|
||||
with open(path, "w", encoding="utf-8") as handle:
|
||||
handle.write(text)
|
||||
os.chmod(path, 0o600)
|
||||
self._append_audit(actor, "write", rel, {"bytes": len(text)})
|
||||
|
||||
def append_lines(self, rel: str, lines: list[str],
|
||||
actor: str = "host") -> None:
|
||||
"""向一个记忆文件追加行(0600),记审计。"""
|
||||
assert_no_secrets({"lines": lines})
|
||||
path = self.resolve(rel)
|
||||
parent = os.path.dirname(path)
|
||||
if parent:
|
||||
os.makedirs(parent, mode=0o700, exist_ok=True)
|
||||
with self._lock:
|
||||
with open(path, "a", encoding="utf-8") as handle:
|
||||
for line in lines:
|
||||
handle.write(line if line.endswith("\n") else line + "\n")
|
||||
os.chmod(path, 0o600)
|
||||
self._append_audit(actor, "append", rel,
|
||||
{"lines": len(lines)})
|
||||
|
||||
def remove_lines(self, rel: str, needle: str,
|
||||
actor: str = "host") -> int:
|
||||
"""删掉一个记忆文件里含 needle 的行,返回行数,记审计。"""
|
||||
path = self.resolve(rel)
|
||||
with self._lock:
|
||||
removed = _remove_lines_containing(path, needle)
|
||||
if removed:
|
||||
self._append_audit(actor, "forget-lines", rel,
|
||||
{"needle": needle, "removed": removed})
|
||||
return removed
|
||||
|
||||
def _append_audit(self, actor: str, action: str, target: str,
|
||||
detail: Mapping[str, object]) -> None:
|
||||
"""审计追加到家目录 memory/audit.jsonl(O_APPEND)。"""
|
||||
audit_path = os.path.join(self._home, _MEMORY_SUBDIR,
|
||||
_AUDIT_FILENAME)
|
||||
with open(audit_path, "a", encoding="utf-8") as handle:
|
||||
handle.write(_audit_line(actor, action, target, detail))
|
||||
|
||||
|
||||
class MemorySystemClient:
|
||||
"""记忆系统 CLI 客户端(subprocess 调真实 CLI,不重造引擎)。
|
||||
|
||||
cli_path 指向 `memory-system/dist/cli.js`,node_path 指向 node
|
||||
可执行文件。所有调用都有超时;非零退出即 MemoryError(stderr
|
||||
截断 500 字符,绝不含凭证值——CLI 输出本就不含凭证)。
|
||||
"""
|
||||
|
||||
def __init__(self, cli_path: str, node_path: str = "node") -> None:
|
||||
self._cli_path = cli_path
|
||||
self._node_path = node_path
|
||||
|
||||
@property
|
||||
def cli_path(self) -> str:
|
||||
"""CLI 脚本路径。"""
|
||||
return self._cli_path
|
||||
|
||||
def available(self) -> bool:
|
||||
"""CLI 脚本与 node 是否都存在。"""
|
||||
import shutil
|
||||
if not os.path.isfile(self._cli_path):
|
||||
return False
|
||||
return shutil.which(self._node_path) is not None
|
||||
|
||||
def _run(self, args: list[str],
|
||||
timeout: int = _CLI_TIMEOUT_SECONDS) -> dict[str, object]:
|
||||
"""跑一次 CLI,stdout 解析为 JSON dict。"""
|
||||
cmd = [self._node_path, self._cli_path] + args
|
||||
try:
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True,
|
||||
timeout=timeout)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
raise MemoryError("memory cli timeout: %s" % " ".join(args[:2]))
|
||||
if proc.returncode != 0:
|
||||
raise MemoryError("memory cli failed (%d): %s" % (
|
||||
proc.returncode, proc.stderr.strip()[:500]))
|
||||
try:
|
||||
parsed = json.loads(proc.stdout)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise MemoryError("memory cli bad json: %s" % exc)
|
||||
if not isinstance(parsed, dict):
|
||||
raise MemoryError("memory cli bad json: not an object")
|
||||
return parsed
|
||||
|
||||
def search(self, home: str, *queries: str) -> dict[str, object]:
|
||||
"""词汇检索(CLI search),返回 {"hits": [...]}。"""
|
||||
return self._run(["search", home] + list(queries))
|
||||
|
||||
def search_semantic(self, home: str, query: str) -> dict[str, object]:
|
||||
"""混合语义 + 词汇检索(无 embedder 时退化为纯词汇)。"""
|
||||
return self._run(["search-semantic", home, query])
|
||||
|
||||
def get(self, home: str, citation: str) -> dict[str, object]:
|
||||
"""按引用取一条记忆(MEMORY.md#L7 或 memory/...#L7-L9)。"""
|
||||
return self._run(["get", home, citation])
|
||||
|
||||
def explain(self, home: str, claim: str) -> dict[str, object]:
|
||||
"""解释一条 claim 的来源与替代关系。"""
|
||||
return self._run(["explain", home, claim])
|
||||
|
||||
def forget_plan(self, home: str, query: str) -> dict[str, object]:
|
||||
"""忘记计划:列出 query 命中的 claims,不删任何东西。"""
|
||||
return self._run(["forget-plan", home, query])
|
||||
|
||||
def forget_confirm(self, home: str, query: str,
|
||||
actor: str = "cli") -> dict[str, object]:
|
||||
"""确认忘记:删命中的 claims,记审计,返回结果。"""
|
||||
return self._run(["forget-confirm", home, query,
|
||||
"--actor", actor])
|
||||
|
||||
def backup(self, home: str,
|
||||
out: str | None = None) -> dict[str, object]:
|
||||
"""整 home 快照备份(文件级 + manifest sha256)。"""
|
||||
args = ["backup", home]
|
||||
if out is not None:
|
||||
args += ["--out", out]
|
||||
return self._run(args)
|
||||
|
||||
def restore(self, home: str, snapshot: str,
|
||||
actor: str = "cli") -> dict[str, object]:
|
||||
"""从快照恢复(先验 manifest,失败则 home 不动)。"""
|
||||
return self._run(["restore", home, snapshot,
|
||||
"--actor", actor])
|
||||
|
||||
def export(self, home: str, out_dir: str) -> dict[str, object]:
|
||||
"""导出全部记忆为 Markdown 目录。"""
|
||||
return self._run(["export", home, out_dir])
|
||||
|
||||
|
||||
def build_pull_command(ssh_cmd: list[str], remote_home: str) -> list[str]:
|
||||
"""构造拉取命令:ssh … tar -czf - -C <家目录> .(只打包内容)。
|
||||
|
||||
绝不用旧形状 ``-C <父目录> <家目录名>``:它会把顶层目录名
|
||||
打进包,解到镜像后文件落在 ``<local>/memory/MEMORY.md``,
|
||||
而 ``MemoryFiles(mirror)`` 读的是 ``<local>/MEMORY.md``——
|
||||
拉取与读取差一层,R12 真机 P7 实测 ``host_pull_files=1``
|
||||
却报 ``memory file missing: MEMORY.md``。
|
||||
"""
|
||||
return list(ssh_cmd) + ["tar -czf - -C %s ."
|
||||
% remote_home.rstrip("/")]
|
||||
|
||||
|
||||
def pull_memory_home(ssh_cmd: list[str], remote_home: str,
|
||||
local_home: str,
|
||||
timeout: int = _CLI_TIMEOUT_SECONDS) -> dict[str, int]:
|
||||
"""经 SSH + tar 把客机记忆家目录拉到宿主镜像(覆盖式)。
|
||||
|
||||
ssh_cmd 是已构造好的 ssh 基础命令(端口/密钥/用户已定,见
|
||||
cloud.chv.ssh_transport.build_ssh_command)。返回拉取到的
|
||||
文件数;ssh 或 tar 失败即 MemoryError。
|
||||
"""
|
||||
os.makedirs(local_home, mode=0o700, exist_ok=True)
|
||||
pull_cmd = build_pull_command(ssh_cmd, remote_home)
|
||||
fetch = subprocess.Popen(pull_cmd, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE)
|
||||
try:
|
||||
extract = subprocess.run(
|
||||
["tar", "-xzf", "-", "-C", local_home],
|
||||
stdin=fetch.stdout, capture_output=True, text=True,
|
||||
timeout=timeout)
|
||||
finally:
|
||||
if fetch.stdout is not None:
|
||||
fetch.stdout.close()
|
||||
fetch_rc = fetch.wait()
|
||||
err = ""
|
||||
if fetch.stderr is not None:
|
||||
err = fetch.stderr.read().decode("utf-8", "replace")[:500]
|
||||
fetch.stderr.close()
|
||||
if fetch_rc != 0:
|
||||
raise MemoryError("memory pull ssh failed (%d): %s" % (fetch_rc, err))
|
||||
if extract.returncode != 0:
|
||||
raise MemoryError("memory pull tar failed: %s"
|
||||
% extract.stderr.strip()[:500])
|
||||
count = 0
|
||||
for _root, _dirs, names in os.walk(local_home):
|
||||
count += len(names)
|
||||
return {"files": count}
|
||||
|
||||
|
||||
def forget(memory_files: MemoryFiles,
|
||||
client: MemorySystemClient | None, query: str,
|
||||
actor: str = "host") -> dict[str, object]:
|
||||
"""忘记指定内容:bank 忘记 + 本地 Markdown 清行 + 验证查不到。
|
||||
|
||||
client 在位时走真实 CLI(plan → confirm → 验证 search 为空);
|
||||
client 缺位时只做本地清行,bank 忘记记为待办(不谎称已忘)。
|
||||
返回 {"mode", "claims", "lines_removed", "verified"}。
|
||||
"""
|
||||
assert_no_secrets({"query": query})
|
||||
if client is not None and client.available():
|
||||
plan = client.forget_plan(memory_files.home, query)
|
||||
claims = plan.get("claims")
|
||||
claim_list = claims if isinstance(claims, list) else []
|
||||
client.forget_confirm(memory_files.home, query, actor=actor)
|
||||
removed = _forget_local_lines(memory_files, query, actor)
|
||||
result = client.search(memory_files.home, query)
|
||||
hits = result.get("hits")
|
||||
hit_list = hits if isinstance(hits, list) else []
|
||||
if hit_list:
|
||||
raise MemoryError(
|
||||
"forget verify failed: %d hits remain" % len(hit_list))
|
||||
return {"mode": "cli", "claims": len(claim_list),
|
||||
"lines_removed": removed, "verified": True}
|
||||
removed = _forget_local_lines(memory_files, query, actor)
|
||||
memory_files._append_audit(actor, "forget-pending", query,
|
||||
{"reason": "cli-unavailable",
|
||||
"lines_removed": removed})
|
||||
return {"mode": "local-only", "claims": 0,
|
||||
"lines_removed": removed, "verified": False}
|
||||
|
||||
|
||||
def _forget_local_lines(memory_files: MemoryFiles, query: str,
|
||||
actor: str) -> int:
|
||||
"""从镜像家目录全部 Markdown 里清掉含 query 的行。"""
|
||||
removed = 0
|
||||
for rel in memory_files.list_files():
|
||||
if rel.endswith(".md"):
|
||||
removed += memory_files.remove_lines(rel, query, actor=actor)
|
||||
return removed
|
||||
@@ -1,142 +0,0 @@
|
||||
"""CHV 上一世残留 VM 清理(生产路径唯一入口)。
|
||||
|
||||
本模块是叶子模块,只依赖标准库,绝不 import
|
||||
cloud.chv.experimental_pool(实验接口)与 cloud.sandbox。
|
||||
生产代码(worker 启动/关闭)只调
|
||||
reconcile_orphans_for_executor(),它直接读 registry 文件、
|
||||
对每个上一世条目调 executor.shutdown_vm(),不构造任何
|
||||
池对象、不调 prewarm/acquire/release。
|
||||
|
||||
Registry 语义与旧 WarmPool._load_registry 完全一致
|
||||
(feat-chv-arch95plus 的 fail-safe 口径,本轨只搬位置、
|
||||
不改语义):
|
||||
- 文件:state_dir/warm-pool.json,内容为
|
||||
{<合成名>: {"state", "vm", "task_id", "snapshot_url"}}。
|
||||
- 读到的 warm/assigned 条目一律当 orphan:旧 VMM 进程
|
||||
已随旧 worker 消失,无法验证它还是当初那个干净快照
|
||||
VM,绝不发给任务,只能清理。
|
||||
- 损坏 fail-safe:文件不是合法 JSON / 顶层不是 dict 时,
|
||||
整个文件 quarantine 成 <path>.corrupt,清理从空开始;
|
||||
单条目结构非法(无 vm、或 state 不在 warm/assigned)
|
||||
时只丢该条,不连坐其它条目。
|
||||
- 清理后落盘为空 dict(原子 tmp + rename);写失败只记
|
||||
日志警告,不炸 worker 启动/关闭。shutdown_vm 抛错时该
|
||||
条目仍计入已清理(旧 reconcile 语义:尽力清理、不因
|
||||
单条失败卡住整批),_emit 失败同样 best-effort。
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
_LOGGER = logging.getLogger("cloud.cloud_hypervisor")
|
||||
|
||||
_REGISTRY_FILENAME = "warm-pool.json"
|
||||
|
||||
|
||||
def registry_path_for_executor(executor: object) -> str | None:
|
||||
"""执行器的 registry 路径:state_dir 下的 warm-pool.json。
|
||||
|
||||
无 state_dir 时返回 None(调用方按“无残留”处理)。"""
|
||||
state_dir = getattr(executor, "state_dir", None)
|
||||
if not state_dir:
|
||||
return None
|
||||
return os.path.join(state_dir, _REGISTRY_FILENAME)
|
||||
|
||||
|
||||
def _quarantine_registry(path: str) -> None:
|
||||
"""把损坏的 registry 文件挪到 <path>.corrupt 旁(尽力而为)。"""
|
||||
try:
|
||||
os.replace(path, path + ".corrupt")
|
||||
except OSError:
|
||||
_LOGGER.warning("chv_pool_registry_quarantine_failed "
|
||||
"path=%s", path)
|
||||
|
||||
|
||||
def _read_registry(path: str) -> tuple[dict | None, bool]:
|
||||
"""读 registry 原始 dict:(data, corrupt)。
|
||||
|
||||
无文件 → (None, False);损坏 → quarantine 后 (None, True);
|
||||
顶层非 dict 同样算损坏。调用方按 corrupt 决定是否记
|
||||
registry_corrupt(实验池需要区分“无文件”与“损坏”)。"""
|
||||
try:
|
||||
with open(path, "r") as fh:
|
||||
data = json.load(fh)
|
||||
except FileNotFoundError:
|
||||
return None, False
|
||||
except (OSError, ValueError):
|
||||
_quarantine_registry(path)
|
||||
return None, True
|
||||
if not isinstance(data, dict):
|
||||
_quarantine_registry(path)
|
||||
return None, True
|
||||
return data, False
|
||||
|
||||
|
||||
def _write_registry(path: str, data: dict) -> None:
|
||||
"""原子落盘 registry(随机 tmp 名 + 双 fsync,P2-18);
|
||||
写失败只记日志。"""
|
||||
try:
|
||||
from cloud.chv.atomic_file import atomic_write_text
|
||||
import json
|
||||
directory = os.path.dirname(path)
|
||||
if directory:
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
atomic_write_text(path, json.dumps(data))
|
||||
except OSError:
|
||||
_LOGGER.warning("chv_pool_registry_save_failed path=%s", path)
|
||||
|
||||
|
||||
def _orphan_entries(data: dict) -> list[dict]:
|
||||
"""从原始 registry dict 里筛出可清理的上一世条目。
|
||||
|
||||
只留 state 为 warm/assigned 且有 vm 的条目;非法条目
|
||||
静默丢弃(不连坐)。返回条目为 {"vm", "task_id",
|
||||
"snapshot_url"} 的新 dict,不改原数据。"""
|
||||
entries: list[dict] = []
|
||||
for entry in data.values():
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
if not entry.get("vm"):
|
||||
continue
|
||||
if entry.get("state") not in ("warm", "assigned"):
|
||||
continue
|
||||
entries.append({
|
||||
"vm": entry["vm"],
|
||||
"task_id": entry.get("task_id"),
|
||||
"snapshot_url": entry.get("snapshot_url"),
|
||||
})
|
||||
return entries
|
||||
|
||||
|
||||
def reconcile_orphans_for_executor(executor: object) -> list[str]:
|
||||
"""生产路径唯一的残留清理入口:返回已清理的 VM 名列表。
|
||||
|
||||
只做清理:读 registry → 每个上一世条目尽力
|
||||
shutdown_vm → 落盘为空 → best-effort 审计事件。
|
||||
绝不构造 WarmPool、绝不调 prewarm/acquire/release,
|
||||
绝不把任何条目发给任务。无 registry 路径时返回 []。"""
|
||||
path = registry_path_for_executor(executor)
|
||||
if not path:
|
||||
return []
|
||||
data, _corrupt = _read_registry(path)
|
||||
if data is None:
|
||||
return []
|
||||
entries = _orphan_entries(data)
|
||||
cleaned: list[str] = []
|
||||
for entry in entries:
|
||||
vm = entry["vm"]
|
||||
try:
|
||||
executor.shutdown_vm(vm) # type: ignore[attr-defined]
|
||||
except Exception:
|
||||
pass
|
||||
cleaned.append(vm)
|
||||
if cleaned:
|
||||
_write_registry(path, {})
|
||||
try:
|
||||
executor._emit( # type: ignore[attr-defined]
|
||||
"warm_pool_reconcile", {"id": None},
|
||||
"cleaned=%s" % ",".join(cleaned))
|
||||
except Exception:
|
||||
pass
|
||||
return cleaned
|
||||
@@ -1,769 +0,0 @@
|
||||
"""支付安全门(feat-chv-muse-p10,叶子模块,纯标准库)。
|
||||
|
||||
对齐 Muse 官方安全文(security.muse.ai 2026-09-08,见
|
||||
docs/chv-muse-p10-gap.md):支付时发行一次性卡号给商家而非
|
||||
真实卡号,绑定特定商家、特定金额、有限有效期;已保存支付
|
||||
凭证的网站每次购买都弹人工审批并展示确切购买详情;未购物
|
||||
过的网站走内置合规钱包(Stripe Link 首发、Shop Pay 次之)。
|
||||
|
||||
本模块只做客内 agent 侧的支付安全门(检测、路由、限额、
|
||||
审批、绑定、审计),不产生真实扣款:
|
||||
- ``PaymentCredentialVault``:已保存支付方式的隔离存储,
|
||||
走 P3 authd 模式(只活内存、白名单、按次释放绑定、审计
|
||||
绝无凭证值),但另立库——只存不透明支付方式 id + 掩码
|
||||
标签,原始卡号 PAN 拒绝入库;
|
||||
- ``OneTimeCard``:一次性卡号的绑定与兑现校验(商家 +
|
||||
金额 + 货币 + 有效期,单次有效);
|
||||
- ``SimulatedWalletBackend``:沙箱钱包(假 PAN 只在后端
|
||||
私有内存里,调用方只见掩码标签与 card_id;``charge``
|
||||
只记账不联网),供测试与演示;真实钱包集成是待用户
|
||||
另行批准的事项,见差距表 §3;
|
||||
- ``PaymentGate``:编排——结账检测 → 路由 → 限额 →
|
||||
P5 对话外审批(确切购买详情进 payload)→ 铸一次性卡
|
||||
→ 兑现校验 → 记账 → 审计。无审批的支付一律拒绝。
|
||||
|
||||
真实一次性虚拟卡由 Link 在其侧发行(连接器文档原文),
|
||||
本轨的卡是绑定与兑现校验的等价模型,不是发卡行。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import secrets
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
|
||||
from cloud.chv.approval_channel import (
|
||||
Answer,
|
||||
request_approval,
|
||||
resolve_via_channel,
|
||||
verify_answer,
|
||||
)
|
||||
from cloud.chv.checkout_detect import CheckoutVerdict
|
||||
from cloud.chv.sentinel import ApprovalQueue, EgressRequest
|
||||
|
||||
# 钱包提供方白名单(连接器文档现状:stripe-link 首发、
|
||||
# shop-pay 已可用;未知提供方一律拒绝)。
|
||||
ALLOWED_WALLET_PROVIDERS = frozenset({"stripe-link", "shop-pay"})
|
||||
|
||||
# 一次性卡默认有效期(秒)与按次释放默认有效期(秒)。
|
||||
DEFAULT_CARD_TTL_SECONDS = 900
|
||||
DEFAULT_RELEASE_TTL_SECONDS = 300
|
||||
|
||||
# 审计里绝不出现的字段名(命中即丢弃并标 redacted)。
|
||||
_SECRET_FIELD_NAMES = ("card_number", "cardnumber", "pan", "cvv",
|
||||
"cvc", "token", "secret", "password")
|
||||
|
||||
# 沙箱钱包的假卡号段(Stripe 测试卡号段,永不产生真实扣款)。
|
||||
_TEST_PAN_PREFIX = "400000000000"
|
||||
|
||||
|
||||
class PaymentRefusal(Exception):
|
||||
"""支付拒绝:kind + 原因,不含任何卡号与凭证值。"""
|
||||
|
||||
def __init__(self, kind: str, message: str) -> None:
|
||||
super().__init__("payment_refused:%s: %s" % (kind, message))
|
||||
self.kind = kind
|
||||
|
||||
|
||||
def luhn_valid(number: str) -> bool:
|
||||
"""Luhn 校验(只用于识别“像卡号”的数字串,拒绝入库)。"""
|
||||
digits = [int(c) for c in number if c.isdigit()]
|
||||
if len(digits) < 13 or len(digits) > 19:
|
||||
return False
|
||||
total = 0
|
||||
for i, d in enumerate(reversed(digits)):
|
||||
if i % 2 == 1:
|
||||
d *= 2
|
||||
if d > 9:
|
||||
d -= 9
|
||||
total += d
|
||||
return total % 10 == 0
|
||||
|
||||
|
||||
def _looks_like_pan(value: str) -> bool:
|
||||
"""值是否像原始卡号:去空格/横线后 13–19 位数字且过
|
||||
Luhn。命中即视为 PAN,拒绝入库、拒绝进审计。"""
|
||||
compact = value.replace(" ", "").replace("-", "")
|
||||
return compact.isdigit() and luhn_valid(compact)
|
||||
|
||||
|
||||
def _masked_label(pan: str, brand: str = "Visa") -> str:
|
||||
"""假卡号的掩码标签(只露后 4 位,审计与审批里唯一
|
||||
允许出现的卡号派生形式)。"""
|
||||
return "%s ....%s" % (brand, pan[-4:])
|
||||
|
||||
|
||||
@dataclass
|
||||
class PaymentMethod:
|
||||
"""一个已保存的支付方式(只含不透明 id + 掩码标签,
|
||||
无真实卡号)。``merchant_host`` 是该凭证保存于的商家
|
||||
主机(None = 未限定商家,释放时仍须绑定本次商家)。"""
|
||||
|
||||
provider: str
|
||||
method_id: str
|
||||
masked_label: str
|
||||
merchant_host: str | None = None
|
||||
|
||||
def describe(self) -> dict:
|
||||
"""审计/审批用描述(不含任何凭证值)。"""
|
||||
return {"provider": self.provider, "method_id": self.method_id,
|
||||
"masked_label": self.masked_label,
|
||||
"merchant_host": self.merchant_host}
|
||||
|
||||
|
||||
@dataclass
|
||||
class PaymentRelease:
|
||||
"""一次支付方式按次释放(替身):绑定到一次具体的购买。
|
||||
兑现后交出的是不透明 method_id + 掩码标签,不是卡号。"""
|
||||
|
||||
release_id: str
|
||||
provider: str
|
||||
method_id: str
|
||||
masked_label: str
|
||||
merchant: str
|
||||
amount_minor: int
|
||||
currency: str
|
||||
expires_at: float
|
||||
uses_left: int = 1
|
||||
|
||||
def describe(self) -> dict:
|
||||
"""审计用描述(只含绑定,不含值)。"""
|
||||
return {"release_id": self.release_id,
|
||||
"provider": self.provider, "method_id": self.method_id,
|
||||
"masked_label": self.masked_label,
|
||||
"merchant": self.merchant,
|
||||
"amount_minor": self.amount_minor,
|
||||
"currency": self.currency,
|
||||
"expires_at": self.expires_at,
|
||||
"uses_left": self.uses_left}
|
||||
|
||||
def __repr__(self) -> str: # 绝不含任何凭证值
|
||||
return ("PaymentRelease(id=%s provider=%s merchant=%s "
|
||||
"amount=%d %s)" % (self.release_id, self.provider,
|
||||
self.merchant, self.amount_minor,
|
||||
self.currency))
|
||||
|
||||
|
||||
class PaymentCredentialVault:
|
||||
"""只活内存的支付凭证保险库(一个 VM 一个实例)。
|
||||
|
||||
- 值只存在内存 dict 里,不落盘、不进日志;
|
||||
- 只存不透明支付方式 id + 掩码标签:原始卡号 PAN、
|
||||
未知提供方、空 id 一律拒绝入库;
|
||||
- ``release()`` 按次发放:绑定 {商家, 金额, 货币},
|
||||
单次有效、ttl 后过期;
|
||||
- ``consume()`` 单次兑现:兑现一次即删除,过期也兑现
|
||||
不了;兑现失败不留可重试的残值。
|
||||
clock 可注入(测试用假时钟测过期),默认 time.time。"""
|
||||
|
||||
def __init__(self, clock: Callable[[], float] | None = None) -> None:
|
||||
self._store: dict[tuple[str, str], PaymentMethod] = {}
|
||||
self._releases: dict[str, PaymentRelease] = {}
|
||||
self._clock = clock or time.time
|
||||
|
||||
def __repr__(self) -> str: # 绝不含任何凭证值
|
||||
return "PaymentCredentialVault(providers=%s)" % sorted(
|
||||
{p for (p, _m) in self._store})
|
||||
|
||||
def store(self, provider: str, method_id: str,
|
||||
masked_label: str, merchant_host: str | None = None
|
||||
) -> None:
|
||||
"""入库:白名单校验 + PAN 拒绝。原始卡号、未知提供方、
|
||||
空 id、空标签一律拒绝入库(PaymentRefusal),绝不静默
|
||||
收下。"""
|
||||
if provider not in ALLOWED_WALLET_PROVIDERS:
|
||||
raise PaymentRefusal(
|
||||
"unknown_provider",
|
||||
"钱包提供方 %s 不在白名单内" % provider)
|
||||
if not isinstance(method_id, str) or not method_id.strip():
|
||||
raise PaymentRefusal("empty_method_id", "支付方式 id 为空")
|
||||
if _looks_like_pan(method_id) or _looks_like_pan(masked_label):
|
||||
raise PaymentRefusal(
|
||||
"raw_pan_refused",
|
||||
"原始卡号不得入库,只存不透明 id 与掩码标签")
|
||||
if not isinstance(masked_label, str) or not masked_label.strip():
|
||||
raise PaymentRefusal("empty_label", "掩码标签为空")
|
||||
key = (provider, method_id)
|
||||
self._store[key] = PaymentMethod(
|
||||
provider, method_id, masked_label, merchant_host)
|
||||
|
||||
def has(self, provider: str, method_id: str) -> bool:
|
||||
"""该(提供方,方式 id)是否已入库(不返回值)。"""
|
||||
return (provider, method_id) in self._store
|
||||
|
||||
def methods_for_merchant(self, merchant_host: str) -> list:
|
||||
"""该商家主机上已保存的支付方式(只返回描述对象,
|
||||
不含任何凭证值)。"""
|
||||
return [m for m in self._store.values()
|
||||
if m.merchant_host is None
|
||||
or m.merchant_host == merchant_host]
|
||||
|
||||
def all_methods(self) -> list:
|
||||
"""全部已保存支付方式(描述对象,不含任何凭证值)。"""
|
||||
return list(self._store.values())
|
||||
|
||||
def forget(self, provider: str, method_id: str) -> bool:
|
||||
"""删一条入库支付方式;同时作废其全部未兑现释放。
|
||||
未知键返回 False,不抛错。"""
|
||||
gone = self._store.pop((provider, method_id), None) is not None
|
||||
if gone:
|
||||
doomed = [rid for rid, rel in self._releases.items()
|
||||
if rel.provider == provider
|
||||
and rel.method_id == method_id]
|
||||
for rid in doomed:
|
||||
self._releases.pop(rid, None)
|
||||
return gone
|
||||
|
||||
def release(self, provider: str, method_id: str, merchant: str,
|
||||
amount_minor: int, currency: str,
|
||||
ttl_seconds: int = DEFAULT_RELEASE_TTL_SECONDS,
|
||||
max_uses: int = 1) -> PaymentRelease:
|
||||
"""按次发放:绑定须与入库作用域一致(merchant_host
|
||||
有记录时须一致),金额须 > 0。发放的释放单次有效、
|
||||
ttl 后过期;须经 consume() 兑现一次才交出方式引用。"""
|
||||
method = self._store.get((provider, method_id))
|
||||
if method is None:
|
||||
raise PaymentRefusal(
|
||||
"method_not_stored",
|
||||
"支付方式 %s 未入库" % method_id)
|
||||
if method.merchant_host is not None \
|
||||
and method.merchant_host != merchant:
|
||||
raise PaymentRefusal(
|
||||
"scope_mismatch", "支付方式作用域商家不一致,拒绝释放")
|
||||
if amount_minor <= 0:
|
||||
raise PaymentRefusal("bad_amount", "金额须大于 0")
|
||||
release = PaymentRelease(
|
||||
release_id=secrets.token_hex(8),
|
||||
provider=provider, method_id=method_id,
|
||||
masked_label=method.masked_label, merchant=merchant,
|
||||
amount_minor=amount_minor, currency=currency,
|
||||
expires_at=self._clock() + ttl_seconds,
|
||||
uses_left=max_uses)
|
||||
self._releases[release.release_id] = release
|
||||
return release
|
||||
|
||||
def consume(self, release_id: str) -> PaymentMethod:
|
||||
"""兑现一次释放:过期/用完/未知 id 一律拒绝
|
||||
(PaymentRefusal,不含值)。兑现成功返回支付方式
|
||||
引用(不透明 id + 掩码标签),并把该释放删除。"""
|
||||
release = self._releases.get(release_id)
|
||||
if release is None:
|
||||
raise PaymentRefusal(
|
||||
"release_unknown", "释放记录不存在或已兑现")
|
||||
if self._clock() >= release.expires_at:
|
||||
self._releases.pop(release_id, None)
|
||||
raise PaymentRefusal("release_expired", "释放已过期")
|
||||
if release.uses_left <= 0:
|
||||
self._releases.pop(release_id, None)
|
||||
raise PaymentRefusal("release_exhausted", "释放已用完")
|
||||
release.uses_left -= 1
|
||||
method = self._store.get((release.provider, release.method_id))
|
||||
self._releases.pop(release_id, None)
|
||||
if method is None:
|
||||
raise PaymentRefusal(
|
||||
"method_gone", "支付方式已删除,释放作废")
|
||||
return method
|
||||
|
||||
|
||||
@dataclass
|
||||
class PurchaseDetails:
|
||||
"""一次购买的确切详情(审批单上展示给用户的全部内容,
|
||||
也是防篡改摘要的输入)。金额以 minor units 计(分/日元
|
||||
等无小数货币按其最小单位),货币为三位字母码,不换算。"""
|
||||
|
||||
merchant: str
|
||||
merchant_host: str
|
||||
items: tuple
|
||||
amount_minor: int
|
||||
currency: str
|
||||
wallet_provider: str
|
||||
|
||||
def total_minor(self) -> int:
|
||||
"""商品小计(数量 × 单价之和),须等于 amount_minor。"""
|
||||
return sum(qty * unit for (_n, qty, unit) in self.items)
|
||||
|
||||
def validate(self) -> None:
|
||||
"""校验详情自洽:金额 > 0、货币三位码、商品非空、
|
||||
小计等于总额、提供方在白名单内。任一失败即
|
||||
ValueError(调用方 bug,不走支付拒绝通道)。"""
|
||||
if self.amount_minor <= 0:
|
||||
raise ValueError("amount_minor must be > 0")
|
||||
if len(self.currency) != 3 or not self.currency.isalpha():
|
||||
raise ValueError("currency must be a 3-letter code")
|
||||
if not self.items:
|
||||
raise ValueError("items must not be empty")
|
||||
if self.total_minor() != self.amount_minor:
|
||||
raise ValueError("items total != amount_minor")
|
||||
if self.wallet_provider not in ALLOWED_WALLET_PROVIDERS:
|
||||
raise ValueError("unknown wallet_provider")
|
||||
|
||||
def describe(self) -> dict:
|
||||
"""审批 payload 里的确切购买详情(商家、商品、数量、
|
||||
金额、货币、提供方;无任何卡号与凭证值)。"""
|
||||
return {"merchant": self.merchant,
|
||||
"merchant_host": self.merchant_host,
|
||||
"items": [{"name": n, "qty": q, "unit_minor": u}
|
||||
for (n, q, u) in self.items],
|
||||
"amount_minor": self.amount_minor,
|
||||
"currency": self.currency,
|
||||
"wallet_provider": self.wallet_provider}
|
||||
|
||||
def digest(self) -> str:
|
||||
"""购买详情摘要(SHA-256,规范 JSON):审批时记下,
|
||||
扣款时重算——详情被篡改(换商家/改金额/换商品)即
|
||||
摘要对不上,支付拒绝。"""
|
||||
canonical = json.dumps(self.describe(), sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=False)
|
||||
return hashlib.sha256(canonical.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
@dataclass
|
||||
class OneTimeCard:
|
||||
"""一张一次性卡号(绑定与兑现校验的等价模型)。
|
||||
|
||||
绑定 {商家, 金额, 货币, 有效期},单次有效。真实一次
|
||||
性虚拟卡由 Link 在其侧发行(连接器文档原文),本类
|
||||
只做绑定校验——调用方只见 card_id 与掩码标签,卡号
|
||||
本体只活在钱包后端私有内存里。"""
|
||||
|
||||
card_id: str
|
||||
merchant: str
|
||||
amount_minor: int
|
||||
currency: str
|
||||
expires_at: float
|
||||
masked_label: str
|
||||
uses_left: int = 1
|
||||
|
||||
def describe(self) -> dict:
|
||||
"""审计用描述(只含绑定与掩码标签,不含卡号)。"""
|
||||
return {"card_id": self.card_id, "merchant": self.merchant,
|
||||
"amount_minor": self.amount_minor,
|
||||
"currency": self.currency,
|
||||
"expires_at": self.expires_at,
|
||||
"masked_label": self.masked_label,
|
||||
"uses_left": self.uses_left}
|
||||
|
||||
def __repr__(self) -> str: # 绝不含卡号
|
||||
return ("OneTimeCard(id=%s merchant=%s amount=%d %s)"
|
||||
% (self.card_id, self.merchant, self.amount_minor,
|
||||
self.currency))
|
||||
|
||||
def authorize(self, merchant: str, amount_minor: int,
|
||||
currency: str, now: float) -> tuple:
|
||||
"""兑现校验:用完/过期/换商家/超额/换货币任一命中
|
||||
即 (False, 原因);全过则消耗一次并返回 (True,
|
||||
"ok")。单次有效,用过即废。"""
|
||||
if self.uses_left <= 0:
|
||||
return False, "card_exhausted"
|
||||
if now >= self.expires_at:
|
||||
return False, "card_expired"
|
||||
if merchant != self.merchant:
|
||||
return False, "wrong_merchant"
|
||||
if currency != self.currency:
|
||||
return False, "wrong_currency"
|
||||
if amount_minor > self.amount_minor:
|
||||
return False, "amount_exceeded"
|
||||
self.uses_left -= 1
|
||||
return True, "ok"
|
||||
|
||||
|
||||
@dataclass
|
||||
class SpendLimits:
|
||||
"""钱包侧限额(minor units,无货币、不换算——按
|
||||
stripe-link 连接器文档口径)。None = 该档未设置。
|
||||
``daily_spent_minor`` / ``thirty_day_spent_minor`` 是
|
||||
调用方传入的已用额度(本模块不记账)。"""
|
||||
|
||||
per_purchase_minor: int | None = None
|
||||
daily_minor: int | None = None
|
||||
thirty_day_minor: int | None = None
|
||||
daily_spent_minor: int = 0
|
||||
thirty_day_spent_minor: int = 0
|
||||
|
||||
def check(self, amount_minor: int) -> tuple:
|
||||
"""逐项比 minor units:超任一档即 (False, 原因);
|
||||
三档全为 None(限额不可用)→ (True,
|
||||
"limits_unavailable"),绝不编造上限。"""
|
||||
if self.per_purchase_minor is not None \
|
||||
and amount_minor > self.per_purchase_minor:
|
||||
return False, "over_per_purchase_limit"
|
||||
if self.daily_minor is not None \
|
||||
and self.daily_spent_minor + amount_minor \
|
||||
> self.daily_minor:
|
||||
return False, "over_daily_limit"
|
||||
if self.thirty_day_minor is not None \
|
||||
and self.thirty_day_spent_minor + amount_minor \
|
||||
> self.thirty_day_minor:
|
||||
return False, "over_thirty_day_limit"
|
||||
if self.per_purchase_minor is None \
|
||||
and self.daily_minor is None \
|
||||
and self.thirty_day_minor is None:
|
||||
return True, "limits_unavailable"
|
||||
return True, "ok"
|
||||
|
||||
|
||||
class PaymentAudit:
|
||||
"""支付审计(内存态记录表)。
|
||||
|
||||
事件只含允许字段(商家、金额、货币、掩码标签、
|
||||
pending_id、原因、路由、提供方);字段名命中
|
||||
``_SECRET_FIELD_NAMES`` 一律丢弃并标 ``redacted``——
|
||||
卡号与凭证值绝不进审计。"""
|
||||
|
||||
ALLOWED_FIELDS = frozenset({
|
||||
"merchant", "merchant_host", "amount_minor", "currency",
|
||||
"masked_label", "pending_id", "reason", "route",
|
||||
"wallet_provider", "card_id", "limits_checked",
|
||||
})
|
||||
|
||||
def __init__(self, clock: Callable[[], float] | None = None) -> None:
|
||||
self._records: list = []
|
||||
self._clock = clock or time.time
|
||||
|
||||
def record(self, event: str, fields: dict | None = None,
|
||||
now: float | None = None) -> dict:
|
||||
"""记一条审计(只留允许字段;秘密字段名丢弃并标
|
||||
redacted)。返回记下的记录副本。"""
|
||||
at = now if now is not None else self._clock()
|
||||
kept: dict = {"event": event, "ts": at}
|
||||
redacted = False
|
||||
for key, value in (fields or {}).items():
|
||||
if key in _SECRET_FIELD_NAMES:
|
||||
redacted = True
|
||||
continue
|
||||
if key in self.ALLOWED_FIELDS:
|
||||
kept[key] = value
|
||||
if redacted:
|
||||
kept["redacted"] = True
|
||||
self._records.append(dict(kept))
|
||||
return dict(kept)
|
||||
|
||||
def list_records(self) -> list:
|
||||
"""全部审计记录(副本,按记录时间排序)。"""
|
||||
return [dict(r) for r in self._records]
|
||||
|
||||
|
||||
class SimulatedWalletBackend:
|
||||
"""沙箱钱包(测试与演示用,绝不产生真实扣款)。
|
||||
|
||||
- 假 PAN 只存在 ``self._pans`` 私有内存里,调用方只见
|
||||
card_id 与掩码标签;``__repr__`` 与账本里绝无卡号;
|
||||
- ``mint_card`` 按购买详情铸一次性卡(绑定商家 + 金额
|
||||
+ 货币 + 900 秒有效期);
|
||||
- ``charge`` 只记账不联网:绑定校验通过即记一笔
|
||||
``simulated_paid``,否则记拒绝原因;
|
||||
- ``void_card`` 作废一张卡(取消待支付时调用)。
|
||||
真实钱包集成是待用户另行批准的事项(见差距表 §3)。"""
|
||||
|
||||
def __init__(self, providers: tuple = ("stripe-link", "shop-pay"),
|
||||
clock: Callable[[], float] | None = None) -> None:
|
||||
self._providers = tuple(providers)
|
||||
self._vault = PaymentCredentialVault(clock=clock)
|
||||
self._pans: dict[str, str] = {}
|
||||
self._cards: dict[str, OneTimeCard] = {}
|
||||
self._voided: set = set()
|
||||
self._ledger: list = []
|
||||
self._clock = clock or time.time
|
||||
self.fail_next_charge = False
|
||||
|
||||
def __repr__(self) -> str: # 绝不含任何卡号
|
||||
return "SimulatedWalletBackend(providers=%s)" % list(
|
||||
self._providers)
|
||||
|
||||
def supports(self, provider: str) -> bool:
|
||||
"""该钱包提供方是否可用(沙箱里由构造参数决定,
|
||||
真实侧由钱包连接状态决定)。"""
|
||||
return provider in self._providers
|
||||
|
||||
def add_method(self, provider: str, method_id: str,
|
||||
masked_label: str, merchant_host: str | None = None
|
||||
) -> None:
|
||||
"""往沙箱钱包里加一个已保存支付方式(只存不透明
|
||||
id + 掩码标签,走同一入库校验)。"""
|
||||
self._vault.store(provider, method_id, masked_label,
|
||||
merchant_host)
|
||||
|
||||
def list_methods(self, merchant_host: str | None = None) -> list:
|
||||
"""已保存支付方式(描述对象;merchant_host 给了
|
||||
则只返回该商家的)。"""
|
||||
if merchant_host is None:
|
||||
return [m.describe()
|
||||
for m in self._vault.all_methods()]
|
||||
return [m.describe() for m in
|
||||
self._vault.methods_for_merchant(merchant_host)]
|
||||
|
||||
def mint_card(self, details: PurchaseDetails,
|
||||
now: float | None = None) -> OneTimeCard:
|
||||
"""按购买详情铸一张一次性卡:假 PAN 只进私有内存,
|
||||
返回的卡对象只有 card_id 与掩码标签。"""
|
||||
at = now if now is not None else self._clock()
|
||||
card_id = "otc_" + secrets.token_hex(8)
|
||||
pan = _TEST_PAN_PREFIX + "%04d" % secrets.randbelow(10000)
|
||||
self._pans[card_id] = pan
|
||||
card = OneTimeCard(
|
||||
card_id=card_id, merchant=details.merchant,
|
||||
amount_minor=details.amount_minor,
|
||||
currency=details.currency,
|
||||
expires_at=at + DEFAULT_CARD_TTL_SECONDS,
|
||||
masked_label=_masked_label(pan))
|
||||
self._cards[card_id] = card
|
||||
return card
|
||||
|
||||
def charge(self, card: OneTimeCard, details: PurchaseDetails,
|
||||
now: float | None = None) -> dict:
|
||||
"""模拟扣款:只记账不联网。绑定校验通过即记一笔
|
||||
simulated_paid;``fail_next_charge`` 为真时模拟
|
||||
钱包侧失败(扣款失败即 void 该卡)。"""
|
||||
at = now if now is not None else self._clock()
|
||||
entry: dict
|
||||
if self.fail_next_charge:
|
||||
self.fail_next_charge = False
|
||||
self.void_card(card.card_id)
|
||||
entry = {"status": "simulated_failed",
|
||||
"reason": "wallet_declined",
|
||||
"card_id": card.card_id}
|
||||
self._ledger.append(dict(entry))
|
||||
return dict(entry)
|
||||
ok, reason = card.authorize(
|
||||
details.merchant, details.amount_minor,
|
||||
details.currency, at)
|
||||
if not ok:
|
||||
entry = {"status": "refused", "reason": reason,
|
||||
"card_id": card.card_id}
|
||||
self._ledger.append(dict(entry))
|
||||
return dict(entry)
|
||||
entry = {"status": "simulated_paid",
|
||||
"card_id": card.card_id,
|
||||
"merchant": details.merchant,
|
||||
"amount_minor": details.amount_minor,
|
||||
"currency": details.currency,
|
||||
"masked_label": card.masked_label}
|
||||
self._ledger.append(dict(entry))
|
||||
return dict(entry)
|
||||
|
||||
def void_card(self, card_id: str) -> bool:
|
||||
"""作废一张一次性卡(取消待支付时调用;已作废或
|
||||
未知 id 返回 False,不抛错)。"""
|
||||
if card_id in self._voided or card_id not in self._cards:
|
||||
return False
|
||||
self._voided.add(card_id)
|
||||
card = self._cards[card_id]
|
||||
card.uses_left = 0
|
||||
return True
|
||||
|
||||
def ledger(self) -> list:
|
||||
"""模拟账本(副本;只含记账条目,无任何卡号)。"""
|
||||
return [dict(e) for e in self._ledger]
|
||||
|
||||
|
||||
def _verdict_of(answer: Answer | dict) -> str:
|
||||
"""答复对象或同款 dict 里的 verdict(verify_answer 已
|
||||
校验过合法性,这里只取值;取不到即空串)。"""
|
||||
if isinstance(answer, dict):
|
||||
value = answer.get("verdict", "")
|
||||
return value if isinstance(value, str) else ""
|
||||
return getattr(answer, "verdict", "") or ""
|
||||
|
||||
|
||||
class PaymentGate:
|
||||
"""支付安全门:一次购买的完整编排。
|
||||
|
||||
``begin_purchase``:结账检测 → 路由(已保存凭证 /
|
||||
合规钱包 stripe-link 优先、shop-pay 次之)→ 限额 →
|
||||
P5 对话外审批(确切购买详情进 payload)。无审批不支付。
|
||||
``complete_purchase``:验签(不消费)→ 验详情摘要
|
||||
(篡改即拒,审批保持待处理)→ 单次消费答复 → 铸一次
|
||||
性卡 → 兑现校验 → 记账 → 审计。审批被拒即停止,不扣款。
|
||||
"""
|
||||
|
||||
def __init__(self, queue: ApprovalQueue,
|
||||
wallet: SimulatedWalletBackend,
|
||||
vault: PaymentCredentialVault,
|
||||
channel_secret: str,
|
||||
clock: Callable[[], float] | None = None,
|
||||
audit: PaymentAudit | None = None) -> None:
|
||||
self._queue = queue
|
||||
self._wallet = wallet
|
||||
self._vault = vault
|
||||
self._secret = channel_secret
|
||||
self._clock = clock or time.time
|
||||
self._audit = audit or PaymentAudit(clock=clock)
|
||||
self._pending: dict[str, dict] = {}
|
||||
|
||||
def _route(self, details: PurchaseDetails) -> tuple:
|
||||
"""路由:商家有已保存凭证 → saved;无 → 合规钱包
|
||||
(stripe-link 优先、shop-pay 次之);两家都不支持
|
||||
→ (None, "no_wallet_route")。"""
|
||||
saved = self._vault.methods_for_merchant(
|
||||
details.merchant_host)
|
||||
if saved:
|
||||
return "saved", saved[0].provider
|
||||
if self._wallet.supports("stripe-link"):
|
||||
return "wallet", "stripe-link"
|
||||
if self._wallet.supports("shop-pay"):
|
||||
return "wallet", "shop-pay"
|
||||
return "", "no_wallet_route"
|
||||
|
||||
def _refuse(self, details: PurchaseDetails, reason: str,
|
||||
at: float, limits_checked: bool | None = None) -> dict:
|
||||
"""记一条 payment_refused 审计并返回统一的拒绝结果。"""
|
||||
fields: dict = {"merchant": details.merchant,
|
||||
"merchant_host": details.merchant_host,
|
||||
"amount_minor": details.amount_minor,
|
||||
"currency": details.currency,
|
||||
"wallet_provider": details.wallet_provider,
|
||||
"reason": reason}
|
||||
if limits_checked is not None:
|
||||
fields["limits_checked"] = limits_checked
|
||||
self._audit.record("payment_refused", fields, now=at)
|
||||
out: dict = {"ok": False, "reason": reason}
|
||||
if limits_checked is not None:
|
||||
out["limits_checked"] = limits_checked
|
||||
return out
|
||||
|
||||
def _request_payment_approval(self, details: PurchaseDetails,
|
||||
route: str, provider: str,
|
||||
limits_checked: bool,
|
||||
at: float) -> dict:
|
||||
"""记入 Sentinel 待审批队列并经 P5 通道发审批请求
|
||||
(确切购买详情进 payload)。返回 begin 的成功结果。"""
|
||||
request = EgressRequest(
|
||||
connector="payment", method="purchase", scheme="https",
|
||||
host=details.merchant_host, port=443, path="/checkout")
|
||||
pending_id = self._queue.record(request, ["payment_approval"])
|
||||
record = next(r for r in self._queue.list_pending()
|
||||
if r["pending_id"] == pending_id)
|
||||
approval = request_approval(
|
||||
record, self._secret, now=at,
|
||||
details=details.describe())
|
||||
self._pending[pending_id] = {
|
||||
"details": details, "digest": details.digest(),
|
||||
"route": route, "provider": provider,
|
||||
"card_id": None, "limits_checked": limits_checked}
|
||||
self._audit.record(
|
||||
"payment_approval_requested",
|
||||
{"merchant": details.merchant,
|
||||
"merchant_host": details.merchant_host,
|
||||
"amount_minor": details.amount_minor,
|
||||
"currency": details.currency,
|
||||
"wallet_provider": details.wallet_provider,
|
||||
"pending_id": pending_id, "route": route,
|
||||
"limits_checked": limits_checked}, now=at)
|
||||
return {"ok": True, "pending_id": pending_id,
|
||||
"payload": approval.payload(), "route": route,
|
||||
"provider": provider}
|
||||
|
||||
def begin_purchase(self, details: PurchaseDetails,
|
||||
checkout: CheckoutVerdict,
|
||||
limits: SpendLimits | None = None,
|
||||
now: float | None = None) -> dict:
|
||||
"""开始一次购买:返回 {"ok": True, "pending_id",
|
||||
"payload", "route", "provider"},或 {"ok": False,
|
||||
"reason"}。审批请求发给客户端 UI,payload 的 details
|
||||
就是用户看到的确切购买详情。"""
|
||||
details.validate()
|
||||
at = now if now is not None else self._clock()
|
||||
if not checkout.is_checkout:
|
||||
return self._refuse(details, "not_checkout", at)
|
||||
route, provider = self._route(details)
|
||||
if not route:
|
||||
return self._refuse(details, provider, at)
|
||||
limits_checked = False
|
||||
if limits is not None:
|
||||
ok, reason = limits.check(details.amount_minor)
|
||||
if not ok:
|
||||
return self._refuse(details, reason, at, True)
|
||||
limits_checked = reason != "limits_unavailable"
|
||||
return self._request_payment_approval(
|
||||
details, route, provider, limits_checked, at)
|
||||
|
||||
def _charge_after_approval(self, pending_id: str,
|
||||
details: PurchaseDetails,
|
||||
base: dict, at: float) -> dict:
|
||||
"""审批通过后:铸一次性卡 → 兑现校验 → 记账。扣款
|
||||
失败即 {"ok": False, "reason"},不扣款;成功即
|
||||
{"ok": True, "charge", "card_id", "masked_label"}。"""
|
||||
card = self._wallet.mint_card(details, now=at)
|
||||
self._pending[pending_id]["card_id"] = card.card_id
|
||||
result = self._wallet.charge(card, details, now=at)
|
||||
if result.get("status") != "simulated_paid":
|
||||
self._audit.record(
|
||||
"payment_refused",
|
||||
dict(base, reason=result.get("reason", "charge_failed"),
|
||||
card_id=card.card_id), now=at)
|
||||
self._pending.pop(pending_id, None)
|
||||
return {"ok": False,
|
||||
"reason": result.get("reason", "charge_failed")}
|
||||
self._audit.record(
|
||||
"payment_completed",
|
||||
dict(base, card_id=card.card_id,
|
||||
masked_label=card.masked_label), now=at)
|
||||
self._pending.pop(pending_id, None)
|
||||
return {"ok": True, "charge": result,
|
||||
"card_id": card.card_id,
|
||||
"masked_label": card.masked_label}
|
||||
|
||||
def complete_purchase(self, pending_id: str,
|
||||
answer: Answer | dict,
|
||||
actual_details: PurchaseDetails,
|
||||
now: float | None = None) -> dict:
|
||||
"""完成一次购买:验签 → 验详情摘要 → 单次消费答复
|
||||
→ 铸一次性卡 → 兑现校验 → 记账。任一步失败即
|
||||
{"ok": False, "reason"},不扣款。"""
|
||||
pending = self._pending.get(pending_id)
|
||||
if pending is None:
|
||||
return {"ok": False, "reason": "unknown_pending"}
|
||||
at = now if now is not None else self._clock()
|
||||
details = pending["details"]
|
||||
base = {"merchant": details.merchant,
|
||||
"amount_minor": details.amount_minor,
|
||||
"currency": details.currency,
|
||||
"pending_id": pending_id}
|
||||
if not verify_answer(answer, self._secret, now=at):
|
||||
self._audit.record("payment_refused",
|
||||
dict(base, reason="invalid_answer"),
|
||||
now=at)
|
||||
return {"ok": False, "reason": "invalid_answer"}
|
||||
if actual_details.digest() != pending["digest"]:
|
||||
self._audit.record("payment_refused",
|
||||
dict(base, reason="details_tampered"),
|
||||
now=at)
|
||||
return {"ok": False, "reason": "details_tampered"}
|
||||
if not resolve_via_channel(self._queue, answer, self._secret,
|
||||
now=at):
|
||||
return {"ok": False, "reason": "resolve_failed"}
|
||||
if _verdict_of(answer) == "deny":
|
||||
self._audit.record("payment_denied", base, now=at)
|
||||
self._pending.pop(pending_id, None)
|
||||
return {"ok": False, "reason": "denied"}
|
||||
return self._charge_after_approval(pending_id, details,
|
||||
base, at)
|
||||
|
||||
def cancel_purchase(self, pending_id: str,
|
||||
now: float | None = None) -> dict:
|
||||
"""取消一笔待支付:审批保持未消费(用户可重新发起),
|
||||
已铸的卡即 void(取消待支付 void 虚拟卡,连接器文档
|
||||
口径)。未知 pending 返回 False,不抛错。"""
|
||||
pending = self._pending.pop(pending_id, None)
|
||||
if pending is None:
|
||||
return {"ok": False, "reason": "unknown_pending"}
|
||||
at = now if now is not None else self._clock()
|
||||
card_id = pending.get("card_id")
|
||||
if card_id is not None:
|
||||
self._wallet.void_card(card_id)
|
||||
self._audit.record("payment_card_voided",
|
||||
{"pending_id": pending_id,
|
||||
"card_id": card_id}, now=at)
|
||||
self._audit.record(
|
||||
"payment_cancelled",
|
||||
{"merchant": pending["details"].merchant,
|
||||
"amount_minor": pending["details"].amount_minor,
|
||||
"currency": pending["details"].currency,
|
||||
"pending_id": pending_id}, now=at)
|
||||
return {"ok": True, "reason": "cancelled"}
|
||||
|
||||
def pending_ids(self) -> list:
|
||||
"""全部待处理购买的 pending_id(副本)。"""
|
||||
return list(self._pending)
|
||||
@@ -1,177 +0,0 @@
|
||||
"""默认策略(feat-chv-muse-p5-B,叶子模块,纯标准库)。
|
||||
|
||||
默认策略规则表 + 谨慎度三档(``DefaultPolicy``)。只读方法
|
||||
(GET/HEAD)+ 已允许主机/路径 + 低风险判定(无 body 写操作、
|
||||
无敏感路径段)→ 自动放行候选;其余走正常审批流。
|
||||
|
||||
谨慎度:
|
||||
- default(默认):已允许主机 + 路径前缀匹配即候选;
|
||||
- cautious:自动放行收窄——只放行已明确允许过的精确主机 +
|
||||
精确路径(前缀规则不再放行),ask 增多;
|
||||
- permissive:放宽自动放行范围——只读 + 非敏感路径即候选,
|
||||
不必先前已允许;但写操作(POST/PUT/DELETE/PATCH)与敏感
|
||||
路径(/admin、/settings、/login、token 相关)永远不自动放行。
|
||||
|
||||
``auto_allow_eligible(request, tainted)`` 只是“候选”判定,不
|
||||
是放行 verdict:污点进程一律 False(与 P5-A 污点追踪器口径
|
||||
一致,tainted 由调用方传);未知连接器仍由 SentinelPolicy
|
||||
的 evaluate() 挡。档位切换与规则增删(``set_caution`` /
|
||||
``add_rule`` / ``remove_rule``)每次变化记审计事件
|
||||
``sentinel_policy_changed``(旧值 → 新值,无值)。
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
from cloud.chv.sentinel import EgressRequest
|
||||
|
||||
READ_METHODS = frozenset({"GET", "HEAD"})
|
||||
WRITE_METHODS = frozenset({"POST", "PUT", "DELETE", "PATCH"})
|
||||
CAUTION_LEVELS = ("default", "cautious", "permissive")
|
||||
_SENSITIVE_MARKERS = ("/admin", "/settings", "/login", "token")
|
||||
_EVENT_POLICY_CHANGED = "sentinel_policy_changed"
|
||||
|
||||
|
||||
def _is_sensitive_path(path: str) -> bool:
|
||||
"""敏感路径:/admin、/settings、/login 或 token 相关。"""
|
||||
lowered = (path or "").lower()
|
||||
return any(m in lowered for m in _SENSITIVE_MARKERS)
|
||||
|
||||
|
||||
def _rule_key(connector: str, host: str, path: str,
|
||||
methods: frozenset) -> tuple:
|
||||
"""规则内部形态:(连接器,主机小写,路径,方法集)。"""
|
||||
return (connector, (host or "").lower(), path, methods)
|
||||
|
||||
|
||||
def _rule_dict(rule: tuple) -> dict:
|
||||
"""规则的审计/展示形态(只含连接器/主机/路径/方法)。"""
|
||||
return {"connector": rule[0], "host": rule[1],
|
||||
"path": rule[2], "methods": sorted(rule[3])}
|
||||
|
||||
|
||||
def _rule_matches(rule: tuple, connector: str, host: str,
|
||||
path: str,
|
||||
methods: Iterable[str] | None) -> bool:
|
||||
"""该规则是否命中(连接器,主机,路径)[+ 方法]。"""
|
||||
if rule[0] != connector or rule[1] != (host or "").lower():
|
||||
return False
|
||||
if rule[2] != path:
|
||||
return False
|
||||
if methods is None:
|
||||
return True
|
||||
return rule[3] == frozenset(m.upper() for m in methods)
|
||||
|
||||
|
||||
class DefaultPolicy:
|
||||
"""默认策略:规则表(已允许主机 + 路径)+ 谨慎度三档。
|
||||
|
||||
规则表由调用方播种(``allow_entries``,每条是
|
||||
(连接器,主机,路径前缀)三元组,方法默认只读)或
|
||||
``add_rule`` 增补;``audit_log`` 收全部策略变化审计。
|
||||
"""
|
||||
|
||||
def __init__(self, caution: str = "default",
|
||||
allow_entries: Iterable[tuple[str, str, str]] = (),
|
||||
audit: list | None = None) -> None:
|
||||
if caution not in CAUTION_LEVELS:
|
||||
raise ValueError("unknown_caution:%s" % caution)
|
||||
self.caution = caution
|
||||
self._rules: list = []
|
||||
for connector, host, path in allow_entries:
|
||||
self._rules.append(_rule_key(
|
||||
connector, host, path, READ_METHODS))
|
||||
self.audit_log = audit if audit is not None else []
|
||||
|
||||
def _audit_change(self, change: str, old_value: object,
|
||||
new_value: object) -> None:
|
||||
"""记一条策略变化审计(旧值 → 新值,无值)。"""
|
||||
self.audit_log.append(
|
||||
{"event": _EVENT_POLICY_CHANGED, "change": change,
|
||||
"old_value": old_value, "new_value": new_value})
|
||||
|
||||
def set_caution(self, level: str) -> bool:
|
||||
"""切换谨慎度:变化记审计并返回 True;非法档位抛
|
||||
ValueError;与现值相同则不记审计、返回 False。"""
|
||||
if level not in CAUTION_LEVELS:
|
||||
raise ValueError("unknown_caution:%s" % level)
|
||||
if level == self.caution:
|
||||
return False
|
||||
old = self.caution
|
||||
self.caution = level
|
||||
self._audit_change("set_caution", old, level)
|
||||
return True
|
||||
|
||||
def add_rule(self, connector: str, host: str, path: str,
|
||||
methods: Iterable[str] = READ_METHODS) -> bool:
|
||||
"""增一条已允许规则(连接器 + 主机 + 路径前缀 + 方法)。
|
||||
已存在相同规则返回 False(不记审计);新增记审计。"""
|
||||
rule = _rule_key(connector, host, path,
|
||||
frozenset(m.upper() for m in methods))
|
||||
if rule in self._rules:
|
||||
return False
|
||||
self._rules.append(rule)
|
||||
self._audit_change("add_rule", None, _rule_dict(rule))
|
||||
return True
|
||||
|
||||
def remove_rule(self, connector: str, host: str, path: str,
|
||||
methods: Iterable[str] | None = None) -> bool:
|
||||
"""删规则:methods 为 None 时按(连接器,主机,路径)
|
||||
删全部方法变体;否则只删精确匹配。删到记审计。"""
|
||||
removed = False
|
||||
kept = []
|
||||
for rule in self._rules:
|
||||
if _rule_matches(rule, connector, host, path, methods):
|
||||
removed = True
|
||||
self._audit_change("remove_rule",
|
||||
_rule_dict(rule), None)
|
||||
else:
|
||||
kept.append(rule)
|
||||
self._rules = kept
|
||||
return removed
|
||||
|
||||
def list_rules(self) -> list:
|
||||
"""全部已允许规则(副本)。"""
|
||||
return [_rule_dict(rule) for rule in self._rules]
|
||||
|
||||
def _allowlisted(self, request: EgressRequest,
|
||||
exact: bool) -> bool:
|
||||
"""主机 + 路径是否在规则表内(exact 为真时路径须全等,
|
||||
否则前缀匹配即算;方法也须在规则方法集内)。"""
|
||||
for rule in self._rules:
|
||||
if rule[0] != request.connector:
|
||||
continue
|
||||
if rule[1] != request.host:
|
||||
continue
|
||||
if request.method not in rule[3]:
|
||||
continue
|
||||
if exact:
|
||||
if request.path == rule[2]:
|
||||
return True
|
||||
elif request.path.startswith(rule[2]):
|
||||
return True
|
||||
return False
|
||||
|
||||
def auto_allow_eligible(self, request: EgressRequest,
|
||||
tainted: bool) -> bool:
|
||||
"""是否自动放行候选(只是候选判定,不是 verdict)。
|
||||
|
||||
污点进程一律 False;写操作与敏感路径在任何档位都不
|
||||
自动放行;default 档须已允许主机 + 路径前缀;cautious
|
||||
档须精确主机 + 精确路径;permissive 档只读 + 非敏感
|
||||
即候选(不必先前已允许)。
|
||||
"""
|
||||
if tainted:
|
||||
return False
|
||||
if request.method not in READ_METHODS:
|
||||
return False
|
||||
if _is_sensitive_path(request.path):
|
||||
return False
|
||||
if self.caution == "permissive":
|
||||
return True
|
||||
if self.caution == "cautious":
|
||||
return self._allowlisted(request, exact=True)
|
||||
return self._allowlisted(request, exact=False)
|
||||
|
||||
|
||||
__all__ = ["READ_METHODS", "WRITE_METHODS", "CAUTION_LEVELS",
|
||||
"DefaultPolicy"]
|
||||
@@ -1,221 +0,0 @@
|
||||
"""P8 presentation:后台任务完成后“是否值得呈现”的评估,与
|
||||
通知去重节流账本。
|
||||
|
||||
对齐 Muse B2(官方设计文):完成后台工作后评估是否值得呈现,
|
||||
只在真正有新内容或需要输入时通知用户。
|
||||
|
||||
判定是确定性规则(不是模型判断):
|
||||
- needs_input:需要用户输入 -> 通知。
|
||||
- failed/interrupted:失败、需要决策 -> 通知。
|
||||
- completed 且有实质新内容(新产物、改动文件、或显式 notify
|
||||
标志)-> 通知;否则常规完成只记日志。
|
||||
- cancelled / 未知终态 -> 只记日志(未知终态 fail-closed,
|
||||
不通知)。
|
||||
|
||||
账本:宿主级 AppStateStore(round-12 已迁移,见
|
||||
docs/chv-muse-p8-gap.md §6):state_dir/app-state.sqlite 的
|
||||
``p8-ledger`` 命名空间,一行存整份账本;旧
|
||||
notification-ledger.json 首次加载时自动导入并改名
|
||||
``.migrated``;行损坏/库打不开时回空账本(去重信息丢了只会
|
||||
导致一次重复通知,不影响任务本身),下次成功保存即覆盖自愈。
|
||||
同一任务同一原因在节流窗口(3600 秒,本项目自定)内只通知一次。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
from cloud.chv.app_state import AppStateStore, host_state_db_path
|
||||
|
||||
_LOGGER = logging.getLogger("cloud.cloud_hypervisor")
|
||||
|
||||
# 同一任务同一原因的通知节流窗口(秒,本项目自定)。
|
||||
NOTIFY_THROTTLE_SECONDS = 3600
|
||||
_LEDGER_FILENAME = "notification-ledger.json"
|
||||
_KV_NAMESPACE = "p8-ledger"
|
||||
_KV_KEY = "registry"
|
||||
_MAX_LEDGER_ENTRIES = 500
|
||||
|
||||
|
||||
def evaluate_presentation(result: dict) -> dict:
|
||||
"""评估一次后台任务终态是否值得呈现。
|
||||
|
||||
result 键:status(completed/failed/interrupted/cancelled/
|
||||
needs_input)、new_artifacts(list)、changed_files(list)、
|
||||
notify(bool,任务显式要求通知)。返回 {"present", "reason"};
|
||||
present 为 False 时调用方只记日志、不通知。"""
|
||||
if not isinstance(result, dict):
|
||||
return {"present": False, "reason": "unknown_status"}
|
||||
status = result.get("status")
|
||||
if status == "needs_input":
|
||||
return {"present": True, "reason": "needs_input"}
|
||||
if status in ("failed", "interrupted"):
|
||||
return {"present": True, "reason": "failed_needs_decision"}
|
||||
if status == "completed":
|
||||
return _evaluate_completed(result)
|
||||
if status == "cancelled":
|
||||
return {"present": False, "reason": "cancelled"}
|
||||
return {"present": False, "reason": "unknown_status"}
|
||||
|
||||
|
||||
def _evaluate_completed(result: dict) -> dict:
|
||||
"""完成态分支:有实质新内容才通知。"""
|
||||
artifacts = result.get("new_artifacts")
|
||||
changed = result.get("changed_files")
|
||||
has_new = bool(result.get("notify"))
|
||||
if isinstance(artifacts, list) and artifacts:
|
||||
has_new = True
|
||||
if isinstance(changed, list) and changed:
|
||||
has_new = True
|
||||
if has_new:
|
||||
return {"present": True, "reason": "substantive_new_content"}
|
||||
return {"present": False, "reason": "routine_completed"}
|
||||
|
||||
|
||||
def _open_store(state_dir: str) -> AppStateStore:
|
||||
"""打开宿主级状态库(每次调用新建连接,用完即关)。"""
|
||||
store = AppStateStore(host_state_db_path(state_dir))
|
||||
store.open()
|
||||
return store
|
||||
|
||||
|
||||
def _ledger_path(state_dir: str) -> str:
|
||||
"""旧版通知账本文件路径(只用于一次性迁移导入)。"""
|
||||
return os.path.join(state_dir, _LEDGER_FILENAME)
|
||||
|
||||
|
||||
def _quarantine(path: str) -> None:
|
||||
"""把损坏的账本挪到 <path>.corrupt 旁(尽力而为)。"""
|
||||
try:
|
||||
os.replace(path, path + ".corrupt")
|
||||
except OSError:
|
||||
_LOGGER.warning("chv_ledger_quarantine_failed path=%s", path)
|
||||
|
||||
|
||||
def _load_ledger(state_dir: str) -> dict:
|
||||
"""读账本:KV 行缺失时走一次性旧文件导入;行损坏/库打不
|
||||
开时回空结构(去重信息丢了只会导致一次重复通知,不影响
|
||||
任务本身;下次成功保存即覆盖自愈)。"""
|
||||
try:
|
||||
with _open_store(state_dir) as store:
|
||||
raw = store.kv_get(_KV_NAMESPACE, _KV_KEY)
|
||||
except (sqlite3.Error, OSError):
|
||||
_LOGGER.warning("chv_ledger_store_unreadable state_dir=%s",
|
||||
state_dir)
|
||||
return {"notified": {}}
|
||||
if raw is None:
|
||||
return _import_legacy(state_dir)
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except ValueError:
|
||||
_LOGGER.warning("chv_ledger_corrupt state_dir=%s", state_dir)
|
||||
return {"notified": {}}
|
||||
if not isinstance(data, dict) \
|
||||
or not isinstance(data.get("notified"), dict):
|
||||
return {"notified": {}}
|
||||
return data
|
||||
|
||||
|
||||
def _import_legacy(state_dir: str) -> dict:
|
||||
"""一次性导入旧 notification-ledger.json:合法即进 KV 并
|
||||
把文件改名 ``.migrated``;无文件回空;损坏则 quarantine
|
||||
后回空(旧口径)。导入失败时返回读到的数据、文件保留,
|
||||
下次加载重试。"""
|
||||
path = _ledger_path(state_dir)
|
||||
try:
|
||||
with open(path, "r") as fh:
|
||||
data = json.load(fh)
|
||||
except FileNotFoundError:
|
||||
return {"notified": {}}
|
||||
except (OSError, ValueError):
|
||||
_quarantine(path)
|
||||
return {"notified": {}}
|
||||
if not isinstance(data, dict) \
|
||||
or not isinstance(data.get("notified"), dict):
|
||||
_quarantine(path)
|
||||
return {"notified": {}}
|
||||
try:
|
||||
with _open_store(state_dir) as store:
|
||||
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
|
||||
except (sqlite3.Error, OSError):
|
||||
_LOGGER.warning("chv_ledger_import_failed state_dir=%s",
|
||||
state_dir)
|
||||
return data
|
||||
try:
|
||||
os.replace(path, path + ".migrated")
|
||||
except OSError:
|
||||
pass
|
||||
return data
|
||||
|
||||
|
||||
def _save_ledger(state_dir: str, data: dict) -> None:
|
||||
"""落盘账本到 KV 行(截断到上限);写失败只记日志。"""
|
||||
notified = data.get("notified", {})
|
||||
if len(notified) > _MAX_LEDGER_ENTRIES:
|
||||
items = sorted(notified.items(), key=lambda kv: kv[1])
|
||||
data["notified"] = dict(items[-_MAX_LEDGER_ENTRIES:])
|
||||
try:
|
||||
with _open_store(state_dir) as store:
|
||||
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
|
||||
except (sqlite3.Error, OSError):
|
||||
_LOGGER.warning("chv_ledger_save_failed state_dir=%s", state_dir)
|
||||
|
||||
|
||||
class NotificationLedger:
|
||||
"""通知去重节流账本(每任务每原因节流)。
|
||||
|
||||
should_notify(task_id, reason) 在窗口内未通知过时回 True
|
||||
并记下这次通知;窗口内已通知过回 False。账本落在宿主级
|
||||
AppStateStore(round-12 已迁移,见模块 docstring)。
|
||||
"""
|
||||
|
||||
def __init__(self, state_dir: str) -> None:
|
||||
"""建账本:state_dir 为空即 ValueError。"""
|
||||
if not state_dir or not isinstance(state_dir, str):
|
||||
raise ValueError("state_dir required")
|
||||
self.state_dir = state_dir
|
||||
|
||||
def should_notify(self, task_id: str, reason: str,
|
||||
now: float | None = None) -> bool:
|
||||
"""该任务该原因现在能不能通知:窗口内没通知过就记下
|
||||
并回 True,否则回 False。task_id/reason 为空即
|
||||
ValueError。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
if not task_id or not isinstance(task_id, str):
|
||||
raise ValueError("task_id required")
|
||||
if not reason or not isinstance(reason, str):
|
||||
raise ValueError("reason required")
|
||||
key = "%s:%s" % (task_id, reason)
|
||||
data = _load_ledger(self.state_dir)
|
||||
notified = data.get("notified", {})
|
||||
last = notified.get(key)
|
||||
if isinstance(last, (int, float)) \
|
||||
and now - last < NOTIFY_THROTTLE_SECONDS:
|
||||
return False
|
||||
notified[key] = now
|
||||
_save_ledger(self.state_dir, data)
|
||||
return True
|
||||
|
||||
|
||||
def present_or_log(result: dict, ledger: NotificationLedger,
|
||||
task_id: str,
|
||||
now: float | None = None) -> dict:
|
||||
"""评估 + 去重的一站式入口(worker/push 终态派发处调它)。
|
||||
|
||||
返回 {"present", "reason", "notified"}:present 是评估结论,
|
||||
notified 是本次是否真的发出通知(评估通过但窗口内已通知
|
||||
过时 notified 为 False、只记日志)。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
decision = evaluate_presentation(result)
|
||||
if not decision["present"]:
|
||||
return {"present": False, "reason": decision["reason"],
|
||||
"notified": False}
|
||||
notified = ledger.should_notify(task_id, decision["reason"], now)
|
||||
return {"present": True, "reason": decision["reason"],
|
||||
"notified": notified}
|
||||
@@ -1,312 +0,0 @@
|
||||
"""privsep worker 协议与客户端(feat-chv-muse-p6,叶子模块)。
|
||||
|
||||
对齐 Muse 官方安全文(security.muse.ai 2026-09-08,见
|
||||
docs/chv-muse-p6-gap.md):单元内的连接器 CLI 只解析参数、通过
|
||||
Unix socket 传递类型化参数与文件描述符;真正业务逻辑由 systemd
|
||||
沙箱化的 privsep worker 执行;每个 worker 由 cgroup 标识、有明确
|
||||
凭证白名单(改参数也索取不到别的凭证)。
|
||||
|
||||
本模块是协议 + 客户端 + 注册表(只用标准库,不依赖 cloud.*,
|
||||
客内 worker 守护进程可直接 import 它):
|
||||
- `WORKER_SPECS`:本期只认一个 worker(gitea),凭证白名单只含
|
||||
GITEA_TOKEN,操作只含 pull/push/status/ping,仓库只限
|
||||
~/workspace 下;
|
||||
- `WorkerRequest` / `WorkerResponse`:类型化 JSON 行(≤64KB),
|
||||
请求里绝不带任何凭证字段——凭证只由 git 经 P3 的 credential
|
||||
helper 向 authd 按次取,worker 与客户端都碰不到 token 值;
|
||||
- `PrivsepClient`:发请求前先按白名单校验(越权请求不出门),
|
||||
worker 崩溃/超时只返回类型化失败(WorkerRefusal),不抛原始
|
||||
异常、不拖垮调用方;返回的外部数据先过安全扫描再交给调用方,
|
||||
block 即阻断(内容不往下流)、flag 即标记复核。
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
import time
|
||||
|
||||
from cloud.chv.safety import (
|
||||
InjectionScanner,
|
||||
check_context_input,
|
||||
mark_untrusted,
|
||||
)
|
||||
|
||||
# 客内工作区根(worker 的仓库 confinement 基准)与默认 socket。
|
||||
GUEST_WORKSPACE = "/home/chv/workspace"
|
||||
DEFAULT_SOCKET_PATH = "/run/chv-gitea/worker.sock"
|
||||
MAX_MESSAGE_BYTES = 65536
|
||||
DEFAULT_TIMEOUT_SECONDS = 300
|
||||
|
||||
|
||||
class WorkerRefusal(Exception):
|
||||
"""privsep 拒绝:kind + 原因,不含任何凭证值。调用方按类型化
|
||||
失败处理,不把它当成 agent 崩溃。"""
|
||||
|
||||
def __init__(self, kind: str, message: str) -> None:
|
||||
super().__init__("privsep_refused:%s: %s" % (kind, message))
|
||||
self.kind = kind
|
||||
|
||||
|
||||
class WorkerSpec:
|
||||
"""一个 privsep worker 的注册信息:连接器 + 凭证白名单 +
|
||||
操作白名单 + socket + systemd 单元。"""
|
||||
|
||||
def __init__(self, connector: str, allowed_credentials: frozenset,
|
||||
operations: frozenset, socket_path: str,
|
||||
unit: str) -> None:
|
||||
self.connector = connector
|
||||
self.allowed_credentials = frozenset(allowed_credentials)
|
||||
self.operations = frozenset(operations)
|
||||
self.socket_path = socket_path
|
||||
self.unit = unit
|
||||
|
||||
|
||||
WORKER_SPECS: dict[str, WorkerSpec] = {
|
||||
"gitea": WorkerSpec(
|
||||
connector="gitea",
|
||||
allowed_credentials=frozenset({"GITEA_TOKEN"}),
|
||||
operations=frozenset({"pull", "push", "status", "ping"}),
|
||||
socket_path=DEFAULT_SOCKET_PATH,
|
||||
unit="chv-gitea-worker.service",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def spec_for(connector: str) -> WorkerSpec:
|
||||
"""取某连接器的 worker 注册信息;未知连接器即拒绝(不设默认
|
||||
放行,调用方不得以“没注册”绕过白名单)。"""
|
||||
spec = WORKER_SPECS.get(connector)
|
||||
if spec is None:
|
||||
raise WorkerRefusal(
|
||||
"unknown_connector",
|
||||
"连接器 %s 没有注册的 privsep worker" % connector)
|
||||
return spec
|
||||
|
||||
|
||||
def worker_cgroup_for(pid: int) -> str:
|
||||
"""读 /proc/<pid>/cgroup 的首行(worker 的 cgroup 标识,审计
|
||||
用)。读不到即空串,不抛错。"""
|
||||
try:
|
||||
with open("/proc/%d/cgroup" % pid) as fh:
|
||||
first = fh.readline().strip()
|
||||
return first
|
||||
except OSError:
|
||||
return ""
|
||||
|
||||
|
||||
def _repo_within_workspace(repo: str, workspace: str) -> str:
|
||||
"""仓库路径 confinement:realpath 后必须在 workspace 之下,
|
||||
且不能就是 workspace 本身。返回解析后的绝对路径,否则
|
||||
WorkerRefusal。"""
|
||||
base = os.path.realpath(workspace)
|
||||
target = os.path.realpath(repo)
|
||||
if os.path.commonpath([base, target]) != base or target == base:
|
||||
raise WorkerRefusal(
|
||||
"repo_outside_workspace",
|
||||
"仓目录只允许 %s 下" % workspace)
|
||||
return target
|
||||
|
||||
|
||||
class WorkerRequest:
|
||||
"""一次类型化 worker 请求:操作 + 连接器 + 仓库 + 请求 id。
|
||||
请求里绝无凭证字段(凭证只走 authd 替身,见差距表 §1)。"""
|
||||
|
||||
def __init__(self, op: str, connector: str,
|
||||
repo: str | None, request_id: str) -> None:
|
||||
self.op = op
|
||||
self.connector = connector
|
||||
self.repo = repo
|
||||
self.request_id = request_id
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""序列化(JSON 行用)。"""
|
||||
return {"op": self.op, "connector": self.connector,
|
||||
"repo": self.repo, "request_id": self.request_id}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls: type["WorkerRequest"], data: dict,
|
||||
workspace: str = GUEST_WORKSPACE) -> "WorkerRequest":
|
||||
"""反序列化 + 校验:未知连接器/操作、仓库越界、请求里带
|
||||
凭证字段一律拒绝。"""
|
||||
if not isinstance(data, dict):
|
||||
raise WorkerRefusal(
|
||||
"bad_request", "请求须是 JSON 对象")
|
||||
if "credential" in data or "token" in data \
|
||||
or "password" in data:
|
||||
raise WorkerRefusal(
|
||||
"credential_in_request",
|
||||
"请求里绝不带凭证字段")
|
||||
connector = data.get("connector")
|
||||
if not isinstance(connector, str):
|
||||
raise WorkerRefusal(
|
||||
"bad_request", "connector 缺失或不是字符串")
|
||||
spec = spec_for(connector)
|
||||
op = data.get("op")
|
||||
if not isinstance(op, str):
|
||||
raise WorkerRefusal(
|
||||
"bad_request", "op 缺失或不是字符串")
|
||||
if op not in spec.operations:
|
||||
raise WorkerRefusal(
|
||||
"operation_not_allowed",
|
||||
"操作 %r 不在 %s 的白名单内" % (op, connector))
|
||||
repo = data.get("repo")
|
||||
if repo is not None and repo != "":
|
||||
if not isinstance(repo, str):
|
||||
raise WorkerRefusal(
|
||||
"bad_request", "repo 不是字符串")
|
||||
repo = _repo_within_workspace(repo, workspace)
|
||||
else:
|
||||
repo = None
|
||||
request_id = data.get("request_id")
|
||||
if not isinstance(request_id, str) or not request_id:
|
||||
raise WorkerRefusal(
|
||||
"bad_request", "request_id 缺失或不是字符串")
|
||||
return cls(op, connector, repo, request_id)
|
||||
|
||||
|
||||
class WorkerResponse:
|
||||
"""一次类型化 worker 响应:ok + git 输出 + 返回码 + 安全扫描
|
||||
摘要 + 请求 id。ok 为 False 时 error 说明原因,不含值。"""
|
||||
|
||||
def __init__(self, ok: bool, stdout: str, stderr: str,
|
||||
returncode: int, safety: dict, request_id: str,
|
||||
error: str | None = None) -> None:
|
||||
self.ok = ok
|
||||
self.stdout = stdout
|
||||
self.stderr = stderr
|
||||
self.returncode = returncode
|
||||
self.safety = dict(safety)
|
||||
self.request_id = request_id
|
||||
self.error = error
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
"""序列化(JSON 行用)。"""
|
||||
return {"ok": self.ok, "stdout": self.stdout,
|
||||
"stderr": self.stderr, "returncode": self.returncode,
|
||||
"safety": self.safety, "request_id": self.request_id,
|
||||
"error": self.error}
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls: type["WorkerResponse"], data: dict
|
||||
) -> "WorkerResponse":
|
||||
"""反序列化 + 校验:形状不对即拒绝,不把脏回包当成成功。"""
|
||||
if not isinstance(data, dict) or "ok" not in data:
|
||||
raise WorkerRefusal(
|
||||
"bad_response", "worker 回包格式不对")
|
||||
if not isinstance(data["ok"], bool):
|
||||
raise WorkerRefusal(
|
||||
"bad_response", "worker 回包 ok 不是布尔值")
|
||||
request_id = data.get("request_id")
|
||||
if not isinstance(request_id, str):
|
||||
raise WorkerRefusal(
|
||||
"bad_response", "worker 回包 request_id 不对")
|
||||
return cls(bool(data["ok"]), str(data.get("stdout") or ""),
|
||||
str(data.get("stderr") or ""),
|
||||
int(data.get("returncode") or 0),
|
||||
dict(data.get("safety") or {}), request_id,
|
||||
data.get("error"))
|
||||
|
||||
|
||||
class PrivsepClient:
|
||||
"""privsep 客户端(agent 侧薄 CLI 的宿主版):发请求前按白名单
|
||||
校验,worker 崩溃/超时只给类型化失败;返回的外部数据先过安全
|
||||
扫描再交给调用方——block 即阻断(内容不往下流,只剩原因),
|
||||
flag 即原样返回、附扫描摘要标记复核。"""
|
||||
|
||||
def __init__(self, socket_path: str = DEFAULT_SOCKET_PATH,
|
||||
connector: str = "gitea",
|
||||
timeout: int = DEFAULT_TIMEOUT_SECONDS,
|
||||
scanner: InjectionScanner | None = None,
|
||||
workspace: str = GUEST_WORKSPACE) -> None:
|
||||
self.socket_path = socket_path
|
||||
self.connector = connector
|
||||
self.timeout = timeout
|
||||
self.scanner = scanner or InjectionScanner()
|
||||
self.workspace = workspace
|
||||
|
||||
def _check_request(self, request: WorkerRequest) -> WorkerSpec:
|
||||
"""客户端白名单校验:连接器须注册、操作须在白名单内;
|
||||
请求的 repo 须在 workspace 下。越权请求不出门。"""
|
||||
spec = spec_for(request.connector)
|
||||
if request.op not in spec.operations:
|
||||
raise WorkerRefusal(
|
||||
"operation_not_allowed",
|
||||
"操作 %r 不在 %s 的白名单内"
|
||||
% (request.op, request.connector))
|
||||
if request.repo is not None:
|
||||
_repo_within_workspace(request.repo, self.workspace)
|
||||
return spec
|
||||
|
||||
def _send(self, payload: dict) -> dict:
|
||||
"""一请求一响应(JSON 行,≤64KB)。socket 层面的失败
|
||||
全部转成 WorkerRefusal,不抛原始异常。"""
|
||||
try:
|
||||
conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
conn.settimeout(self.timeout)
|
||||
conn.connect(self.socket_path)
|
||||
except OSError as exc:
|
||||
raise WorkerRefusal(
|
||||
"worker_unavailable",
|
||||
"worker socket 连不上:%s" % exc)
|
||||
try:
|
||||
conn.sendall((json.dumps(payload) + "\n").encode("utf-8"))
|
||||
data = b""
|
||||
while len(data) < MAX_MESSAGE_BYTES:
|
||||
chunk = conn.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
data += chunk
|
||||
if data.endswith(b"\n"):
|
||||
break
|
||||
except OSError as exc:
|
||||
raise WorkerRefusal(
|
||||
"worker_io_failed",
|
||||
"worker 通信失败:%s" % exc)
|
||||
finally:
|
||||
conn.close()
|
||||
try:
|
||||
parsed = json.loads(data.decode("utf-8"))
|
||||
except (ValueError, UnicodeDecodeError):
|
||||
raise WorkerRefusal(
|
||||
"bad_response", "worker 回包不是合法 JSON")
|
||||
if not isinstance(parsed, dict):
|
||||
raise WorkerRefusal(
|
||||
"bad_response", "worker 回包不是 JSON 对象")
|
||||
return parsed
|
||||
|
||||
def call(self, request: WorkerRequest) -> WorkerResponse:
|
||||
"""发一次类型化请求,返回扫描过的类型化响应。"""
|
||||
self._check_request(request)
|
||||
raw = self._send(request.to_dict())
|
||||
response = WorkerResponse.from_dict(raw)
|
||||
if response.request_id != request.request_id:
|
||||
raise WorkerRefusal(
|
||||
"bad_response",
|
||||
"worker 回包 request_id 与请求不一致")
|
||||
source = "gitea:%s" % (request.repo or "unknown")
|
||||
checked = check_context_input(
|
||||
mark_untrusted(response.stdout + "\n" + response.stderr,
|
||||
source),
|
||||
scanner=self.scanner)
|
||||
worker_verdict = response.safety.get("verdict")
|
||||
if worker_verdict in ("block", "flag"):
|
||||
effective = worker_verdict
|
||||
else:
|
||||
effective = checked.verdict
|
||||
response.safety = checked.describe()
|
||||
if effective == "block":
|
||||
rules = response.safety.get("rules", [])
|
||||
if not rules:
|
||||
rules = [f.rule for f in checked.findings]
|
||||
return WorkerResponse(
|
||||
False, "", "", response.returncode,
|
||||
response.safety, response.request_id,
|
||||
"safety_blocked:%s" % ",".join(rules))
|
||||
return response
|
||||
|
||||
def ping(self) -> WorkerResponse:
|
||||
"""worker 存活探针(不跑 git,只走一遍协议与校验)。"""
|
||||
request = WorkerRequest(
|
||||
"ping", self.connector, None,
|
||||
"ping-%d" % int(time.time() * 1000))
|
||||
return self.call(request)
|
||||
@@ -1,515 +0,0 @@
|
||||
"""每用户常驻 VM 池(生产模块,P1 对齐 Muse,见 ADR-17)。
|
||||
|
||||
每用户一台常驻 VM:任务到时 resume 已暂停的 VM,跑完
|
||||
再 pause,不销毁;池空或任何一步失败都回落冷启动,
|
||||
绝不因池坏而丢任务。常驻 VM 的盘是每用户持久工作区盘
|
||||
(state_dir/chv-resident-<用户>/disk.raw),不是每任务
|
||||
新盘——隔离单元在常驻路径上从“每任务”变成“每用户”
|
||||
(单租户可接受,多租户重审前不得把常驻当租户隔离)。
|
||||
|
||||
状态机:cold → provisioning → paused → running → paused。
|
||||
acquire(user):有 paused 条目时 resume_vm + 等 SSH 就绪
|
||||
(复用执行器 wait_guest_ready,不另写判定),返回
|
||||
mode=resident;无可用/失败时返回 mode=cold。release(user):
|
||||
pause_vm,不销毁;pause 失败时条目留在 running(不删
|
||||
不销毁,worker 关闭时 reconcile 清理)。provision(user):
|
||||
wake_vm + pause_vm 建 paused 条目(惰性建池,失败回落)。
|
||||
reclaim_idle():paused 超 CHV_RESIDENT_IDLE_MINUTES(默认
|
||||
0=不自动关机)时 hibernate(pause + snapshot + shutdown)
|
||||
并删条目,快照文件留作恢复物,restore 是后续工作。
|
||||
|
||||
Registry(state_dir/resident-pool.json):文件 IO 与损坏
|
||||
语义复用 cloud.chv.orphans(原子写、损坏 quarantine、
|
||||
单条目非法只丢该条);重启后读到的条目一律 orphan,不
|
||||
复用,只能 reconcile 清理。条目按用户键隔离:release
|
||||
绝不把用过的 VM 放回给另一用户;同一用户同一时间只跑
|
||||
一个常驻任务,并发第二个任务回落冷启动。
|
||||
|
||||
诚实边界:resume 失败时 wait_guest_ready 的失败路径会
|
||||
stop_vm(含删工作区盘),条目随之丢弃——该路径的工作区
|
||||
以 Gitea 同步为准;带密钥 extra_env 的任务不走常驻
|
||||
(seed 是建池时定的,密钥只能冷启动随 seed 投递)。
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
|
||||
from cloud.chv.orphans import _read_registry, _write_registry
|
||||
|
||||
_LOGGER = logging.getLogger("cloud.cloud_hypervisor")
|
||||
|
||||
_REGISTRY_FILENAME = "resident-pool.json"
|
||||
_RESIDENT_STATES = ("paused", "running", "provisioning")
|
||||
_DEFAULT_USER = "default"
|
||||
|
||||
|
||||
def resident_user_key(user_id: str | None) -> str:
|
||||
"""用户键:非空即原样消毒(非法字符变 -),空/缺省
|
||||
即 "default"(当前单租户:任务 dict 无 user_id 时全
|
||||
部落到同一常驻 VM;多租户重审前不得当租户边界用)。"""
|
||||
raw = _DEFAULT_USER if user_id is None else str(user_id)
|
||||
key = re.sub(r"[^A-Za-z0-9_.-]", "-", raw).strip(".-")
|
||||
return key or _DEFAULT_USER
|
||||
|
||||
|
||||
def resident_task_for_user(user_id: str | None) -> dict:
|
||||
"""常驻伪任务:执行器按它算 VM 名/known_hosts/socket。
|
||||
vm_name({"id": "resident-alice"}) = "chv-resident-alice"。"""
|
||||
return {"id": "resident-" + resident_user_key(user_id)}
|
||||
|
||||
|
||||
def resident_registry_path_for_executor(executor: object) -> str | None:
|
||||
"""常驻 registry 路径:state_dir/resident-pool.json。
|
||||
无 state_dir 时返回 None(调用方按“无残留”处理)。"""
|
||||
state_dir = getattr(executor, "state_dir", None)
|
||||
if not state_dir:
|
||||
return None
|
||||
return os.path.join(state_dir, _REGISTRY_FILENAME)
|
||||
|
||||
|
||||
def reconcile_resident_orphans_for_executor(executor: object) -> list[str]:
|
||||
"""生产路径的常驻残留清理入口:返回已清理的 VM 名。
|
||||
只做清理:读 resident-pool.json → 每个上一世条目尽力
|
||||
shutdown_vm → 落盘为空 → best-effort 审计。绝不构造
|
||||
ResidentPool、绝不把任何条目发给任务。无 registry 路
|
||||
径时返回 []。文件 IO 与损坏语义复用 orphans。"""
|
||||
path = resident_registry_path_for_executor(executor)
|
||||
if not path:
|
||||
return []
|
||||
data, _corrupt = _read_registry(path)
|
||||
if data is None:
|
||||
return []
|
||||
cleaned: list[str] = []
|
||||
for entry in data.values():
|
||||
if not isinstance(entry, dict):
|
||||
continue
|
||||
if not entry.get("vm"):
|
||||
continue
|
||||
if entry.get("state") not in _RESIDENT_STATES:
|
||||
continue
|
||||
vm = str(entry["vm"])
|
||||
try:
|
||||
executor.shutdown_vm(vm) # type: ignore[attr-defined]
|
||||
except Exception:
|
||||
pass
|
||||
cleaned.append(vm)
|
||||
if cleaned:
|
||||
_write_registry(path, {})
|
||||
try:
|
||||
executor._emit( # type: ignore[attr-defined]
|
||||
"resident_pool_reconcile", {"id": None},
|
||||
"cleaned=%s" % ",".join(cleaned))
|
||||
except Exception:
|
||||
pass
|
||||
return cleaned
|
||||
|
||||
|
||||
class ResidentPool:
|
||||
"""每用户常驻 VM 池(生产路径,见模块 docstring)。
|
||||
|
||||
同一 state_dir 只活一个池实例:registry 只在构造时
|
||||
读一次,重启后条目一律 orphan(见 _load_registry)。
|
||||
公共方法全部加锁:并发 acquire/release/provision 只
|
||||
放行一路,其余回 cold/False,不把同一 VM 发给两个任务。
|
||||
"""
|
||||
|
||||
def __init__(self, executor: object, idle_minutes: int | None = None,
|
||||
registry_path: str | None = None) -> None:
|
||||
"""建池并读 registry:读到的条目一律 orphan(见模块
|
||||
docstring)。idle_minutes 未给时读执行器属性或
|
||||
CHV_RESIDENT_IDLE_MINUTES,非法/负数即 0(不自动关机)。"""
|
||||
self.executor = executor
|
||||
self._lock = threading.Lock()
|
||||
self._busy: set[str] = set()
|
||||
self._entries: dict = {}
|
||||
self.registry_path = registry_path
|
||||
if self.registry_path is None:
|
||||
self.registry_path = \
|
||||
resident_registry_path_for_executor(executor)
|
||||
if idle_minutes is None:
|
||||
idle_minutes = self._idle_minutes_from_env()
|
||||
self.idle_minutes = idle_minutes if idle_minutes > 0 else 0
|
||||
self.registry_corrupt = False
|
||||
self.registry_corrupt_entries = 0
|
||||
self._load_registry()
|
||||
|
||||
# -- registry ------------------------------------------------------
|
||||
def _load_registry(self) -> None:
|
||||
"""载入 registry:读到的条目一律标 orphan(绝不直接
|
||||
当 paused 发给任务,旧 VMM 进程已随旧 worker 消失)。
|
||||
无文件时为空池;损坏时 quarantine 并从空开始;单条
|
||||
目非法只丢该条并计数,不连坐其它条目。"""
|
||||
if not self.registry_path:
|
||||
return
|
||||
data, corrupt = _read_registry(self.registry_path)
|
||||
if corrupt:
|
||||
self.registry_corrupt = True
|
||||
return
|
||||
if data is None:
|
||||
return
|
||||
for key, entry in data.items():
|
||||
if not isinstance(entry, dict) \
|
||||
or not entry.get("vm") \
|
||||
or entry.get("state") not in _RESIDENT_STATES:
|
||||
self.registry_corrupt_entries += 1
|
||||
continue
|
||||
self._entries[key] = {
|
||||
"state": "orphan",
|
||||
"vm": entry["vm"],
|
||||
"user_id": entry.get("user_id") or key,
|
||||
"paused_at": None,
|
||||
"last_used": None,
|
||||
}
|
||||
|
||||
def _save_registry(self) -> None:
|
||||
"""落盘当前条目(只写 paused/running/provisioning;
|
||||
orphan 是待清理的上一世条目,reconcile 后即删,不写
|
||||
回)。调用方须已持锁;写失败只记日志,不炸任务流。"""
|
||||
if not self.registry_path:
|
||||
return
|
||||
data = {}
|
||||
for key, entry in self._entries.items():
|
||||
if entry["state"] not in _RESIDENT_STATES:
|
||||
continue
|
||||
data[key] = {
|
||||
"state": entry["state"],
|
||||
"vm": entry.get("vm"),
|
||||
"user_id": entry.get("user_id"),
|
||||
"paused_at": entry.get("paused_at"),
|
||||
"last_used": entry.get("last_used"),
|
||||
}
|
||||
_write_registry(self.registry_path, data)
|
||||
|
||||
def _emit(self, event: str, task: dict, detail: str) -> None:
|
||||
"""best-effort 审计:执行器无 _emit 或抛错都不炸
|
||||
池操作(与 WarmPool 同款)。"""
|
||||
emit = getattr(self.executor, "_emit", None)
|
||||
if not callable(emit):
|
||||
return
|
||||
try:
|
||||
emit(event, task, detail)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _idle_minutes_from_env(self) -> int:
|
||||
"""闲置阈值:执行器属性优先(P1-B 若经 config 落
|
||||
属性,直接复用),否则读 CHV_RESIDENT_IDLE_MINUTES;
|
||||
非数字/负数一律 0(不自动关机,不炸建池)。"""
|
||||
raw = getattr(self.executor, "resident_idle_minutes", None)
|
||||
if raw is None:
|
||||
env = getattr(self.executor, "_env", None)
|
||||
if env is None:
|
||||
env = os.environ
|
||||
raw = env.get("CHV_RESIDENT_IDLE_MINUTES", 0)
|
||||
if raw is None:
|
||||
return 0
|
||||
try:
|
||||
minutes = int(raw)
|
||||
except (TypeError, ValueError):
|
||||
return 0
|
||||
return minutes if minutes > 0 else 0
|
||||
|
||||
def _vm_name(self, resident_task: dict) -> str:
|
||||
"""常驻 VM 名:优先执行器 vm_name(与冷启动同款派
|
||||
生),鸭子型执行器无该方法时退回 chv- 前缀拼法。"""
|
||||
vm_name = getattr(self.executor, "vm_name", None)
|
||||
if callable(vm_name):
|
||||
try:
|
||||
return str(vm_name(resident_task))
|
||||
except Exception:
|
||||
pass
|
||||
return "chv-" + str(resident_task.get("id"))
|
||||
|
||||
# -- 计数 ----------------------------------------------------------
|
||||
def paused_count(self) -> int:
|
||||
"""当前 paused(可分配)的常驻条目数。"""
|
||||
with self._lock:
|
||||
return sum(1 for e in self._entries.values()
|
||||
if e.get("state") == "paused")
|
||||
|
||||
def orphan_count(self) -> int:
|
||||
"""上一世残留条目数(待 reconcile,不可分配)。"""
|
||||
with self._lock:
|
||||
return sum(1 for e in self._entries.values()
|
||||
if e.get("state") == "orphan")
|
||||
|
||||
# -- acquire / release ----------------------------------------------
|
||||
def _claim_paused(self, user_key: str) -> dict | None:
|
||||
"""占一条 paused 条目:加忙标记并返回其快照;无条目/
|
||||
非 paused/正忙时返回 None(调用方回落冷启动)。"""
|
||||
with self._lock:
|
||||
entry = self._entries.get(user_key)
|
||||
if entry is None or entry.get("state") != "paused":
|
||||
return None
|
||||
if user_key in self._busy:
|
||||
return None
|
||||
self._busy.add(user_key)
|
||||
return dict(entry)
|
||||
|
||||
def _resume_and_wait(self, resident_task: dict) -> bool:
|
||||
"""resume + 等 SSH 就绪(复用执行器判定)。
|
||||
resume 缺失的鸭子型执行器跳过 resume(与 WarmPool
|
||||
同款宽容);SSH 未就绪即 False(wait_guest_ready
|
||||
的失败路径已自行 stop_vm,调用方只须删条目)。"""
|
||||
resume = getattr(self.executor, "resume_vm", None)
|
||||
if callable(resume):
|
||||
try:
|
||||
resume(resident_task)
|
||||
except Exception:
|
||||
return False
|
||||
wait = getattr(self.executor, "wait_guest_ready", None)
|
||||
if not callable(wait):
|
||||
return True
|
||||
try:
|
||||
return bool(wait(resident_task))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _settle_acquire(self, user_key: str, ok: bool) -> None:
|
||||
"""acquire 收尾:成功标 running,失败删条目;都清
|
||||
忙标记并落盘。"""
|
||||
with self._lock:
|
||||
self._busy.discard(user_key)
|
||||
entry = self._entries.get(user_key)
|
||||
if entry is None:
|
||||
return
|
||||
if ok:
|
||||
entry["state"] = "running"
|
||||
entry["last_used"] = time.time()
|
||||
else:
|
||||
self._entries.pop(user_key, None)
|
||||
self._save_registry()
|
||||
|
||||
def acquire(self, user_id: str | None) -> dict:
|
||||
"""取常驻 VM:paused 条目 → resume + SSH 就绪 →
|
||||
mode=resident;无条目/正忙/失败 → mode=cold(调用
|
||||
方回落冷启动)。失败条目直接丢弃(VM 本身已被
|
||||
wait_guest_ready 的失败路径关掉,或 resume 已拒)。"""
|
||||
user_key = resident_user_key(user_id)
|
||||
entry = self._claim_paused(user_key)
|
||||
if entry is None:
|
||||
return {"mode": "cold", "vm": None, "task": None}
|
||||
resident_task = resident_task_for_user(user_id)
|
||||
ok = self._resume_and_wait(resident_task)
|
||||
self._settle_acquire(user_key, ok)
|
||||
if not ok:
|
||||
return {"mode": "cold", "vm": None, "task": None}
|
||||
self._emit("resident_pool_acquire", resident_task,
|
||||
"vm=%s user=%s" % (entry.get("vm"), user_key))
|
||||
return {"mode": "resident", "vm": entry.get("vm"),
|
||||
"task": resident_task}
|
||||
|
||||
def _pause(self, resident_task: dict) -> bool:
|
||||
"""pause_vm 尽力而为:失败返回 False,不抛。"""
|
||||
pause = getattr(self.executor, "pause_vm", None)
|
||||
if not callable(pause):
|
||||
return False
|
||||
try:
|
||||
pause(resident_task)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def release(self, user_id: str | None) -> bool:
|
||||
"""还常驻 VM:running 条目 → pause_vm,不销毁。
|
||||
无条目/非 running/正忙 → False;pause 失败时条目
|
||||
留在 running(不删不销毁,worker 关闭时 reconcile
|
||||
清理),返回 False。只动本用户的条目。"""
|
||||
user_key = resident_user_key(user_id)
|
||||
resident_task = resident_task_for_user(user_id)
|
||||
with self._lock:
|
||||
entry = self._entries.get(user_key)
|
||||
if entry is None or entry.get("state") != "running":
|
||||
return False
|
||||
if user_key in self._busy:
|
||||
return False
|
||||
self._busy.add(user_key)
|
||||
paused = self._pause(resident_task)
|
||||
with self._lock:
|
||||
self._busy.discard(user_key)
|
||||
entry = self._entries.get(user_key)
|
||||
if entry is None or not paused:
|
||||
return False
|
||||
entry["state"] = "paused"
|
||||
entry["paused_at"] = time.time()
|
||||
self._save_registry()
|
||||
self._emit("resident_pool_release", resident_task,
|
||||
"vm=%s user=%s" % (entry.get("vm"), user_key))
|
||||
return True
|
||||
|
||||
# -- provision -------------------------------------------------------
|
||||
def _wake_and_pause(self, resident_task: dict) -> bool:
|
||||
"""建池:wake_vm(冷启动全套,含 SSH 就绪)再
|
||||
pause_vm。任一步失败都尽力 shutdown,不留半醒的
|
||||
VM;调用方随后删条目回落冷启动。"""
|
||||
try:
|
||||
wake = getattr(self.executor, "wake_vm", None)
|
||||
pause = getattr(self.executor, "pause_vm", None)
|
||||
if not callable(wake) or not callable(pause):
|
||||
return False
|
||||
wake(resident_task)
|
||||
pause(resident_task)
|
||||
return True
|
||||
except Exception:
|
||||
shutdown = getattr(self.executor, "shutdown_vm", None)
|
||||
if callable(shutdown):
|
||||
try:
|
||||
shutdown(resident_task)
|
||||
except Exception:
|
||||
pass
|
||||
return False
|
||||
|
||||
def _finish_provision(self, user_key: str, ok: bool) -> str | None:
|
||||
"""建池收尾:成功标 paused 并记时间,失败删条目;
|
||||
都清忙标记并落盘。返回 VM 名或 None。"""
|
||||
with self._lock:
|
||||
self._busy.discard(user_key)
|
||||
entry = self._entries.get(user_key)
|
||||
if entry is None:
|
||||
return None
|
||||
if not ok:
|
||||
self._entries.pop(user_key, None)
|
||||
self._save_registry()
|
||||
return None
|
||||
entry["state"] = "paused"
|
||||
entry["paused_at"] = time.time()
|
||||
self._save_registry()
|
||||
return str(entry.get("vm"))
|
||||
|
||||
def provision(self, user_id: str | None) -> dict:
|
||||
"""建常驻 VM:已有 paused/running 条目直接返回(不
|
||||
重 wake);否则 wake + pause,成功落 paused 条目,
|
||||
失败删条目回 cold。并发建池只放行一路,其余回 cold。"""
|
||||
user_key = resident_user_key(user_id)
|
||||
resident_task = resident_task_for_user(user_id)
|
||||
with self._lock:
|
||||
entry = self._entries.get(user_key)
|
||||
if entry is not None and entry.get("state") in (
|
||||
"paused", "running"):
|
||||
return {"mode": "resident", "vm": entry.get("vm"),
|
||||
"task": resident_task}
|
||||
if user_key in self._busy:
|
||||
return {"mode": "cold", "vm": None, "task": None}
|
||||
self._busy.add(user_key)
|
||||
self._entries[user_key] = {
|
||||
"state": "provisioning",
|
||||
"vm": self._vm_name(resident_task),
|
||||
"user_id": user_id if user_id is not None
|
||||
else _DEFAULT_USER,
|
||||
"paused_at": None, "last_used": None}
|
||||
self._save_registry()
|
||||
ok = self._wake_and_pause(resident_task)
|
||||
vm = self._finish_provision(user_key, ok)
|
||||
if vm is None:
|
||||
return {"mode": "cold", "vm": None, "task": None}
|
||||
self._emit("resident_pool_provision", resident_task,
|
||||
"vm=%s user=%s" % (vm, user_key))
|
||||
return {"mode": "resident", "vm": vm, "task": resident_task}
|
||||
|
||||
# -- 空闲回收 --------------------------------------------------------
|
||||
def _hibernate_one(self, user_key: str) -> str | None:
|
||||
"""单条目休眠:hibernate_vm(快照到 state_dir 内
|
||||
resident.snapshot,成功后删条目)。失败留条目下次
|
||||
再试,返回 None;无 state_dir 或无 hibernate 入口
|
||||
同样留条目。"""
|
||||
with self._lock:
|
||||
entry = self._entries.get(user_key)
|
||||
if entry is None or entry.get("state") != "paused":
|
||||
return None
|
||||
if user_key in self._busy:
|
||||
return None
|
||||
self._busy.add(user_key)
|
||||
resident_task = resident_task_for_user(
|
||||
entry.get("user_id"))
|
||||
vm = str(entry.get("vm"))
|
||||
state_dir = getattr(self.executor, "state_dir", None)
|
||||
hibernate = getattr(self.executor, "hibernate_vm", None)
|
||||
if not state_dir or not callable(hibernate):
|
||||
with self._lock:
|
||||
self._busy.discard(user_key)
|
||||
return None
|
||||
snapshot_url = "file://" + os.path.join(
|
||||
state_dir, vm, "resident.snapshot")
|
||||
try:
|
||||
hibernate(resident_task, snapshot_url)
|
||||
except Exception:
|
||||
with self._lock:
|
||||
self._busy.discard(user_key)
|
||||
return None
|
||||
with self._lock:
|
||||
self._busy.discard(user_key)
|
||||
self._entries.pop(user_key, None)
|
||||
self._save_registry()
|
||||
self._emit("resident_pool_reclaim", resident_task,
|
||||
"vm=%s snapshot=%s" % (vm, snapshot_url))
|
||||
return vm
|
||||
|
||||
def reclaim_idle(self, now: float | None = None) -> list[str]:
|
||||
"""空闲回收:paused 超阈值的条目 hibernate(pause +
|
||||
snapshot + shutdown)并删条目。阈值 0 即关闭回收;
|
||||
未到阈值/正忙/无 paused_at 的条目不动。返回回收的
|
||||
VM 名。"""
|
||||
if self.idle_minutes <= 0:
|
||||
return []
|
||||
if now is None:
|
||||
now = time.time()
|
||||
due: list[str] = []
|
||||
with self._lock:
|
||||
for key, entry in self._entries.items():
|
||||
if entry.get("state") != "paused":
|
||||
continue
|
||||
if key in self._busy:
|
||||
continue
|
||||
paused_at = entry.get("paused_at")
|
||||
if paused_at is None:
|
||||
continue
|
||||
if now - paused_at > self.idle_minutes * 60:
|
||||
due.append(key)
|
||||
reclaimed: list[str] = []
|
||||
for key in due:
|
||||
vm = self._hibernate_one(key)
|
||||
if vm is not None:
|
||||
reclaimed.append(vm)
|
||||
return reclaimed
|
||||
|
||||
# -- 孤儿清理 --------------------------------------------------------
|
||||
def _shutdown_one(self, user_key: str) -> str | None:
|
||||
"""单条目尽力 shutdown 并从内存删掉(失败仍计入
|
||||
已清理,与 orphans 旧语义一致:不因单条卡住整批)。
|
||||
返回 VM 名或 None(条目已不在)。"""
|
||||
with self._lock:
|
||||
entry = self._entries.get(user_key)
|
||||
if entry is None:
|
||||
return None
|
||||
vm = str(entry.get("vm"))
|
||||
shutdown = getattr(self.executor, "shutdown_vm", None)
|
||||
if callable(shutdown):
|
||||
try:
|
||||
shutdown(vm)
|
||||
except Exception:
|
||||
pass
|
||||
with self._lock:
|
||||
self._entries.pop(user_key, None)
|
||||
return vm
|
||||
|
||||
def reconcile_orphans(self) -> list[str]:
|
||||
"""清理本池内存里的 orphan 条目(重启后读到的上一
|
||||
世条目):每个尽力 shutdown_vm,条目清掉并落盘。
|
||||
生产启动/关闭不走本方法,走模块级 facade。"""
|
||||
with self._lock:
|
||||
orphans = [k for k, e in self._entries.items()
|
||||
if e.get("state") == "orphan"]
|
||||
cleaned: list[str] = []
|
||||
for key in orphans:
|
||||
vm = self._shutdown_one(key)
|
||||
if vm is not None:
|
||||
cleaned.append(vm)
|
||||
if cleaned:
|
||||
with self._lock:
|
||||
self._save_registry()
|
||||
self._emit("resident_pool_reconcile", {"id": None},
|
||||
"cleaned=%s" % ",".join(cleaned))
|
||||
return cleaned
|
||||
@@ -1,281 +0,0 @@
|
||||
"""确定性安全检查层(feat-chv-muse-p6,叶子模块)。
|
||||
|
||||
对齐 Muse 官方安全文(security.muse.ai 2026-09-08,见
|
||||
docs/chv-muse-p6-gap.md):Harness 层外部数据进模型上下文时标注为
|
||||
“不受信任输入”;独立分类器层多组提示注入检测分类器并行运行,覆盖
|
||||
经文件与工具调用进入上下文的全部外部数据;hatch-safety 检查进出
|
||||
核心模型推理的请求/响应。
|
||||
|
||||
本模块是确定性规则版,不是模型分类器,不冒充(见差距表 §3.3):
|
||||
同一输入永远同一 verdict,可复现、可测试;能拦什么、不能拦什么
|
||||
单列在差距表里。只用标准库,不依赖 cloud.*,客内 worker 守护进程
|
||||
可直接 import 它(见 chv-guest/chv-gitea-worker.py)。
|
||||
|
||||
三件套:
|
||||
- 来源标记:`UntrustedData`(内容 + 来源 + 可信标志),随数据走;
|
||||
- 注入扫描:`InjectionScanner`,分级 block/flag/allow;
|
||||
- 推理边界审计:`AuditLogger`,只记指纹与 verdict,不记值;密钥
|
||||
与 token 值先过 `redact_secrets` 再记,绝不进审计。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
|
||||
_LOGGER = logging.getLogger("chv_safety")
|
||||
|
||||
# 注入规则:(rule_id, tier, 正则)。tier 只分 block / flag 两档;
|
||||
# 无命中即 allow。block 只给高置信模式;flag 给中置信,命中即
|
||||
# 标记复核、不自动执行后续写操作。
|
||||
_RULES: tuple[tuple[str, str, str], ...] = (
|
||||
# shell 执行链:curl/wget 的输出直接 piped 给 shell。
|
||||
("shell_pipe", "block",
|
||||
r"(?i)\b(curl|wget)\b[^|\n]{0,120}\|\s*(sh|bash|zsh|dash)\b"),
|
||||
# 命令替换里藏危险动词。
|
||||
("cmd_substitution", "block",
|
||||
r"\$\([^)]*\b(rm|sudo|eval|exec|curl|wget|chmod|chown|kill|dd)\b"),
|
||||
("backtick_command", "block",
|
||||
r"`[^`]*\b(rm|sudo|eval|exec|curl|wget)\b[^`]*`"),
|
||||
# 直接指令覆盖(英文)。
|
||||
("instruction_override_en", "block",
|
||||
r"(?i)\b(ignore|disregard|override|forget)\s+(all\s+)?"
|
||||
r"(previous|prior|above|your|system)\s+(system\s+)?"
|
||||
r"(instructions?|prompts?|rules?|guidelines?)"),
|
||||
("instruction_role_en", "block",
|
||||
r"(?i)\byou are now\b.{0,40}\b(follow|obey|execute|do)\b"),
|
||||
# 直接指令覆盖(中文)。
|
||||
("instruction_override_zh", "block",
|
||||
r"忽略(之前|以前|上述)?(的)?(所有)?(的)?"
|
||||
r"(指令|提示|指示|系统提示)"),
|
||||
("instruction_role_zh", "block",
|
||||
r"你现在是.{0,20}(助手|模型|系统|AI|机器人)"),
|
||||
# rm -rf 根目录 / 家目录(路径后不能再跟内容,/tmp/xxx 不算)。
|
||||
("rm_rf_root", "block",
|
||||
r"\brm\s+-[a-z]*r[a-z]*f\s+(/|~|/home|/etc|/usr)(?![\w/])"),
|
||||
# URL 里带 userinfo 口令(凭证泄露形态)。
|
||||
("url_userinfo", "block",
|
||||
r"(?i)\bhttps?://[^\s/:@]+:[^\s@/]+@[^\s/]+"),
|
||||
# data:/javascript: 伪协议。
|
||||
("dangerous_scheme", "block",
|
||||
r"(?i)\b(data|javascript|vbscript):[^\s]*"),
|
||||
# 代码执行原语(中置信:只 flag,不 block——正常文本里也可能
|
||||
# 出现这些词,block 会误伤)。
|
||||
("code_exec_primitive", "flag",
|
||||
r"(?i)\b(eval\s*\(|exec\s*\(|os\.system|subprocess\.|__import__\s*\()"),
|
||||
# 指令性文本(中置信)。
|
||||
("instruction_like_en", "flag",
|
||||
r"(?i)\b(follow these instructions|do not tell the user|"
|
||||
r"keep this (secret|confidential)|do not (mention|reveal|disclose))"),
|
||||
("instruction_like_zh", "flag",
|
||||
r"(不要告诉用户|不要让用户知道|秘密执行|不要透露|不要提及)"),
|
||||
# 可疑 URL(中置信):http 明文、IP 字面量主机、短链。
|
||||
("suspicious_url", "flag",
|
||||
r"(?i)\bhttp://(?:\d{1,3}\.){3}\d{1,3}\b|"
|
||||
r"\bhttps?://(?:bit\.ly|t\.co|tinyurl\.com|goo\.gl|is\.gd)/"),
|
||||
# 可疑拼接:分隔线后紧跟指令性文本。
|
||||
("suspicious_concatenation", "flag",
|
||||
r"(?m)^[-=*_]{3,}\s*$\n?.{0,200}?"
|
||||
r"(ignore|disregard|you are now|忽略|你现在是)"),
|
||||
)
|
||||
|
||||
# 脱敏模式:(正则, 替换)。只留键名/结构,值一律换成 [redacted]。
|
||||
_REDACT_RULES: tuple[tuple[str, str], ...] = (
|
||||
(r"(?i)(gitea_token|token|password|passwd|secret|api[_-]?key|"
|
||||
r"access[_-]?key)\s*[:=]\s*[\"']?\S+", r"\1=[redacted]"),
|
||||
(r"sk-[A-Za-z0-9]{10,}", "[redacted]"),
|
||||
(r"ghp_[A-Za-z0-9]{10,}", "[redacted]"),
|
||||
(r"gho_[A-Za-z0-9]{10,}", "[redacted]"),
|
||||
(r"github_pat_[A-Za-z0-9_]{10,}", "[redacted]"),
|
||||
(r"(?i)Authorization:[^\n]*", "Authorization: [redacted]"),
|
||||
(r"(?i)\bhttps?://[^\s/:@]+:[^\s@/]+@[^\s/]+",
|
||||
"https://[redacted]@"),
|
||||
)
|
||||
|
||||
_COMPILED_RULES: tuple[tuple[str, str, "re.Pattern[str]"], ...] = tuple(
|
||||
(rule_id, tier, re.compile(pattern))
|
||||
for rule_id, tier, pattern in _RULES
|
||||
)
|
||||
|
||||
_COMPILED_REDACT: tuple[tuple["re.Pattern[str]", str], ...] = tuple(
|
||||
(re.compile(pattern), repl) for pattern, repl in _REDACT_RULES
|
||||
)
|
||||
|
||||
|
||||
def redact_secrets(text: str) -> str:
|
||||
"""脱敏:把疑似密钥/口令的值换成 [redacted],只留键名与结构。
|
||||
输入非 str 即返回空串,绝不抛错。"""
|
||||
if not isinstance(text, str):
|
||||
return ""
|
||||
out = text
|
||||
for pattern, repl in _COMPILED_REDACT:
|
||||
out = pattern.sub(repl, out)
|
||||
return out
|
||||
|
||||
|
||||
def content_fingerprint(text: str) -> str:
|
||||
"""内容指纹(审计里唯一允许出现的内容派生形式):先脱敏再
|
||||
SHA-256,取前 16 位 hex。指纹不可逆推出原值。"""
|
||||
cleaned = redact_secrets(text)
|
||||
return hashlib.sha256(cleaned.encode("utf-8")).hexdigest()[:16]
|
||||
|
||||
|
||||
class Finding:
|
||||
"""一条规则命中:规则 id + 分级 + 截断后的证据片段(已脱敏)。"""
|
||||
|
||||
def __init__(self, rule: str, tier: str, excerpt: str) -> None:
|
||||
self.rule = rule
|
||||
self.tier = tier
|
||||
self.excerpt = redact_secrets(excerpt)[:120]
|
||||
|
||||
def describe(self) -> dict:
|
||||
"""审计用描述(只含规则与分级,不含原文)。"""
|
||||
return {"rule": self.rule, "tier": self.tier}
|
||||
|
||||
|
||||
class ScanResult:
|
||||
"""一次扫描结果:verdict + 命中列表。verdict 只取 block /
|
||||
flag / allow 三档;block 命中一个即 block。"""
|
||||
|
||||
def __init__(self, verdict: str, findings: list) -> None:
|
||||
self.verdict = verdict
|
||||
self.findings = list(findings)
|
||||
|
||||
def describe(self) -> dict:
|
||||
"""审计用描述(只含 verdict 与规则名,不含原文)。"""
|
||||
return {"verdict": self.verdict,
|
||||
"rules": [f.rule for f in self.findings]}
|
||||
|
||||
|
||||
class InjectionScanner:
|
||||
"""确定性注入扫描器:规则全跑完再定级;同一输入永远同一
|
||||
verdict。空文本即 allow,不抛错。"""
|
||||
|
||||
def scan(self, text: str) -> ScanResult:
|
||||
"""扫描一段外部文本,返回分级结果。"""
|
||||
if not isinstance(text, str) or not text:
|
||||
return ScanResult("allow", [])
|
||||
findings = []
|
||||
for rule_id, tier, pattern in _COMPILED_RULES:
|
||||
match = pattern.search(text)
|
||||
if match is not None:
|
||||
findings.append(Finding(rule_id, tier, match.group(0)))
|
||||
verdict = "allow"
|
||||
if any(f.tier == "block" for f in findings):
|
||||
verdict = "block"
|
||||
elif findings:
|
||||
verdict = "flag"
|
||||
return ScanResult(verdict, findings)
|
||||
|
||||
|
||||
class UntrustedData:
|
||||
"""带来源标记的外部数据:内容 + 来源 + 可信标志 + 收到时间。
|
||||
|
||||
默认不受信任(trusted=False);只有调用方核实过本地来源
|
||||
(如本仓代码、本地配置)才用 mark_trusted。标记随数据走,
|
||||
交给模型前须经 check_context_input。"""
|
||||
|
||||
def __init__(self, content: str, source: str, trusted: bool,
|
||||
received_at: float) -> None:
|
||||
self.content = content
|
||||
self.source = source
|
||||
self.trusted = trusted
|
||||
self.received_at = received_at
|
||||
|
||||
def describe(self) -> dict:
|
||||
"""审计用描述(只含来源与可信标志,不含内容)。"""
|
||||
return {"source": self.source, "trusted": self.trusted,
|
||||
"length": len(self.content),
|
||||
"fingerprint": content_fingerprint(self.content)}
|
||||
|
||||
|
||||
def mark_untrusted(content: str, source: str) -> UntrustedData:
|
||||
"""把一段外部数据标成不受信任(来源如 gitea:<owner/repo>)。"""
|
||||
return UntrustedData(content, source, False, time.time())
|
||||
|
||||
|
||||
def mark_trusted(content: str, source: str) -> UntrustedData:
|
||||
"""把一段已核实本地来源的数据标成可信。只给本地来源用;
|
||||
远端拉回的数据一律走 mark_untrusted,绝不直接标可信。"""
|
||||
return UntrustedData(content, source, True, time.time())
|
||||
|
||||
|
||||
def check_context_input(data: object,
|
||||
scanner: InjectionScanner | None = None
|
||||
) -> ScanResult:
|
||||
"""数据进模型上下文前的检查入口:扫一遍再定级。
|
||||
|
||||
受信任的数据只 flag、不 block(本仓控制面自身的文本不应被
|
||||
规则直接阻断,flag 进审计复核);不受信任的数据按规则全分级。
|
||||
返回的 ScanResult 由调用方执行:block 即阻断、flag 即标记复核。
|
||||
"""
|
||||
text = ""
|
||||
trusted = False
|
||||
if isinstance(data, UntrustedData):
|
||||
text = data.content
|
||||
trusted = data.trusted
|
||||
elif isinstance(data, str):
|
||||
text = data
|
||||
result = (scanner or InjectionScanner()).scan(text)
|
||||
if trusted and result.verdict == "block":
|
||||
capped = [Finding(f.rule, "flag", f.excerpt)
|
||||
for f in result.findings]
|
||||
return ScanResult("flag", capped)
|
||||
return result
|
||||
|
||||
|
||||
class AuditLogger:
|
||||
"""推理边界审计(hatch-safety 的确定性等价):记进出核心模型
|
||||
推理的请求/响应。只记任务 id、kind、内容指纹、长度、verdict;
|
||||
内容值绝不进审计(先脱敏再算指纹)。emit 回调失败只记日志,
|
||||
绝不抛错、不中断任务(沿用执行器 _emit 的既定语义)。"""
|
||||
|
||||
def __init__(self, emit: Callable[[dict], None] | None = None
|
||||
) -> None:
|
||||
self._emit = emit
|
||||
|
||||
def _record(self, event: str, task_id: str, kind: str,
|
||||
content: str, verdict: str) -> dict:
|
||||
record = {"event": event, "task_id": task_id, "kind": kind,
|
||||
"fingerprint": content_fingerprint(content),
|
||||
"length": len(content) if isinstance(content, str)
|
||||
else 0,
|
||||
"verdict": verdict}
|
||||
try:
|
||||
if self._emit is not None:
|
||||
self._emit(record)
|
||||
else:
|
||||
_LOGGER.info("safety_audit %s", record)
|
||||
except Exception as exc: # 审计失败不中断任务
|
||||
_LOGGER.error("safety_audit_failed event=%s error=%s",
|
||||
event, exc)
|
||||
return record
|
||||
|
||||
def log_request(self, task_id: str, kind: str, content: str
|
||||
) -> dict:
|
||||
"""记一次进推理的请求(任务 → 模型)。"""
|
||||
return self._record("safety_inference_request", task_id,
|
||||
kind, content, "allow")
|
||||
|
||||
def log_response(self, task_id: str, kind: str, content: str,
|
||||
verdict: str = "allow") -> dict:
|
||||
"""记一次出推理的响应(模型 → 调用方),附扫描 verdict。"""
|
||||
return self._record("safety_inference_response", task_id,
|
||||
kind, content, verdict)
|
||||
|
||||
|
||||
DEFAULT_AUDIT_LOGGER = AuditLogger()
|
||||
|
||||
|
||||
def audit_inference_request(task_id: str, kind: str, content: str
|
||||
) -> dict:
|
||||
"""模块级快捷方式:记一次进推理的请求(默认 logger)。"""
|
||||
return DEFAULT_AUDIT_LOGGER.log_request(task_id, kind, content)
|
||||
|
||||
|
||||
def audit_inference_response(task_id: str, kind: str, content: str,
|
||||
verdict: str = "allow") -> dict:
|
||||
"""模块级快捷方式:记一次出推理的响应(默认 logger)。"""
|
||||
return DEFAULT_AUDIT_LOGGER.log_response(task_id, kind, content,
|
||||
verdict)
|
||||
@@ -1,489 +0,0 @@
|
||||
"""Sentinel 式出口审批决策引擎(feat-chv-muse-p3,叶子模块)。
|
||||
|
||||
对齐 Muse 官方安全文(security.muse.ai 2026-09-08,见
|
||||
docs/chv-muse-p3-gap.md):Sentinel 是连接器动作与全部网络出口的
|
||||
唯一权限机构;L4 与 L7 评估(主机名、解析后与最终目的 IP、端口、
|
||||
协议、HTTP 方法、路径、解码后的实际请求);SSRF 防护(防止公开
|
||||
主机名在 DNS 解析后指向私有基础设施);审批是严格能力
|
||||
(strict capabilities),绑定到特定连接器/目的地与用例,支持
|
||||
一次性/会话级/任务级/时限级授权;审批 UI 不在对话内。
|
||||
|
||||
本模块只是决策层(纯函数 + 内存态队列),不是网络层强制:
|
||||
- 它不拦截客内流量。客内直连的强制仍靠 Tinyproxy 白名单 +
|
||||
宿主防火墙(第三棒落地中);eBPF cgroup 拦截 + LSM hook 污点
|
||||
传播是后续工作,见差距表 §3.4,不得把本模块写成“出口已全部
|
||||
经 Sentinel”。
|
||||
- 本期两个接线点:① 执行器 wake 时的 Gitea 出口预审(起 VMM 前
|
||||
fail-closed);② 客内 authd 放凭证前评估。两者都调本模块的
|
||||
`evaluate()`,决策记录进审计(无值)。
|
||||
- `ask` 的语义:主机不在白名单、也不在拒绝名单时返回 ask,并记入
|
||||
`ApprovalQueue` 待审批;调用方遇到 ask 必须停止执行、不得自动
|
||||
放行。审批 UI 不在对话内——本期只留队列,不实现任何对话内审批
|
||||
入口(见差距表 §1)。
|
||||
|
||||
本模块只用标准库,不依赖 cloud.*,客内 authd 守护进程可直接
|
||||
import 它(见 chv-guest/authd.py)。解析器可注入(`resolve`
|
||||
参数),默认 `socket.getaddrinfo`;测试用假解析器,绝不联网。
|
||||
审计记录里绝不含凭证值、也绝不含解析出的 IP 以外的内容——
|
||||
reason 只含网段名与策略名。
|
||||
|
||||
P5 纵深(feat-chv-muse-p5):evaluate() 可接三件套——污点追踪器
|
||||
(taint.py:污点进程失去自动放行、回到正常审批流)、默认策略
|
||||
(policy.py:干净进程 + 只读 + 已允许 + 低风险可自动放行,
|
||||
三档谨慎度)、五档严格能力(capabilities.py:一次/会话/任务/
|
||||
时限/永久,默认仍单次);审批答复只走 approval_channel.py 的
|
||||
独立通道(nonce + 过期 + HMAC),对话内容伪造无效;eBPF 程序
|
||||
源码与加载器在 ebpf_taint.py(本机实测加载被 seccomp 挡,见
|
||||
docs/chv-muse-p5-gap.md §3)。顶层仍只用标准库(客内 authd
|
||||
直接 import 不受影响);五档能力签发是函数内懒 import,只在
|
||||
tier != "one-shot" 时触发。
|
||||
"""
|
||||
|
||||
import ipaddress
|
||||
import socket
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable, Iterable
|
||||
from typing import Protocol
|
||||
|
||||
|
||||
class _TaintTracker(Protocol):
|
||||
"""污点追踪器口径(P5-A 的 TaintTracker 须满足):只读
|
||||
一个方法,不持有追踪器实例、不 import taint 模块——
|
||||
客内 authd 直接 import 本模块时不受影响。"""
|
||||
|
||||
def is_tainted(self, pid: int) -> bool: ...
|
||||
|
||||
|
||||
class _DefaultPolicy(Protocol):
|
||||
"""默认策略口径(P5-B 的 DefaultPolicy 须满足)。"""
|
||||
|
||||
def auto_allow_eligible(self, request: "EgressRequest",
|
||||
tainted: bool) -> bool: ...
|
||||
|
||||
# SSRF 挡截:解析到这些网段的 IP 一律 deny。公开主机名解析到
|
||||
# 私有地址即挡(Muse 安全文的 SSRF 防护口径);解析失败也
|
||||
# deny(fail-closed,不把“解析不出”当成“可以放行”)。
|
||||
_BLOCKED_NETWORKS = tuple(
|
||||
ipaddress.ip_network(net) for net in (
|
||||
"0.0.0.0/8", # 本网络 / 未指定
|
||||
"10.0.0.0/8", # RFC1918
|
||||
"100.64.0.0/10", # CGNAT 共享地址空间
|
||||
"127.0.0.0/8", # 回环
|
||||
"169.254.0.0/16", # 链路本地
|
||||
"172.16.0.0/12", # RFC1918
|
||||
"192.0.0.0/24", # IETF 协议分配
|
||||
"192.0.2.0/24", # 文档 TEST-NET-1
|
||||
"192.88.99.0/24", # 6to4 中继(已废弃)
|
||||
"192.168.0.0/16", # RFC1918
|
||||
# 注意:198.18.0.0/15(基准测试)故意不在表里——本部署的
|
||||
# egress/DNS 基础设施自用该网段,gitea.tose.top 在此环境
|
||||
# 真实解析到 198.18.139.152;挡掉它会把真 Gitea 主机也拒了。
|
||||
"198.51.100.0/24", # 文档 TEST-NET-2
|
||||
"203.0.113.0/24", # 文档 TEST-NET-3
|
||||
"224.0.0.0/4", # 组播
|
||||
"240.0.0.0/4", # 保留
|
||||
"255.255.255.255/32",
|
||||
"::1/128", # 回环
|
||||
"::/128", # 未指定
|
||||
"fc00::/7", # 唯一本地
|
||||
"fe80::/10", # 链路本地
|
||||
"ff00::/8", # 组播
|
||||
))
|
||||
|
||||
# 每连接器的 L7 路径策略:(方法, 匹配方式, 模式)。
|
||||
# gitea:API 只放 /api/v1/;git 智能 HTTP 只放三条(clone/pull/push
|
||||
# 的实际路径,见 git http-protocol)加裸 .git(round-11 真机修复
|
||||
# 层 4:git 智能 HTTP 的发现请求只发裸 .git 路径,裸路径只放 GET、
|
||||
# 一律当 pull;POST 裸 .git 仍拒绝)。其它路径(Web UI、登录页、
|
||||
# 任意管理接口)一律不在本连接器的能力内。
|
||||
# feat-chv-muse-p9 的 inference 条目:推理连接器只放
|
||||
# POST /v1/responses 与 GET /v1/models(/v1/models 是否必需待
|
||||
# 真机核实);scheme/端口见下面两表。本文件其余内容与 P5 的
|
||||
# 25bcd35 一致,合入时以本文件为准(它是 P3+P5+P9 的并集)。
|
||||
_CONNECTOR_PATH_RULES = {
|
||||
"gitea": (
|
||||
("GET", "prefix", "/api/v1/"),
|
||||
("POST", "prefix", "/api/v1/"),
|
||||
("GET", "suffix", ".git/info/refs"),
|
||||
("GET", "suffix", ".git"),
|
||||
("POST", "suffix", ".git/git-upload-pack"),
|
||||
("POST", "suffix", ".git/git-receive-pack"),
|
||||
),
|
||||
"inference": (
|
||||
("POST", "prefix", "/v1/responses"),
|
||||
("GET", "prefix", "/v1/models"),
|
||||
),
|
||||
}
|
||||
|
||||
# 每连接器允许的 scheme/端口(默认:只 https/443;http 只在
|
||||
# 显式放行时才出现,本期没有任何连接器放 http)。注意:推理的
|
||||
# http 本地中继(字面环回/私网 IP,如 http://172.17.0.1:28084)
|
||||
# 不走 evaluate(见 inference_channel 的本地中继分支),不要
|
||||
# “顺手”把 http 加进下面两表——那会放宽公网路径的 scheme 检查。
|
||||
_CONNECTOR_SCHEMES = {"gitea": frozenset({"https"}),
|
||||
"inference": frozenset({"https"})}
|
||||
_CONNECTOR_PORTS = {"gitea": frozenset({443}),
|
||||
"inference": frozenset({443})}
|
||||
|
||||
# 默认放凭证有效期(秒):与 authd 按次释放同款 300 秒。
|
||||
DEFAULT_CAPABILITY_TTL_SECONDS = 300
|
||||
|
||||
|
||||
def _blocked_network_for(ip_text: str):
|
||||
"""该 IP 落在哪个挡截网段里(无即 None)。非法 IP 文本
|
||||
按“挡不住也说不清”处理:返回 0.0.0.0/8(deny),绝不
|
||||
把解析异常当成放行信号。"""
|
||||
try:
|
||||
addr = ipaddress.ip_address(ip_text)
|
||||
except ValueError:
|
||||
return _BLOCKED_NETWORKS[0]
|
||||
for net in _BLOCKED_NETWORKS:
|
||||
if addr.version == net.version and addr in net:
|
||||
return net
|
||||
return None
|
||||
|
||||
|
||||
def _default_resolve(host: str, port: int) -> list:
|
||||
"""默认解析器:socket.getaddrinfo 去重后的 IP 列表。"""
|
||||
infos = socket.getaddrinfo(host, port, proto=socket.IPPROTO_TCP)
|
||||
out = []
|
||||
for info in infos:
|
||||
ip = info[4][0]
|
||||
if ip not in out:
|
||||
out.append(ip)
|
||||
return out
|
||||
|
||||
|
||||
class EgressRequest:
|
||||
"""一次出口请求(决策输入):连接器 + 方法 + 目标 + 解析后
|
||||
IP(解析在 evaluate 里做,这里只记目标描述,不记任何凭证值)。"""
|
||||
|
||||
def __init__(self, connector: str, method: str, scheme: str,
|
||||
host: str, port: int, path: str) -> None:
|
||||
self.connector = connector
|
||||
self.method = (method or "").upper()
|
||||
self.scheme = (scheme or "").lower()
|
||||
self.host = (host or "").lower()
|
||||
self.port = int(port)
|
||||
self.path = path or "/"
|
||||
|
||||
def describe(self) -> dict:
|
||||
"""审计用描述(只含连接器/目标/方法,不含值)。"""
|
||||
return {"connector": self.connector, "method": self.method,
|
||||
"scheme": self.scheme, "host": self.host,
|
||||
"port": self.port, "path": self.path}
|
||||
|
||||
|
||||
class Capability:
|
||||
"""严格能力(Muse 口径:绑定到特定连接器/目的地与用例)。
|
||||
|
||||
一次 evaluate().capability 只对:一个连接器、一个主机、声明
|
||||
的方法与路径模式、在有效期内、剩余次数内有效。越界使用
|
||||
(换主机、换方法、换路径、过期、用完)一律 `matches()` 为
|
||||
False,调用方必须重新走 evaluate,不得自行续期。"""
|
||||
|
||||
def __init__(self, connector: str, host: str,
|
||||
methods: Iterable[str],
|
||||
path_rules: Iterable[tuple[str, str, str]],
|
||||
ttl_seconds: int = DEFAULT_CAPABILITY_TTL_SECONDS,
|
||||
max_uses: int = 1, now: float | None = None) -> None:
|
||||
self.id = uuid.uuid4().hex[:16]
|
||||
self.connector = connector
|
||||
self.host = (host or "").lower()
|
||||
self.methods = frozenset(methods)
|
||||
self.path_rules = tuple(path_rules)
|
||||
# None = 不限(五档能力的 session/task/timed/permanent
|
||||
# 用;单次档仍是 int,行为不变)。
|
||||
self.expires_at: float | None = (
|
||||
(now if now is not None else time.time())
|
||||
+ ttl_seconds)
|
||||
self.max_uses: int | None = max_uses
|
||||
self.uses_left: int | None = max_uses
|
||||
|
||||
def matches(self, request: EgressRequest,
|
||||
now: float | None = None) -> bool:
|
||||
"""该请求是否在能力绑定内(不消耗次数,只判定)。"""
|
||||
if request.connector != self.connector:
|
||||
return False
|
||||
if request.host != self.host:
|
||||
return False
|
||||
if request.method not in self.methods:
|
||||
return False
|
||||
if not _path_allowed(request, self.path_rules):
|
||||
return False
|
||||
at = now if now is not None else time.time()
|
||||
if self.expires_at is not None and at >= self.expires_at:
|
||||
return False
|
||||
if self.uses_left is not None and self.uses_left <= 0:
|
||||
return False
|
||||
return True
|
||||
|
||||
def use(self, request: EgressRequest,
|
||||
now: float | None = None) -> bool:
|
||||
"""消耗一次能力:绑定内才扣次数并返回 True,否则 False
|
||||
(不抛错,调用方按 deny 处理)。"""
|
||||
if not self.matches(request, now=now):
|
||||
return False
|
||||
if self.uses_left is not None:
|
||||
self.uses_left -= 1
|
||||
return True
|
||||
|
||||
|
||||
class Decision:
|
||||
"""一次审批决策:allow / deny / ask + 原因 + 能力(仅 allow
|
||||
时有)+ 审计记录(无值)。"""
|
||||
|
||||
def __init__(self, verdict: str, reasons: Iterable[str],
|
||||
capability: Capability | None = None,
|
||||
tainted: bool = False) -> None:
|
||||
assert verdict in ("allow", "deny", "ask")
|
||||
self.verdict = verdict
|
||||
self.reasons = list(reasons)
|
||||
self.capability = capability
|
||||
self.tainted = tainted
|
||||
|
||||
def audit(self, request: EgressRequest) -> dict:
|
||||
"""审计记录:事件名按 verdict,detail 只含目标描述与原因。"""
|
||||
record = request.describe()
|
||||
record["event"] = "sentinel_egress_" + self.verdict
|
||||
record["verdict"] = self.verdict
|
||||
record["reasons"] = list(self.reasons)
|
||||
if self.tainted:
|
||||
record["tainted"] = True
|
||||
if self.capability is not None:
|
||||
record["capability_id"] = self.capability.id
|
||||
return record
|
||||
|
||||
|
||||
def _path_allowed(request: EgressRequest, rules) -> bool:
|
||||
"""路径是否在该连接器的 L7 规则内。"""
|
||||
for method, kind, pattern in rules:
|
||||
if request.method != method:
|
||||
continue
|
||||
if kind == "prefix" and request.path.startswith(pattern):
|
||||
return True
|
||||
if kind == "suffix" and request.path.endswith(pattern):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _scope_str(scope: object | None, key: str) -> str | None:
|
||||
"""从 scope(dict 或带属性的对象)取一键,只认 str,
|
||||
别的当缺失(fail-closed,不猜)。"""
|
||||
if scope is None:
|
||||
return None
|
||||
value = scope.get(key) if isinstance(scope, dict) \
|
||||
else getattr(scope, key, None)
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def _one_shot_capability(request: EgressRequest, rules: tuple,
|
||||
now: float | None) -> Capability:
|
||||
"""P3 默认能力:单次、300 秒。"""
|
||||
return Capability(
|
||||
request.connector, request.host, {request.method}, rules,
|
||||
ttl_seconds=DEFAULT_CAPABILITY_TTL_SECONDS,
|
||||
max_uses=1, now=now)
|
||||
|
||||
|
||||
def _issue_capability(request: EgressRequest, rules: tuple,
|
||||
tier: str, scope: object | None,
|
||||
now: float | None) -> tuple[Capability, bool]:
|
||||
"""按档位签发严格能力;非法组合(缺用例/会话 id、
|
||||
timed 缺 ttl、permanent 未显式开启)回落单次(更严格)
|
||||
并报告 fallback=True,不抛错。"""
|
||||
if tier == "one-shot":
|
||||
return _one_shot_capability(request, rules, now), False
|
||||
from cloud.chv.capabilities import CapabilityIssuer
|
||||
issuer = CapabilityIssuer()
|
||||
try:
|
||||
cap = issuer.issue(
|
||||
tier, request.connector, request.host,
|
||||
{request.method}, rules,
|
||||
use_case=_scope_str(scope, "use_case") or "",
|
||||
session_id=_scope_str(scope, "session_id"),
|
||||
task_id=_scope_str(scope, "task_id"), now=now)
|
||||
except ValueError:
|
||||
return _one_shot_capability(request, rules, now), True
|
||||
return cap, False
|
||||
|
||||
|
||||
def _is_tainted(tracker: _TaintTracker | None,
|
||||
pid: int | None) -> bool:
|
||||
"""该 pid 是否已污点:未接追踪器或未给 pid 即 False
|
||||
(不阻断评估,污点只关自动放行)。"""
|
||||
if tracker is None or pid is None:
|
||||
return False
|
||||
return bool(tracker.is_tainted(pid))
|
||||
|
||||
|
||||
def _safety_verdict(policy: "SentinelPolicy",
|
||||
request: EgressRequest, rules: tuple,
|
||||
resolve: Callable[[str, int], list] | None
|
||||
) -> Decision | None:
|
||||
"""步骤 2–6:scheme/端口/拒绝名单/解析失败/SSRF 私网
|
||||
IP → deny;全部通过返回 None(继续往下走)。"""
|
||||
if request.scheme not in _CONNECTOR_SCHEMES.get(
|
||||
request.connector, frozenset()):
|
||||
return Decision("deny", ["scheme_not_allowed"])
|
||||
if request.port not in _CONNECTOR_PORTS.get(
|
||||
request.connector, frozenset()):
|
||||
return Decision("deny", ["port_not_allowed"])
|
||||
if request.host in policy.deny_hosts:
|
||||
return Decision("deny", ["host_denied"])
|
||||
resolver = resolve or _default_resolve
|
||||
try:
|
||||
ips = resolver(request.host, request.port)
|
||||
except Exception:
|
||||
return Decision("deny", ["dns_resolution_failed"])
|
||||
if not ips:
|
||||
return Decision("deny", ["dns_resolution_failed"])
|
||||
for ip in ips:
|
||||
net = _blocked_network_for(ip)
|
||||
if net is not None:
|
||||
return Decision("deny", ["ssrf_private_ip:%s" % net])
|
||||
return None
|
||||
|
||||
|
||||
def _auto_allow_decision(request: EgressRequest, rules: tuple,
|
||||
default_policy: _DefaultPolicy | None,
|
||||
tainted: bool, tier: str,
|
||||
scope: object | None,
|
||||
now: float | None) -> Decision | None:
|
||||
"""默认策略自动放行:未接策略、污点进程、不合候选条件
|
||||
时返回 None(走正常审批流);命中即 allow + 按档位签发
|
||||
能力(仍须过连接器 L7 路径规则)。"""
|
||||
if default_policy is None:
|
||||
return None
|
||||
if not default_policy.auto_allow_eligible(request, tainted):
|
||||
return None
|
||||
if not _path_allowed(request, rules):
|
||||
return None
|
||||
capability, fallback = _issue_capability(
|
||||
request, rules, tier, scope, now)
|
||||
reasons = ["policy_auto_allow"]
|
||||
if fallback:
|
||||
reasons.append("capability_tier_fallback")
|
||||
return Decision("allow", reasons, capability=capability,
|
||||
tainted=tainted)
|
||||
|
||||
|
||||
class SentinelPolicy:
|
||||
"""Sentinel 策略:白名单主机 + 拒绝主机 + 连接器 L7 规则。
|
||||
|
||||
默认拒绝(default-deny):主机不在白名单、也不在拒绝名单时
|
||||
返回 ask(不是 allow);拒绝名单命中、SSRF 命中、scheme/
|
||||
端口/路径越界时返回 deny。白名单只放行“已知且解析干净”的
|
||||
目标——白名单不是“解析后是什么都行”,SSRF 检查对白名单
|
||||
主机同样生效(DNS 可被投毒,白名单主机解析到私网 IP 仍 deny)。"""
|
||||
|
||||
def __init__(self, allow_hosts: Iterable[str] = (),
|
||||
deny_hosts: Iterable[str] = (),
|
||||
connector_paths: dict[str, tuple] | None = None
|
||||
) -> None:
|
||||
self.allow_hosts = frozenset(
|
||||
h.lower() for h in (allow_hosts or ()))
|
||||
self.deny_hosts = frozenset(
|
||||
h.lower() for h in (deny_hosts or ()))
|
||||
self.connector_paths = dict(
|
||||
connector_paths if connector_paths is not None
|
||||
else _CONNECTOR_PATH_RULES)
|
||||
|
||||
@classmethod
|
||||
def from_env(cls: type["SentinelPolicy"], env: dict | None,
|
||||
extra_allow_hosts: Iterable[str] = ()
|
||||
) -> "SentinelPolicy":
|
||||
"""从宿主环境变量建策略:CHV_SENTINEL_ALLOW_HOSTS(逗号
|
||||
分隔主机名)+ 调用方追加的白名单(执行器 wake 预审时把
|
||||
配置的 Gitea 主机追加进来——只放行配置的 Gitea 主机,
|
||||
不是放行任意主机)。"""
|
||||
raw = (env or {}).get("CHV_SENTINEL_ALLOW_HOSTS") or ""
|
||||
hosts = [h.strip().lower() for h in raw.split(",") if h.strip()]
|
||||
hosts += [h.lower() for h in (extra_allow_hosts or ())]
|
||||
return cls(allow_hosts=hosts)
|
||||
|
||||
def evaluate(self, request: EgressRequest,
|
||||
resolve: Callable[[str, int], list] | None = None,
|
||||
now: float | None = None,
|
||||
taint_tracker: _TaintTracker | None = None,
|
||||
pid: int | None = None,
|
||||
default_policy: _DefaultPolicy | None = None,
|
||||
tier: str = "one-shot",
|
||||
scope: object | None = None) -> Decision:
|
||||
"""评估一次出口请求(按顺序,首个命中即定 verdict):
|
||||
|
||||
1–6. 安全检查(未知连接器/scheme/端口/拒绝名单/
|
||||
解析失败/SSRF 私网 IP)→ deny;
|
||||
7. 默认策略自动放行(须接 default_policy;干净进程
|
||||
+ 只读 + 已允许 + 低风险才放行;污点进程一律不放,
|
||||
回到正常审批流);
|
||||
8. 主机不在白名单 → ask;9. 方法/路径越界 → deny;
|
||||
10. 全部通过 → allow + 按档位签发严格能力(默认
|
||||
单次、300 秒;tier/scope 由调用方传)。
|
||||
不传接线参数时行为与 P3 完全一致。
|
||||
"""
|
||||
rules = self.connector_paths.get(request.connector)
|
||||
if rules is None:
|
||||
return Decision("deny", ["unknown_connector"])
|
||||
denied = _safety_verdict(self, request, rules, resolve)
|
||||
if denied is not None:
|
||||
return denied
|
||||
tainted = _is_tainted(taint_tracker, pid)
|
||||
auto = _auto_allow_decision(
|
||||
request, rules, default_policy, tainted,
|
||||
tier, scope, now)
|
||||
if auto is not None:
|
||||
return auto
|
||||
if request.host not in self.allow_hosts:
|
||||
return Decision("ask", ["host_not_allowlisted"],
|
||||
tainted=tainted)
|
||||
if not _path_allowed(request, rules):
|
||||
return Decision("deny", ["path_not_allowed"],
|
||||
tainted=tainted)
|
||||
capability, fallback = _issue_capability(
|
||||
request, rules, tier, scope, now)
|
||||
reasons = ["policy_allow"]
|
||||
if fallback:
|
||||
reasons.append("capability_tier_fallback")
|
||||
return Decision("allow", reasons,
|
||||
capability=capability, tainted=tainted)
|
||||
|
||||
class ApprovalQueue:
|
||||
"""ask 待审批队列(内存态):ask 出现即记入,执行停止。
|
||||
|
||||
设计约束(Muse 口径):审批 UI 不在对话内。本期只留队列与
|
||||
查询/标记接口,不实现任何对话内审批入口,也不自动放行。
|
||||
队列是内存态,进程重启即清——ask 本来就意味着“这次执行
|
||||
已停止”,重启后须重新申请。"""
|
||||
|
||||
def __init__(self, clock: Callable[[], float] | None = None) -> None:
|
||||
self._pending: dict[str, dict] = {}
|
||||
self._clock = clock or time.time
|
||||
|
||||
def record(self, request: EgressRequest,
|
||||
reasons: Iterable[str]) -> str:
|
||||
"""记入一条待审批,返回 pending id(只含目标描述与原因)。"""
|
||||
pending_id = uuid.uuid4().hex[:16]
|
||||
record = request.describe()
|
||||
record["pending_id"] = pending_id
|
||||
record["reasons"] = list(reasons)
|
||||
record["ts"] = self._clock()
|
||||
self._pending[pending_id] = record
|
||||
return pending_id
|
||||
|
||||
def list_pending(self) -> list:
|
||||
"""全部待审批(副本,按记录时间排序)。"""
|
||||
return [dict(v) for _, v in sorted(
|
||||
self._pending.items(), key=lambda kv: kv[1]["ts"])]
|
||||
|
||||
def resolve(self, pending_id: str, verdict: str) -> bool:
|
||||
"""标记一条待审批已处理(verdict 只记 allow/deny,不在
|
||||
这里执行放行——放行须调用方重新走 evaluate)。未知 id
|
||||
返回 False,不抛错。"""
|
||||
if pending_id not in self._pending:
|
||||
return False
|
||||
if verdict not in ("allow", "deny"):
|
||||
return False
|
||||
self._pending.pop(pending_id)
|
||||
return True
|
||||
@@ -1,256 +0,0 @@
|
||||
"""Cloud Hypervisor 客机 SSH 传输(叶子模块,不依赖
|
||||
cloud.sandbox,避免循环导入)。
|
||||
|
||||
安全模型(本轨的核心变化,替代 #216 的
|
||||
StrictHostKeyChecking=no + UserKnownHostsFile=/dev/null):
|
||||
|
||||
- 每 VM 一个独立 known_hosts 文件:
|
||||
state_dir/<vm 名>/known_hosts(与每任务磁盘同目录,
|
||||
任务销毁时不自动删——known_hosts 留在宿主 state_dir 里,
|
||||
供事后审计该 VM 当时钉死的 key;VM 名本身已含任务 id,
|
||||
不会串到别的 VM 上)。
|
||||
- 首次信任(TOFU,边界必须写清,不假装是强验证):
|
||||
首次启动时客机 host key 未知,执行器用 ssh-keyscan 经
|
||||
宿主转发端口(127.0.0.1:<port>,只走本机回环)抓一次
|
||||
当时呈现的 key,写入该 VM 的 known_hosts 文件;此后
|
||||
同一 VM 的每次 SSH 都用 StrictHostKeyChecking=yes +
|
||||
该文件校验,VM 被替换 / 中间人换 key 时 SSH 直接拒绝,
|
||||
不再静默接受。首次捕获本身不验证“这台客机就是我们要
|
||||
的那台”——它只防首次之后的替换;更强的方案是镜像构建
|
||||
/cloud-init 时把客机 host key 预置进 known_hosts(未来
|
||||
工作,见 docs/chv-production-readiness.md 的待办),本轨
|
||||
在 mock 测试里只能验证“捕获→写文件→后续命令带钉死
|
||||
选项”的管线,绝不声称 mock 验证了首次密钥的真实性。
|
||||
- 密钥红线不变且收紧(feat-chv-secfix,H2/M1):argv 里只有
|
||||
key 文件路径,绝无 key 值;网关密钥类变量(见
|
||||
_CHV_SECRET_ENV_NAMES,与 Docker 的 _DOCKER_SECRET_ENV_NAMES
|
||||
同款“只传名”口径,但 CHV 更进一步:值连宿主子进程环境
|
||||
都不进,改走 seed 盘里的客内文件,见 hostkeys.write_seed)
|
||||
出现在 extra_env 里时本函数直接 ValueError——传输层
|
||||
fail-closed,任何调用方都不能再把密钥值拼回 argv。
|
||||
非密钥 extra_env 仍可走 env 前缀(值在 argv 里可见,
|
||||
但按定义不是密钥)。
|
||||
- 远程 shell 转义(feat-chv-secfix,M1):SSH 协议的 exec
|
||||
请求本身是一条字符串,ssh 客户端把多个参数用空格拼成
|
||||
远程 shell 命令——此前 inner 参数原样进 argv,prompt 里
|
||||
的 ;/$()/引号/换行会在远程 shell 里被二次解析。现在每个
|
||||
远程 token 都经 shlex.quote 引用后仍以列表元素形式返回
|
||||
(ssh 拼接后远程解析结果与本地 list 语义一致);客内
|
||||
agent 本来就是 root,本项修的是“任务执行语义不被 prompt
|
||||
改写 + 审计不失真”,不是“防客内提权”(客内 root 无
|
||||
需注入即可为所欲为,边界见执行器文档)。
|
||||
"""
|
||||
|
||||
import os
|
||||
import shlex
|
||||
|
||||
from cloud.chv.atomic_file import atomic_write_text
|
||||
from cloud.chv.gitea_sync import (
|
||||
GITEA_SYNC_PLAIN_KEYS,
|
||||
GITEA_SYNC_SECRET_KEYS,
|
||||
)
|
||||
|
||||
# CHV 密钥类环境变量名:值绝不进 ssh argv。Docker 线是
|
||||
# “-e 只传名、值由 CLI 子进程环境提供”(值仍在宿主进程
|
||||
# 环境里,/proc/<pid>/environ 对同用户可见);CHV 线是
|
||||
# “值只进 seed 盘客内文件”(宿主进程表与宿主进程环境里
|
||||
# 都没有值),比 Docker 线更严——差异见执行器文档。
|
||||
_CHV_SECRET_ENV_NAMES = frozenset({"SUB2API_API_KEY"}) \
|
||||
| GITEA_SYNC_SECRET_KEYS
|
||||
|
||||
# CHV guest-env 类非密钥变量名:值可进 SSH env 前缀(按定义
|
||||
# 不是密钥),但 chv-sync boot 在 cloud-init 时运行、SSH 之
|
||||
# 前,只能读 seed 投递的 /etc/chv/guest-env,所以这些键也
|
||||
# 必须随 seed 写进 guest-env(见 split_guest_env 与执行器
|
||||
# wake_vm),只走 SSH 前缀是不够的。
|
||||
_CHV_GUEST_ENV_NAMES = frozenset(GITEA_SYNC_PLAIN_KEYS)
|
||||
|
||||
|
||||
def split_secret_env(extra_env: dict | None) -> tuple[dict, dict]:
|
||||
"""把 extra_env 拆成(密钥子集,非密钥子集)。"""
|
||||
secrets, plain = {}, {}
|
||||
for key, value in dict(extra_env or {}).items():
|
||||
if key in _CHV_SECRET_ENV_NAMES:
|
||||
secrets[key] = value
|
||||
else:
|
||||
plain[key] = value
|
||||
return secrets, plain
|
||||
|
||||
|
||||
def split_guest_env(extra_env: dict | None) -> tuple[dict, dict]:
|
||||
"""把 extra_env 拆成(须写进 seed guest-env 的子集,
|
||||
非密钥子集)。
|
||||
|
||||
guest-env 子集 = 密钥子集 + _CHV_GUEST_ENV_NAMES 命中的
|
||||
非密钥同步变量(GITEA_URL 等 boot 时就要用的键);第二
|
||||
个返回仍是全部非密钥子集(含 guest-env 命中键——它们
|
||||
任务期也要经 SSH env 前缀进客机环境,seed 只解决
|
||||
boot 时可见性,不替代任务期前缀)。"""
|
||||
secrets, plain = split_secret_env(extra_env)
|
||||
guest_env = dict(secrets)
|
||||
for key, value in plain.items():
|
||||
if key in _CHV_GUEST_ENV_NAMES:
|
||||
guest_env[key] = value
|
||||
return guest_env, plain
|
||||
|
||||
|
||||
def _merge_no_proxy(extra_env, egress_proxy):
|
||||
"""调用方 NO_PROXY/no_proxy 与 egress 的 localhost、
|
||||
127.0.0.1 去重合并(单条 NO_PROXY 的值)。"""
|
||||
parts = []
|
||||
if extra_env:
|
||||
for key in ("NO_PROXY", "no_proxy"):
|
||||
val = extra_env.get(key)
|
||||
if val:
|
||||
parts += [p.strip() for p in str(val).split(",")
|
||||
if p.strip()]
|
||||
merged = []
|
||||
for part in parts:
|
||||
if part not in merged:
|
||||
merged.append(part)
|
||||
if egress_proxy:
|
||||
for part in ("localhost", "127.0.0.1"):
|
||||
if part not in merged:
|
||||
merged.append(part)
|
||||
return merged
|
||||
|
||||
|
||||
def _env_assignments(extra_env, egress_proxy):
|
||||
"""env 前缀的赋值列表(feat-chv-closeup,H1 附带)。
|
||||
|
||||
NO_PROXY 覆盖顺序已核实并修掉:旧拼装是 extra_env 的
|
||||
赋值在前、egress 的 NO_PROXY 在后,`env` 遇重复赋值以
|
||||
后者为准(实测 env NO_PROXY=a NO_PROXY=b → b),调用方
|
||||
传的 NO_PROXY 被 egress 的 localhost,127.0.0.1 静默覆盖。
|
||||
现改为合并(见 _merge_no_proxy),单条 NO_PROXY 发出。
|
||||
egress 代理本身须赢:egress 已设时调用方的
|
||||
HTTPS_PROXY/HTTP_PROXY(含小写,curl 等工具读小写优先,
|
||||
留着会架空 egress)一律不发出,只发 egress 的三条;
|
||||
无 egress 时调用方的值原样保留(只去重,不丢)。"""
|
||||
skip = {"NO_PROXY", "no_proxy"}
|
||||
if egress_proxy:
|
||||
skip |= {"HTTPS_PROXY", "HTTP_PROXY",
|
||||
"https_proxy", "http_proxy"}
|
||||
assignments = []
|
||||
if extra_env:
|
||||
for key in sorted(extra_env):
|
||||
if key in skip:
|
||||
continue
|
||||
assignments.append("%s=%s"
|
||||
% (key, shlex.quote(
|
||||
str(extra_env[key]))))
|
||||
merged = _merge_no_proxy(extra_env, egress_proxy)
|
||||
if egress_proxy:
|
||||
assignments += [
|
||||
"HTTPS_PROXY=%s" % shlex.quote(egress_proxy),
|
||||
"HTTP_PROXY=%s" % shlex.quote(egress_proxy),
|
||||
"NO_PROXY=%s" % shlex.quote(",".join(merged))]
|
||||
elif merged:
|
||||
assignments += [
|
||||
"NO_PROXY=%s" % shlex.quote(",".join(merged))]
|
||||
return assignments
|
||||
|
||||
|
||||
def remote_command(inner_cmd: list, egress_proxy: str | None = None,
|
||||
extra_env: dict | None = None,
|
||||
guest_env_file: str | None = None) -> list:
|
||||
"""构造远程 shell 命令的 token 列表(不含 ssh 前缀)。
|
||||
|
||||
每个 token 都经 shlex.quote:ssh 客户端把 argv 用空格拼
|
||||
成一条远程命令时,远程 shell 解析回来的恰好是本地 list
|
||||
语义。guest_env_file 给定时,远程命令先 `set -a; . <文件>;
|
||||
set +a`(把 seed 投递的密钥文件 source 进环境并 export),
|
||||
再接 env 前缀与客内命令;文件路径本身不是密钥。
|
||||
extra_env 含密钥名时 ValueError(调用方须先 split 出
|
||||
密钥子集走 seed 投递,见 split_secret_env)。env 前缀的
|
||||
拼装见 _env_assignments(NO_PROXY 合并、egress 优先)。"""
|
||||
if extra_env:
|
||||
secrets, _ = split_secret_env(extra_env)
|
||||
if secrets:
|
||||
raise ValueError(
|
||||
"密钥类环境变量 %s 不得进 ssh argv,须走 seed "
|
||||
"文件投递" % ",".join(sorted(secrets)))
|
||||
tokens = []
|
||||
if guest_env_file:
|
||||
tokens += ["set", "-a;", ".",
|
||||
shlex.quote(guest_env_file) + ";", "set", "+a;"]
|
||||
env_assignments = _env_assignments(extra_env, egress_proxy)
|
||||
inner = [shlex.quote(str(arg)) for arg in inner_cmd]
|
||||
if env_assignments:
|
||||
tokens += ["env"] + env_assignments + inner
|
||||
else:
|
||||
tokens += inner
|
||||
return tokens
|
||||
|
||||
|
||||
def known_hosts_path(state_dir: str, vm_name: str) -> str:
|
||||
"""该 VM 的 known_hosts 文件路径:state_dir/<vm>/known_hosts。"""
|
||||
return os.path.join(state_dir, vm_name, "known_hosts")
|
||||
|
||||
|
||||
def build_ssh_command(ssh_port: int, ssh_key: str | None,
|
||||
ssh_user: str, egress_proxy: str | None,
|
||||
known_hosts_file: str, inner_cmd: list,
|
||||
ssh_host: str = "127.0.0.1",
|
||||
extra_env: dict | None = None,
|
||||
guest_env_file: str | None = None) -> list:
|
||||
"""构造 SSH 命令:ssh -p <端口> -i <key 文件> + 钉死选项
|
||||
+ user@<ssh_host> + 客内命令(远程 token 均经 shlex.quote,
|
||||
见 remote_command)。
|
||||
|
||||
feat-chv-fix(P0-4):ssh_host 不再在传输层写死回环,
|
||||
由执行器解析传入(显式 CHV_SSH_HOST / net 客机 IP /
|
||||
显式转发宣称下的回环);本函数缺省回环仅为旧调用点
|
||||
兼容,执行器路径绝不依赖该缺省。
|
||||
feat-chv-secfix(H2):extra_env 只接受非密钥变量;
|
||||
密钥类变量须先经 split_secret_env 拆出、走 seed 盘客内
|
||||
文件投递,调用方把 guest_env_file 传进来(客内 seed
|
||||
写出的 /etc/chv/guest-env),本函数在远程命令前 source
|
||||
它;密钥值绝不出现在本函数返回的任何 argv 元素里。
|
||||
egress_proxy 已设时客内命令前加 env 前缀(代理地址本身
|
||||
不是密钥;客机出口只走该白名单代理——但这只是 env 约
|
||||
定,网络层强制见宿主防火墙部署清单,代码不声称已隔离)。"""
|
||||
remote = remote_command(inner_cmd, egress_proxy=egress_proxy,
|
||||
extra_env=extra_env,
|
||||
guest_env_file=guest_env_file)
|
||||
return ["ssh", "-p", str(ssh_port),
|
||||
"-i", ssh_key,
|
||||
"-o", "StrictHostKeyChecking=yes",
|
||||
"-o", "UserKnownHostsFile=%s" % known_hosts_file,
|
||||
"-o", "BatchMode=yes",
|
||||
"%s@%s" % (ssh_user, ssh_host)] + remote
|
||||
|
||||
|
||||
def parse_keyscan_output(text: str, host: str = "127.0.0.1") -> list:
|
||||
"""从 ssh-keyscan 的 stdout 里挑出目标主机的 host key
|
||||
行:形如 "[<host>]:<port> ssh-ed25519 AAAA..."(-p 时
|
||||
的标准写法)或 "<host> ssh-rsa AAAA..."。其它主机行、
|
||||
注释行、空行一律丢掉,绝不把陌生主机的 key 写进文件。
|
||||
host 缺省回环仅为旧调用点兼容;客机 IP 方案下应传
|
||||
解析出的客机 IP。"""
|
||||
kept = []
|
||||
for line in (text or "").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
parts = line.split()
|
||||
if len(parts) < 3:
|
||||
continue
|
||||
line_host, keytype = parts[0], parts[1]
|
||||
if line_host != host \
|
||||
and not line_host.startswith("[%s]:" % host):
|
||||
continue
|
||||
if not keytype.startswith("ssh-"):
|
||||
continue
|
||||
kept.append(line)
|
||||
return kept
|
||||
|
||||
|
||||
def write_known_hosts(path: str, key_lines: list) -> None:
|
||||
"""原子写 known_hosts:随机 tmp 名 + fsync 后 rename
|
||||
(feat-chv-closeup,P2-18:旧固定 `<path>.tmp` 无 fsync),
|
||||
文件权限 0600(只含公钥行,仍按敏感文件对待)。"""
|
||||
directory = os.path.dirname(path)
|
||||
if directory:
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
atomic_write_text(path, "\n".join(key_lines) + "\n", mode=0o600)
|
||||
@@ -1,440 +0,0 @@
|
||||
"""P8 swarm:子代理群——建群、派代理、隔离、工具注册与审计。
|
||||
|
||||
对齐 Muse B3(官方安全文):启动成群的子代理、构建自己的工具;
|
||||
VM 需要足够资源处理并发子代理与 crons。资源上限是本项目自定
|
||||
(见 docs/chv-muse-p8-gap.md §4),不是 Meta 官方值。
|
||||
|
||||
隔离模型:每代理一个独立任务 id + 独立工作区(父任务工作区下
|
||||
按代理 id 分目录,`resolve_inside` 门守着);代理之间互不可见
|
||||
对方工作区(`check_agent_path` 否定测试钉住)。真正执行在代
|
||||
理自己的 VM/单元里(P1 常驻池 + P2 单元是底座),本层只做登
|
||||
记、派发与审计,不执行工具代码。
|
||||
|
||||
存储:宿主级 AppStateStore(round-12 已迁移,见
|
||||
docs/chv-muse-p8-gap.md §6):state_dir/app-state.sqlite 的
|
||||
``p8-swarm`` 命名空间,一行存整份登记文档(原子写由 SQLite
|
||||
事务保证);旧 swarms.json 首次加载时自动导入并改名
|
||||
``.migrated``;KV 行损坏/库打不开时回空结构(fail-safe,
|
||||
orphans 同款口径),下次成功保存即覆盖自愈。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sqlite3
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
|
||||
from cloud.chv.app_state import AppStateStore, host_state_db_path
|
||||
from cloud.chv.compat215 import resolve_inside
|
||||
|
||||
_LOGGER = logging.getLogger("cloud.cloud_hypervisor")
|
||||
|
||||
MAX_AGENTS_PER_SWARM = 8
|
||||
MAX_AGENTS_LIMIT = 32
|
||||
MAX_SWARMS_PER_PROJECT = 10
|
||||
MAX_TOOL_NAME_LEN = 100
|
||||
MAX_AGENT_PROMPT_LEN = 20000
|
||||
MAX_AUDIT_ENTRIES = 200
|
||||
MAX_INVOCATION_ENTRIES = 200
|
||||
AGENT_STATUSES = ("pending", "running", "done", "failed")
|
||||
_REGISTRY_FILENAME = "swarms.json"
|
||||
_KV_NAMESPACE = "p8-swarm"
|
||||
_KV_KEY = "registry"
|
||||
|
||||
|
||||
def _open_store(state_dir: str) -> AppStateStore:
|
||||
"""打开宿主级状态库(每次调用新建连接,用完即关)。"""
|
||||
store = AppStateStore(host_state_db_path(state_dir))
|
||||
store.open()
|
||||
return store
|
||||
|
||||
|
||||
def _registry_path(state_dir: str) -> str:
|
||||
"""旧版群登记文件路径(只用于一次性迁移导入)。"""
|
||||
return os.path.join(state_dir, _REGISTRY_FILENAME)
|
||||
|
||||
|
||||
def _quarantine(path: str) -> None:
|
||||
"""把损坏的登记文件挪到 <path>.corrupt 旁(尽力而为)。"""
|
||||
try:
|
||||
os.replace(path, path + ".corrupt")
|
||||
except OSError:
|
||||
_LOGGER.warning("chv_swarm_registry_quarantine_failed "
|
||||
"path=%s", path)
|
||||
|
||||
|
||||
def _empty_registry() -> dict:
|
||||
"""空登记结构。"""
|
||||
return {"swarms": {}, "tools": {}, "invocations": [],
|
||||
"audit": []}
|
||||
|
||||
|
||||
def _load_registry(state_dir: str) -> dict:
|
||||
"""读登记:KV 行缺失时走一次性旧文件导入;行损坏/库打不
|
||||
开时回空结构(损坏丢状态是 orphans 同款 fail-safe 口径,
|
||||
见差距表;下次成功保存即覆盖自愈)。"""
|
||||
try:
|
||||
with _open_store(state_dir) as store:
|
||||
raw = store.kv_get(_KV_NAMESPACE, _KV_KEY)
|
||||
except (sqlite3.Error, OSError):
|
||||
_LOGGER.warning("chv_swarm_registry_store_unreadable "
|
||||
"state_dir=%s", state_dir)
|
||||
return _empty_registry()
|
||||
if raw is None:
|
||||
return _import_legacy(state_dir)
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except ValueError:
|
||||
_LOGGER.warning("chv_swarm_registry_corrupt state_dir=%s",
|
||||
state_dir)
|
||||
return _empty_registry()
|
||||
if not isinstance(data, dict):
|
||||
return _empty_registry()
|
||||
data.setdefault("swarms", {})
|
||||
data.setdefault("tools", {})
|
||||
data.setdefault("invocations", [])
|
||||
data.setdefault("audit", [])
|
||||
return data
|
||||
|
||||
|
||||
def _import_legacy(state_dir: str) -> dict:
|
||||
"""一次性导入旧 swarms.json:合法即进 KV 并把文件改名
|
||||
``.migrated``;无文件回空;损坏则 quarantine 后回空(旧
|
||||
口径)。导入失败(库写不进)时返回读到的数据、文件保留,
|
||||
下次加载重试。"""
|
||||
path = _registry_path(state_dir)
|
||||
try:
|
||||
with open(path, "r") as fh:
|
||||
data = json.load(fh)
|
||||
except FileNotFoundError:
|
||||
return _empty_registry()
|
||||
except (OSError, ValueError):
|
||||
_quarantine(path)
|
||||
return _empty_registry()
|
||||
if not isinstance(data, dict) \
|
||||
or not isinstance(data.get("swarms", {}), dict) \
|
||||
or not isinstance(data.get("tools", {}), dict) \
|
||||
or not isinstance(data.get("invocations", []), list) \
|
||||
or not isinstance(data.get("audit", []), list):
|
||||
_quarantine(path)
|
||||
return _empty_registry()
|
||||
data.setdefault("swarms", {})
|
||||
data.setdefault("tools", {})
|
||||
data.setdefault("invocations", [])
|
||||
data.setdefault("audit", [])
|
||||
try:
|
||||
with _open_store(state_dir) as store:
|
||||
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
|
||||
except (sqlite3.Error, OSError):
|
||||
_LOGGER.warning("chv_swarm_registry_import_failed "
|
||||
"state_dir=%s", state_dir)
|
||||
return data
|
||||
try:
|
||||
os.replace(path, path + ".migrated")
|
||||
except OSError:
|
||||
pass
|
||||
return data
|
||||
|
||||
|
||||
def _save_registry(state_dir: str, data: dict) -> None:
|
||||
"""落盘登记到 KV 行(审计与调用记录各截断到上限);写失
|
||||
败只记日志,不炸调用方。"""
|
||||
data["audit"] = data.get("audit", [])[-MAX_AUDIT_ENTRIES:]
|
||||
data["invocations"] = data.get("invocations", [])[
|
||||
-MAX_INVOCATION_ENTRIES:]
|
||||
try:
|
||||
with _open_store(state_dir) as store:
|
||||
store.kv_put(_KV_NAMESPACE, _KV_KEY, json.dumps(data))
|
||||
except (sqlite3.Error, OSError):
|
||||
_LOGGER.warning("chv_swarm_registry_save_failed "
|
||||
"state_dir=%s", state_dir)
|
||||
|
||||
|
||||
def _audit(data: dict, action: str, detail: str, now: float) -> None:
|
||||
"""记一条审计(调用方负责落盘)。"""
|
||||
data.setdefault("audit", []).append(
|
||||
{"action": action, "detail": detail, "created_at": now})
|
||||
|
||||
|
||||
def _get_swarm_or_raise(data: dict, swarm_id: str) -> dict:
|
||||
"""取群,不存在即 ValueError(调用方转 404)。"""
|
||||
swarm = data.get("swarms", {}).get(swarm_id)
|
||||
if not isinstance(swarm, dict):
|
||||
raise ValueError("swarm not found")
|
||||
return swarm
|
||||
|
||||
|
||||
def create_swarm(state_dir: str, project_id: str,
|
||||
parent_task_id: str, max_agents: int = 8,
|
||||
now: float | None = None) -> dict:
|
||||
"""建群:每项目群上限 10(本项目自定),每群代理上限
|
||||
1–32(默认 8,本项目自定)。返回群 dict。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
if isinstance(max_agents, bool) \
|
||||
or not isinstance(max_agents, int):
|
||||
raise ValueError("max_agents must be an integer")
|
||||
if not 1 <= max_agents <= MAX_AGENTS_LIMIT:
|
||||
raise ValueError("max_agents must be between 1 and %d"
|
||||
% MAX_AGENTS_LIMIT)
|
||||
if not project_id or not parent_task_id:
|
||||
raise ValueError("project_id and parent_task_id required")
|
||||
data = _load_registry(state_dir)
|
||||
count = sum(1 for s in data["swarms"].values()
|
||||
if isinstance(s, dict)
|
||||
and s.get("project_id") == project_id)
|
||||
if count >= MAX_SWARMS_PER_PROJECT:
|
||||
raise ValueError("too many swarms in this project")
|
||||
swarm_id = uuid.uuid4().hex
|
||||
swarm = {
|
||||
"id": swarm_id,
|
||||
"project_id": project_id,
|
||||
"parent_task_id": parent_task_id,
|
||||
"max_agents": max_agents,
|
||||
"status": "active",
|
||||
"agents": {},
|
||||
"created_at": now,
|
||||
}
|
||||
data["swarms"][swarm_id] = swarm
|
||||
_audit(data, "swarm_created",
|
||||
"%s parent=%s" % (swarm_id, parent_task_id), now)
|
||||
_save_registry(state_dir, data)
|
||||
return dict(swarm)
|
||||
|
||||
|
||||
def get_swarm(state_dir: str, swarm_id: str) -> dict | None:
|
||||
"""取群;不存在回 None。"""
|
||||
data = _load_registry(state_dir)
|
||||
swarm = data.get("swarms", {}).get(swarm_id)
|
||||
if not isinstance(swarm, dict):
|
||||
return None
|
||||
return dict(swarm)
|
||||
|
||||
|
||||
def spawn_agent(state_dir: str, swarm_id: str, prompt: str,
|
||||
workspace_root: str,
|
||||
now: float | None = None) -> dict:
|
||||
"""派一个子代理:群须 active、人数未满上限;代理得独立
|
||||
任务 id + 独立工作区(父工作区下按代理 id 分目录,
|
||||
resolve_inside 门守着,越界即 ValueError)。返回代理 dict。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
if not prompt or not isinstance(prompt, str) \
|
||||
or not prompt.strip():
|
||||
raise ValueError("prompt required")
|
||||
if len(prompt) > MAX_AGENT_PROMPT_LEN:
|
||||
raise ValueError("prompt too long")
|
||||
if not workspace_root or not isinstance(workspace_root, str):
|
||||
raise ValueError("workspace_root required")
|
||||
data = _load_registry(state_dir)
|
||||
swarm = _get_swarm_or_raise(data, swarm_id)
|
||||
if swarm.get("status") != "active":
|
||||
raise ValueError("swarm is not active")
|
||||
agents = swarm.setdefault("agents", {})
|
||||
if len(agents) >= int(swarm["max_agents"]):
|
||||
raise ValueError("swarm agent limit reached")
|
||||
agent_id = uuid.uuid4().hex
|
||||
workspace = resolve_inside(workspace_root, agent_id)
|
||||
if workspace is None:
|
||||
raise ValueError("workspace_root is not usable")
|
||||
agent = {
|
||||
"id": agent_id,
|
||||
"task_id": uuid.uuid4().hex,
|
||||
"prompt": prompt.strip(),
|
||||
"workspace": workspace,
|
||||
"status": "pending",
|
||||
"created_at": now,
|
||||
}
|
||||
agents[agent_id] = agent
|
||||
_audit(data, "swarm_agent_spawned",
|
||||
"%s agent=%s" % (swarm_id, agent_id), now)
|
||||
_save_registry(state_dir, data)
|
||||
return dict(agent)
|
||||
|
||||
|
||||
def list_agents(state_dir: str, swarm_id: str) -> list[dict]:
|
||||
"""列出某群的全部代理(按创建时间排序)。群不存在即
|
||||
ValueError。"""
|
||||
data = _load_registry(state_dir)
|
||||
swarm = _get_swarm_or_raise(data, swarm_id)
|
||||
agents = [dict(a) for a in
|
||||
swarm.get("agents", {}).values()
|
||||
if isinstance(a, dict)]
|
||||
agents.sort(key=lambda a: (a.get("created_at") or 0,
|
||||
a.get("id") or ""))
|
||||
return agents
|
||||
|
||||
|
||||
def agent_workspace(state_dir: str, swarm_id: str,
|
||||
agent_id: str) -> str | None:
|
||||
"""取某代理的工作区路径;群或代理不存在回 None。"""
|
||||
data = _load_registry(state_dir)
|
||||
swarm = data.get("swarms", {}).get(swarm_id)
|
||||
if not isinstance(swarm, dict):
|
||||
return None
|
||||
agent = swarm.get("agents", {}).get(agent_id)
|
||||
if not isinstance(agent, dict):
|
||||
return None
|
||||
workspace = agent.get("workspace")
|
||||
return workspace if isinstance(workspace, str) else None
|
||||
|
||||
|
||||
def check_agent_path(state_dir: str, swarm_id: str, agent_id: str,
|
||||
path: str) -> bool:
|
||||
"""隔离门:path 是否在该代理自己的工作区内(realpath +
|
||||
commonpath 校验,符号链接逃逸、.. 逃逸、跨盘符一律 False)。
|
||||
群或代理不存在时同样 False(fail-closed)。"""
|
||||
workspace = agent_workspace(state_dir, swarm_id, agent_id)
|
||||
if not workspace or not isinstance(path, str) or not path:
|
||||
return False
|
||||
try:
|
||||
rel = os.path.relpath(path, workspace)
|
||||
except ValueError:
|
||||
return False
|
||||
return resolve_inside(workspace, rel) is not None
|
||||
|
||||
|
||||
def _get_agent_or_raise(data: dict, swarm_id: str,
|
||||
agent_id: str) -> tuple[dict, dict]:
|
||||
"""取(群,代理),任一不存在即 ValueError。"""
|
||||
swarm = _get_swarm_or_raise(data, swarm_id)
|
||||
agent = swarm.get("agents", {}).get(agent_id)
|
||||
if not isinstance(agent, dict):
|
||||
raise ValueError("agent not found")
|
||||
return swarm, agent
|
||||
|
||||
|
||||
def _refresh_swarm_status(swarm: dict) -> None:
|
||||
"""群终态推导:代理全终态后,全部 done 即 done、有 failed
|
||||
即 failed;否则保持 active。"""
|
||||
agents = [a for a in swarm.get("agents", {}).values()
|
||||
if isinstance(a, dict)]
|
||||
if not agents:
|
||||
return
|
||||
if any(a.get("status") not in ("done", "failed")
|
||||
for a in agents):
|
||||
return
|
||||
if any(a.get("status") == "failed" for a in agents):
|
||||
swarm["status"] = "failed"
|
||||
else:
|
||||
swarm["status"] = "done"
|
||||
|
||||
|
||||
def set_agent_status(state_dir: str, swarm_id: str, agent_id: str,
|
||||
status: str,
|
||||
now: float | None = None) -> dict:
|
||||
"""改代理状态(pending/running/done/failed);非法状态或
|
||||
代理不存在即 ValueError。改完推导群终态,返回代理 dict。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
if status not in AGENT_STATUSES:
|
||||
raise ValueError("invalid agent status")
|
||||
data = _load_registry(state_dir)
|
||||
swarm, agent = _get_agent_or_raise(data, swarm_id, agent_id)
|
||||
agent["status"] = status
|
||||
_refresh_swarm_status(swarm)
|
||||
_audit(data, "swarm_agent_status",
|
||||
"%s agent=%s status=%s" % (swarm_id, agent_id, status),
|
||||
now)
|
||||
_save_registry(state_dir, data)
|
||||
return dict(agent)
|
||||
|
||||
|
||||
def register_tool(state_dir: str, swarm_id: str, agent_id: str,
|
||||
name: str, description: str,
|
||||
now: float | None = None) -> dict:
|
||||
"""子代理构建自己的工具:登记进群内工具表(名字在该代理
|
||||
内唯一,1–100 字符),带审计。登记只登记,不执行。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
if not name or not isinstance(name, str) or not name.strip():
|
||||
raise ValueError("tool name required")
|
||||
name = name.strip()
|
||||
if len(name) > MAX_TOOL_NAME_LEN:
|
||||
raise ValueError("tool name too long")
|
||||
if not isinstance(description, str):
|
||||
raise ValueError("description must be a string")
|
||||
data = _load_registry(state_dir)
|
||||
_get_agent_or_raise(data, swarm_id, agent_id)
|
||||
tools = data.setdefault("tools", {})
|
||||
by_agent = tools.setdefault(swarm_id, {}).setdefault(agent_id, {})
|
||||
if name in by_agent:
|
||||
raise ValueError("tool already registered for this agent")
|
||||
tool = {"name": name, "description": description,
|
||||
"agent_id": agent_id, "swarm_id": swarm_id,
|
||||
"created_at": now}
|
||||
by_agent[name] = tool
|
||||
_audit(data, "swarm_tool_registered",
|
||||
"%s agent=%s tool=%s" % (swarm_id, agent_id, name), now)
|
||||
_save_registry(state_dir, data)
|
||||
return dict(tool)
|
||||
|
||||
|
||||
def record_tool_invocation(state_dir: str, swarm_id: str,
|
||||
agent_id: str, name: str,
|
||||
args_summary: str,
|
||||
now: float | None = None) -> dict:
|
||||
"""记一次工具调用意图(登记 + 审计):工具须是该代理自己
|
||||
注册的,别代理的工具名在这里叫不动。
|
||||
|
||||
注意:本函数**不执行工具代码**——执行在子代理自己的 VM/
|
||||
单元里(执行器派发代理任务时发生)。这里只留下“谁、在
|
||||
哪群、调了哪个工具、参数摘要”的审计记录。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
if not isinstance(args_summary, str):
|
||||
raise ValueError("args_summary must be a string")
|
||||
data = _load_registry(state_dir)
|
||||
_get_agent_or_raise(data, swarm_id, agent_id)
|
||||
by_agent = data.get("tools", {}).get(swarm_id, {}).get(agent_id,
|
||||
{})
|
||||
if name not in by_agent:
|
||||
raise ValueError("tool not registered for this agent")
|
||||
record = {
|
||||
"id": uuid.uuid4().hex,
|
||||
"swarm_id": swarm_id,
|
||||
"agent_id": agent_id,
|
||||
"name": name,
|
||||
"args_summary": args_summary[:500],
|
||||
"created_at": now,
|
||||
}
|
||||
data.setdefault("invocations", []).append(record)
|
||||
_audit(data, "swarm_tool_invoked",
|
||||
"%s agent=%s tool=%s" % (swarm_id, agent_id, name), now)
|
||||
_save_registry(state_dir, data)
|
||||
return dict(record)
|
||||
|
||||
|
||||
def dispatch_agent(state_dir: str, swarm_id: str, agent_id: str,
|
||||
runner: Callable[[dict], dict],
|
||||
now: float | None = None) -> dict:
|
||||
"""派发代理去执行:代理须是 pending;runner 是调用方注入
|
||||
的执行函数(生产上接执行器 dispatch,测试里传 fake),
|
||||
收到任务 dict {"id", "prompt", "swarm_id", "agent_id",
|
||||
"workspace"},返回其执行结果。
|
||||
|
||||
派发前把代理置为 running;runner 抛错时代理状态不动
|
||||
(调用方按失败处理,不在这里把异常吞成 done)。"""
|
||||
if now is None:
|
||||
now = time.time()
|
||||
data = _load_registry(state_dir)
|
||||
swarm, agent = _get_agent_or_raise(data, swarm_id, agent_id)
|
||||
if agent.get("status") != "pending":
|
||||
raise ValueError("agent is not pending")
|
||||
task = {
|
||||
"id": agent["task_id"],
|
||||
"prompt": agent["prompt"],
|
||||
"swarm_id": swarm_id,
|
||||
"agent_id": agent_id,
|
||||
"workspace": agent["workspace"],
|
||||
}
|
||||
agent["status"] = "running"
|
||||
_audit(data, "swarm_agent_dispatched",
|
||||
"%s agent=%s task=%s"
|
||||
% (swarm_id, agent_id, agent["task_id"]), now)
|
||||
_save_registry(state_dir, data)
|
||||
return runner(task)
|
||||
@@ -1,171 +0,0 @@
|
||||
"""污点追踪器(对齐 Muse P5-A,叶子模块,纯标准库)。
|
||||
|
||||
Muse 口径(见 docs/chv-muse-p5-gap.md §1 污点出口行):每个工具
|
||||
执行进程初始干净,读取用户数据后变“污点”;干净且符合窄范围
|
||||
自动放行策略并通过出站 URL 其他检查的请求可免打扰放行,污点
|
||||
或不可验证进程失去自动放行、回到正常审批流。
|
||||
|
||||
本模块只是用户态追踪器,不是强制层:
|
||||
- “读用户数据”事件靠调用方上报(工具执行层在读用户数据前
|
||||
调 `mark_tainted`,或 eBPF 通了之后由 LSM hook 回调上报)。
|
||||
没接 LSM hook 之前漏报是可能的,本模块不宣称“所有读用户
|
||||
数据的行为都被捕获”。
|
||||
- 污点是单向的:没有清除入口,只能随进程退出消失。
|
||||
- `source` 只记“哪类用户数据”(见 `_SOURCE_CATEGORIES`),
|
||||
绝不记内容;不在分类表里的上报值一律归一化为 `unknown`,
|
||||
原始字符串不会落进任何记录。审计记录里只有事件名、pid、
|
||||
原因与分类,绝无用户数据内容、无凭证值。
|
||||
- 未知 pid(未注册)一律不可自动放行:`is_tainted` 对它报
|
||||
False(没证据说它污),`auto_allow_eligible` 对它也报
|
||||
False(没证据说它干净可信)。
|
||||
"""
|
||||
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
|
||||
# 允许的 source 分类:只记“哪类用户数据”,不记内容。
|
||||
_SOURCE_CATEGORIES = frozenset({
|
||||
"gitea_repo",
|
||||
"user_file",
|
||||
"user_memory",
|
||||
"chat_history",
|
||||
"credential_vault",
|
||||
"tool_output",
|
||||
"unknown",
|
||||
})
|
||||
|
||||
# 审计事件名:沿用 P3 的 `sentinel_*` 契约(见 gap §1 审计行)。
|
||||
_EVENT_MARKED = "sentinel_taint_marked"
|
||||
_EVENT_INHERITED = "sentinel_taint_inherited"
|
||||
_EVENT_REGISTERED = "sentinel_taint_registered"
|
||||
_EVENT_RELEASED = "sentinel_taint_released"
|
||||
|
||||
|
||||
def _normalize_source(source: str) -> str:
|
||||
"""source 归一化为分类表里的值,不在表里的记 `unknown`。
|
||||
|
||||
调用方误把内容当 source 传进来时,原始字符串在这里就被
|
||||
丢掉,绝不会进入进程记录或审计记录。"""
|
||||
value = (source or "").strip().lower()
|
||||
if value in _SOURCE_CATEGORIES:
|
||||
return value
|
||||
return "unknown"
|
||||
|
||||
|
||||
class TaintTracker:
|
||||
"""进程污点两态追踪器(内存态,进程退出即清)。
|
||||
|
||||
每个进程一条记录:pid、cgroup、连接器、用例、是否污点。
|
||||
污点只能由 `mark_tainted` 置位、由 `fork_child` 继承、由
|
||||
`process_exit` 随进程消失;没有反向清除入口。
|
||||
"""
|
||||
|
||||
def __init__(self, clock: Callable[[], float] | None = None
|
||||
) -> None:
|
||||
self._records: dict[int, dict] = {}
|
||||
self._audit: list[dict] = []
|
||||
self._clock = clock or time.time
|
||||
|
||||
def _emit(self, event: str, pid: int, reason: str,
|
||||
source: str = "") -> None:
|
||||
"""记一条审计(事件名 + pid + 原因 + 分类,无内容)"""
|
||||
record = {"event": event, "pid": pid, "reason": reason,
|
||||
"ts": self._clock()}
|
||||
if source:
|
||||
record["source"] = source
|
||||
self._audit.append(record)
|
||||
|
||||
def register(self, pid: int, cgroup: str = "",
|
||||
connector: str = "", use_case: str = "") -> bool:
|
||||
"""登记一个干净进程。pid 非法或已登记返回 False。"""
|
||||
if pid <= 0 or pid in self._records:
|
||||
return False
|
||||
self._records[pid] = {
|
||||
"pid": pid, "cgroup": cgroup,
|
||||
"connector": connector, "use_case": use_case,
|
||||
"tainted": False,
|
||||
}
|
||||
self._emit(_EVENT_REGISTERED, pid, "process_registered")
|
||||
return True
|
||||
|
||||
def mark_tainted(self, pid: int, source: str) -> bool:
|
||||
"""上报“读用户数据”事件:该进程变污点,不可逆。
|
||||
|
||||
未登记的 pid 也接受上报(fail-closed:读了用户数据的
|
||||
进程先记为污点,不等它补登记);source 只存分类。"""
|
||||
category = _normalize_source(source)
|
||||
record = self._records.get(pid)
|
||||
if record is None:
|
||||
self._records[pid] = {
|
||||
"pid": pid, "cgroup": "", "connector": "",
|
||||
"use_case": "", "tainted": True,
|
||||
}
|
||||
else:
|
||||
record["tainted"] = True
|
||||
reason = "read_user_data:" + category
|
||||
self._emit(_EVENT_MARKED, pid, reason, source=category)
|
||||
return True
|
||||
|
||||
def fork_child(self, parent_pid: int, child_pid: int) -> bool:
|
||||
"""子进程继承污点:父污则子污,父干净则子干净。
|
||||
|
||||
父未登记或子 pid 非法返回 False,不建记录;子已登记
|
||||
时只改它的污点位,不覆盖它自己的 cgroup/连接器/用例。"""
|
||||
if child_pid <= 0 or child_pid == parent_pid:
|
||||
return False
|
||||
parent = self._records.get(parent_pid)
|
||||
if parent is None:
|
||||
return False
|
||||
child = self._records.get(child_pid)
|
||||
if child is None:
|
||||
self._records[child_pid] = {
|
||||
"pid": child_pid, "cgroup": parent["cgroup"],
|
||||
"connector": parent["connector"],
|
||||
"use_case": parent["use_case"],
|
||||
"tainted": parent["tainted"],
|
||||
}
|
||||
else:
|
||||
child["tainted"] = parent["tainted"]
|
||||
if parent["tainted"]:
|
||||
self._emit(_EVENT_INHERITED, child_pid,
|
||||
"inherited_from_parent:%d" % parent_pid)
|
||||
return True
|
||||
|
||||
def process_exit(self, pid: int) -> bool:
|
||||
"""进程退出:记录连同污点一起消失。未知 pid 返回 False。"""
|
||||
if pid not in self._records:
|
||||
return False
|
||||
del self._records[pid]
|
||||
self._emit(_EVENT_RELEASED, pid, "process_exit")
|
||||
return True
|
||||
|
||||
def is_tainted(self, pid: int) -> bool:
|
||||
"""该进程是否污点。未知 pid 报 False(无证据说它污)。"""
|
||||
record = self._records.get(pid)
|
||||
return bool(record and record["tainted"])
|
||||
|
||||
def is_registered(self, pid: int) -> bool:
|
||||
"""该 pid 是否已登记。"""
|
||||
return pid in self._records
|
||||
|
||||
def state(self, pid: int) -> dict | None:
|
||||
"""该进程记录的副本(无记录返回 None)。"""
|
||||
record = self._records.get(pid)
|
||||
if record is None:
|
||||
return None
|
||||
return dict(record)
|
||||
|
||||
def auto_allow_eligible(self, pid: int) -> bool:
|
||||
"""出口自动放行资格:已登记且干净才 True。
|
||||
|
||||
污点进程一律 False;未知 pid(不可验证)也一律 False。
|
||||
这只是决策输入,不放行任何请求——放行仍须走 Sentinel
|
||||
的出站 URL 其他检查(见 gap §1)。"""
|
||||
record = self._records.get(pid)
|
||||
if record is None:
|
||||
return False
|
||||
return not record["tainted"]
|
||||
|
||||
def audit(self) -> list:
|
||||
"""全部审计记录的副本(事件名 + pid + 原因 + 分类)。"""
|
||||
return [dict(record) for record in self._audit]
|
||||
@@ -1,211 +0,0 @@
|
||||
"""遥测最小化 + 训练退出开关(feat-chv-muse-p9,叶子模块)。
|
||||
|
||||
对齐 Muse 官方安全文(security.muse.ai 2026-09-08,见
|
||||
docs/chv-muse-p9-gap.md):VM 发送有限数据用于推理与遥测
|
||||
(安全文未给字段级清单——本模块的字段清单按最小化原则自定,
|
||||
见 `TELEMETRY_FIELDS`);推理轨迹在去除关键个人身份信息后可
|
||||
用于训练新检查点,用户可一键退出训练。
|
||||
|
||||
诚实边界(见差距表 §4):
|
||||
- 遥测目前只落盘 `<state_dir>/telemetry.jsonl`,没有向外部遥测
|
||||
端点发送的路径;落盘失败也不影响任务(`emit_telemetry`
|
||||
永不抛错)。
|
||||
- 训练轨迹目前没有上传路径:开关管的是“轨迹类事件是否产生”;
|
||||
上传通道本身不存在,开关是为上传通道预留的门禁。
|
||||
- 字段白名单是构造性保证:不在白名单里的键一律丢弃;token 数
|
||||
只取 host.py 回传的 usage,未回传记 `null`,不估算、不编造。
|
||||
- 训练开关默认开是**本项目决定,非 Meta 官方值**(报告注明
|
||||
帮助中心原文未逐字确认默认开启)。
|
||||
|
||||
本模块只用标准库 + `atomic_file`(开关状态文件原子写)。
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
from cloud.chv.atomic_file import atomic_write_text
|
||||
|
||||
_LOGGER = logging.getLogger(__name__)
|
||||
|
||||
# 遥测字段白名单(自定,按最小化原则):只含任务级元数据。
|
||||
# 绝不含对话原文、文件内容、密钥、个人身份信息——白名单外
|
||||
# 的键在 sanitize 时一律丢弃,不是“记下来再脱敏”。
|
||||
TELEMETRY_FIELDS = frozenset({
|
||||
"task_id", "vm_id", "provider", "model", "status",
|
||||
"started_at", "ended_at", "duration_ms",
|
||||
"tokens_in", "tokens_out", "tokens_total",
|
||||
"turns", "tool_calls", "sampled",
|
||||
})
|
||||
|
||||
# 轨迹类事件字段白名单(自定):只含回合级元数据,不含任何文本。
|
||||
TRAJECTORY_FIELDS = frozenset({
|
||||
"task_id", "turns", "tools", "duration_ms",
|
||||
})
|
||||
|
||||
TRAINING_OPT_FILENAME = "training-opt.json"
|
||||
TRAINING_OPT_AUDIT_FILENAME = "training-opt-audit.jsonl"
|
||||
TELEMETRY_FILENAME = "telemetry.jsonl"
|
||||
DEFAULT_TELEMETRY_SAMPLE_RATE = 1.0
|
||||
|
||||
|
||||
def telemetry_sample_rate(env: dict | None = None) -> float:
|
||||
"""遥测采样率:`CHV_TELEMETRY_SAMPLE_RATE`,0.0–1.0,默认 1.0
|
||||
(全量,任务级元数据量小)。非法值即 ValueError。"""
|
||||
values = env if env is not None else os.environ # type: ignore[assignment]
|
||||
raw = values.get("CHV_TELEMETRY_SAMPLE_RATE")
|
||||
if raw is None or str(raw).strip() == "":
|
||||
return DEFAULT_TELEMETRY_SAMPLE_RATE
|
||||
try:
|
||||
rate = float(str(raw).strip())
|
||||
except ValueError:
|
||||
raise ValueError(
|
||||
"CHV_TELEMETRY_SAMPLE_RATE 必须是数字: %r" % raw)
|
||||
if not 0.0 <= rate <= 1.0:
|
||||
raise ValueError(
|
||||
"CHV_TELEMETRY_SAMPLE_RATE 必须在 0.0-1.0 之间: %s" % rate)
|
||||
return rate
|
||||
|
||||
|
||||
def sampled(task_id: str, rate: float) -> bool:
|
||||
"""确定性采样:同一 task_id 每次判定一致(可测试);rate=0
|
||||
全丢,rate=1 全留。"""
|
||||
digest = hashlib.sha256(task_id.encode("utf-8")).hexdigest()
|
||||
return int(digest[:8], 16) % 10000 < rate * 10000
|
||||
|
||||
|
||||
def _is_scalar(value: object) -> bool:
|
||||
"""遥测值只认标量(str/int/float/bool/None);dict/list
|
||||
一律丢弃——不把结构化内容带进遥测。"""
|
||||
return value is None or isinstance(value, (str, int, float, bool))
|
||||
|
||||
|
||||
def sanitize_telemetry_event(event: dict) -> dict:
|
||||
"""按白名单过滤遥测事件:不在白名单里的键丢弃;值不是标量
|
||||
的丢弃;`tools` 字段例外——只认 str 列表(工具名),其它
|
||||
元素一律去掉。返回新 dict,不改原 event。"""
|
||||
clean: dict = {}
|
||||
for key, value in (event or {}).items():
|
||||
if key not in TELEMETRY_FIELDS and key not in TRAJECTORY_FIELDS:
|
||||
continue
|
||||
if key == "tools":
|
||||
if isinstance(value, (list, tuple)):
|
||||
clean[key] = [t for t in value if isinstance(t, str)]
|
||||
continue
|
||||
if _is_scalar(value):
|
||||
clean[key] = value
|
||||
return clean
|
||||
|
||||
|
||||
def build_task_telemetry(task_id: str, vm_id: str, provider: str,
|
||||
model: str | None, status: str,
|
||||
started_at: float, ended_at: float,
|
||||
duration_ms: int,
|
||||
tokens: dict | None = None,
|
||||
turns: int | None = None,
|
||||
tool_calls: int | None = None,
|
||||
env: dict | None = None) -> dict | None:
|
||||
"""建一条任务级遥测事件;采样未命中返回 None(不产生事件,
|
||||
不是产生后丢弃)。tokens 取 host.py 回传的 usage,未回传
|
||||
时三个 token 字段记 None,不编造。"""
|
||||
rate = telemetry_sample_rate(env)
|
||||
if not sampled(task_id, rate):
|
||||
return None
|
||||
tokens = tokens or {}
|
||||
event = {
|
||||
"task_id": task_id, "vm_id": vm_id, "provider": provider,
|
||||
"model": model, "status": status,
|
||||
"started_at": started_at, "ended_at": ended_at,
|
||||
"duration_ms": duration_ms,
|
||||
"tokens_in": tokens.get("input_tokens"),
|
||||
"tokens_out": tokens.get("output_tokens"),
|
||||
"tokens_total": tokens.get("total_tokens"),
|
||||
"turns": turns, "tool_calls": tool_calls, "sampled": True,
|
||||
}
|
||||
return sanitize_telemetry_event(event)
|
||||
|
||||
|
||||
def build_trajectory_event(task_id: str, turns: int,
|
||||
tools: list, duration_ms: int,
|
||||
state_dir: str) -> dict | None:
|
||||
"""建一条轨迹类事件(训练用):只含回合级元数据(回合数、
|
||||
工具名、耗时),不含任何文本。训练退出开关关时返回 None
|
||||
(事件不再产生)。"""
|
||||
if not training_enabled(state_dir):
|
||||
return None
|
||||
return sanitize_telemetry_event({
|
||||
"task_id": task_id, "turns": turns,
|
||||
"tools": list(tools or []), "duration_ms": duration_ms,
|
||||
})
|
||||
|
||||
|
||||
def _append_jsonl(path: str, record: dict) -> bool:
|
||||
"""best-effort 追加一行 JSONL:失败返回 False,永不抛错
|
||||
(遥测/审计落盘失败绝不影响任务)。"""
|
||||
try:
|
||||
line = json.dumps(record, ensure_ascii=False) + "\n"
|
||||
with open(path, "a", encoding="utf-8") as fh:
|
||||
fh.write(line)
|
||||
fh.flush()
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001 - 落盘失败只记日志
|
||||
_LOGGER.warning("telemetry_append_failed path=%s error=%s",
|
||||
path, exc)
|
||||
return False
|
||||
|
||||
|
||||
def emit_telemetry(state_dir: str, event: dict) -> bool:
|
||||
"""发一条遥测事件:先过白名单再落盘 `<state_dir>/telemetry.jsonl`。
|
||||
落盘失败返回 False,永不抛错、不影响任务。"""
|
||||
if not state_dir or not isinstance(event, dict):
|
||||
return False
|
||||
clean = sanitize_telemetry_event(event)
|
||||
if not clean:
|
||||
return False
|
||||
return _append_jsonl(
|
||||
os.path.join(state_dir, TELEMETRY_FILENAME), clean)
|
||||
|
||||
|
||||
def training_enabled(state_dir: str) -> bool:
|
||||
"""训练退出开关状态:默认开(本项目决定,非 Meta 官方值)。
|
||||
状态文件缺失即默认开;损坏/不可解析/缺 `enabled` 键时
|
||||
fail-closed 当作关处理并记日志(损坏不等于同意上传)。"""
|
||||
path = os.path.join(state_dir, TRAINING_OPT_FILENAME)
|
||||
try:
|
||||
with open(path, encoding="utf-8") as fh:
|
||||
data = json.load(fh)
|
||||
except FileNotFoundError:
|
||||
return True
|
||||
except Exception as exc: # noqa: BLE001 - 损坏即关
|
||||
_LOGGER.warning("training_opt_corrupt path=%s error=%s",
|
||||
path, exc)
|
||||
return False
|
||||
if not isinstance(data, dict) or "enabled" not in data:
|
||||
return False
|
||||
return bool(data["enabled"])
|
||||
|
||||
|
||||
def set_training_enabled(state_dir: str, enabled: bool,
|
||||
actor: str = "user") -> bool:
|
||||
"""一键退出/恢复训练:状态原子写 `<state_dir>/training-opt.json`
|
||||
(0600),变更追加进 `training-opt-audit.jsonl`。`enabled`
|
||||
不是 bool 即 TypeError;写失败返回 False,永不抛错。
|
||||
关后立即生效:后续轨迹事件不再产生。"""
|
||||
if not isinstance(enabled, bool):
|
||||
raise TypeError("enabled 必须是 bool: %r" % (enabled,))
|
||||
record = {"enabled": enabled, "updated_at": time.time(),
|
||||
"actor": str(actor)[:64]}
|
||||
try:
|
||||
atomic_write_text(
|
||||
os.path.join(state_dir, TRAINING_OPT_FILENAME),
|
||||
json.dumps(record, ensure_ascii=False))
|
||||
except Exception as exc: # noqa: BLE001 - 写失败只记日志
|
||||
_LOGGER.warning("training_opt_write_failed error=%s", exc)
|
||||
return False
|
||||
audit = dict(record)
|
||||
audit["event"] = "training_opt_changed"
|
||||
_append_jsonl(
|
||||
os.path.join(state_dir, TRAINING_OPT_AUDIT_FILENAME), audit)
|
||||
return True
|
||||
@@ -1,223 +0,0 @@
|
||||
"""宿主侧 vsock notify 最小实现(P1,feat-chv-muse-p1)。
|
||||
|
||||
方向只一个:客机 → 宿主。客机连宿主 CID 2 的
|
||||
VSOCK_NOTIFY_PORT 端口发 ready/心跳(客内脚本
|
||||
cloud/chv/guest/vsock_notify.py);宿主在
|
||||
``<vsock socket>_<port>`` 这个 Unix socket 上监听接收。
|
||||
|
||||
形态依据(CHV 官方 docs/vsock.md,upstream 原文):
|
||||
- ``--vsock cid=3,socket=/tmp/ch.vsock``:cid 是客机 CID,
|
||||
socket 是宿主侧 Unix socket 路径;
|
||||
- 客 → 宿主:宿主须先在 ``<socket>_<port>`` 上监听
|
||||
(如 ``/tmp/ch.vsock_1234``),客机连众所周知的 CID 2
|
||||
(Host);
|
||||
- 只支持 stream socket。
|
||||
|
||||
只做就绪辅助:wait_ready() 收到 ready 即返回,供
|
||||
_run_with_vm/常驻路径在 SSH 轮询之外多一个早期信号;
|
||||
SSH 仍是唯一命令通道与就绪判定——vsock 超时、无消息、
|
||||
监听失败一律返回 None,绝不因此判任务失败、绝不抛到
|
||||
调用方。消息是 JSON 行:
|
||||
|
||||
{"type": "ready"|"heartbeat", "vm": "<vm 名>", "ts": <unix 秒>}
|
||||
|
||||
不做宿主 → 客机方向(客机不监听,host.py 协议仍走 SSH)。
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import time
|
||||
|
||||
_LOGGER = logging.getLogger("cloud.cloud_hypervisor")
|
||||
|
||||
# 宿主 CID(vsock 众所周知值,CHV 官方 docs/vsock.md)。
|
||||
HOST_CID = 2
|
||||
# 客机 CID 默认值(与 CHV_VSOCK_CID 默认值一致;0/1/2 保留)。
|
||||
DEFAULT_GUEST_CID = 3
|
||||
# notify 端口:客机连 (CID 2, 本端口) 发 ready/心跳;
|
||||
# 宿主监听 <socket>_<本端口>。与客内脚本的端口须一致。
|
||||
VSOCK_NOTIFY_PORT = 1024
|
||||
# 单行上限:客机发来的行超长即丢弃该连接(防脏数据撑内存)。
|
||||
MAX_LINE_BYTES = 65536
|
||||
# 合法消息类型。
|
||||
VALID_TYPES = ("ready", "heartbeat")
|
||||
|
||||
|
||||
def vsock_socket_path(state_dir: str, vm_name: str) -> str:
|
||||
"""该 VM 的 vsock socket 路径(放进 VmConfig 的值)。
|
||||
|
||||
与 build_vm_config 的派生一致:任务盘同目录的
|
||||
``state_dir/<vm>/vsock.sock``。vm_name 须与
|
||||
api_socket/任务盘用的名字相同(executor.vm_name(task)
|
||||
的返回值,含 chv- 前缀),这里不二次加前缀。"""
|
||||
return os.path.join(state_dir, vm_name, "vsock.sock")
|
||||
|
||||
|
||||
def notify_socket_path(socket_path: str,
|
||||
port: int = VSOCK_NOTIFY_PORT) -> str:
|
||||
"""宿主侧监听路径:``<socket>_<port>``(CHV 官方约定)。"""
|
||||
return "%s_%d" % (socket_path, port)
|
||||
|
||||
|
||||
def encode_notify(message_type: str, vm: str,
|
||||
ts: float | None = None) -> bytes:
|
||||
"""编码一条 notify 消息为 JSON 行(含换行)。
|
||||
|
||||
type 非法或 vm 为空即 ValueError(调用方是可信的客内
|
||||
脚本/宿主代码,编码错误是编程错误,早炸)。"""
|
||||
if message_type not in VALID_TYPES:
|
||||
raise ValueError("vsock notify 类型非法:%r" % (message_type,))
|
||||
if not vm:
|
||||
raise ValueError("vsock notify vm 不能为空")
|
||||
if ts is None:
|
||||
ts = time.time()
|
||||
line = json.dumps({"type": message_type, "vm": vm, "ts": ts},
|
||||
separators=(",", ":"))
|
||||
return (line + "\n").encode("utf-8")
|
||||
|
||||
|
||||
def decode_notify(line: bytes | str) -> dict | None:
|
||||
"""解码一行 notify 消息:合法返回 dict,否则返回 None。
|
||||
|
||||
客机字节不可信,任何畸形(非 JSON、非对象、类型不在
|
||||
ready/heartbeat 内、vm 空/非字符串、ts 非数字)一律
|
||||
None,绝不抛异常。"""
|
||||
try:
|
||||
if isinstance(line, bytes):
|
||||
line = line.decode("utf-8")
|
||||
obj = json.loads(line)
|
||||
except (UnicodeDecodeError, ValueError):
|
||||
return None
|
||||
if not isinstance(obj, dict):
|
||||
return None
|
||||
if obj.get("type") not in VALID_TYPES:
|
||||
return None
|
||||
if not isinstance(obj.get("vm"), str) or not obj.get("vm"):
|
||||
return None
|
||||
if not isinstance(obj.get("ts"), (int, float)) \
|
||||
or isinstance(obj.get("ts"), bool):
|
||||
return None
|
||||
return obj
|
||||
|
||||
|
||||
def _bind_notify_listener(listener, notify_path: str) -> bool:
|
||||
"""bind notify socket:残留文件先删一次再重试一次;
|
||||
两次都失败即 False(调用方记 warning 后返回 None,
|
||||
与原来直抛到外层 except 的行为一致)。"""
|
||||
try:
|
||||
listener.bind(notify_path)
|
||||
return True
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
os.unlink(notify_path)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
listener.bind(notify_path)
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def listen_notify(notify_path: str, timeout: float,
|
||||
want_type: str | None = None) -> dict | None:
|
||||
"""在 notify_path(Unix socket)上监听,等一条 notify。
|
||||
|
||||
收到首条合法消息即返回(want_type 给定时跳过其它类型
|
||||
继续等,直到超时);超时、客机断开未发合法消息、监听
|
||||
失败一律返回 None。监听用的 socket 文件用完即删(尽力
|
||||
而为);bind 前已存在的同名残留文件先删一次再 bind。
|
||||
timeout <= 0 即不等,直接返回 None。"""
|
||||
if timeout is None or timeout <= 0:
|
||||
return None
|
||||
deadline = time.monotonic() + timeout
|
||||
listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
try:
|
||||
if not _bind_notify_listener(listener, notify_path):
|
||||
_LOGGER.warning("vsock notify 监听失败:%s", notify_path)
|
||||
return None
|
||||
listener.listen(1)
|
||||
while True:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
return None
|
||||
listener.settimeout(remaining)
|
||||
try:
|
||||
conn, _ = listener.accept()
|
||||
except socket.timeout:
|
||||
return None
|
||||
except OSError:
|
||||
_LOGGER.warning("vsock notify accept 失败:%s",
|
||||
notify_path)
|
||||
return None
|
||||
try:
|
||||
return _read_notify(conn, deadline, want_type)
|
||||
finally:
|
||||
try:
|
||||
conn.close()
|
||||
except OSError:
|
||||
pass
|
||||
except OSError:
|
||||
_LOGGER.warning("vsock notify 监听失败:%s", notify_path)
|
||||
return None
|
||||
finally:
|
||||
try:
|
||||
listener.close()
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
os.unlink(notify_path)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _read_notify(conn, deadline: float,
|
||||
want_type: str | None) -> dict | None:
|
||||
"""从一条已接受的连接上读 JSON 行,直到合法消息或超时/
|
||||
断开。want_type 给定时跳过其它合法类型继续读。"""
|
||||
try:
|
||||
stream = conn.makefile("rb")
|
||||
except OSError:
|
||||
return None
|
||||
try:
|
||||
while True:
|
||||
remaining = deadline - time.monotonic()
|
||||
if remaining <= 0:
|
||||
return None
|
||||
try:
|
||||
conn.settimeout(remaining)
|
||||
line = stream.readline(MAX_LINE_BYTES + 1)
|
||||
except (OSError, ValueError):
|
||||
return None
|
||||
if not line:
|
||||
return None
|
||||
if len(line) > MAX_LINE_BYTES:
|
||||
return None
|
||||
msg = decode_notify(line)
|
||||
if msg is None:
|
||||
continue
|
||||
if want_type is not None and msg["type"] != want_type:
|
||||
continue
|
||||
return msg
|
||||
finally:
|
||||
try:
|
||||
stream.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def wait_ready(state_dir: str, vm_name: str,
|
||||
timeout: float = 30,
|
||||
port: int = VSOCK_NOTIFY_PORT) -> dict | None:
|
||||
"""等一台 VM 的 vsock ready(就绪辅助,非 SSH 判定)。
|
||||
|
||||
在 ``state_dir/<vm>/vsock.sock_<port>`` 上监听,直到
|
||||
收到 {"type": "ready"} 或超时。返回 ready 消息 dict,
|
||||
超时/失败返回 None。调用方(_run_with_vm/常驻路径)
|
||||
拿到 None 时继续走 SSH 就绪轮询,绝不因此失败。"""
|
||||
path = notify_socket_path(
|
||||
vsock_socket_path(state_dir, vm_name), port)
|
||||
return listen_notify(path, timeout, want_type="ready")
|
||||
@@ -1,231 +0,0 @@
|
||||
"""CHV worker 适配模块(to10 轨,见 ADR-13)。
|
||||
|
||||
worker.py 里散落的 CHV 强相关接线抽到这里,worker 只留
|
||||
薄包装(方法名、签名、调用点不变,旧测试继续走 worker
|
||||
方法)。本模块只依赖标准库 + cloud.chv 叶子模块 +
|
||||
cloud.logging_config,不 import cloud.worker,绝无循环。
|
||||
|
||||
抽入本模块的(行为与错误文案逐字搬运):
|
||||
- call_goal_resume_breakpoint:通用断点调用(原 worker
|
||||
顶层函数 _call_goal_resume_breakpoint 的本体)。
|
||||
- make_chv_audit_callback:VM 生命周期事件进审计的回调
|
||||
(原 Worker.__init__ 里的闭包本体)。
|
||||
- reconcile_warm_pool:生产残留清理 facade 调用 + 审计
|
||||
(原 Worker._reconcile_warm_pool 的本体)。
|
||||
- reconcile_resident_pool:常驻池残留清理 facade 调用 +
|
||||
审计(新,feat-chv-muse-p1;调 resident_pool 的模块级
|
||||
facade,不构造 ResidentPool、不碰实验接口)。
|
||||
- cancel_watchdog_loop / quota_watchdog_loop:同步路径
|
||||
两条看门狗循环(原 Worker._chv_*_watchdog 的本体,
|
||||
self 依赖全部改成回调参数)。
|
||||
- watchdog_vm_name:看门狗启动时的 VM 名派生(原
|
||||
_run_attempt 里的 try/except 块)。
|
||||
|
||||
明确不抽的(见 ADR-13 诚实清单,worker 通用流程):
|
||||
cancel() 里的 cancel_vm 分支、cleanup_stale_containers
|
||||
的 CHV 分支、网关守卫、_executor_name 的 CHV 分支、
|
||||
_run_attempt 的看门狗线程启动、VM 死亡写回——它们都
|
||||
嵌在 Docker/Local 共用的通用流程里,硬抽只会把通用
|
||||
流程撕成两半,行为不变的收益为零。
|
||||
"""
|
||||
|
||||
import inspect
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
|
||||
from cloud.chv.executor import CloudHypervisorExecutor
|
||||
from cloud.logging_config import log_event
|
||||
|
||||
|
||||
def is_chv_executor(executor: object) -> bool:
|
||||
"""是否为 CHV 执行器(worker 各处 isinstance 的同款判断)。"""
|
||||
return isinstance(executor, CloudHypervisorExecutor)
|
||||
|
||||
|
||||
def call_goal_resume_breakpoint(executor: object, task: dict,
|
||||
model_provider: str | None = None,
|
||||
extra_env: dict | None = None) -> object:
|
||||
"""通用断点调用:执行器提供 goal_resume_breakpoint_task
|
||||
时调用它。
|
||||
|
||||
Local 仍走原路径(调用方直接传 model_provider,不经过
|
||||
这里,以免改生产语义);这里只用于非 Local 执行器:
|
||||
签名接受 model_provider 时传它(CHV/Docker 均接受),
|
||||
否则只传 task(旧假执行器单参)。feat-chv-secfix(H2):
|
||||
CHV 的 gateway_env 不再经 SSH env 前缀注入(密钥值进
|
||||
argv 的旧路径已废除),改走 provisioned seed 盘客内文件
|
||||
投递;CHV 没有 Local 那样的进程 env 通道,seed 文件是
|
||||
唯一通道,keyscan 模式下密钥投递 fail-closed。"""
|
||||
method = getattr(executor, "goal_resume_breakpoint_task")
|
||||
try:
|
||||
params = inspect.signature(method).parameters
|
||||
except (TypeError, ValueError):
|
||||
return method(task)
|
||||
kwargs: dict = {}
|
||||
if "model_provider" in params:
|
||||
kwargs["model_provider"] = model_provider
|
||||
if "extra_env" in params and extra_env is not None:
|
||||
kwargs["extra_env"] = extra_env
|
||||
return method(task, **kwargs)
|
||||
|
||||
|
||||
def make_chv_audit_callback(audit_fn: Callable) -> Callable:
|
||||
"""VM 生命周期事件进审计的回调(原 __init__ 闭包本体)。
|
||||
|
||||
audit_fn 即 Worker._audit(best-effort,失败不影响任务);
|
||||
时间线展示 VM 事件时读审计表即可,不另起一套 VM 事件表;
|
||||
成本/配额仍走 worker 现有 quota 语义,本回调绝不记账。"""
|
||||
|
||||
def _chv_audit(event, task, detail, kind=None):
|
||||
try:
|
||||
project_id = task.get("project_id") \
|
||||
if isinstance(task, dict) else None
|
||||
audit_fn(project_id, "chv_vm_" + event, detail or "")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return _chv_audit
|
||||
|
||||
|
||||
def reconcile_warm_pool(executor: object, audit_fn: Callable,
|
||||
when: str) -> list:
|
||||
"""显式调生产残留清理 facade,返回已清理 VM 名。
|
||||
|
||||
仅 CHV worker 有效(其它执行器返回 [])。实现是
|
||||
cloud.chv.orphans.reconcile_orphans_for_executor():
|
||||
直接读 registry 文件做清理,不构造 WarmPool、不碰
|
||||
实验接口。调用失败不炸主流程,但进审计(失败本身
|
||||
也审计,不静默丢弃)。orphan 条目绝不发给任务,
|
||||
reconcile 只做清理。"""
|
||||
if not isinstance(executor, CloudHypervisorExecutor):
|
||||
return []
|
||||
try:
|
||||
from cloud.chv.orphans import reconcile_orphans_for_executor as _reconcile
|
||||
cleaned = _reconcile(executor) or []
|
||||
except Exception as exc:
|
||||
try:
|
||||
audit_fn(None, "chv_warm_pool_reconcile_failed",
|
||||
"when=%s error=%s" % (when, exc))
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
if cleaned:
|
||||
try:
|
||||
audit_fn(None, "chv_warm_pool_reconcile",
|
||||
"when=%s cleaned=%s" % (when, ",".join(cleaned)))
|
||||
except Exception:
|
||||
pass
|
||||
return cleaned
|
||||
|
||||
|
||||
def reconcile_resident_pool(executor: object, audit_fn: Callable,
|
||||
when: str) -> list:
|
||||
"""显式调常驻残留清理 facade,返回已清理 VM 名。
|
||||
|
||||
仅 CHV worker 有效(其它执行器返回 [])。实现是
|
||||
cloud.chv.resident_pool.reconcile_resident_orphans_
|
||||
for_executor():直接读 resident-pool.json 做清理,不
|
||||
构造 ResidentPool、不碰实验接口。调用失败不炸主流程,
|
||||
但进审计(失败本身也审计,不静默丢弃)。orphan 条目
|
||||
绝不发给任务,reconcile 只做清理。"""
|
||||
if not isinstance(executor, CloudHypervisorExecutor):
|
||||
return []
|
||||
try:
|
||||
from cloud.chv.resident_pool import (
|
||||
reconcile_resident_orphans_for_executor as _reconcile)
|
||||
cleaned = _reconcile(executor) or []
|
||||
except Exception as exc:
|
||||
try:
|
||||
audit_fn(None, "chv_resident_pool_reconcile_failed",
|
||||
"when=%s error=%s" % (when, exc))
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
if cleaned:
|
||||
try:
|
||||
audit_fn(None, "chv_resident_pool_reconcile",
|
||||
"when=%s cleaned=%s" % (when, ",".join(cleaned)))
|
||||
except Exception:
|
||||
pass
|
||||
return cleaned
|
||||
|
||||
|
||||
def cancel_watchdog_loop(task_id: str, vm_name: str | None,
|
||||
stop_event: threading.Event,
|
||||
cancel_requested: set,
|
||||
get_task: Callable,
|
||||
cancel_vm: Callable | None,
|
||||
logger: object) -> None:
|
||||
"""CHV 同步路径的取消看门狗(feat-chv-fix,P1-2,Docker
|
||||
取消看门狗同款模式):同步 SSH 阻塞时 worker 的 cancel()
|
||||
只能改库、清 _running 项,阻塞中的 run 本身不会醒;
|
||||
本线程每 1 秒查库/取消标志,命中时调 cancel_vm 真关
|
||||
VM,然后退出。查库失败时当没取消,不误杀。"""
|
||||
while not stop_event.wait(1.0):
|
||||
try:
|
||||
if task_id in cancel_requested:
|
||||
cancelled = True
|
||||
else:
|
||||
db_task = get_task(task_id)
|
||||
cancelled = (db_task is not None and
|
||||
db_task.get("status") == "interrupted")
|
||||
except Exception:
|
||||
cancelled = False
|
||||
if not cancelled:
|
||||
continue
|
||||
try:
|
||||
if cancel_vm is not None and vm_name:
|
||||
cancel_vm(vm_name)
|
||||
log_event(logger, "worker_chv_killed",
|
||||
task_id=task_id,
|
||||
message="cancel watchdog killed VM %s "
|
||||
"(task interrupted/cancelled)"
|
||||
% vm_name)
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
|
||||
|
||||
def quota_watchdog_loop(task_id: str, vm_name: str | None,
|
||||
stop_event: threading.Event,
|
||||
quota_check: Callable | None,
|
||||
mark_quota_hit: Callable,
|
||||
cancel_vm: Callable | None,
|
||||
logger: object,
|
||||
interval: float = 30.0) -> None:
|
||||
"""CHV 同步路径的配额看门狗(feat-chv-fix,P1-2,Docker
|
||||
配额看门狗同款):命中时记 _quota_hit_tasks 并调
|
||||
cancel_vm 真关 VM;_run_attempt 在同步返回后把结果
|
||||
改写成 quota_exceeded,走统一配额中断收尾。"""
|
||||
while not stop_event.wait(interval):
|
||||
try:
|
||||
over = bool(quota_check()) if quota_check else False
|
||||
except Exception:
|
||||
over = False
|
||||
if not over:
|
||||
continue
|
||||
mark_quota_hit(task_id)
|
||||
try:
|
||||
if cancel_vm is not None and vm_name:
|
||||
cancel_vm(vm_name)
|
||||
log_event(logger, "worker_chv_killed",
|
||||
task_id=task_id,
|
||||
message="quota watchdog killed VM %s "
|
||||
"(quota exceeded)" % vm_name)
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
|
||||
|
||||
def watchdog_vm_name(executor: object, task: dict) -> str | None:
|
||||
"""看门狗启动时的 VM 名派生(原 _run_attempt 块):
|
||||
executor.vm_name(task) 抛错时回 None(不建看门狗),
|
||||
绝不炸主派发流程。executor 无 vm_name 时等同抛错
|
||||
(原代码里 AttributeError 同样被 except 吞掉回 None)。"""
|
||||
get_vm_name = getattr(executor, "vm_name", None)
|
||||
if not callable(get_vm_name):
|
||||
return None
|
||||
try:
|
||||
return get_vm_name(task)
|
||||
except Exception:
|
||||
return None
|
||||
@@ -31,20 +31,20 @@ npm run build:console
|
||||
`npm run build:console` = `tsc -b && vite build`(vite-plugin-singlefile
|
||||
把 JS/CSS 全部内联成一个 `dist/index.html`),再由
|
||||
`scripts/copy-build.mjs` **原样**拷到
|
||||
`../console.html`(即 GET /console 现读的文件),并打印字节数与
|
||||
sha256、校验拷后内容与 dist 一致。脚本不改动产物内容。
|
||||
旧目标 `../console-redesign/console-react.html` 在仓库里不存在,
|
||||
已废弃,不再靠手工拷兜底。
|
||||
`../console-redesign/console-react.html`(审阅用),并打印字节数与
|
||||
sha256。脚本不改动产物内容。
|
||||
|
||||
## 自动部署链路(不留手工步骤)
|
||||
|
||||
```
|
||||
改 src → npm run build:console → 产物直接拷到 cloud/console.html → 提交 → 部署
|
||||
改 src → npm run build:console → 产物拷到 cloud/console.html → 提交 → 部署
|
||||
```
|
||||
|
||||
具体到仓库里的一步(本项目不部署生产):
|
||||
具体到仓库里的一步(由主 agent 在审阅后执行,本项目不部署生产):
|
||||
|
||||
```bash
|
||||
cp console-redesign/console-react.html <仓库>/cloud/console.html
|
||||
# 或在仓库分支里直接用构建产物覆盖 cloud/console.html
|
||||
git add cloud/console.html && git commit -m "console: React 版"
|
||||
# 随后走仓库已有的 PR → CI → 合并 → deploy.py git 模式部署
|
||||
```
|
||||
@@ -58,10 +58,10 @@ git add cloud/console.html && git commit -m "console: React 版"
|
||||
## 怎么验证
|
||||
|
||||
```bash
|
||||
npm run build:console # 必须 0 退出,tsc 无错误,且拷后与 dist/index.html 内容一致
|
||||
ls -l ../console.html
|
||||
sha256sum ../console.html dist/index.html # 两者 sha 应相同
|
||||
grep -E 'src="/assets|href="/assets' ../console.html # 应无输出
|
||||
npm run build:console # 必须 0 退出,tsc 无错误
|
||||
ls -l ../console-redesign/console-react.html
|
||||
sha256sum ../console-redesign/console-react.html
|
||||
grep -E 'src="/assets|href="/assets' ../console-redesign/console-react.html # 应无输出
|
||||
```
|
||||
|
||||
部署后另行核对(主 agent 执行):公网 `/console` 的字节数/sha 与
|
||||
|
||||
Generated
+1
-252
@@ -17,8 +17,7 @@
|
||||
"@vitejs/plugin-react": "^4.4.1",
|
||||
"typescript": "~5.8.3",
|
||||
"vite": "^6.3.5",
|
||||
"vite-plugin-singlefile": "^2.2.0",
|
||||
"vitest": "^5.0.5"
|
||||
"vite-plugin-singlefile": "^2.2.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@babel/code-frame": {
|
||||
@@ -1214,24 +1213,6 @@
|
||||
"@babel/types": "^7.28.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/chai": {
|
||||
"version": "5.2.3",
|
||||
"resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz",
|
||||
"integrity": "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/deep-eql": "*",
|
||||
"assertion-error": "^2.0.1"
|
||||
}
|
||||
},
|
||||
"node_modules/@types/deep-eql": {
|
||||
"version": "4.0.2",
|
||||
"resolved": "https://registry.npmjs.org/@types/deep-eql/-/deep-eql-4.0.2.tgz",
|
||||
"integrity": "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/@types/estree": {
|
||||
"version": "1.0.9",
|
||||
"resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz",
|
||||
@@ -1280,26 +1261,6 @@
|
||||
"vite": "^4.2.0 || ^5.0.0 || ^6.0.0 || ^7.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@vitest/spy": {
|
||||
"version": "5.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/spy/-/spy-5.0.5.tgz",
|
||||
"integrity": "sha512-lwJRExuH417257Q5wv9QYugN9IMJkoxcrKrYxGPQkBzl7wlNpTUxD/5yNX4ESKVUIShKPgkGchYPFMWOOk0wzw==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"funding": {
|
||||
"url": "https://opencollective.com/vitest"
|
||||
}
|
||||
},
|
||||
"node_modules/assertion-error": {
|
||||
"version": "2.0.1",
|
||||
"resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-2.0.1.tgz",
|
||||
"integrity": "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12"
|
||||
}
|
||||
},
|
||||
"node_modules/baseline-browser-mapping": {
|
||||
"version": "2.11.34",
|
||||
"resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.34.tgz",
|
||||
@@ -1368,16 +1329,6 @@
|
||||
],
|
||||
"license": "CC-BY-4.0"
|
||||
},
|
||||
"node_modules/chai": {
|
||||
"version": "6.3.0",
|
||||
"resolved": "https://registry.npmjs.org/chai/-/chai-6.3.0.tgz",
|
||||
"integrity": "sha512-XWAtwJ6OHO+tj0EKCs0Y2UamnyOxseZWltU4x2U2wh8g4AigdjwvtUjvLP2tqkA/avxHEtzxNaqGq/YGNwckKg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/convert-source-map": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz",
|
||||
@@ -1417,13 +1368,6 @@
|
||||
"dev": true,
|
||||
"license": "ISC"
|
||||
},
|
||||
"node_modules/es-module-lexer": {
|
||||
"version": "2.3.2",
|
||||
"resolved": "https://registry.npmjs.org/es-module-lexer/-/es-module-lexer-2.3.2.tgz",
|
||||
"integrity": "sha512-poHGpORABojJJucnV9KbOavETW8lBVnphkW77ER5/BQ5Fz7oXSoCNek7IH3vR5nRjdsEz926ibFYX8KtLQmdyw==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/esbuild": {
|
||||
"version": "0.25.12",
|
||||
"resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.25.12.tgz",
|
||||
@@ -1476,26 +1420,6 @@
|
||||
"node": ">=6"
|
||||
}
|
||||
},
|
||||
"node_modules/estree-walker": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/estree-walker/-/estree-walker-3.0.3.tgz",
|
||||
"integrity": "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/estree": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/expect-type": {
|
||||
"version": "1.4.0",
|
||||
"resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz",
|
||||
"integrity": "sha512-KfYbmpRm0VbLjEvVa9yGwCi9GI34xvi7A/HXYWQO65CSD2u3MczUJSuwXKFIxlGsgBQizV9q5J9NHj4VG0n+pA==",
|
||||
"dev": true,
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=12.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/fdir": {
|
||||
"version": "6.5.0",
|
||||
"resolved": "https://registry.npmjs.org/fdir/-/fdir-6.5.0.tgz",
|
||||
@@ -1582,16 +1506,6 @@
|
||||
"yallist": "^3.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/magic-string": {
|
||||
"version": "1.4.5",
|
||||
"resolved": "https://registry.npmjs.org/magic-string/-/magic-string-1.4.5.tgz",
|
||||
"integrity": "sha512-oLLC2+YvQUfAa+vxoufE/yc1icb87M9iXIRqAoR/2EhE/8cb3SF/+Gi0c2By1B+5Vz2e2vQtUvHiu1+63Znh+w==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@jridgewell/sourcemap-codec": "^1.6.0"
|
||||
}
|
||||
},
|
||||
"node_modules/ms": {
|
||||
"version": "2.1.3",
|
||||
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
|
||||
@@ -1628,20 +1542,6 @@
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/obug": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/obug/-/obug-3.0.0.tgz",
|
||||
"integrity": "sha512-5vvB5+W7ePv+p3uqxi+RcW1XAzLW0/hxt3/4X4Lc4qHudzOhmBiBwOY6DRob4WnanAEGvNcLjF+KNOufrUoEQw==",
|
||||
"dev": true,
|
||||
"funding": [
|
||||
"https://github.com/sponsors/sxzz",
|
||||
"https://opencollective.com/debug"
|
||||
],
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=12.20.0"
|
||||
}
|
||||
},
|
||||
"node_modules/picocolors": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz",
|
||||
@@ -1794,33 +1694,6 @@
|
||||
"node": ">=0.10.0"
|
||||
}
|
||||
},
|
||||
"node_modules/std-env": {
|
||||
"version": "4.3.0",
|
||||
"resolved": "https://registry.npmjs.org/std-env/-/std-env-4.3.0.tgz",
|
||||
"integrity": "sha512-OtU/EgQ1kIm5KwqQpBC6ZEMXrZRui11w8zgfTWp8cdO9B8OaPsbA8bTHO2P+HNo1VlUTGMVBwPhydu6poeXiag==",
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/tinybench": {
|
||||
"version": "6.2.2",
|
||||
"resolved": "https://registry.npmjs.org/tinybench/-/tinybench-6.2.2.tgz",
|
||||
"integrity": "sha512-nIZ2N2dFpjBsg2hgitKu7mA+FvsIJuGJOwysRngmVRRmWgR/UyPjwJj8c/fT/dhm8EkHnF5EOogRf+0Z2qrimA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=20.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/tinyexec": {
|
||||
"version": "1.3.1",
|
||||
"resolved": "https://registry.npmjs.org/tinyexec/-/tinyexec-1.3.1.tgz",
|
||||
"integrity": "sha512-GCvB3aoys96IuDFBMcTB46JOR6mdMtAToqwiW8JlWhsoh1mhHi/xn9ss/Dg7N555GiJyEt2qzoG/NHCwM6h1EA==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=18"
|
||||
}
|
||||
},
|
||||
"node_modules/tinyglobby": {
|
||||
"version": "0.2.17",
|
||||
"resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.17.tgz",
|
||||
@@ -1980,130 +1853,6 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/vitest": {
|
||||
"version": "5.0.5",
|
||||
"resolved": "https://registry.npmjs.org/vitest/-/vitest-5.0.5.tgz",
|
||||
"integrity": "sha512-cPODOMF8YnkiiYtDdHo6n4wWmBrOBBbOIYH2d33M4l3CdNn72h+qMmO6uXffUN6A5AZFXNND/d+krS9mgkXRfg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@types/chai": "^5.2.2",
|
||||
"@vitest/mocker": "5.0.5",
|
||||
"chai": "^6.3.0",
|
||||
"es-module-lexer": "^2.3.2",
|
||||
"expect-type": "^1.4.0",
|
||||
"magic-string": "^1.4.3",
|
||||
"obug": "^3.0.0",
|
||||
"picomatch": "^4.0.7",
|
||||
"std-env": "^4.3.0",
|
||||
"tinybench": "^6.2.2",
|
||||
"tinyexec": "^1.3.1",
|
||||
"tinyglobby": "^0.2.17",
|
||||
"why-is-node-running": "3.2.1"
|
||||
},
|
||||
"bin": {
|
||||
"vitest": "vitest.mjs"
|
||||
},
|
||||
"engines": {
|
||||
"node": "^22.12.0 || ^24.0.0 || >=26.0.0"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://opencollective.com/vitest"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@edge-runtime/vm": "*",
|
||||
"@opentelemetry/api": "^1.9.0",
|
||||
"@types/node": "^22.0.0 || >=24.0.0",
|
||||
"@vitest/browser-playwright": "5.0.5",
|
||||
"@vitest/browser-preview": "5.0.5",
|
||||
"@vitest/browser-webdriverio": "^5.0.0-beta.5 || >=5.0.0",
|
||||
"@vitest/coverage-istanbul": "5.0.5",
|
||||
"@vitest/coverage-v8": "5.0.5",
|
||||
"@vitest/ui": "5.0.5",
|
||||
"happy-dom": "*",
|
||||
"jsdom": "*",
|
||||
"vite": "^6.4.0 || ^7.0.0 || ^8.0.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@edge-runtime/vm": {
|
||||
"optional": true
|
||||
},
|
||||
"@opentelemetry/api": {
|
||||
"optional": true
|
||||
},
|
||||
"@types/node": {
|
||||
"optional": true
|
||||
},
|
||||
"@vitest/browser-playwright": {
|
||||
"optional": true
|
||||
},
|
||||
"@vitest/browser-preview": {
|
||||
"optional": true
|
||||
},
|
||||
"@vitest/browser-webdriverio": {
|
||||
"optional": true
|
||||
},
|
||||
"@vitest/coverage-istanbul": {
|
||||
"optional": true
|
||||
},
|
||||
"@vitest/coverage-v8": {
|
||||
"optional": true
|
||||
},
|
||||
"@vitest/ui": {
|
||||
"optional": true
|
||||
},
|
||||
"happy-dom": {
|
||||
"optional": true
|
||||
},
|
||||
"jsdom": {
|
||||
"optional": true
|
||||
},
|
||||
"vite": {
|
||||
"optional": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/vitest/node_modules/@vitest/mocker": {
|
||||
"version": "5.0.5",
|
||||
"resolved": "https://registry.npmjs.org/@vitest/mocker/-/mocker-5.0.5.tgz",
|
||||
"integrity": "sha512-xLL3lkDegJZw+MGoGaCCsFayDZthGgpAtr3Vziyfc4p7xH89K7KRPcRBBh/kEYm0Aqkt5DtTmkYpBtRV/FrHZg==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"@jridgewell/trace-mapping": "0.3.31",
|
||||
"@vitest/spy": "5.0.5",
|
||||
"estree-walker": "^3.0.3",
|
||||
"magic-string": "^1.4.3"
|
||||
},
|
||||
"funding": {
|
||||
"url": "https://opencollective.com/vitest"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"msw": "^2.4.9 || ^3.0.0",
|
||||
"vite": "^6.0.0 || ^7.0.0 || ^8.0.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"msw": {
|
||||
"optional": true
|
||||
},
|
||||
"vite": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/why-is-node-running": {
|
||||
"version": "3.2.1",
|
||||
"resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-3.2.1.tgz",
|
||||
"integrity": "sha512-Tb2FUhB4vUsGQlfSquQLYkApkuPAFQXGFzxWKHHumVz2dK+X1RUm/HnID4+TfIGYJ1kTcwOaCk/buYCEJr6YjQ==",
|
||||
"dev": true,
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"why-is-node-running": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.11"
|
||||
}
|
||||
},
|
||||
"node_modules/yallist": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
|
||||
|
||||
@@ -7,8 +7,7 @@
|
||||
"build": "tsc -b && vite build",
|
||||
"dev": "vite",
|
||||
"preview": "vite preview",
|
||||
"build:console": "npm run build && node scripts/copy-build.mjs",
|
||||
"test": "vitest run"
|
||||
"build:console": "npm run build && node scripts/copy-build.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"react": "^19.1.0",
|
||||
@@ -20,7 +19,6 @@
|
||||
"@vitejs/plugin-react": "^4.4.1",
|
||||
"typescript": "~5.8.3",
|
||||
"vite": "^6.3.5",
|
||||
"vite-plugin-singlefile": "^2.2.0",
|
||||
"vitest": "^5.0.5"
|
||||
"vite-plugin-singlefile": "^2.2.0"
|
||||
}
|
||||
}
|
||||
@@ -1,23 +1,17 @@
|
||||
// build:console 的第二步:把 vite-plugin-singlefile 产出的单文件
|
||||
// 原样拷到可部署位置。绝不改动产物内容——以后改页面只改 src 再构建。
|
||||
// 目标就是 GET /console 现读的文件 cloud/console.html
|
||||
// (cloud/server.py 与 console.html 同目录、每次请求现读),
|
||||
// 不再拷到仓库里不存在的 cloud/console-redesign/console-react.html,
|
||||
// 也不再靠手工拷兜底:npm run build:console 一次跑通即产物就位。
|
||||
import { copyFileSync } from "node:fs";
|
||||
import { createHash, timingSafeEqual } from "node:crypto";
|
||||
// 在本施工目录,目标是 ../console-redesign/console-react.html(主 agent
|
||||
// 审阅用);进仓库后,主 agent 把同一文件拷到 cloud/console.html。
|
||||
import { copyFileSync, statSync } from "node:fs";
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const root = join(dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const src = join(root, "dist", "index.html");
|
||||
const dest = join(root, "..", "console.html");
|
||||
const dest = join(root, "..", "console-redesign", "console-react.html");
|
||||
copyFileSync(src, dest);
|
||||
const srcBuf = readFileSync(src);
|
||||
const destBuf = readFileSync(dest);
|
||||
if (srcBuf.length !== destBuf.length || !timingSafeEqual(srcBuf, destBuf)) {
|
||||
throw new Error("copy-build: dest 与 dist/index.html 内容不一致,构建失败");
|
||||
}
|
||||
console.log(`console.html ${destBuf.length} bytes sha256=${createHash("sha256").update(destBuf).digest("hex")}`);
|
||||
const buf = readFileSync(dest);
|
||||
console.log(`console-react.html ${buf.length} bytes sha256=${createHash("sha256").update(buf).digest("hex")}`);
|
||||
console.log(`built: ${src} -> ${dest}`);
|
||||
@@ -9,8 +9,7 @@ import ConversationSection from "./sections/ConversationSection";
|
||||
import TaskDetailPanel from "./sections/TaskDetailPanel";
|
||||
import ApiKeySection from "./sections/ApiKeySection";
|
||||
import AdminSection from "./sections/AdminSection";
|
||||
import ProjectsSection from "./sections/ProjectsSection";
|
||||
import MembersPanel from "./sections/MembersPanel";
|
||||
import ProjectNav from "./sections/ProjectNav";
|
||||
import EnvSection from "./sections/EnvSection";
|
||||
import EnvContextPanel from "./sections/EnvContextPanel";
|
||||
import WorkersSection from "./sections/WorkersSection";
|
||||
@@ -25,7 +24,10 @@ import TodosSection from "./sections/TodosSection";
|
||||
// data-cards row on top of a long stacked page, no detail stack
|
||||
// hanging under the conversation. The new shell is an IDE-style
|
||||
// workspace that never scrolls as one page:
|
||||
// 左侧栏:导航(按工作流分组)+ 数据卡片(紧凑版,始终可见)
|
||||
// 左侧栏:导航(按工作流分组)+ 项目模块(ProjectNav:项目列表 +
|
||||
// 创建项目 + 成员,id nav-projects,是全站唯一的项目选择器;
|
||||
// 因此不再有独立的“项目与成员”视图,旧 ProjectsSection 已删,
|
||||
// 工作台也不再带项目选择器)+ 数据卡片(紧凑版,始终可见)
|
||||
// 任务工作台:左 = 任务列表轨(搜索/筛选/分页),中 = 派发 + 对话,
|
||||
// 右 = 任务详情检查器(概览/协作/评审与审批/线程四个内页签,
|
||||
// 见 TaskDetailPanel)
|
||||
@@ -36,11 +38,10 @@ import TodosSection from "./sections/TodosSection";
|
||||
// unmounted with a view. The view is synced to location.hash
|
||||
// (#view-tasks etc.) so a reload or a shared link lands on the
|
||||
// same view; unknown hashes fall back to the task workspace.
|
||||
type ViewId = "tasks" | "todos" | "projects" | "resources" | "automation" | "admin";
|
||||
type ViewId = "tasks" | "todos" | "resources" | "automation" | "admin";
|
||||
const VIEWS: Array<[ViewId, string, string]> = [
|
||||
["tasks", "任务工作台", "任务"],
|
||||
["todos", "项目待办", "任务"],
|
||||
["projects", "项目与成员", "管理"],
|
||||
["resources", "环境与 Workers", "管理"],
|
||||
["automation", "技能与自动化", "管理"],
|
||||
["admin", "系统与 Key", "管理"],
|
||||
@@ -67,6 +68,9 @@ function SideNav({ view, setView }: { view: ViewId; setView: (v: ViewId) => void
|
||||
</div>
|
||||
))}
|
||||
</nav>
|
||||
<div id="nav-projects" className="shell-project-nav">
|
||||
<ProjectNav />
|
||||
</div>
|
||||
<div className="shell-stats">
|
||||
<div className="shell-stats-label">概览</div>
|
||||
<DataCards />
|
||||
@@ -138,7 +142,6 @@ export default function App() {
|
||||
) : (
|
||||
<main className="page">
|
||||
{view === "todos" ? (<><TodosSection /></>)
|
||||
: view === "projects" ? (<><ProjectsSection /><MembersPanel /></>)
|
||||
: view === "resources" ? (<><EnvSection /><EnvContextPanel /><WorkersSection /><HealthPanel /></>)
|
||||
: view === "automation" ? (<><SkillsSection /><SkillsToolsPanel /><AutomationsSection /><TriageSection /></>)
|
||||
: (<><AdminSection /><ApiKeySection /></>)}
|
||||
|
||||
@@ -1,85 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { chvAttachmentBlocked, chvAvailability, chvAvailabilityText, chvQueueHint, chvVmBadge, chvVmDisplay, chvVmStatusText, countChvWorkers, isWorkerAlive } from "./chv";
|
||||
|
||||
// closeup 轨补边角:上一轨 chv.test.ts 已覆主路径后仍缺的
|
||||
// 显式 false、undefined、优先级与“旧 bug 文案不复发”断言。
|
||||
// 只测 chv.ts 纯逻辑,不渲染组件、不发请求。
|
||||
describe("chvVmDisplay 边角(closeup)", () => {
|
||||
it("字段显式 false 与 absent 同属 not-reported", () => {
|
||||
const d = chvVmDisplay({ executor: "cloud-hypervisor", chv_vm_dead: false });
|
||||
expect(d.state).toBe("not-reported");
|
||||
expect(d.text).toContain("不代表 VM 活着");
|
||||
});
|
||||
it("任务 undefined 与 null 同属 unknown", () => {
|
||||
expect(chvVmDisplay(undefined).state).toBe("unknown");
|
||||
expect(chvVmDisplay(undefined).text).toContain("无法判断");
|
||||
});
|
||||
it("非 CHV 且无 executor 字段时也是 not-applicable", () => {
|
||||
expect(chvVmDisplay({}).state).toBe("not-applicable");
|
||||
expect(chvVmDisplay({ executor: undefined }).text).toBe("");
|
||||
expect(chvVmStatusText({ executor: "cloud-hypervisor", chv_vm_dead: false })).toContain("未报告 VM 死亡");
|
||||
});
|
||||
it("列表徽标:显式 false 不出徽标", () => {
|
||||
expect(chvVmBadge({ executor: "cloud-hypervisor", chv_vm_dead: false })).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("chvAvailability 优先级(closeup)", () => {
|
||||
it("加载失败优先于聚合文案:不把失败写成 aggregate-only", () => {
|
||||
expect(chvAvailability(null, "完整 worker 清单仅全局管理员可见", "加载失败:x")).toEqual(
|
||||
{ state: "unknown", reason: "load-failed" },
|
||||
);
|
||||
});
|
||||
it("有清单时忽略错误与聚合文案", () => {
|
||||
expect(chvAvailability([{ executor: "cloud-hypervisor", alive: false }], "完整 worker 清单仅全局管理员可见", "加载失败:x")).toEqual(
|
||||
{ state: "known", online: 0, total: 1 },
|
||||
);
|
||||
});
|
||||
it("空清单与空文案是 not-loaded,不是已知 0", () => {
|
||||
expect(chvAvailability(null, "", "")).toEqual({ state: "unknown", reason: "not-loaded" });
|
||||
});
|
||||
it("空 worker 数组是已知 0(与 null 区分)", () => {
|
||||
expect(chvAvailability([], "", "")).toEqual({ state: "known", online: 0, total: 0 });
|
||||
expect(countChvWorkers([])).toEqual({ online: 0, total: 0 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("chvAvailabilityText:Workers 旧 bug 不复发(closeup)", () => {
|
||||
it("load-failed / not-loaded 绝不写“仅全局管理员可见”", () => {
|
||||
expect(chvAvailabilityText({ state: "unknown", reason: "load-failed" })).not.toContain("仅全局管理员可见");
|
||||
expect(chvAvailabilityText({ state: "unknown", reason: "not-loaded" })).not.toContain("仅全局管理员可见");
|
||||
});
|
||||
it("known 总数为 0 时仍是已知 0,不写未知", () => {
|
||||
const text = chvAvailabilityText({ state: "known", online: 0, total: 0 });
|
||||
expect(text).toContain("在线 0 / 共 0");
|
||||
expect(text).not.toContain("未知");
|
||||
});
|
||||
});
|
||||
|
||||
describe("附件拦截与排队提示边角(closeup)", () => {
|
||||
it("executor 为 undefined/null 时附件不拦", () => {
|
||||
expect(chvAttachmentBlocked(undefined, true)).toBe(false);
|
||||
expect(chvAttachmentBlocked(null, true)).toBe(false);
|
||||
});
|
||||
it("排队提示:未知三原因一律为空串", () => {
|
||||
for (const reason of ["aggregate-only", "load-failed", "not-loaded"] as const) {
|
||||
expect(chvQueueHint("cloud-hypervisor", "queued", { state: "unknown", reason })).toBe("");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("isWorkerAlive 边界(closeup)", () => {
|
||||
// 精确边界不用活时钟测:原写法先取 Date.now() 再让被测函数
|
||||
// 内部重取一次时钟,age = 120 + 两次取时钟的差,两次取时钟
|
||||
// 跨过毫秒边界时 age = 120.001,<=120 判定为 false,约 7%
|
||||
// 概率抖动(30 轮 2 败,均是本用例)。生产逻辑 <=120 本身
|
||||
// 没错,错的是用活时钟测精确边界。
|
||||
it("恰好 120 秒算在线(<=120,用不取时钟的字段确定性测)", () => {
|
||||
expect(isWorkerAlive({ executor: "cloud-hypervisor", last_seen_seconds_ago: 120 })).toBe(true);
|
||||
expect(isWorkerAlive({ executor: "cloud-hypervisor", last_seen_seconds_ago: 121 })).toBe(false);
|
||||
});
|
||||
it("活时钟只测带 1 秒余量的内侧,不测精确边界", () => {
|
||||
const now = Date.now() / 1000;
|
||||
expect(isWorkerAlive({ executor: "cloud-hypervisor", last_seen: now - 119 })).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -1,155 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { chvAttachmentBlocked, chvAvailability, chvAvailabilityText, chvQueueHint, chvVmBadge, chvVmDisplay, chvVmStatusText, countChvWorkers, isChvExecutor, isWorkerAlive } from "./chv";
|
||||
|
||||
// 纯逻辑测试:只测 chv.ts,不渲染组件、不发请求、不读 localStorage。
|
||||
// alive 规则与 WorkersSection 现有口径一致:alive 字段优先,
|
||||
// 缺失时用 last_seen 距今 <= 120 秒判定在线。
|
||||
describe("isWorkerAlive", () => {
|
||||
it("alive 字段优先", () => {
|
||||
expect(isWorkerAlive({ executor: "cloud-hypervisor", alive: true })).toBe(true);
|
||||
expect(isWorkerAlive({ executor: "cloud-hypervisor", alive: false, last_seen: Date.now() / 1000 })).toBe(false);
|
||||
});
|
||||
it("无 alive 时用 last_seen 120 秒门槛", () => {
|
||||
const now = Date.now() / 1000;
|
||||
expect(isWorkerAlive({ executor: "cloud-hypervisor", last_seen: now - 10 })).toBe(true);
|
||||
expect(isWorkerAlive({ executor: "cloud-hypervisor", last_seen: now - 121 })).toBe(false);
|
||||
expect(isWorkerAlive({ executor: "cloud-hypervisor" })).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("countChvWorkers", () => {
|
||||
it("只数 cloud-hypervisor,不数 docker/local", () => {
|
||||
const now = Date.now() / 1000;
|
||||
expect(countChvWorkers([
|
||||
{ executor: "cloud-hypervisor", alive: true },
|
||||
{ executor: "cloud-hypervisor", last_seen: now - 9999 },
|
||||
{ executor: "docker", alive: true },
|
||||
])).toEqual({ online: 1, total: 2 });
|
||||
});
|
||||
});
|
||||
|
||||
describe("chvAvailability", () => {
|
||||
it("有清单时按清单数", () => {
|
||||
expect(chvAvailability([{ executor: "cloud-hypervisor", alive: true }], "", "")).toEqual(
|
||||
{ state: "known", online: 1, total: 1 },
|
||||
);
|
||||
});
|
||||
it("无清单但聚合文案时是未知,不是 0", () => {
|
||||
expect(chvAvailability(null, "在线 2 / 离线 1(共 3;完整 worker 清单仅全局管理员可见)", "")).toEqual(
|
||||
{ state: "unknown", reason: "aggregate-only" },
|
||||
);
|
||||
});
|
||||
it("无 worker 注册时是已知 0", () => {
|
||||
expect(chvAvailability(null, "无 worker 注册", "")).toEqual({ state: "known", online: 0, total: 0 });
|
||||
});
|
||||
it("加载失败时是未知", () => {
|
||||
expect(chvAvailability(null, "", "加载失败:x")).toEqual({ state: "unknown", reason: "load-failed" });
|
||||
});
|
||||
it("尚未加载时是未知", () => {
|
||||
expect(chvAvailability(null, "未加载", "")).toEqual({ state: "unknown", reason: "not-loaded" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("isChvExecutor", () => {
|
||||
it("精确值才算 CHV,不做大小写/前缀宽松匹配", () => {
|
||||
expect(isChvExecutor("cloud-hypervisor")).toBe(true);
|
||||
expect(isChvExecutor("Cloud-Hypervisor")).toBe(false);
|
||||
expect(isChvExecutor("cloud-hypervisor-x")).toBe(false);
|
||||
expect(isChvExecutor("docker")).toBe(false);
|
||||
expect(isChvExecutor(undefined)).toBe(false);
|
||||
expect(isChvExecutor(null)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("chvAttachmentBlocked", () => {
|
||||
it("仅 CHV + 带附件时拦(后端 400 的前端提示口径)", () => {
|
||||
expect(chvAttachmentBlocked("cloud-hypervisor", true)).toBe(true);
|
||||
expect(chvAttachmentBlocked("cloud-hypervisor", false)).toBe(false);
|
||||
expect(chvAttachmentBlocked("docker", true)).toBe(false);
|
||||
expect(chvAttachmentBlocked("cloud", true)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("chvVmStatusText", () => {
|
||||
it("已报告死亡时给死亡文案", () => {
|
||||
expect(chvVmStatusText({ executor: "cloud-hypervisor", chv_vm_dead: true })).toContain("VM 中途死亡");
|
||||
});
|
||||
it("CHV 无字段时给未报告文案,绝不写成活着", () => {
|
||||
const text = chvVmStatusText({ executor: "cloud-hypervisor" });
|
||||
expect(text).toContain("未报告 VM 死亡");
|
||||
expect(text).toContain("不代表 VM 活着");
|
||||
});
|
||||
it("非 CHV 任务不显示 VM 行", () => {
|
||||
expect(chvVmStatusText({ executor: "docker" })).toBe("");
|
||||
expect(chvVmStatusText({ executor: "docker", chv_vm_dead: true })).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("chvVmBadge", () => {
|
||||
it("只有 CHV + 已报告死亡时才出徽标", () => {
|
||||
expect(chvVmBadge({ executor: "cloud-hypervisor", chv_vm_dead: true })).toBe("VM 已报告死亡");
|
||||
expect(chvVmBadge({ executor: "cloud-hypervisor" })).toBe("");
|
||||
expect(chvVmBadge({ executor: "docker", chv_vm_dead: true })).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("chvQueueHint", () => {
|
||||
it("仅 CHV + 排队 + 已知零在线时给提示", () => {
|
||||
const knownZero = { state: "known" as const, online: 0, total: 0 };
|
||||
expect(chvQueueHint("cloud-hypervisor", "queued", knownZero)).toContain("可能原因");
|
||||
expect(chvQueueHint("docker", "queued", knownZero)).toBe("");
|
||||
expect(chvQueueHint("cloud-hypervisor", "running", knownZero)).toBe("");
|
||||
expect(chvQueueHint("cloud-hypervisor", "queued", { state: "unknown" as const, reason: "aggregate-only" })).toBe("");
|
||||
expect(chvQueueHint("cloud-hypervisor", "queued", { state: "known" as const, online: 1, total: 1 })).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("chvVmDisplay(to10 四态)", () => {
|
||||
it("字段 present=true → dead", () => {
|
||||
const d = chvVmDisplay({ executor: "cloud-hypervisor", chv_vm_dead: true });
|
||||
expect(d.state).toBe("dead");
|
||||
expect(d.text).toContain("VM 中途死亡");
|
||||
});
|
||||
it("字段 absent → not-reported,不猜 live", () => {
|
||||
const d = chvVmDisplay({ executor: "cloud-hypervisor" });
|
||||
expect(d.state).toBe("not-reported");
|
||||
expect(d.text).toContain("不代表 VM 活着");
|
||||
});
|
||||
it("任务未加载 → unknown", () => {
|
||||
const d = chvVmDisplay(null);
|
||||
expect(d.state).toBe("unknown");
|
||||
expect(d.text).toContain("无法判断");
|
||||
});
|
||||
it("非 CHV → not-applicable,text 为空", () => {
|
||||
const d = chvVmDisplay({ executor: "docker", chv_vm_dead: true });
|
||||
expect(d.state).toBe("not-applicable");
|
||||
expect(d.text).toBe("");
|
||||
});
|
||||
it("旧入口仍是 display 的 text(unknown/not-applicable 回空)", () => {
|
||||
expect(chvVmStatusText({ executor: "cloud-hypervisor", chv_vm_dead: true })).toContain("VM 中途死亡");
|
||||
expect(chvVmStatusText(null)).toBe("");
|
||||
expect(chvVmStatusText({ executor: "docker" })).toBe("");
|
||||
});
|
||||
});
|
||||
|
||||
describe("chvAvailabilityText(to10 四处统一文案)", () => {
|
||||
it("known 零在线:不拦提交、后端是最终权威", () => {
|
||||
const text = chvAvailabilityText({ state: "known", online: 0, total: 2 });
|
||||
expect(text).toContain("在线 0 / 共 2");
|
||||
expect(text).toContain("不拦提交");
|
||||
expect(text).toContain("最终权威");
|
||||
});
|
||||
it("known 非零:只有在线数,无排队警告", () => {
|
||||
const text = chvAvailabilityText({ state: "known", online: 1, total: 1 });
|
||||
expect(text).toContain("在线 1 / 共 1");
|
||||
expect(text).not.toContain("排队");
|
||||
});
|
||||
it("未知三原因各有不同文案,绝不写成 0", () => {
|
||||
expect(chvAvailabilityText({ state: "unknown", reason: "aggregate-only" })).toContain("仅全局管理员可见");
|
||||
expect(chvAvailabilityText({ state: "unknown", reason: "load-failed" })).toContain("加载失败");
|
||||
expect(chvAvailabilityText({ state: "unknown", reason: "not-loaded" })).toContain("尚未加载");
|
||||
for (const reason of ["aggregate-only", "load-failed", "not-loaded"] as const) {
|
||||
expect(chvAvailabilityText({ state: "unknown", reason })).not.toContain("在线 0");
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -1,163 +0,0 @@
|
||||
import type { Worker } from "./types";
|
||||
import { workerAgeText } from "./client";
|
||||
|
||||
// CHV 可用性纯逻辑:不发请求、不拼 URL,只从 ctx 已有的
|
||||
// /workers 数据推断。非管理员的 /workers 只返回聚合数
|
||||
// (workers_total/online/offline),没有按执行器的分项,
|
||||
// 此时 CHV 在线数是未知,绝不能把未知写成 0,也绝不能
|
||||
// 把聚合在线数当成 CHV 在线数。
|
||||
export function isWorkerAlive(w: Worker): boolean {
|
||||
if (w.alive !== undefined && w.alive !== null) return !!w.alive;
|
||||
const info = workerAgeText(w);
|
||||
return info.age !== null && info.age <= 120;
|
||||
}
|
||||
|
||||
export function countChvWorkers(workers: Worker[]): { online: number; total: number } {
|
||||
const chv = workers.filter((w) => w.executor === "cloud-hypervisor");
|
||||
return { online: chv.filter(isWorkerAlive).length, total: chv.length };
|
||||
}
|
||||
|
||||
export type ChvAvailability =
|
||||
| { state: "known"; online: number; total: number }
|
||||
| { state: "unknown"; reason: "not-loaded" | "aggregate-only" | "load-failed" };
|
||||
|
||||
export function chvAvailability(
|
||||
workers: Worker[] | null,
|
||||
workersSummary: string,
|
||||
workersError: string,
|
||||
): ChvAvailability {
|
||||
if (workers) return { state: "known", ...countChvWorkers(workers) };
|
||||
if (workersError) return { state: "unknown", reason: "load-failed" };
|
||||
if (workersSummary.includes("无 worker 注册")) return { state: "known", online: 0, total: 0 };
|
||||
if (workersSummary.includes("完整 worker 清单仅全局管理员可见")) {
|
||||
return { state: "unknown", reason: "aggregate-only" };
|
||||
}
|
||||
return { state: "unknown", reason: "not-loaded" };
|
||||
}
|
||||
|
||||
// 执行器选择谓词:派发下拉与列表筛选都以字符串相等为准,
|
||||
// 不做大小写/前缀宽松匹配(后端白名单是精确值,见 server
|
||||
// POST /tasks 的 executor 白名单;宽松匹配会把非法值放进
|
||||
// “CHV 任务”分支,造成前端提示与后端拒绝不一致)。
|
||||
export function isChvExecutor(executor: string | undefined | null): boolean {
|
||||
return executor === "cloud-hypervisor";
|
||||
}
|
||||
|
||||
// 附件守卫谓词:CHV + 带附件 → 后端建任务即拒绝(server
|
||||
// POST /tasks 的附件绑定分支,错误文案“附件不支持
|
||||
// cloud-hypervisor 执行器”)。本函数只作前端提示口径,
|
||||
// 绝不代替后端白名单:前端照常提交,后端拒绝为准。
|
||||
export function chvAttachmentBlocked(
|
||||
executor: string | undefined | null,
|
||||
hasAttachments: boolean,
|
||||
): boolean {
|
||||
return isChvExecutor(executor) && hasAttachments;
|
||||
}
|
||||
|
||||
// VM 只读状态:四种情形各有明确文案,绝不编造第五种,
|
||||
// 绝不猜 live(to10 精确化,见 ADR-15):
|
||||
// - dead:CHV 任务且 chv_vm_dead 为 true(字段已 present;
|
||||
// 来源:worker 终局写回的 interrupted_reason,见 server
|
||||
// _chv_vm_dead_from_row);
|
||||
// - not-reported:CHV 任务但字段 absent(后端只在死亡时
|
||||
// 才返回该字段;absent 只表示“未报告”,且明写“后端无
|
||||
// live 状态字段,不代表 VM 活着”,server 没有 VM 登记表);
|
||||
// - unknown:任务详情本身未加载(task 为 null),无法判断;
|
||||
// - not-applicable:非 CHV 任务,不显示 VM 行(text 为空)。
|
||||
export type ChvVmDisplay =
|
||||
| { state: "dead"; text: string }
|
||||
| { state: "not-reported"; text: string }
|
||||
| { state: "unknown"; text: string }
|
||||
| { state: "not-applicable"; text: string };
|
||||
|
||||
export function chvVmDisplay(task: {
|
||||
executor?: string | null;
|
||||
chv_vm_dead?: boolean;
|
||||
} | null | undefined): ChvVmDisplay {
|
||||
if (task === null || task === undefined) {
|
||||
return {
|
||||
state: "unknown",
|
||||
text: "任务详情未加载,无法判断 VM 状态(后端无 live 状态字段,详情加载后才显示)",
|
||||
};
|
||||
}
|
||||
// 死亡文案同样先过执行器门:后端只对 CHV 任务派生该字段,
|
||||
// 万一某天非 CHV 行误带该字段,前端也不把 docker 任务
|
||||
// 写成“VM 死亡”。
|
||||
if (task.chv_vm_dead && isChvExecutor(task.executor)) {
|
||||
return {
|
||||
state: "dead",
|
||||
text: "VM 中途死亡,任务未完成、断点可续跑(后端只读字段 chv_vm_dead)",
|
||||
};
|
||||
}
|
||||
if (isChvExecutor(task.executor)) {
|
||||
return {
|
||||
state: "not-reported",
|
||||
text: "未报告 VM 死亡(后端无 live 状态字段,不代表 VM 活着)",
|
||||
};
|
||||
}
|
||||
return { state: "not-applicable", text: "" };
|
||||
}
|
||||
|
||||
// 旧入口(保留给已测调用点):即 chvVmDisplay 的 text,
|
||||
// not-applicable 与 unknown 均回空串(调用点只在已加载的
|
||||
// CHV 行上用它,unknown 分支由详情页的新入口处理)。
|
||||
export function chvVmStatusText(task: {
|
||||
executor?: string | null;
|
||||
chv_vm_dead?: boolean;
|
||||
} | null | undefined): string {
|
||||
const display = chvVmDisplay(task);
|
||||
return display.state === "not-applicable" ||
|
||||
display.state === "unknown"
|
||||
? ""
|
||||
: display.text;
|
||||
}
|
||||
|
||||
// CHV 可用性统一文案(to10):新建任务、Workers、环境
|
||||
// 三处此前各写各的,Workers 的未知分支甚至恒写“仅全局
|
||||
// 管理员可见”(加载失败/未加载时也是这句,是文案 bug)。
|
||||
// 四处统一走本函数:known 零在线时不拦提交、后端是最终
|
||||
// 权威;未知时按 reason 分三句,绝不把未知写成 0。
|
||||
export function chvAvailabilityText(
|
||||
availability: ChvAvailability,
|
||||
): string {
|
||||
if (availability.state === "known") {
|
||||
const base = `CHV worker 在线 ${availability.online} / 共 ${availability.total}(来自 /workers)`;
|
||||
return availability.online === 0
|
||||
? `${base}:当前无 CHV worker 在线,任务将一直排队,请先确认宿主 worker 已起;本提示不拦提交,worker 随后上线可认领,后端是最终权威。`
|
||||
: base;
|
||||
}
|
||||
if (availability.reason === "aggregate-only") {
|
||||
return "CHV worker 在线数未知:完整 worker 清单仅全局管理员可见,非管理员只能看到聚合数,无法按执行器分项。";
|
||||
}
|
||||
if (availability.reason === "load-failed") {
|
||||
return "CHV worker 在线数未知:/workers 加载失败,请到 Workers 区刷新后重试。";
|
||||
}
|
||||
return "CHV worker 在线数未知:/workers 尚未加载,请到 Workers 区刷新。";
|
||||
}
|
||||
|
||||
// 列表行徽标:只有已报告死亡时才出现短徽标;未报告时
|
||||
// 列表行不写 VM 状态(详情里才写“未报告”长文案,避免
|
||||
// 列表行被未知状态刷屏)。
|
||||
export function chvVmBadge(task: {
|
||||
executor?: string | null;
|
||||
chv_vm_dead?: boolean;
|
||||
}): string {
|
||||
return task.chv_vm_dead && isChvExecutor(task.executor)
|
||||
? "VM 已报告死亡"
|
||||
: "";
|
||||
}
|
||||
|
||||
// 排队原因提示:只在“已知零 CHV worker 在线 + CHV 任务排队”时
|
||||
// 给出,文案明确写“可能原因”,且不编造后端未返回的
|
||||
// VM 状态/IP/快照。未知时返回空串(不提示、不猜测)。
|
||||
export function chvQueueHint(
|
||||
executor: string | undefined | null,
|
||||
status: string | undefined | null,
|
||||
availability: ChvAvailability,
|
||||
): string {
|
||||
if (executor !== "cloud-hypervisor" || status !== "queued") return "";
|
||||
if (availability.state === "known" && availability.online === 0) {
|
||||
return "可能原因:当前无 cloud-hypervisor worker 在线,任务在排队(基于 /workers 推断;VM 状态/IP/快照等后端未返回字段不展示)。请先确认 KVM 宿主上的 CHV worker 已起,worker 上线后可认领。";
|
||||
}
|
||||
return "";
|
||||
}
|
||||
@@ -93,6 +93,7 @@ export function memberErr(e: Error): string {
|
||||
if (m.includes("forbidden")) return "没有权限:只有项目 owner 或全局管理员可以管理成员";
|
||||
if (m.includes("already_member")) return "该用户已是成员,改角色请用成员行的下拉框";
|
||||
if (m.includes("cannot demote the only owner")) return "不能把唯一 owner 降为非 owner:请先把另一位成员升为 owner";
|
||||
if (m.includes("cannot remove the only owner")) return "不能移除唯一 owner:请先把另一位成员升为 owner";
|
||||
if (m.includes("user not found")) return "找不到该用户:请核对 user_id 或用户名是否正确";
|
||||
if (m.includes("member not found")) return "该用户不是本项目成员";
|
||||
if (m.includes("unknown role")) return "角色无效:只能是 viewer / contributor / maintainer / owner";
|
||||
|
||||
@@ -21,10 +21,6 @@ export interface Task {
|
||||
coordinator_part?: string; goal_mode?: number | boolean; verify_command?: string;
|
||||
no_time_limit?: number | boolean;
|
||||
environment_id?: string; attachments?: string[]; model?: string; effort?: string;
|
||||
// feat-chv-arch95plus 只读字段:仅 CHV 任务且 worker 已把
|
||||
// VM 死亡写进 interrupted_reason 时 server 才返回 true,
|
||||
// 其它情形字段不存在(绝不把“未报告死亡”当成“VM 活着”)。
|
||||
chv_vm_dead?: boolean;
|
||||
child_model?: string; child_effort?: string; approval?: string;
|
||||
result_text?: string; error_text?: string;
|
||||
}
|
||||
@@ -40,7 +36,9 @@ export interface Followup { mode?: string; status?: string; sender_task_id?: str
|
||||
export interface ChatHistoryMessage { id: string; role: "user" | "assistant" | "tool" | "status"; source?: string; kind?: string; label?: string; text: string; ts_ms?: number | null }
|
||||
export interface ChatHistory { task_id?: string; status?: string; messages: ChatHistoryMessage[]; continue_messages_stored?: number; legacy_continue_count?: number | null; trace_note?: string }
|
||||
export interface PeerMessage { task_id?: string; mode?: string; status?: string; body?: string }
|
||||
export interface Automation { id: string; name?: string; prompt?: string; repo_url?: string; status?: string; due?: boolean; interval_seconds?: number; next_run_at?: number; last_run_at?: number; created_at?: number; cancelled_at?: number; run_once?: boolean }
|
||||
export interface Automation { id: string; name?: string; prompt?: string; repo_url?: string; status?: string; due?: boolean; interval_seconds?: number; next_run_at?: number; last_run_at?: number; last_claimed_at?: number; created_by?: string; created_at?: number; cancelled_at?: number; first_run_at?: number; run_once?: boolean; project_id?: string }
|
||||
export interface AutomationRun { id?: string; automation_id?: string; project_id?: string; task_id?: string; status?: string; detail?: string; created_at?: number }
|
||||
export interface AutomationDetail extends Automation { runs?: AutomationRun[]; runs_note?: string; heartbeat_binding?: unknown }
|
||||
export interface Member { user_id: string; username?: string; role?: string }
|
||||
export interface SkillItem { name?: string; id?: string; description?: string; detail?: string }
|
||||
export interface McpServer { name?: string; id?: string; status?: string; detail?: string; details?: string; state?: string }
|
||||
@@ -49,7 +47,7 @@ export interface Agent { kind?: string; name?: string; status?: string; prompt?:
|
||||
export interface ThreadInfo { id?: string; name?: string; thread_name?: string; archived?: boolean | null; cwd?: string; turns?: number; turn_count?: number; turns_count?: number }
|
||||
export interface Triage {
|
||||
counts?: { automations_due?: number; failed_tasks?: number; pending_followups?: number; recent_runs?: number };
|
||||
automations_due?: Array<{ name?: string; interval_seconds?: number }>;
|
||||
automations_due?: Array<{ id?: string; name?: string; status?: string; due?: boolean; interval_seconds?: number; next_run_at?: number }>;
|
||||
failed_tasks?: Array<{ id?: string; status?: string; prompt?: string }>;
|
||||
pending_followups?: Array<{ task_id?: string; body?: string }>;
|
||||
recent_runs?: Array<{ automation_id?: string; status?: string; task_id?: string; task_status?: string }>;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { createContext, useContext } from "react";
|
||||
import type { Automation, Env, Member, Project, Review, Approval, Followup, PeerMessage, Task, TaskCost, ThreadInfo, Triage, Worker, SkillItem, McpServer, Agent, DispatchBody, ReviewBody, TraceEntry, ChatHistory } from "./api/types";
|
||||
import type { Automation, AutomationDetail, Env, Member, Project, Review, Approval, Followup, PeerMessage, Task, TaskCost, ThreadInfo, Triage, Worker, SkillItem, McpServer, Agent, DispatchBody, ReviewBody, TraceEntry, ChatHistory } from "./api/types";
|
||||
|
||||
export interface ConsoleCtx {
|
||||
keySaved: boolean; keyState: string;
|
||||
@@ -7,13 +7,13 @@ export interface ConsoleCtx {
|
||||
quotaText: string;
|
||||
healthText: string; refreshHealth: () => void;
|
||||
adminGate: string; adminBody: boolean; adminResult: string; adminUserId: string; setAdminUserId: (v: string) => void;
|
||||
adminQuota: () => void; adminSweep: () => void; adminBackup: () => void;
|
||||
adminQuota: () => void; adminSetQuota: (uid: string, usdText: string) => void; adminSweep: () => void; adminBackup: () => void;
|
||||
projects: Project[]; projectId: string; projectMsg: string;
|
||||
selectProject: (id: string) => void; refreshProjects: () => void;
|
||||
createProject: (name: string, repo: string) => Promise<void>;
|
||||
githubRepos: Array<{ clone_url: string; repo_full_name: string }>;
|
||||
workers: Worker[] | null; workersSummary: string; workersError: string; refreshWorkers: () => void;
|
||||
members: Member[]; memberMsg: string; memberError: string; refreshMembers: () => void; inviteMember: (raw: string, role: string) => Promise<void>; changeMemberRole: (uid: string, role: string) => Promise<void>;
|
||||
members: Member[]; memberMsg: string; memberError: string; refreshMembers: () => void; inviteMember: (raw: string, role: string) => Promise<void>; changeMemberRole: (uid: string, role: string) => Promise<void>; removeMember: (uid: string) => void;
|
||||
skills: SkillItem[]; apps: SkillItem[]; mcpServers: McpServer[]; skillMsg: string; skillError: string; mcpMsg: string; mcpOauthMsg: string; mcpOauthBox: string;
|
||||
refreshSkills: () => void; reloadMcp: () => void; mcpOauth: (name: string) => Promise<void>;
|
||||
envs: Env[]; envCount: string; envMsg: string; envRepoLine: string; envRepoBar: boolean; envRepoMsg: string; envSelect: string; setEnvSelect: (v: string) => void;
|
||||
@@ -35,7 +35,7 @@ export interface ConsoleCtx {
|
||||
followups: Followup[];
|
||||
approvals: Approval[]; approvalMsg: string; refreshApprovals: () => void; decideApproval: (id: string, decision: string) => void;
|
||||
peerMessages: PeerMessage[]; peerMsg: string; postPeer: (mode: string, target: string, text: string) => Promise<void>;
|
||||
automations: Automation[]; autoMsg: string; refreshAutomations: () => void; createAutomation: (name: string, prompt: string, interval: string, repo: string) => Promise<void>; updateAutomation: (a: Automation, name: string, prompt: string, interval: string, repo: string) => Promise<boolean>; cancelAutomation: (id: string) => void;
|
||||
automations: Automation[]; autoMsg: string; autoDetail: AutomationDetail | null; autoDetailMsg: string; refreshAutomations: () => void; createAutomation: (name: string, prompt: string, interval: string, repo: string) => Promise<void>; updateAutomation: (a: Automation, name: string, prompt: string, interval: string, repo: string) => Promise<boolean>; cancelAutomation: (id: string) => void; loadAutomationDetail: (id: string) => void; closeAutomationDetail: () => void;
|
||||
triage: Triage | null; triageError: string; refreshTriage: () => void;
|
||||
threadText: string; threadGateNote: string; threadMsg: string; threadList: ThreadInfo[]; threadAction: (kind: string, name: string) => void;
|
||||
dispatch: (body: DispatchBody, atts: Array<{ path: string; file: File }>) => Promise<string>; dispatchMsg: string;
|
||||
|
||||
@@ -253,6 +253,17 @@ export function useCatalog() {
|
||||
} catch (e: any) { setMemberMsg(memberErr(e)); }
|
||||
}, []);
|
||||
|
||||
const removeMember = useCallback(async (uid: string) => {
|
||||
const pid = projectIdRef.current;
|
||||
if (!pid) return;
|
||||
if (!window.confirm("确定移除该成员吗?移除后其项目权限立即失效(唯一 owner 不可移除,服务端会拒绝)。")) return;
|
||||
try {
|
||||
await api(paths.projectMember(pid, uid), "DELETE");
|
||||
setMemberMsg("成员已移除");
|
||||
await refreshMembers(pid);
|
||||
} catch (e: any) { setMemberMsg(memberErr(e)); }
|
||||
}, [refreshMembers]);
|
||||
|
||||
const reloadMcp = useCallback(async () => {
|
||||
const pid = projectIdRef.current;
|
||||
if (!pid) { setMcpMsg("先选项目"); return; }
|
||||
@@ -289,6 +300,24 @@ export function useCatalog() {
|
||||
} catch (e: any) { setAdminResult(adminErr(e)); }
|
||||
}, [adminUserId]);
|
||||
|
||||
const adminSetQuota = useCallback(async (uid: string, usdText: string) => {
|
||||
const target = uid.trim();
|
||||
if (!target) { setAdminResult("请填要设置配额的用户 id"); return; }
|
||||
// POST /admin/quotas {user_id, quota_usd_micros}: integer micros or
|
||||
// null for unlimited, >= 0. The console takes dollars and converts.
|
||||
let micros: number | null;
|
||||
if (!usdText.trim()) micros = null;
|
||||
else {
|
||||
const usd = Number(usdText.trim());
|
||||
if (!Number.isFinite(usd) || usd < 0) { setAdminResult("配额金额无效:请输入 >= 0 的美元数,留空表示不限额"); return; }
|
||||
micros = Math.round(usd * 1000000);
|
||||
}
|
||||
try {
|
||||
const d = await api<{ used_usd_micros?: number; quota_usd_micros?: number | null; remaining_usd_micros?: number | null; month?: string }>(paths.adminQuotas, "POST", { user_id: target, quota_usd_micros: micros });
|
||||
setAdminResult("配额已设置:用户 " + target.slice(0, 8) + ":本月用量 " + fmtUsd(d.used_usd_micros) + " / 配额 " + fmtUsd(d.quota_usd_micros) + (d.remaining_usd_micros !== null && d.remaining_usd_micros !== undefined ? "(剩余 " + fmtUsd(d.remaining_usd_micros) + ")" : "") + " | 月份 " + (d.month || ""));
|
||||
} catch (e: any) { setAdminResult(adminErr(e)); }
|
||||
}, []);
|
||||
|
||||
const adminSweep = useCallback(async () => {
|
||||
if (!window.confirm("确定手动触发 sweep 吗?超阈值的陈旧 running 任务会被标为 interrupted。")) return;
|
||||
try {
|
||||
@@ -314,6 +343,6 @@ export function useCatalog() {
|
||||
models, modelCustomVisible,
|
||||
refreshHealth, loadQuota, probeAdmin, refreshProjects, loadGithubRepos, loadRelayModels, refreshWorkers,
|
||||
refreshEnvs, refreshMembers, refreshSkills, saveKeyInput, clearKeyInput, createProject, createEnv,
|
||||
archiveEnv, saveEnvRepo, inviteMember, changeMemberRole, reloadMcp, mcpOauth, adminQuota, adminSweep, adminBackup,
|
||||
archiveEnv, saveEnvRepo, inviteMember, changeMemberRole, removeMember, reloadMcp, mcpOauth, adminQuota, adminSetQuota, adminSweep, adminBackup,
|
||||
};
|
||||
}
|
||||
@@ -108,14 +108,14 @@ export function useConsoleStore(): ConsoleCtx {
|
||||
healthText: catalog.healthText, refreshHealth: catalog.refreshHealth,
|
||||
adminGate: catalog.adminGate, adminBody: catalog.adminBody, adminResult: catalog.adminResult,
|
||||
adminUserId: catalog.adminUserId, setAdminUserId: catalog.setAdminUserId,
|
||||
adminQuota: catalog.adminQuota, adminSweep: catalog.adminSweep, adminBackup: catalog.adminBackup,
|
||||
adminQuota: catalog.adminQuota, adminSetQuota: catalog.adminSetQuota, adminSweep: catalog.adminSweep, adminBackup: catalog.adminBackup,
|
||||
projects: catalog.projects, projectId: catalog.projectId, projectMsg: catalog.projectMsg,
|
||||
selectProject, refreshProjects: catalog.refreshProjects,
|
||||
createProject: catalog.createProject,
|
||||
githubRepos: catalog.githubRepos,
|
||||
workers: catalog.workers, workersSummary: catalog.workersSummary, workersError: catalog.workersError, refreshWorkers: catalog.refreshWorkers,
|
||||
members: catalog.members, memberMsg: catalog.memberMsg, memberError: catalog.memberError,
|
||||
refreshMembers: () => void catalog.refreshMembers(), inviteMember: catalog.inviteMember, changeMemberRole: catalog.changeMemberRole,
|
||||
refreshMembers: () => void catalog.refreshMembers(), inviteMember: catalog.inviteMember, changeMemberRole: catalog.changeMemberRole, removeMember: catalog.removeMember,
|
||||
skills: catalog.skills, apps: catalog.apps, mcpServers: catalog.mcpServers, skillMsg: catalog.skillMsg, skillError: catalog.skillError,
|
||||
mcpMsg: catalog.mcpMsg, mcpOauthMsg: catalog.mcpOauthMsg, mcpOauthBox: catalog.mcpOauthBox,
|
||||
refreshSkills: () => void catalog.refreshSkills(), reloadMcp: catalog.reloadMcp, mcpOauth: catalog.mcpOauth,
|
||||
@@ -144,7 +144,7 @@ export function useConsoleStore(): ConsoleCtx {
|
||||
followups: detail.followups,
|
||||
approvals: detail.approvals, approvalMsg: detail.approvalMsg, refreshApprovals: detail.refreshApprovals, decideApproval: detail.decideApproval,
|
||||
peerMessages: detail.peerMessages, peerMsg: detail.peerMsg, postPeer: detail.postPeer,
|
||||
automations: tools.automations, autoMsg: tools.autoMsg, refreshAutomations: () => void tools.refreshAutomations(), createAutomation: tools.createAutomation, updateAutomation: tools.updateAutomation, cancelAutomation: tools.cancelAutomation,
|
||||
automations: tools.automations, autoMsg: tools.autoMsg, autoDetail: tools.autoDetail, autoDetailMsg: tools.autoDetailMsg, refreshAutomations: () => void tools.refreshAutomations(), createAutomation: tools.createAutomation, updateAutomation: tools.updateAutomation, cancelAutomation: tools.cancelAutomation, loadAutomationDetail: tools.loadAutomationDetail, closeAutomationDetail: tools.closeAutomationDetail,
|
||||
triage: tools.triage, triageError: tools.triageError, refreshTriage: () => void tools.refreshTriage(),
|
||||
threadText: detail.threadText, threadGateNote: detail.threadGateNote, threadMsg: detail.threadMsg, threadList: detail.threadList, threadAction: detail.threadAction,
|
||||
dispatch: async (body, atts) => {
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
import { useCallback, useState } from "react";
|
||||
import { api } from "../api/client";
|
||||
import { paths } from "../api/paths";
|
||||
import type { Automation, Triage } from "../api/types";
|
||||
import type { Automation, AutomationDetail, Triage } from "../api/types";
|
||||
|
||||
// Project-scoped tools shown in the centre for their own views:
|
||||
// automations (create/list/cancel) and the triage inbox (4 groups).
|
||||
export function useProjectTools(getProjectId: () => string) {
|
||||
const [automations, setAutomations] = useState<Automation[]>([]);
|
||||
const [autoMsg, setAutoMsg] = useState("");
|
||||
const [autoDetail, setAutoDetail] = useState<AutomationDetail | null>(null);
|
||||
const [autoDetailMsg, setAutoDetailMsg] = useState("");
|
||||
const [triage, setTriage] = useState<Triage | null>(null);
|
||||
const [triageError, setTriageError] = useState("");
|
||||
|
||||
@@ -66,10 +68,26 @@ export function useProjectTools(getProjectId: () => string) {
|
||||
const pid = getProjectId();
|
||||
if (!pid) return;
|
||||
void api(paths.projectAutomationCancel(pid, id), "POST", {})
|
||||
.then(() => { setAutoMsg("自动化已取消"); void refreshAutomations(pid); })
|
||||
.then(() => { setAutoMsg("自动化已取消"); setAutoDetail((prev) => (prev && prev.id === id ? null : prev)); void refreshAutomations(pid); })
|
||||
.catch((e: any) => setAutoMsg("取消失败:" + e.message));
|
||||
}, [getProjectId, refreshAutomations]);
|
||||
|
||||
// GET /projects/{pid}/automations/{aid}: full definition + recent
|
||||
// runs (+ runs_note, heartbeat_binding). Verified against origin/main
|
||||
// cloud/server.py _automation_get / cloud/automations.py get_automation.
|
||||
const loadAutomationDetail = useCallback(async (id: string) => {
|
||||
const pid = getProjectId();
|
||||
if (!pid) { setAutoDetailMsg("先选项目"); return; }
|
||||
try {
|
||||
const d = await api<AutomationDetail>(paths.projectAutomation(pid, id));
|
||||
setAutoDetail(d);
|
||||
setAutoDetailMsg("");
|
||||
} catch (e: any) { setAutoDetailMsg("详情加载失败:" + e.message); }
|
||||
}, [getProjectId]);
|
||||
|
||||
const closeAutomationDetail = useCallback(() => { setAutoDetail(null); setAutoDetailMsg(""); }, []);
|
||||
|
||||
|
||||
const refreshTriage = useCallback(async (pid?: string) => {
|
||||
const id = pid !== undefined ? pid : getProjectId();
|
||||
if (!id) { setTriage(null); return; }
|
||||
@@ -80,5 +98,5 @@ export function useProjectTools(getProjectId: () => string) {
|
||||
} catch (e: any) { setTriageError("分诊加载失败:" + e.message); }
|
||||
}, [getProjectId]);
|
||||
|
||||
return { automations, autoMsg, triage, triageError, refreshAutomations, createAutomation, updateAutomation, cancelAutomation, refreshTriage };
|
||||
return { automations, autoMsg, autoDetail, autoDetailMsg, triage, triageError, refreshAutomations, createAutomation, updateAutomation, cancelAutomation, loadAutomationDetail, closeAutomationDetail, refreshTriage };
|
||||
}
|
||||
@@ -1,8 +1,11 @@
|
||||
import { useState } from "react";
|
||||
import { useConsole } from "../context";
|
||||
import { Card, Bar } from "../components/ui";
|
||||
|
||||
export default function AdminSection() {
|
||||
const ctx = useConsole();
|
||||
const [setUser, setSetUser] = useState("");
|
||||
const [setUsd, setSetUsd] = useState("");
|
||||
return (
|
||||
<Card id="adminCard" title={<>管理区 <span className="muted">(仅全局管理员)</span></>}>
|
||||
<div className="muted">{ctx.adminGate}</div>
|
||||
@@ -12,6 +15,11 @@ export default function AdminSection() {
|
||||
<input type="text" placeholder="要查配额的用户 id" value={ctx.adminUserId} onChange={(e) => ctx.setAdminUserId(e.target.value)} />
|
||||
<button onClick={ctx.adminQuota}>查该用户配额</button>
|
||||
</Bar>
|
||||
<Bar>
|
||||
<input type="text" placeholder="要设置配额的用户 id" value={setUser} onChange={(e) => setSetUser(e.target.value)} />
|
||||
<input type="text" placeholder="美元配额(留空 = 不限额)" value={setUsd} onChange={(e) => setSetUsd(e.target.value)} />
|
||||
<button className="primary" onClick={() => ctx.adminSetQuota(setUser, setUsd)}>设置配额</button>
|
||||
</Bar>
|
||||
<Bar>
|
||||
<button onClick={ctx.adminSweep}>手动触发 sweep(清理陈旧 running 任务)</button>
|
||||
<button onClick={ctx.adminBackup}>手动触发备份</button>
|
||||
|
||||
@@ -2,7 +2,6 @@ import { useRef, useState } from "react";
|
||||
import { useConsole } from "../context";
|
||||
import { Card, Bar } from "../components/ui";
|
||||
import { gatherAttachments } from "../api/client";
|
||||
import { chvAttachmentBlocked, chvAvailability, chvAvailabilityText, isChvExecutor } from "../api/chv";
|
||||
import { EFFORTS } from "../api/types";
|
||||
|
||||
export default function DispatchSection() {
|
||||
@@ -23,15 +22,7 @@ export default function DispatchSection() {
|
||||
const filesRef = useRef<HTMLInputElement>(null);
|
||||
const dirRef = useRef<HTMLInputElement>(null);
|
||||
const imgRef = useRef<HTMLInputElement>(null);
|
||||
const [hasAttachments, setHasAttachments] = useState(false);
|
||||
const [msg, setMsg] = useState("");
|
||||
const chvAvail = chvAvailability(ctx.workers, ctx.workersSummary, ctx.workersError);
|
||||
const isChv = isChvExecutor(executor);
|
||||
function refreshAttachments() {
|
||||
setHasAttachments(
|
||||
!!(filesRef.current?.files?.length || dirRef.current?.files?.length || imgRef.current?.files?.length),
|
||||
);
|
||||
}
|
||||
|
||||
async function submit() {
|
||||
setMsg("");
|
||||
@@ -69,7 +60,6 @@ export default function DispatchSection() {
|
||||
if (goalMode || goalInvincible || planMode) { setMsg("协调员任务不能与 goal/计划模式同用:协调员任务本身不跑模型回合,请取消 goal/计划勾选,或清空协调员 JSON"); return; }
|
||||
}
|
||||
const atts = gatherAttachments(filesRef.current?.files || null, dirRef.current?.files || null, imgRef.current?.files || null);
|
||||
if (chvAttachmentBlocked(executor, atts.length > 0)) { setMsg("cloud-hypervisor 任务不支持附件:请先清空附件再提交(后端对带附件的 CHV 任务会 400)"); return; }
|
||||
try {
|
||||
const result = await ctx.dispatch(body, atts);
|
||||
setMsg(result);
|
||||
@@ -77,7 +67,6 @@ export default function DispatchSection() {
|
||||
if (filesRef.current) filesRef.current.value = "";
|
||||
if (dirRef.current) dirRef.current.value = "";
|
||||
if (imgRef.current) imgRef.current.value = "";
|
||||
setHasAttachments(false);
|
||||
} catch (e: any) { setMsg("提交失败:" + (e?.message || String(e))); }
|
||||
}
|
||||
|
||||
@@ -89,19 +78,12 @@ export default function DispatchSection() {
|
||||
{ctx.envs.map((e) => <option key={e.id} value={e.id} disabled={e.status !== "ready"}>{e.name}({e.status})</option>)}
|
||||
</select>
|
||||
<select aria-label="执行器" value={executor} onChange={(e) => setExecutor(e.target.value)}>
|
||||
<option value="docker">docker(默认,多用户下隔离最强)</option><option value="local">local</option><option value="cloud">cloud</option><option value="cloud-hypervisor">cloud-hypervisor(实验,默认关闭)</option>
|
||||
<option value="docker">docker(默认,多用户下隔离最强)</option><option value="local">local</option><option value="cloud">cloud</option>
|
||||
</select>
|
||||
<label><input type="checkbox" checked={goalMode} onChange={(e) => setGoalMode(e.target.checked)} /> goal 模式</label>
|
||||
<label><input type="checkbox" checked={goalInvincible} onChange={(e) => setGoalInvincible(e.target.checked)} /> goal 模式无敌版</label>
|
||||
<label><input type="checkbox" checked={planMode} onChange={(e) => setPlanMode(e.target.checked)} /> 计划模式(只读)</label>
|
||||
</Bar>
|
||||
{isChv ? <div className="muted">实验执行器,默认关闭:需后端已开启 CHV worker 且宿主有 KVM,否则任务无人认领、一直排队;带附件时建任务即被后端拒绝(错误文案原样显示在下方)。前端只传 executor 值,不绕过后端白名单;默认仍是 docker。</div> : null}
|
||||
{isChv ? (
|
||||
<div className="muted" aria-live="polite">
|
||||
{chvAvailabilityText(chvAvail)}
|
||||
{" "}<button type="button" onClick={ctx.refreshWorkers}>刷新 CHV 可用性</button>
|
||||
</div>
|
||||
) : null}
|
||||
<textarea className="prompt stacked" placeholder="任务 prompt(必填)" value={prompt} onChange={(e) => setPrompt(e.target.value)} />
|
||||
<textarea className="coord stacked" placeholder="协调员子任务 JSON(可选),留空为普通任务" value={coordParts} onChange={(e) => setCoordParts(e.target.value)} />
|
||||
<div className="muted">协调员格式:[{"{"}"name":"后端","prompt":"做后端"{"}"},{"{"}"name":"前端","prompt":"做前端"{"}"}];提交后 worker 并行建子任务、等齐、合流汇总。不能与 goal/计划模式同用。协调员与 goal/计划互斥:勾了 goal 或计划模式时不要填协调员 JSON,提交前会校验并拦截。子任务费用记父任务创建人账;父任务创建时按分工数预留配额,余额不足时整个父任务被挡下。</div>
|
||||
@@ -135,14 +117,13 @@ export default function DispatchSection() {
|
||||
{goalInvincible ? <div className="muted">goal 模式无敌版:goal 模式 + 任务不限时(去掉 worker/docker 总超时;单轮 600 秒护栏还在,不限的是任务总时长与总轮次)。服务端默认的自动续跑与断点镜像同时生效。</div> : null}
|
||||
<Bar>
|
||||
<span className="muted">附件(放进任务工作区,图片与普通文件不区分):</span>
|
||||
<input ref={filesRef} type="file" multiple title="文件" aria-label="附件文件" disabled={isChv} onChange={refreshAttachments} />
|
||||
<input ref={dirRef} type="file" multiple title="文件夹" aria-label="附件文件夹" disabled={isChv} onChange={refreshAttachments} {...({ webkitdirectory: "" } as any)} />
|
||||
<input ref={imgRef} type="file" accept="image/*" multiple title="图片" aria-label="附件图片" disabled={isChv} onChange={refreshAttachments} />
|
||||
<input ref={filesRef} type="file" multiple title="文件" />
|
||||
<input ref={dirRef} type="file" multiple title="文件夹" {...({ webkitdirectory: "" } as any)} />
|
||||
<input ref={imgRef} type="file" accept="image/*" multiple title="图片" />
|
||||
</Bar>
|
||||
{isChv ? <div className="muted">cloud-hypervisor 任务不支持附件:附件输入已禁用(后端对带附件的 CHV 任务会 400,错误文案原样显示在下方;切换回其他执行器可恢复附件输入)。{hasAttachments ? <>当前仍有已选附件,请先清空再提交。 <button type="button" onClick={() => { if (filesRef.current) filesRef.current.value = ""; if (dirRef.current) dirRef.current.value = ""; if (imgRef.current) imgRef.current.value = ""; setHasAttachments(false); }}>清空附件</button></> : ""}</div> : null}
|
||||
<Bar>
|
||||
<button className="seal" onClick={submit}>提交任务</button>
|
||||
<span className="muted" role="status" aria-live="polite">{msg || ctx.dispatchMsg}</span>
|
||||
<span className="muted">{msg || ctx.dispatchMsg}</span>
|
||||
</Bar>
|
||||
</Card>
|
||||
);
|
||||
|
||||
@@ -2,14 +2,12 @@ import { useState } from "react";
|
||||
import { useConsole } from "../context";
|
||||
import { Card, Bar, DataListRaw } from "../components/ui";
|
||||
import { shortId } from "../api/client";
|
||||
import { chvAvailability, chvAvailabilityText } from "../api/chv";
|
||||
|
||||
export default function EnvSection() {
|
||||
const ctx = useConsole();
|
||||
const [name, setName] = useState("");
|
||||
const [branch, setBranch] = useState("");
|
||||
const [executor, setExecutor] = useState("");
|
||||
const chvAvail = chvAvailability(ctx.workers, ctx.workersSummary, ctx.workersError);
|
||||
return (
|
||||
<Card id="nav-env" title={<>环境 <span className="muted">(选、建、管:当前项目的基准检出)</span></>}>
|
||||
<Bar>
|
||||
@@ -33,11 +31,10 @@ export default function EnvSection() {
|
||||
<input type="text" placeholder="新环境名" value={name} onChange={(e) => setName(e.target.value)} />
|
||||
<input type="text" placeholder="分支(可选,空=默认分支)" value={branch} onChange={(e) => setBranch(e.target.value)} />
|
||||
<select aria-label="新环境执行器" value={executor} onChange={(e) => setExecutor(e.target.value)}>
|
||||
<option value="">执行器:不限</option><option value="local">local</option><option value="docker">docker</option><option value="cloud-hypervisor">cloud-hypervisor(实验,默认关闭)</option>
|
||||
<option value="">执行器:不限</option><option value="local">local</option><option value="docker">docker</option>
|
||||
</select>
|
||||
<button onClick={async () => { await ctx.createEnv(name, branch, executor); setName(""); setBranch(""); }}>新建环境</button>
|
||||
</Bar>
|
||||
{executor === "cloud-hypervisor" ? <div className="muted" aria-live="polite">实验执行器,默认关闭:环境创建白名单同样允许 cloud-hypervisor(后端环境接口 allowlist:local / docker / cloud-hypervisor,不含 cloud);仍需后端有 KVM 宿主上的 CHV worker 才会认领,否则任务排队。前端只传值,不绕过后端白名单。 {chvAvailabilityText(chvAvail)}</div> : null}
|
||||
<span className="muted">{ctx.envMsg}</span>
|
||||
</Card>
|
||||
);
|
||||
|
||||
@@ -37,6 +37,7 @@ export default function MembersPanel() {
|
||||
<select aria-label="成员角色" value={m.role || "viewer"} onChange={(e) => void ctx.changeMemberRole(m.user_id, e.target.value)}>
|
||||
{ROLES.map((r) => <option key={r} value={r}>{r}</option>)}
|
||||
</select>
|
||||
<button onClick={() => void ctx.removeMember(m.user_id)}>移除</button>
|
||||
</li>
|
||||
);
|
||||
})}
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
import { useState } from "react";
|
||||
import { useConsole } from "../context";
|
||||
import { Card, Bar } from "../components/ui";
|
||||
import { shortId } from "../api/client";
|
||||
import MembersPanel from "./MembersPanel";
|
||||
|
||||
// Left sidebar: the one project selector for the whole console.
|
||||
// Lists every project (name + short id + my role), click selects it;
|
||||
// below the list is the create-project form (same fields as the old
|
||||
// workbench ProjectsSection: name, GitHub repo select, repo_url) and
|
||||
// the selected project's members. The workbench no longer carries any
|
||||
// project-module UI; it only consumes ctx.projectId.
|
||||
export default function ProjectNav() {
|
||||
const ctx = useConsole();
|
||||
const [name, setName] = useState("");
|
||||
const [repoSelect, setRepoSelect] = useState("");
|
||||
const [repo, setRepo] = useState("");
|
||||
return (
|
||||
<div className="project-nav">
|
||||
<Card title="项目">
|
||||
<Bar><button onClick={ctx.refreshProjects}>刷新项目</button></Bar>
|
||||
{ctx.projects.length === 0 ? <div className="muted">暂无项目</div> : (
|
||||
<ul className="project-nav-list">
|
||||
{ctx.projects.map((p) => (
|
||||
<li key={p.id} className={"project-nav-item" + (p.id === ctx.projectId ? " selected" : "")} onClick={() => ctx.selectProject(p.id)} title={p.id}>
|
||||
<span className="project-nav-name">{p.name}</span>
|
||||
<span className="project-nav-meta">{shortId(p.id)}{p.my_role ? " · " + p.my_role : ""}</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
<h3>创建项目</h3>
|
||||
<Bar>
|
||||
<input type="text" placeholder="新项目名" value={name} onChange={(e) => setName(e.target.value)} />
|
||||
</Bar>
|
||||
<Bar>
|
||||
<select aria-label="从 GitHub 仓库选" value={repoSelect} onChange={(e) => { setRepoSelect(e.target.value); if (e.target.value) setRepo(e.target.value); }}>
|
||||
<option value="">(从 GitHub App 安装仓库里选,或手动输入)</option>
|
||||
{ctx.githubRepos.map((r) => <option key={r.clone_url} value={r.clone_url}>{r.repo_full_name}</option>)}
|
||||
</select>
|
||||
</Bar>
|
||||
<Bar>
|
||||
<input type="text" placeholder="repo_url(可选)" value={repo} onChange={(e) => setRepo(e.target.value)} />
|
||||
<button className="primary" onClick={async () => { await ctx.createProject(name, repo); setName(""); setRepo(""); setRepoSelect(""); }}>创建项目</button>
|
||||
</Bar>
|
||||
<span className="muted">{ctx.projectMsg}</span>
|
||||
</Card>
|
||||
<MembersPanel />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Loaded 100 of 424 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user