fix(chv-guest): 客机镜像构建缺口补齐(B-02/B-03/B-04/B-05/B-06/B-07/B-08/B-09/B-10/B-13/B-15)
- B-10(实测):virt-customize chroot 里 sshd -t 必败,双因: 无 host key(exit 1)+ 缺 /run/sshd(exit 255);修为 mkdir -p /run/sshd + ssh-keygen -A 临时生成,断言通过后 立删临时 key,镜像仍不带 host key(已在 WSL 真机预演验证) - B-08:Playwright 装在第二次定制,须重写 pip freeze 否则 BUILDINFO 里无 playwright;Chromium 以 ms-playwright 缓存 目录名记录进 guest-versions-chromium.txt - B-09:codex --version 须含 CODEX_VERSION;REPO_COMMIT 须 等于 HEAD 短 hash;工作树脏即拒绝构建 - B-02/B-13:BUILDINFO 生成抽成 buildinfo.py(可单元测试), 补 disk.raw/vmlinux sha256+大小、ubuntu 镜像 sha、cmdline、 playwright 开关、image_sha256 字段 - generate_lockfile.py 改写:版本唯一来源改为 BUILDINFO.versions(客机读回),codex 三件套取 codex_version、/app/host.py 取 repo_commit、chromium 取 versions.chromium;不再查构建宿主 dpkg/pip(旧方法不成立) - check_lockfile.py 补强:条目格式、contents 全覆盖、冻结 清单双向比对 - B-03/B-04 输入格式校验、B-05 curl 重试、B-06 哈希不符 删文件重下、B-07 磁盘预检、B-15 set -euo pipefail - 同步改 test_chv_to10.py:generate 不再查宿主,改断言客机 版本来源 + 不回落宿主
This commit is contained in:
1 parent
8c8d62f70d
commit
7f18c2d13b
5 files changed
+371
-95
No files matched your search
+66
-31
@@ -19,6 +19,10 @@
|
||||
# sshd 加固由构建写入 /etc/ssh/sshd_config.d/99-chv.conf
|
||||
# 并经 sshd -t + grep 断言(README 的“只允许 key 登录”
|
||||
# 从此是构建断言,不再只是文档声称)。
|
||||
# B-10:virt-customize 的 chroot 里 sshd -t 须先 ssh-keygen -A
|
||||
# 临时生成 host key 并 mkdir -p /run/sshd,否则分别报无 host
|
||||
# key / 缺 privilege separation 目录而失败;断言通过后立即删
|
||||
# 临时 key,镜像仍不带 host key(per-VM 预置才生效)。
|
||||
#
|
||||
# 客机 host key 硬性要求(见 README §5,补缺口轨新增):
|
||||
# 客机必须装 cloud-init 且支持 NoCloud seed,首启先应用
|
||||
@@ -26,7 +30,7 @@
|
||||
# 执行器的 per-VM 预置钉死即失效。本脚本在镜像里写
|
||||
# /etc/cloud/cloud.cfg.d/99-chv-hostkey.cfg(ssh_deletekeys:
|
||||
# false、ssh_genkeytypes: []),把“不重生成”钉进镜像。
|
||||
set -eu
|
||||
set -euo pipefail
|
||||
|
||||
DRY_RUN=0
|
||||
if [ "${1:-}" = "--dry-run" ]; then
|
||||
@@ -40,6 +44,35 @@ OUT="${1:-out}"
|
||||
: "${REPO_COMMIT:?REPO_COMMIT 未设:构建时仓库 commit 短 hash}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_DIR="${REPO_DIR:-$(cd "$SCRIPT_DIR/.." && pwd)}"
|
||||
# B-03/B-04:输入格式校验(干跑与全量都做,报错点提前,
|
||||
# 不等到 curl 404 或 sha256sum -c 才失败)。
|
||||
if ! printf '%s' "$UBUNTU_VERSION" | grep -Eq '^[0-9]+\.[0-9]+$'; then
|
||||
echo "UBUNTU_VERSION 格式须为 NN.NN:$UBUNTU_VERSION" >&2; exit 1
|
||||
fi
|
||||
if ! printf '%s' "$REPO_COMMIT" | grep -Eq '^[0-9a-f]{7,40}$'; then
|
||||
echo "REPO_COMMIT 须为 7-40 位 hex:$REPO_COMMIT" >&2; exit 1
|
||||
fi
|
||||
if ! printf '%s' "$CODEX_VERSION" | grep -Eq '^[0-9]+\.[0-9]+\.[0-9]+'; then
|
||||
echo "CODEX_VERSION 格式异常:$CODEX_VERSION" >&2; exit 1
|
||||
fi
|
||||
if [ -n "${UBUNTU_IMAGE_SHA256:-}" ] && ! printf '%s' "$UBUNTU_IMAGE_SHA256" | grep -Eq '^[0-9a-f]{64}$'; then
|
||||
echo "UBUNTU_IMAGE_SHA256 须为 64 位 hex" >&2; exit 1
|
||||
fi
|
||||
# B-09:版本声称须核对,不只进 tag。codex 二进制真实版本
|
||||
# 须包含 CODEX_VERSION;REPO_COMMIT 须等于仓库 HEAD 短 hash;
|
||||
# 工作树脏即拒绝构建(tag 会谎称干净 commit)。
|
||||
CODEX_ACTUAL_VERSION="$("$CODEX_BIN_DIR/codex" --version 2>/dev/null || true)"
|
||||
case "$CODEX_ACTUAL_VERSION" in
|
||||
*"$CODEX_VERSION"*) ;;
|
||||
*) echo "codex 二进制版本与 CODEX_VERSION 不符:got [$CODEX_ACTUAL_VERSION] want 含 [$CODEX_VERSION]" >&2; exit 1 ;;
|
||||
esac
|
||||
HEAD_SHORT="$(git -C "$REPO_DIR" rev-parse --short HEAD)"
|
||||
if [ "$HEAD_SHORT" != "$REPO_COMMIT" ]; then
|
||||
echo "REPO_COMMIT 与仓库 HEAD 不符:$REPO_COMMIT != $HEAD_SHORT" >&2; exit 1
|
||||
fi
|
||||
if [ -n "$(git -C "$REPO_DIR" status --porcelain)" ]; then
|
||||
echo "仓库工作树有未提交修改,拒绝构建(tag 会谎称干净 commit)" >&2; exit 1
|
||||
fi
|
||||
TAG="chv-guest-${UBUNTU_VERSION}-${CODEX_VERSION}-${REPO_COMMIT}"
|
||||
UBUNTU_IMAGE_URL="${UBUNTU_IMAGE_URL:-https://cloud-images.ubuntu.com/releases/${UBUNTU_VERSION}/release/ubuntu-${UBUNTU_VERSION}-server-cloudimg-amd64.img}"
|
||||
|
||||
@@ -81,6 +114,13 @@ if [ "$DRY_RUN" = "0" ]; then
|
||||
fi
|
||||
|
||||
run mkdir -p "$OUT"
|
||||
# B-07:磁盘空间预检(仅全量模式;干跑不建目录,无法 df)。
|
||||
if [ "$DRY_RUN" = "0" ]; then
|
||||
avail_kb="$(df -k --output=avail "$OUT" | tail -1 | tr -d ' ')"
|
||||
if [ "${avail_kb:-0}" -lt 20971520 ]; then
|
||||
echo "磁盘不足:$OUT 所在盘可用 ${avail_kb}KB,需 ≥20GB" >&2; exit 1
|
||||
fi
|
||||
fi
|
||||
IMG="$OUT/ubuntu-${UBUNTU_VERSION}-server-cloudimg-amd64.img"
|
||||
# feat-chv-secfix(M4):云镜像 SHA256 改为必填(干跑只验
|
||||
# “已设”,不验文件;全量模式下载后验哈希)。未设即失败,
|
||||
@@ -90,15 +130,22 @@ if [ -z "${UBUNTU_IMAGE_SHA256:-}" ]; then
|
||||
exit 1
|
||||
fi
|
||||
if [ "$DRY_RUN" = "1" ]; then
|
||||
echo "DRY-RUN: curl -fL $UBUNTU_IMAGE_URL -o $IMG"
|
||||
echo "DRY-RUN: curl -fL --retry 3 --retry-all-errors --connect-timeout 30 $UBUNTU_IMAGE_URL -o $IMG"
|
||||
echo "DRY-RUN: [ -s $IMG ] || exit 1"
|
||||
echo "DRY-RUN: sha256sum -c (UBUNTU_IMAGE_SHA256)"
|
||||
else
|
||||
if [ ! -s "$IMG" ]; then
|
||||
curl -fL "$UBUNTU_IMAGE_URL" -o "$IMG"
|
||||
curl -fL --retry 3 --retry-all-errors --connect-timeout 30 "$UBUNTU_IMAGE_URL" -o "$IMG"
|
||||
fi
|
||||
[ -s "$IMG" ] || { echo "下载的云镜像为空:$IMG" >&2; exit 1; }
|
||||
# B-06:哈希不符时删文件重下一次,不直接退出让操作员手动清理。
|
||||
if ! echo "$UBUNTU_IMAGE_SHA256 $IMG" | sha256sum -c -; then
|
||||
echo "云镜像哈希不符,删除后重下一次:$IMG" >&2
|
||||
rm -f "$IMG"
|
||||
curl -fL --retry 3 --retry-all-errors --connect-timeout 30 "$UBUNTU_IMAGE_URL" -o "$IMG"
|
||||
[ -s "$IMG" ] || { echo "重下的云镜像为空:$IMG" >&2; exit 1; }
|
||||
echo "$UBUNTU_IMAGE_SHA256 $IMG" | sha256sum -c -
|
||||
fi
|
||||
fi
|
||||
run qemu-img convert -f qcow2 -O raw "$IMG" "$OUT/disk.raw"
|
||||
# 清单软件(照抄 cloud/docker/Dockerfile 的 apt/pip 清单,
|
||||
@@ -115,11 +162,18 @@ run virt-customize -a "$OUT/disk.raw" \
|
||||
--copy-in "$REPO_DIR/host.py:/app/" \
|
||||
--copy-in "$REPO_DIR/cloud:/app/" \
|
||||
--run-command 'printf "ssh_deletekeys: false\nssh_genkeytypes: []\n" > /etc/cloud/cloud.cfg.d/99-chv-hostkey.cfg; systemctl enable ssh || true' \
|
||||
--run-command 'printf "PasswordAuthentication no\nPermitRootLogin prohibit-password\nChallengeResponseAuthentication no\nKbdInteractiveAuthentication no\nPubkeyAuthentication yes\n" > /etc/ssh/sshd_config.d/99-chv.conf; sshd -t && grep -q "^PasswordAuthentication no" /etc/ssh/sshd_config.d/99-chv.conf && grep -q "^PermitRootLogin prohibit-password" /etc/ssh/sshd_config.d/99-chv.conf' \
|
||||
--run-command 'printf "PasswordAuthentication no\nPermitRootLogin prohibit-password\nChallengeResponseAuthentication no\nKbdInteractiveAuthentication no\nPubkeyAuthentication yes\n" > /etc/ssh/sshd_config.d/99-chv.conf && mkdir -p /run/sshd && ssh-keygen -A && sshd -t && grep -q "^PasswordAuthentication no" /etc/ssh/sshd_config.d/99-chv.conf && grep -q "^PermitRootLogin prohibit-password" /etc/ssh/sshd_config.d/99-chv.conf && rm -f /etc/ssh/ssh_host_* && test -z "$(ls /etc/ssh/ssh_host_* 2>/dev/null)"' \
|
||||
--run-command 'dpkg-query -W -f "${Package} ${Version}\n" > /app/guest-versions-dpkg.txt; pip3 freeze > /app/guest-versions-pip.txt || true'
|
||||
if [ "${PLAYWRIGHT:-0}" = "1" ]; then
|
||||
run virt-customize -a "$OUT/disk.raw" \
|
||||
--run-command 'pip3 install --no-cache-dir playwright; python3 -m playwright install --with-deps chromium'
|
||||
# B-08:pip freeze 是在第一次定制时写的,Playwright 装在第二次
|
||||
# 定制,须重写 freeze,否则 BUILDINFO.versions.pip 里没有
|
||||
# playwright;Chromium 版本以 ms-playwright 缓存目录名记录
|
||||
# (如 chromium-1200),写进 /app/guest-versions-chromium.txt。
|
||||
run virt-customize -a "$OUT/disk.raw" \
|
||||
--run-command 'dpkg-query -W -f "${Package} ${Version}\n" > /app/guest-versions-dpkg.txt; pip3 freeze > /app/guest-versions-pip.txt || true' \
|
||||
--run-command 'ls /root/.cache/ms-playwright/ > /app/guest-versions-chromium.txt 2>/dev/null || true'
|
||||
fi
|
||||
run virt-get-kernel -a "$OUT/disk.raw" -o "$OUT/vmlinux"
|
||||
# feat-chv-secfix(M4):构建时把客机内实际装好的版本
|
||||
@@ -131,43 +185,24 @@ run virt-get-kernel -a "$OUT/disk.raw" -o "$OUT/vmlinux"
|
||||
if [ "$DRY_RUN" = "1" ]; then
|
||||
echo "DRY-RUN: virt-cat -a $OUT/disk.raw /app/guest-versions-dpkg.txt > $OUT/guest-versions-dpkg.txt"
|
||||
echo "DRY-RUN: virt-cat -a $OUT/disk.raw /app/guest-versions-pip.txt > $OUT/guest-versions-pip.txt"
|
||||
if [ "${PLAYWRIGHT:-0}" = "1" ]; then
|
||||
echo "DRY-RUN: virt-cat -a $OUT/disk.raw /app/guest-versions-chromium.txt > $OUT/guest-versions-chromium.txt"
|
||||
fi
|
||||
else
|
||||
virt-cat -a "$OUT/disk.raw" /app/guest-versions-dpkg.txt > "$OUT/guest-versions-dpkg.txt"
|
||||
virt-cat -a "$OUT/disk.raw" /app/guest-versions-pip.txt > "$OUT/guest-versions-pip.txt"
|
||||
if [ "${PLAYWRIGHT:-0}" = "1" ]; then
|
||||
virt-cat -a "$OUT/disk.raw" /app/guest-versions-chromium.txt > "$OUT/guest-versions-chromium.txt" || true
|
||||
fi
|
||||
fi
|
||||
# 产物校验(干跑只打印,不执行校验命令的副作用部分)。
|
||||
if [ "$DRY_RUN" = "1" ]; then
|
||||
echo "DRY-RUN: test -s $OUT/vmlinux && test -s $OUT/disk.raw"
|
||||
echo "DRY-RUN: python3 生成 $OUT/BUILDINFO.json 并校验 tag=$TAG"
|
||||
echo "DRY-RUN: python3 $SCRIPT_DIR/buildinfo.py $OUT $TAG $UBUNTU_VERSION $CODEX_VERSION $REPO_COMMIT <sha256> ${PLAYWRIGHT:-0}"
|
||||
else
|
||||
test -s "$OUT/vmlinux" || { echo "内核产物为空:$OUT/vmlinux" >&2; exit 1; }
|
||||
test -s "$OUT/disk.raw" || { echo "磁盘产物为空:$OUT/disk.raw" >&2; exit 1; }
|
||||
python3 - "$OUT/BUILDINFO.json" "$TAG" "$UBUNTU_VERSION" "$CODEX_VERSION" "$REPO_COMMIT" <<'PYEOF'
|
||||
import json, sys, time
|
||||
path, tag, ubuntu, codex, commit = sys.argv[1:6]
|
||||
def _read_lines(p):
|
||||
try:
|
||||
with open(p) as fh:
|
||||
return [l.strip() for l in fh if l.strip()]
|
||||
except OSError:
|
||||
return []
|
||||
import os
|
||||
outdir = os.path.dirname(path)
|
||||
info = {"tag": tag, "ubuntu": ubuntu, "codex_version": codex,
|
||||
"repo_commit": commit, "built_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"contents": ["git", "pandoc", "poppler-utils", "latexmk", "texlive-latex-base",
|
||||
"openssh-server", "cloud-init", "qemu-guest-agent",
|
||||
"python-docx", "openpyxl", "python-pptx", "PyMuPDF", "Pillow",
|
||||
"codex", "codex-code-mode-host", "codex-resources/bwrap", "/app/host.py"],
|
||||
"versions": {
|
||||
"dpkg": _read_lines(os.path.join(outdir, "guest-versions-dpkg.txt")),
|
||||
"pip": _read_lines(os.path.join(outdir, "guest-versions-pip.txt")),
|
||||
},
|
||||
"ubuntu_image_sha256_set": True}
|
||||
with open(path, "w") as fh:
|
||||
json.dump(info, fh, indent=2)
|
||||
PYEOF
|
||||
python3 -c "import json; d=json.load(open('$OUT/BUILDINFO.json')); assert d['tag'] == '$TAG', d; assert d['versions']['dpkg'], 'BUILDINFO versions.dpkg 为空'; assert d['versions']['pip'], 'BUILDINFO versions.pip 为空'"
|
||||
python3 "$SCRIPT_DIR/buildinfo.py" "$OUT" "$TAG" "$UBUNTU_VERSION" "$CODEX_VERSION" "$REPO_COMMIT" "$UBUNTU_IMAGE_SHA256" "${PLAYWRIGHT:-0}"
|
||||
fi
|
||||
echo "tag=$TAG"
|
||||
echo "out=$OUT"
|
||||
|
||||
@@ -0,0 +1,118 @@
|
||||
#!/usr/bin/env python3
|
||||
"""生成 CHV 客机镜像的 BUILDINFO.json(build.sh 调用)。
|
||||
|
||||
从 build.sh 抽出(B-02),使其可独立单元测试。输入全部是
|
||||
已落盘的构建产物与读回文件,不编造版本号:versions 来自
|
||||
客机读回的 dpkg-query / pip freeze(Playwright=1 时另有
|
||||
ms-playwright 缓存目录名作 chromium 标识);artifacts 的
|
||||
sha256/size 是对产物文件实测的;image_sha256 即 disk.raw
|
||||
的 sha256,供 generate_lockfile.py 沿用。
|
||||
|
||||
用法:
|
||||
python3 buildinfo.py OUT_DIR TAG UBUNTU CODEX_VERSION \\
|
||||
REPO_COMMIT UBUNTU_IMAGE_SHA256 PLAYWRIGHT(0|1)
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
BASE_CONTENTS = [
|
||||
"git", "pandoc", "poppler-utils", "latexmk",
|
||||
"texlive-latex-base", "openssh-server", "cloud-init",
|
||||
"qemu-guest-agent", "python-docx", "openpyxl",
|
||||
"python-pptx", "PyMuPDF", "Pillow", "codex",
|
||||
"codex-code-mode-host", "codex-resources/bwrap",
|
||||
"/app/host.py",
|
||||
]
|
||||
|
||||
# 执行器默认内核 cmdline(config.py),构建产物须与它一起记录(B-12),
|
||||
# 否则换宿主/换执行器配置时无法复现。
|
||||
KERNEL_CMDLINE = "console=hvc0 root=/dev/vda1 rw"
|
||||
|
||||
|
||||
def _read_lines(path):
|
||||
try:
|
||||
with open(path) as fh:
|
||||
return [line.strip() for line in fh if line.strip()]
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
|
||||
def _sha256_size(path):
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
with open(path, "rb") as fh:
|
||||
for chunk in iter(lambda: fh.read(1 << 20), b""):
|
||||
digest.update(chunk)
|
||||
size += len(chunk)
|
||||
return digest.hexdigest(), size
|
||||
|
||||
|
||||
def build(outdir, tag, ubuntu, codex_version, repo_commit,
|
||||
ubuntu_image_sha256, playwright):
|
||||
disk_path = os.path.join(outdir, "disk.raw")
|
||||
vmlinux_path = os.path.join(outdir, "vmlinux")
|
||||
disk_sha, disk_size = _sha256_size(disk_path)
|
||||
vmlinux_sha, vmlinux_size = _sha256_size(vmlinux_path)
|
||||
contents = list(BASE_CONTENTS)
|
||||
versions = {
|
||||
"dpkg": _read_lines(
|
||||
os.path.join(outdir, "guest-versions-dpkg.txt")),
|
||||
"pip": _read_lines(
|
||||
os.path.join(outdir, "guest-versions-pip.txt")),
|
||||
}
|
||||
if playwright:
|
||||
contents += ["playwright", "chromium"]
|
||||
versions["chromium"] = _read_lines(
|
||||
os.path.join(outdir, "guest-versions-chromium.txt"))
|
||||
info = {
|
||||
"tag": tag,
|
||||
"ubuntu": ubuntu,
|
||||
"codex_version": codex_version,
|
||||
"repo_commit": repo_commit,
|
||||
"built_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"contents": contents,
|
||||
"versions": versions,
|
||||
"ubuntu_image_sha256": ubuntu_image_sha256,
|
||||
"ubuntu_image_sha256_set": True,
|
||||
"kernel_cmdline": KERNEL_CMDLINE,
|
||||
"playwright": bool(playwright),
|
||||
"image_sha256": disk_sha,
|
||||
"artifacts": {
|
||||
"disk.raw": {"sha256": disk_sha, "size": disk_size},
|
||||
"vmlinux": {"sha256": vmlinux_sha, "size": vmlinux_size},
|
||||
},
|
||||
}
|
||||
path = os.path.join(outdir, "BUILDINFO.json")
|
||||
with open(path, "w") as fh:
|
||||
json.dump(info, fh, indent=2)
|
||||
fh.write("\n")
|
||||
return info
|
||||
|
||||
|
||||
def main(argv):
|
||||
if len(argv) != 8:
|
||||
print("用法:buildinfo.py OUT TAG UBUNTU CODEX_VERSION "
|
||||
"REPO_COMMIT UBUNTU_IMAGE_SHA256 PLAYWRIGHT",
|
||||
file=sys.stderr)
|
||||
return 2
|
||||
_, outdir, tag, ubuntu, codex_version, repo_commit, \
|
||||
ubuntu_image_sha256, playwright = argv
|
||||
info = build(outdir, tag, ubuntu, codex_version, repo_commit,
|
||||
ubuntu_image_sha256, playwright == "1")
|
||||
if not info["versions"]["dpkg"]:
|
||||
print("BUILDINFO versions.dpkg 为空", file=sys.stderr)
|
||||
return 1
|
||||
if not info["versions"]["pip"]:
|
||||
print("BUILDINFO versions.pip 为空", file=sys.stderr)
|
||||
return 1
|
||||
if info["tag"] != tag:
|
||||
print("BUILDINFO tag 不一致", file=sys.stderr)
|
||||
return 1
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main(sys.argv))
|
||||
@@ -5,9 +5,9 @@
|
||||
宿主上解析、pip 版本在客机里解析,没有构建宿主就编不出
|
||||
版本号——本脚本绝不编造。构建时 build.sh 已从客机读回
|
||||
dpkg + pip freeze 写进 BUILDINFO.versions(事后 lockfile),
|
||||
本脚本校验它:versions 非空、tag 与预期一致、可选的冻结
|
||||
清单(--frozen JSON,形如 {"dpkg": ["pkg version", ...]})
|
||||
逐项比对,漂移即退出非 0。
|
||||
本脚本校验它:versions 非空、每条有版本、contents 全覆盖、
|
||||
tag 与预期一致、可选的冻结清单(--frozen JSON,形如
|
||||
{"dpkg": ["pkg version", ...]})双向比对,漂移即退出非 0。
|
||||
|
||||
用法:
|
||||
python3 check_lockfile.py out/BUILDINFO.json [--tag TAG]
|
||||
@@ -16,6 +16,47 @@ dpkg + pip freeze 写进 BUILDINFO.versions(事后 lockfile),
|
||||
import json
|
||||
import sys
|
||||
|
||||
# 非包名的内容项:版本来自 BUILDINFO 的声明字段,不是 versions 列表。
|
||||
DECLARED_SOURCES = {
|
||||
"codex": "codex_version",
|
||||
"codex-code-mode-host": "codex_version",
|
||||
"codex-resources/bwrap": "codex_version",
|
||||
"/app/host.py": "repo_commit",
|
||||
}
|
||||
|
||||
|
||||
def _names(entries, pip_style=False):
|
||||
"""versions 条目 → 小写名集合。dpkg 行是 "name version",
|
||||
pip freeze 行是 "name==version"。"""
|
||||
names = set()
|
||||
for entry in entries or []:
|
||||
text = str(entry)
|
||||
if pip_style and "==" in text:
|
||||
names.add(text.split("==", 1)[0].lower())
|
||||
else:
|
||||
parts = text.split(None, 1)
|
||||
if parts:
|
||||
names.add(parts[0].lower())
|
||||
return names
|
||||
|
||||
|
||||
def _check_format(kind, entries):
|
||||
"""每条须带非空版本:dpkg "name version",pip "name==version"。"""
|
||||
for entry in entries or []:
|
||||
text = str(entry)
|
||||
if kind == "pip" and "==" in text:
|
||||
if not text.split("==", 1)[1].strip():
|
||||
print("版本为空的条目:%s %r" % (kind, entry),
|
||||
file=sys.stderr)
|
||||
return False
|
||||
else:
|
||||
parts = text.split()
|
||||
if len(parts) < 2 or not parts[1]:
|
||||
print("版本为空的条目:%s %r" % (kind, entry),
|
||||
file=sys.stderr)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def main(argv):
|
||||
if len(argv) < 2:
|
||||
@@ -47,20 +88,46 @@ def main(argv):
|
||||
print("BUILDINFO.versions.dpkg/pip 为空,供应链清单缺失",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
if not _check_format("dpkg", versions.get("dpkg")):
|
||||
return 1
|
||||
if not _check_format("pip", versions.get("pip")):
|
||||
return 1
|
||||
# contents 全覆盖:每项须在 dpkg/pip 名单里,或有声明字段版本。
|
||||
covered = _names(versions.get("dpkg")) | _names(
|
||||
versions.get("pip"), pip_style=True)
|
||||
missing = []
|
||||
for name in data.get("contents") or []:
|
||||
if name.lower() in covered:
|
||||
continue
|
||||
source = DECLARED_SOURCES.get(name)
|
||||
if source and data.get(source):
|
||||
continue
|
||||
if name.lower() == "chromium" and versions.get("chromium"):
|
||||
continue
|
||||
missing.append(name)
|
||||
if missing:
|
||||
print("contents 未覆盖:%s" % sorted(missing), file=sys.stderr)
|
||||
return 1
|
||||
if frozen_path:
|
||||
with open(frozen_path) as fh:
|
||||
frozen = json.load(fh)
|
||||
for kind in ("dpkg", "pip"):
|
||||
want = set(frozen.get(kind) or [])
|
||||
got = set(versions.get(kind) or [])
|
||||
missing = want - got
|
||||
if missing:
|
||||
absent = want - got
|
||||
if absent:
|
||||
print("%s 漂移:冻结清单里有、构建产物里无:%s"
|
||||
% (kind, sorted(missing)), file=sys.stderr)
|
||||
% (kind, sorted(absent)), file=sys.stderr)
|
||||
return 1
|
||||
print("lockfile 校验通过:tag=%s dpkg=%d pip=%d"
|
||||
extra = got - want
|
||||
if extra:
|
||||
print("%s 漂移:构建产物里有、冻结清单里无:%s"
|
||||
% (kind, sorted(extra)), file=sys.stderr)
|
||||
return 1
|
||||
print("lockfile 校验通过:tag=%s dpkg=%d pip=%d contents=%d"
|
||||
% (data.get("tag"), len(versions["dpkg"]),
|
||||
len(versions["pip"])))
|
||||
len(versions["pip"]),
|
||||
len(data.get("contents") or [])))
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -1,67 +1,79 @@
|
||||
#!/usr/bin/env python3
|
||||
"""在构建宿主上生成 CHV 客机镜像的供应链 lockfile(to10 轨)。
|
||||
"""在构建宿主上生成 CHV 客机镜像的供应链 lockfile。
|
||||
|
||||
事后校验见 check_lockfile.py(校验 BUILDINFO.versions);
|
||||
本脚本是事前/事中生成端:在构建宿主上跑,读 BUILDINFO
|
||||
的 tag 与 contents(仅名字清单,不含版本号),逐项从本
|
||||
机 dpkg / pip 查真实版本,写出 lockfile.json:
|
||||
本脚本是生成端:读 BUILDINFO.json,以其中的 versions
|
||||
(build.sh 已从客机读回的 dpkg-query / pip freeze,
|
||||
Playwright=1 时另有 chromium 标识)为唯一版本来源,
|
||||
逐项给 contents 清单配版本,写出 lockfile.json:
|
||||
|
||||
{"tag": ..., "image_sha256": ... | null,
|
||||
"generated_at": UTC ISO 时间,
|
||||
"versions": {"dpkg": ["name version", ...],
|
||||
"pip": ["name version", ...]},
|
||||
"pip": ["name version", ...],
|
||||
"declared": ["name version", ...]},
|
||||
"unresolved": ["查不到版本的名", ...]}
|
||||
|
||||
绝不编造版本号:查不到的进 unresolved,不猜、不填占位
|
||||
版本。--verify 模式校验一份 lockfile:tag 与 BUILDINFO
|
||||
一致、dpkg/pip 非空、每条都有非空版本、unresolved 为空;
|
||||
任一不满足即退出非 0(unresolved 非空 = 清单不完整,
|
||||
不是“生成成功”)。
|
||||
版本来源映射(绝不编造版本号,查不到的进 unresolved):
|
||||
- dpkg 包名 → versions.dpkg("name version" 行)
|
||||
- pip 包名 → versions.pip("name==version" 行)
|
||||
- codex / codex-code-mode-host / codex-resources/bwrap
|
||||
→ BUILDINFO.codex_version(随 codex 一起发布)
|
||||
- /app/host.py → BUILDINFO.repo_commit
|
||||
- chromium → versions.chromium(ms-playwright 缓存目录名)
|
||||
|
||||
注意:旧版逐项查构建宿主的 dpkg/pip,方法不成立——宿主
|
||||
版本与客机版本没有必然关系,且 /app/host.py、codex 三件
|
||||
套在宿主上永远查不到,--verify 永远过不了。本版只读
|
||||
BUILDINFO.versions,不再现场查宿主。
|
||||
|
||||
用法(均在构建宿主上跑):
|
||||
python3 generate_lockfile.py BUILDINFO.json --output lockfile.json
|
||||
python3 generate_lockfile.py --verify lockfile.json \
|
||||
--buildinfo BUILDINFO.json
|
||||
本轨无构建宿主:脚本已落地且可跑(见回执里的 /tmp 合成
|
||||
BUILDINFO 实测),真实清单待构建宿主产出,绝不把合成
|
||||
数据写成生产清单。
|
||||
"""
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
|
||||
|
||||
def _dpkg_version(name):
|
||||
"""dpkg 查版本,查不到回 None(不抛)。"""
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["dpkg-query", "-W", "-f=${Version}", name],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
except Exception:
|
||||
return None
|
||||
if out.returncode != 0:
|
||||
return None
|
||||
version = out.stdout.strip()
|
||||
return version or None
|
||||
# 非包名的内容项:版本来自 BUILDINFO 的声明字段,不是 dpkg/pip。
|
||||
DECLARED_SOURCES = {
|
||||
"codex": "codex_version",
|
||||
"codex-code-mode-host": "codex_version",
|
||||
"codex-resources/bwrap": "codex_version",
|
||||
"/app/host.py": "repo_commit",
|
||||
}
|
||||
|
||||
|
||||
def _pip_version(name):
|
||||
"""pip 查版本,查不到回 None(不抛)。"""
|
||||
try:
|
||||
out = subprocess.run(
|
||||
[sys.executable, "-m", "pip", "show", name],
|
||||
capture_output=True, text=True, timeout=30)
|
||||
except Exception:
|
||||
return None
|
||||
if out.returncode != 0:
|
||||
return None
|
||||
for line in out.stdout.splitlines():
|
||||
if line.startswith("Version:"):
|
||||
version = line.split(":", 1)[1].strip()
|
||||
return version or None
|
||||
return None
|
||||
def _index_dpkg(lines):
|
||||
"""dpkg 行 "name version" → {小写名: "name version"}。"""
|
||||
index = {}
|
||||
for line in lines or []:
|
||||
parts = str(line).split(None, 1)
|
||||
if len(parts) == 2:
|
||||
index[parts[0].lower()] = "%s %s" % (parts[0], parts[1])
|
||||
return index
|
||||
|
||||
|
||||
def _index_pip(lines):
|
||||
"""pip freeze 行 "name==version" → {小写名: "name version"}。"""
|
||||
index = {}
|
||||
for line in lines or []:
|
||||
if "==" in str(line):
|
||||
name, version = str(line).split("==", 1)
|
||||
index[name.lower()] = "%s %s" % (name, version.strip())
|
||||
return index
|
||||
|
||||
|
||||
def _index_chromium(lines):
|
||||
"""chromium 标识行(如 chromium-1200)→ {"chromium": 行}。"""
|
||||
index = {}
|
||||
for line in lines or []:
|
||||
text = str(line).strip()
|
||||
if text:
|
||||
index["chromium"] = "chromium %s" % text
|
||||
return index
|
||||
|
||||
|
||||
def generate(buildinfo_path, output_path):
|
||||
@@ -71,15 +83,32 @@ def generate(buildinfo_path, output_path):
|
||||
if not isinstance(names, list):
|
||||
print("BUILDINFO.contents 不是列表", file=sys.stderr)
|
||||
return 2
|
||||
dpkg, pip, unresolved = [], [], []
|
||||
versions = buildinfo.get("versions") or {}
|
||||
dpkg_index = _index_dpkg(versions.get("dpkg"))
|
||||
pip_index = _index_pip(versions.get("pip"))
|
||||
chromium_index = _index_chromium(versions.get("chromium"))
|
||||
dpkg, pip, declared, unresolved = [], [], [], []
|
||||
for name in names:
|
||||
version = _dpkg_version(name)
|
||||
if version is not None:
|
||||
dpkg.append("%s %s" % (name, version))
|
||||
key = name.lower()
|
||||
if name in DECLARED_SOURCES:
|
||||
version = buildinfo.get(DECLARED_SOURCES[name])
|
||||
if version:
|
||||
declared.append("%s %s" % (name, version))
|
||||
else:
|
||||
unresolved.append(name)
|
||||
continue
|
||||
version = _pip_version(name)
|
||||
if version is not None:
|
||||
pip.append("%s %s" % (name, version))
|
||||
if key == "chromium":
|
||||
entry = chromium_index.get("chromium")
|
||||
if entry:
|
||||
declared.append(entry)
|
||||
else:
|
||||
unresolved.append(name)
|
||||
continue
|
||||
if key in dpkg_index:
|
||||
dpkg.append(dpkg_index[key])
|
||||
continue
|
||||
if key in pip_index:
|
||||
pip.append(pip_index[key])
|
||||
continue
|
||||
unresolved.append(name)
|
||||
lockfile = {
|
||||
@@ -87,7 +116,8 @@ def generate(buildinfo_path, output_path):
|
||||
"image_sha256": buildinfo.get("image_sha256")
|
||||
or buildinfo.get("sha256"),
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"versions": {"dpkg": sorted(dpkg), "pip": sorted(pip)},
|
||||
"versions": {"dpkg": sorted(dpkg), "pip": sorted(pip),
|
||||
"declared": sorted(declared)},
|
||||
"unresolved": sorted(unresolved),
|
||||
}
|
||||
with open(output_path, "w") as fh:
|
||||
@@ -129,9 +159,10 @@ def verify(lockfile_path, buildinfo_path):
|
||||
print("清单不完整,unresolved 非空:%s" % sorted(unresolved),
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
print("lockfile 校验通过:tag=%s dpkg=%d pip=%d"
|
||||
print("lockfile 校验通过:tag=%s dpkg=%d pip=%d declared=%d"
|
||||
% (lockfile.get("tag"), len(versions["dpkg"]),
|
||||
len(versions["pip"])))
|
||||
len(versions["pip"]),
|
||||
len(versions.get("declared") or [])))
|
||||
return 0
|
||||
|
||||
|
||||
|
||||
@@ -359,21 +359,46 @@ class LockfileScriptTest(unittest.TestCase):
|
||||
spec.loader.exec_module(mod)
|
||||
return mod
|
||||
|
||||
def test_generate_reads_real_dpkg_version(self):
|
||||
def test_generate_reads_guest_versions(self):
|
||||
# C 轨纠正(报告 §3.2):版本唯一来源是 BUILDINFO.versions
|
||||
#(客机读回),不是构建宿主的 dpkg/pip;codex 类声明项
|
||||
# 取 BUILDINFO 的 codex_version / repo_commit。
|
||||
import json as _json
|
||||
mod = self._load_script()
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
bi = os.path.join(tmp, "bi.json")
|
||||
out = os.path.join(tmp, "lock.json")
|
||||
with open(bi, "w") as fh:
|
||||
_json.dump({"tag": "t", "contents": ["coreutils"]},
|
||||
_json.dump({"tag": "t",
|
||||
"contents": ["coreutils", "codex"],
|
||||
"codex_version": "0.164.0-alpha.9",
|
||||
"versions": {"dpkg": ["coreutils 9.4-3"],
|
||||
"pip": []}},
|
||||
fh)
|
||||
self.assertEqual(mod.generate(bi, out), 0)
|
||||
lock = _json.load(open(out))
|
||||
self.assertEqual(lock["tag"], "t")
|
||||
self.assertEqual(lock["unresolved"], [])
|
||||
self.assertTrue(lock["versions"]["dpkg"][0].startswith(
|
||||
"coreutils "))
|
||||
self.assertEqual(lock["versions"]["dpkg"],
|
||||
["coreutils 9.4-3"])
|
||||
self.assertEqual(lock["versions"]["declared"],
|
||||
["codex 0.164.0-alpha.9"])
|
||||
|
||||
def test_generate_does_not_query_host(self):
|
||||
# 宿主上装着 coreutils,但 BUILDINFO.versions 里没有 →
|
||||
# unresolved,绝不回落到宿主查询(旧方法不成立)。
|
||||
import json as _json
|
||||
mod = self._load_script()
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
bi = os.path.join(tmp, "bi.json")
|
||||
out = os.path.join(tmp, "lock.json")
|
||||
with open(bi, "w") as fh:
|
||||
_json.dump({"tag": "t", "contents": ["coreutils"],
|
||||
"versions": {"dpkg": [], "pip": []}},
|
||||
fh)
|
||||
self.assertEqual(mod.generate(bi, out), 0)
|
||||
lock = _json.load(open(out))
|
||||
self.assertEqual(lock["unresolved"], ["coreutils"])
|
||||
|
||||
def test_verify_fails_on_unresolved(self):
|
||||
import json as _json
|
||||
|
||||
Reference in new issue
Block a user