残留批次A：成员删除、PR needs_local_worker 落库、推送 DNS TOCTOU
- 成员删除：DELETE /projects/{id}/members/{user_id}，限 owner 或
全局管理员，禁删唯一 owner，记 remove_member 审计
（AuthStore.remove_member + server 分发，沿用 PATCH 守卫口径）
- PR needs_local_worker 落库：非 local 扫到 queued 任务时 CAS
写行状态为 needs_local_worker；claim/find_open 同步认该状态，
标记过的行仍可被 local worker 认领、不被搁置；另修
claim/find_open 对裸 sqlite3 tuple 行的 dict() 兜底
（TypeError/ValueError）
- 推送 DNS TOCTOU：发送改走建连时解析+判段的连接类，解析
一次、只拨判过的 IP，Host/SNI 仍为原域名；opener 直接构造
以排除 ProxyHandler 改写（build_opener 默认会塞代理）
This commit is contained in:
host committed
1 parent
6e3eb7ba11
commit
9bb9a50e4e
10 files changed
+725
-38
No files matched your search
@@ -0,0 +1,161 @@
|
||||
"""成员删除端点单测（残留批次 A 任务 1）。
|
||||
|
||||
DELETE /projects/{id}/members/{user_id}：
|
||||
- 限项目 owner 或全局管理员，其余 403；
|
||||
- 目标非成员 404，项目不存在 404；
|
||||
- 删唯一 owner 时 409（owner 与全局管理员均受此限）；
|
||||
- 成功后成员列表无此人，且记 remove_member 审计。
|
||||
|
||||
测试用的 key、用户 id 全是假值；不联网、不碰生产。
|
||||
"""
|
||||
|
||||
import http.client
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.server import create_server
|
||||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.auth import AuthStore
|
||||
from cloud.server import create_server
|
||||
|
||||
|
||||
def _req(port, method, path, body=None, key=None):
|
||||
conn = http.client.HTTPConnection("127.0.0.1", port, timeout=10)
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if key:
|
||||
headers["X-API-Key"] = key
|
||||
conn.request(method, path,
|
||||
json.dumps(body) if body is not None else None, headers)
|
||||
resp = conn.getresponse()
|
||||
raw = resp.read()
|
||||
conn.close()
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except ValueError:
|
||||
data = None
|
||||
return resp.status, data
|
||||
|
||||
|
||||
class MemberDeleteTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
self.trace_dir = os.path.join(self.tmp.name, "host-traces")
|
||||
os.makedirs(self.trace_dir, exist_ok=True)
|
||||
self.auth = AuthStore(self.db_path)
|
||||
self.owner_id, self.owner_key = self.auth.create_user("owner")
|
||||
self.maint_id, self.maint_key = self.auth.create_user("maint")
|
||||
self.member_id, self.member_key = self.auth.create_user("member")
|
||||
self.admin_id, self.admin_key = self.auth.create_user("admin")
|
||||
self.outsider_id, self.out_key = self.auth.create_user("outsider")
|
||||
self._old_admins = os.environ.get("CODEX_GLOBAL_ADMINS")
|
||||
os.environ["CODEX_GLOBAL_ADMINS"] = self.admin_id
|
||||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||||
self.server.trace_dir = self.trace_dir
|
||||
self.port = self.server.server_address[1]
|
||||
self.thread = threading.Thread(
|
||||
target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
status, proj = _req(self.port, "POST", "/projects",
|
||||
{"name": "demo"}, self.owner_key)
|
||||
self.assertEqual(status, 201, proj)
|
||||
self.project_id = proj["id"]
|
||||
self.auth.set_member(self.project_id, self.owner_id,
|
||||
self.maint_id, "maintainer")
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join(timeout=5)
|
||||
try:
|
||||
self.auth.conn.close()
|
||||
except Exception:
|
||||
pass
|
||||
self.tmp.cleanup()
|
||||
if self._old_admins is None:
|
||||
os.environ.pop("CODEX_GLOBAL_ADMINS", None)
|
||||
else:
|
||||
os.environ["CODEX_GLOBAL_ADMINS"] = self._old_admins
|
||||
|
||||
def _delete(self, target, key):
|
||||
return _req(self.port, "DELETE",
|
||||
"/projects/%s/members/%s" % (self.project_id, target),
|
||||
key=key)
|
||||
|
||||
def _invite(self, target, role="contributor", key=None):
|
||||
return _req(self.port, "POST",
|
||||
"/projects/%s/members" % self.project_id,
|
||||
{"user_id": target, "role": role},
|
||||
key or self.owner_key)
|
||||
|
||||
def _member_ids(self, key=None):
|
||||
status, data = _req(self.port, "GET",
|
||||
"/projects/%s/members" % self.project_id,
|
||||
key=key or self.owner_key)
|
||||
self.assertEqual(status, 200, data)
|
||||
return {m["user_id"] for m in data["members"]}
|
||||
|
||||
def _audit_actions(self):
|
||||
return [(a["action"], a["detail"])
|
||||
for a in self.auth.list_audit(self.project_id)]
|
||||
|
||||
def test_owner_delete_member_and_audit(self):
|
||||
self.assertEqual(self._invite(self.member_id)[0], 201)
|
||||
status, data = self._delete(self.member_id, self.owner_key)
|
||||
self.assertEqual(status, 200, data)
|
||||
self.assertEqual(data["user_id"], self.member_id)
|
||||
self.assertNotIn(self.member_id, self._member_ids())
|
||||
self.assertIn(("remove_member", self.member_id),
|
||||
self._audit_actions())
|
||||
|
||||
def test_non_owner_and_outsider_denied(self):
|
||||
self.assertEqual(self._invite(self.member_id)[0], 201)
|
||||
status, _ = self._delete(self.member_id, self.maint_key)
|
||||
self.assertEqual(status, 403)
|
||||
status, _ = self._delete(self.member_id, self.out_key)
|
||||
self.assertEqual(status, 403)
|
||||
# 拒绝后成员仍在。
|
||||
self.assertIn(self.member_id, self._member_ids())
|
||||
|
||||
def test_global_admin_delete_without_membership(self):
|
||||
self.assertEqual(self._invite(self.member_id)[0], 201)
|
||||
status, data = self._delete(self.member_id, self.admin_key)
|
||||
self.assertEqual(status, 200, data)
|
||||
self.assertNotIn(self.member_id, self._member_ids())
|
||||
self.assertIn(("remove_member", self.member_id),
|
||||
self._audit_actions())
|
||||
|
||||
def test_delete_non_member_and_missing_project(self):
|
||||
status, _ = self._delete("ghost", self.owner_key)
|
||||
self.assertEqual(status, 404)
|
||||
status, _ = _req(self.port, "DELETE",
|
||||
"/projects/no-such/members/%s" % self.member_id,
|
||||
key=self.owner_key)
|
||||
self.assertEqual(status, 404)
|
||||
|
||||
def test_cannot_remove_only_owner(self):
|
||||
status, data = self._delete(self.owner_id, self.owner_key)
|
||||
self.assertEqual(status, 409, data)
|
||||
status, _ = self._delete(self.owner_id, self.admin_key)
|
||||
self.assertEqual(status, 409)
|
||||
# 升第二位 owner 后，删原 owner 放行。
|
||||
self.assertEqual(self._invite(self.member_id)[0], 201)
|
||||
status, _ = _req(self.port, "PATCH",
|
||||
"/projects/%s/members/%s"
|
||||
% (self.project_id, self.member_id),
|
||||
{"role": "owner"}, self.owner_key)
|
||||
self.assertEqual(status, 200)
|
||||
status, _ = self._delete(self.owner_id, self.maint_key)
|
||||
# maint 不是 owner 也不是全局管理员：仍 403（守卫先行）。
|
||||
self.assertEqual(status, 403)
|
||||
status, data = self._delete(self.owner_id, self.member_key)
|
||||
self.assertEqual(status, 200, data)
|
||||
self.assertNotIn(self.owner_id, self._member_ids(self.member_key))
|
||||
@@ -449,7 +449,8 @@ class P1WorkerTestCase(unittest.TestCase):
|
||||
self.assertEqual(_status(self.db_path, "T1"), "failed")
|
||||
self.assertEqual(self.calls, [])
|
||||
|
||||
# 15. PR 非 local：显式 needs_local_worker，任务与排队都不动
|
||||
# 15. PR 非 local：显式 needs_local_worker，行状态持久化，
|
||||
# 任务本身不动，标记过的行仍可被 local worker 认领
|
||||
def test_pr_non_local_needs_local_worker(self):
|
||||
conn = sqlite3.connect(self.db_path)
|
||||
try:
|
||||
@@ -470,9 +471,14 @@ class P1WorkerTestCase(unittest.TestCase):
|
||||
row = conn.execute(
|
||||
"SELECT status FROM pr_jobs WHERE id = ?",
|
||||
(job_id,)).fetchone()
|
||||
self.assertEqual(row[0], "needs_local_worker")
|
||||
# 标记过的行仍可被 local worker 认领，不会被搁置。
|
||||
from cloud.pr_exec import claim_pr_job
|
||||
claimed = claim_pr_job(conn, "test-host")
|
||||
self.assertIsNotNone(claimed)
|
||||
self.assertEqual(claimed["id"], job_id)
|
||||
finally:
|
||||
conn.close()
|
||||
self.assertEqual(row[0], "queued")
|
||||
|
||||
# 16. 多文件签名 + 签名缺失/已变时认领 fail-closed
|
||||
def test_code_signature_multi_file_and_claim_refusal(self):
|
||||
|
||||
@@ -308,7 +308,7 @@ class WorkerPrJobsTestCase(unittest.TestCase):
|
||||
finally:
|
||||
conn.close()
|
||||
# P1：Docker worker 不再静默返回空，改为显式 needs_local_worker，
|
||||
# 任务保持 queued 等 local worker 认领。
|
||||
# 行状态持久化为 needs_local_worker、仍可被 local worker 认领。
|
||||
jobs = worker._run_pr_jobs()
|
||||
self.assertEqual(len(jobs), 1)
|
||||
self.assertEqual(jobs[0]["status"], "needs_local_worker")
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
"""PR needs_local_worker 持久化单测（残留批次 A 任务 2）。
|
||||
|
||||
- 非 local worker 扫到 queued 的 pr 任务后，行状态持久化为
|
||||
needs_local_worker（重读仍在），且返回条目状态一致；
|
||||
- 标记过的行仍可被 local worker 认领（claim 后 running，可
|
||||
正常 finish），不会被搁置；
|
||||
- find_open_pr_job 仍把标记过的行算未完成（/pr 不重排）；
|
||||
- 标记是 CAS：已 running 的行不被覆盖；重复扫不重复记。
|
||||
|
||||
测试用的 key、主机名、任务 id 全是假值；不联网、不碰生产。
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
from cloud.db import connect as _db_connect
|
||||
from cloud.pr_exec import (
|
||||
claim_pr_job, enqueue_pr_job, find_open_pr_job,
|
||||
mark_pr_job_needs_local_worker)
|
||||
from cloud.sandbox import DockerExecutor
|
||||
from cloud.worker import Worker
|
||||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.db import connect as _db_connect
|
||||
from cloud.pr_exec import (
|
||||
claim_pr_job, enqueue_pr_job, find_open_pr_job,
|
||||
mark_pr_job_needs_local_worker)
|
||||
from cloud.sandbox import DockerExecutor
|
||||
from cloud.worker import Worker
|
||||
|
||||
|
||||
class PrNeedsLocalTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||||
|
||||
def _enqueue_t1(self, host="host-a"):
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
enqueue_pr_job(conn, "t1", "p1", host)
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def _row(self, task_id):
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
return conn.execute(
|
||||
"SELECT id, status, error FROM pr_jobs"
|
||||
" WHERE task_id = ?", (task_id,)).fetchone()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def test_docker_peek_persists_mark(self):
|
||||
worker = Worker(self.db_path, "/nonexistent/host.py",
|
||||
env_key="TEST_KEY",
|
||||
executor=DockerExecutor("/fake/host.py"))
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
enqueue_pr_job(conn, "t1", "p1", worker.host_name)
|
||||
finally:
|
||||
conn.close()
|
||||
jobs = worker._run_pr_jobs()
|
||||
self.assertEqual(len(jobs), 1)
|
||||
self.assertEqual(jobs[0]["status"], "needs_local_worker")
|
||||
self.assertEqual(jobs[0]["task_id"], "t1")
|
||||
# 重读：行状态已持久化，不只是返回值里有。
|
||||
row = self._row("t1")
|
||||
self.assertEqual(row[1], "needs_local_worker")
|
||||
self.assertIn("local worker", row[2])
|
||||
# 再扫一次：已标记的行不再是 queued，不重复返回、不重复记。
|
||||
self.assertEqual(worker._run_pr_jobs(), [])
|
||||
|
||||
def test_marked_row_still_claimable_by_local(self):
|
||||
self._enqueue_t1()
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
job_id = conn.execute(
|
||||
"SELECT id FROM pr_jobs WHERE task_id = 't1'"
|
||||
).fetchone()[0]
|
||||
self.assertTrue(mark_pr_job_needs_local_worker(
|
||||
conn, job_id, "needs local worker"))
|
||||
# 标记后仍算未完成：/pr 去重不受影响。
|
||||
self.assertIsNotNone(find_open_pr_job(conn, "t1"))
|
||||
# local worker 认领标记过的行。
|
||||
claimed = claim_pr_job(conn, "host-a")
|
||||
self.assertIsNotNone(claimed)
|
||||
self.assertEqual(claimed["task_id"], "t1")
|
||||
row = conn.execute(
|
||||
"SELECT status FROM pr_jobs WHERE task_id = 't1'"
|
||||
).fetchone()
|
||||
self.assertEqual(row[0], "running")
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
def test_mark_is_cas_on_queued_only(self):
|
||||
self._enqueue_t1()
|
||||
conn = _db_connect(self.db_path)
|
||||
try:
|
||||
claimed = claim_pr_job(conn, "host-a")
|
||||
self.assertIsNotNone(claimed)
|
||||
# 已 running：标记必须失败、不覆盖状态。
|
||||
self.assertFalse(mark_pr_job_needs_local_worker(
|
||||
conn, claimed["id"], "late mark"))
|
||||
row = conn.execute(
|
||||
"SELECT status FROM pr_jobs WHERE task_id = 't1'"
|
||||
).fetchone()
|
||||
self.assertEqual(row[0], "running")
|
||||
finally:
|
||||
conn.close()
|
||||
@@ -0,0 +1,172 @@
|
||||
"""推送 DNS TOCTOU 单测（残留批次 A 任务 3）。
|
||||
|
||||
旧版：validate_push_target 解析一次，urlopen 建连时又解析
|
||||
一次；两次解析结果不一致（DNS 重绑定）时可能拨向未验证
|
||||
的内网 IP。新版：建连时解析一次并判段、只拨判过的 IP，
|
||||
Host 头与 TLS SNI 仍为原域名。
|
||||
|
||||
- 两次解析不一致（先公网、后内网）时：发送失败，且一次
|
||||
都没有拨向内网 IP；
|
||||
- 混合解析结果（先内网、后公网）时：只拨公网 IP；
|
||||
- HTTP 真服务器：域名经 resolver 钉到 127.0.0.1，服务器
|
||||
收到的 Host 仍为原域名；
|
||||
- HTTPS 连接类：拨号地址为判过的 IP，wrap_socket 的
|
||||
server_hostname 仍为原域名（SNI 不变）。
|
||||
|
||||
测试用的 IP、域名、key 全是假值；除本地回环服务器外不联网。
|
||||
"""
|
||||
|
||||
import http.client
|
||||
import http.server
|
||||
import os
|
||||
import sys
|
||||
import threading
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
try:
|
||||
from cloud.push import (
|
||||
_checked_connection_class, _post_json, send_webhook)
|
||||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from cloud.push import (
|
||||
_checked_connection_class, _post_json, send_webhook)
|
||||
|
||||
PUBLIC_IP = "8.8.8.8"
|
||||
PRIVATE_IP = "169.254.169.254"
|
||||
|
||||
|
||||
class _FlipFlopResolver:
|
||||
"""第一次返回公网 IP，之后返回内网 IP：模拟重绑定。
|
||||
|
||||
旧版两次解析恰好踩中这个窗口；新版第二次解析判段后
|
||||
直接拒绝，绝不拨号。
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.calls = 0
|
||||
|
||||
def __call__(self, host):
|
||||
self.calls += 1
|
||||
return [PUBLIC_IP] if self.calls == 1 else [PRIVATE_IP]
|
||||
|
||||
|
||||
class _DialRecorder:
|
||||
"""记录拨号地址后直接失败：测试只断言拨向，不跑完 HTTP。"""
|
||||
|
||||
def __init__(self):
|
||||
self.calls = []
|
||||
|
||||
def __call__(self, address, timeout=None, source_address=None):
|
||||
self.calls.append(address)
|
||||
raise OSError("dial stopped by test")
|
||||
|
||||
|
||||
class _DummySocket:
|
||||
def setsockopt(self, *args):
|
||||
pass
|
||||
|
||||
|
||||
class _RecordingHandler(http.server.BaseHTTPRequestHandler):
|
||||
requests = []
|
||||
|
||||
def do_POST(self):
|
||||
length = int(self.headers.get("Content-Length", 0))
|
||||
body = self.rfile.read(length) if length else b""
|
||||
_RecordingHandler.requests.append({
|
||||
"host": self.headers.get("Host"),
|
||||
"body": body,
|
||||
})
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Length", "2")
|
||||
self.end_headers()
|
||||
self.wfile.write(b"{}")
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
|
||||
class PushToctouTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._old_allow = os.environ.get("CODEX_PUSH_ALLOW_PRIVATE_TARGETS")
|
||||
os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None)
|
||||
_RecordingHandler.requests = []
|
||||
|
||||
def tearDown(self):
|
||||
if self._old_allow is None:
|
||||
os.environ.pop("CODEX_PUSH_ALLOW_PRIVATE_TARGETS", None)
|
||||
else:
|
||||
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = self._old_allow
|
||||
|
||||
def _message(self, target, resolver=None):
|
||||
msg = {"event_type": "completed", "project_id": "proj-a",
|
||||
"payload": {"task_id": "t-1"}, "target": target}
|
||||
if resolver is not None:
|
||||
msg["_resolver"] = resolver
|
||||
return msg
|
||||
|
||||
def test_rebinding_second_resolution_never_dialed(self):
|
||||
resolver = _FlipFlopResolver()
|
||||
dials = _DialRecorder()
|
||||
with mock.patch("socket.create_connection", dials):
|
||||
with self.assertRaises(RuntimeError):
|
||||
send_webhook(self._message(
|
||||
"http://hook.example.invalid/hook", resolver))
|
||||
# 两次解析确实都发生了（一次预检、一次建连），但建连侧
|
||||
# 判段后拒绝：内网 IP 一次都没被拨。
|
||||
self.assertEqual(resolver.calls, 2)
|
||||
self.assertEqual(dials.calls, [])
|
||||
|
||||
def test_mixed_resolution_dials_only_public(self):
|
||||
dials = _DialRecorder()
|
||||
with mock.patch("socket.create_connection", dials):
|
||||
ok, _detail = _post_json(
|
||||
"http://hook.example.invalid/hook", {"a": 1},
|
||||
resolver=lambda host: [PRIVATE_IP, PUBLIC_IP])
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual(dials.calls, [(PUBLIC_IP, 80)])
|
||||
|
||||
def test_http_pinned_host_header_preserved(self):
|
||||
os.environ["CODEX_PUSH_ALLOW_PRIVATE_TARGETS"] = "1"
|
||||
server = http.server.HTTPServer(("127.0.0.1", 0),
|
||||
_RecordingHandler)
|
||||
port = server.server_address[1]
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
try:
|
||||
send_webhook(self._message(
|
||||
"http://webhook.test.invalid:%d/hook" % port,
|
||||
resolver=lambda host: ["127.0.0.1"]))
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join(timeout=5)
|
||||
self.assertEqual(len(_RecordingHandler.requests), 1)
|
||||
host_header = _RecordingHandler.requests[0]["host"]
|
||||
# 拨的是 127.0.0.1，但 Host 仍为原域名（虚拟主机依赖它）。
|
||||
self.assertEqual(host_header.split(":")[0],
|
||||
"webhook.test.invalid")
|
||||
|
||||
def test_https_sni_preserved_with_pinned_ip(self):
|
||||
wrap_calls = []
|
||||
|
||||
class FakeContext:
|
||||
def wrap_socket(self, sock, server_hostname=None):
|
||||
wrap_calls.append(server_hostname)
|
||||
return sock
|
||||
|
||||
cls = _checked_connection_class(http.client.HTTPSConnection,
|
||||
lambda host: [PUBLIC_IP])
|
||||
conn = cls("example.com", 443)
|
||||
conn._context = FakeContext()
|
||||
dials = []
|
||||
|
||||
def fake_create(address, timeout=None, source_address=None):
|
||||
dials.append(address)
|
||||
return _DummySocket()
|
||||
|
||||
with mock.patch("socket.create_connection", fake_create):
|
||||
conn.connect()
|
||||
self.assertEqual(dials, [(PUBLIC_IP, 443)])
|
||||
self.assertEqual(wrap_calls, ["example.com"])
|
||||
Reference in new issue
Block a user