host
bed3fc0dff
Projects gain repo_url (validated: https:// or git@ only, no userinfo/password, no file:// or local paths; old DBs get the column via idempotent ALTER in both SQLite and Postgres init). Worker resolves a project's repo_url to a local checkout under --repos-dir (default ./repos) before dispatch: clone if absent, verify remote origin matches and fetch if present, never force reset/clean, never push (push needs write credentials the control plane does not store). Resolve failure fails the task with a clear error, never completed. Docker workers need repos_dir under a same-path extra mount (e.g. /var/lib/codex-app-server/repos). Also: driver-missing test is now mocked instead of depending on the host really having no psycopg driver installed. Deploy package: Caddyfile template exposing only /webhooks/* and /oauth/* (tasks/projects/admin stay 403) plus a webhook-secret generator script that never prints the secret. Tests: host 27 pass, cloud 286 pass (8 skipped are environment-dependent real-Postgres tests).
398 lines
16 KiB
Python
398 lines
16 KiB
Python
"""项目级 repo_url 单测：校验、API、老库迁移、resolve、worker 派发。
|
||
|
||
只用标准库与假 runner/假执行器，不跑真实模型回合、不联网、
|
||
不用真实 git clone（clone 逻辑用 mock runner 模拟 git 的副作用：
|
||
建出目标目录 + .git；真实公开仓库 clone 列入真实环境验证清单）。
|
||
测试里出现的 URL 与密钥均为占位值，绝无真实凭证。
|
||
"""
|
||
|
||
import http.client
|
||
import json
|
||
import os
|
||
import sqlite3
|
||
import sys
|
||
import tempfile
|
||
import threading
|
||
import unittest
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
try:
|
||
from cloud.auth import AuthStore
|
||
from cloud.repos import (
|
||
RepoError,
|
||
checkout_dir,
|
||
resolve_repo,
|
||
validate_repo_url,
|
||
)
|
||
from cloud.server import create_server, init_db
|
||
from cloud.worker import Worker
|
||
except ImportError: # discover -s cloud/tests 时把项目根目录补进 sys.path
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from cloud.auth import AuthStore
|
||
from cloud.repos import (
|
||
RepoError,
|
||
checkout_dir,
|
||
resolve_repo,
|
||
validate_repo_url,
|
||
)
|
||
from cloud.server import create_server, init_db
|
||
from cloud.worker import Worker
|
||
|
||
|
||
def _git_runner_factory(behavior, calls, side_effect=None):
|
||
"""behavior: dict，键为命令首段特征，值为 (returncode, stdout, stderr)。
|
||
|
||
clone 命令默认成功且模拟 git 副作用（建出目标目录 + .git）；
|
||
调用方可覆写任意命令的行为。
|
||
"""
|
||
def runner(cmd):
|
||
calls.append(list(cmd))
|
||
if cmd[:2] == ["git", "clone"]:
|
||
key = "clone"
|
||
rc, out, err = behavior.get(
|
||
key, (0, "", ""))
|
||
if rc == 0 and side_effect is None:
|
||
target = cmd[-1]
|
||
os.makedirs(os.path.join(target, ".git"), exist_ok=True)
|
||
return SimpleNamespace(returncode=rc, stdout=out, stderr=err)
|
||
if cmd[:3] == ["git", "-C", cmd[2]] and "remote" in cmd:
|
||
rc, out, err = behavior.get(
|
||
"get-url", (0, behavior.get("origin", ""), ""))
|
||
return SimpleNamespace(returncode=rc, stdout=out, stderr=err)
|
||
if "fetch" in cmd:
|
||
rc, out, err = behavior.get("fetch", (0, "", ""))
|
||
return SimpleNamespace(returncode=rc, stdout=out, stderr=err)
|
||
return SimpleNamespace(returncode=0, stdout="", stderr="")
|
||
return runner
|
||
|
||
|
||
class ValidateRepoUrlTestCase(unittest.TestCase):
|
||
def test_valid_https_and_git_scp(self):
|
||
self.assertEqual(
|
||
validate_repo_url("https://github.com/openqj/codex-app-server.git"),
|
||
"https://github.com/openqj/codex-app-server.git")
|
||
self.assertEqual(
|
||
validate_repo_url("git@github.com:openqj/codex-app-server.git"),
|
||
"git@github.com:openqj/codex-app-server.git")
|
||
|
||
def test_rejects_userinfo_file_ssh_local_empty(self):
|
||
bad = [
|
||
"https://user:pass@github.com/o/r.git",
|
||
"https://user@github.com/o/r.git",
|
||
"file:///tmp/repo",
|
||
"ssh://git@github.com/o/r.git",
|
||
"ssh://git:pass@github.com/o/r.git",
|
||
"/tmp/local-repo",
|
||
"./relative",
|
||
"",
|
||
" ",
|
||
"https://",
|
||
"git@github.com",
|
||
"https://github.com/o/r.git\n",
|
||
]
|
||
for url in bad:
|
||
with self.subTest(url=url):
|
||
with self.assertRaises(ValueError):
|
||
validate_repo_url(url)
|
||
with self.assertRaises(ValueError):
|
||
validate_repo_url(None)
|
||
with self.assertRaises(ValueError):
|
||
validate_repo_url(123)
|
||
|
||
|
||
class CheckoutDirTestCase(unittest.TestCase):
|
||
def test_sanitizes_and_stays_inside(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
base = os.path.join(tmp.name, "repos")
|
||
path = checkout_dir(base, "abc123")
|
||
self.assertEqual(path, os.path.join(os.path.abspath(base), "abc123"))
|
||
# 穿越尝试被过滤，仍在 repos_dir 之下。
|
||
path2 = checkout_dir(base, "../../etc")
|
||
self.assertTrue(os.path.commonpath(
|
||
[os.path.abspath(base), path2]) == os.path.abspath(base))
|
||
tmp.cleanup()
|
||
|
||
|
||
class ResolveRepoTestCase(unittest.TestCase):
|
||
def test_clone_when_missing(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
calls = []
|
||
runner = _git_runner_factory({}, calls)
|
||
url = "https://github.com/openqj/codex-app-server.git"
|
||
path = resolve_repo(url, os.path.join(tmp.name, "repos"),
|
||
"proj1", runner=runner)
|
||
self.assertTrue(os.path.isdir(os.path.join(path, ".git")))
|
||
self.assertEqual(calls[0][:3], ["git", "clone", url])
|
||
tmp.cleanup()
|
||
|
||
def test_clone_failure_mentions_no_stored_credentials(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
calls = []
|
||
runner = _git_runner_factory(
|
||
{"clone": (128, "", "fatal: Authentication failed")}, calls)
|
||
with self.assertRaises(RepoError) as ctx:
|
||
resolve_repo("https://github.com/o/private.git",
|
||
os.path.join(tmp.name, "repos"),
|
||
"proj1", runner=runner)
|
||
self.assertIn("控制面不存仓库凭证", str(ctx.exception))
|
||
tmp.cleanup()
|
||
|
||
def test_existing_matching_origin_fetches(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
repos = os.path.join(tmp.name, "repos")
|
||
target = checkout_dir(repos, "proj1")
|
||
os.makedirs(os.path.join(target, ".git"))
|
||
url = "https://github.com/openqj/codex-app-server.git"
|
||
calls = []
|
||
runner = _git_runner_factory({"origin": url}, calls)
|
||
path = resolve_repo(url, repos, "proj1", runner=runner)
|
||
self.assertEqual(path, target)
|
||
self.assertTrue(any("fetch" in c for c in calls))
|
||
tmp.cleanup()
|
||
|
||
def test_existing_mismatched_origin_refuses(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
repos = os.path.join(tmp.name, "repos")
|
||
target = checkout_dir(repos, "proj1")
|
||
os.makedirs(os.path.join(target, ".git"))
|
||
calls = []
|
||
runner = _git_runner_factory(
|
||
{"origin": "https://github.com/other/repo.git"}, calls)
|
||
with self.assertRaises(RepoError) as ctx:
|
||
resolve_repo("https://github.com/openqj/codex-app-server.git",
|
||
repos, "proj1", runner=runner)
|
||
self.assertIn("不一致", str(ctx.exception))
|
||
tmp.cleanup()
|
||
|
||
def test_existing_not_git_refuses(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
repos = os.path.join(tmp.name, "repos")
|
||
os.makedirs(checkout_dir(repos, "proj1"))
|
||
calls = []
|
||
runner = _git_runner_factory({}, calls)
|
||
with self.assertRaises(RepoError):
|
||
resolve_repo("https://github.com/o/r.git",
|
||
repos, "proj1", runner=runner)
|
||
self.assertEqual(calls, [])
|
||
tmp.cleanup()
|
||
|
||
def test_fetch_failure_refuses(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
repos = os.path.join(tmp.name, "repos")
|
||
target = checkout_dir(repos, "proj1")
|
||
os.makedirs(os.path.join(target, ".git"))
|
||
url = "https://github.com/o/r.git"
|
||
calls = []
|
||
runner = _git_runner_factory(
|
||
{"origin": url, "fetch": (128, "", "fatal: unable to access")},
|
||
calls)
|
||
with self.assertRaises(RepoError) as ctx:
|
||
resolve_repo(url, repos, "proj1", runner=runner)
|
||
self.assertIn("fetch", str(ctx.exception))
|
||
tmp.cleanup()
|
||
|
||
|
||
class _FakeExecutor:
|
||
def __init__(self):
|
||
self.seen = []
|
||
|
||
def _host_name(self, task):
|
||
return task["id"]
|
||
|
||
def dispatch_task(self, task):
|
||
self.seen.append(dict(task))
|
||
return {"executor": "local", "task_id": task["id"],
|
||
"status": "completed", "stdout": "ok", "returncode": 0,
|
||
"host_status": None}
|
||
|
||
|
||
def _make_worker_db(tmp, project_repo_url="__unset__"):
|
||
"""建库 + 项目行 + queued 任务行，返回 (db_path, project_id)。"""
|
||
db_path = os.path.join(tmp, "control.db")
|
||
init_db(db_path)
|
||
project_id = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6"
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
if project_repo_url == "__unset__":
|
||
conn.execute(
|
||
"INSERT INTO projects(id, name, owner_id, created_at)"
|
||
" VALUES(?, 'p', 'owner', 0)", (project_id,))
|
||
else:
|
||
conn.execute(
|
||
"INSERT INTO projects(id, name, owner_id, created_at,"
|
||
" repo_url) VALUES(?, 'p', 'owner', 0, ?)",
|
||
(project_id, project_repo_url))
|
||
conn.execute(
|
||
"INSERT INTO tasks(id, project_id, prompt, status,"
|
||
" depends_on, priority, created_by, created_at, updated_at,"
|
||
" last_heartbeat) VALUES('t1', ?, 'do it', 'queued',"
|
||
" '[]', 0, 'system', 0, 0, NULL)", (project_id,))
|
||
conn.commit()
|
||
finally:
|
||
conn.close()
|
||
return db_path, project_id
|
||
|
||
|
||
class WorkerProjectRepoTestCase(unittest.TestCase):
|
||
def test_dispatch_uses_project_repo(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
url = "https://github.com/openqj/codex-app-server.git"
|
||
db_path, _ = _make_worker_db(tmp.name, url)
|
||
repos = os.path.join(tmp.name, "repos")
|
||
calls = []
|
||
executor = _FakeExecutor()
|
||
worker = Worker(db_path, "host.py", repo=None,
|
||
env={"XCM_API_KEY": "test-placeholder"},
|
||
env_key=None, executor=executor,
|
||
repos_dir=repos,
|
||
repos_runner=_git_runner_factory({}, calls),
|
||
retry_backoff_seconds=0)
|
||
result = worker.run_one()
|
||
self.assertEqual(result["status"], "completed")
|
||
self.assertEqual(len(executor.seen), 1)
|
||
self.assertEqual(executor.seen[0].get("repo"),
|
||
checkout_dir(repos,
|
||
"a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6"))
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
status = conn.execute(
|
||
"SELECT status FROM tasks WHERE id='t1'").fetchone()[0]
|
||
finally:
|
||
conn.close()
|
||
self.assertEqual(status, "completed")
|
||
tmp.cleanup()
|
||
|
||
def test_no_repo_url_uses_default(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
db_path, _ = _make_worker_db(tmp.name, "__unset__")
|
||
executor = _FakeExecutor()
|
||
worker = Worker(db_path, "host.py", repo="/default/repo",
|
||
env={"XCM_API_KEY": "test-placeholder"},
|
||
env_key=None, executor=executor,
|
||
repos_dir=os.path.join(tmp.name, "repos"),
|
||
retry_backoff_seconds=0)
|
||
result = worker.run_one()
|
||
self.assertEqual(result["status"], "completed")
|
||
# task 本身无 repo 字段：执行器走自身默认 repo，不被改写。
|
||
self.assertNotIn("repo", executor.seen[0])
|
||
tmp.cleanup()
|
||
|
||
def test_repo_resolve_failure_is_failed_not_completed(self):
|
||
tmp = tempfile.TemporaryDirectory()
|
||
url = "https://github.com/o/private.git"
|
||
db_path, _ = _make_worker_db(tmp.name, url)
|
||
executor = _FakeExecutor()
|
||
worker = Worker(db_path, "host.py", repo=None,
|
||
env={"XCM_API_KEY": "test-placeholder"},
|
||
env_key=None, executor=executor,
|
||
repos_dir=os.path.join(tmp.name, "repos"),
|
||
repos_runner=_git_runner_factory(
|
||
{"clone": (128, "",
|
||
"fatal: Authentication failed")},
|
||
[]),
|
||
retry_backoff_seconds=0)
|
||
result = worker.run_one()
|
||
self.assertEqual(result["status"], "failed")
|
||
self.assertIn("仓库解析失败", result["error"])
|
||
self.assertEqual(executor.seen, [])
|
||
conn = sqlite3.connect(db_path)
|
||
try:
|
||
status, error = conn.execute(
|
||
"SELECT status, error_text FROM tasks WHERE id='t1'"
|
||
).fetchone()
|
||
finally:
|
||
conn.close()
|
||
self.assertEqual(status, "failed")
|
||
self.assertIn("仓库解析失败", error)
|
||
tmp.cleanup()
|
||
|
||
|
||
class ProjectRepoApiTestCase(unittest.TestCase):
|
||
def setUp(self):
|
||
self.tmp = tempfile.TemporaryDirectory()
|
||
self.db_path = os.path.join(self.tmp.name, "control.db")
|
||
# 老库：projects 表不含 repo_url 列，init_db 必须幂等补列。
|
||
conn = sqlite3.connect(self.db_path)
|
||
conn.execute(
|
||
"CREATE TABLE projects(id TEXT PRIMARY KEY, name TEXT,"
|
||
" owner_id TEXT, created_at REAL)")
|
||
conn.execute(
|
||
"CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT,"
|
||
" prompt TEXT, status TEXT, idempotency_key TEXT,"
|
||
" depends_on TEXT, priority INTEGER, created_by TEXT,"
|
||
" created_at REAL, updated_at REAL, last_heartbeat REAL,"
|
||
" UNIQUE(project_id, idempotency_key))")
|
||
conn.commit()
|
||
conn.close()
|
||
auth = AuthStore(self.db_path)
|
||
self.owner_id, self.owner_key = auth.create_user("owner")
|
||
auth.conn.close()
|
||
self.server = create_server(self.db_path, "127.0.0.1", 0)
|
||
self.port = self.server.server_address[1]
|
||
self.thread = threading.Thread(
|
||
target=self.server.serve_forever, daemon=True)
|
||
self.thread.start()
|
||
|
||
def tearDown(self):
|
||
self.server.shutdown()
|
||
self.server.server_close()
|
||
self.thread.join(timeout=5)
|
||
self.tmp.cleanup()
|
||
|
||
def _req(self, method, path, body=None, key=None):
|
||
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
|
||
payload = json.dumps(body).encode("utf-8") if body is not None else None
|
||
headers = {"Content-Type": "application/json"}
|
||
if key is not None:
|
||
headers["X-API-Key"] = key
|
||
try:
|
||
conn.request(method, path, body=payload, headers=headers)
|
||
resp = conn.getresponse()
|
||
return resp.status, json.loads(resp.read() or b"{}")
|
||
finally:
|
||
conn.close()
|
||
|
||
def test_old_db_migrated_and_create_with_repo_url(self):
|
||
conn = sqlite3.connect(self.db_path)
|
||
cols = {r[1] for r in conn.execute(
|
||
"PRAGMA table_info(projects)").fetchall()}
|
||
conn.close()
|
||
self.assertIn("repo_url", cols)
|
||
status, project = self._req(
|
||
"POST", "/projects",
|
||
{"name": "p1",
|
||
"repo_url": "https://github.com/openqj/codex-app-server.git"},
|
||
self.owner_key)
|
||
self.assertEqual(status, 201, project)
|
||
self.assertEqual(
|
||
project["repo_url"],
|
||
"https://github.com/openqj/codex-app-server.git")
|
||
status, listed = self._req("GET", "/projects", key=self.owner_key)
|
||
self.assertEqual(status, 200)
|
||
self.assertEqual(len(listed["projects"]), 1)
|
||
self.assertEqual(
|
||
listed["projects"][0]["repo_url"],
|
||
"https://github.com/openqj/codex-app-server.git")
|
||
|
||
def test_create_without_repo_url_is_none(self):
|
||
status, project = self._req(
|
||
"POST", "/projects", {"name": "p2"}, self.owner_key)
|
||
self.assertEqual(status, 201, project)
|
||
self.assertIsNone(project["repo_url"])
|
||
|
||
def test_create_with_bad_repo_url_is_400_and_not_created(self):
|
||
for bad in ("https://user:pass@github.com/o/r.git",
|
||
"file:///tmp/r", "ssh://git@github.com/o/r.git",
|
||
"/tmp/local"):
|
||
status, body = self._req(
|
||
"POST", "/projects",
|
||
{"name": "bad", "repo_url": bad}, self.owner_key)
|
||
self.assertEqual(status, 400, (bad, body))
|
||
status, listed = self._req("GET", "/projects", key=self.owner_key)
|
||
self.assertEqual(listed["projects"], [])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|