Project repo_url: worker can run tasks from any git repo

Projects gain repo_url (validated: https:// or git@ only, no
userinfo/password, no file:// or local paths; old DBs get the
column via idempotent ALTER in both SQLite and Postgres init).
Worker resolves a project's repo_url to a local checkout under
--repos-dir (default ./repos) before dispatch: clone if absent,
verify remote origin matches and fetch if present, never force
reset/clean, never push (push needs write credentials the
control plane does not store). Resolve failure fails the task
with a clear error, never completed. Docker workers need
repos_dir under a same-path extra mount (e.g.
/var/lib/codex-app-server/repos).
Also: driver-missing test is now mocked instead of depending
on the host really having no psycopg driver installed.
Deploy package: Caddyfile template exposing only /webhooks/*
and /oauth/* (tasks/projects/admin stay 403) plus a
webhook-secret generator script that never prints the secret.
Tests: host 27 pass, cloud 286 pass (8 skipped are
environment-dependent real-Postgres tests).

cloud/tests/test_postgres_backend.py

@@ -288,7 +288,13 @@ class SchemaStatementTestCase(unittest.TestCase):
         conn = db.connect()
         init_postgres_schema(conn)
         executed = [sql for sql, _ in fake.conn.cursor_obj.executed]
-        self.assertEqual(len(executed), len(POSTGRES_SCHEMA))
+        # 改动理由：init_postgres_schema 现在还要给老库 projects 表补
+        # repo_url 列（PRAGMA 查列 + ALTER TABLE），所以执行语句数
+        # 不再精确等于 schema 语句数；断言改为“schema 语句全部执行
+        # 且补列语句也执行”，不放宽 schema 本身的覆盖要求。
+        for stmt in POSTGRES_SCHEMA:
+            self.assertIn(stmt, executed)
+        self.assertTrue(any("repo_url" in sql for sql in executed))
         self.assertTrue(fake.conn.committed)
 
 

cloud/tests/test_prod_gaps.py

@@ -16,6 +16,7 @@ import tempfile
 import threading
 import time
 import unittest
+from unittest import mock
 from pathlib import Path
 from types import SimpleNamespace
 
@@ -309,9 +310,14 @@ class MetricsAndBackupTestCase(unittest.TestCase):
             conn.close()
 
     def test_postgres_backend_without_driver_fails_before_file(self):
+        # 环境无关：用 mock 模拟驱动缺失，而不是依赖本机真的没装驱动
+        # （另一任务已在本 VM 装了 psycopg2 + Postgres，原写法在该环境下
+        # 前提不存在而失败，并非代码回归；CI runner 无驱动时原逻辑同样成立）。
         other_db = os.path.join(self.tmp.name, "pg.db")
-        with self.assertRaises(PostgresDriverMissingError):
-            create_server(other_db, "127.0.0.1", 0, backend="postgres")
+        with mock.patch("cloud.db._load_postgres_driver",
+                         side_effect=PostgresDriverMissingError("no driver")):
+            with self.assertRaises(PostgresDriverMissingError):
+                create_server(other_db, "127.0.0.1", 0, backend="postgres")
         self.assertFalse(os.path.exists(other_db))
 
 

cloud/tests/test_project_repo.py

@@ -0,0 +1,397 @@
+"""项目级 repo_url 单测：校验、API、老库迁移、resolve、worker 派发。
+
+只用标准库与假 runner/假执行器，不跑真实模型回合、不联网、
+不用真实 git clone（clone 逻辑用 mock runner 模拟 git 的副作用：
+建出目标目录 + .git；真实公开仓库 clone 列入真实环境验证清单）。
+测试里出现的 URL 与密钥均为占位值，绝无真实凭证。
+"""
+
+import http.client
+import json
+import os
+import sqlite3
+import sys
+import tempfile
+import threading
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+
+try:
+    from cloud.auth import AuthStore
+    from cloud.repos import (
+        RepoError,
+        checkout_dir,
+        resolve_repo,
+        validate_repo_url,
+    )
+    from cloud.server import create_server, init_db
+    from cloud.worker import Worker
+except ImportError:  # discover -s cloud/tests 时把项目根目录补进 sys.path
+    sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+    from cloud.auth import AuthStore
+    from cloud.repos import (
+        RepoError,
+        checkout_dir,
+        resolve_repo,
+        validate_repo_url,
+    )
+    from cloud.server import create_server, init_db
+    from cloud.worker import Worker
+
+
+def _git_runner_factory(behavior, calls, side_effect=None):
+    """behavior: dict，键为命令首段特征，值为 (returncode, stdout, stderr)。
+
+    clone 命令默认成功且模拟 git 副作用（建出目标目录 + .git）；
+    调用方可覆写任意命令的行为。
+    """
+    def runner(cmd):
+        calls.append(list(cmd))
+        if cmd[:2] == ["git", "clone"]:
+            key = "clone"
+            rc, out, err = behavior.get(
+                key, (0, "", ""))
+            if rc == 0 and side_effect is None:
+                target = cmd[-1]
+                os.makedirs(os.path.join(target, ".git"), exist_ok=True)
+            return SimpleNamespace(returncode=rc, stdout=out, stderr=err)
+        if cmd[:3] == ["git", "-C", cmd[2]] and "remote" in cmd:
+            rc, out, err = behavior.get(
+                "get-url", (0, behavior.get("origin", ""), ""))
+            return SimpleNamespace(returncode=rc, stdout=out, stderr=err)
+        if "fetch" in cmd:
+            rc, out, err = behavior.get("fetch", (0, "", ""))
+            return SimpleNamespace(returncode=rc, stdout=out, stderr=err)
+        return SimpleNamespace(returncode=0, stdout="", stderr="")
+    return runner
+
+
+class ValidateRepoUrlTestCase(unittest.TestCase):
+    def test_valid_https_and_git_scp(self):
+        self.assertEqual(
+            validate_repo_url("https://github.com/openqj/codex-app-server.git"),
+            "https://github.com/openqj/codex-app-server.git")
+        self.assertEqual(
+            validate_repo_url("git@github.com:openqj/codex-app-server.git"),
+            "git@github.com:openqj/codex-app-server.git")
+
+    def test_rejects_userinfo_file_ssh_local_empty(self):
+        bad = [
+            "https://user:pass@github.com/o/r.git",
+            "https://user@github.com/o/r.git",
+            "file:///tmp/repo",
+            "ssh://git@github.com/o/r.git",
+            "ssh://git:pass@github.com/o/r.git",
+            "/tmp/local-repo",
+            "./relative",
+            "",
+            "   ",
+            "https://",
+            "git@github.com",
+            "https://github.com/o/r.git\n",
+        ]
+        for url in bad:
+            with self.subTest(url=url):
+                with self.assertRaises(ValueError):
+                    validate_repo_url(url)
+        with self.assertRaises(ValueError):
+            validate_repo_url(None)
+        with self.assertRaises(ValueError):
+            validate_repo_url(123)
+
+
+class CheckoutDirTestCase(unittest.TestCase):
+    def test_sanitizes_and_stays_inside(self):
+        tmp = tempfile.TemporaryDirectory()
+        base = os.path.join(tmp.name, "repos")
+        path = checkout_dir(base, "abc123")
+        self.assertEqual(path, os.path.join(os.path.abspath(base), "abc123"))
+        # 穿越尝试被过滤，仍在 repos_dir 之下。
+        path2 = checkout_dir(base, "../../etc")
+        self.assertTrue(os.path.commonpath(
+            [os.path.abspath(base), path2]) == os.path.abspath(base))
+        tmp.cleanup()
+
+
+class ResolveRepoTestCase(unittest.TestCase):
+    def test_clone_when_missing(self):
+        tmp = tempfile.TemporaryDirectory()
+        calls = []
+        runner = _git_runner_factory({}, calls)
+        url = "https://github.com/openqj/codex-app-server.git"
+        path = resolve_repo(url, os.path.join(tmp.name, "repos"),
+                            "proj1", runner=runner)
+        self.assertTrue(os.path.isdir(os.path.join(path, ".git")))
+        self.assertEqual(calls[0][:3], ["git", "clone", url])
+        tmp.cleanup()
+
+    def test_clone_failure_mentions_no_stored_credentials(self):
+        tmp = tempfile.TemporaryDirectory()
+        calls = []
+        runner = _git_runner_factory(
+            {"clone": (128, "", "fatal: Authentication failed")}, calls)
+        with self.assertRaises(RepoError) as ctx:
+            resolve_repo("https://github.com/o/private.git",
+                         os.path.join(tmp.name, "repos"),
+                         "proj1", runner=runner)
+        self.assertIn("控制面不存仓库凭证", str(ctx.exception))
+        tmp.cleanup()
+
+    def test_existing_matching_origin_fetches(self):
+        tmp = tempfile.TemporaryDirectory()
+        repos = os.path.join(tmp.name, "repos")
+        target = checkout_dir(repos, "proj1")
+        os.makedirs(os.path.join(target, ".git"))
+        url = "https://github.com/openqj/codex-app-server.git"
+        calls = []
+        runner = _git_runner_factory({"origin": url}, calls)
+        path = resolve_repo(url, repos, "proj1", runner=runner)
+        self.assertEqual(path, target)
+        self.assertTrue(any("fetch" in c for c in calls))
+        tmp.cleanup()
+
+    def test_existing_mismatched_origin_refuses(self):
+        tmp = tempfile.TemporaryDirectory()
+        repos = os.path.join(tmp.name, "repos")
+        target = checkout_dir(repos, "proj1")
+        os.makedirs(os.path.join(target, ".git"))
+        calls = []
+        runner = _git_runner_factory(
+            {"origin": "https://github.com/other/repo.git"}, calls)
+        with self.assertRaises(RepoError) as ctx:
+            resolve_repo("https://github.com/openqj/codex-app-server.git",
+                         repos, "proj1", runner=runner)
+        self.assertIn("不一致", str(ctx.exception))
+        tmp.cleanup()
+
+    def test_existing_not_git_refuses(self):
+        tmp = tempfile.TemporaryDirectory()
+        repos = os.path.join(tmp.name, "repos")
+        os.makedirs(checkout_dir(repos, "proj1"))
+        calls = []
+        runner = _git_runner_factory({}, calls)
+        with self.assertRaises(RepoError):
+            resolve_repo("https://github.com/o/r.git",
+                         repos, "proj1", runner=runner)
+        self.assertEqual(calls, [])
+        tmp.cleanup()
+
+    def test_fetch_failure_refuses(self):
+        tmp = tempfile.TemporaryDirectory()
+        repos = os.path.join(tmp.name, "repos")
+        target = checkout_dir(repos, "proj1")
+        os.makedirs(os.path.join(target, ".git"))
+        url = "https://github.com/o/r.git"
+        calls = []
+        runner = _git_runner_factory(
+            {"origin": url, "fetch": (128, "", "fatal: unable to access")},
+            calls)
+        with self.assertRaises(RepoError) as ctx:
+            resolve_repo(url, repos, "proj1", runner=runner)
+        self.assertIn("fetch", str(ctx.exception))
+        tmp.cleanup()
+
+
+class _FakeExecutor:
+    def __init__(self):
+        self.seen = []
+
+    def _host_name(self, task):
+        return task["id"]
+
+    def dispatch_task(self, task):
+        self.seen.append(dict(task))
+        return {"executor": "local", "task_id": task["id"],
+                "status": "completed", "stdout": "ok", "returncode": 0,
+                "host_status": None}
+
+
+def _make_worker_db(tmp, project_repo_url="__unset__"):
+    """建库 + 项目行 + queued 任务行，返回 (db_path, project_id)。"""
+    db_path = os.path.join(tmp, "control.db")
+    init_db(db_path)
+    project_id = "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6"
+    conn = sqlite3.connect(db_path)
+    try:
+        if project_repo_url == "__unset__":
+            conn.execute(
+                "INSERT INTO projects(id, name, owner_id, created_at)"
+                " VALUES(?, 'p', 'owner', 0)", (project_id,))
+        else:
+            conn.execute(
+                "INSERT INTO projects(id, name, owner_id, created_at,"
+                " repo_url) VALUES(?, 'p', 'owner', 0, ?)",
+                (project_id, project_repo_url))
+        conn.execute(
+            "INSERT INTO tasks(id, project_id, prompt, status,"
+            " depends_on, priority, created_by, created_at, updated_at,"
+            " last_heartbeat) VALUES('t1', ?, 'do it', 'queued',"
+            " '[]', 0, 'system', 0, 0, NULL)", (project_id,))
+        conn.commit()
+    finally:
+        conn.close()
+    return db_path, project_id
+
+
+class WorkerProjectRepoTestCase(unittest.TestCase):
+    def test_dispatch_uses_project_repo(self):
+        tmp = tempfile.TemporaryDirectory()
+        url = "https://github.com/openqj/codex-app-server.git"
+        db_path, _ = _make_worker_db(tmp.name, url)
+        repos = os.path.join(tmp.name, "repos")
+        calls = []
+        executor = _FakeExecutor()
+        worker = Worker(db_path, "host.py", repo=None,
+                        env={"XCM_API_KEY": "test-placeholder"},
+                        env_key=None, executor=executor,
+                        repos_dir=repos,
+                        repos_runner=_git_runner_factory({}, calls),
+                        retry_backoff_seconds=0)
+        result = worker.run_one()
+        self.assertEqual(result["status"], "completed")
+        self.assertEqual(len(executor.seen), 1)
+        self.assertEqual(executor.seen[0].get("repo"),
+                         checkout_dir(repos,
+                                      "a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6"))
+        conn = sqlite3.connect(db_path)
+        try:
+            status = conn.execute(
+                "SELECT status FROM tasks WHERE id='t1'").fetchone()[0]
+        finally:
+            conn.close()
+        self.assertEqual(status, "completed")
+        tmp.cleanup()
+
+    def test_no_repo_url_uses_default(self):
+        tmp = tempfile.TemporaryDirectory()
+        db_path, _ = _make_worker_db(tmp.name, "__unset__")
+        executor = _FakeExecutor()
+        worker = Worker(db_path, "host.py", repo="/default/repo",
+                        env={"XCM_API_KEY": "test-placeholder"},
+                        env_key=None, executor=executor,
+                        repos_dir=os.path.join(tmp.name, "repos"),
+                        retry_backoff_seconds=0)
+        result = worker.run_one()
+        self.assertEqual(result["status"], "completed")
+        # task 本身无 repo 字段：执行器走自身默认 repo，不被改写。
+        self.assertNotIn("repo", executor.seen[0])
+        tmp.cleanup()
+
+    def test_repo_resolve_failure_is_failed_not_completed(self):
+        tmp = tempfile.TemporaryDirectory()
+        url = "https://github.com/o/private.git"
+        db_path, _ = _make_worker_db(tmp.name, url)
+        executor = _FakeExecutor()
+        worker = Worker(db_path, "host.py", repo=None,
+                        env={"XCM_API_KEY": "test-placeholder"},
+                        env_key=None, executor=executor,
+                        repos_dir=os.path.join(tmp.name, "repos"),
+                        repos_runner=_git_runner_factory(
+                            {"clone": (128, "",
+                                       "fatal: Authentication failed")},
+                            []),
+                        retry_backoff_seconds=0)
+        result = worker.run_one()
+        self.assertEqual(result["status"], "failed")
+        self.assertIn("仓库解析失败", result["error"])
+        self.assertEqual(executor.seen, [])
+        conn = sqlite3.connect(db_path)
+        try:
+            status, error = conn.execute(
+                "SELECT status, error_text FROM tasks WHERE id='t1'"
+            ).fetchone()
+        finally:
+            conn.close()
+        self.assertEqual(status, "failed")
+        self.assertIn("仓库解析失败", error)
+        tmp.cleanup()
+
+
+class ProjectRepoApiTestCase(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.db_path = os.path.join(self.tmp.name, "control.db")
+        # 老库：projects 表不含 repo_url 列，init_db 必须幂等补列。
+        conn = sqlite3.connect(self.db_path)
+        conn.execute(
+            "CREATE TABLE projects(id TEXT PRIMARY KEY, name TEXT,"
+            " owner_id TEXT, created_at REAL)")
+        conn.execute(
+            "CREATE TABLE tasks(id TEXT PRIMARY KEY, project_id TEXT,"
+            " prompt TEXT, status TEXT, idempotency_key TEXT,"
+            " depends_on TEXT, priority INTEGER, created_by TEXT,"
+            " created_at REAL, updated_at REAL, last_heartbeat REAL,"
+            " UNIQUE(project_id, idempotency_key))")
+        conn.commit()
+        conn.close()
+        auth = AuthStore(self.db_path)
+        self.owner_id, self.owner_key = auth.create_user("owner")
+        auth.conn.close()
+        self.server = create_server(self.db_path, "127.0.0.1", 0)
+        self.port = self.server.server_address[1]
+        self.thread = threading.Thread(
+            target=self.server.serve_forever, daemon=True)
+        self.thread.start()
+
+    def tearDown(self):
+        self.server.shutdown()
+        self.server.server_close()
+        self.thread.join(timeout=5)
+        self.tmp.cleanup()
+
+    def _req(self, method, path, body=None, key=None):
+        conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
+        payload = json.dumps(body).encode("utf-8") if body is not None else None
+        headers = {"Content-Type": "application/json"}
+        if key is not None:
+            headers["X-API-Key"] = key
+        try:
+            conn.request(method, path, body=payload, headers=headers)
+            resp = conn.getresponse()
+            return resp.status, json.loads(resp.read() or b"{}")
+        finally:
+            conn.close()
+
+    def test_old_db_migrated_and_create_with_repo_url(self):
+        conn = sqlite3.connect(self.db_path)
+        cols = {r[1] for r in conn.execute(
+            "PRAGMA table_info(projects)").fetchall()}
+        conn.close()
+        self.assertIn("repo_url", cols)
+        status, project = self._req(
+            "POST", "/projects",
+            {"name": "p1",
+             "repo_url": "https://github.com/openqj/codex-app-server.git"},
+            self.owner_key)
+        self.assertEqual(status, 201, project)
+        self.assertEqual(
+            project["repo_url"],
+            "https://github.com/openqj/codex-app-server.git")
+        status, listed = self._req("GET", "/projects", key=self.owner_key)
+        self.assertEqual(status, 200)
+        self.assertEqual(len(listed["projects"]), 1)
+        self.assertEqual(
+            listed["projects"][0]["repo_url"],
+            "https://github.com/openqj/codex-app-server.git")
+
+    def test_create_without_repo_url_is_none(self):
+        status, project = self._req(
+            "POST", "/projects", {"name": "p2"}, self.owner_key)
+        self.assertEqual(status, 201, project)
+        self.assertIsNone(project["repo_url"])
+
+    def test_create_with_bad_repo_url_is_400_and_not_created(self):
+        for bad in ("https://user:pass@github.com/o/r.git",
+                    "file:///tmp/r", "ssh://git@github.com/o/r.git",
+                    "/tmp/local"):
+            status, body = self._req(
+                "POST", "/projects",
+                {"name": "bad", "repo_url": bad}, self.owner_key)
+            self.assertEqual(status, 400, (bad, body))
+        status, listed = self._req("GET", "/projects", key=self.owner_key)
+        self.assertEqual(listed["projects"], [])
+
+
+if __name__ == "__main__":
+    unittest.main()