Merge pull request #108 from openqj/feat-auto-memory

云端身份记忆 + 自动提取注入:官方 memories 对齐(确定性 v1)
This commit is contained in:
openqj authored and GitHub committed 2026-10-28 12:19:17 +08:00
commit d129f7bf8c
10 files changed
+2313

No files matched your search

+550
View File
@@ -0,0 +1,550 @@
"""自动记忆:官方 Codex memories 的云端对齐版(v1)。
官方行为(learn.chatgpt.com/docs/customization/memories):
- 开启后,Codex 把先前聊天里的有用上下文转成记忆,供未来工作用;
- 只对符合条件的先前聊天做,后台生成,不在每轮结束时立刻做;
- 聊天闲置够久才生成,避免总结仍在进行的工作;
- 剩余额度低于阈值时跳过本次生成,避免在额度见底时烧钱;
- 记忆存本地文件(~/.codex/memories/),有 generate / use /
disable_on_external_context / min_rate_limit_remaining_percent /
extract_model / consolidation_model 等设置;
- 密钥要脱敏,AGENTS.md 这类硬性规则不靠记忆。
云端对齐方式(本模块只做控制面数据库层,不动 host.py、
不改 CODEX_HOME、不改任何 thread 的创建/续跑/fork/归档):
- generate_memories:完成任务是否可被提取成记忆。默认关,
与官方“本地记忆默认关”一致;用户在
PUT /users/me/auto-memory-settings 里显式打开才生效。
- use_memories:新任务首轮是否注入记忆。注入只改 worker 内存
里的任务副本(prompt 前加一段记忆前缀),库里的原 prompt
不动;续跑、断点续跑、协调员任务不注入(它们已有会话或
本身不跑模型回合)。
- disable_on_external_context:默认开。v1 能可靠识别的“外部
上下文”只有两类:任务间消息(task_followups 里有该任务的
行)与协调员子任务(parent_task_id 非空)。host 线程内的
MCP / web search 调用目前不在 tasks 表上留标记,本模块
诚实地不宣称能识别它们,见 docs/auto-memory.md 的未做。
- min_rate_limit_remaining_percent:官方的剩余额度门,云端
用当月 USD 配额剩余百分比对齐(与 worker 配额口径一致:
无配额行按默认 $10 计,只有显式 NULL 行才是不限额)。
提取本身是确定性文本处理、不调模型,但被提取的记忆随后
会被注入并占用后续任务的 prompt token,所以额度门仍在
提取处执行;查不到配额时 fail-closed(跳过并记
skipped_quota_check),不冒充“额度充足”。
- idle_seconds:任务终态后闲置多久才可提取,默认 900 秒。
终态后仍可被 /continue 续跑,所以“刚完成”不等于“已闲置”。
- 提取内容(v1 确定性版,不调模型):从任务 goal/prompt 首行
与 result_text 前段拼一条 project 级 fact,脱敏后写入
identity_memories。extract_model / consolidation_model 只
存设置、不调模型——调模型提取是下一步显式决策(模型选择
与成本上限待定),v1 不假装已经调了。
- 去重与合并(consolidation 的确定性版):同一任务只提取一次
(memory_extraction_log + 记忆行的 source_thread_id 记任务
id);同一目标的新结果出现时,supersede 旧记忆而不是再
堆一条。自动提取只写 project 级记忆:单个项目的任务结论
若自动写成 user 级,会跨项目泄漏项目上下文,所以 user 级
记忆仍只接受手动创建。
- 任务级覆盖:POST /tasks/{id}/auto-memory-flags 可为单个任务
单独开关 use/generate(NULL = 跟随用户设置),不改 tasks
表结构(建任务的 4 条 INSERT 路径都不用动)。
新表一律 TEXT 主键,不走 IDENTITY 序列,见
cloud/migrations.py 的对齐说明;PG DDL 同步进
cloud/db.py POSTGRES_SCHEMA 与 MIGRATION_TABLES。
"""
import os
import re
import time
from cloud.db import connect as _db_connect
from cloud.identity import IdentityStore
DEFAULT_SETTINGS = {
"generate_memories": 0,
"use_memories": 0,
"disable_on_external_context": 1,
"min_rate_limit_remaining_percent": 20,
"idle_seconds": 900,
"min_duration_seconds": 30,
"min_result_chars": 200,
"extract_model": "",
"consolidation_model": "",
}
_BOOL_FIELDS = ("generate_memories", "use_memories",
"disable_on_external_context")
_INT_FIELDS = {
"min_rate_limit_remaining_percent": (0, 100),
"idle_seconds": (0, 86400),
"min_duration_seconds": (0, 86400),
"min_result_chars": (0, 10000),
}
_STR_FIELDS = ("extract_model", "consolidation_model")
# 脱敏:只处理“看起来像凭证”的形态,不把普通长 token(如
# git sha)一律抹掉。key=value 形态、Authorization/Bearer 头、
# 常见凭证前缀(sk- / ghp_ / gho_ / xoxb- / xoxp-)进黑名单。
_REDACT_PATTERNS = [
(re.compile(
r"(?i)(api[_-]?key|password|passwd|secret|token|credential)"
r"(\s*[:=]\s*)([\"']?)([^\s\"',;]+)"),
r"\1\2\3[REDACTED]"),
(re.compile(r"(?i)(authorization\s*:\s*)(Bearer\s+)?[^\s]+"),
r"\1[REDACTED]"),
(re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._\-]+"),
"Bearer [REDACTED]"),
(re.compile(r"\b(sk-[A-Za-z0-9_\-]{8,}|ghp_[A-Za-z0-9]{8,}|"
r"gho_[A-Za-z0-9]{8,}|xox[baprs]-[A-Za-z0-9\-]{8,})"
r"\b"),
"[REDACTED]"),
]
def redact_text(text):
"""脱敏并清控制字符;非字符串返回空串。"""
if not isinstance(text, str):
return ""
out = text
for pattern, repl in _REDACT_PATTERNS:
out = pattern.sub(repl, out)
out = "".join(c for c in out if ord(c) >= 32 or c in ("\n", "\t"))
return out.strip()
def _default_quota_usd_micros():
"""与 worker/server 同款默认配额解析,避免循环 import。"""
try:
return max(0, int(os.environ.get(
"CLOUD_DEFAULT_QUOTA_USD_MICROS", "10000000")))
except (TypeError, ValueError):
return 10000000
def quota_remaining_percent(db_path, owner):
"""返回当月配额剩余百分比;不限额返回 None。
查库失败抛异常(调用方 fail-closed),全新库无成本行时
已用按 0 计(那是真没有花费,不是查失败)。
"""
import datetime
month = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m")
conn = _db_connect(db_path)
try:
conn.execute(
"CREATE TABLE IF NOT EXISTS user_quotas("
"user_id TEXT PRIMARY KEY, quota_usd_micros BIGINT, "
"updated_at DOUBLE PRECISION)")
conn.commit()
row = conn.execute(
"SELECT quota_usd_micros FROM user_quotas WHERE user_id = ?",
(owner,)).fetchone()
quota = _default_quota_usd_micros() if row is None else row[0]
if quota is None:
return None
if quota <= 0:
return 0.0
try:
used_row = conn.execute(
"SELECT COALESCE(SUM(c.usd_micros), 0)"
" FROM cost_records c JOIN tasks t"
" ON c.task_id = t.id"
" WHERE t.created_by = ? AND c.day LIKE ?",
(owner, month + "%")).fetchone()
used = used_row[0] if used_row else 0
except Exception as exc:
if "no such table" in str(exc):
used = 0
else:
raise
return max(0.0, min(100.0, (quota - int(used or 0)) * 100.0 / quota))
finally:
conn.close()
def _row_get(row, name, default=None):
try:
return row[name]
except (KeyError, IndexError, TypeError):
return default
def _first_line(text, max_len=200):
for line in (text or "").splitlines():
line = " ".join(line.split())
if line:
return redact_text(line)[:max_len]
return ""
def build_extraction_content(task, result_text):
"""从任务拼一条可存的记忆正文;拼不出返回空串。"""
goal = _first_line(task.get("goal") or task.get("prompt") or "")
if not goal:
return ""
outcome = " ".join((result_text or "").split())
outcome = redact_text(outcome)[:800]
if outcome:
content = "任务完成:%s;结果:%s" % (goal, outcome)
else:
content = "任务完成:%s" % goal
return content[:3900]
class AutoMemoryStore:
"""记忆设置、任务级覆盖、提取日志(SQLite 或 Postgres)。"""
def __init__(self, db_path):
self.db_path = db_path
self.conn = _db_connect(db_path)
self.conn.execute(
"CREATE TABLE IF NOT EXISTS memory_settings("
"user_id TEXT PRIMARY KEY, "
"generate_memories INTEGER DEFAULT 0, "
"use_memories INTEGER DEFAULT 0, "
"disable_on_external_context INTEGER DEFAULT 1, "
"min_rate_limit_remaining_percent INTEGER DEFAULT 20, "
"idle_seconds INTEGER DEFAULT 900, "
"min_duration_seconds REAL DEFAULT 30, "
"min_result_chars INTEGER DEFAULT 200, "
"extract_model TEXT, "
"consolidation_model TEXT, "
"updated_at REAL)"
)
self.conn.execute(
"CREATE TABLE IF NOT EXISTS task_memory_flags("
"task_id TEXT PRIMARY KEY, "
"use_memories INTEGER, "
"generate_memories INTEGER, "
"updated_at REAL)"
)
self.conn.execute(
"CREATE TABLE IF NOT EXISTS memory_extraction_log("
"task_id TEXT PRIMARY KEY, "
"owner_user_id TEXT, "
"project_id TEXT, "
"status TEXT, "
"reason TEXT, "
"memory_id TEXT, "
"created_at REAL, "
"processed_at REAL)"
)
self.conn.commit()
def close(self):
try:
self.conn.close()
except Exception:
pass
def get_settings(self, user_id):
row = self.conn.execute(
"SELECT * FROM memory_settings WHERE user_id = ?",
(user_id,)).fetchone()
settings = dict(DEFAULT_SETTINGS)
if row is not None:
d = dict(row)
for key in DEFAULT_SETTINGS:
if d.get(key) is not None:
settings[key] = d[key]
settings["user_id"] = user_id
return settings
def upsert_settings(self, user_id, **kwargs):
"""只改给出的字段;非法值抛 ValueError,不落库。"""
clean = {}
for key, value in kwargs.items():
if key in _BOOL_FIELDS:
if value not in (True, False, 0, 1):
raise ValueError("%s must be a boolean" % key)
clean[key] = 1 if value else 0
elif key in _INT_FIELDS:
lo, hi = _INT_FIELDS[key]
try:
ivalue = int(value)
except (TypeError, ValueError):
raise ValueError("%s must be an integer" % key)
if isinstance(value, bool) or not lo <= ivalue <= hi:
raise ValueError(
"%s must be between %d and %d" % (key, lo, hi))
clean[key] = ivalue
elif key in _STR_FIELDS:
if value is None:
clean[key] = ""
elif not isinstance(value, str) or len(value) > 128:
raise ValueError("%s must be a short string" % key)
else:
clean[key] = value.strip()
else:
raise ValueError("unknown setting: %s" % key)
existing = self.get_settings(user_id)
merged = dict(existing)
merged.update(clean)
now = time.time()
self.conn.execute(
"INSERT OR REPLACE INTO memory_settings("
"user_id, generate_memories, use_memories, "
"disable_on_external_context, "
"min_rate_limit_remaining_percent, idle_seconds, "
"min_duration_seconds, min_result_chars, extract_model, "
"consolidation_model, updated_at) "
"VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(user_id, merged["generate_memories"], merged["use_memories"],
merged["disable_on_external_context"],
merged["min_rate_limit_remaining_percent"],
merged["idle_seconds"], merged["min_duration_seconds"],
merged["min_result_chars"], merged["extract_model"],
merged["consolidation_model"], now))
self.conn.commit()
return self.get_settings(user_id)
def get_task_flags(self, task_id):
row = self.conn.execute(
"SELECT * FROM task_memory_flags WHERE task_id = ?",
(task_id,)).fetchone()
return dict(row) if row is not None else None
def set_task_flags(self, task_id, use_memories=None,
generate_memories=None):
if use_memories is None and generate_memories is None:
raise ValueError(
"at least one of use_memories / generate_memories required")
for name, value in (("use_memories", use_memories),
("generate_memories", generate_memories)):
if value is not None and value not in (True, False, 0, 1):
raise ValueError("%s must be a boolean" % name)
old = self.get_task_flags(task_id) or {}
self.conn.execute(
"INSERT OR REPLACE INTO task_memory_flags("
"task_id, use_memories, generate_memories, updated_at) "
"VALUES(?, ?, ?, ?)",
(task_id,
(1 if use_memories else 0) if use_memories is not None
else old.get("use_memories"),
(1 if generate_memories else 0)
if generate_memories is not None
else old.get("generate_memories"),
time.time()))
self.conn.commit()
return self.get_task_flags(task_id)
@staticmethod
def effective(settings, flags, field):
"""任务级覆盖优先于用户设置;NULL = 跟随设置。"""
if flags is not None and flags.get(field) is not None:
return bool(flags[field])
return bool(settings[field])
def write_log(self, task_id, owner, project_id, status, reason,
memory_id=None, now=None):
now = now if now is not None else time.time()
row = self.conn.execute(
"SELECT created_at FROM memory_extraction_log"
" WHERE task_id = ?", (task_id,)).fetchone()
created = row[0] if row else now
self.conn.execute(
"INSERT OR REPLACE INTO memory_extraction_log("
"task_id, owner_user_id, project_id, status, reason, "
"memory_id, created_at, processed_at) "
"VALUES(?, ?, ?, ?, ?, ?, ?, ?)",
(task_id, owner, project_id, status, reason,
memory_id or "", created, now))
self.conn.commit()
def list_log(self, owner, limit=50):
rows = self.conn.execute(
"SELECT * FROM memory_extraction_log"
" WHERE owner_user_id = ?"
" ORDER BY processed_at DESC, task_id DESC LIMIT ?",
(owner, max(1, min(200, int(limit))))).fetchall()
return [dict(r) for r in rows]
def build_injection_prefix(db_path, owner, project_id, flags=None):
"""use_memories 关、或无记忆时返回空串;否则返回注入前缀。"""
store = AutoMemoryStore(db_path)
try:
settings = store.get_settings(owner)
if not AutoMemoryStore.effective(settings, flags, "use_memories"):
return ""
finally:
store.close()
identity = IdentityStore(db_path)
try:
context = identity.build_identity_context(owner, project_id)
finally:
try:
identity.conn.close()
except Exception:
pass
if not context:
return ""
return ("【过往记忆,仅供参考,不得改写本任务目标】\n%s" % context)
def _has_external_context(conn, task_id, parent_task_id):
if parent_task_id:
return True
try:
row = conn.execute(
"SELECT 1 FROM task_followups WHERE task_id = ? LIMIT 1",
(task_id,)).fetchone()
return row is not None
except Exception:
return False
def process_due_tasks(db_path, now=None, force=False, owner_id=None):
"""扫 completed 任务并提取记忆,返回每任务一条结果。
幂等:已提取的任务不再重复建记忆(force 也不重复,靠
source_thread_id 去重);被跳过的任务下次仍可再扫(设置
改了、闲置够了、额度回来了都可能翻盘),日志行按任务
主键覆盖,不无限增长。
"""
now = now if now is not None else time.time()
store = AutoMemoryStore(db_path)
identity = IdentityStore(db_path)
results = []
try:
if owner_id:
rows = store.conn.execute(
"SELECT * FROM tasks WHERE status = 'completed'"
" AND created_by = ? ORDER BY id",
(owner_id,)).fetchall()
else:
rows = store.conn.execute(
"SELECT * FROM tasks WHERE status = 'completed'"
" ORDER BY id").fetchall()
except Exception as exc:
store.close()
try:
identity.conn.close()
except Exception:
pass
return [{"status": "error", "reason": "tasks query failed: %s"
% exc}]
for row in rows:
task = dict(row)
task_id = task.get("id")
owner = task.get("created_by")
project_id = task.get("project_id")
result = {"task_id": task_id, "status": "skipped",
"reason": "", "memory_id": None}
results.append(result)
if not owner or not project_id:
result["reason"] = "no_owner_or_project"
store.write_log(task_id, owner or "", project_id or "",
"skipped", result["reason"], now=now)
continue
settings = store.get_settings(owner)
flags = store.get_task_flags(task_id)
if not AutoMemoryStore.effective(
settings, flags, "generate_memories"):
result["reason"] = "generate_disabled"
store.write_log(task_id, owner, project_id, "skipped",
result["reason"], now=now)
continue
finished = _row_get(task, "finished_at") \
or _row_get(task, "updated_at") \
or _row_get(task, "created_at")
started = _row_get(task, "started_at")
if not force:
if finished is None or now - float(finished) < \
float(settings["idle_seconds"]):
result["reason"] = "idle_not_reached"
store.write_log(task_id, owner, project_id, "skipped",
result["reason"], now=now)
continue
if started is None or finished is None:
result["reason"] = "no_timing"
store.write_log(task_id, owner, project_id, "skipped",
result["reason"], now=now)
continue
duration = float(finished) - float(started)
if duration < float(settings["min_duration_seconds"]):
result["reason"] = "too_short"
store.write_log(task_id, owner, project_id, "skipped",
result["reason"], now=now)
continue
result_text = _row_get(task, "result_text") or ""
if len(result_text) < int(settings["min_result_chars"]) \
and not force:
result["reason"] = "result_too_short"
store.write_log(task_id, owner, project_id, "skipped",
result["reason"], now=now)
continue
if settings["disable_on_external_context"] and \
_has_external_context(
store.conn, task_id,
_row_get(task, "parent_task_id")):
result["reason"] = "external_context"
store.write_log(task_id, owner, project_id, "skipped",
result["reason"], now=now)
continue
min_remaining = int(
settings["min_rate_limit_remaining_percent"])
if min_remaining > 0:
try:
remaining = quota_remaining_percent(db_path, owner)
except Exception:
result["reason"] = "quota_check_failed"
store.write_log(task_id, owner, project_id, "skipped",
result["reason"], now=now)
continue
if remaining is not None and remaining < min_remaining:
result["reason"] = "quota_low"
store.write_log(task_id, owner, project_id, "skipped",
result["reason"], now=now)
continue
content = build_extraction_content(task, result_text)
if not content:
result["reason"] = "empty_content"
store.write_log(task_id, owner, project_id, "skipped",
result["reason"], now=now)
continue
existing = identity.list_project_memories(project_id)
same_source = [r for r in existing
if r.get("source_thread_id") == task_id]
if same_source:
result["status"] = "extracted"
result["reason"] = "already_extracted"
result["memory_id"] = same_source[0]["id"]
store.write_log(task_id, owner, project_id, "extracted",
result["reason"], result["memory_id"], now=now)
continue
goal_line = _first_line(task.get("goal") or task.get("prompt")
or "")
same_goal = [r for r in existing
if r.get("content", "").startswith(
"任务完成:%s;" % goal_line)] if goal_line else []
if same_goal:
_old, new_id = identity.supersede_memory(
same_goal[0]["id"], owner,
replacement_content=content)
if new_id:
try:
identity.conn.execute(
"UPDATE identity_memories SET source_thread_id = ?"
" WHERE id = ?", (task_id, new_id))
identity.conn.commit()
except Exception:
pass
result["status"] = "extracted"
result["reason"] = "consolidated"
result["memory_id"] = new_id
else:
new_id = identity.add_memory(
owner, "project", "fact", content,
project_id=project_id, source_thread_id=task_id)
result["status"] = "extracted"
result["reason"] = "created"
result["memory_id"] = new_id
store.write_log(task_id, owner, project_id, "extracted",
result["reason"], result["memory_id"], now=now)
store.close()
try:
identity.conn.close()
except Exception:
pass
return results
+49
View File
@@ -946,6 +946,55 @@ POSTGRES_SCHEMA = (
"preference TEXT NOT NULL, " "preference TEXT NOT NULL, "
"created_at DOUBLE PRECISION, " "created_at DOUBLE PRECISION, "
"UNIQUE(project_id, user_id, channel, target))", "UNIQUE(project_id, user_id, channel, target))",
# System-level identity (cloud/identity.py): TEXT primary keys,
# so no IDENTITY sequence to keep in sync in migrations.py.
# DDL must match cloud/identity.py CREATE TABLE statements.
"CREATE TABLE IF NOT EXISTS user_profiles("
"user_id TEXT PRIMARY KEY, "
"display_name TEXT, "
"timezone TEXT, "
"preferences TEXT, "
"updated_at DOUBLE PRECISION)",
"CREATE TABLE IF NOT EXISTS identity_memories("
"id TEXT PRIMARY KEY, "
"owner_user_id TEXT, "
"scope TEXT, "
"project_id TEXT, "
"kind TEXT, "
"content TEXT, "
"source_thread_id TEXT, "
"status TEXT DEFAULT 'active', "
"superseded_by TEXT, "
"created_at DOUBLE PRECISION, "
"updated_at DOUBLE PRECISION)",
# Auto memory (cloud/auto_memory.py): TEXT primary keys,
# so no IDENTITY sequence to keep in sync in migrations.py.
"CREATE TABLE IF NOT EXISTS memory_settings("
"user_id TEXT PRIMARY KEY, "
"generate_memories INTEGER DEFAULT 0, "
"use_memories INTEGER DEFAULT 0, "
"disable_on_external_context INTEGER DEFAULT 1, "
"min_rate_limit_remaining_percent INTEGER DEFAULT 20, "
"idle_seconds INTEGER DEFAULT 900, "
"min_duration_seconds DOUBLE PRECISION DEFAULT 30, "
"min_result_chars INTEGER DEFAULT 200, "
"extract_model TEXT, "
"consolidation_model TEXT, "
"updated_at DOUBLE PRECISION)",
"CREATE TABLE IF NOT EXISTS task_memory_flags("
"task_id TEXT PRIMARY KEY, "
"use_memories INTEGER, "
"generate_memories INTEGER, "
"updated_at DOUBLE PRECISION)",
"CREATE TABLE IF NOT EXISTS memory_extraction_log("
"task_id TEXT PRIMARY KEY, "
"owner_user_id TEXT, "
"project_id TEXT, "
"status TEXT, "
"reason TEXT, "
"memory_id TEXT, "
"created_at DOUBLE PRECISION, "
"processed_at DOUBLE PRECISION)",
) )
+301
View File
@@ -0,0 +1,301 @@
"""System-level identity and memory, cloud layer only.
Design (deliberately NOT touching Codex threads):
- Identity lives in the control-plane database, not in CODEX_HOME.
A Codex thread is never rewritten by this module: no thread_id is
created, resumed, forked or archived here, and no prompt is sent to
any worker. ``build_identity_context`` only *formats* text for a
caller that explicitly asks for it; whether that text ever reaches
a new turn is the caller's decision, recorded separately.
- Two scopes, no cross-user leakage by construction:
- ``user``: owned by one user, visible only to that user, across
projects (project_id must be empty).
- ``project``: owned by a user but visible to project members;
the store does NOT know project membership, so the server must
check project ``read``/``create_task`` permission before calling
the project-scope methods. Project-scope ownership checks in
this module only guard the ``user`` scope; project membership
is enforced one layer up.
- Supersede is a chain, not an overwrite: the old row becomes
``superseded`` and points at its replacement (``superseded_by``),
so a correction stays auditable. ``delete`` is physical removal,
reserved for user-initiated forgetting, mirroring MemoryStore.
- Profiles never hold secrets: preference keys containing password /
secret / token / api_key / credential are rejected, and values are
capped, so a profile cannot become a second credential store.
Tables use TEXT primary keys on purpose: no IDENTITY sequence to
keep in sync in cloud/migrations.py SEQUENCE_TABLES (see the
test_p2_migrations alignment test). The same CREATE TABLE statements
must appear verbatim in cloud/db.py POSTGRES_SCHEMA.
"""
import json
import time
import uuid
from cloud.db import connect as _db_connect
_SCOPES = ("user", "project")
_KINDS = ("fact", "preference", "decision", "convention")
_SECRET_KEY_WORDS = ("password", "secret", "token", "api_key", "apikey",
"private_key", "credential")
_MAX_CONTENT = 4000
_MAX_PREFERENCES_BYTES = 8192
def _clean_str(value, max_len, field):
if not isinstance(value, str):
raise ValueError("%s must be a string" % field)
value = value.strip()
if not value:
raise ValueError("%s required" % field)
if len(value) > max_len:
raise ValueError("%s too long" % field)
if any(ord(c) < 32 and c not in ("\n", "\t") for c in value):
raise ValueError("%s contains control characters" % field)
return value
def _validate_preferences(preferences):
if preferences is None:
return None
if not isinstance(preferences, dict):
raise ValueError("preferences must be an object")
for key in preferences:
if not isinstance(key, str):
raise ValueError("preferences keys must be strings")
lowered = key.lower()
if any(word in lowered for word in _SECRET_KEY_WORDS):
raise ValueError(
"preferences must not hold secrets: %s" % key)
try:
encoded = json.dumps(preferences, ensure_ascii=False)
except (TypeError, ValueError):
raise ValueError("preferences must be JSON-serializable")
if len(encoded.encode("utf-8")) > _MAX_PREFERENCES_BYTES:
raise ValueError("preferences too large")
return preferences
class IdentityStore:
"""User profiles + scoped identity memories (SQLite or Postgres).
Instantiation creates the tables if missing (idempotent), the
same pattern as MemoryStore / KBStore.
"""
def __init__(self, db_path):
self.db_path = db_path
self.conn = _db_connect(db_path)
self.conn.execute(
"CREATE TABLE IF NOT EXISTS user_profiles("
"user_id TEXT PRIMARY KEY, "
"display_name TEXT, "
"timezone TEXT, "
"preferences TEXT, "
"updated_at REAL)"
)
self.conn.execute(
"CREATE TABLE IF NOT EXISTS identity_memories("
"id TEXT PRIMARY KEY, "
"owner_user_id TEXT, "
"scope TEXT, "
"project_id TEXT, "
"kind TEXT, "
"content TEXT, "
"source_thread_id TEXT, "
"status TEXT DEFAULT 'active', "
"superseded_by TEXT, "
"created_at REAL, "
"updated_at REAL)"
)
self.conn.commit()
# -- profiles ------------------------------------------------------
def get_profile(self, user_id):
row = self.conn.execute(
"SELECT * FROM user_profiles WHERE user_id = ?",
(user_id,)).fetchone()
if row is None:
return {"user_id": user_id, "display_name": "",
"timezone": "", "preferences": {},
"updated_at": None}
d = dict(row)
try:
d["preferences"] = json.loads(d["preferences"]) \
if d["preferences"] else {}
except (TypeError, ValueError):
d["preferences"] = {}
return d
def upsert_profile(self, user_id, display_name=None, timezone=None,
preferences=None):
"""Partial update: only fields that are not None change."""
if display_name is not None:
display_name = _clean_str(display_name, 100, "display_name")
if timezone is not None:
timezone = _clean_str(timezone, 64, "timezone")
preferences = _validate_preferences(preferences)
now = time.time()
existing = self.conn.execute(
"SELECT * FROM user_profiles WHERE user_id = ?",
(user_id,)).fetchone()
if existing is None:
self.conn.execute(
"INSERT INTO user_profiles(user_id, display_name, "
"timezone, preferences, updated_at) VALUES(?, ?, ?, ?, ?)",
(user_id, display_name or "",
timezone or "",
json.dumps(preferences) if preferences is not None
else "{}",
now))
else:
row = dict(existing)
try:
old_prefs = json.loads(row["preferences"]) \
if row["preferences"] else {}
except (TypeError, ValueError):
old_prefs = {}
self.conn.execute(
"UPDATE user_profiles SET display_name = ?, "
"timezone = ?, preferences = ?, updated_at = ? "
"WHERE user_id = ?",
(display_name if display_name is not None
else (row["display_name"] or ""),
timezone if timezone is not None
else (row["timezone"] or ""),
json.dumps(preferences) if preferences is not None
else json.dumps(old_prefs),
now, user_id))
self.conn.commit()
return self.get_profile(user_id)
# -- memories -------------------------------------------------------
def add_memory(self, owner_user_id, scope, kind, content,
project_id=None, source_thread_id=None):
if scope not in _SCOPES:
raise ValueError("scope must be 'user' or 'project'")
if kind not in _KINDS:
raise ValueError("kind must be one of %s" % (", ".join(_KINDS),))
content = _clean_str(content, _MAX_CONTENT, "content")
if scope == "user":
if project_id:
raise ValueError(
"user-scope memory must not carry a project_id")
project_id = ""
else:
if not project_id or not isinstance(project_id, str):
raise ValueError(
"project-scope memory requires a project_id")
project_id = project_id.strip()
if source_thread_id is not None:
source_thread_id = _clean_str(
source_thread_id, 128, "source_thread_id")
mem_id = uuid.uuid4().hex
now = time.time()
self.conn.execute(
"INSERT INTO identity_memories(id, owner_user_id, scope, "
"project_id, kind, content, source_thread_id, status, "
"superseded_by, created_at, updated_at) "
"VALUES(?, ?, ?, ?, ?, ?, ?, 'active', '', ?, ?)",
(mem_id, owner_user_id, scope, project_id, kind, content,
source_thread_id or "", now, now))
self.conn.commit()
return mem_id
def get_memory(self, mem_id):
row = self.conn.execute(
"SELECT * FROM identity_memories WHERE id = ?",
(mem_id,)).fetchone()
return dict(row) if row is not None else None
def list_user_memories(self, user_id, include_superseded=False):
sql = ("SELECT * FROM identity_memories "
"WHERE owner_user_id = ? AND scope = 'user'")
if not include_superseded:
sql += " AND status = 'active'"
sql += " ORDER BY created_at DESC, id DESC"
rows = self.conn.execute(sql, (user_id,)).fetchall()
return [dict(r) for r in rows]
def list_project_memories(self, project_id, include_superseded=False):
"""Caller must already hold project ``read`` permission."""
sql = ("SELECT * FROM identity_memories "
"WHERE scope = 'project' AND project_id = ?")
if not include_superseded:
sql += " AND status = 'active'"
sql += " ORDER BY created_at DESC, id DESC"
rows = self.conn.execute(sql, (project_id,)).fetchall()
return [dict(r) for r in rows]
def supersede_memory(self, mem_id, actor_user_id,
replacement_content=None, replacement_kind=None):
"""Mark old row superseded; optionally create its replacement.
User-scope rows may only be superseded by their owner
(PermissionError otherwise). Project-scope membership is
enforced by the server before calling this method.
Returns (old_id, new_id_or_None).
"""
row = self.get_memory(mem_id)
if row is None:
raise KeyError("memory not found")
if row["scope"] == "user" and row["owner_user_id"] != actor_user_id:
raise PermissionError("not the owner of this memory")
new_id = None
if replacement_content is not None:
kind = replacement_kind or row["kind"]
new_id = self.add_memory(
row["owner_user_id"], row["scope"], kind,
replacement_content,
project_id=(row["project_id"] or None)
if row["scope"] == "project" else None,
source_thread_id=(row["source_thread_id"] or None))
self.conn.execute(
"UPDATE identity_memories SET status = 'superseded', "
"superseded_by = ?, updated_at = ? WHERE id = ?",
(new_id or "", time.time(), mem_id))
self.conn.commit()
return mem_id, new_id
def delete_memory(self, mem_id, actor_user_id):
"""Physical removal (user-initiated forgetting only)."""
row = self.get_memory(mem_id)
if row is None:
raise KeyError("memory not found")
if row["scope"] == "user" and row["owner_user_id"] != actor_user_id:
raise PermissionError("not the owner of this memory")
self.conn.execute(
"DELETE FROM identity_memories WHERE id = ?", (mem_id,))
self.conn.commit()
return True
def build_identity_context(self, user_id, project_id=None,
top_k_user=5, top_k_project=5):
"""Format active memories for an explicit caller.
Returns "" when there is nothing active. Never returns the
full history: at most top_k per scope. Project memories are
included only when project_id is given; the caller must have
checked project ``read`` permission first.
"""
sections = []
user_rows = self.list_user_memories(user_id)[:top_k_user]
if user_rows:
lines = ["[用户记忆]"]
for r in user_rows:
lines.append("- [%s] %s" % (r["kind"], r["content"]))
sections.append("\n".join(lines))
if project_id:
proj_rows = self.list_project_memories(
project_id)[:top_k_project]
if proj_rows:
lines = ["[项目记忆]"]
for r in proj_rows:
lines.append("- [%s] %s" % (r["kind"], r["content"]))
sections.append("\n".join(lines))
return "\n".join(sections)
+7
View File
@@ -68,6 +68,13 @@ MIGRATION_TABLES = (
"task_attachments", "task_attachments",
"task_followups", "task_followups",
"environments", "environments",
# Cloud identity batch: TEXT primary keys, no sequence to sync.
"user_profiles",
"identity_memories",
# Auto memory batch: same TEXT-primary-key pattern.
"memory_settings",
"task_memory_flags",
"memory_extraction_log",
) )
_SKIP_TABLES = frozenset({"kb_fts", "sqlite_sequence"}) _SKIP_TABLES = frozenset({"kb_fts", "sqlite_sequence"})
+440
View File
@@ -110,6 +110,9 @@ from cloud.approvals import ensure_approvals_table as _appr_ensure
from cloud.approvals import expire_overdue as _appr_expire from cloud.approvals import expire_overdue as _appr_expire
from cloud.approvals import get_approval as _appr_get from cloud.approvals import get_approval as _appr_get
from cloud.approvals import list_approvals as _appr_list from cloud.approvals import list_approvals as _appr_list
from cloud.auto_memory import AutoMemoryStore
from cloud.auto_memory import process_due_tasks as _auto_process_due
from cloud.identity import IdentityStore
from cloud.kb import KBStore from cloud.kb import KBStore
from cloud.memory import MemoryStore from cloud.memory import MemoryStore
from cloud.pr import PrSkipped, PrStepError from cloud.pr import PrSkipped, PrStepError
@@ -1664,6 +1667,11 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
# 项目补仓库是第一个 PATCH 路由;与其它方法一样走 _dispatch。 # 项目补仓库是第一个 PATCH 路由;与其它方法一样走 _dispatch。
self._dispatch_safe("PATCH") self._dispatch_safe("PATCH")
def do_PUT(self):
# PUT 与 PATCH 同理:此前无入口,/users/me/profile 是
# 第一个 PUT 路由;未知 PUT 路径仍由 _dispatch 回 404。
self._dispatch_safe("PUT")
def _dispatch_safe(self, method): def _dispatch_safe(self, method):
"""分发兜底:_dispatch 里任何未捕获异常都回 500 JSON,而不是 """分发兜底:_dispatch 里任何未捕获异常都回 500 JSON,而不是
让请求线程直接断连(2026-10-27 死锁事件就是这么变成前端 让请求线程直接断连(2026-10-27 死锁事件就是这么变成前端
@@ -1879,6 +1887,34 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._send_json(404, {"error": "not found"}) self._send_json(404, {"error": "not found"})
elif path == "/users/me/quota" and method == "GET": elif path == "/users/me/quota" and method == "GET":
self._user_quota_self(user_id) self._user_quota_self(user_id)
elif path == "/users/me/profile" and method == "GET":
self._identity_profile_get(user_id)
elif path == "/users/me/profile" and method in ("PUT", "POST"):
self._identity_profile_upsert(user_id)
elif path == "/users/me/memories" and method == "GET":
self._identity_user_memories_list(user_id)
elif path == "/users/me/memories" and method == "POST":
self._identity_user_memory_create(user_id)
elif path == "/users/me/identity-context" and method == "GET":
self._identity_context(user_id, query)
elif path == "/users/me/auto-memory-settings" and method == "GET":
self._auto_memory_settings_get(user_id)
elif path == "/users/me/auto-memory-settings" and \
method in ("PUT", "POST"):
self._auto_memory_settings_upsert(user_id)
elif path == "/users/me/auto-memory-log" and method == "GET":
self._auto_memory_log(user_id, query)
elif path == "/users/me/auto-memory/run" and method == "POST":
self._auto_memory_run(user_id)
elif path.startswith("/users/me/memories/"):
_mid = path[len("/users/me/memories/"):]
if _mid and "/" not in _mid and method == "DELETE":
self._identity_user_memory_delete(user_id, _mid)
elif _mid.endswith("/supersede") and method == "POST":
self._identity_user_memory_supersede(
user_id, _mid[: -len("/supersede")].rstrip("/"))
else:
self._send_json(404, {"error": "not found"})
elif path == "/tasks" and method == "POST": elif path == "/tasks" and method == "POST":
self._create_task(user_id) self._create_task(user_id)
elif path == "/uploads" and method == "POST": elif path == "/uploads" and method == "POST":
@@ -1918,6 +1954,9 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
self._retry_task(user_id, rest[: -len("/retry")]) self._retry_task(user_id, rest[: -len("/retry")])
elif rest.endswith("/goal-resume") and method == "POST": elif rest.endswith("/goal-resume") and method == "POST":
self._goal_resume_task(user_id, rest[: -len("/goal-resume")]) self._goal_resume_task(user_id, rest[: -len("/goal-resume")])
elif rest.endswith("/auto-memory-flags") and method == "POST":
self._task_auto_memory_flags(
user_id, rest[: -len("/auto-memory-flags")])
elif rest.endswith("/heartbeat") and method == "POST": elif rest.endswith("/heartbeat") and method == "POST":
self._heartbeat_task(user_id, rest[: -len("/heartbeat")]) self._heartbeat_task(user_id, rest[: -len("/heartbeat")])
elif rest.endswith("/followups") and method == "POST": elif rest.endswith("/followups") and method == "POST":
@@ -2688,6 +2727,48 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
else: else:
self._send_json(404, {"error": "not found"}) self._send_json(404, {"error": "not found"})
return return
# /projects/{id}/identity-memories (project-scope system memory)
if len(parts) == 2 and parts[1] == "identity-memories":
if method == "POST":
err = self._check_project(user_id, project_id, "create_task")
if err:
self._send_json(err[0], {"error": err[1]})
return
self._project_identity_memory_create(user_id, project_id)
elif method == "GET":
err = self._check_project(user_id, project_id, "read")
if err:
self._send_json(err[0], {"error": err[1]})
return
self._project_identity_memories_list(project_id, query)
else:
self._send_json(404, {"error": "not found"})
return
# /projects/{id}/identity-memories/{mid}/supersede
if len(parts) == 4 and parts[1] == "identity-memories" \
and parts[3] == "supersede":
if method != "POST":
self._send_json(404, {"error": "not found"})
return
err = self._check_project(user_id, project_id, "create_task")
if err:
self._send_json(err[0], {"error": err[1]})
return
self._project_identity_memory_supersede(
user_id, project_id, parts[2])
return
# /projects/{id}/identity-memories/{mid}
if len(parts) == 3 and parts[1] == "identity-memories":
if method != "DELETE":
self._send_json(404, {"error": "not found"})
return
err = self._check_project(user_id, project_id, "create_task")
if err:
self._send_json(err[0], {"error": err[1]})
return
self._project_identity_memory_delete(
user_id, project_id, parts[2])
return
# /projects/{id}/memory # /projects/{id}/memory
if len(parts) == 2 and parts[1] == "memory": if len(parts) == 2 and parts[1] == "memory":
if method == "POST": if method == "POST":
@@ -3036,6 +3117,365 @@ class CloudControlPlaneHandler(BaseHTTPRequestHandler):
auth.audit(user_id, project_id, "supersede_memory", str(mid)) auth.audit(user_id, project_id, "supersede_memory", str(mid))
self._send_json(200, {"id": mid, "status": "deprecated"}) self._send_json(200, {"id": mid, "status": "deprecated"})
# -- System-level identity / memory (cloud only) ------------------
# These handlers never touch host.py, CODEX_HOME or any thread:
# they only read/write the control-plane database. The context
# endpoint returns text for an explicit caller; it does not
# inject anything into a task or a thread on its own.
def _identity_store(self):
return IdentityStore(self.server.db_path)
def _identity_close(self, store):
try:
store.conn.close()
except Exception:
pass
def _identity_profile_get(self, user_id):
store = self._identity_store()
try:
self._send_json(200, store.get_profile(user_id))
finally:
self._identity_close(store)
def _identity_profile_upsert(self, user_id):
body, err = self._read_json()
if err:
self._send_json(400, {"error": err})
return
store = self._identity_store()
try:
try:
profile = store.upsert_profile(
user_id,
display_name=body.get("display_name"),
timezone=body.get("timezone"),
preferences=body.get("preferences"))
except ValueError as exc:
self._send_json(400, {"error": str(exc)})
return
finally:
self._identity_close(store)
with _auth_store(self.server.db_path) as auth:
auth.audit(user_id, "", "profile_updated", user_id)
self._send_json(200, profile)
def _identity_user_memories_list(self, user_id):
store = self._identity_store()
try:
rows = store.list_user_memories(user_id)
finally:
self._identity_close(store)
self._send_json(200, {"user_id": user_id, "memories": rows})
def _identity_user_memory_create(self, user_id):
body, err = self._read_json()
if err:
self._send_json(400, {"error": err})
return
store = self._identity_store()
try:
try:
mem_id = store.add_memory(
user_id, "user", body.get("kind"),
body.get("content"),
source_thread_id=body.get("source_thread_id"))
except ValueError as exc:
self._send_json(400, {"error": str(exc)})
return
finally:
self._identity_close(store)
with _auth_store(self.server.db_path) as auth:
auth.audit(user_id, "", "identity_memory_created", mem_id)
self._send_json(201, {"id": mem_id, "scope": "user"})
def _identity_user_memory_supersede(self, user_id, mem_id):
body, err = self._read_json()
if err:
body = {}
store = self._identity_store()
try:
row = store.get_memory(mem_id)
if row is None or row["scope"] != "user" \
or row["owner_user_id"] != user_id:
self._send_json(404, {"error": "memory not found"})
return
try:
_old, new_id = store.supersede_memory(
mem_id, user_id,
replacement_content=body.get("replacement_content"),
replacement_kind=body.get("replacement_kind"))
except (KeyError, PermissionError):
self._send_json(404, {"error": "memory not found"})
return
except ValueError as exc:
self._send_json(400, {"error": str(exc)})
return
finally:
self._identity_close(store)
with _auth_store(self.server.db_path) as auth:
auth.audit(user_id, "", "identity_memory_superseded", mem_id)
self._send_json(200, {"id": mem_id, "status": "superseded",
"replacement_id": new_id})
def _identity_user_memory_delete(self, user_id, mem_id):
store = self._identity_store()
try:
row = store.get_memory(mem_id)
if row is None or row["scope"] != "user" \
or row["owner_user_id"] != user_id:
self._send_json(404, {"error": "memory not found"})
return
try:
store.delete_memory(mem_id, user_id)
except (KeyError, PermissionError):
self._send_json(404, {"error": "memory not found"})
return
finally:
self._identity_close(store)
with _auth_store(self.server.db_path) as auth:
auth.audit(user_id, "", "identity_memory_deleted", mem_id)
self._send_json(200, {"deleted": True, "id": mem_id})
def _identity_context(self, user_id, query):
project_id = None
if query:
vals = query.get("project_id")
if vals:
project_id = vals[0]
if project_id:
err = self._check_project(user_id, project_id, "read")
if err:
self._send_json(err[0], {"error": err[1]})
return
store = self._identity_store()
try:
profile = store.get_profile(user_id)
user_rows = store.list_user_memories(user_id)
proj_rows = store.list_project_memories(project_id) \
if project_id else []
context = store.build_identity_context(user_id, project_id)
finally:
self._identity_close(store)
summaries_context = ""
if project_id:
mem_store = MemoryStore(self.server.db_path)
try:
summaries_context = mem_store.build_prompt_context(
project_id)
finally:
try:
mem_store.conn.close()
except Exception:
pass
self._send_json(200, {
"user_id": user_id,
"project_id": project_id,
"profile": profile,
"user_memories": user_rows,
"project_memories": proj_rows,
"identity_context": context,
# Kept as a separate field on purpose: the existing
# project summaries are NOT merged into identity_context
# and neither field is injected into any task/thread here.
"project_summaries_context": summaries_context,
})
# -- 自动记忆(官方 memories 对齐:设置 + 提取触发) ------------
# 设置与提取日志只归属本人;任务级开关走项目 create_task 门。
def _auto_memory_store(self):
return AutoMemoryStore(self.server.db_path)
def _auto_memory_close(self, store):
try:
store.close()
except Exception:
pass
def _auto_memory_settings_get(self, user_id):
store = self._auto_memory_store()
try:
self._send_json(200, store.get_settings(user_id))
finally:
self._auto_memory_close(store)
def _auto_memory_settings_upsert(self, user_id):
body, err = self._read_json()
if err:
self._send_json(400, {"error": err})
return
if not isinstance(body, dict):
self._send_json(400, {"error": "body must be an object"})
return
store = self._auto_memory_store()
try:
try:
settings = store.upsert_settings(user_id, **body)
except (ValueError, TypeError) as exc:
self._send_json(400, {"error": str(exc)})
return
finally:
self._auto_memory_close(store)
with _auth_store(self.server.db_path) as auth:
auth.audit(user_id, "", "auto_memory_settings_updated", "")
self._send_json(200, settings)
def _auto_memory_log(self, user_id, query):
try:
limit = int((query.get("limit") or ["50"])[0])
except (TypeError, ValueError):
self._send_json(400, {"error": "limit must be an integer"})
return
store = self._auto_memory_store()
try:
rows = store.list_log(user_id, limit=limit)
finally:
self._auto_memory_close(store)
self._send_json(200, {"user_id": user_id, "log": rows})
def _auto_memory_run(self, user_id):
body, err = self._read_json()
if err:
self._send_json(400, {"error": err})
return
force = False
if isinstance(body, dict) and body.get("force") is not None:
if body.get("force") not in (True, False, 0, 1):
self._send_json(400, {"error": "force must be a boolean"})
return
force = bool(body.get("force"))
results = _auto_process_due(
self.server.db_path, force=force, owner_id=user_id)
for item in results:
if item.get("status") == "extracted":
# 提取日志已在 process_due_tasks 里落库,这里只补
# 一条审计(不查任务归属:run 只扫本人的任务)。
task_id = item.get("task_id") or ""
with _auth_store(self.server.db_path) as auth:
auth.audit(user_id, "", "auto_memory_extracted",
task_id)
self._send_json(200, {"user_id": user_id, "results": results})
def _task_auto_memory_flags(self, user_id, task_id):
body, err = self._read_json()
if err:
self._send_json(400, {"error": err})
return
if not isinstance(body, dict):
self._send_json(400, {"error": "body must be an object"})
return
task = self._fetch_task(task_id)
if task is None:
self._send_json(404, {"error": "task not found"})
return
err = self._check_project(
user_id, task["project_id"], "create_task")
if err:
self._send_json(err[0], {"error": err[1]})
return
store = self._auto_memory_store()
try:
try:
flags = store.set_task_flags(
task_id,
use_memories=body.get("use_memories"),
generate_memories=body.get("generate_memories"))
except ValueError as exc:
self._send_json(400, {"error": str(exc)})
return
finally:
self._auto_memory_close(store)
with _auth_store(self.server.db_path) as auth:
auth.audit(user_id, task["project_id"],
"task_auto_memory_flags", task_id)
self._send_json(200, {"task_id": task_id, "flags": flags})
def _project_identity_memory_create(self, user_id, project_id):
body, err = self._read_json()
if err:
self._send_json(400, {"error": err})
return
store = self._identity_store()
try:
try:
mem_id = store.add_memory(
user_id, "project", body.get("kind"),
body.get("content"), project_id=project_id,
source_thread_id=body.get("source_thread_id"))
except ValueError as exc:
self._send_json(400, {"error": str(exc)})
return
finally:
self._identity_close(store)
with _auth_store(self.server.db_path) as auth:
auth.audit(user_id, project_id,
"identity_memory_created", mem_id)
self._send_json(201, {"id": mem_id, "scope": "project",
"project_id": project_id})
def _project_identity_memories_list(self, project_id, query):
try:
top_k = int((query.get("top_k") or ["20"])[0])
except (TypeError, ValueError):
self._send_json(400, {"error": "top_k must be an integer"})
return
store = self._identity_store()
try:
rows = store.list_project_memories(project_id)
finally:
self._identity_close(store)
self._send_json(200, {"project_id": project_id,
"memories": rows[:top_k]})
def _project_identity_memory_supersede(self, user_id, project_id,
mem_id):
body, err = self._read_json()
if err:
body = {}
store = self._identity_store()
try:
row = store.get_memory(mem_id)
if row is None or row["scope"] != "project" \
or row["project_id"] != project_id:
self._send_json(404, {"error": "memory not found"})
return
try:
_old, new_id = store.supersede_memory(
mem_id, user_id,
replacement_content=body.get("replacement_content"),
replacement_kind=body.get("replacement_kind"))
except (KeyError, PermissionError, ValueError) as exc:
if isinstance(exc, ValueError):
self._send_json(400, {"error": str(exc)})
else:
self._send_json(404, {"error": "memory not found"})
return
finally:
self._identity_close(store)
with _auth_store(self.server.db_path) as auth:
auth.audit(user_id, project_id,
"identity_memory_superseded", mem_id)
self._send_json(200, {"id": mem_id, "status": "superseded",
"replacement_id": new_id})
def _project_identity_memory_delete(self, user_id, project_id, mem_id):
store = self._identity_store()
try:
row = store.get_memory(mem_id)
if row is None or row["scope"] != "project" \
or row["project_id"] != project_id:
self._send_json(404, {"error": "memory not found"})
return
store.delete_memory(mem_id, user_id)
finally:
self._identity_close(store)
with _auth_store(self.server.db_path) as auth:
auth.audit(user_id, project_id,
"identity_memory_deleted", mem_id)
self._send_json(200, {"deleted": True, "id": mem_id})
def _kb_index(self, user_id, project_id): def _kb_index(self, user_id, project_id):
body, err = self._read_json() body, err = self._read_json()
if err: if err:
+482
View File
@@ -0,0 +1,482 @@
"""Tests for cloud/auto_memory.py + worker injection + HTTP settings.
Scope, stated so a future reader does not over-read it: v1
deterministic extraction only. No test here calls a model for
extraction, touches host.py, CODEX_HOME or any Codex thread; the
store, the worker hook and the HTTP endpoints only read/write the
control-plane database, and injection only changes the in-memory
task copy the worker hands to its executor.
Run from the repo root:
python3 -m unittest discover -s cloud/tests -p "test_auto_memory.py" -v
"""
import http.client
import json
import os
import sqlite3
import sys
import tempfile
import threading
import time
import unittest
from pathlib import Path
from types import SimpleNamespace
try:
from cloud.auth import AuthStore
from cloud.auto_memory import AutoMemoryStore
from cloud.auto_memory import build_injection_prefix
from cloud.auto_memory import process_due_tasks
from cloud.auto_memory import redact_text
from cloud.db import POSTGRES_SCHEMA
from cloud.identity import IdentityStore
from cloud.migrations import MIGRATION_TABLES
from cloud.server import create_server
from cloud.server import init_db
from cloud.worker import Worker
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.auth import AuthStore
from cloud.auto_memory import AutoMemoryStore
from cloud.auto_memory import build_injection_prefix
from cloud.auto_memory import process_due_tasks
from cloud.auto_memory import redact_text
from cloud.db import POSTGRES_SCHEMA
from cloud.identity import IdentityStore
from cloud.migrations import MIGRATION_TABLES
from cloud.server import create_server
from cloud.server import init_db
from cloud.worker import Worker
def _tmp_db():
fd, path = tempfile.mkstemp(suffix=".db")
os.close(fd)
os.unlink(path)
return path
def _insert_completed(db_path, task_id, owner="u1", project="p1",
prompt="做登录页", result=None, started_ago=2000,
duration=120, goal=None):
now = time.time()
finished = now - 1000
started = finished - duration
conn = sqlite3.connect(db_path)
try:
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status,"
" idempotency_key, depends_on, priority, created_by,"
" created_at, updated_at, last_heartbeat, goal,"
" started_at, finished_at, result_text)"
" VALUES(?, ?, ?, 'completed', NULL, '[]', 0, ?, ?, ?,"
" NULL, ?, ?, ?, ?)",
(task_id, project, prompt, owner, now, finished,
goal, started, finished,
result if result is not None else "登录页已完成。" + "x" * 300),
)
conn.commit()
finally:
conn.close()
return finished
class RedactTest(unittest.TestCase):
def test_key_value_and_bearer_redacted(self):
out = redact_text("api_key=abc123 and password: hunter2")
self.assertNotIn("abc123", out)
self.assertNotIn("hunter2", out)
self.assertIn("[REDACTED]", out)
def test_plain_text_untouched(self):
self.assertEqual(redact_text("hello world"), "hello world")
self.assertEqual(redact_text(None), "")
class StoreTest(unittest.TestCase):
def setUp(self):
self.db = _tmp_db()
self.store = AutoMemoryStore(self.db)
def tearDown(self):
try:
self.store.close()
finally:
if os.path.exists(self.db):
os.unlink(self.db)
def test_defaults_off(self):
settings = self.store.get_settings("u1")
self.assertEqual(settings["generate_memories"], 0)
self.assertEqual(settings["use_memories"], 0)
self.assertEqual(settings["disable_on_external_context"], 1)
self.assertEqual(settings["min_rate_limit_remaining_percent"], 20)
self.assertEqual(settings["idle_seconds"], 900)
def test_upsert_validates_ranges(self):
with self.assertRaises(ValueError):
self.store.upsert_settings("u1",
min_rate_limit_remaining_percent=101)
with self.assertRaises(ValueError):
self.store.upsert_settings("u1", idle_seconds=-1)
with self.assertRaises(ValueError):
self.store.upsert_settings("u1", bogus_field=1)
got = self.store.upsert_settings(
"u1", generate_memories=True, use_memories=1,
idle_seconds=0, min_duration_seconds=0, min_result_chars=0)
self.assertEqual(got["generate_memories"], 1)
self.assertEqual(got["idle_seconds"], 0)
def test_task_flag_overrides_settings(self):
settings = self.store.upsert_settings("u1", generate_memories=False)
self.assertFalse(AutoMemoryStore.effective(settings, None,
"generate_memories"))
self.store.set_task_flags("t1", generate_memories=True)
flags = self.store.get_task_flags("t1")
self.assertTrue(AutoMemoryStore.effective(settings, flags,
"generate_memories"))
with self.assertRaises(ValueError):
self.store.set_task_flags("t2")
def test_pg_schema_and_migration_lists_include_new_tables(self):
schema_text = "\n".join(POSTGRES_SCHEMA)
for table in ("memory_settings", "task_memory_flags",
"memory_extraction_log"):
self.assertIn("CREATE TABLE IF NOT EXISTS %s" % table,
schema_text)
self.assertIn(table, MIGRATION_TABLES)
class ExtractionTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
init_db(self.db_path)
store = AutoMemoryStore(self.db_path)
store.upsert_settings(
"u1", generate_memories=True, use_memories=True,
idle_seconds=900, min_duration_seconds=30,
min_result_chars=200, min_rate_limit_remaining_percent=0)
store.close()
# Unlimited quota: explicit NULL row, so the quota gate passes.
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"INSERT INTO user_quotas(user_id, quota_usd_micros,"
" updated_at) VALUES('u1', NULL, ?)", (time.time(),))
conn.commit()
finally:
conn.close()
def tearDown(self):
self.tmp.cleanup()
def _memories(self):
store = IdentityStore(self.db_path)
try:
return store.list_project_memories("p1")
finally:
store.conn.close()
def test_happy_path_creates_project_memory_and_is_idempotent(self):
_insert_completed(self.db_path, "t1")
results = process_due_tasks(self.db_path)
self.assertEqual(results[0]["status"], "extracted")
self.assertEqual(results[0]["reason"], "created")
memories = self._memories()
self.assertEqual(len(memories), 1)
self.assertEqual(memories[0]["scope"], "project")
self.assertIn("做登录页", memories[0]["content"])
# Second sweep: no duplicate, same memory reused.
results2 = process_due_tasks(self.db_path)
self.assertEqual(results2[0]["reason"], "already_extracted")
self.assertEqual(len(self._memories()), 1)
def test_secret_in_result_is_redacted(self):
_insert_completed(self.db_path, "t1",
result="完成,api_key=should-not-leak " + "y" * 300)
process_due_tasks(self.db_path)
self.assertNotIn("should-not-leak", self._memories()[0]["content"])
def test_idle_gate_and_force(self):
now = time.time()
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status,"
" idempotency_key, depends_on, priority, created_by,"
" created_at, updated_at, last_heartbeat, started_at,"
" finished_at, result_text)"
" VALUES('t1', 'p1', '做登录页', 'completed', NULL,"
" '[]', 0, 'u1', ?, ?, NULL, ?, ?, ?)",
(now, now, now - 120, now, "z" * 300))
conn.commit()
finally:
conn.close()
results = process_due_tasks(self.db_path)
self.assertEqual(results[0]["reason"], "idle_not_reached")
self.assertEqual(self._memories(), [])
results = process_due_tasks(self.db_path, force=True)
self.assertEqual(results[0]["status"], "extracted")
def test_too_short_skipped(self):
_insert_completed(self.db_path, "t1", duration=5)
results = process_due_tasks(self.db_path)
self.assertEqual(results[0]["reason"], "too_short")
self.assertEqual(self._memories(), [])
def test_generate_disabled_skipped(self):
store = AutoMemoryStore(self.db_path)
store.upsert_settings("u1", generate_memories=False)
store.close()
_insert_completed(self.db_path, "t1")
results = process_due_tasks(self.db_path)
self.assertEqual(results[0]["reason"], "generate_disabled")
self.assertEqual(self._memories(), [])
def test_external_context_gate(self):
_insert_completed(self.db_path, "t1")
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"CREATE TABLE IF NOT EXISTS task_followups("
"id TEXT PRIMARY KEY, task_id TEXT, project_id TEXT,"
" body TEXT, status TEXT, created_at REAL,"
" sender_task_id TEXT)")
conn.execute(
"INSERT INTO task_followups(id, task_id, project_id,"
" body, status, created_at, sender_task_id)"
" VALUES('f1', 't1', 'p1', 'hi', 'done', ?, 'other')",
(time.time(),))
conn.commit()
finally:
conn.close()
results = process_due_tasks(self.db_path)
self.assertEqual(results[0]["reason"], "external_context")
store = AutoMemoryStore(self.db_path)
store.upsert_settings("u1", disable_on_external_context=False)
store.close()
results = process_due_tasks(self.db_path)
self.assertEqual(results[0]["status"], "extracted")
def test_quota_gate(self):
import datetime
month = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m")
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"UPDATE user_quotas SET quota_usd_micros = 10000000"
" WHERE user_id = 'u1'")
conn.execute(
"CREATE TABLE IF NOT EXISTS cost_records("
"id INTEGER PRIMARY KEY, task_id TEXT, project_id TEXT,"
" model TEXT, input_tokens INTEGER, output_tokens INTEGER,"
" cached_input_tokens INTEGER, total_tokens INTEGER,"
" usd_micros INTEGER, source TEXT, fallback_count INTEGER,"
" day TEXT, created_at REAL)")
conn.execute(
"INSERT INTO cost_records(task_id, project_id, model,"
" input_tokens, output_tokens, cached_input_tokens,"
" total_tokens, usd_micros, source, fallback_count, day,"
" created_at) VALUES('t1', 'p1', 'm', 0, 0, 0, 0, 9500000,"
" 'actual', 0, ?, ?)", (month + "-01", time.time()))
conn.commit()
finally:
conn.close()
store = AutoMemoryStore(self.db_path)
store.upsert_settings(
"u1", min_rate_limit_remaining_percent=20)
store.close()
_insert_completed(self.db_path, "t1")
results = process_due_tasks(self.db_path)
self.assertEqual(results[0]["reason"], "quota_low")
self.assertEqual(self._memories(), [])
def test_same_goal_consolidates_instead_of_duplicating(self):
_insert_completed(self.db_path, "t1", result="第一版完成。" + "a" * 300)
process_due_tasks(self.db_path)
self.assertEqual(len(self._memories()), 1)
_insert_completed(self.db_path, "t2", result="第二版完成。" + "b" * 300)
results = process_due_tasks(self.db_path)
extracted = [r for r in results if r["task_id"] == "t2"][0]
self.assertEqual(extracted["reason"], "consolidated")
memories = self._memories()
self.assertEqual(len(memories), 1)
self.assertIn("第二版完成", memories[0]["content"])
class InjectionTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
init_db(self.db_path)
identity = IdentityStore(self.db_path)
identity.add_memory("u1", "project", "convention", "提交前先跑测试",
project_id="p1")
identity.conn.close()
store = AutoMemoryStore(self.db_path)
store.upsert_settings("u1", use_memories=True)
store.close()
self.env = {"XCM_API_KEY": "test-placeholder"}
self.calls = []
def tearDown(self):
self.tmp.cleanup()
def _completed_runner(self, cmd):
if "dispatch" in cmd:
name = cmd[cmd.index("--name") + 1]
else:
name = cmd[cmd.index("--task") + 1]
return SimpleNamespace(
returncode=0,
stdout=json.dumps({"task": name, "status": "completed",
"text": "done"}),
stderr="")
def _recording_runner(self, inner):
def fake(cmd):
self.calls.append(list(cmd))
return inner(cmd)
return fake
def _insert_queued(self, task_id, prompt="做新功能", owner="u1"):
now = time.time()
conn = sqlite3.connect(self.db_path)
try:
conn.execute(
"INSERT INTO tasks(id, project_id, prompt, status,"
" idempotency_key, depends_on, priority, created_by,"
" created_at, updated_at, last_heartbeat)"
" VALUES(?, 'p1', ?, 'queued', NULL, '[]', 0, ?, ?, ?,"
" NULL)",
(task_id, prompt, owner, now, now),
)
conn.commit()
finally:
conn.close()
def test_worker_prepends_memory_to_first_round_only(self):
self._insert_queued("n1")
worker = Worker(self.db_path, "/fake/host.py",
runner=self._recording_runner(self._completed_runner),
env=self.env)
results = worker.run_all()
self.assertEqual([r["status"] for r in results], ["completed"])
dispatch = [c for c in self.calls if "dispatch" in c][0]
task_arg = dispatch[dispatch.index("--task") + 1]
self.assertIn("提交前先跑测试", task_arg)
self.assertIn("做新功能", task_arg)
# DB prompt is unchanged: injection only touched the copy.
conn = sqlite3.connect(self.db_path)
try:
row = conn.execute(
"SELECT prompt FROM tasks WHERE id = 'n1'").fetchone()
finally:
conn.close()
self.assertEqual(row[0], "做新功能")
def test_injection_off_by_default(self):
store = AutoMemoryStore(self.db_path)
store.upsert_settings("u1", use_memories=False)
store.close()
self.assertEqual(
build_injection_prefix(self.db_path, "u1", "p1"), "")
class EndpointsTest(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
self.auth = AuthStore(self.db_path)
self.owner_id, self.owner_key = self.auth.create_user("owner")
self.server = create_server(self.db_path, "127.0.0.1", 0)
self.port = self.server.server_address[1]
self.thread = threading.Thread(
target=self.server.serve_forever, daemon=True)
self.thread.start()
status, self.project = self._req(
"POST", "/projects", {"name": "demo"}, self.owner_key)
self.assertEqual(status, 201)
def tearDown(self):
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=5)
try:
self.auth.conn.close()
except Exception:
pass
self.tmp.cleanup()
def _req(self, method, path, body=None, key=None):
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
hdrs = {}
payload = None
if body is not None:
payload = json.dumps(body).encode("utf-8")
hdrs["Content-Type"] = "application/json"
if key is not None:
hdrs["X-API-Key"] = key
try:
conn.request(method, path, body=payload, headers=hdrs)
resp = conn.getresponse()
data = resp.read()
finally:
conn.close()
try:
parsed = json.loads(data.decode("utf-8")) if data else None
except ValueError:
parsed = None
return resp.status, parsed
def test_settings_get_put_and_validation(self):
status, body = self._req(
"GET", "/users/me/auto-memory-settings", key=self.owner_key)
self.assertEqual(status, 200)
self.assertEqual(body["generate_memories"], 0)
status, body = self._req(
"PUT", "/users/me/auto-memory-settings",
{"generate_memories": True, "use_memories": True},
self.owner_key)
self.assertEqual(status, 200, body)
self.assertEqual(body["generate_memories"], 1)
status, _ = self._req(
"PUT", "/users/me/auto-memory-settings",
{"idle_seconds": -5}, self.owner_key)
self.assertEqual(status, 400)
def test_run_and_log_and_task_flags(self):
status, body = self._req(
"POST", "/users/me/auto-memory/run", {"force": True},
self.owner_key)
self.assertEqual(status, 200, body)
self.assertEqual(body["results"], [])
status, body = self._req(
"GET", "/users/me/auto-memory-log", key=self.owner_key)
self.assertEqual(status, 200)
self.assertEqual(body["log"], [])
# Task flags: unknown task 404, real task 200.
status, _ = self._req(
"POST", "/tasks/nope/auto-memory-flags",
{"use_memories": False}, self.owner_key)
self.assertEqual(status, 404)
status, task = self._req(
"POST", "/tasks",
{"project_id": self.project["id"], "prompt": "做事"},
self.owner_key)
self.assertEqual(status, 201, task)
status, body = self._req(
"POST", "/tasks/%s/auto-memory-flags" % task["id"],
{"use_memories": False}, self.owner_key)
self.assertEqual(status, 200, body)
self.assertEqual(body["flags"]["use_memories"], 0)
status, _ = self._req(
"POST", "/tasks/%s/auto-memory-flags" % task["id"],
{}, self.owner_key)
self.assertEqual(status, 400)
if __name__ == "__main__":
unittest.main()
+259
View File
@@ -0,0 +1,259 @@
"""Tests for cloud/identity.py (system-level identity + memory).
Scope of this batch, stated so a future reader does not over-read
it: cloud layer only. No test here touches host.py, CODEX_HOME or
any Codex thread; the store and the HTTP endpoints only read/write
the control-plane database.
Run from the repo root:
python3 -m unittest discover -s cloud/tests -p "test_identity_memory.py" -v
"""
import http.client
import json
import os
import sys
import tempfile
import threading
import unittest
from pathlib import Path
try:
from cloud.auth import AuthStore
from cloud.db import POSTGRES_SCHEMA
from cloud.identity import IdentityStore
from cloud.migrations import MIGRATION_TABLES
from cloud.server import create_server
except ImportError:
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from cloud.auth import AuthStore
from cloud.db import POSTGRES_SCHEMA
from cloud.identity import IdentityStore
from cloud.migrations import MIGRATION_TABLES
from cloud.server import create_server
def _tmp_db():
fd, path = tempfile.mkstemp(suffix=".db")
os.close(fd)
os.unlink(path)
return path
class IdentityStoreTest(unittest.TestCase):
def setUp(self):
self.db = _tmp_db()
self.store = IdentityStore(self.db)
def tearDown(self):
try:
self.store.conn.close()
finally:
if os.path.exists(self.db):
os.unlink(self.db)
def test_profile_partial_update_keeps_other_fields(self):
self.store.upsert_profile(
"u1", display_name="Alice", timezone="Asia/Shanghai",
preferences={"lang": "zh"})
got = self.store.upsert_profile("u1", display_name="Alicia")
self.assertEqual(got["display_name"], "Alicia")
self.assertEqual(got["timezone"], "Asia/Shanghai")
self.assertEqual(got["preferences"], {"lang": "zh"})
def test_profile_rejects_secret_preferences(self):
with self.assertRaises(ValueError):
self.store.upsert_profile("u1", preferences={"api_key": "x"})
with self.assertRaises(ValueError):
self.store.upsert_profile("u1", preferences={"token": "x"})
def test_user_memory_visible_only_to_owner(self):
self.store.add_memory("u1", "user", "preference", "喜欢简洁中文")
self.assertEqual(len(self.store.list_user_memories("u1")), 1)
self.assertEqual(self.store.list_user_memories("u2"), [])
self.assertNotIn("喜欢简洁中文",
self.store.build_identity_context("u2"))
def test_scope_project_validation(self):
with self.assertRaises(ValueError):
self.store.add_memory("u1", "user", "fact", "x",
project_id="p1")
with self.assertRaises(ValueError):
self.store.add_memory("u1", "project", "fact", "x")
with self.assertRaises(ValueError):
self.store.add_memory("u1", "project", "nope", "x",
project_id="p1")
with self.assertRaises(ValueError):
self.store.add_memory("u1", "project", "fact", "",
project_id="p1")
def test_supersede_chain_hides_old_from_context(self):
old = self.store.add_memory("u1", "user", "preference", "旧偏好")
_old, new = self.store.supersede_memory(
old, "u1", replacement_content="新偏好")
self.assertIsNotNone(new)
ctx = self.store.build_identity_context("u1")
self.assertIn("新偏好", ctx)
self.assertNotIn("旧偏好", ctx)
row = self.store.get_memory(old)
self.assertEqual(row["status"], "superseded")
self.assertEqual(row["superseded_by"], new)
def test_supersede_user_memory_by_non_owner_forbidden(self):
mid = self.store.add_memory("u1", "user", "fact", "我的事实")
with self.assertRaises(PermissionError):
self.store.supersede_memory(mid, "u2")
with self.assertRaises(PermissionError):
self.store.delete_memory(mid, "u2")
def test_context_caps_and_project_section(self):
for i in range(7):
self.store.add_memory("u1", "user", "fact", "用户事实%d" % i)
self.store.add_memory("u1", "project", "convention", "项目约定",
project_id="p1")
ctx = self.store.build_identity_context("u1", "p1")
self.assertIn("[用户记忆]", ctx)
self.assertIn("[项目记忆]", ctx)
self.assertIn("项目约定", ctx)
# Only the newest 5 user facts may appear.
self.assertNotIn("用户事实0", ctx)
self.assertNotIn("用户事实1", ctx)
self.assertIn("用户事实6", ctx)
# Without a project, no project section leaks in.
self.assertNotIn("[项目记忆]",
self.store.build_identity_context("u1"))
def test_pg_schema_and_migration_lists_include_new_tables(self):
schema_text = "\n".join(POSTGRES_SCHEMA)
self.assertIn("CREATE TABLE IF NOT EXISTS user_profiles", schema_text)
self.assertIn("CREATE TABLE IF NOT EXISTS identity_memories",
schema_text)
self.assertIn("user_profiles", MIGRATION_TABLES)
self.assertIn("identity_memories", MIGRATION_TABLES)
class IdentityEndpointsTest(unittest.TestCase):
"""HTTP-level: routes, self-only user scope, project permission."""
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.db_path = os.path.join(self.tmp.name, "control.db")
self.auth = AuthStore(self.db_path)
self.owner_id, self.owner_key = self.auth.create_user("owner")
self.viewer_id, self.viewer_key = self.auth.create_user("viewer")
self.outsider_id, self.outsider_key = self.auth.create_user("outsider")
self.server = create_server(self.db_path, "127.0.0.1", 0)
self.port = self.server.server_address[1]
self.thread = threading.Thread(
target=self.server.serve_forever, daemon=True)
self.thread.start()
status, self.project = self._req(
"POST", "/projects", {"name": "demo"}, self.owner_key)
self.assertEqual(status, 201)
self.auth.set_member(
self.project["id"], self.owner_id, self.viewer_id, "viewer")
def tearDown(self):
self.server.shutdown()
self.server.server_close()
self.thread.join(timeout=5)
try:
self.auth.conn.close()
except Exception:
pass
self.tmp.cleanup()
def _req(self, method, path, body=None, key=None):
conn = http.client.HTTPConnection("127.0.0.1", self.port, timeout=10)
hdrs = {}
payload = None
if body is not None:
payload = json.dumps(body).encode("utf-8")
hdrs["Content-Type"] = "application/json"
if key is not None:
hdrs["X-API-Key"] = key
try:
conn.request(method, path, body=payload, headers=hdrs)
resp = conn.getresponse()
data = resp.read()
finally:
conn.close()
try:
parsed = json.loads(data.decode("utf-8")) if data else None
except ValueError:
parsed = None
return resp.status, parsed
def _pid(self):
return self.project["id"]
def test_profile_and_user_memory_roundtrip(self):
status, body = self._req(
"PUT", "/users/me/profile",
{"display_name": "Alice", "timezone": "Asia/Shanghai",
"preferences": {"lang": "zh"}}, self.owner_key)
self.assertEqual(status, 200, body)
self.assertEqual(body["display_name"], "Alice")
status, body = self._req(
"GET", "/users/me/profile", key=self.owner_key)
self.assertEqual(status, 200)
self.assertEqual(body["preferences"], {"lang": "zh"})
status, body = self._req(
"POST", "/users/me/memories",
{"kind": "preference", "content": "喜欢简洁中文"},
self.owner_key)
self.assertEqual(status, 201, body)
status, body = self._req(
"GET", "/users/me/memories", key=self.viewer_key)
self.assertEqual(status, 200)
# Viewer asks for *their own* memories: owner's must not leak.
self.assertEqual(body["memories"], [])
status, body = self._req(
"GET", "/users/me/identity-context", key=self.owner_key)
self.assertEqual(status, 200)
self.assertIn("喜欢简洁中文", body["identity_context"])
def test_project_memory_permission_and_outsider_403(self):
status, body = self._req(
"POST", "/projects/%s/identity-memories" % self._pid(),
{"kind": "convention", "content": "提交前先跑测试"},
self.owner_key)
self.assertEqual(status, 201, body)
status, body = self._req(
"GET", "/projects/%s/identity-memories" % self._pid(),
key=self.viewer_key)
self.assertEqual(status, 200)
self.assertEqual(len(body["memories"]), 1)
status, _ = self._req(
"GET", "/projects/%s/identity-memories" % self._pid(),
key=self.outsider_key)
self.assertEqual(status, 403)
status, _ = self._req(
"GET", "/users/me/identity-context?project_id=%s" % self._pid(),
key=self.outsider_key)
self.assertEqual(status, 403)
def test_user_memory_supersede_and_delete_via_http(self):
status, created = self._req(
"POST", "/users/me/memories",
{"kind": "fact", "content": "旧事实"}, self.owner_key)
self.assertEqual(status, 201)
mid = created["id"]
status, body = self._req(
"POST", "/users/me/memories/%s/supersede" % mid,
{"replacement_content": "新事实"}, self.viewer_key)
# Another user's memory: 404, not a leak-shaped 403.
self.assertEqual(status, 404)
status, body = self._req(
"POST", "/users/me/memories/%s/supersede" % mid,
{"replacement_content": "新事实"}, self.owner_key)
self.assertEqual(status, 200, body)
self.assertEqual(body["status"], "superseded")
status, body = self._req(
"DELETE", "/users/me/memories/%s" % body["replacement_id"],
key=self.owner_key)
self.assertEqual(status, 200)
if __name__ == "__main__":
unittest.main()
+81
View File
@@ -145,6 +145,9 @@ from cloud.cost import CostStore, estimate_tokens, price_usd_micros, \
from cloud.db import connect as _db_connect from cloud.db import connect as _db_connect
from cloud.logging_config import get_logger, log_event, setup_logging from cloud.logging_config import get_logger, log_event, setup_logging
from cloud.models import ModelRouter, is_fallbackable, record_fallback from cloud.models import ModelRouter, is_fallbackable, record_fallback
from cloud.auto_memory import AutoMemoryStore
from cloud.auto_memory import build_injection_prefix as _auto_prefix
from cloud.auto_memory import process_due_tasks as _auto_process_due
from cloud.peer_messages import deliver_peer_message as _deliver_peer_message from cloud.peer_messages import deliver_peer_message as _deliver_peer_message
from cloud.peer_messages import peer_message_prefix as _peer_prefix from cloud.peer_messages import peer_message_prefix as _peer_prefix
from cloud.peer_messages import row_get as _row_get from cloud.peer_messages import row_get as _row_get
@@ -977,6 +980,78 @@ class Worker:
status=event_type, status=event_type,
message="notify exception: %s" % exc) message="notify exception: %s" % exc)
# -- 自动记忆 ------------------------------------------------------
# 注入失败绝不挡任务(fail-open:无记忆跑也比不跑强);提取
# 失败只记日志(run_loop 轮次边界已包一层,见 process 方法)。
def _apply_auto_memory_injection(self, task):
"""首轮注入记忆:只改内存副本,原 prompt/goal 不落库。
跳过:协调员任务(不跑模型回合)、续跑与断点续跑(已有
会话历史)、无属主、use_memories 关、无记忆。goal 任务
同时改内存里的 goal(执行器取 goal 优先于 prompt),
前缀里写明“仅供参考,不得改写本任务目标”。
"""
if task.get("coordinator_parts"):
return task
if task.get("_resume") or task.get("_goal_resume_breakpoint"):
return task
owner = task.get("created_by")
if not owner:
return task
try:
store = AutoMemoryStore(self.db_path)
try:
flags = store.get_task_flags(task["id"])
finally:
store.close()
prefix = _auto_prefix(
self.db_path, owner, task["project_id"], flags=flags)
except Exception as exc:
try:
self.logger.warning("auto memory injection failed: %s",
exc)
except Exception:
pass
return task
if not prefix:
return task
new_task = dict(task)
prompt = task.get("prompt") or ""
new_task["prompt"] = prefix + "\n\n" + prompt
if task.get("goal_mode") or task.get("verify_command"):
new_task["goal"] = prefix + "\n\n" + (
task.get("goal") or prompt)
new_task["_memory_injected"] = True
self._audit(task["project_id"], "worker_memory_injected",
task["id"])
return new_task
def process_auto_memories(self, now=None, force=False,
owner_id=None):
"""扫 completed 任务并提取记忆(run_loop 轮次边界调用)。
返回 process_due_tasks 的结果列表;提取本身失败时返回
空列表并只记日志,不炸循环。提取成功的任务各补一条
worker 侧审计(服务端手动触发路径另有自己的审计)。
"""
try:
results = _auto_process_due(
self.db_path, now=now, force=force, owner_id=owner_id)
except Exception as exc:
try:
self.logger.warning("auto memory sweep failed: %s", exc)
except Exception:
pass
return []
for item in results:
if item.get("status") == "extracted":
# project_id 不在结果里:审计 detail 只带任务 id,
# project 字段用空串(与系统审计同款占位)。
self._audit("", "worker_memory_extracted",
item.get("task_id") or "")
return results
# -- 认领 ---------------------------------------------------------- # -- 认领 ----------------------------------------------------------
def _budget_exceeded(self): def _budget_exceeded(self):
return budget_exceeded(self.usage_turns, self.max_turns, return budget_exceeded(self.usage_turns, self.max_turns,
@@ -3158,6 +3233,9 @@ class Worker:
else: else:
task["prompt"] = resume_message task["prompt"] = resume_message
task["_resume"] = True task["_resume"] = True
# 自动记忆注入(官方 use_memories):首轮才注入,见
# _apply_auto_memory_injection 的跳过规则;只改内存副本。
task = self._apply_auto_memory_injection(task)
self._audit(task["project_id"], "worker_claim", self._audit(task["project_id"], "worker_claim",
"%s executor=%s" % (task_id, self._executor_name())) "%s executor=%s" % (task_id, self._executor_name()))
# log_event 只接受 task_id/project_id/status/message 四个 # log_event 只接受 task_id/project_id/status/message 四个
@@ -4122,6 +4200,9 @@ class Worker:
# G11/G12 后台投递:摘要(每小时)与 outbox 重试, # G11/G12 后台投递:摘要(每小时)与 outbox 重试,
# 轮次边界做,不中断在跑任务,失败只记日志。 # 轮次边界做,不中断在跑任务,失败只记日志。
self._process_push_due() self._process_push_due()
# 自动记忆提取:同款轮次边界后台扫,失败只记日志
# (process_auto_memories 内部已兜底,不炸循环)。
self.process_auto_memories()
# 自动陈旧回收:死 worker 的 running 任务在这里被 # 自动陈旧回收:死 worker 的 running 任务在这里被
# 标 interrupted;活任务心跳新鲜,不会被扫到。 # 标 interrupted;活任务心跳新鲜,不会被扫到。
if sweep_threshold is not None: if sweep_threshold is not None:
+93
View File
@@ -0,0 +1,93 @@
# 自动记忆(官方 Codex memories 的云端对齐版,v1)
官方功能见 learn.chatgpt.com/docs/customization/memories:开启后,
Codex 把先前聊天的有用上下文转成记忆,供未来工作用;后台生成、
闲置后才生成、剩余额度低时跳过、有 generate / use /
disable_on_external_context 等设置、密钥脱敏。
本批把这套行为搬到云端控制面(`cloud/auto_memory.py`),不改
host.py、不改 CODEX_HOME、不改 thread 的创建/续跑/fork/归档。
云端记忆与官方本地记忆的区别要先说清:官方记忆存在单机
`~/.codex/memories/` 里、跟一台机器走;云端记忆存在控制面
数据库里,跟用户/项目走,换 worker、换执行器都还在。
## 开关(默认全关,与官方“本地记忆默认关”一致)
- `GET/PUT /users/me/auto-memory-settings`:本人设置。
- `generate_memories`:完成任务是否可被提取。默认 0。
- `use_memories`:新任务首轮是否注入记忆。默认 0。
- `disable_on_external_context`:默认 1,见下。
- `min_rate_limit_remaining_percent`:默认 20,见下。
- `idle_seconds`:默认 900;`min_duration_seconds`:默认 30;
`min_result_chars`:默认 200。
- `extract_model` / `consolidation_model`:只存设置,v1 不调
模型(见“未做”)。
- `POST /tasks/{id}/auto-memory-flags`:单个任务覆盖 use/generate
(NULL = 跟随用户设置),权限为项目 create_task。不改 tasks
表结构,建任务的各条 INSERT 路径都不用动。
- `POST /users/me/auto-memory/run`:手动触发本人任务的提取扫荡
(body 可带 `force: true` 跳过闲置/时长/长度门,不跳过开关、
外部上下文门与额度门);`GET /users/me/auto-memory-log`:看
本人任务的提取日志(按任务主键覆盖,不无限增长)。
## 提取:什么时候、从哪来、写到哪
触发:worker `run_loop` 每轮结束调 `process_auto_memories()`;
也可经上面的手动触发接口立即跑。只扫 `status='completed'` 的
任务(failed / interrupted / merged 不提取)。
资格(任一不满足即跳过并记日志,下次仍可再扫):
1. generate 开关(任务级覆盖优先于用户设置);
2. 终态闲置够 `idle_seconds`(终态后仍可被 /continue 续跑,
所以“刚完成”不等于“已闲置”;`force` 可跳过);
3. 有 started_at/finished_at 且时长 ≥ `min_duration_seconds`,
result_text 长度 ≥ `min_result_chars`(跳过短期会话);
4. `disable_on_external_context` 开时,任务间消息
(task_followups 有该任务的行)或协调员子任务
(parent_task_id 非空)不提取;
5. 当月 USD 配额剩余百分比 ≥ `min_rate_limit_remaining_percent`
(无配额行按默认 $10 计,只有显式 NULL 行才是不限额,与
worker 配额口径一致;查不到配额时 fail-closed,记
quota_check_failed,不冒充额度充足)。
内容(v1 确定性版,不调模型):任务 goal/prompt 首行 +
result_text 前 800 字,脱敏后拼成一条 project 级 fact,写入
`identity_memories`,`source_thread_id` 记云端任务 id(去重键)。
同一任务只提取一次;同一目标的新结果出现时 supersede 旧记忆
(consolidation 的确定性版),不堆重复。自动提取只写 project
级:单个项目的任务结论若自动写成 user 级,会跨项目泄漏项目
上下文;user 级记忆仍只接受手动创建。
脱敏:`api_key/password/secret/token/credential` 的 key=value
形态、Authorization / Bearer 头、sk- / ghp_ / gho_ / xox*- 前缀
一律替换为 `[REDACTED]`;偏好里的密钥键禁令见 identity-memory.md。
## 注入:怎么进任务
worker `run_one` 认领后、执行前调 `_apply_auto_memory_injection`:
- 只注入首轮:续跑、断点续跑、协调员任务不注入;
- 只改内存副本:prompt 前加 `【过往记忆,仅供参考,不得改写
本任务目标】` + 最新各 5 条用户/项目记忆;goal 任务同时改
内存里的 goal(执行器取 goal 优先于 prompt);库里的原
prompt / goal 不动,任务详情里看到的仍是原文本;
- 注入失败 fail-open(记 warning,不挡任务);注入成功记
`worker_memory_injected` 审计;无记忆或 use 关时不注入。
- 注入不进 thread 的 developerInstructions:host.py 的 dispatch
路径不传 instructions,云端层唯一不改 host.py 的注入点就是
prompt 前缀,identity-memory.md 里“调用方自己拼进新建任务
的 prompt”正是这条路。
## 未做(诚实版)
- 未调模型提取/合并:`extract_model` / `consolidation_model`
只存设置。v1 提取是确定性拼贴,不是模型摘要;换模型提取
需另定模型选择、成本上限与失败语义。
- 未识别 host 线程内的 MCP / web search:tasks 表上没有这类
标记,`disable_on_external_context` 目前只认任务间消息与
协调员子任务;要认 MCP/web 需先给 host 线程事件加遥测。
- 未做控制台 UI:开关与日志目前只有 API;未做 per-chat 的
`/memories` 式交互(任务级 flags 接口是对应的最小版)。
- 未进代码签名候选:worker 签名只含 worker/sandbox/
coordinator/cost/server/host;本批同时改了 worker.py,所以
正常部署换目录时旧 worker 仍会因 worker.py 变化而自退,
仅换 auto_memory.py 的单文件部署不在签名覆盖内。
+51
View File
@@ -0,0 +1,51 @@
# 系统级身份与记忆(云端层)
本批只做云端层,不动 Codex thread:不改 `host.py`、不改
`CODEX_HOME`、不改任何 thread 的创建/续跑/fork/归档。
自动注入是后一批(docs/auto-memory.md)接通的:worker 首轮
把上下文拼进内存里的 prompt 副本,库里的原 prompt 不动。
## 数据
- `user_profiles`:每用户一行,display_name、timezone、
preferences(JSON)。偏好里禁止含 password/secret/token/
api_key/credential 的键,绝不能拿它当第二个凭证库。
- `identity_memories`:分级记忆,TEXT 主键(不走 IDENTITY
序列,见 `cloud/migrations.py` 对齐说明)。
- `user`:只归属用户一人可见,跨项目;不得带 project_id。
- `project`:归属用户创建、项目成员可见;读/写权限由
server 的项目 `read` / `create_task` 门先卡,store 层
不认项目成员关系。
- `kind`:fact / preference / decision / convention。
- `source_thread_id`:仅作来源标注,不建立 thread 依赖。
- 旧项目记忆 `summaries` 表不动;`identity-context` 返回的
项目摘要上下文是独立字段,不与身份上下文合并。
## 语义
- 修正用 supersede 链:旧行变 `superseded` 并指向新行,
旧内容不再进上下文,但可审计;删除是物理删除,仅用于
用户主动遗忘。
- 上下文只取最新 active 各 5 条,不全量倒历史。
- 他人用户级记忆在 HTTP 层回 404(不是 403),避免用
状态码探出记忆存在。
## 接口
- `GET/PUT /users/me/profile`(本人)
- `GET/POST /users/me/memories`、`POST /users/me/memories/{id}/supersede`、`DELETE /users/me/memories/{id}`(本人)
- `GET/POST /projects/{id}/identity-memories`、`POST /projects/{id}/identity-memories/{id}/supersede`、`DELETE /projects/{id}/identity-memories/{id}`(项目门)
- `GET /users/me/identity-context?project_id=...`:显式取用,
返回 profile、两级记忆、身份上下文、项目摘要上下文四个
独立字段;调用方要把上下文放进新任务时,应自己拼进
新建任务的 prompt 并留痕,本接口不代劳。
## 未做
- 自动提取与注入已由 docs/auto-memory.md 一批接通(worker
首轮 prompt 前缀注入 + run_loop 轮次边界提取扫荡):注入点
定为首轮 prompt(host.py dispatch 不传 developerInstructions,
云端层不改 host.py 的唯一注入点就是 prompt 前缀),去重与
合并用确定性 supersede,成本上限用配额剩余百分比门。
- 未做控制台 UI;未做跨引擎记忆(本批记忆与 Codex
thread 无关,换引擎时云端记忆仍可用)。