# ============================================================ # 智能兜底（Pi Agent）黄金测试 —— 全部确定性（注入 fake runner， # 不依赖真实 node/pi/网络/LLM）。覆盖 P1-DESIGN §6 测试矩阵 13 例。 # ============================================================ from __future__ import annotations import itertools import json import subprocess import uuid from pathlib import Path from typing import get_args import pytest from server.agent_core import fallback_lane, harness from server.agent_core.assistant import reply as assistant_reply from server.agent_core.feature_flags import load_feature_flags from server.agent_core.providers import reset_provider from server.agent_core.tool_runtime import check_tool from server.aps_domain.workflow import handle_intent from server.contracts import IntentName, IntentResult from server.integrations.pi_bridge import PiBridge, ToolBridgeViolation from server.state.seed import seed_world class FakeStore: """照 test_assistant.py 的 seed_world 版。""" def __init__(self): self.data = seed_world() self._counters: dict[str, int] = {} def next_id(self, kind: str) -> int: self._counters[kind] = self._counters.get(kind, 0) + 1 return self._counters[kind] def save(self) -> None: pass @pytest.fixture(autouse=True) def _isolate(tmp_path, monkeypatch): """环境隔离：run 目录与开关文件指向 tmp；清掉 LLM env 保证离线确定性。""" monkeypatch.setenv("APS_FALLBACK_DIR", str(tmp_path / "fb")) monkeypatch.setenv("APS_FEATURES_PATH", str(tmp_path / "features.json")) monkeypatch.delenv("LLM_API_KEY", raising=False) monkeypatch.delenv("LLM_BASE_URL", raising=False) monkeypatch.delenv("LLM_MODEL", raising=False) monkeypatch.delenv("LLM_PROVIDER", raising=False) reset_provider() yield reset_provider() # --------------------------------------------------------------------------- # fake runner 剧本构造（唯一注入点 = propose_reply(..., runner=..., config=...)） # --------------------------------------------------------------------------- def _read_issued_call_ids(work_dir: Path) -> list[str]: calls = work_dir.parent / "calls.jsonl" if not calls.exists(): return [] return [json.loads(line)["call_id"] for line in calls.read_text(encoding="utf-8").splitlines() if line.strip() and json.loads(line).get("status") == "issued"] def make_success_runner(report_template: str): """1 次 read + stop 报告；报告模板里的 {call_id} 在 resume 时读 calls.jsonl 填成真实签发的凭证（确定性：生成器是惰性的，凭证在 tool 事件处理后立即可读）。""" def runner(task: str, work_dir: Path): yield {"type": "tool_execution_start", "toolName": "read", "toolCallId": "t1"} yield {"type": "tool_execution_end", "toolName": "read", "toolCallId": "t1", "result": "snapshot ok"} ids = _read_issued_call_ids(work_dir) report = report_template.format(call_id=ids[0] if ids else "call-missing") yield {"type": "message_update", "delta": {"text": report}} yield {"type": "message_end", "message": {"role": "assistant", "stopReason": "stop", "content": [{"type": "text", "text": report}]}} yield {"type": "agent_end", "messages": [{"role": "assistant", "stopReason": "stop", "content": [{"type": "text", "text": report}]}]} return runner def _cfg(tmp_path: Path, **kw) -> fallback_lane.FallbackConfig: return fallback_lane.FallbackConfig(pi_home=str(tmp_path / "pi-home"), **kw) def _write_features(tmp_path: Path, features: dict) -> None: (tmp_path / "features.json").write_text( json.dumps({"version": 1, "features": features}, ensure_ascii=False), encoding="utf-8") def _intent(query: str) -> IntentResult: return IntentResult(intent="unknown", params={"query": query}, confidence=0.1, source="LLM") def _fb_audits(store: FakeStore) -> list[dict]: return [e for e in store.data.get("auditEvents", []) if e.get("action") == "agent.fallback.propose"] # --------------------------------------------------------------------------- # 1-2. 开关关 = 原行为不变；开关开 = propose 路径触发 # --------------------------------------------------------------------------- async def test_flag_off_preserves_original_behavior(tmp_path): store = FakeStore() direct = await assistant_reply(store.data, "随便说说", history=[], session_id="s1") reply = await handle_intent(store, "s1", _intent("随便说说")) assert reply.text == direct.text # 原话术逐字节不变 assert _fb_audits(store) == [] # 零审计噪音 assert not (tmp_path / "fb").exists() # 零 run 目录副作用 async def test_flag_on_success_proposes(tmp_path): _write_features(tmp_path, {"fallback": True}) store = FakeStore() reply = await fallback_lane.propose_reply( store, "s1", _intent("帮我分析下订单结构"), runner=make_success_runner("status: success\n\n报告正文：订单结构 [callId: {call_id}]"), config=_cfg(tmp_path)) assert reply is not None assert "[智能兜底 · 草稿]" in reply.text assert "报告正文：订单结构" in reply.text audits = _fb_audits(store) assert len(audits) == 1 assert audits[0]["result"] == "SUCCESS" assert audits[0]["power"] == "P1" assert harness.power_of("agent.fallback.propose") == "P1" runs = [p for p in (tmp_path / "fb").iterdir() if p.is_dir() and p.name != "pi-home"] assert len(runs) == 1 assert (runs[0] / "result.json").is_file() assert (runs[0] / "calls.jsonl").is_file() assert (runs[0] / "outbox" / "report.md").is_file() # --------------------------------------------------------------------------- # 3-4. 未登记意图仍拒绝；审计落链 # --------------------------------------------------------------------------- async def test_unregistered_intent_still_denied(tmp_path): _write_features(tmp_path, {"fallback": True}) store = FakeStore() denied = check_tool(store, _intent("随便说说")) # unknown 未登记 → 拒绝 assert denied is not None assert "不是已登记的操作" in denied.text denied_audits = [e for e in store.data.get("auditEvents", []) if e.get("action") == "tool.denied"] assert denied_audits and denied_audits[0]["result"] == "DENIED" # agent.fallback.propose 只是权力登记动作名，绝不是意图枚举成员 assert "agent.fallback.propose" not in get_args(IntentName) async def test_audit_chain_links(tmp_path): _write_features(tmp_path, {"fallback": True}) store = FakeStore() reply = await fallback_lane.propose_reply( store, "s1", _intent("分析订单"), runner=make_success_runner("status: success\n\n报告 [callId: {call_id}]"), config=_cfg(tmp_path)) assert reply is not None events = store.data.get("auditEvents", []) assert len(events) >= 2 # tool.run + 完成事件 for prev, cur in itertools.pairwise(events): assert cur["prevHash"] == prev["hash"] # 链不断 tool_runs = [e for e in events if e.get("action") == "tool.run"] assert any(str(e.get("actor", "")).startswith("pi-fallback:") for e in tool_runs) # --------------------------------------------------------------------------- # 5-8. 熔断 / 伪造凭证 / 异常：全部显式失败 # --------------------------------------------------------------------------- async def test_breaker_timeout_explicit_failure(tmp_path): _write_features(tmp_path, {"fallback": True}) store = FakeStore() def heartbeat_runner(task: str, work_dir: Path): while True: yield {"type": "harness_heartbeat"} reply = await fallback_lane.propose_reply( store, "s1", _intent("随便说说"), runner=heartbeat_runner, config=_cfg(tmp_path, timeout_sec=0.05)) assert reply is not None # 绝不抛、永远有回复 assert "智能兜底本次未完成" in reply.text audits = _fb_audits(store) assert audits[0]["result"] == "FAILED" assert audits[0]["rationale"]["stopReason"].startswith("breaker:timeout") async def test_breaker_max_steps(tmp_path): _write_features(tmp_path, {"fallback": True}) store = FakeStore() def busy_runner(task: str, work_dir: Path): for i in range(3): yield {"type": "tool_execution_start", "toolName": "read", "toolCallId": f"t{i}"} yield {"type": "agent_end", "messages": []} await fallback_lane.propose_reply( store, "s1", _intent("随便说说"), runner=busy_runner, config=_cfg(tmp_path, max_steps=2)) audits = _fb_audits(store) assert audits[0]["result"] == "FAILED" assert audits[0]["rationale"]["stopReason"].startswith("breaker:max_steps") async def test_forged_callid_rejected(tmp_path): _write_features(tmp_path, {"fallback": True}) store = FakeStore() forged = "call-" + str(uuid.uuid4()) report = f"status: success\n\n编造的数据结论 [callId: {forged}]" def forging_runner(task: str, work_dir: Path): yield {"type": "message_end", "message": {"role": "assistant", "stopReason": "stop", "content": [{"type": "text", "text": report}]}} yield {"type": "agent_end", "messages": []} reply = await fallback_lane.propose_reply( store, "s1", _intent("随便说说"), runner=forging_runner, config=_cfg(tmp_path)) assert "智能兜底本次未完成" in reply.text audits = _fb_audits(store) assert audits[0]["result"] == "FAILED" assert audits[0]["rationale"]["stopReason"] == "forged_citation" runs = [p for p in (tmp_path / "fb").iterdir() if p.is_dir() and p.name != "pi-home"] check = PiBridge("x", runs[0]).validate_report_citations(report) assert check["valid"] is False and check["missing"] == [forged] async def test_runner_exception_fails_explicit(tmp_path): _write_features(tmp_path, {"fallback": True}) store = FakeStore() def boom_runner(task: str, work_dir: Path): raise RuntimeError("spawn exploded") yield # pragma: no cover - 保持生成器形态 reply = await fallback_lane.propose_reply( store, "s1", _intent("随便说说"), runner=boom_runner, config=_cfg(tmp_path)) assert reply is not None # propose_reply 不抛 assert "智能兜底本次未完成" in reply.text audits = _fb_audits(store) assert audits[0]["result"] == "FAILED" assert audits[0]["rationale"]["stopReason"] == "harness_error" # --------------------------------------------------------------------------- # 9-11. 「默认关」开关语义 # --------------------------------------------------------------------------- def test_fallback_default_off_when_config_missing(tmp_path): result = load_feature_flags(str(tmp_path / "features.json")) assert result["features"]["fallback"]["enabled"] is False assert result["defaultOff"] == ["fallback"] assert all(info["enabled"] for key, info in result["features"].items() if key not in result["defaultOff"]) def test_fallback_explicit_true_enables(tmp_path): _write_features(tmp_path, {"fallback": True}) result = load_feature_flags(str(tmp_path / "features.json")) assert result["source"] == "file" assert result["features"]["fallback"]["enabled"] is True def test_fallback_non_bool_stays_off(tmp_path): _write_features(tmp_path, {"fallback": "yes", "orders": "no"}) result = load_feature_flags(str(tmp_path / "features.json")) assert result["features"]["fallback"]["enabled"] is False # 非 bool 回退各自默认 assert result["features"]["orders"]["enabled"] is True # 开的就是开、关的就是关 assert result["error"] is not None and "fallback" in result["error"] # --------------------------------------------------------------------------- # 12-13. 路径越界拦截；运行时不可用显式回话术 # --------------------------------------------------------------------------- def test_fs_read_path_escape_blocked(tmp_path): run_dir = tmp_path / "run" run_dir.mkdir() (run_dir / "ok.txt").write_text("界内内容", encoding="utf-8") bridge = PiBridge("rb-1", run_dir) assert bridge.handle_fs_read("ok.txt") == "界内内容" with pytest.raises(ToolBridgeViolation): bridge.handle_fs_read("../../server/contracts.py") with pytest.raises(ToolBridgeViolation): bridge.handle_fs_read(str(tmp_path / "outside.txt")) # --------------------------------------------------------------------------- # 14. 守卫模板 format 防回归（P1 冒烟 Bug A：注释行未转义花括号曾致真实 runner 100% 失败） # --------------------------------------------------------------------------- def test_write_guard_extension_real_template_format(tmp_path): """真实模板 str.format 不抛 + 生成守卫文件含字面量 { block: true（fake runner 结构性摸不到这条路径，本条是真实 runner 模板回归的唯一网）。""" run_dir = tmp_path / "fb-20990101-000000-abcdef" run_dir.mkdir() guard = fallback_lane.write_guard_extension(run_dir) # 不抛 KeyError 即过半 assert guard.is_file() and guard.name == f"guard-{run_dir.name}.ts" content = guard.read_text(encoding="utf-8") assert "{ block: true" in content # 字面量花括号必须真实出现 assert "{RUN_ROOT_POSIX}" not in content # 占位符必须被替换干净 assert run_dir.as_posix() in content def test_write_guard_readonly_mode_preserves_p1_semantics(tmp_path): """readonly 模式（默认）逐字节保持 P1 围墙语义：bash/edit/write 全禁、 无 WRITE_DIRS 放行面（P2 守卫模板参数化对 P1 的唯一约束）。""" run_dir = tmp_path / "fb-20990101-000000-abcdef" run_dir.mkdir() default_content = fallback_lane.write_guard_extension(run_dir).read_text(encoding="utf-8") explicit = fallback_lane.write_guard_extension(run_dir, mode="readonly") assert explicit.read_text(encoding="utf-8") == default_content assert "disabled by fallback guard (read-only lane)" in default_content # P1 全禁原文 assert "WRITE_DIRS" not in default_content # 无写放行面 assert 'name === "edit"' in default_content # edit 仍在全禁名单 def test_write_guard_plan_mode_opens_work_outbox_only(tmp_path): """plan/execute 模式（v2 模板）：write/edit 仅放行 run 目录内 work/+outbox/， bash 仍全禁、只读工具防逃逸不变。""" run_dir = tmp_path / "fb-20990101-000000-bcdef0" run_dir.mkdir() for mode in ("plan", "execute"): guard = fallback_lane.write_guard_extension(run_dir, mode=mode) content = guard.read_text(encoding="utf-8") assert "{ block: true" in content # 字面量花括号真实出现 assert "{RUN_ROOT_POSIX}" not in content # 占位符替换干净 assert "{MODE_LABEL}" not in content assert "WRITE_DIRS" in content assert "write outside work/outbox (fallback guard)" in content assert 'name === "bash"' in content # bash 仍全禁 assert "path escapes run root" in content # 防逃逸不变 assert "read-only lane" not in content # 不再是 P1 全禁语义 async def test_runtime_unavailable_falls_back_to_canned_reply(tmp_path, monkeypatch): _write_features(tmp_path, {"fallback": True}) monkeypatch.setenv("APS_FALLBACK_PI_CLI", str(tmp_path / "no-such-cli.js")) popen_calls = [] monkeypatch.setattr(subprocess, "Popen", lambda *a, **kw: popen_calls.append((a, kw))) store = FakeStore() direct = await assistant_reply(store.data, "随便说说", history=[], session_id="s1") reply = await fallback_lane.propose_reply(store, "s1", _intent("随便说说")) assert reply is not None assert popen_calls == [] # 不可用即显式失败，不触子进程 assert "智能兜底本次未完成" in reply.text assert "兜底运行时不可用" in reply.text assert direct.text in reply.text # 原话术仍在 audits = _fb_audits(store) assert audits[0]["result"] == "FAILED" assert audits[0]["rationale"]["stopReason"].startswith("unavailable:")