373 lines
16 KiB
Python
373 lines
16 KiB
Python
# ============================================================
|
||
# 智能兜底（Pi Agent）黄金测试 —— 全部确定性（注入 fake runner，
|
||
# 不依赖真实 node/pi/网络/LLM）。覆盖 P1-DESIGN §6 测试矩阵 13 例。
|
||
# ============================================================
|
||
from __future__ import annotations
|
||
|
||
import itertools
|
||
import json
|
||
import subprocess
|
||
import uuid
|
||
from pathlib import Path
|
||
from typing import get_args
|
||
|
||
import pytest
|
||
|
||
from server.agent_core import fallback_lane, harness
|
||
from server.agent_core.assistant import reply as assistant_reply
|
||
from server.agent_core.feature_flags import load_feature_flags
|
||
from server.agent_core.providers import reset_provider
|
||
from server.agent_core.tool_runtime import check_tool
|
||
from server.aps_domain.workflow import handle_intent
|
||
from server.contracts import IntentName, IntentResult
|
||
from server.integrations.pi_bridge import PiBridge, ToolBridgeViolation
|
||
from server.state.seed import seed_world
|
||
|
||
|
||
class FakeStore:
|
||
"""照 test_assistant.py 的 seed_world 版。"""
|
||
|
||
def __init__(self):
|
||
self.data = seed_world()
|
||
self._counters: dict[str, int] = {}
|
||
|
||
def next_id(self, kind: str) -> int:
|
||
self._counters[kind] = self._counters.get(kind, 0) + 1
|
||
return self._counters[kind]
|
||
|
||
def save(self) -> None:
|
||
pass
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _isolate(tmp_path, monkeypatch):
|
||
"""环境隔离：run 目录与开关文件指向 tmp；清掉 LLM env 保证离线确定性。"""
|
||
monkeypatch.setenv("APS_FALLBACK_DIR", str(tmp_path / "fb"))
|
||
monkeypatch.setenv("APS_FEATURES_PATH", str(tmp_path / "features.json"))
|
||
monkeypatch.delenv("LLM_API_KEY", raising=False)
|
||
monkeypatch.delenv("LLM_BASE_URL", raising=False)
|
||
monkeypatch.delenv("LLM_MODEL", raising=False)
|
||
monkeypatch.delenv("LLM_PROVIDER", raising=False)
|
||
reset_provider()
|
||
yield
|
||
reset_provider()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# fake runner 剧本构造（唯一注入点 = propose_reply(..., runner=..., config=...)）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _read_issued_call_ids(work_dir: Path) -> list[str]:
|
||
calls = work_dir.parent / "calls.jsonl"
|
||
if not calls.exists():
|
||
return []
|
||
return [json.loads(line)["call_id"]
|
||
for line in calls.read_text(encoding="utf-8").splitlines()
|
||
if line.strip() and json.loads(line).get("status") == "issued"]
|
||
|
||
|
||
def make_success_runner(report_template: str):
|
||
"""1 次 read + stop 报告；报告模板里的 {call_id} 在 resume 时读 calls.jsonl
|
||
填成真实签发的凭证（确定性：生成器是惰性的，凭证在 tool 事件处理后立即可读）。"""
|
||
|
||
def runner(task: str, work_dir: Path):
|
||
yield {"type": "tool_execution_start", "toolName": "read", "toolCallId": "t1"}
|
||
yield {"type": "tool_execution_end", "toolName": "read", "toolCallId": "t1",
|
||
"result": "snapshot ok"}
|
||
ids = _read_issued_call_ids(work_dir)
|
||
report = report_template.format(call_id=ids[0] if ids else "call-missing")
|
||
yield {"type": "message_update", "delta": {"text": report}}
|
||
yield {"type": "message_end", "message": {"role": "assistant", "stopReason": "stop",
|
||
"content": [{"type": "text", "text": report}]}}
|
||
yield {"type": "agent_end", "messages": [{"role": "assistant", "stopReason": "stop",
|
||
"content": [{"type": "text", "text": report}]}]}
|
||
|
||
return runner
|
||
|
||
|
||
def _cfg(tmp_path: Path, **kw) -> fallback_lane.FallbackConfig:
|
||
return fallback_lane.FallbackConfig(pi_home=str(tmp_path / "pi-home"), **kw)
|
||
|
||
|
||
def _write_features(tmp_path: Path, features: dict) -> None:
|
||
(tmp_path / "features.json").write_text(
|
||
json.dumps({"version": 1, "features": features}, ensure_ascii=False),
|
||
encoding="utf-8")
|
||
|
||
|
||
def _intent(query: str) -> IntentResult:
|
||
return IntentResult(intent="unknown", params={"query": query},
|
||
confidence=0.1, source="LLM")
|
||
|
||
|
||
def _fb_audits(store: FakeStore) -> list[dict]:
|
||
return [e for e in store.data.get("auditEvents", [])
|
||
if e.get("action") == "agent.fallback.propose"]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 1-2. 开关关 = 原行为不变；开关开 = propose 路径触发
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def test_flag_off_preserves_original_behavior(tmp_path):
|
||
store = FakeStore()
|
||
direct = await assistant_reply(store.data, "随便说说", history=[], session_id="s1")
|
||
reply = await handle_intent(store, "s1", _intent("随便说说"))
|
||
assert reply.text == direct.text # 原话术逐字节不变
|
||
assert _fb_audits(store) == [] # 零审计噪音
|
||
assert not (tmp_path / "fb").exists() # 零 run 目录副作用
|
||
|
||
|
||
async def test_flag_on_success_proposes(tmp_path):
|
||
_write_features(tmp_path, {"fallback": True})
|
||
store = FakeStore()
|
||
reply = await fallback_lane.propose_reply(
|
||
store, "s1", _intent("帮我分析下订单结构"),
|
||
runner=make_success_runner("status: success\n\n报告正文：订单结构 [callId: {call_id}]"),
|
||
config=_cfg(tmp_path))
|
||
assert reply is not None
|
||
assert "[智能兜底 · 草稿]" in reply.text
|
||
assert "报告正文：订单结构" in reply.text
|
||
|
||
audits = _fb_audits(store)
|
||
assert len(audits) == 1
|
||
assert audits[0]["result"] == "SUCCESS"
|
||
assert audits[0]["power"] == "P1"
|
||
assert harness.power_of("agent.fallback.propose") == "P1"
|
||
|
||
runs = [p for p in (tmp_path / "fb").iterdir() if p.is_dir() and p.name != "pi-home"]
|
||
assert len(runs) == 1
|
||
assert (runs[0] / "result.json").is_file()
|
||
assert (runs[0] / "calls.jsonl").is_file()
|
||
assert (runs[0] / "outbox" / "report.md").is_file()
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 3-4. 未登记意图仍拒绝；审计落链
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def test_unregistered_intent_still_denied(tmp_path):
|
||
_write_features(tmp_path, {"fallback": True})
|
||
store = FakeStore()
|
||
denied = check_tool(store, _intent("随便说说")) # unknown 未登记 → 拒绝
|
||
assert denied is not None
|
||
assert "不是已登记的操作" in denied.text
|
||
denied_audits = [e for e in store.data.get("auditEvents", [])
|
||
if e.get("action") == "tool.denied"]
|
||
assert denied_audits and denied_audits[0]["result"] == "DENIED"
|
||
# agent.fallback.propose 只是权力登记动作名，绝不是意图枚举成员
|
||
assert "agent.fallback.propose" not in get_args(IntentName)
|
||
|
||
|
||
async def test_audit_chain_links(tmp_path):
|
||
_write_features(tmp_path, {"fallback": True})
|
||
store = FakeStore()
|
||
reply = await fallback_lane.propose_reply(
|
||
store, "s1", _intent("分析订单"),
|
||
runner=make_success_runner("status: success\n\n报告 [callId: {call_id}]"),
|
||
config=_cfg(tmp_path))
|
||
assert reply is not None
|
||
events = store.data.get("auditEvents", [])
|
||
assert len(events) >= 2 # tool.run + 完成事件
|
||
for prev, cur in itertools.pairwise(events):
|
||
assert cur["prevHash"] == prev["hash"] # 链不断
|
||
tool_runs = [e for e in events if e.get("action") == "tool.run"]
|
||
assert any(str(e.get("actor", "")).startswith("pi-fallback:") for e in tool_runs)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 5-8. 熔断 / 伪造凭证 / 异常：全部显式失败
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
async def test_breaker_timeout_explicit_failure(tmp_path):
|
||
_write_features(tmp_path, {"fallback": True})
|
||
store = FakeStore()
|
||
|
||
def heartbeat_runner(task: str, work_dir: Path):
|
||
while True:
|
||
yield {"type": "harness_heartbeat"}
|
||
|
||
reply = await fallback_lane.propose_reply(
|
||
store, "s1", _intent("随便说说"), runner=heartbeat_runner,
|
||
config=_cfg(tmp_path, timeout_sec=0.05))
|
||
assert reply is not None # 绝不抛、永远有回复
|
||
assert "智能兜底本次未完成" in reply.text
|
||
audits = _fb_audits(store)
|
||
assert audits[0]["result"] == "FAILED"
|
||
assert audits[0]["rationale"]["stopReason"].startswith("breaker:timeout")
|
||
|
||
|
||
async def test_breaker_max_steps(tmp_path):
|
||
_write_features(tmp_path, {"fallback": True})
|
||
store = FakeStore()
|
||
|
||
def busy_runner(task: str, work_dir: Path):
|
||
for i in range(3):
|
||
yield {"type": "tool_execution_start", "toolName": "read",
|
||
"toolCallId": f"t{i}"}
|
||
yield {"type": "agent_end", "messages": []}
|
||
|
||
await fallback_lane.propose_reply(
|
||
store, "s1", _intent("随便说说"), runner=busy_runner,
|
||
config=_cfg(tmp_path, max_steps=2))
|
||
audits = _fb_audits(store)
|
||
assert audits[0]["result"] == "FAILED"
|
||
assert audits[0]["rationale"]["stopReason"].startswith("breaker:max_steps")
|
||
|
||
|
||
async def test_forged_callid_rejected(tmp_path):
|
||
_write_features(tmp_path, {"fallback": True})
|
||
store = FakeStore()
|
||
forged = "call-" + str(uuid.uuid4())
|
||
report = f"status: success\n\n编造的数据结论 [callId: {forged}]"
|
||
|
||
def forging_runner(task: str, work_dir: Path):
|
||
yield {"type": "message_end", "message": {"role": "assistant",
|
||
"stopReason": "stop", "content": [{"type": "text", "text": report}]}}
|
||
yield {"type": "agent_end", "messages": []}
|
||
|
||
reply = await fallback_lane.propose_reply(
|
||
store, "s1", _intent("随便说说"), runner=forging_runner, config=_cfg(tmp_path))
|
||
assert "智能兜底本次未完成" in reply.text
|
||
audits = _fb_audits(store)
|
||
assert audits[0]["result"] == "FAILED"
|
||
assert audits[0]["rationale"]["stopReason"] == "forged_citation"
|
||
|
||
runs = [p for p in (tmp_path / "fb").iterdir() if p.is_dir() and p.name != "pi-home"]
|
||
check = PiBridge("x", runs[0]).validate_report_citations(report)
|
||
assert check["valid"] is False and check["missing"] == [forged]
|
||
|
||
|
||
async def test_runner_exception_fails_explicit(tmp_path):
|
||
_write_features(tmp_path, {"fallback": True})
|
||
store = FakeStore()
|
||
|
||
def boom_runner(task: str, work_dir: Path):
|
||
raise RuntimeError("spawn exploded")
|
||
yield # pragma: no cover - 保持生成器形态
|
||
|
||
reply = await fallback_lane.propose_reply(
|
||
store, "s1", _intent("随便说说"), runner=boom_runner, config=_cfg(tmp_path))
|
||
assert reply is not None # propose_reply 不抛
|
||
assert "智能兜底本次未完成" in reply.text
|
||
audits = _fb_audits(store)
|
||
assert audits[0]["result"] == "FAILED"
|
||
assert audits[0]["rationale"]["stopReason"] == "harness_error"
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 9-11. 「默认关」开关语义
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_fallback_default_off_when_config_missing(tmp_path):
|
||
result = load_feature_flags(str(tmp_path / "features.json"))
|
||
assert result["features"]["fallback"]["enabled"] is False
|
||
assert result["defaultOff"] == ["fallback"]
|
||
assert all(info["enabled"] for key, info in result["features"].items()
|
||
if key not in result["defaultOff"])
|
||
|
||
|
||
def test_fallback_explicit_true_enables(tmp_path):
|
||
_write_features(tmp_path, {"fallback": True})
|
||
result = load_feature_flags(str(tmp_path / "features.json"))
|
||
assert result["source"] == "file"
|
||
assert result["features"]["fallback"]["enabled"] is True
|
||
|
||
|
||
def test_fallback_non_bool_stays_off(tmp_path):
|
||
_write_features(tmp_path, {"fallback": "yes", "orders": "no"})
|
||
result = load_feature_flags(str(tmp_path / "features.json"))
|
||
assert result["features"]["fallback"]["enabled"] is False # 非 bool 回退各自默认
|
||
assert result["features"]["orders"]["enabled"] is True # 开的就是开、关的就是关
|
||
assert result["error"] is not None and "fallback" in result["error"]
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 12-13. 路径越界拦截；运行时不可用显式回话术
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_fs_read_path_escape_blocked(tmp_path):
|
||
run_dir = tmp_path / "run"
|
||
run_dir.mkdir()
|
||
(run_dir / "ok.txt").write_text("界内内容", encoding="utf-8")
|
||
bridge = PiBridge("rb-1", run_dir)
|
||
assert bridge.handle_fs_read("ok.txt") == "界内内容"
|
||
with pytest.raises(ToolBridgeViolation):
|
||
bridge.handle_fs_read("../../server/contracts.py")
|
||
with pytest.raises(ToolBridgeViolation):
|
||
bridge.handle_fs_read(str(tmp_path / "outside.txt"))
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 14. 守卫模板 format 防回归（P1 冒烟 Bug A：注释行未转义花括号曾致真实 runner 100% 失败）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def test_write_guard_extension_real_template_format(tmp_path):
|
||
"""真实模板 str.format 不抛 + 生成守卫文件含字面量 { block: true（fake runner
|
||
结构性摸不到这条路径，本条是真实 runner 模板回归的唯一网）。"""
|
||
run_dir = tmp_path / "fb-20990101-000000-abcdef"
|
||
run_dir.mkdir()
|
||
guard = fallback_lane.write_guard_extension(run_dir) # 不抛 KeyError 即过半
|
||
assert guard.is_file() and guard.name == f"guard-{run_dir.name}.ts"
|
||
content = guard.read_text(encoding="utf-8")
|
||
assert "{ block: true" in content # 字面量花括号必须真实出现
|
||
assert "{RUN_ROOT_POSIX}" not in content # 占位符必须被替换干净
|
||
assert run_dir.as_posix() in content
|
||
|
||
|
||
def test_write_guard_readonly_mode_preserves_p1_semantics(tmp_path):
|
||
"""readonly 模式（默认）逐字节保持 P1 围墙语义：bash/edit/write 全禁、
|
||
无 WRITE_DIRS 放行面（P2 守卫模板参数化对 P1 的唯一约束）。"""
|
||
run_dir = tmp_path / "fb-20990101-000000-abcdef"
|
||
run_dir.mkdir()
|
||
default_content = fallback_lane.write_guard_extension(run_dir).read_text(encoding="utf-8")
|
||
explicit = fallback_lane.write_guard_extension(run_dir, mode="readonly")
|
||
assert explicit.read_text(encoding="utf-8") == default_content
|
||
assert "disabled by fallback guard (read-only lane)" in default_content # P1 全禁原文
|
||
assert "WRITE_DIRS" not in default_content # 无写放行面
|
||
assert 'name === "edit"' in default_content # edit 仍在全禁名单
|
||
|
||
|
||
def test_write_guard_plan_mode_opens_work_outbox_only(tmp_path):
|
||
"""plan/execute 模式（v2 模板）：write/edit 仅放行 run 目录内 work/+outbox/，
|
||
bash 仍全禁、只读工具防逃逸不变。"""
|
||
run_dir = tmp_path / "fb-20990101-000000-bcdef0"
|
||
run_dir.mkdir()
|
||
for mode in ("plan", "execute"):
|
||
guard = fallback_lane.write_guard_extension(run_dir, mode=mode)
|
||
content = guard.read_text(encoding="utf-8")
|
||
assert "{ block: true" in content # 字面量花括号真实出现
|
||
assert "{RUN_ROOT_POSIX}" not in content # 占位符替换干净
|
||
assert "{MODE_LABEL}" not in content
|
||
assert "WRITE_DIRS" in content
|
||
assert "write outside work/outbox (fallback guard)" in content
|
||
assert 'name === "bash"' in content # bash 仍全禁
|
||
assert "path escapes run root" in content # 防逃逸不变
|
||
assert "read-only lane" not in content # 不再是 P1 全禁语义
|
||
|
||
|
||
async def test_runtime_unavailable_falls_back_to_canned_reply(tmp_path, monkeypatch):
|
||
_write_features(tmp_path, {"fallback": True})
|
||
monkeypatch.setenv("APS_FALLBACK_PI_CLI", str(tmp_path / "no-such-cli.js"))
|
||
popen_calls = []
|
||
monkeypatch.setattr(subprocess, "Popen",
|
||
lambda *a, **kw: popen_calls.append((a, kw)))
|
||
store = FakeStore()
|
||
direct = await assistant_reply(store.data, "随便说说", history=[], session_id="s1")
|
||
reply = await fallback_lane.propose_reply(store, "s1", _intent("随便说说"))
|
||
assert reply is not None
|
||
assert popen_calls == [] # 不可用即显式失败，不触子进程
|
||
assert "智能兜底本次未完成" in reply.text
|
||
assert "兜底运行时不可用" in reply.text
|
||
assert direct.text in reply.text # 原话术仍在
|
||
audits = _fb_audits(store)
|
||
assert audits[0]["result"] == "FAILED"
|
||
assert audits[0]["rationale"]["stopReason"].startswith("unavailable:")
|