aps-agent/tests/golden/test_fallback_lane.py

# ============================================================
# 智能兜底（Pi Agent）黄金测试 —— 全部确定性（注入 fake runner，
# 不依赖真实 node/pi/网络/LLM）。覆盖 P1-DESIGN §6 测试矩阵 13 例。
# ============================================================
from __future__ import annotations

import itertools
import json
import subprocess
import uuid
from pathlib import Path
from typing import get_args

import pytest

from server.agent_core import fallback_lane, harness
from server.agent_core.feature_flags import load_feature_flags
from server.agent_core.providers import reset_provider
from server.agent_core.tool_runtime import check_tool
from server.aps_domain.workflow import handle_intent
from server.contracts import IntentName, IntentResult
from server.integrations.pi_bridge import PiBridge, ToolBridgeViolation
from server.state.seed import seed_world


class FakeStore:
    """seed_world 版内存 store。"""

    def __init__(self):
        self.data = seed_world()
        self._counters: dict[str, int] = {}

    def next_id(self, kind: str) -> int:
        self._counters[kind] = self._counters.get(kind, 0) + 1
        return self._counters[kind]

    def save(self) -> None:
        pass


@pytest.fixture(autouse=True)
def _isolate(tmp_path, monkeypatch):
    """环境隔离：run 目录与开关文件指向 tmp；清掉 LLM env 保证离线确定性。"""
    monkeypatch.setenv("APS_FALLBACK_DIR", str(tmp_path / "fb"))
    monkeypatch.setenv("APS_FEATURES_PATH", str(tmp_path / "features.json"))
    monkeypatch.delenv("LLM_API_KEY", raising=False)
    monkeypatch.delenv("LLM_BASE_URL", raising=False)
    monkeypatch.delenv("LLM_MODEL", raising=False)
    monkeypatch.delenv("LLM_PROVIDER", raising=False)
    reset_provider()
    yield
    reset_provider()


# ---------------------------------------------------------------------------
# fake runner 剧本构造（唯一注入点 = propose_reply(..., runner=..., config=...)）
# ---------------------------------------------------------------------------


def _read_issued_call_ids(work_dir: Path) -> list[str]:
    calls = work_dir.parent / "calls.jsonl"
    if not calls.exists():
        return []
    return [json.loads(line)["call_id"]
            for line in calls.read_text(encoding="utf-8").splitlines()
            if line.strip() and json.loads(line).get("status") == "issued"]


def make_success_runner(report_template: str):
    """1 次 read + stop 报告；报告模板里的 {call_id} 在 resume 时读 calls.jsonl
    填成真实签发的凭证（确定性：生成器是惰性的，凭证在 tool 事件处理后立即可读）。"""

    def runner(task: str, work_dir: Path):
        yield {"type": "tool_execution_start", "toolName": "read", "toolCallId": "t1"}
        yield {"type": "tool_execution_end", "toolName": "read", "toolCallId": "t1",
               "result": "snapshot ok"}
        ids = _read_issued_call_ids(work_dir)
        report = report_template.format(call_id=ids[0] if ids else "call-missing")
        yield {"type": "message_update", "delta": {"text": report}}
        yield {"type": "message_end", "message": {"role": "assistant", "stopReason": "stop",
               "content": [{"type": "text", "text": report}]}}
        yield {"type": "agent_end", "messages": [{"role": "assistant", "stopReason": "stop",
               "content": [{"type": "text", "text": report}]}]}

    return runner


def _cfg(tmp_path: Path, **kw) -> fallback_lane.FallbackConfig:
    return fallback_lane.FallbackConfig(pi_home=str(tmp_path / "pi-home"), **kw)


def _write_features(tmp_path: Path, features: dict) -> None:
    (tmp_path / "features.json").write_text(
        json.dumps({"version": 1, "features": features}, ensure_ascii=False),
        encoding="utf-8")


def _intent(query: str) -> IntentResult:
    return IntentResult(intent="unknown", params={"query": query},
                        confidence=0.1, source="LLM")


def _fb_audits(store: FakeStore) -> list[dict]:
    return [e for e in store.data.get("auditEvents", [])
            if e.get("action") == "agent.fallback.propose"]


# ---------------------------------------------------------------------------
# 1-2. 旧开关不再拦截 unknown：Pi 主入口恒定；开关开 = propose 路径触发
# ---------------------------------------------------------------------------


async def test_legacy_flag_off_still_routes_unknown_to_pi_primary(tmp_path):
    store = FakeStore()
    reply = await handle_intent(store, "s1", _intent("随便说说"))
    # Pi 是唯一自然语言入口：旧开关关也不再走本地话术，而是显式失败
    assert "智能助手服务暂不可用" in reply.text
    audits = _fb_audits(store)
    assert len(audits) == 1 and audits[0]["result"] == "FAILED"
    assert audits[0]["rationale"]["stopReason"].startswith("unavailable:")


def test_chat_timeout_budget_covers_real_multi_step_llm_runs(monkeypatch):
    """真实 LLM 多步工具调用实测 45~80s；默认闸必须留余量，且仍可被环境变量覆盖。"""
    monkeypatch.delenv("APS_FALLBACK_TIMEOUT_SEC", raising=False)
    assert fallback_lane.FallbackConfig().timeout_sec >= 180.0
    assert fallback_lane.FallbackConfig.from_env().timeout_sec >= 180.0

    monkeypatch.setenv("APS_FALLBACK_TIMEOUT_SEC", "45")
    assert fallback_lane.FallbackConfig.from_env().timeout_sec == 45.0


async def test_provider_outage_is_reported_as_service_unavailable(tmp_path):
    """模型网关欠费/限流/鉴权报「服务暂不可用」；APS 自身失败保持原话术。"""
    store = FakeStore()
    outcome = fallback_lane.FallbackOutcome(
        run_id="fb-20990101-000000-abc123", ok=False, run_dir=str(tmp_path),
        stop_reason="error",
        error_message=("429 Your account org-demo <ak-***> is suspended due to "
                       "insufficient balance, please recharge your account"),
    )
    reply = await fallback_lane._compose_failure_reply(
        store, "立即排产", [], "s1", outcome, primary=True)
    assert reply.text == "智能助手服务暂不可用，本次未执行任何操作。请稍后重试或联系管理员。"

    outcome.stop_reason = "breaker:timeout(90.3s>90.0s)"
    outcome.error_message = "熔断触发，运行显式标记失败"
    reply = await fallback_lane._compose_failure_reply(
        store, "立即排产", [], "s1", outcome, primary=True)
    assert reply.text == "本次处理未完成，未执行任何操作。请稍后重试。"

    outcome.stop_reason = "error:empty_report"
    outcome.error_message = "stopReason=stop 但最终报告为空，按失败处理"
    reply = await fallback_lane._compose_failure_reply(
        store, "立即排产", [], "s1", outcome, primary=True)
    assert reply.text == "本次处理未完成，未执行任何操作。请稍后重试。"
    assert not (tmp_path / "fb").exists()                  # 不可用不建 run 目录


async def test_flag_on_success_proposes(tmp_path):
    _write_features(tmp_path, {"fallback": True})
    store = FakeStore()
    reply = await fallback_lane.propose_reply(
        store, "s1", _intent("帮我分析下订单结构"),
        runner=make_success_runner("status: success\n\n报告正文：订单结构 [callId: {call_id}]"),
        config=_cfg(tmp_path))
    assert reply is not None
    assert "[智能兜底 · 草稿]" in reply.text
    assert "报告正文：订单结构" in reply.text

    audits = _fb_audits(store)
    assert len(audits) == 1
    assert audits[0]["result"] == "SUCCESS"
    assert audits[0]["power"] == "P1"
    assert harness.power_of("agent.fallback.propose") == "P1"

    runs = [p for p in (tmp_path / "fb").iterdir() if p.is_dir() and p.name != "pi-home"]
    assert len(runs) == 1
    assert (runs[0] / "result.json").is_file()
    assert (runs[0] / "calls.jsonl").is_file()
    assert (runs[0] / "outbox" / "report.md").is_file()


# ---------------------------------------------------------------------------
# 3-4. 未登记意图仍拒绝；审计落链
# ---------------------------------------------------------------------------


async def test_unregistered_intent_still_denied(tmp_path):
    _write_features(tmp_path, {"fallback": True})
    store = FakeStore()
    denied = check_tool(store, _intent("随便说说"))        # unknown 未登记 → 拒绝
    assert denied is not None
    assert "不是已登记的操作" in denied.text
    denied_audits = [e for e in store.data.get("auditEvents", [])
                     if e.get("action") == "tool.denied"]
    assert denied_audits and denied_audits[0]["result"] == "DENIED"
    # agent.fallback.propose 只是权力登记动作名，绝不是意图枚举成员
    assert "agent.fallback.propose" not in get_args(IntentName)


async def test_audit_chain_links(tmp_path):
    _write_features(tmp_path, {"fallback": True})
    store = FakeStore()
    reply = await fallback_lane.propose_reply(
        store, "s1", _intent("分析订单"),
        runner=make_success_runner("status: success\n\n报告 [callId: {call_id}]"),
        config=_cfg(tmp_path))
    assert reply is not None
    events = store.data.get("auditEvents", [])
    assert len(events) >= 2                                # tool.run + 完成事件
    for prev, cur in itertools.pairwise(events):
        assert cur["prevHash"] == prev["hash"]             # 链不断
    tool_runs = [e for e in events if e.get("action") == "tool.run"]
    assert any(str(e.get("actor", "")).startswith("pi-fallback:") for e in tool_runs)


# ---------------------------------------------------------------------------
# 5-8. 熔断 / 伪造凭证 / 异常：全部显式失败
# ---------------------------------------------------------------------------


async def test_breaker_timeout_explicit_failure(tmp_path):
    _write_features(tmp_path, {"fallback": True})
    store = FakeStore()

    def heartbeat_runner(task: str, work_dir: Path):
        while True:
            yield {"type": "harness_heartbeat"}

    reply = await fallback_lane.propose_reply(
        store, "s1", _intent("随便说说"), runner=heartbeat_runner,
        config=_cfg(tmp_path, timeout_sec=0.05))
    assert reply is not None                               # 绝不抛、永远有回复
    assert reply.text == "本次处理未完成，未执行任何操作。请稍后重试。"
    audits = _fb_audits(store)
    assert audits[0]["result"] == "FAILED"
    assert audits[0]["rationale"]["stopReason"].startswith("breaker:timeout")


async def test_breaker_max_steps(tmp_path):
    _write_features(tmp_path, {"fallback": True})
    store = FakeStore()

    def busy_runner(task: str, work_dir: Path):
        for i in range(3):
            yield {"type": "tool_execution_start", "toolName": "read",
                   "toolCallId": f"t{i}"}
        yield {"type": "agent_end", "messages": []}

    await fallback_lane.propose_reply(
        store, "s1", _intent("随便说说"), runner=busy_runner,
        config=_cfg(tmp_path, max_steps=2))
    audits = _fb_audits(store)
    assert audits[0]["result"] == "FAILED"
    assert audits[0]["rationale"]["stopReason"].startswith("breaker:max_steps")


async def test_forged_callid_rejected(tmp_path):
    _write_features(tmp_path, {"fallback": True})
    store = FakeStore()
    forged = "call-" + str(uuid.uuid4())
    report = f"status: success\n\n编造的数据结论 [callId: {forged}]"

    def forging_runner(task: str, work_dir: Path):
        yield {"type": "message_end", "message": {"role": "assistant",
               "stopReason": "stop", "content": [{"type": "text", "text": report}]}}
        yield {"type": "agent_end", "messages": []}

    reply = await fallback_lane.propose_reply(
        store, "s1", _intent("随便说说"), runner=forging_runner, config=_cfg(tmp_path))
    assert reply.text == "本次处理未完成，未执行任何操作。请稍后重试。"
    audits = _fb_audits(store)
    assert audits[0]["result"] == "FAILED"
    assert audits[0]["rationale"]["stopReason"] == "forged_citation"

    runs = [p for p in (tmp_path / "fb").iterdir() if p.is_dir() and p.name != "pi-home"]
    check = PiBridge("x", runs[0]).validate_report_citations(report)
    assert check["valid"] is False and check["missing"] == [forged]


async def test_runner_exception_fails_explicit(tmp_path):
    _write_features(tmp_path, {"fallback": True})
    store = FakeStore()

    def boom_runner(task: str, work_dir: Path):
        raise RuntimeError("spawn exploded")
        yield  # pragma: no cover - 保持生成器形态

    reply = await fallback_lane.propose_reply(
        store, "s1", _intent("随便说说"), runner=boom_runner, config=_cfg(tmp_path))
    assert reply is not None                               # propose_reply 不抛
    assert reply.text == "本次处理未完成，未执行任何操作。请稍后重试。"
    audits = _fb_audits(store)
    assert audits[0]["result"] == "FAILED"
    assert audits[0]["rationale"]["stopReason"] == "harness_error"


# ---------------------------------------------------------------------------
# 9-11. 「默认关」开关语义
# ---------------------------------------------------------------------------


def test_fallback_default_off_when_config_missing(tmp_path):
    result = load_feature_flags(str(tmp_path / "features.json"))
    assert result["features"]["fallback"]["enabled"] is False
    assert result["defaultOff"] == ["fallback"]
    assert all(info["enabled"] for key, info in result["features"].items()
               if key not in result["defaultOff"])


def test_fallback_explicit_true_enables(tmp_path):
    _write_features(tmp_path, {"fallback": True})
    result = load_feature_flags(str(tmp_path / "features.json"))
    assert result["source"] == "file"
    assert result["features"]["fallback"]["enabled"] is True


def test_fallback_non_bool_stays_off(tmp_path):
    _write_features(tmp_path, {"fallback": "yes", "orders": "no"})
    result = load_feature_flags(str(tmp_path / "features.json"))
    assert result["features"]["fallback"]["enabled"] is False   # 非 bool 回退各自默认
    assert result["features"]["orders"]["enabled"] is True      # 开的就是开、关的就是关
    assert result["error"] is not None and "fallback" in result["error"]


# ---------------------------------------------------------------------------
# 12-13. 路径越界拦截；运行时不可用显式回话术
# ---------------------------------------------------------------------------


def test_fs_read_path_escape_blocked(tmp_path):
    run_dir = tmp_path / "run"
    run_dir.mkdir()
    (run_dir / "ok.txt").write_text("界内内容", encoding="utf-8")
    bridge = PiBridge("rb-1", run_dir)
    assert bridge.handle_fs_read("ok.txt") == "界内内容"
    with pytest.raises(ToolBridgeViolation):
        bridge.handle_fs_read("../../server/contracts.py")
    with pytest.raises(ToolBridgeViolation):
        bridge.handle_fs_read(str(tmp_path / "outside.txt"))


# ---------------------------------------------------------------------------
# 14. 守卫模板 format 防回归（P1 冒烟 Bug A：注释行未转义花括号曾致真实 runner 100% 失败）
# ---------------------------------------------------------------------------


def test_write_guard_extension_real_template_format(tmp_path):
    """真实模板 str.format 不抛 + 生成守卫文件含字面量 { block: true（fake runner
    结构性摸不到这条路径，本条是真实 runner 模板回归的唯一网）。"""
    run_dir = tmp_path / "fb-20990101-000000-abcdef"
    run_dir.mkdir()
    guard = fallback_lane.write_guard_extension(run_dir)   # 不抛 KeyError 即过半
    assert guard.is_file() and guard.name == f"guard-{run_dir.name}.ts"
    content = guard.read_text(encoding="utf-8")
    assert "{ block: true" in content                       # 字面量花括号必须真实出现
    assert "{RUN_ROOT_POSIX}" not in content                # 占位符必须被替换干净
    assert run_dir.as_posix() in content


def test_write_guard_readonly_mode_preserves_p1_semantics(tmp_path):
    """readonly 模式（默认）逐字节保持 P1 围墙语义：bash/edit/write 全禁、
    无 WRITE_DIRS 放行面（P2 守卫模板参数化对 P1 的唯一约束）。"""
    run_dir = tmp_path / "fb-20990101-000000-abcdef"
    run_dir.mkdir()
    default_content = fallback_lane.write_guard_extension(run_dir).read_text(encoding="utf-8")
    explicit = fallback_lane.write_guard_extension(run_dir, mode="readonly")
    assert explicit.read_text(encoding="utf-8") == default_content
    assert "disabled by fallback guard (read-only lane)" in default_content  # P1 全禁原文
    assert "WRITE_DIRS" not in default_content              # 无写放行面
    assert 'name === "edit"' in default_content             # edit 仍在全禁名单


def test_write_guard_plan_mode_opens_work_outbox_only(tmp_path):
    """plan/execute 模式（v2 模板）：write/edit 仅放行 run 目录内 work/+outbox/，
    bash 仍全禁、只读工具防逃逸不变。"""
    run_dir = tmp_path / "fb-20990101-000000-bcdef0"
    run_dir.mkdir()
    for mode in ("plan", "execute"):
        guard = fallback_lane.write_guard_extension(run_dir, mode=mode)
        content = guard.read_text(encoding="utf-8")
        assert "{ block: true" in content                   # 字面量花括号真实出现
        assert "{RUN_ROOT_POSIX}" not in content            # 占位符替换干净
        assert "{MODE_LABEL}" not in content
        assert "WRITE_DIRS" in content
        assert "write outside work/outbox (fallback guard)" in content
        assert 'name === "bash"' in content                 # bash 仍全禁
        assert "path escapes run root" in content           # 防逃逸不变
        assert "read-only lane" not in content              # 不再是 P1 全禁语义


async def test_runtime_unavailable_fails_explicit(tmp_path, monkeypatch):
    _write_features(tmp_path, {"fallback": True})
    monkeypatch.setenv("APS_FALLBACK_PI_CLI", str(tmp_path / "no-such-cli.js"))
    popen_calls = []
    monkeypatch.setattr(subprocess, "Popen",
                        lambda *a, **kw: popen_calls.append((a, kw)))
    store = FakeStore()
    reply = await fallback_lane.propose_reply(store, "s1", _intent("随便说说"))
    assert reply is not None
    assert popen_calls == []                               # 不可用即显式失败，不触子进程
    assert "智能助手服务暂不可用" in reply.text
    assert "未执行任何操作" in reply.text
    audits = _fb_audits(store)
    assert audits[0]["result"] == "FAILED"
    assert audits[0]["rationale"]["stopReason"].startswith("unavailable:")