# -*- coding: utf-8 -*- """ tests/real_llm_chain.py — 真实 LLM 全链路验证(Agent-C 任务 1) ============================================================== 链路:计划草稿生成 →(模拟)确认 → 真实 Pi headless 沙箱执行(真实 Kimi K2 模型) → 报告落 outbox(带 callId 凭证)→ validate_report_citations 校验。 凭证机制(真实模式):编排器 on_tool_event 回调把 pi 事件流里真实发生的每次 tool_execution_start/end 登记进 ToolBridge(pi toolCallId ↔ 桥 callId 双向映射), 报告附录中的证据链引用这些真实 callId —— 与 mock 模式同一套校验代码。 用法(Git Bash,工作区根): .venv/Scripts/python.exe poc/pi-fallback/tests/real_llm_chain.py [--timeout 600] """ from __future__ import annotations import argparse import json import sys import time from pathlib import Path from _common import load_llm_env, patch_child_env, POC_ROOT sys.path.insert(0, str(POC_ROOT)) from orchestrator import BreakerConfig, Orchestrator # noqa: E402 from sandbox import create_run_dirs, new_run_id, validate_path, write_guard_extension # noqa: E402 from tool_bridge import ToolBridge # noqa: E402 from demo_task import prepare_inbox, REPORT_NAME, PLAN_NAME # noqa: E402 # pi 内置只读工具 → 桥注册表工具的映射(真实模式下 pi 侧工具名是 read/grep/find/ls) PI_TOOL_MAP = {"read": "fs_read", "grep": "fs_read", "find": "fs_read", "ls": "fs_read"} TASK = ( "你是一个数据分析 agent。当前工作目录的上一级有 inbox/05_orders.csv(订单数据," "列为 orderNo,productCode,quantity,dueDate,priority,customerName,customerLevel,status)。" "请用 read 工具读取它(路径 ../inbox/05_orders.csv),分析:" "1) 订单状态(status)分布;2) 优先级(priority)分布;3) 客户等级(customerLevel)结构。" "然后把完整的 Markdown 分析报告作为你的最后一条消息输出(不要尝试写文件,你没有写权限)," "报告包含:数据概览、三项分布的具体数字、排产建议。" ) def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--timeout", type=float, default=600.0) ap.add_argument("--model", default="moonshot/kimi-k2-0711-preview") args = ap.parse_args() creds = load_llm_env() if not creds["LLM_API_KEY"]: print("[real-chain] 未在 .env 找到 LLM_API_KEY,无法跑真实链路", file=sys.stderr) return 2 patch_child_env(creds["LLM_API_KEY"]) print(f"[real-chain] 端点={creds['LLM_BASE_URL']} 模型={args.model} key=已注入子进程环境(未落盘)") # —— propose:计划草稿 + 模拟确认 —— run_id = new_run_id("realchain") dirs = create_run_dirs(run_id) run_dir = dirs["root"] inbox_csv, source_note = prepare_inbox(dirs["inbox"]) (run_dir / PLAN_NAME).write_text( f"# 执行计划草稿(propose 阶段产物)\n\n- runId: `{run_id}`\n- 任务: 真实 LLM 分析 inbox 订单数据\n" f"- 数据来源: {source_note}\n- 模型: {args.model}\n\n" "| 步 | 工具 | 权力 | 动作 |\n|----|------|------|------|\n" "| 1 | read(→fs_read) | P0 | 读 inbox/05_orders.csv |\n" "| 2 | (LLM 推理) | - | 状态/优先级/客户等级统计 |\n" f"| 3 | fs_write(编排器侧) | P1 | 报告落 outbox/{REPORT_NAME} 并附证据链 |\n", encoding="utf-8") (run_dir / "确认记录.md").write_text( f"# 模拟确认\n\nrunId: {run_id}\n计划已(模拟)人工确认,进入 execute 阶段。\n", encoding="utf-8") print(f"[real-chain] run_id={run_id} 素材={inbox_csv.name}({source_note})") # —— execute:真实 pi headless,桥凭证随真实工具事件登记 —— guard = write_guard_extension(run_dir) bridge = ToolBridge(run_id, run_dir) id_map: dict[str, str] = {} # pi toolCallId -> bridge callId def on_tool_event(ev: dict) -> None: tcid = ev.get("toolCallId", "") if ev.get("type") == "tool_execution_start": mapped = PI_TOOL_MAP.get(ev.get("toolName", ""), "fs_read") id_map[tcid] = bridge.issue_call(mapped, ev.get("input") or {}) elif ev.get("type") == "tool_execution_end": cid = id_map.get(tcid) if cid: bridge.complete_call(cid, {"isError": ev.get("isError", False)}, ok=not ev.get("isError", False)) orch = Orchestrator(BreakerConfig(timeout_sec=args.timeout)) runner = orch.pi_headless_runner(model=args.model, tools="read,grep,find,ls", guard_ext=guard) res = orch.run(TASK, runner, run_id=run_id, on_tool_event=on_tool_event) print(f"[real-chain] 编排结果: success={res.success} stop_reason={res.stop_reason} " f"steps={res.steps} output={res.output_bytes}B elapsed={res.elapsed_sec:.1f}s") if res.error_message: print(f"[real-chain] error: {res.error_message[:300]}") # —— 报告落账 + 证据链 + 凭证校验 —— verdict = {"run_id": run_id, "success": res.success, "stop_reason": res.stop_reason, "steps": res.steps, "elapsed_sec": round(res.elapsed_sec, 1)} if res.success and res.final_text.strip(): evidence_lines = ["", "", "---", "", "## 证据链(桥层凭证,由编排器据真实事件流登记)", ""] for tcid, cid in id_map.items(): evidence_lines.append(f"- 工具调用 `{tcid}` → 凭证 [callId: {cid}]") cid_w = bridge.issue_call("fs_write", {"path": f"outbox/{REPORT_NAME}"}) target = validate_path(dirs["outbox"] / REPORT_NAME, run_dir, zone="outbox") target.write_text(res.final_text + "\n".join(evidence_lines) + "\n", encoding="utf-8") bridge.complete_call(cid_w, str(target)) cid_e = bridge.issue_call("report_emit", {"md": REPORT_NAME}) bridge.complete_call(cid_e, str(target)) v = bridge.validate_report_citations(target.read_text(encoding="utf-8")) verdict.update({"report": str(target), "citation_valid": v["valid"], "cited": len(v["cited"]), "issued": len(v["issued"]), "missing": v["missing"]}) print(f"[real-chain] 报告: {target}") print(f"[real-chain] 凭证校验: valid={v['valid']} cited={len(v['cited'])} " f"issued={len(v['issued'])} missing={v['missing']}") else: verdict["report"] = None print("[real-chain] 未产出报告(success=false 或 final_text 为空)", file=sys.stderr) blocked = run_dir / "guard-blocked-calls.jsonl" verdict["guard_blocked"] = blocked.read_text(encoding="utf-8").count("\n") if blocked.exists() else 0 (run_dir / "verdict.json").write_text(json.dumps(verdict, ensure_ascii=False, indent=2), encoding="utf-8") print(f"[real-chain] verdict: {json.dumps({k: v for k, v in verdict.items() if k != 'missing'}, ensure_ascii=False)}") return 0 if (res.success and verdict.get("citation_valid")) else 2 if __name__ == "__main__": sys.exit(main())