930 lines
44 KiB
Python
930 lines
44 KiB
Python
"""Real Pi CLI + real LLM + real workbook product-chat acceptance harness.
|
|
|
|
This module is intentionally opt-in. It never starts or calls the live web or
|
|
desktop services; all product calls go through an in-process FastAPI TestClient
|
|
and all persistence paths are redirected below one temporary artifact root.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import tempfile
|
|
import time
|
|
import traceback
|
|
import uuid
|
|
from collections.abc import Callable
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from fastapi.testclient import TestClient
|
|
|
|
from tests.auth_provider import install_test_auth
|
|
from tests.workbook_acceptance import load_expectations, resolve_source
|
|
|
|
OPT_IN_ENV = "APS_REAL_PI_E2E"
|
|
ARTIFACT_ENV = "APS_REAL_PI_E2E_ARTIFACT_DIR"
|
|
REPRO_COMMAND = (
|
|
".\\scripts\\run-pi-real-workbook-e2e.ps1 "
|
|
"-SourcePath \"$env:ROUND87_SOURCE\" -ArtifactRoot \"<ARTIFACT_ROOT>\""
|
|
)
|
|
|
|
|
|
class RealPiE2EEnvironmentError(RuntimeError):
|
|
"""The opt-in run is missing a required real-Pi/runtime dependency."""
|
|
|
|
|
|
def _sha256_bytes(raw: bytes) -> str:
|
|
return hashlib.sha256(raw).hexdigest()
|
|
|
|
|
|
def _sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def _json_text(value: Any) -> str:
|
|
return json.dumps(value, ensure_ascii=False, indent=2, default=str)
|
|
|
|
|
|
def _read_json(path: Path) -> Any:
|
|
return json.loads(path.read_text(encoding="utf-8"))
|
|
|
|
|
|
def _read_jsonl(path: Path) -> list[dict[str, Any]]:
|
|
if not path.is_file():
|
|
return []
|
|
rows: list[dict[str, Any]] = []
|
|
for line in path.read_text(encoding="utf-8", errors="replace").splitlines():
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
value = json.loads(line)
|
|
except json.JSONDecodeError:
|
|
rows.append({"parseError": line})
|
|
continue
|
|
rows.append(value if isinstance(value, dict) else {"value": value})
|
|
return rows
|
|
|
|
|
|
def _safe_name(value: str) -> str:
|
|
return re.sub(r"[^A-Za-z0-9_.-]+", "-", value).strip("-") or "step"
|
|
|
|
|
|
def _parse_sse(raw: str) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
|
|
chunks: list[dict[str, Any]] = []
|
|
events: list[dict[str, Any]] = []
|
|
for line in raw.splitlines():
|
|
if line.startswith("data:"):
|
|
payload_text = line[5:].lstrip()
|
|
record: dict[str, Any] = {"raw": line}
|
|
try:
|
|
event = json.loads(payload_text)
|
|
except json.JSONDecodeError as exc:
|
|
record["parseError"] = str(exc)
|
|
else:
|
|
record["event"] = event
|
|
if isinstance(event, dict):
|
|
events.append(event)
|
|
chunks.append(record)
|
|
elif line.strip():
|
|
chunks.append({"raw": line})
|
|
return chunks, events
|
|
|
|
|
|
def _block(result: ChatResult, block_type: str) -> dict[str, Any] | None:
|
|
return next((row for row in result.blocks if row.get("type") == block_type), None)
|
|
|
|
|
|
@dataclass
|
|
class PiRunEvidence:
|
|
run_id: str
|
|
path: Path
|
|
inventory: dict[str, dict[str, Any]]
|
|
events: list[dict[str, Any]]
|
|
calls: list[dict[str, Any]]
|
|
result: dict[str, Any] | None
|
|
chat_tools: list[dict[str, Any]]
|
|
|
|
def to_json(self) -> dict[str, Any]:
|
|
return {
|
|
"runId": self.run_id,
|
|
"path": str(self.path),
|
|
"inventory": self.inventory,
|
|
"events": self.events,
|
|
"calls": self.calls,
|
|
"result": self.result,
|
|
"chatTools": self.chat_tools,
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class ChatResult:
|
|
label: str
|
|
prompt: str
|
|
attempt: int
|
|
status_code: int
|
|
raw_sse: str
|
|
sse_chunks: list[dict[str, Any]]
|
|
events: list[dict[str, Any]]
|
|
blocks: list[dict[str, Any]]
|
|
text: str
|
|
errors: list[str]
|
|
new_run_ids: list[str]
|
|
pi_runs: list[PiRunEvidence] = field(default_factory=list)
|
|
|
|
def to_json(self) -> dict[str, Any]:
|
|
return {
|
|
"label": self.label,
|
|
"prompt": self.prompt,
|
|
"attempt": self.attempt,
|
|
"statusCode": self.status_code,
|
|
"text": self.text,
|
|
"errors": self.errors,
|
|
"blocks": self.blocks,
|
|
"newRunIds": self.new_run_ids,
|
|
"sseChunkCount": len(self.sse_chunks),
|
|
}
|
|
|
|
|
|
class RealPiWorkbookE2E:
|
|
"""Run one isolated product-chat acceptance flow with the real Pi runner."""
|
|
|
|
def __init__(self, monkeypatch: Any, *, source_path: str | Path | None = None,
|
|
artifact_base: str | Path | None = None) -> None:
|
|
if os.environ.get(OPT_IN_ENV) != "1":
|
|
raise RealPiE2EEnvironmentError(
|
|
f"{OPT_IN_ENV}=1 is required; ordinary pytest must not invoke the real model."
|
|
)
|
|
self.monkeypatch = monkeypatch
|
|
self.repo_root = Path(__file__).resolve().parents[2]
|
|
self.expectations = load_expectations()
|
|
self.source_path_override = Path(source_path).expanduser() if source_path is not None else None
|
|
configured_base = artifact_base or os.environ.get(ARTIFACT_ENV)
|
|
self.artifact_base = (
|
|
Path(configured_base).expanduser().resolve()
|
|
if configured_base
|
|
else Path(tempfile.mkdtemp(prefix="aps-pi-real-e2e-")).resolve()
|
|
)
|
|
self.artifact_base.mkdir(parents=True, exist_ok=True)
|
|
self.artifact_root = self.artifact_base / "pending-source-resolution"
|
|
self.artifact_root.mkdir(parents=True, exist_ok=True)
|
|
self.evidence_dir = self.artifact_root / "evidence"
|
|
self.evidence_dir.mkdir(parents=True, exist_ok=True)
|
|
|
|
self.source: Path | None = None
|
|
self.source_sha_before = ""
|
|
self.source_sha_after: str | None = None
|
|
self.client: TestClient | None = None
|
|
self.project_id = ""
|
|
self.session_id = ""
|
|
self.pi_cli: Path | None = None
|
|
self.pi_cli_sha256 = ""
|
|
self.node_bin = ""
|
|
self.node_version = ""
|
|
self.steps: list[dict[str, Any]] = []
|
|
self.pi_runs: list[dict[str, Any]] = []
|
|
self.assertions: list[str] = []
|
|
self.persisted_messages: list[dict[str, Any]] = []
|
|
|
|
@property
|
|
def fallback_root(self) -> Path:
|
|
configured = (os.environ.get("APS_FALLBACK_DIR") or "").strip()
|
|
return Path(configured).resolve() if configured else self.artifact_root / "runtime" / "fallback"
|
|
|
|
def _record_assertion(self, text: str) -> None:
|
|
self.assertions.append(text)
|
|
|
|
def _redact(self, text: str) -> str:
|
|
secret = (os.environ.get("LLM_API_KEY") or "").strip()
|
|
return text.replace(secret, "***REDACTED***") if secret else text
|
|
|
|
def _write_json(self, path: Path, payload: Any) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(_json_text(payload), encoding="utf-8")
|
|
|
|
@staticmethod
|
|
def _reset_runtime_caches() -> None:
|
|
"""Reset path-bound singletons before and after an isolated run."""
|
|
try:
|
|
from server.db.database import reset_engine
|
|
|
|
reset_engine()
|
|
except Exception: # noqa: BLE001, S110 - best-effort cache reset before isolation
|
|
pass
|
|
try:
|
|
from server.state import store as world_store
|
|
|
|
with world_store._stores_lock:
|
|
world_store._stores.clear()
|
|
except Exception: # noqa: BLE001, S110 - best-effort cache reset before isolation
|
|
pass
|
|
try:
|
|
from server.state.branches import reset_branch_stores
|
|
|
|
reset_branch_stores()
|
|
except Exception: # noqa: BLE001, S110 - best-effort cache reset before isolation
|
|
pass
|
|
try:
|
|
from server.agent_core.plan_runtime import reset_plan_stores
|
|
|
|
reset_plan_stores()
|
|
except Exception: # noqa: BLE001, S110 - best-effort cache reset before isolation
|
|
pass
|
|
try:
|
|
from server.agent_core.mcp_bus import reset_mcp_bus
|
|
|
|
reset_mcp_bus()
|
|
except Exception: # noqa: BLE001, S110 - best-effort cache reset before isolation
|
|
pass
|
|
try:
|
|
from server.knowledge import assets, embedding, preferences
|
|
|
|
with assets._stores_lock:
|
|
assets._stores.clear()
|
|
with preferences._stores_lock:
|
|
preferences._stores.clear()
|
|
with embedding._stores_lock:
|
|
embedding._stores.clear()
|
|
embedding._provider = None
|
|
except Exception: # noqa: BLE001, S110 - best-effort cache reset before isolation
|
|
pass
|
|
try:
|
|
from server.agent_core import fallback_lane
|
|
|
|
fallback_lane._MODEL_CACHE.update({"model": None, "note": "", "ts": 0.0})
|
|
except Exception: # noqa: BLE001, S110 - best-effort cache reset before isolation
|
|
pass
|
|
|
|
def _configure_environment(self) -> None:
|
|
runtime = self.artifact_root / "runtime"
|
|
data = runtime / "data"
|
|
home = runtime / "home"
|
|
for path in (
|
|
runtime, data, home, runtime / "fallback", runtime / "pi-home",
|
|
data / "audit-ledger", data / "audit-mirror", data / "branches",
|
|
):
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
path_env: dict[str, Path] = {
|
|
"APS_HOME": home,
|
|
"APS_DATA_DIR": data,
|
|
"APS_DB_PATH": data / "master.db",
|
|
"APS_WORLD_PATH": data / "world.json",
|
|
"APS_PROJECTS_PATH": runtime / "sessions" / "workspace.json",
|
|
"APS_KNOWLEDGE_PATH": data / "knowledge.json",
|
|
"APS_CHECKPOINT_PATH": data / "checkpoints.json",
|
|
"APS_PREFERENCE_PATH": data / "preferences.json",
|
|
"APS_EMBEDDINGS_PATH": data / "embeddings.json",
|
|
"APS_APPROVAL_PATH": data / "approvals.json",
|
|
"APS_AUDIT_LEDGER_DIR": data / "audit-ledger",
|
|
"APS_AUDIT_MIRROR_DIR": data / "audit-mirror",
|
|
"APS_AUTOMATION_STATE_PATH": data / "automation.json",
|
|
"APS_MCP_BUS_PATH": data / "mcp-bus.json",
|
|
"APS_BRANCH_PATH": data / "branches.json",
|
|
"APS_BRANCH_DIR": data / "branches",
|
|
"APS_GOLDEN_CACHE": data / "golden-tests.json",
|
|
"APS_MESH_PATH": data / "mesh.json",
|
|
"APS_FALLBACK_DIR": runtime / "fallback",
|
|
"APS_FALLBACK_PI_HOME": runtime / "pi-home",
|
|
}
|
|
for name, value in path_env.items():
|
|
self.monkeypatch.setenv(name, str(value))
|
|
for name in (
|
|
"APS_DATABASE_URL", "APS_DB_DISABLED", "APS_SEED_PACK",
|
|
"APS_WORKBOOK_PROFILE_DIR", "MES_HTTP_BASE_URL", "APS_FALLBACK_MODEL",
|
|
):
|
|
self.monkeypatch.delenv(name, raising=False)
|
|
self.monkeypatch.setenv("APS_MODE", "web")
|
|
self.monkeypatch.setenv("APS_AUTH_ENABLED", "1")
|
|
self.monkeypatch.setenv("APS_SEED_DEMO", "0")
|
|
self.monkeypatch.setenv("APS_APPROVAL_BACKEND", "file")
|
|
self.monkeypatch.setenv("APS_AUTH_COOKIE_SECURE", "0")
|
|
self.monkeypatch.setenv("APS_AUTOMATION_DRIVER", "0")
|
|
self.monkeypatch.setenv("APS_AUDIT_MIRROR", "1")
|
|
self.monkeypatch.setenv("EMBEDDING_PROVIDER", "off")
|
|
self.monkeypatch.setenv("ROUND87_REQUIRED", "1")
|
|
self.monkeypatch.setenv("ROUND87_SOURCE", str(self.source))
|
|
self.monkeypatch.setenv("APS_REAL_PI_E2E_ARTIFACT_DIR", str(self.artifact_base))
|
|
self.monkeypatch.setenv("APS_FALLBACK_TIMEOUT_SEC", os.environ.get("APS_FALLBACK_TIMEOUT_SEC", "300"))
|
|
self.monkeypatch.setenv("APS_FALLBACK_EXEC_TIMEOUT_SEC", os.environ.get("APS_FALLBACK_EXEC_TIMEOUT_SEC", "300"))
|
|
self.monkeypatch.setenv("APS_FALLBACK_MAX_STEPS", os.environ.get("APS_FALLBACK_MAX_STEPS", "50"))
|
|
self.monkeypatch.setenv("APS_FALLBACK_EXEC_MAX_STEPS", os.environ.get("APS_FALLBACK_EXEC_MAX_STEPS", "50"))
|
|
|
|
|
|
def _resolve_runtime_paths(self) -> None:
|
|
configured_cli = (os.environ.get("APS_FALLBACK_PI_CLI") or "").strip()
|
|
candidates: list[Path] = []
|
|
if configured_cli:
|
|
candidates.append(Path(configured_cli).expanduser())
|
|
candidates.append(
|
|
self.repo_root / "poc" / "pi-fallback" / "runtime" / "node_modules"
|
|
/ "@mariozechner" / "pi-coding-agent" / "dist" / "cli.js"
|
|
)
|
|
cli: Path | None = None
|
|
for candidate in candidates:
|
|
resolved = candidate if candidate.is_absolute() else (self.repo_root / candidate)
|
|
resolved = resolved.resolve()
|
|
if resolved.is_file():
|
|
cli = resolved
|
|
break
|
|
if cli is None:
|
|
raise RealPiE2EEnvironmentError(
|
|
"Real Pi CLI not found. Set APS_FALLBACK_PI_CLI to pi-coding-agent/dist/cli.js."
|
|
)
|
|
self.pi_cli = cli
|
|
self.pi_cli_sha256 = _sha256_file(cli)
|
|
self.monkeypatch.setenv("APS_FALLBACK_PI_CLI", str(cli))
|
|
configured_node = (os.environ.get("APS_FALLBACK_NODE") or "").strip()
|
|
if configured_node:
|
|
node_candidate = Path(configured_node).expanduser()
|
|
if not node_candidate.is_absolute():
|
|
node_candidate = (self.repo_root / node_candidate).resolve()
|
|
node = str(node_candidate) if node_candidate.is_file() else shutil.which(configured_node)
|
|
else:
|
|
node = shutil.which("node")
|
|
if not node:
|
|
raise RealPiE2EEnvironmentError("Node.js is required for the real Pi CLI but was not found.")
|
|
self.node_bin = str(node)
|
|
self.monkeypatch.setenv("APS_FALLBACK_NODE", self.node_bin)
|
|
version = subprocess.run(
|
|
[self.node_bin, "--version"], capture_output=True, text=True, timeout=15, check=False,
|
|
)
|
|
if version.returncode != 0:
|
|
raise RealPiE2EEnvironmentError(f"Node.js version probe failed: {version.stderr.strip()}")
|
|
self.node_version = version.stdout.strip()
|
|
if not (os.environ.get("LLM_BASE_URL") or "").strip():
|
|
raise RealPiE2EEnvironmentError("LLM_BASE_URL is required for the real Pi LLM run.")
|
|
if not (os.environ.get("LLM_API_KEY") or "").strip():
|
|
raise RealPiE2EEnvironmentError("LLM_API_KEY is required for the real Pi LLM run.")
|
|
|
|
def prepare(self) -> None:
|
|
selected = self.source_path_override if self.source_path_override is not None else os.environ.get("ROUND87_SOURCE")
|
|
source = resolve_source(selected, required=True, expectations=self.expectations)
|
|
if source is None:
|
|
raise RealPiE2EEnvironmentError("ROUND87_SOURCE did not resolve to a workbook.")
|
|
self.source = source
|
|
self.source_sha_before = _sha256_file(source)
|
|
stamp = time.strftime("%Y%m%d-%H%M%S")
|
|
self.artifact_root = self.artifact_base / f"{stamp}-{uuid.uuid4().hex[:8]}-{self.source_sha_before[:12]}"
|
|
self.artifact_root.mkdir(parents=True, exist_ok=True)
|
|
self.evidence_dir = self.artifact_root / "evidence"
|
|
self.evidence_dir.mkdir(parents=True, exist_ok=True)
|
|
self._reset_runtime_caches()
|
|
self._configure_environment()
|
|
self._resolve_runtime_paths()
|
|
from server.agent_core import harness
|
|
|
|
harness.configure_approval_store(str(self.artifact_root / "runtime" / "data" / "approvals.json"))
|
|
self._reset_runtime_caches()
|
|
install_test_auth(self.monkeypatch, "real-pi-workbook-acceptance")
|
|
from server.gateway.app import create_app
|
|
|
|
self.client = TestClient(create_app(), raise_server_exceptions=False)
|
|
self.client.__enter__()
|
|
login = self.client.post(
|
|
"/api/auth/login",
|
|
json={"method": "password", "username": "planner", "password": "test"},
|
|
)
|
|
if login.status_code != 200:
|
|
raise AssertionError(f"test login failed: {login.status_code} {login.text}")
|
|
created = self.client.post("/api/projects", json={"name": f"真实 Pi 工作簿验收 {self.source.stem}"})
|
|
if created.status_code != 200:
|
|
raise AssertionError(f"project create failed: {created.status_code} {created.text}")
|
|
created_json = created.json()
|
|
self.project_id = str(created_json["project"]["id"])
|
|
self.session_id = str(created_json["session"]["id"])
|
|
raw = self.source.read_bytes()
|
|
if _sha256_bytes(raw) != self.source_sha_before:
|
|
raise AssertionError("source workbook changed before upload")
|
|
upload = self.client.post(
|
|
f"/api/projects/{self.project_id}/files/upload",
|
|
files=[(
|
|
"files",
|
|
(self.source.name, raw,
|
|
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"),
|
|
)],
|
|
)
|
|
if upload.status_code != 200:
|
|
raise AssertionError(f"workbook upload failed: {upload.status_code} {upload.text}")
|
|
if self.source.name not in (upload.json().get("saved") or []):
|
|
raise AssertionError(f"upload response did not include the source workbook: {upload.text}")
|
|
self._record_assertion("real workbook uploaded through /api/projects/{id}/files/upload")
|
|
|
|
def close(self) -> None:
|
|
try:
|
|
if self.client is not None:
|
|
self.client.__exit__(None, None, None)
|
|
finally:
|
|
self.client = None
|
|
self._reset_runtime_caches()
|
|
|
|
def _get_json(self, path: str) -> dict[str, Any]:
|
|
if self.client is None:
|
|
raise RuntimeError("client is not ready")
|
|
response = self.client.get(path)
|
|
if response.status_code != 200:
|
|
raise AssertionError(f"GET {path} failed: {response.status_code} {response.text}")
|
|
value = response.json()
|
|
if not isinstance(value, dict):
|
|
raise TypeError(f"GET {path} did not return an object")
|
|
return value
|
|
|
|
def _run_ids(self) -> set[str]:
|
|
root = self.fallback_root
|
|
if not root.is_dir():
|
|
return set()
|
|
return {row.name for row in root.iterdir() if row.is_dir() and row.name.startswith("fb-")}
|
|
|
|
def _capture_pi_runs(self, new_run_ids: set[str]) -> list[PiRunEvidence]:
|
|
captured: list[PiRunEvidence] = []
|
|
root = self.fallback_root
|
|
for run_id in sorted(new_run_ids):
|
|
run_dir = root / run_id
|
|
inventory: dict[str, dict[str, Any]] = {}
|
|
for path in sorted(run_dir.rglob("*")):
|
|
if not path.is_file():
|
|
continue
|
|
rel = path.relative_to(run_dir).as_posix()
|
|
inventory[rel] = {
|
|
"path": str(path), "size": path.stat().st_size, "sha256": _sha256_file(path),
|
|
}
|
|
chat_tools: list[dict[str, Any]] = []
|
|
tools_dir = run_dir / "outbox" / "chat-tools"
|
|
if tools_dir.is_dir():
|
|
for request_path in sorted(tools_dir.glob("*.json")):
|
|
if request_path.name.endswith(".result.json"):
|
|
continue
|
|
result_path = tools_dir / f"{request_path.stem}.result.json"
|
|
request_value: Any = None
|
|
result_value: Any = None
|
|
try:
|
|
request_value = _read_json(request_path)
|
|
except (OSError, ValueError) as exc:
|
|
request_value = {"parseError": str(exc)}
|
|
if result_path.is_file():
|
|
try:
|
|
result_value = _read_json(result_path)
|
|
except (OSError, ValueError) as exc:
|
|
result_value = {"parseError": str(exc)}
|
|
chat_tools.append({
|
|
"requestFile": str(request_path),
|
|
"resultFile": str(result_path) if result_path.is_file() else None,
|
|
"request": request_value,
|
|
"result": result_value,
|
|
})
|
|
result_path = run_dir / "result.json"
|
|
result_value: dict[str, Any] | None = None
|
|
if result_path.is_file():
|
|
try:
|
|
result_value = _read_json(result_path)
|
|
except (OSError, ValueError) as exc:
|
|
result_value = {"parseError": str(exc)}
|
|
evidence = PiRunEvidence(
|
|
run_id=run_id,
|
|
path=run_dir,
|
|
inventory=inventory,
|
|
events=_read_jsonl(run_dir / "events.jsonl"),
|
|
calls=_read_jsonl(run_dir / "calls.jsonl"),
|
|
result=result_value,
|
|
chat_tools=chat_tools,
|
|
)
|
|
self._write_json(self.evidence_dir / "pi-runs" / f"{run_id}.json", evidence.to_json())
|
|
self.pi_runs.append({"runId": run_id, "path": str(run_dir), "chatToolCount": len(chat_tools)})
|
|
captured.append(evidence)
|
|
return captured
|
|
|
|
|
|
def chat(self, prompt: str, *, label: str, attempt: int = 1) -> ChatResult:
|
|
if self.client is None:
|
|
raise RuntimeError("client is not ready")
|
|
before = self._run_ids()
|
|
response = self.client.post(
|
|
"/api/chat",
|
|
json={"text": prompt, "sessionId": self.session_id, "projectId": self.project_id},
|
|
)
|
|
raw = response.text
|
|
chunks, events = _parse_sse(raw)
|
|
blocks = [
|
|
row["block"] for row in events
|
|
if row.get("type") == "block" and isinstance(row.get("block"), dict)
|
|
]
|
|
text = "".join(str(row.get("text") or "") for row in events if row.get("type") == "token")
|
|
errors = [str(row.get("message") or "") for row in events if row.get("type") == "error"]
|
|
new_ids = self._run_ids() - before
|
|
result = ChatResult(
|
|
label=label, prompt=prompt, attempt=attempt, status_code=response.status_code,
|
|
raw_sse=raw, sse_chunks=chunks, events=events, blocks=blocks, text=text,
|
|
errors=errors, new_run_ids=sorted(new_ids), pi_runs=self._capture_pi_runs(new_ids),
|
|
)
|
|
self._write_json(
|
|
self.evidence_dir / "chats" / f"{_safe_name(label)}-{attempt}.json", result.to_json(),
|
|
)
|
|
(self.evidence_dir / "chats" / f"{_safe_name(label)}-{attempt}.sse.txt").write_text(
|
|
raw, encoding="utf-8",
|
|
)
|
|
self.steps.append(result.to_json())
|
|
return result
|
|
|
|
def chat_until(self, prompt: str, predicate: Callable[[ChatResult], bool], *,
|
|
label: str, attempts: int = 2) -> ChatResult:
|
|
last: ChatResult | None = None
|
|
for attempt in range(1, attempts + 1):
|
|
last = self.chat(prompt, label=label, attempt=attempt)
|
|
if predicate(last):
|
|
return last
|
|
if last.new_run_ids and not last.errors:
|
|
break
|
|
assert last is not None
|
|
return last
|
|
|
|
def _fail_with_chat(self, result: ChatResult, message: str) -> None:
|
|
tail = result.text[-600:] if result.text else result.raw_sse[-600:]
|
|
raise AssertionError(f"{message}; status={result.status_code}; tail={tail!r}")
|
|
|
|
def _assert_initial_world(self) -> None:
|
|
master = self._get_json("/api/master")
|
|
flex = self._get_json("/api/flex/world")
|
|
if master.get("materials") != []:
|
|
raise AssertionError("isolated project unexpectedly contains master materials")
|
|
if flex.get("orders") != [] or flex.get("latestVersion") is not None or flex.get("workOrders") != []:
|
|
raise AssertionError("isolated project unexpectedly contains scheduling state")
|
|
self._record_assertion("isolated APS world starts empty")
|
|
|
|
|
|
def _failure_injection_then_retry(self) -> ChatResult:
|
|
"""Prove fail-closed recovery before the real model analysis."""
|
|
key = (os.environ.get("LLM_API_KEY") or "").strip()
|
|
if not key:
|
|
raise RealPiE2EEnvironmentError("LLM_API_KEY must be present before failure injection")
|
|
failed: ChatResult | None = None
|
|
try:
|
|
self.monkeypatch.delenv("LLM_API_KEY", raising=False)
|
|
failed = self.chat("分析一下数据文件", label="failure-injection-missing-llm-key")
|
|
finally:
|
|
self.monkeypatch.setenv("LLM_API_KEY", key)
|
|
if failed is None:
|
|
raise AssertionError("failure injection did not return a chat result")
|
|
if failed.errors:
|
|
raise AssertionError(f"failure injection leaked an SSE error: {failed.errors}")
|
|
if failed.new_run_ids:
|
|
raise AssertionError("failure injection unexpectedly started a Pi run")
|
|
if _block(failed, "folder-pack") is not None or _block(failed, "confirm-card") is not None:
|
|
raise AssertionError("failure injection unexpectedly produced a business review card")
|
|
if not any(token in failed.text for token in ("暂不可用", "未执行")):
|
|
self._fail_with_chat(failed, "failure injection did not fail closed with a visible message")
|
|
master = self._get_json("/api/master")
|
|
flex = self._get_json("/api/flex/world")
|
|
if master.get("materials") != [] or flex.get("orders") not in ([], None):
|
|
raise AssertionError("failure injection wrote business state")
|
|
self._record_assertion("missing LLM key failed closed; business world unchanged; then key restored")
|
|
|
|
analysis = self.chat_until(
|
|
"分析一下数据文件",
|
|
lambda row: _block(row, "folder-pack") is not None and _block(row, "confirm-card") is not None,
|
|
label="real-pi-analysis",
|
|
attempts=2,
|
|
)
|
|
folder = _block(analysis, "folder-pack")
|
|
card = _block(analysis, "confirm-card")
|
|
if analysis.errors:
|
|
self._fail_with_chat(analysis, "real Pi analysis returned an SSE error")
|
|
if folder is None or card is None:
|
|
self._fail_with_chat(analysis, "real Pi analysis did not return folder-pack + confirmation card")
|
|
props = folder["props"]
|
|
expected_counts = self.expectations["entityCounts"]
|
|
if props.get("summary") != self.expectations["reviewCounts"]:
|
|
raise AssertionError(f"folder-pack summary mismatch: {props.get('summary')!r}")
|
|
entity_counts = props.get("entityCounts") or {}
|
|
for key_name in ("orders", "materials", "routing", "equipment", "personnel", "sandboxOrders"):
|
|
if entity_counts.get(key_name) != expected_counts.get(key_name):
|
|
raise AssertionError(
|
|
f"folder-pack entity count {key_name}={entity_counts.get(key_name)!r}, "
|
|
f"oracle={expected_counts.get(key_name)!r}"
|
|
)
|
|
if len(props.get("sheetSummary") or []) != len(self.expectations["sheets"]):
|
|
raise AssertionError("folder-pack sheet count differs from the independent oracle")
|
|
source_sha = ((props.get("source") or {}).get("sha256") or "").lower()
|
|
if source_sha != self.expectations["sourceSha256"].lower():
|
|
raise AssertionError("folder-pack source SHA does not match the independent oracle")
|
|
if card["props"].get("action") != "import.commit":
|
|
raise AssertionError(f"unexpected adoption action: {card['props'].get('action')!r}")
|
|
if props.get("adopted") is not False:
|
|
raise AssertionError("folder-pack must remain unreviewed before confirmation")
|
|
if _block(analysis, "flex-schedule") is not None:
|
|
raise AssertionError("real Pi must not schedule before adoption")
|
|
if self._get_json("/api/master").get("materials") != []:
|
|
raise AssertionError("analysis wrote master data before confirmation")
|
|
self._record_assertion(
|
|
"real Pi folder.analyze returned oracle counts/SHA and an import.commit confirmation card"
|
|
)
|
|
return analysis
|
|
|
|
|
|
def _confirm_adoption(self, analysis: ChatResult) -> dict[str, Any]:
|
|
card = _block(analysis, "confirm-card")
|
|
if card is None:
|
|
raise AssertionError("analysis confirmation card is missing")
|
|
confirm_id = str(card["props"].get("confirmId") or "")
|
|
if not confirm_id:
|
|
raise AssertionError("confirmation card has no confirmId")
|
|
if self.client is None:
|
|
raise RuntimeError("client is not ready")
|
|
response = self.client.post(
|
|
"/api/actions/confirm",
|
|
json={"sessionId": self.session_id, "confirmId": confirm_id, "approve": True},
|
|
)
|
|
if response.status_code != 200:
|
|
raise AssertionError(f"adoption confirmation failed: {response.status_code} {response.text}")
|
|
payload = response.json()
|
|
if payload.get("errorCode"):
|
|
raise AssertionError(f"adoption confirmation returned errorCode={payload['errorCode']}: {payload}")
|
|
if payload.get("refresh") is not True:
|
|
raise AssertionError(f"adoption did not request product refresh: {payload}")
|
|
self._write_json(self.evidence_dir / "adoption-confirm.json", payload)
|
|
expected_world = self.expectations["worldCounts"]
|
|
master = self._get_json("/api/master")
|
|
flex = self._get_json("/api/flex/world")
|
|
if len(master.get("materials") or []) != expected_world["flexMaterials"]:
|
|
raise AssertionError("adopted material count differs from the oracle")
|
|
if len(master.get("equipment") or []) != expected_world["flexEquipment"]:
|
|
raise AssertionError("adopted equipment count differs from the oracle")
|
|
routing_steps = sum(len(row.get("steps") or []) for row in master.get("routings") or [])
|
|
if routing_steps != expected_world["flexRoutings"]:
|
|
raise AssertionError("adopted routing step count differs from the oracle")
|
|
if len(flex.get("orders") or []) != expected_world["flexOrders"]:
|
|
raise AssertionError("adopted formal order count differs from the oracle")
|
|
if flex.get("latestVersion") is not None or flex.get("workOrders") != []:
|
|
raise AssertionError("adoption unexpectedly created a schedule")
|
|
self._record_assertion(
|
|
"confirmation adopted the workbook; material/equipment/routing/order counts match oracle"
|
|
)
|
|
return payload
|
|
|
|
def _schedule(self) -> ChatResult:
|
|
result = self.chat_until(
|
|
"立即排产",
|
|
lambda row: _block(row, "flex-schedule") is not None,
|
|
label="real-pi-immediate-schedule",
|
|
attempts=2,
|
|
)
|
|
if result.errors:
|
|
self._fail_with_chat(result, "real Pi scheduling returned an SSE error")
|
|
if _block(result, "flex-schedule") is None:
|
|
self._fail_with_chat(result, "real Pi scheduling did not return flex-schedule")
|
|
self._record_assertion("real Pi selected the scheduling tool for the natural-language command 立即排产")
|
|
return result
|
|
|
|
|
|
def _assert_schedule_and_no_dispatch(self, schedule: ChatResult) -> None:
|
|
block = _block(schedule, "flex-schedule")
|
|
if block is None:
|
|
raise AssertionError("flex-schedule block is missing")
|
|
props = block["props"]
|
|
stats = props.get("stats") or {}
|
|
expected_orders = int(self.expectations["entityCounts"]["orders"])
|
|
formal = {str(value) for value in self.expectations["formalOrderNos"]}
|
|
sandbox = {str(value) for value in self.expectations["sandboxOrderNos"]}
|
|
plan = self.expectations["trialPlan"]
|
|
scheduled_expected = {str(value) for value in plan["scheduledOrderNos"]}
|
|
blocked_expected = {str(value) for value in plan["blockedOrderNos"]}
|
|
if props.get("trialOnly") is not True:
|
|
raise AssertionError(f"flex-schedule trialOnly must be true: {props.get('trialOnly')!r}")
|
|
if props.get("productionReady") is not False:
|
|
raise AssertionError(f"flex-schedule productionReady must be false: {props.get('productionReady')!r}")
|
|
if stats.get("orderCount") != expected_orders:
|
|
raise AssertionError(f"flex-schedule orderCount mismatch: {stats.get('orderCount')!r}")
|
|
if stats.get("blockedOrderCount") != len(blocked_expected):
|
|
raise AssertionError(f"flex-schedule blockedOrderCount mismatch: {stats.get('blockedOrderCount')!r}")
|
|
if int(stats.get("woCount") or 0) <= 0:
|
|
raise AssertionError(f"trial draft must carry work orders: {stats.get('woCount')!r}")
|
|
if int(stats.get("vlCount") or 0) != len(scheduled_expected):
|
|
raise AssertionError(f"flex-schedule virtual line count mismatch: {stats.get('vlCount')!r}")
|
|
if props.get("downloadUrl") or props.get("filename"):
|
|
raise AssertionError("trial schedule must not expose a published/downloadable production file")
|
|
lines = props.get("lines") or []
|
|
rows = {str(row.get("orderNo")): row for row in lines if row.get("orderNo")}
|
|
if set(rows) != formal:
|
|
raise AssertionError(f"flex-schedule line orders differ from formal oracle: {sorted(rows)}")
|
|
if set(rows) & sandbox:
|
|
raise AssertionError("sandbox order leaked into the formal trial schedule")
|
|
scheduled = {order_no for order_no, row in rows.items() if row.get("status") == "scheduled"}
|
|
blocked = {order_no for order_no, row in rows.items() if row.get("status") == "blocked"}
|
|
if scheduled != scheduled_expected:
|
|
raise AssertionError(f"trial scheduled orders differ from oracle: {sorted(scheduled)}")
|
|
if blocked != blocked_expected:
|
|
raise AssertionError(f"trial blocked orders differ from oracle: {sorted(blocked)}")
|
|
for order_no in sorted(blocked):
|
|
row = rows[order_no]
|
|
if not str(row.get("blockReason") or "").strip():
|
|
raise AssertionError(f"blocked order has no real reason: {row}")
|
|
if row.get("steps"):
|
|
raise AssertionError(f"blocked order carries schedule steps: {row}")
|
|
for order_no in sorted(scheduled):
|
|
if not rows[order_no].get("steps"):
|
|
raise AssertionError(f"scheduled order carries no work-order steps: {rows[order_no]}")
|
|
conflicts = props.get("conflicts") or []
|
|
assumptions = [item for item in conflicts if item.get("severity") == plan["assumptionSeverity"]]
|
|
if {str(item.get("type")) for item in assumptions} != set(plan["assumptionTypes"]):
|
|
raise AssertionError(f"trial assumptions differ from oracle: {sorted(str(i.get('type')) for i in assumptions)}")
|
|
for item in assumptions:
|
|
if not str(item.get("description") or "").startswith(plan["assumptionDescriptionPrefix"]):
|
|
raise AssertionError(f"trial assumption is not marked as such: {item}")
|
|
if not str(item.get("suggestion") or item.get("suggestedSolution") or "").strip():
|
|
raise AssertionError(f"trial assumption has no follow-up advice: {item}")
|
|
blocking = [item for item in conflicts if item.get("severity") == "CRITICAL"]
|
|
blocking_types = {str(item.get("type")) for item in blocking}
|
|
if not blocking or not blocking_types <= set(plan["blockedReasonTypes"]):
|
|
raise AssertionError(f"blocking conflict types differ from oracle: {sorted(blocking_types)}")
|
|
if {str(item.get("orderNo")) for item in blocking} != blocked_expected:
|
|
raise AssertionError(f"blocking conflicts do not match blocked orders: {blocking}")
|
|
self._record_assertion(
|
|
f"trial schedule: {len(scheduled)} scheduled / {len(blocked)} blocked on real shortage, "
|
|
f"{len(assumptions)} explicit trial assumptions, {stats.get('woCount')} draft work orders"
|
|
)
|
|
|
|
world = self._get_json("/api/flex/world")
|
|
latest = world.get("latestVersion")
|
|
if not isinstance(latest, dict):
|
|
raise TypeError("flex world has no latest schedule version")
|
|
if latest.get("status") != "DRAFT":
|
|
raise AssertionError(f"latest schedule version is not DRAFT: {latest.get('status')!r}")
|
|
if int(latest.get("woCount") or 0) != int(stats.get("woCount") or 0):
|
|
raise AssertionError(f"latest version work order count differs from the block: {latest}")
|
|
if int(latest.get("vlCount") or 0) != int(stats.get("vlCount") or 0):
|
|
raise AssertionError(f"latest version virtual line count differs from the block: {latest}")
|
|
if latest.get("publishedAt") or latest.get("dispatchedAt"):
|
|
raise AssertionError("latest draft version was published or dispatched")
|
|
world_work_orders = world.get("workOrders") or []
|
|
if len(world_work_orders) != int(stats.get("woCount") or 0):
|
|
raise AssertionError(f"flex world work order count differs from the block: {len(world_work_orders)}")
|
|
if any(str(row.get("status") or "") != "PENDING" for row in world_work_orders):
|
|
raise AssertionError("trial work orders must all stay PENDING")
|
|
world_orders = {str(row.get("orderNo")) for row in world.get("orders") or []}
|
|
if world_orders != formal:
|
|
raise AssertionError(f"flex world orders differ from formal oracle: {sorted(world_orders)}")
|
|
if world_orders & sandbox:
|
|
raise AssertionError("sandbox order leaked into flex world")
|
|
if not world.get("conflicts"):
|
|
raise AssertionError("flex world has no real blocking conflicts")
|
|
mes = self._get_json("/api/mes/execution")
|
|
if mes.get("total") != 0 or mes.get("rows") != []:
|
|
raise AssertionError(f"MES execution is not empty: {mes}")
|
|
pending = self._get_json("/api/gov/pending").get("pending") or []
|
|
forbidden = {"schedule.publish", "mes.dispatch"}
|
|
if any(str(row.get("action") or "") in forbidden for row in pending):
|
|
raise AssertionError("a publish or MES dispatch confirmation is pending")
|
|
audit = self._get_json("/api/gov/audit?limit=200")
|
|
audit_actions = [str(row.get("action") or "") for row in audit.get("events") or []]
|
|
leaked = sorted(set(audit_actions) & forbidden)
|
|
if leaked:
|
|
raise AssertionError(f"publish/MES actions leaked into audit: {leaked}")
|
|
if any(row.get("mesExternalId") for row in world.get("workOrders") or []):
|
|
raise AssertionError("work order carries a MES external id")
|
|
self._record_assertion("latest version remains DRAFT; no publish, MES execution, or dispatch action")
|
|
self._write_json(self.evidence_dir / "final-flex-world.json", world)
|
|
self._write_json(self.evidence_dir / "final-mes-execution.json", mes)
|
|
self._write_json(self.evidence_dir / "final-gov-pending.json", {"pending": pending})
|
|
self._write_json(self.evidence_dir / "final-gov-audit.json", audit)
|
|
|
|
|
|
def _replace_and_reload(self, messages: list[dict[str, Any]], expected_blocks: list[str]) -> list[dict[str, Any]]:
|
|
if self.client is None:
|
|
raise RuntimeError("client is not ready")
|
|
response = self.client.put(
|
|
f"/api/sessions/{self.session_id}/messages",
|
|
json={"messages": messages},
|
|
)
|
|
if response.status_code != 200:
|
|
raise AssertionError(f"message persistence failed: {response.status_code} {response.text}")
|
|
loaded = self._get_json(f"/api/sessions/{self.session_id}/messages")
|
|
restored = loaded.get("messages") or []
|
|
if len(restored) != len(messages):
|
|
raise AssertionError(f"message count mismatch after reload: {len(restored)} != {len(messages)}")
|
|
for block_type in expected_blocks:
|
|
if not any(
|
|
isinstance(block, dict) and block.get("type") == block_type
|
|
for message in restored
|
|
for block in (message.get("blocks") or [])
|
|
):
|
|
raise AssertionError(f"persisted message lost block type {block_type}")
|
|
self.persisted_messages = messages
|
|
self._write_json(self.evidence_dir / "session-messages.json", loaded)
|
|
return restored
|
|
|
|
def _persist_after_analysis(self, analysis: ChatResult) -> None:
|
|
self._replace_and_reload(
|
|
[
|
|
{"role": "user", "text": "分析一下数据文件"},
|
|
{"role": "agent", "text": analysis.text, "blocks": analysis.blocks},
|
|
],
|
|
["folder-pack", "confirm-card"],
|
|
)
|
|
self._record_assertion("analysis blocks persisted and reloaded through session message APIs")
|
|
|
|
def _persist_after_schedule(self, analysis: ChatResult, schedule: ChatResult) -> None:
|
|
self._replace_and_reload(
|
|
[
|
|
{"role": "user", "text": "分析一下数据文件"},
|
|
{"role": "agent", "text": analysis.text, "blocks": analysis.blocks},
|
|
{"role": "user", "text": "立即排产"},
|
|
{"role": "agent", "text": schedule.text, "blocks": schedule.blocks},
|
|
],
|
|
["folder-pack", "confirm-card", "flex-schedule"],
|
|
)
|
|
self._record_assertion("full analysis-to-schedule conversation persisted and reloaded")
|
|
|
|
def _source_hash_now(self) -> str | None:
|
|
if self.source is None or not self.source.is_file():
|
|
return None
|
|
return _sha256_file(self.source)
|
|
|
|
|
|
def _summary(self, status: str, error: BaseException | None = None) -> dict[str, Any]:
|
|
source_after = self._source_hash_now()
|
|
payload: dict[str, Any] = {
|
|
"schemaVersion": 1,
|
|
"status": status,
|
|
"startedAt": getattr(self, "_started_at", None),
|
|
"finishedAt": time.strftime("%Y-%m-%dT%H:%M:%S%z"),
|
|
"artifactRoot": str(self.artifact_root),
|
|
"source": {
|
|
"path": str(self.source) if self.source else None,
|
|
"sha256Before": self.source_sha_before,
|
|
"sha256After": source_after,
|
|
"unchanged": bool(self.source and source_after == self.source_sha_before),
|
|
},
|
|
"oracle": {
|
|
"sourceSha256": self.expectations["sourceSha256"],
|
|
"entityCounts": self.expectations["entityCounts"],
|
|
"formalOrderNos": self.expectations["formalOrderNos"],
|
|
"sandboxOrderNos": self.expectations["sandboxOrderNos"],
|
|
},
|
|
"runtime": {
|
|
"piCli": str(self.pi_cli) if self.pi_cli else None,
|
|
"piCliSha256": self.pi_cli_sha256,
|
|
"node": self.node_bin,
|
|
"nodeVersion": self.node_version,
|
|
"fallbackDir": str(self.fallback_root),
|
|
"piHome": os.environ.get("APS_FALLBACK_PI_HOME"),
|
|
"llmConfigured": bool(
|
|
(os.environ.get("LLM_BASE_URL") or "").strip()
|
|
and (os.environ.get("LLM_API_KEY") or "").strip()
|
|
),
|
|
"llmProvider": os.environ.get("LLM_PROVIDER") or None,
|
|
"llmModel": os.environ.get("LLM_MODEL") or None,
|
|
"authEnabled": os.environ.get("APS_AUTH_ENABLED"),
|
|
"seedDemo": os.environ.get("APS_SEED_DEMO"),
|
|
},
|
|
"steps": self.steps,
|
|
"piRuns": self.pi_runs,
|
|
"assertions": self.assertions,
|
|
"projectId": self.project_id,
|
|
"sessionId": self.session_id,
|
|
"error": None,
|
|
}
|
|
if error is not None:
|
|
payload["error"] = {
|
|
"type": type(error).__name__,
|
|
"message": self._redact(str(error)),
|
|
"traceback": self._redact(
|
|
"".join(traceback.format_exception(type(error), error, error.__traceback__))
|
|
),
|
|
}
|
|
return payload
|
|
|
|
def _write_summary(self, status: str, error: BaseException | None = None) -> dict[str, Any]:
|
|
payload = self._summary(status, error)
|
|
self._write_json(self.artifact_root / "latest-summary.json", payload)
|
|
return payload
|
|
|
|
def run(self) -> dict[str, Any]:
|
|
self._started_at = time.strftime("%Y-%m-%dT%H:%M:%S%z")
|
|
try:
|
|
self.prepare()
|
|
self._assert_initial_world()
|
|
analysis = self._failure_injection_then_retry()
|
|
self._confirm_adoption(analysis)
|
|
self._persist_after_analysis(analysis)
|
|
schedule = self._schedule()
|
|
self._assert_schedule_and_no_dispatch(schedule)
|
|
self._persist_after_schedule(analysis, schedule)
|
|
self.source_sha_after = self._source_hash_now()
|
|
if self.source_sha_after != self.source_sha_before:
|
|
raise AssertionError("source workbook SHA changed during the E2E run")
|
|
self._record_assertion("source workbook SHA unchanged before and after the product flow")
|
|
return self._write_summary("passed")
|
|
except BaseException as exc:
|
|
self.source_sha_after = self._source_hash_now()
|
|
self._write_summary("failed", exc)
|
|
raise
|
|
finally:
|
|
self.close()
|