768 lines
35 KiB
Python
768 lines
35 KiB
Python
# P3 S6 真实冒烟（Agent-N）：MES 断连 → Pi 待补录清单 → 逐笔确认卡 → 批准补录
|
||
# → 人工触发「恢复」→ readiness probe → 对账报告（三档）→ 审计链完整。
|
||
# 对应 P3-DESIGN §9.1 验收点 A–H。真实 server + 真实 Pi + 真实 LLM（kimi-k2.6）
|
||
# + 真实 MES HTTP 适配器（断连=未监听端口；恢复=冒烟脚本内嵌假 MES HTTP 服务）。
|
||
#
|
||
# 分阶段执行（单命令 300s 预算约束；阶段间状态落 site4/ + s6_results.json）：
|
||
# python smoke_s6_reconcile.py setup —— 搭临时现场（幂等重建）
|
||
# python smoke_s6_reconcile.py outage —— 断连期：readiness failed + 补录全链
|
||
# python smoke_s6_reconcile.py recovery —— 恢复期：假 MES 上线 + 对账协议
|
||
# python smoke_s6_reconcile.py control —— 对照组：删白名单重发补录请求
|
||
# python smoke_s6_reconcile.py finalize —— 收尾校验 + 汇总退出码
|
||
#
|
||
# 已知设施取舍（如实记录）：
|
||
# - 断连前现场（已下发工单 + 断连期间完工事实）由脚本直接种子到租户世界与
|
||
# CSV 导出文件（watcher 投 inbox）——这是冒烟设施，不是产品能力声明；
|
||
# - 假 MES 是冒烟脚本内嵌的最小 HTTP 服务（/health + GET /work-orders/{id}），
|
||
# 用于让真实 HttpMesClient 走真实 HTTP 协议完成恢复后对账；
|
||
# - LLM key 只经 server 子进程环境从仓库根 .env 流入，本脚本不读不打印不落盘。
|
||
import csv
|
||
import hashlib
|
||
import json
|
||
import os
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
import urllib.request
|
||
from http.cookiejar import CookieJar
|
||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||
from pathlib import Path
|
||
|
||
HERE = Path(__file__).resolve().parent # poc/pi-fallback/smoke
|
||
REPO = HERE.parents[2]
|
||
SITE = HERE / "site4"
|
||
APS_HOME = SITE / "aps-home"
|
||
DATA = APS_HOME / "data"
|
||
PORT = 8394
|
||
MES_PORT = 8641 # 断连期无人监听；恢复期假 MES 上线
|
||
BASE = f"http://127.0.0.1:{PORT}"
|
||
DEVICE_ID = "smoke-n-device-s6-0001"
|
||
RESULTS = HERE / "s6_results.json"
|
||
MOCK_TENANT = hashlib.sha256(f"{SITE.resolve()}|tenant".encode()).hexdigest()[:32]
|
||
MOCK_SECRET = hashlib.sha256(f"{SITE.resolve()}|license".encode()).hexdigest()
|
||
|
||
# 口令逐条避让 intent.py 快路（无 完工报工/工序报工/报工回流/下发MES/执行进度/
|
||
# 分析..文件/导入 等触发）与 _RECONCILE_SIGNALS（对账/恢复）——落 LLM 分类 →
|
||
# assistant.reply → 兜底分支 → S6 归类（命中 连不上/补录/mes 信号）。
|
||
QUERY_BACKFILL = ("车间 MES 连不上了，今天实际干完的活系统里都还没登记"
|
||
"（车间导出的 mes-导出-车间完工.csv 已放进 inbox）。"
|
||
"帮我核对这份车间导出表，把断连期间落下的进度逐笔补登进系统，别漏行。")
|
||
QUERY_RECONCILE = "MES 恢复了，对一下账"
|
||
|
||
PRODUCT_CODE = "28200003654300" # demo 世界唯一成品物料
|
||
WO_SEED = [(9001, "EXT-1001"), (9002, "EXT-1002")]
|
||
CSV_ROWS = [("EXT-1001", "100", "10", "COMPLETED"),
|
||
("EXT-1002", "60", "6", "RUNNING")]
|
||
|
||
|
||
def load_results() -> dict:
|
||
if RESULTS.is_file():
|
||
return json.loads(RESULTS.read_text(encoding="utf-8"))
|
||
return {"startedAt": time.strftime("%Y-%m-%d %H:%M:%S"), "checks": {}}
|
||
|
||
|
||
R = load_results()
|
||
|
||
|
||
def check(name: str, ok: bool, detail: str = "") -> bool:
|
||
R["checks"][name] = {"ok": bool(ok), "detail": detail}
|
||
print(f"[{'PASS' if ok else 'FAIL'}] {name}" + (f" — {detail}" if detail else ""))
|
||
return ok
|
||
|
||
|
||
def dump_results() -> None:
|
||
RESULTS.write_text(json.dumps(R, ensure_ascii=False, indent=2, default=str),
|
||
encoding="utf-8")
|
||
|
||
|
||
def http(opener, method, path, body=None, headers=None, timeout=30):
|
||
req = urllib.request.Request(BASE + path, method=method)
|
||
for k, v in (headers or {}).items():
|
||
req.add_header(k, v)
|
||
data = None
|
||
if body is not None:
|
||
data = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
||
req.add_header("Content-Type", "application/json")
|
||
return opener.open(req, data=data, timeout=timeout)
|
||
|
||
|
||
def desktop_headers():
|
||
return {"x-aps-client": "desktop", "x-aps-device-id": DEVICE_ID}
|
||
|
||
|
||
def tenant_world_path() -> Path:
|
||
device_hash = hashlib.sha256(DEVICE_ID.encode("utf-8")).hexdigest()
|
||
user_id = int(device_hash[:15], 16)
|
||
return DATA / "tenants" / MOCK_TENANT / "projects" / f"personal-{user_id}" / "world.json"
|
||
|
||
|
||
def world_fingerprint_of(path: Path) -> str:
|
||
sys.path.insert(0, str(REPO))
|
||
from server.agent_core.harness import world_fingerprint
|
||
return world_fingerprint(json.loads(path.read_text(encoding="utf-8")))
|
||
|
||
|
||
def start_server(env_extra: dict, log_name: str):
|
||
env = dict(os.environ)
|
||
env.update({
|
||
"APS_HOME": str(APS_HOME),
|
||
"APS_FEATURES_PATH": str(SITE / "features.json"),
|
||
"APS_FALLBACK_HIGHRISK_PATH": str(SITE / "fallback-highrisk.json"),
|
||
"APS_WORLD_PATH": str(DATA / "world.json"),
|
||
"APS_DB_PATH": str(DATA / "master.db"),
|
||
"APS_LICENSE_PROVIDER": "mock",
|
||
"APS_MOCK_LICENSE_SECRET": MOCK_SECRET,
|
||
"APS_MOCK_LICENSE_TENANT_UUID": MOCK_TENANT,
|
||
"APS_PORT": str(PORT),
|
||
"APS_FALLBACK_MODEL": "aps-fallback/kimi-k2.6",
|
||
"APS_FALLBACK_TIMEOUT_SEC": "170",
|
||
"MES_HTTP_BASE_URL": f"http://127.0.0.1:{MES_PORT}",
|
||
"MES_HTTP_TIMEOUT_SECONDS": "2",
|
||
"MES_HTTP_MAX_RETRIES": "1",
|
||
# 本机系统代理（注册表 127.0.0.1:7897）会劫持 httpx 对本地端口的请求，
|
||
# 把「拒连」包装成代理错误页（UPSTREAM_ERROR）——显式绕过，冒烟环境层修正
|
||
"NO_PROXY": "127.0.0.1,localhost",
|
||
"no_proxy": "127.0.0.1,localhost",
|
||
"PYTHONIOENCODING": "utf-8",
|
||
})
|
||
env.pop("APS_FALLBACK_DIR", None)
|
||
env.update(env_extra)
|
||
server_log = open(HERE / log_name, "w", encoding="utf-8")
|
||
proc = subprocess.Popen(
|
||
[sys.executable, "-m", "uvicorn", "server.main:app",
|
||
"--host", "127.0.0.1", "--port", str(PORT)],
|
||
cwd=str(REPO), env=env, stdout=server_log, stderr=subprocess.STDOUT)
|
||
return proc, server_log
|
||
|
||
|
||
def wait_health(proc, seconds=90) -> dict | None:
|
||
for _ in range(seconds):
|
||
if proc.poll() is not None:
|
||
return None
|
||
try:
|
||
with urllib.request.urlopen(BASE + "/api/health", timeout=3) as r:
|
||
return json.loads(r.read().decode("utf-8"))
|
||
except Exception:
|
||
time.sleep(1)
|
||
return None
|
||
|
||
|
||
def kill_server(proc, server_log) -> None:
|
||
if proc.poll() is None:
|
||
subprocess.run(["taskkill", "/PID", str(proc.pid), "/T", "/F"],
|
||
capture_output=True, timeout=30, check=False)
|
||
try:
|
||
proc.wait(timeout=20)
|
||
except Exception:
|
||
pass
|
||
server_log.close()
|
||
|
||
|
||
def make_opener():
|
||
cj = CookieJar()
|
||
return urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
|
||
|
||
|
||
def activate_license(opener) -> None:
|
||
resp = http(opener, "POST", "/api/auth/license/activate",
|
||
{"code": "APS-DAY-DEMO", "deviceId": DEVICE_ID},
|
||
headers=desktop_headers())
|
||
resp.read()
|
||
|
||
|
||
def parse_sse(raw: str) -> list[dict]:
|
||
events = []
|
||
for line in raw.splitlines():
|
||
line = line.strip()
|
||
if line.startswith("data:"):
|
||
try:
|
||
events.append(json.loads(line[5:].strip()))
|
||
except json.JSONDecodeError:
|
||
pass
|
||
return events
|
||
|
||
|
||
class InboxWatcher:
|
||
"""把车间导出 CSV 投入新建 run 目录的 inbox/（模拟「给文件」环节）。"""
|
||
|
||
def __init__(self, fb_root: Path, payload: Path):
|
||
self.fb_root = fb_root
|
||
self.payload = payload
|
||
self.delivered_to = ""
|
||
self._stop = threading.Event()
|
||
self._thread = threading.Thread(target=self._loop, daemon=True)
|
||
|
||
def _loop(self):
|
||
while not self._stop.is_set() and not self.delivered_to:
|
||
try:
|
||
if self.fb_root.is_dir():
|
||
for d in sorted(self.fb_root.glob("fb-*")):
|
||
if d.is_dir():
|
||
dst = d / "inbox" / self.payload.name
|
||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||
shutil.copy(self.payload, dst)
|
||
self.delivered_to = str(dst)
|
||
return
|
||
except Exception:
|
||
pass
|
||
time.sleep(0.25)
|
||
|
||
def __enter__(self):
|
||
self._thread.start()
|
||
return self
|
||
|
||
def __exit__(self, *exc):
|
||
self._stop.set()
|
||
self._thread.join(timeout=5)
|
||
|
||
|
||
class FakeMes:
|
||
"""内嵌最小 MES HTTP 服务：GET /health + GET /work-orders/{id}（真实 HTTP 协议）。"""
|
||
|
||
def __init__(self, port: int, dataset: dict[str, dict]):
|
||
self.port = port
|
||
self.dataset = dataset
|
||
self.server = None
|
||
self.thread = None
|
||
|
||
def _handler(self):
|
||
dataset = self.dataset
|
||
|
||
class Handler(BaseHTTPRequestHandler):
|
||
def log_message(self, *args): # 静音
|
||
pass
|
||
|
||
def do_GET(self):
|
||
if self.path == "/health":
|
||
body = {"system": "MES-HTTP-FAKE", "plant": "CNWH",
|
||
"woCount": len(dataset), "openCount": 0,
|
||
"reportCount": 0, "updatedAt": "2026-09-05T00:00:00"}
|
||
elif self.path.startswith("/work-orders/"):
|
||
ext_id = self.path.rsplit("/", 1)[-1]
|
||
body = dataset.get(ext_id)
|
||
if body is None:
|
||
self.send_response(404)
|
||
self.end_headers()
|
||
return
|
||
else:
|
||
self.send_response(404)
|
||
self.end_headers()
|
||
return
|
||
payload = json.dumps(body, ensure_ascii=False).encode("utf-8")
|
||
self.send_response(200)
|
||
self.send_header("Content-Type", "application/json")
|
||
self.send_header("Content-Length", str(len(payload)))
|
||
self.end_headers()
|
||
self.wfile.write(payload)
|
||
|
||
return Handler
|
||
|
||
def __enter__(self):
|
||
self.server = ThreadingHTTPServer(("127.0.0.1", self.port), self._handler())
|
||
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
||
self.thread.start()
|
||
return self
|
||
|
||
def __exit__(self, *exc):
|
||
self.server.shutdown()
|
||
self.server.server_close()
|
||
self.thread.join(timeout=5)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# phase: setup —— 搭临时现场（幂等重建 site4）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _robocopy_purge(target: Path) -> None:
|
||
"""robocopy 空目录镜像 purge：绕开 Windows MAX_PATH（pi 会话目录名编码完整
|
||
工作路径 + 中文，rmtree 递归在超长路径上必败——冒烟环境层对策）。"""
|
||
empty = target.parent / "_empty_purge"
|
||
empty.mkdir(exist_ok=True)
|
||
subprocess.run(["robocopy", str(empty), str(target), "/MIR",
|
||
"/NFL", "/NDL", "/NJH", "/NJS", "/NP"],
|
||
capture_output=True, timeout=120, check=False)
|
||
try:
|
||
empty.rmdir()
|
||
target.rmdir()
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def phase_setup() -> int:
|
||
if SITE.exists():
|
||
for attempt in range(30):
|
||
try:
|
||
shutil.rmtree(SITE)
|
||
break
|
||
except OSError:
|
||
if attempt == 29:
|
||
_robocopy_purge(SITE)
|
||
if SITE.exists():
|
||
raise
|
||
time.sleep(1.0)
|
||
DATA.mkdir(parents=True)
|
||
shutil.copy(REPO / "server" / "data" / "world.json", DATA / "world.json")
|
||
tworld = tenant_world_path()
|
||
tworld.parent.mkdir(parents=True)
|
||
seed = json.loads((REPO / "server" / "data" / "world.json").read_text(encoding="utf-8"))
|
||
# 断连前现场：2 张已下发工单（世界侧投影 + mesLinks 链路），进度 0。
|
||
# 种进 flexWorkOrders：mes.apply_report 默认 track=flex 查该表
|
||
# （真实 Pi 计划未申报 track → 默认 flex；golden 的 track=fixed 是测试构造）
|
||
seed.setdefault("flexWorkOrders", [])
|
||
seed.setdefault("mesLinks", [])
|
||
for wo_id, ext_id in WO_SEED:
|
||
seed["flexWorkOrders"].append({
|
||
"id": wo_id, "mesExternalId": ext_id, "status": "RUNNING",
|
||
"progressPct": 0, "qtyDone": 0, "productCode": PRODUCT_CODE})
|
||
seed["mesLinks"].append({
|
||
"kind": "dispatch", "woId": wo_id, "externalWoId": ext_id,
|
||
"idemKey": f"idem-s6-{wo_id}"})
|
||
tworld.write_text(json.dumps(seed, ensure_ascii=False), encoding="utf-8")
|
||
(SITE / "features.json").write_text(
|
||
json.dumps({"version": 1, "features": {"fallback": True}}, ensure_ascii=False),
|
||
encoding="utf-8")
|
||
whitelist = {
|
||
"whitelistVersion": 1, "updatedAt": "2026-09-05T09:00:00",
|
||
"updatedBy": "smoke-s6",
|
||
"scenarios": {
|
||
"S4": {"enabled": True,
|
||
"intents": ["flex.simulate_due", "flex.compare",
|
||
"scenario.compare", "scenario.sensitivity"],
|
||
"roles": ["planner", "admin"]},
|
||
"S6": {"enabled": True, "intents": ["mes.report"],
|
||
"roles": ["planner", "admin"], "maxItemsPerRun": 20},
|
||
"S7": {"enabled": True,
|
||
"intents": ["agent.fallback.ops.config.apply",
|
||
"agent.fallback.policy.update"],
|
||
"roles": ["ops", "admin"]}}}
|
||
(SITE / "fallback-highrisk.json").write_text(
|
||
json.dumps(whitelist, ensure_ascii=False, indent=2), encoding="utf-8")
|
||
# 断连期间车间实际完工事实（MES 导出表）
|
||
with open(SITE / "mes-导出-车间完工.csv", "w", encoding="utf-8-sig", newline="") as f:
|
||
w = csv.writer(f)
|
||
w.writerow(["外部单号", "完工进度%", "完工数量", "状态"])
|
||
w.writerows(CSV_ROWS)
|
||
R.clear()
|
||
R.update({"startedAt": time.strftime("%Y-%m-%d %H:%M:%S"), "checks": {},
|
||
"site": str(SITE), "tenantWorld": str(tworld),
|
||
"queryBackfill": QUERY_BACKFILL, "queryReconcile": QUERY_RECONCILE})
|
||
check("setup.site_ready", tworld.is_file()
|
||
and (SITE / "fallback-highrisk.json").is_file())
|
||
dump_results()
|
||
return 0
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# phase: outage —— 断连注入 + 段 A/B/C 全链（验收点 A/B/C/D）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def _chat(opener, query: str, timeout: int = 190) -> tuple[str, list[dict]]:
|
||
resp = http(opener, "POST", "/api/chat", {"text": query},
|
||
headers=desktop_headers(), timeout=timeout)
|
||
raw = resp.read().decode("utf-8", errors="replace")
|
||
return raw, parse_sse(raw)
|
||
|
||
|
||
def _confirm(opener, confirm_id: str, note: str) -> dict:
|
||
resp = http(opener, "POST", "/api/actions/confirm",
|
||
{"confirmId": confirm_id, "approve": True, "note": note},
|
||
headers=desktop_headers(), timeout=120)
|
||
return json.loads(resp.read().decode("utf-8"))
|
||
|
||
|
||
def phase_outage() -> int:
|
||
tworld = tenant_world_path()
|
||
# 前置自检：断连注入端口必须无人监听（否则「断连」前提不成立，冒烟无效）
|
||
try:
|
||
urllib.request.urlopen(f"http://127.0.0.1:{MES_PORT}/health", timeout=2)
|
||
check("A.mes_outage_injected", False,
|
||
f"端口 {MES_PORT} 已有服务监听，断连前提不成立")
|
||
dump_results()
|
||
return 1
|
||
except Exception:
|
||
pass
|
||
proc, server_log = start_server({}, "server_s6_outage.log")
|
||
R["serverPidOutage"] = proc.pid
|
||
try:
|
||
health = wait_health(proc)
|
||
if not health:
|
||
check("A0.outage_server_health", False, "server 90s 内未就绪（见 server_s6_outage.log）")
|
||
dump_results()
|
||
return 1
|
||
check("A0.outage_server_health", True)
|
||
opener = make_opener()
|
||
activate_license(opener)
|
||
|
||
# 验收点 A：断连注入（readiness 探测 → connectivity=failed + CONNECT_FAILED）
|
||
resp = http(opener, "POST", "/api/integrations/mes/readiness/probe",
|
||
{}, headers=desktop_headers())
|
||
probe = json.loads(resp.read().decode("utf-8"))
|
||
R["readinessOutage"] = probe
|
||
check("A.mes_outage_injected",
|
||
probe.get("connectivity") == "failed"
|
||
and (probe.get("lastError") or {}).get("code") == "MES_HTTP_CONNECT_FAILED",
|
||
f"connectivity={probe.get('connectivity')} "
|
||
f"code={(probe.get('lastError') or {}).get('code')}")
|
||
|
||
# 验收点 B：断连前现场已种子（世界内 mesLinks=2 + 导出文件待投 inbox）
|
||
world0 = json.loads(tworld.read_text(encoding="utf-8"))
|
||
links = [l for l in world0.get("mesLinks") or [] if l.get("kind") == "dispatch"]
|
||
check("B.prescene_seeded", len(links) == 2
|
||
and (SITE / "mes-导出-车间完工.csv").is_file(),
|
||
f"mesLinks={len(links)}")
|
||
|
||
# 验收点 C：断连对话 → S6 兜底（集成状态注入 + Pi 待补录清单/计划）
|
||
fb_root = DATA / "fallback"
|
||
runs_before = {p.name for p in fb_root.glob("fb-*")} if fb_root.is_dir() else set()
|
||
t0 = time.monotonic()
|
||
with InboxWatcher(fb_root, SITE / "mes-导出-车间完工.csv") as watcher:
|
||
raw, events = _chat(opener, QUERY_BACKFILL)
|
||
R["chatElapsedSec"] = round(time.monotonic() - t0, 1)
|
||
R["csvDeliveredTo"] = watcher.delivered_to
|
||
(HERE / "chat_s6_outage.sse").write_text(raw, encoding="utf-8")
|
||
reply_text = "".join(e.get("text", "") for e in events if e.get("type") == "token")
|
||
(HERE / "reply_s6_outage.txt").write_text(reply_text, encoding="utf-8")
|
||
intent_ev = next((e for e in events if e.get("type") == "intent"), None)
|
||
intent_name = ((intent_ev or {}).get("intent") or {}).get("intent") \
|
||
if isinstance((intent_ev or {}).get("intent"), dict) \
|
||
else (intent_ev or {}).get("intent")
|
||
check("C1.intent_lands_assistant_reply", intent_name == "assistant.reply",
|
||
f"intent={intent_name}")
|
||
runs_after = {p.name for p in fb_root.glob("fb-*")} if fb_root.is_dir() else set()
|
||
new_runs = sorted(runs_after - runs_before)
|
||
run_dir = fb_root / new_runs[-1] if new_runs else None
|
||
R["outageRunDir"] = str(run_dir) if run_dir else None
|
||
status_md = ""
|
||
if run_dir and (run_dir / "inbox" / "integration-status.md").is_file():
|
||
status_md = (run_dir / "inbox" / "integration-status.md") \
|
||
.read_text(encoding="utf-8")
|
||
check("C2.integration_status_injected",
|
||
"connectivity=failed" in status_md and "未连接" in status_md,
|
||
status_md.replace("\n", " ")[:200])
|
||
plan_doc = None
|
||
if run_dir and (run_dir / "outbox" / "plan.json").is_file():
|
||
plan_doc = json.loads((run_dir / "outbox" / "plan.json")
|
||
.read_text(encoding="utf-8"))
|
||
R["outagePlan"] = plan_doc
|
||
check("C3.s6_plan_produced",
|
||
bool(plan_doc) and plan_doc.get("scenario") == "S6"
|
||
and all(s.get("intent") == "mes.report" and s.get("mode") == "frozen"
|
||
for s in (plan_doc or {}).get("steps", [])),
|
||
f"scenario={(plan_doc or {}).get('scenario')} "
|
||
f"steps={len((plan_doc or {}).get('steps', []))}")
|
||
|
||
# 验收点 D：逐笔确认卡（张数 == 清单项数）+ 逐张批准逐笔落账 + 独立 checkpoint
|
||
blocks = [e["block"] for e in events
|
||
if e.get("type") == "block" and isinstance(e.get("block"), dict)]
|
||
cards = [b for b in blocks
|
||
if (b.get("props") or {}).get("action") == "agent.fallback.execute"]
|
||
R["outageCards"] = [{"confirmId": (b.get("props") or {}).get("confirmId"),
|
||
"title": (b.get("props") or {}).get("title"),
|
||
"summary": (b.get("props") or {}).get("summary")}
|
||
for b in cards]
|
||
n_steps = len((plan_doc or {}).get("steps", []))
|
||
check("D1.per_item_cards", bool(cards) and len(cards) == n_steps and n_steps > 0,
|
||
f"cards={len(cards)} steps={n_steps}")
|
||
if not cards:
|
||
dump_results()
|
||
return 1
|
||
# 卡摘要全部由结构化字段再生成（逐笔列明 woId / 补录内容）
|
||
summary_blob = json.dumps(R["outageCards"], ensure_ascii=False)
|
||
check("D2.card_summary_machine_rendered",
|
||
"工单 woId=" in summary_blob and "计划指纹 sha256:" in summary_blob)
|
||
|
||
confirm_results = []
|
||
for i, card in enumerate(cards, 1):
|
||
cid = card["props"]["confirmId"]
|
||
confirm_results.append(_confirm(opener, cid, f"P3 S6 冒烟批准第 {i} 笔"))
|
||
R["confirmResults"] = [
|
||
{"message": str(r.get("message") or "")[:300]} for r in confirm_results]
|
||
all_ok = all("兜底计划已执行完成" in str(r.get("message") or "")
|
||
for r in confirm_results)
|
||
check("D3.each_card_executed", all_ok,
|
||
f"{sum('兜底计划已执行完成' in str(r.get('message') or '') for r in confirm_results)}"
|
||
f"/{len(confirm_results)} 笔执行完成")
|
||
|
||
# 落账对账：按各卡冻结参数逐项核对（run 目录冻结计划 steps.params 是
|
||
# 唯一事实源——approvals.json 历史条目只存 paramsHash 不存 params；
|
||
# 修复登记：原实现读 history[].record.params 恒为空，该路径此前从未在
|
||
# 通过态被验证，Agent-O 修复轮改为读冻结计划）
|
||
world_after = json.loads(tworld.read_text(encoding="utf-8"))
|
||
wos = {w["id"]: w
|
||
for table in ("flexWorkOrders", "workOrders")
|
||
for w in world_after.get(table) or []}
|
||
applied = [dict(s.get("params") or {})
|
||
for s in (plan_doc or {}).get("steps", [])]
|
||
R["appliedParams"] = applied
|
||
mismatches = []
|
||
for p in applied:
|
||
wo = wos.get(p.get("woId"))
|
||
if wo is None:
|
||
mismatches.append(f"woId={p.get('woId')} 不存在")
|
||
continue
|
||
if p.get("finish") and wo.get("status") != "COMPLETED":
|
||
mismatches.append(f"woId={p['woId']} 未完工（{wo.get('status')}）")
|
||
if p.get("progressPct") is not None \
|
||
and wo.get("progressPct") != p.get("progressPct"):
|
||
mismatches.append(f"woId={p['woId']} 进度 {wo.get('progressPct')}"
|
||
f"!={p.get('progressPct')}")
|
||
check("D4.backfill_applied_per_card",
|
||
bool(applied) and not mismatches,
|
||
f"落账 {len(applied)} 笔；mismatches={mismatches}")
|
||
|
||
# 独立 checkpoint 对（每笔一对）
|
||
ck_file = tworld.parent / "checkpoints.json"
|
||
ck = json.loads(ck_file.read_text(encoding="utf-8")) if ck_file.is_file() else {}
|
||
reasons = [p.get("reason") for p in (ck.get("pairs") or [])]
|
||
n_pairs = reasons.count("auto:fallback.execute")
|
||
n_posts = reasons.count("auto:fallback.execute.post")
|
||
check("D5.independent_checkpoint_pairs",
|
||
n_pairs == len(cards) and n_posts == len(cards),
|
||
f"before={n_pairs} after={n_posts} cards={len(cards)}")
|
||
|
||
# 第二张卡成功执行 = 偏差⑦（信封哈希同步重算）真实链路直接证据
|
||
check("D6.second_card_envelope_ok", len(confirm_results) >= 2 and all_ok,
|
||
"多卡逐张批准全成功（偏差⑦修复的真实链路复证）")
|
||
|
||
# 审计链完整性
|
||
audits = world_after.get("auditEvents") or []
|
||
sys.path.insert(0, str(REPO))
|
||
from server.agent_core.registry import verify_audit_chain
|
||
chain = verify_audit_chain(audits)
|
||
stage_ev = [a for a in audits if a.get("action") == "agent.fallback.execute.stage"]
|
||
exec_ev = [a for a in audits if a.get("action") == "agent.fallback.execute"
|
||
and a.get("category") == "WORLD_WRITE" and a.get("result") == "SUCCESS"]
|
||
check("G1.audit_chain_outage",
|
||
bool(chain.get("ok")) and len(stage_ev) == len(cards)
|
||
and len(exec_ev) == len(cards),
|
||
f"chain.ok={chain.get('ok')} stage={len(stage_ev)} exec={len(exec_ev)}")
|
||
finally:
|
||
kill_server(proc, server_log)
|
||
dump_results()
|
||
failed = [k for k, v in R["checks"].items() if not v["ok"]]
|
||
return 0 if not failed else 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# phase: recovery —— 恢复注入（假 MES 上线）+ 段 D 对账协议（验收点 E/F）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def phase_recovery() -> int:
|
||
tworld = tenant_world_path()
|
||
# 用补录后的真实本地状态构造假 MES 数据集：
|
||
# EXT-1001 完全镜像本地 → MATCH；EXT-1002 进度改成一个不同值 → DRIFT。
|
||
world_now = json.loads(tworld.read_text(encoding="utf-8"))
|
||
wos = {w.get("mesExternalId"): w
|
||
for table in ("flexWorkOrders", "workOrders")
|
||
for w in world_now.get(table) or []}
|
||
dataset = {}
|
||
local1 = wos.get("EXT-1001") or {}
|
||
local2 = wos.get("EXT-1002") or {}
|
||
dataset["EXT-1001"] = {"id": "EXT-1001",
|
||
"status": local1.get("status"),
|
||
"progressPct": local1.get("progressPct"),
|
||
"qtyDone": local1.get("qtyDone")}
|
||
drift_pct = 40 if local2.get("progressPct") != 40 else 70
|
||
dataset["EXT-1002"] = {"id": "EXT-1002",
|
||
"status": local2.get("status"),
|
||
"progressPct": drift_pct,
|
||
"qtyDone": local2.get("qtyDone")}
|
||
R["fakeMesDataset"] = dataset
|
||
R["reconcileExpect"] = {"EXT-1001": "MATCH", "EXT-1002": "DRIFT"}
|
||
|
||
fp_before = world_fingerprint_of(tworld)
|
||
proc, server_log = start_server({}, "server_s6_recovery.log")
|
||
try:
|
||
with FakeMes(MES_PORT, dataset):
|
||
health = wait_health(proc)
|
||
if not health:
|
||
check("E0.recovery_server_health", False,
|
||
"server 90s 内未就绪（见 server_s6_recovery.log）")
|
||
dump_results()
|
||
return 1
|
||
check("E0.recovery_server_health", True)
|
||
opener = make_opener()
|
||
activate_license(opener)
|
||
|
||
# 验收点 E：恢复注入（readiness → connectivity=ok）
|
||
resp = http(opener, "POST", "/api/integrations/mes/readiness/probe",
|
||
{}, headers=desktop_headers())
|
||
probe = json.loads(resp.read().decode("utf-8"))
|
||
R["readinessRecovery"] = probe
|
||
check("E.mes_recovered", probe.get("connectivity") == "ok",
|
||
f"connectivity={probe.get('connectivity')}")
|
||
|
||
# 验收点 F：人工触发对账 → 报告三档 + 证据链冻结 + ALGO_RUN 审计
|
||
raw, events = _chat(opener, QUERY_RECONCILE)
|
||
(HERE / "chat_s6_recovery.sse").write_text(raw, encoding="utf-8")
|
||
reply_text = "".join(e.get("text", "") for e in events
|
||
if e.get("type") == "token")
|
||
(HERE / "reply_s6_recovery.txt").write_text(reply_text, encoding="utf-8")
|
||
R["reconcileReplyHead"] = reply_text[:600]
|
||
fb_root = DATA / "fallback"
|
||
runs = sorted([p for p in fb_root.glob("fb-*") if p.is_dir()],
|
||
key=lambda p: p.stat().st_mtime) if fb_root.is_dir() else []
|
||
run_dir = runs[-1] if runs else None
|
||
R["reconcileRunDir"] = str(run_dir) if run_dir else None
|
||
report = ""
|
||
if run_dir and (run_dir / "outbox" / "reconcile-report.md").is_file():
|
||
report = (run_dir / "outbox" / "reconcile-report.md") \
|
||
.read_text(encoding="utf-8")
|
||
R["reconcileReport"] = report
|
||
check("F1.reconcile_reply_counts",
|
||
"一致 1" in reply_text and "漂移 1" in reply_text,
|
||
reply_text.replace("\n", " ")[:200])
|
||
check("F2.reconcile_report_verdicts",
|
||
"| EXT-1001 | 9001 | MATCH |" in report
|
||
and "| EXT-1002 | 9002 | DRIFT |" in report,
|
||
report.replace("\n", " ")[:300])
|
||
manifest = {}
|
||
if run_dir and (run_dir / "outbox" / "reconcile-manifest.json").is_file():
|
||
manifest = json.loads((run_dir / "outbox" / "reconcile-manifest.json")
|
||
.read_text(encoding="utf-8"))
|
||
sha_ok = bool(manifest.get("items")) and all(
|
||
run_dir and (run_dir / it["path"]).is_file()
|
||
and hashlib.sha256((run_dir / it["path"]).read_text(encoding="utf-8")
|
||
.encode("utf-8")).hexdigest() == it["sha256"]
|
||
for it in manifest.get("items", []))
|
||
check("F3.evidence_frozen", sha_ok,
|
||
f"manifest items={len(manifest.get('items') or [])} sha256 全部应验")
|
||
world_after = json.loads(tworld.read_text(encoding="utf-8"))
|
||
recon_audits = [a for a in world_after.get("auditEvents") or []
|
||
if a.get("action") == "agent.fallback.reconcile"]
|
||
rationale = (recon_audits[-1].get("rationale") or {}) if recon_audits else {}
|
||
check("F4.reconcile_audit",
|
||
bool(recon_audits) and recon_audits[-1].get("category") == "ALGO_RUN"
|
||
and bool(rationale.get("manifestSha256"))
|
||
and rationale.get("match") == 1 and rationale.get("drift") == 1,
|
||
f"rationale={json.dumps(rationale, ensure_ascii=False)[:200]}")
|
||
# 对账纯读：世界指纹前后相等（审计键不干扰指纹口径）
|
||
fp_after = world_fingerprint_of(tworld)
|
||
check("F5.reconcile_readonly", fp_before == fp_after,
|
||
f"fp {fp_before[:12]} == {fp_after[:12]}")
|
||
# 假 MES 与真实 HTTP 适配器在环的直接证据：外部原文落盘内容即数据集
|
||
ext_file = run_dir / "inbox" / "external" / "EXT-1002.json" if run_dir else None
|
||
ext_payload = ext_file.read_text(encoding="utf-8") \
|
||
if ext_file and ext_file.is_file() else ""
|
||
check("F6.external_snapshot_real",
|
||
f'"progressPct": {drift_pct}' in ext_payload
|
||
and "UNTRUSTED_DATA" in ext_payload,
|
||
f"外部原文含漂移值 {drift_pct}")
|
||
finally:
|
||
kill_server(proc, server_log)
|
||
dump_results()
|
||
failed = [k for k, v in R["checks"].items() if not v["ok"]]
|
||
return 0 if not failed else 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# phase: control —— 对照组：删白名单重发补录请求（验收点 G）
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def phase_control() -> int:
|
||
tworld = tenant_world_path()
|
||
wl_path = SITE / "fallback-highrisk.json"
|
||
wl_backup = SITE / "fallback-highrisk.json.disabled"
|
||
if wl_path.is_file():
|
||
wl_path.rename(wl_backup) # 白名单缺失 → fail-closed 全拒
|
||
proc, server_log = start_server({}, "server_s6_control.log")
|
||
try:
|
||
health = wait_health(proc)
|
||
if not health:
|
||
check("G0.control_server_health", False,
|
||
"server 90s 内未就绪（见 server_s6_control.log）")
|
||
dump_results()
|
||
return 1
|
||
opener = make_opener()
|
||
activate_license(opener)
|
||
world_before = json.loads(tworld.read_text(encoding="utf-8"))
|
||
wos_before = {w["id"]: dict(w)
|
||
for table in ("flexWorkOrders", "workOrders")
|
||
for w in world_before.get(table) or []}
|
||
|
||
raw, events = _chat(opener, QUERY_BACKFILL)
|
||
(HERE / "chat_s6_control.sse").write_text(raw, encoding="utf-8")
|
||
reply_text = "".join(e.get("text", "") for e in events if e.get("type") == "token")
|
||
(HERE / "reply_s6_control.txt").write_text(reply_text, encoding="utf-8")
|
||
R["controlReplyHead"] = reply_text[:600]
|
||
blocks = [e["block"] for e in events
|
||
if e.get("type") == "block" and isinstance(e.get("block"), dict)]
|
||
s6_cards = [b for b in blocks
|
||
if "补录" in str((b.get("props") or {}).get("title") or "")]
|
||
world_after = json.loads(tworld.read_text(encoding="utf-8"))
|
||
wos_after = {w["id"]: dict(w)
|
||
for table in ("flexWorkOrders", "workOrders")
|
||
for w in world_after.get(table) or []}
|
||
check("G.control_no_s6_card_no_write",
|
||
not s6_cards and wos_before == wos_after,
|
||
f"S6 卡={len(s6_cards)}；工单表逐行一致={wos_before == wos_after}"
|
||
f"（白名单缺失 → S6 入口物理关闭，分类降级走 P1/P2 路径）")
|
||
finally:
|
||
kill_server(proc, server_log)
|
||
if wl_backup.is_file():
|
||
wl_backup.rename(wl_path) # 还原现场供查验
|
||
dump_results()
|
||
failed = [k for k, v in R["checks"].items() if not v["ok"]]
|
||
return 0 if not failed else 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# phase: finalize —— 收尾校验（验收点 H）+ 汇总
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def phase_finalize() -> int:
|
||
time.sleep(2)
|
||
port_free = False
|
||
try:
|
||
urllib.request.urlopen(BASE + "/api/health", timeout=3)
|
||
except Exception:
|
||
port_free = True
|
||
mes_free = False
|
||
try:
|
||
urllib.request.urlopen(f"http://127.0.0.1:{MES_PORT}/health", timeout=3)
|
||
except Exception:
|
||
mes_free = True
|
||
wmi = ""
|
||
ps_exe = shutil.which("powershell.exe") or shutil.which("powershell") or "powershell.exe"
|
||
try:
|
||
wmi = subprocess.run(
|
||
[ps_exe, "-NoProfile", "-Command",
|
||
("Get-CimInstance Win32_Process | Where-Object "
|
||
"{$_.CommandLine -like '*pi-coding-agent*cli.js*' "
|
||
"-and $_.CommandLine -notlike '*Get-CimInstance*'} "
|
||
"| Select-Object -ExpandProperty ProcessId")],
|
||
capture_output=True, text=True, timeout=60, check=False).stdout.strip()
|
||
except Exception as exc:
|
||
R["piResidualCheckError"] = str(exc)
|
||
R["piResidualPids"] = [x for x in wmi.splitlines() if x.strip()]
|
||
git = subprocess.run(["git", "status", "--short", "server/data"],
|
||
cwd=str(REPO), capture_output=True, text=True, timeout=30,
|
||
check=False).stdout.strip()
|
||
R["gitServerData"] = git or "(clean)"
|
||
check("H.cleanup", port_free and mes_free and not R["piResidualPids"] and not git,
|
||
f"端口释放={port_free}/{mes_free} pi 残留={R['piResidualPids']} git={git or 'clean'}")
|
||
|
||
dump_results()
|
||
failed = [k for k, v in R["checks"].items() if not v["ok"]]
|
||
total = len(R["checks"])
|
||
print(json.dumps({"passed": total - len(failed), "total": total,
|
||
"failed": failed}, ensure_ascii=False, indent=2))
|
||
return 0 if not failed else 1
|
||
|
||
|
||
PHASES = {"setup": phase_setup, "outage": phase_outage, "recovery": phase_recovery,
|
||
"control": phase_control, "finalize": phase_finalize}
|
||
|
||
if __name__ == "__main__":
|
||
if len(sys.argv) != 2 or sys.argv[1] not in PHASES:
|
||
print(f"usage: {sys.argv[0]} <{'|'.join(PHASES)}>")
|
||
sys.exit(2)
|
||
sys.exit(PHASES[sys.argv[1]]())
|