340 lines
15 KiB
Python
340 lines
15 KiB
Python
# ============================================================
|
||
# 参数优化网关黄金测试（矩阵 87 行 · 方向 W）
|
||
# 覆盖：端点契约 / 优化触发（split→回放→候选） /
|
||
# P2 确认卡 promote（批准升级、拒绝留 GRAY） /
|
||
# 手动回滚 P1 / 线上观测劣化自动回滚+审计
|
||
# ============================================================
|
||
from __future__ import annotations
|
||
|
||
import pytest
|
||
from fastapi.testclient import TestClient
|
||
|
||
from server.agent_core import harness
|
||
from server.agent_core.param_opt import ParameterOptimizer
|
||
from server.auth.context import bind_identity, reset_identity
|
||
from server.state.seed import seed_world
|
||
from server.timeutil import add_minutes, fmt_date, today0
|
||
|
||
|
||
def _tight_world():
|
||
"""6 单紧张交期世界（与 test_param_opt 同构）。"""
|
||
w = seed_world()
|
||
base = today0()
|
||
order_date = fmt_date(base)
|
||
product = next(m for m in w["materials"] if m["type"] == "FINISHED_PRODUCT")
|
||
sos = []
|
||
for i in range(6):
|
||
oid = i + 1
|
||
level = "VIP" if i % 2 == 0 else "C"
|
||
due = fmt_date(add_minutes(base, (2, 2, 3, 3, 4, 5)[i] * 24 * 60))
|
||
sos.append({
|
||
"id": oid, "orderNo": f"SO-PT-{oid}", "customerId": f"C{oid}",
|
||
"customerName": f"客户{level}", "customerLevel": level,
|
||
"orderDate": order_date, "deliveryDate": due,
|
||
"priority": 5, "manualPriority": None, "status": "CONFIRMED",
|
||
"source": "MANUAL", "specialRequirements": "", "totalAmount": 1000,
|
||
"isRush": False, "rushStrategy": None, "changes": [],
|
||
"createdBy": "test", "createdAt": order_date + " 09:00",
|
||
"updatedAt": order_date + " 09:00",
|
||
"items": [{
|
||
"id": oid * 10, "orderId": oid, "lineNo": 1,
|
||
"productId": product["id"], "productName": product["name"],
|
||
"productCode": product["code"], "quantity": 300,
|
||
"unit": "件", "bomVersion": "V1.0", "routingVersion": "V1.0",
|
||
"status": "PENDING", "note": "",
|
||
}],
|
||
})
|
||
w["salesOrders"] = sos
|
||
w["scheduleVersions"] = []
|
||
w["productionOrders"] = []
|
||
w["workOrders"] = []
|
||
w["conflicts"] = []
|
||
return w
|
||
|
||
|
||
class _WorldStore:
|
||
"""测试用世界存储：与确认卡记录的租户/世界键匹配。"""
|
||
|
||
def __init__(self, world: dict, *, tenant_uuid: str = "tenant-param-opt",
|
||
world_key: str = "personal-1001"):
|
||
self.tenant_uuid = tenant_uuid
|
||
self.world_key = world_key
|
||
self.data = world
|
||
|
||
def next_id(self, _kind: str) -> int:
|
||
return len(self.data.get("auditEvents") or []) + 1
|
||
|
||
def save(self) -> None:
|
||
pass
|
||
|
||
|
||
class _CheckpointStore:
|
||
def __init__(self) -> None:
|
||
self.pairs: list[dict] = []
|
||
|
||
def create(self, *_args, **_kwargs) -> dict:
|
||
pair = {"pairId": f"pair-{len(self.pairs) + 1}"}
|
||
self.pairs.append(pair)
|
||
return pair
|
||
|
||
|
||
class _ProjectStore:
|
||
def active_world_key(self) -> str:
|
||
return "default"
|
||
|
||
def require_active_write(self) -> None:
|
||
pass
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _clear_pending():
|
||
harness._approval_store.clear()
|
||
yield
|
||
harness._approval_store.clear()
|
||
|
||
|
||
@pytest.fixture
|
||
def client_factory(monkeypatch):
|
||
"""返回 (client, store, provider)；用测试认证代替 JMS。"""
|
||
import server.gateway.app as gateway_module
|
||
import server.state.projects as projects_module
|
||
from tests.auth_provider import install_test_auth
|
||
|
||
provider = install_test_auth(monkeypatch, "tenant-param-opt")
|
||
|
||
def factory(world: dict | None = None):
|
||
store = _WorldStore(world if world is not None else _tight_world())
|
||
monkeypatch.setattr(gateway_module, "get_store", lambda: store)
|
||
monkeypatch.setattr(gateway_module, "get_checkpoints", lambda: _CheckpointStore())
|
||
monkeypatch.setattr(projects_module, "get_project_store", lambda: _ProjectStore())
|
||
client = TestClient(gateway_module.create_app())
|
||
login = client.post("/api/auth/login", json={"username": "planner", "password": "test"})
|
||
assert login.status_code == 200, login.text
|
||
return client, store, provider
|
||
|
||
return factory
|
||
|
||
|
||
def _as(identity, callback):
|
||
token = bind_identity(identity)
|
||
try:
|
||
return callback()
|
||
finally:
|
||
reset_identity(token)
|
||
|
||
|
||
def _audit_actions(world: dict) -> list[str]:
|
||
return [e.get("action") for e in world.get("auditEvents") or []]
|
||
|
||
|
||
# ---------------- 端点契约 / 优化触发 ----------------
|
||
def test_params_experiments_contract_empty(client_factory):
|
||
"""GET /api/params/experiments 契约：无实验时返回空列表。"""
|
||
client, _, _ = client_factory()
|
||
resp = client.get("/api/params/experiments")
|
||
assert resp.status_code == 200, resp.text
|
||
assert resp.json() == {"experiments": []}
|
||
|
||
|
||
def test_params_optimize_creates_gray_experiments_and_audits(client_factory):
|
||
"""POST /api/params/optimize：split→回放→候选，产出 GRAY 实验且审计 param.optimize。"""
|
||
client, store, _ = client_factory()
|
||
resp = client.post("/api/params/optimize", json={
|
||
"sessionId": "opt-session",
|
||
"strategy": "COMPREHENSIVE",
|
||
"scope": {"lineIds": [1]},
|
||
"candidates": [{"freezeWindowHours": 0.0}, {"freezeWindowHours": 48.0}],
|
||
})
|
||
assert resp.status_code == 200, resp.text
|
||
data = resp.json()
|
||
assert data["summary"]["total"] == 2
|
||
assert data["summary"]["gray"] == 1 # freeze 48h 训练回放退化被拒绝
|
||
assert data["summary"]["rejected"] == 1
|
||
gray = data["results"][0]
|
||
assert gray["stage"] == "gray" and gray["experimentId"]
|
||
exp = next(e for e in store.data["paramExperiments"]
|
||
if e["id"] == gray["experimentId"])
|
||
assert exp["status"] == "GRAY"
|
||
assert exp["scope"] == {"lineIds": [1]}
|
||
assert "split" in exp and "trainReplay" in exp # 集隔离与回放证据均在
|
||
assert "param.optimize" in _audit_actions(store.data)
|
||
# 实验列表可查
|
||
resp2 = client.get("/api/params/experiments")
|
||
assert len(resp2.json()["experiments"]) == 1
|
||
assert resp2.json()["experiments"][0]["id"] == exp["id"]
|
||
# 主干参数未动（灰度不全量写入）
|
||
assert store.data["scheduleParams"]["freezeWindowHours"] == 24.0
|
||
|
||
|
||
def test_params_optimize_default_candidates_from_sensitivity(client_factory):
|
||
"""默认候选：敏感性分析方向→候选→回放→GRAY（闭环驱动）。"""
|
||
client, store, _ = client_factory()
|
||
resp = client.post("/api/params/optimize", json={"sessionId": "opt2"})
|
||
assert resp.status_code == 200, resp.text
|
||
data = resp.json()
|
||
assert data["summary"]["total"] >= 3
|
||
assert data["summary"]["gray"] >= 1
|
||
assert data["summary"]["rejected"] == 0
|
||
assert len(store.data.get("paramExperiments") or []) == data["summary"]["total"]
|
||
|
||
|
||
# ---------------- P2 确认卡 promote ----------------
|
||
def test_promote_stages_confirm_card_and_approve_activates(client_factory):
|
||
"""promote 经 P2 门禁：出卡后仍为 GRAY；批准后验证回放通过 → FULL+全量生效。"""
|
||
world = _tight_world()
|
||
opt = ParameterOptimizer()
|
||
g = opt.promote_gray(world, {"freezeWindowHours": 0.0})
|
||
exp_id = g["experiment"]["id"]
|
||
client, _, provider = client_factory(world)
|
||
resp = client.post(f"/api/params/experiments/{exp_id}/promote",
|
||
json={"sessionId": "promote-session"})
|
||
assert resp.status_code == 200, resp.text
|
||
data = resp.json()
|
||
block = data["block"]
|
||
assert block["type"] == "confirm-card"
|
||
confirm_id = block["props"]["confirmId"]
|
||
assert data["message"].find("P2") >= 0
|
||
# 卡面文案含实验摘要/候选参数/回放 KPI
|
||
card_lines = "\n".join(block["props"].get("summary") or [])
|
||
assert "freezeWindowHours=0.0" in card_lines
|
||
assert "GRAY" in card_lines and "KPI" in card_lines
|
||
assert "GRAY→ACTIVE" in data["message"] or "升级" in data["message"]
|
||
assert world["paramExperiments"][0]["status"] == "GRAY" # 确认前不动
|
||
assert "param.promote.stage" in _audit_actions(world)
|
||
# 批准执行（单次确认，P2 不需二次确认）
|
||
resp2 = client.post("/api/actions/confirm",
|
||
json={"sessionId": "promote-session", "confirmId": confirm_id,
|
||
"approve": True, "note": "同意升级"})
|
||
assert resp2.status_code == 200, resp2.text
|
||
assert resp2.json()["secondConfirmRequired"] is False
|
||
rec = world["paramExperiments"][0]
|
||
assert rec["status"] == "FULL"
|
||
assert world["scheduleParams"]["freezeWindowHours"] == 0.0
|
||
assert rec["validationReplay"]["degraded"] is False
|
||
assert rec["productionBaseline"] is None # 激活时无观测，等首个观测自校准
|
||
audits = _audit_actions(world)
|
||
assert "param.promote" in audits
|
||
assert "param.promote.stage" in audits
|
||
promote_evt = next(e for e in world["auditEvents"] if e["action"] == "param.promote")
|
||
assert promote_evt["power"] == "P2"
|
||
assert promote_evt["rationale"]["confirmId"] == confirm_id
|
||
# 批准历史记录 note
|
||
identity = provider._identity("planner")
|
||
history = _as(identity, harness.list_approval_history)
|
||
assert history[-1]["note"] == "同意升级"
|
||
assert history[-1]["status"] == "APPROVED"
|
||
|
||
|
||
def test_promote_reject_keeps_gray(client_factory):
|
||
"""拒绝则留在 GRAY：无任何世界写入，审计 param.experiment.promote.reject。"""
|
||
world = _tight_world()
|
||
opt = ParameterOptimizer()
|
||
g = opt.promote_gray(world, {"freezeWindowHours": 0.0})
|
||
exp_id = g["experiment"]["id"]
|
||
client, _, _ = client_factory(world)
|
||
resp = client.post(f"/api/params/experiments/{exp_id}/promote", json={"sessionId": "s"})
|
||
confirm_id = resp.json()["block"]["props"]["confirmId"]
|
||
resp2 = client.post("/api/actions/confirm",
|
||
json={"sessionId": "s", "confirmId": confirm_id, "approve": False})
|
||
assert resp2.status_code == 200, resp2.text
|
||
assert "拒绝" in resp2.json()["message"] or "取消" in resp2.json()["message"] or "DENIED" in resp2.json()["message"].upper() or "驳回" in resp2.json()["message"]
|
||
rec = world["paramExperiments"][0]
|
||
assert rec["status"] == "GRAY" # 留在 GRAY
|
||
assert world["scheduleParams"]["freezeWindowHours"] == 24.0
|
||
assert "param.experiment.promote.reject" in _audit_actions(world)
|
||
|
||
|
||
def test_promote_unknown_or_non_gray_rejected(client_factory):
|
||
"""契约保护：不存在实验 / 非 GRAY 实验不出卡。"""
|
||
client, store, _ = client_factory()
|
||
resp = client.post("/api/params/experiments/px-none/promote", json={"sessionId": "s"})
|
||
assert resp.status_code == 200 and "error" in resp.json()
|
||
# 已升级实验不可再提交
|
||
world = store.data
|
||
opt = ParameterOptimizer()
|
||
g = opt.promote_gray(world, {"freezeWindowHours": 0.0})
|
||
rec = opt.finalize(world, g["experiment"]["id"])
|
||
assert rec["status"] == "FULL"
|
||
resp2 = client.post(f"/api/params/experiments/{g['experiment']['id']}/promote",
|
||
json={"sessionId": "s"})
|
||
assert "error" in resp2.json()
|
||
|
||
|
||
# ---------------- 手动回滚（P1） ----------------
|
||
def test_manual_rollback_p1_direct(client_factory):
|
||
"""手动回滚 P1：直接执行，恢复上一参数版本，审计 param.rollback。"""
|
||
world = _tight_world()
|
||
opt = ParameterOptimizer()
|
||
g = opt.promote_gray(world, {"freezeWindowHours": 0.0})
|
||
exp_id = g["experiment"]["id"]
|
||
opt.finalize(world, exp_id) # → FULL，freeze=0.0
|
||
assert world["scheduleParams"]["freezeWindowHours"] == 0.0
|
||
client, _, _ = client_factory(world)
|
||
resp = client.post(f"/api/params/experiments/{exp_id}/rollback",
|
||
json={"sessionId": "s", "reason": "人工干预"})
|
||
assert resp.status_code == 200, resp.text
|
||
rec = world["paramExperiments"][0]
|
||
assert rec["status"] == "ROLLED_BACK"
|
||
assert rec["rollbackReason"] == "人工干预"
|
||
assert world["scheduleParams"]["freezeWindowHours"] == 24.0 # 恢复上一版本
|
||
evt = next(e for e in world["auditEvents"] if e["action"] == "param.rollback")
|
||
assert evt["power"] == "P1"
|
||
assert evt["rationale"]["reason"] == "人工干预"
|
||
|
||
|
||
# ---------------- 线上观测驱动回滚监控 ----------------
|
||
def test_observation_degradation_auto_rollback_via_endpoint(client_factory):
|
||
"""观测回调：连续劣化达阈值自动回滚+审计，观测数据落盘 paramObservations。"""
|
||
world = _tight_world()
|
||
opt = ParameterOptimizer(max_consecutive_degraded=2)
|
||
g = opt.promote_gray(world, {"freezeWindowHours": 0.0})
|
||
exp_id = g["experiment"]["id"]
|
||
opt.finalize(world, exp_id) # → FULL
|
||
client, _, _ = client_factory(world)
|
||
# 基线自校准（首个观测）
|
||
r1 = client.post("/api/params/observations", json={"sessionId": "obs", "kpi": {"tardiness": 10.0}})
|
||
assert r1.status_code == 200, r1.text
|
||
assert r1.json()["autoRolledBack"] == []
|
||
# 劣化 1
|
||
r2 = client.post("/api/params/observations", json={"sessionId": "obs", "kpi": {"tardiness": 60.0}})
|
||
assert r2.status_code == 200, r2.text
|
||
assert r2.json()["autoRolledBack"] == []
|
||
assert world["paramExperiments"][0]["degradedStreak"] == 1
|
||
# 劣化 2 → 自动回滚
|
||
r3 = client.post("/api/params/observations", json={"sessionId": "obs", "kpi": {"tardiness": 90.0}})
|
||
assert r3.status_code == 200, r3.text
|
||
assert r3.json()["autoRolledBack"] == [exp_id]
|
||
rec = world["paramExperiments"][0]
|
||
assert rec["status"] == "ROLLED_BACK"
|
||
assert rec["rollbackReason"] == "production-degraded"
|
||
assert world["scheduleParams"]["freezeWindowHours"] == 24.0
|
||
# 观测数据落盘
|
||
assert len(world.get("paramObservations") or []) == 3
|
||
assert world["paramObservations"][-1]["kpi"] == {"tardiness": 90.0}
|
||
# 审计
|
||
evts = [e for e in world["auditEvents"]
|
||
if e["action"] == "param.experiment.auto_rolled_back"]
|
||
assert len(evts) == 1
|
||
assert evts[0]["target"] == {"type": "PARAM_EXPERIMENT", "id": exp_id}
|
||
assert evts[0]["power"] == "P1"
|
||
assert evts[0]["rationale"]["degradedStreak"] == 2
|
||
assert evts[0]["rationale"]["observationId"] == world["paramObservations"][-1]["id"]
|
||
|
||
|
||
def test_observation_endpoint_configurable_threshold(client_factory):
|
||
"""可配置连续劣化阈值：maxConsecutiveDegraded=1 单次劣化即回滚。"""
|
||
world = _tight_world()
|
||
opt = ParameterOptimizer()
|
||
g = opt.promote_gray(world, {"freezeWindowHours": 0.0})
|
||
exp_id = g["experiment"]["id"]
|
||
opt.finalize(world, exp_id)
|
||
client, _, _ = client_factory(world)
|
||
r1 = client.post("/api/params/observations", json={"sessionId": "obs", "kpi": {"tardiness": 10.0}})
|
||
assert r1.status_code == 200
|
||
r2 = client.post("/api/params/observations",
|
||
json={"sessionId": "obs", "kpi": {"tardiness": 70.0},
|
||
"maxConsecutiveDegraded": 1})
|
||
assert r2.status_code == 200, r2.text
|
||
assert r2.json()["autoRolledBack"] == [exp_id]
|
||
assert world["paramExperiments"][0]["status"] == "ROLLED_BACK"
|