278 lines
13 KiB
Python
278 lines
13 KiB
Python
|
# ============================================================
|
|||
|
|
# 参数优化闭环黄金测试（矩阵 87 行 · 方向 E）
|
|||
|
|
# 覆盖：集隔离、回放（沙盒重排）、灰度受限生效、退化自动回滚、
|
|||
|
|
# 验证通过全量生效、敏感性分析衔接（候选方向）
|
|||
|
|
# ============================================================
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
import json
|
|||
|
|
|
|||
|
|
import pytest
|
|||
|
|
|
|||
|
|
from server.agent_core.param_opt import ParameterOptimizer, optimize
|
|||
|
|
from server.aps_domain.sensitivity import run_sensitivity
|
|||
|
|
from server.state.seed import seed_world
|
|||
|
|
from server.timeutil import add_minutes, fmt_date, today0
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _tight_world():
|
|||
|
|
"""6 单紧张交期世界：基线（冻结 24h）总延期 > 0，参数方向稳定可复现。
|
|||
|
|
|
|||
|
|
构造口径与 test_params.py 的 _two_orders_world 一致（在演示主数据上
|
|||
|
|
覆盖订单集），qty=300 使单产线满负荷，2/3/4/5 天交期产生延期压力。
|
|||
|
|
"""
|
|||
|
|
w = seed_world()
|
|||
|
|
base = today0()
|
|||
|
|
order_date = fmt_date(base)
|
|||
|
|
product = next(m for m in w["materials"] if m["type"] == "FINISHED_PRODUCT")
|
|||
|
|
sos = []
|
|||
|
|
for i in range(6):
|
|||
|
|
oid = i + 1
|
|||
|
|
level = "VIP" if i % 2 == 0 else "C"
|
|||
|
|
due = fmt_date(add_minutes(base, (2, 2, 3, 3, 4, 5)[i] * 24 * 60))
|
|||
|
|
sos.append({
|
|||
|
|
"id": oid, "orderNo": f"SO-PT-{oid}", "customerId": f"C{oid}",
|
|||
|
|
"customerName": f"客户{level}", "customerLevel": level,
|
|||
|
|
"orderDate": order_date, "deliveryDate": due,
|
|||
|
|
"priority": 5, "manualPriority": None, "status": "CONFIRMED",
|
|||
|
|
"source": "MANUAL", "specialRequirements": "", "totalAmount": 1000,
|
|||
|
|
"isRush": False, "rushStrategy": None, "changes": [],
|
|||
|
|
"createdBy": "test", "createdAt": order_date + " 09:00",
|
|||
|
|
"updatedAt": order_date + " 09:00",
|
|||
|
|
"items": [{
|
|||
|
|
"id": oid * 10, "orderId": oid, "lineNo": 1,
|
|||
|
|
"productId": product["id"], "productName": product["name"],
|
|||
|
|
"productCode": product["code"], "quantity": 300,
|
|||
|
|
"unit": "件", "bomVersion": "V1.0", "routingVersion": "V1.0",
|
|||
|
|
"status": "PENDING", "note": "",
|
|||
|
|
}],
|
|||
|
|
})
|
|||
|
|
w["salesOrders"] = sos
|
|||
|
|
w["scheduleVersions"] = []
|
|||
|
|
w["productionOrders"] = []
|
|||
|
|
w["workOrders"] = []
|
|||
|
|
w["conflicts"] = []
|
|||
|
|
return w
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_split_orders_isolation():
|
|||
|
|
"""训练/验证集隔离：互斥、并集全量、同种子可复现、不同种子不同切分。"""
|
|||
|
|
w = _tight_world()
|
|||
|
|
opt = ParameterOptimizer(seed=20260802)
|
|||
|
|
s1 = opt.split_orders(w)
|
|||
|
|
s2 = opt.split_orders(w) # 同种子 → 相同切分
|
|||
|
|
assert s1 == s2
|
|||
|
|
assert not (set(s1["train"]) & set(s1["validation"]))
|
|||
|
|
all_ids = {so["id"] for so in w["salesOrders"]}
|
|||
|
|
assert set(s1["train"]) | set(s1["validation"]) == all_ids
|
|||
|
|
assert len(s1["train"]) >= 1 and len(s1["validation"]) >= 1
|
|||
|
|
s3 = opt.split_orders(w, seed=42) # 不同种子 → 不同切分
|
|||
|
|
assert s1["train"] != s3["train"]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_replay_sandbox_no_mutation_and_directions():
|
|||
|
|
"""回放：沙盒重排零副作用；改善/退化方向与引擎信号一致；越界参数拒绝。"""
|
|||
|
|
w = _tight_world()
|
|||
|
|
before = json.dumps(w, ensure_ascii=False, sort_keys=True)
|
|||
|
|
opt = ParameterOptimizer()
|
|||
|
|
improved = opt.replay(w, {"freezeWindowHours": 0.0})
|
|||
|
|
assert improved["degraded"] is False
|
|||
|
|
assert improved["deltas"]["tardiness"] < -1 # 冻结 0h 相对 24h 明显改善延期
|
|||
|
|
degraded = opt.replay(w, {"freezeWindowHours": 48.0})
|
|||
|
|
assert degraded["degraded"] is True
|
|||
|
|
assert degraded["deltas"]["tardiness"] > 1 # 冻结 48h 明显恶化延期
|
|||
|
|
after = json.dumps(w, ensure_ascii=False, sort_keys=True)
|
|||
|
|
assert before == after # 回放不写主干
|
|||
|
|
with pytest.raises(ValueError):
|
|||
|
|
opt.replay(w, {"freezeWindowHours": 999.0}) # 越界参数拒绝
|
|||
|
|
with pytest.raises(ValueError):
|
|||
|
|
opt.replay(w, {"unknownKey": 1.0}) # 未知键拒绝
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_gray_restricted_effect():
|
|||
|
|
"""灰度受限生效：命中作用域的查询才拿到候选值，其余保持基线。"""
|
|||
|
|
w = _tight_world()
|
|||
|
|
opt = ParameterOptimizer()
|
|||
|
|
g = opt.promote_gray(w, {"freezeWindowHours": 0.0}, scope={"lineIds": [1]})
|
|||
|
|
assert g["accepted"] is True
|
|||
|
|
assert g["experiment"]["status"] == "GRAY"
|
|||
|
|
assert opt.active_params(w, {"lineIds": [1]})["freezeWindowHours"] == 0.0
|
|||
|
|
assert opt.active_params(w, {"lineIds": [2]})["freezeWindowHours"] == 24.0
|
|||
|
|
assert opt.active_params(w)["freezeWindowHours"] == 24.0 # 无作用域不扩散
|
|||
|
|
# 灰度阶段不动 scheduleParams（全量写入在 finalize）
|
|||
|
|
assert w["scheduleParams"]["freezeWindowHours"] == 24.0
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_replay_gate_rejects_degrading_candidate():
|
|||
|
|
"""回放门禁：训练集回放退化的候选在灰度前被拦截，世界零写入。"""
|
|||
|
|
w = _tight_world()
|
|||
|
|
opt = ParameterOptimizer()
|
|||
|
|
g = opt.promote_gray(w, {"freezeWindowHours": 48.0})
|
|||
|
|
assert g["accepted"] is False
|
|||
|
|
assert g["reason"] == "replay-degraded"
|
|||
|
|
assert g["replay"]["degraded"] is True
|
|||
|
|
assert len(w.get("paramExperiments") or []) == 0
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_rollback_on_degradation():
|
|||
|
|
"""效果退化自动回滚：金丝雀灰度后验证集回放退化 → 自动回滚到上一版本。"""
|
|||
|
|
w = _tight_world()
|
|||
|
|
opt = ParameterOptimizer()
|
|||
|
|
g = opt.promote_gray(w, {"freezeWindowHours": 48.0}, scope={"lineIds": [1]},
|
|||
|
|
require_replay_pass=False)
|
|||
|
|
assert g["accepted"] is True
|
|||
|
|
assert g["experiment"]["trainReplay"]["degraded"] is True
|
|||
|
|
rec = opt.finalize(w, g["experiment"]["id"])
|
|||
|
|
assert rec["status"] == "ROLLED_BACK"
|
|||
|
|
assert rec["rollbackReason"] == "validation-degraded"
|
|||
|
|
assert "validationReplay" in rec and rec["validationReplay"]["degraded"] is True
|
|||
|
|
# 回滚后：受限参数不再生效，基线参数保持
|
|||
|
|
assert opt.active_params(w, {"lineIds": [1]})["freezeWindowHours"] == 24.0
|
|||
|
|
assert w["scheduleParams"]["freezeWindowHours"] == 24.0
|
|||
|
|
assert len(opt.history(w)) == 1
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_finalize_full_apply_and_manual_rollback():
|
|||
|
|
"""验证通过 → 全量生效（留 before/after 证据）；显式回滚恢复上一版本。"""
|
|||
|
|
w = _tight_world()
|
|||
|
|
opt = ParameterOptimizer()
|
|||
|
|
g = opt.promote_gray(w, {"freezeWindowHours": 0.0}, scope={"lineIds": [1]})
|
|||
|
|
rec = opt.finalize(w, g["experiment"]["id"])
|
|||
|
|
assert rec["status"] == "FULL"
|
|||
|
|
assert w["scheduleParams"]["freezeWindowHours"] == 0.0
|
|||
|
|
assert rec["fullApply"]["before"]["freezeWindowHours"] == 24.0
|
|||
|
|
assert rec["fullApply"]["after"]["freezeWindowHours"] == 0.0
|
|||
|
|
rb = opt.rollback(w, g["experiment"]["id"])
|
|||
|
|
assert rb["status"] == "ROLLED_BACK"
|
|||
|
|
assert w["scheduleParams"]["freezeWindowHours"] == 24.0 # 恢复上一参数版本
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_candidates_from_sensitivity():
|
|||
|
|
"""与敏感性分析衔接：Tornado 产出 → 候选方向，且候选回放不退化。"""
|
|||
|
|
w = _tight_world()
|
|||
|
|
opt = ParameterOptimizer()
|
|||
|
|
report = run_sensitivity(w, strategy="COMPREHENSIVE")
|
|||
|
|
cands = opt.candidates_from_sensitivity(report)
|
|||
|
|
assert cands, "敏感性报告应产出候选"
|
|||
|
|
# 摆幅最大因子优先：冻结窗口（低档 0h 改善延期）
|
|||
|
|
assert cands[0]["factorId"] == "freezeWindowHours"
|
|||
|
|
assert cands[0]["candidate"]["freezeWindowHours"] == 0.0
|
|||
|
|
# 效率因子高档（×1.2）方向
|
|||
|
|
eff = next(c for c in cands if c["factorId"] == "lineEfficiency")
|
|||
|
|
assert eff["candidate"]["efficiencyScale"] == 1.2
|
|||
|
|
# 每个候选按改善方向回放都不退化
|
|||
|
|
for c in cands[:2]:
|
|||
|
|
r = opt.replay(w, c["candidate"])
|
|||
|
|
assert r["degraded"] is False
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_pipeline_end_to_end():
|
|||
|
|
"""一键闭环：敏感性取方向 → 逐候选灰度→收口，产出实验记录。"""
|
|||
|
|
w = _tight_world()
|
|||
|
|
out = optimize(w, scope={"lineIds": [1]})
|
|||
|
|
assert out["summary"]["total"] >= 3
|
|||
|
|
assert out["summary"]["rolledBack"] == 0 # 敏感性方向候选不应退化
|
|||
|
|
assert out["summary"]["active"] >= 1
|
|||
|
|
assert any(r["stage"] == "full" for r in out["results"])
|
|||
|
|
assert len(w.get("paramExperiments") or []) == out["summary"]["total"]
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ============================================================
|
|||
|
|
# 矩阵 87 补充：线上观测回调单元（观测落盘 / 基线 / 连续劣化自动回滚 / 审计）
|
|||
|
|
# ============================================================
|
|||
|
|
def _activate_full(w, candidate=None):
|
|||
|
|
"""helper: 灰度→finalize 升级为 FULL，返回实验 id。"""
|
|||
|
|
opt = ParameterOptimizer()
|
|||
|
|
cand = candidate or {"freezeWindowHours": 0.0}
|
|||
|
|
g = opt.promote_gray(w, cand)
|
|||
|
|
assert g["accepted"] is True
|
|||
|
|
rec = opt.finalize(w, g["experiment"]["id"])
|
|||
|
|
assert rec["status"] == "FULL"
|
|||
|
|
return g["experiment"]["id"]
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_observation_persisted_and_self_calibrating_baseline():
|
|||
|
|
"""观测回调：观测落盘 paramObservations；首个观测自校准为生效前基线。"""
|
|||
|
|
w = _tight_world()
|
|||
|
|
opt = ParameterOptimizer()
|
|||
|
|
exp_id = _activate_full(w)
|
|||
|
|
rec0 = w["paramExperiments"][0]
|
|||
|
|
assert rec0["productionBaseline"] is None # 激活时无观测
|
|||
|
|
out = opt.record_observation(w, {"tardiness": 5.0})
|
|||
|
|
assert out["autoRolledBack"] == []
|
|||
|
|
assert len(w["paramObservations"]) == 1
|
|||
|
|
obs = w["paramObservations"][0]
|
|||
|
|
assert obs["kpi"] == {"tardiness": 5.0}
|
|||
|
|
assert obs["evaluations"][0]["experimentId"] == exp_id
|
|||
|
|
assert obs["evaluations"][0]["degraded"] is False # 首个观测即基线，不算劣化
|
|||
|
|
assert rec0["productionBaseline"] == {"tardiness": 5.0}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_observation_uses_pre_activation_baseline():
|
|||
|
|
"""激活前已有生产观测：finalize 捕获最近一次观测为生效前基线。"""
|
|||
|
|
w = _tight_world()
|
|||
|
|
opt = ParameterOptimizer()
|
|||
|
|
opt.record_observation(w, {"tardiness": 10.0}) # 生产基线观测（激活前）
|
|||
|
|
g = opt.promote_gray(w, {"freezeWindowHours": 0.0})
|
|||
|
|
rec = opt.finalize(w, g["experiment"]["id"])
|
|||
|
|
assert rec["productionBaseline"] == {"tardiness": 10.0}
|
|||
|
|
assert rec["degradedStreak"] == 0
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_consecutive_degradation_auto_rollback_with_audit():
|
|||
|
|
"""连续劣化自动回滚：streak 达阈值 → ROLLED_BACK + 恢复基线参数 + 审计。"""
|
|||
|
|
w = _tight_world()
|
|||
|
|
opt = ParameterOptimizer(max_consecutive_degraded=2)
|
|||
|
|
exp_id = _activate_full(w)
|
|||
|
|
opt.record_observation(w, {"tardiness": 10.0}) # 基线自校准
|
|||
|
|
opt.record_observation(w, {"tardiness": 50.0}) # 劣化 1
|
|||
|
|
rec = w["paramExperiments"][0]
|
|||
|
|
assert rec["degradedStreak"] == 1
|
|||
|
|
assert rec["status"] == "FULL"
|
|||
|
|
out = opt.record_observation(w, {"tardiness": 80.0}) # 劣化 2 → 回滚
|
|||
|
|
assert out["autoRolledBack"] == [exp_id]
|
|||
|
|
rec = w["paramExperiments"][0]
|
|||
|
|
assert rec["status"] == "ROLLED_BACK"
|
|||
|
|
assert rec["rollbackReason"] == "production-degraded"
|
|||
|
|
assert rec["autoRollback"]["degradedStreak"] == 2
|
|||
|
|
assert w["scheduleParams"]["freezeWindowHours"] == 24.0 # 恢复上一版本
|
|||
|
|
events = [e for e in w["auditEvents"]
|
|||
|
|
if e["action"] == "param.experiment.auto_rolled_back"]
|
|||
|
|
assert len(events) == 1
|
|||
|
|
assert events[0]["target"] == {"type": "PARAM_EXPERIMENT", "id": exp_id}
|
|||
|
|
assert events[0]["power"] == "P1"
|
|||
|
|
assert events[0]["rationale"]["degradedStreak"] == 2
|
|||
|
|
assert events[0]["rationale"]["reason"] == "production-degraded"
|
|||
|
|
assert len(w["paramObservations"]) == 3
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_observation_recovery_resets_streak():
|
|||
|
|
"""观测恢复：非劣化观测清零连败计数，不触发回滚。"""
|
|||
|
|
w = _tight_world()
|
|||
|
|
opt = ParameterOptimizer(max_consecutive_degraded=2)
|
|||
|
|
_activate_full(w)
|
|||
|
|
opt.record_observation(w, {"tardiness": 10.0})
|
|||
|
|
opt.record_observation(w, {"tardiness": 60.0}) # 劣化 1
|
|||
|
|
opt.record_observation(w, {"tardiness": 8.0}) # 恢复 → 清零
|
|||
|
|
rec = w["paramExperiments"][0]
|
|||
|
|
assert rec["degradedStreak"] == 0
|
|||
|
|
assert rec["status"] == "FULL"
|
|||
|
|
assert w["scheduleParams"]["freezeWindowHours"] == 0.0
|
|||
|
|
assert [e["action"] for e in w["auditEvents"]] == [] # 无自动回滚审计
|
|||
|
|
assert len(w["paramObservations"]) == 3
|
|||
|
|
|
|||
|
|
|
|||
|
|
def test_observation_monitors_only_active_full():
|
|||
|
|
"""只监控 ACTIVE/FULL：GRAY 与已回滚实验不参与观测评估。"""
|
|||
|
|
w = _tight_world()
|
|||
|
|
opt = ParameterOptimizer()
|
|||
|
|
g = opt.promote_gray(w, {"freezeWindowHours": 0.0}) # GRAY
|
|||
|
|
out = opt.record_observation(w, {"tardiness": 10.0})
|
|||
|
|
assert out["observation"]["evaluations"] == []
|
|||
|
|
assert w["paramExperiments"][0]["status"] == "GRAY"
|
|||
|
|
opt.finalize(w, g["experiment"]["id"]) # → FULL
|
|||
|
|
opt.rollback(w, g["experiment"]["id"]) # → ROLLED_BACK
|
|||
|
|
out2 = opt.record_observation(w, {"tardiness": 100.0})
|
|||
|
|
assert out2["observation"]["evaluations"] == []
|
|||
|
|
assert out2["autoRolledBack"] == []
|