945 lines
41 KiB
Python
945 lines
41 KiB
Python
# ============================================================
|
||
# Saga/补偿编排框架（moduleId: domain-saga, 可重用 ✔）
|
||
# round-39 方向 P：跨内部世界状态与外部系统的 Saga/补偿
|
||
# ① 编排单元：step（name/action/幂等键 idem_key/超时 timeout_sec/重试 max_retries/补偿 compensation_action）
|
||
# ② 恢复队列：saga 记录随世界状态持久化（宽容加载），resume/replay 未完成 saga
|
||
# ③ 补偿注册表：动作→补偿动作映射；失败自动执行补偿链；补偿也失败 → MANUAL_TAKEOVER
|
||
# ④ 超时/重试：每步超时上限 + 重试次数（指数退避简化），超限走补偿
|
||
# ⑤ 审计：每次 step 状态迁移写 saga.step.*（含 idemKey/status/compensation）
|
||
# ⑥ 第一个 Saga 实例：WMS 缺料→重排→MES 下发→回执 闭环（server/aps_domain/wms_events.py）
|
||
# ============================================================
|
||
from __future__ import annotations
|
||
|
||
import copy
|
||
import time
|
||
import uuid
|
||
from dataclasses import dataclass
|
||
from datetime import datetime
|
||
from typing import Any, Callable
|
||
|
||
from server.timeutil import fmt_dt
|
||
|
||
World = dict[str, Any]
|
||
|
||
# ------------------------------------------------------------
|
||
# 状态机
|
||
# ------------------------------------------------------------
|
||
SAGA_STATUSES = ("PENDING", "RUNNING", "WAITING_HUMAN", "SUCCEEDED", "FAILED",
|
||
"COMPENSATING", "COMPENSATED", "MANUAL_TAKEOVER")
|
||
STEP_STATUSES = ("PENDING", "RUNNING", "WAITING_HUMAN", "SUCCEEDED", "FAILED",
|
||
"COMPENSATED", "SKIPPED")
|
||
|
||
_TERMINAL = ("SUCCEEDED", "COMPENSATED", "MANUAL_TAKEOVER")
|
||
_RECOVERABLE = ("PENDING", "RUNNING", "WAITING_HUMAN", "FAILED")
|
||
|
||
|
||
class SagaError(Exception):
|
||
"""Saga 编排基础异常。"""
|
||
|
||
|
||
class SagaStepTimeout(SagaError):
|
||
"""步骤超时（超过 timeout_sec 或动作主动抛出）。"""
|
||
|
||
|
||
class SagaStepFailed(SagaError):
|
||
"""步骤业务失败（动作可主动抛出，等价于抛普通异常）。"""
|
||
|
||
|
||
class SagaCompensationError(SagaError):
|
||
"""补偿动作失败（触发 MANUAL_TAKEOVER）。"""
|
||
|
||
|
||
class SagaNotFound(KeyError):
|
||
"""saga 记录不存在。"""
|
||
|
||
|
||
def _now() -> str:
|
||
return fmt_dt(datetime.now())
|
||
|
||
def _jsonable(value: Any) -> Any:
|
||
"""把步骤结果递归转成可 JSON 持久化形态（pydantic BaseModel → dict，其余兜底 str）。"""
|
||
if hasattr(value, "model_dump"):
|
||
return _jsonable(value.model_dump())
|
||
if isinstance(value, dict):
|
||
return {str(k): _jsonable(v) for k, v in value.items()}
|
||
if isinstance(value, (list, tuple)):
|
||
return [_jsonable(v) for v in value]
|
||
if value is None or isinstance(value, (str, int, float, bool)):
|
||
return value
|
||
return str(value)
|
||
|
||
|
||
# ------------------------------------------------------------
|
||
# ① 编排单元
|
||
# ------------------------------------------------------------
|
||
@dataclass
|
||
class SagaStep:
|
||
"""单个 saga 步骤：动作 + 幂等键 + 超时 + 重试 + 补偿动作。"""
|
||
|
||
name: str
|
||
action: str
|
||
idem_key: str = ""
|
||
timeout_sec: float = 30.0
|
||
max_retries: int = 2
|
||
compensation_action: str | None = None
|
||
gate: bool = False
|
||
|
||
@classmethod
|
||
def from_mapping(cls, raw: dict[str, Any]) -> SagaStep:
|
||
d = dict(raw or {})
|
||
keys = {"name", "action", "idem_key", "timeout_sec", "max_retries",
|
||
"compensation_action", "gate", "idemKey", "timeoutSec",
|
||
"maxRetries", "compensationAction"}
|
||
clean: dict[str, Any] = {}
|
||
for k, v in d.items():
|
||
if k not in keys:
|
||
continue
|
||
clean[k] = v
|
||
if "idemKey" in clean and "idem_key" not in clean:
|
||
clean["idem_key"] = clean.pop("idemKey")
|
||
if "timeoutSec" in clean and "timeout_sec" not in clean:
|
||
clean["timeout_sec"] = clean.pop("timeoutSec")
|
||
if "maxRetries" in clean and "max_retries" not in clean:
|
||
clean["max_retries"] = clean.pop("maxRetries")
|
||
if "compensationAction" in clean and "compensation_action" not in clean:
|
||
clean["compensation_action"] = clean.pop("compensationAction")
|
||
return cls(**clean)
|
||
|
||
def to_dict(self) -> dict[str, Any]:
|
||
return {
|
||
"name": self.name,
|
||
"action": self.action,
|
||
"idemKey": self.idem_key,
|
||
"timeoutSec": float(self.timeout_sec),
|
||
"maxRetries": int(self.max_retries),
|
||
"compensationAction": self.compensation_action,
|
||
"gate": bool(self.gate),
|
||
"status": "PENDING",
|
||
"attempts": 0,
|
||
"timeouts": 0,
|
||
"startedAt": None,
|
||
"finishedAt": None,
|
||
"result": None,
|
||
"error": None,
|
||
"compensation": None,
|
||
}
|
||
# ------------------------------------------------------------
|
||
# Saga 编排器
|
||
# ------------------------------------------------------------
|
||
class SagaCoordinator:
|
||
"""Saga 编排器：步骤执行 / 重试 / 超时 / 补偿链 / 人工接管 / 持久化恢复。
|
||
|
||
- saga 记录持久化在 store.data["sagas"]（随世界状态落盘，宽容加载）；
|
||
- 幂等键登记在 store.data["sagaIdem"]（跨 saga 去重 / 恢复时安全重放）；
|
||
- 补偿注册表 registry: {动作名: {"fn": callable(ctx)->dict, "compensation": 补偿动作名}}；
|
||
- 补偿动作也是注册表条目：{补偿动作名: {"fn": callable(ctx)->dict}}。
|
||
"""
|
||
|
||
def __init__(self, store, *, actor: str = "saga",
|
||
registry: dict[str, dict[str, Any]] | None = None,
|
||
clock: Callable[[], float] | None = None,
|
||
sleep_fn: Callable[[float], None] | None = None,
|
||
backoff_base: float = 0.0, backoff_cap: float = 60.0,
|
||
second_approver: Any = None):
|
||
self.store = store
|
||
self.actor = actor
|
||
self.registry = registry if registry is not None else default_registry()
|
||
self.clock = clock or time.monotonic
|
||
self.sleep_fn = sleep_fn or time.sleep
|
||
self.backoff_base = backoff_base
|
||
self.backoff_cap = backoff_cap
|
||
self.second_approver = second_approver
|
||
self._ensure_tables()
|
||
|
||
# ---------------- 持久化（随世界状态，宽容加载） ----------------
|
||
def _ensure_tables(self) -> None:
|
||
world = self.store.data
|
||
sagas = world.setdefault("sagas", [])
|
||
world["sagas"] = [s for s in sagas if isinstance(s, dict)]
|
||
idem = world.setdefault("sagaIdem", {})
|
||
if not isinstance(idem, dict):
|
||
world["sagaIdem"] = {}
|
||
|
||
def _save(self) -> None:
|
||
self.store.save()
|
||
|
||
def _find(self, saga_id: str) -> dict[str, Any]:
|
||
for s in self.store.data.get("sagas", []):
|
||
if s.get("id") == saga_id:
|
||
return s
|
||
raise SagaNotFound(saga_id)
|
||
|
||
# ---------------- 创建 ----------------
|
||
def create_saga(self, name: str, steps: list[SagaStep | dict[str, Any]],
|
||
context: dict[str, Any] | None = None, *,
|
||
dedupe_key: str | None = None) -> dict[str, Any]:
|
||
"""创建并持久化一条 saga；dedupe_key 相同（如 WMS 事件 eventId）时复用已有记录（幂等）。"""
|
||
if dedupe_key:
|
||
existing = next((s for s in self.store.data.get("sagas", [])
|
||
if s.get("name") == name and s.get("dedupeKey") == dedupe_key), None)
|
||
if existing is not None:
|
||
return existing
|
||
saga_id = f"saga-{uuid.uuid4().hex[:10]}"
|
||
step_records: list[dict[str, Any]] = []
|
||
for raw in steps:
|
||
step_records.append(raw.to_dict() if isinstance(raw, SagaStep)
|
||
else SagaStep.from_mapping(dict(raw)).to_dict())
|
||
record: dict[str, Any] = {
|
||
"id": saga_id,
|
||
"name": name,
|
||
"status": "PENDING",
|
||
"createdAt": _now(),
|
||
"updatedAt": _now(),
|
||
"context": _jsonable(copy.deepcopy(context or {})),
|
||
"dedupeKey": dedupe_key,
|
||
"steps": step_records,
|
||
"manualTakeover": False,
|
||
"takeoverReason": None,
|
||
"compensationReason": None,
|
||
"auditRefs": [f"saga:{saga_id}"],
|
||
}
|
||
self.store.data.setdefault("sagas", []).append(record)
|
||
self._audit_saga(record, "saga.created", result="PENDING")
|
||
self._save()
|
||
return record
|
||
|
||
# ---------------- 执行 ----------------
|
||
def run(self, saga_id: str, *, auto_approve: bool = True) -> dict[str, Any]:
|
||
"""执行/续跑一条 saga：按序执行未完成步骤；失败自动进入补偿链。
|
||
|
||
- auto_approve=True：gate 步骤由编排器代行人工确认（测试/自动化模式）；
|
||
- auto_approve=False：gate 步骤停在 WAITING_HUMAN（中间态可见，人工门禁）。
|
||
"""
|
||
saga = self._find(saga_id)
|
||
if saga["status"] in ("SUCCEEDED", "COMPENSATED"):
|
||
return saga
|
||
if saga["status"] == "MANUAL_TAKEOVER" or saga.get("manualTakeover"):
|
||
return saga # 人工接管中：需显式 retry 恢复自动化
|
||
saga["status"] = "RUNNING"
|
||
saga["updatedAt"] = _now()
|
||
self._audit_saga(saga, "saga.run", result="RUNNING")
|
||
for step in saga["steps"]:
|
||
if step["status"] in ("SUCCEEDED", "COMPENSATED", "SKIPPED"):
|
||
continue
|
||
if step.get("gate") and not auto_approve:
|
||
step["status"] = "WAITING_HUMAN"
|
||
saga["status"] = "WAITING_HUMAN"
|
||
saga["updatedAt"] = _now()
|
||
self._audit_step(saga, step, "saga.step.waiting_human", result="WAITING_HUMAN")
|
||
self._save()
|
||
return saga
|
||
outcome = self._execute_step(saga, step, auto_approve=auto_approve)
|
||
if outcome == "FAILED":
|
||
self._compensate(saga, reason=f"step:{step['name']}")
|
||
return saga
|
||
saga["status"] = "SUCCEEDED"
|
||
saga["updatedAt"] = _now()
|
||
self._audit_saga(saga, "saga.succeeded", result="SUCCEEDED")
|
||
self._save()
|
||
return saga
|
||
|
||
def retry(self, saga_id: str, *, auto_approve: bool = True) -> dict[str, Any]:
|
||
"""显式重试：回滚失败步的已发生副作用（写前快照），重置重试预算后重跑。
|
||
|
||
人工接管（MANUAL_TAKEOVER）后调用 retry 可恢复自动化。
|
||
"""
|
||
saga = self._find(saga_id)
|
||
if saga["status"] in ("SUCCEEDED", "COMPENSATED"):
|
||
return saga
|
||
ctx = self._ctx(saga, None)
|
||
for step in saga["steps"]:
|
||
if step["status"] == "FAILED":
|
||
if step.get("checkpointId"):
|
||
try:
|
||
_restore_from_checkpoint({**ctx, "step": step})
|
||
except SagaCompensationError:
|
||
pass # 快照缺失时保留现场，交由 run 决定
|
||
step["status"] = "PENDING"
|
||
step["error"] = None
|
||
step["attempts"] = 0
|
||
step["timeouts"] = 0
|
||
step["result"] = None
|
||
step["finishedAt"] = None
|
||
elif step["status"] == "WAITING_HUMAN":
|
||
step["status"] = "PENDING"
|
||
saga["manualTakeover"] = False
|
||
saga["takeoverReason"] = None
|
||
saga["compensationReason"] = None
|
||
saga["status"] = "PENDING"
|
||
saga["updatedAt"] = _now()
|
||
self._audit_saga(saga, "saga.retry", result="RUNNING")
|
||
self._save()
|
||
return self.run(saga_id, auto_approve=auto_approve)
|
||
|
||
def compensate(self, saga_id: str, *, reason: str = "manual") -> dict[str, Any]:
|
||
"""手动补偿（P2 人工动作）：对已成功（或失败但留有写前快照）的步骤逆序补偿。"""
|
||
saga = self._find(saga_id)
|
||
if saga["status"] == "COMPENSATED":
|
||
return saga
|
||
self._compensate(saga, reason=reason)
|
||
return saga
|
||
|
||
def takeover(self, saga_id: str, *, reason: str = "manual") -> dict[str, Any]:
|
||
"""人工接管：停止自动执行，标记 MANUAL_TAKEOVER（可经 retry 恢复自动化）。"""
|
||
saga = self._find(saga_id)
|
||
if saga["status"] in _TERMINAL:
|
||
return saga
|
||
saga["manualTakeover"] = True
|
||
saga["takeoverReason"] = reason
|
||
saga["status"] = "MANUAL_TAKEOVER"
|
||
saga["updatedAt"] = _now()
|
||
self._audit_saga(saga, "saga.manual_takeover", result="MANUAL_TAKEOVER",
|
||
extra={"reason": reason})
|
||
self._save()
|
||
return saga
|
||
|
||
def resume_all(self, *, auto_approve: bool = True) -> list[dict[str, Any]]:
|
||
"""恢复队列：重放所有未完成（非终态、非人工接管）的 saga。"""
|
||
out: list[dict[str, Any]] = []
|
||
for saga in list(self.store.data.get("sagas", [])):
|
||
if saga["status"] in _RECOVERABLE and not saga.get("manualTakeover"):
|
||
out.append(self.run(saga["id"], auto_approve=auto_approve))
|
||
return out
|
||
|
||
def list(self, limit: int | None = None) -> list[dict[str, Any]]:
|
||
sagas = list(reversed(self.store.data.get("sagas", [])))
|
||
return sagas if not limit else sagas[: int(limit)]
|
||
|
||
def get(self, saga_id: str) -> dict[str, Any] | None:
|
||
try:
|
||
return self._find(saga_id)
|
||
except SagaNotFound:
|
||
return None
|
||
|
||
# ---------------- 步骤执行 ----------------
|
||
def _execute_step(self, saga: dict[str, Any], step: dict[str, Any], *,
|
||
auto_approve: bool) -> str:
|
||
act = self.registry.get(step["action"])
|
||
if act is None:
|
||
step["status"] = "FAILED"
|
||
step["error"] = f"未注册动作 {step['action']}"
|
||
step["finishedAt"] = _now()
|
||
self._audit_step(saga, step, "saga.step.failed", result="FAILED")
|
||
self._save()
|
||
return "FAILED"
|
||
# 幂等重放：同 idem_key 已被（其他/先前）saga 成功执行 → 复用记录，不重复外部效果
|
||
idem_key = step.get("idemKey") or ""
|
||
if idem_key and step["status"] != "FAILED":
|
||
prior = self.store.data.setdefault("sagaIdem", {}).get(idem_key)
|
||
if prior and prior.get("step") == step["name"]:
|
||
step["result"] = _jsonable(copy.deepcopy(prior.get("result")))
|
||
step["status"] = "SUCCEEDED"
|
||
step["finishedAt"] = _now()
|
||
self._audit_step(saga, step, "saga.step.idempotent_replayed", result="SUCCEEDED")
|
||
self._save()
|
||
return "OK"
|
||
# 补偿性步骤执行前：自动建写前快照（回滚锚点）
|
||
if step.get("compensationAction") and not step.get("checkpointId"):
|
||
step["checkpointId"] = self._snapshot(saga, step)
|
||
max_attempts = int(step.get("maxRetries") or 0) + 1
|
||
for attempt in range(max_attempts):
|
||
step["status"] = "RUNNING"
|
||
step["attempts"] = int(step.get("attempts") or 0) + 1
|
||
step["startedAt"] = _now()
|
||
step["error"] = None
|
||
started = self.clock()
|
||
try:
|
||
result = act["fn"](self._ctx(saga, step))
|
||
elapsed = self.clock() - started
|
||
if float(step.get("timeoutSec") or 0) > 0 and elapsed > float(step["timeoutSec"]):
|
||
raise SagaStepTimeout(
|
||
f"步骤超时 {elapsed:.1f}s > {step['timeoutSec']}s")
|
||
step["result"] = _jsonable(result if isinstance(result, dict) else {"value": result})
|
||
step["status"] = "SUCCEEDED"
|
||
step["finishedAt"] = _now()
|
||
if idem_key:
|
||
self.store.data.setdefault("sagaIdem", {})[idem_key] = {
|
||
"sagaId": saga["id"], "step": step["name"],
|
||
"result": step["result"], "at": _now()}
|
||
self._audit_step(saga, step, "saga.step.succeeded", result="SUCCEEDED")
|
||
self._save()
|
||
return "OK"
|
||
except SagaStepTimeout as exc:
|
||
step["timeouts"] = int(step.get("timeouts") or 0) + 1
|
||
step["error"] = f"timeout: {exc}"
|
||
self._audit_step(saga, step, "saga.step.timeout", result="TIMEOUT")
|
||
except Exception as exc:
|
||
step["error"] = f"{type(exc).__name__}: {exc}"
|
||
self._audit_step(saga, step, "saga.step.failed", result="FAILED")
|
||
step["finishedAt"] = _now()
|
||
if attempt < max_attempts - 1:
|
||
delay = min(float(self.backoff_base) * (2 ** attempt), float(self.backoff_cap))
|
||
self._audit_step(saga, step, "saga.step.retry", result="RETRY",
|
||
extra={"delaySec": delay, "attempt": step["attempts"]})
|
||
self._save()
|
||
if delay > 0:
|
||
self.sleep_fn(delay)
|
||
continue
|
||
break
|
||
step["status"] = "FAILED"
|
||
step["finishedAt"] = _now()
|
||
self._audit_step(saga, step, "saga.step.failed", result="FAILED")
|
||
self._save()
|
||
return "FAILED"
|
||
|
||
# ---------------- 补偿链 ----------------
|
||
def _compensate(self, saga: dict[str, Any], *, reason: str) -> None:
|
||
saga["status"] = "COMPENSATING"
|
||
saga["compensationReason"] = reason
|
||
saga["updatedAt"] = _now()
|
||
self._audit_saga(saga, "saga.compensating", result="COMPENSATING",
|
||
extra={"reason": reason})
|
||
failed = False
|
||
for step in reversed(saga["steps"]):
|
||
if not self._compensatable(step):
|
||
continue
|
||
comp_action = step["compensationAction"]
|
||
comp = self.registry.get(comp_action)
|
||
if comp is None:
|
||
step["compensation"] = {"status": "FAILED",
|
||
"error": f"未注册补偿动作 {comp_action}", "at": _now()}
|
||
failed = True
|
||
self._audit_step(saga, step, "saga.step.compensation_failed", result="FAILED",
|
||
extra={"compensation": comp_action})
|
||
self._save()
|
||
break
|
||
try:
|
||
res = comp["fn"](self._ctx(saga, step))
|
||
step["compensation"] = {
|
||
"status": "SUCCEEDED",
|
||
"action": comp_action,
|
||
"result": _jsonable(res if isinstance(res, dict) else {"value": res}),
|
||
"at": _now(),
|
||
}
|
||
if step.get("idemKey"):
|
||
self.store.data.setdefault("sagaIdem", {}).pop(step["idemKey"], None)
|
||
self._audit_step(saga, step, "saga.step.compensated", result="SUCCEEDED",
|
||
extra={"compensation": comp_action})
|
||
self._save()
|
||
except Exception as exc:
|
||
step["compensation"] = {
|
||
"status": "FAILED", "action": comp_action,
|
||
"error": f"{type(exc).__name__}: {exc}", "at": _now(),
|
||
}
|
||
failed = True
|
||
self._audit_step(saga, step, "saga.step.compensation_failed", result="FAILED",
|
||
extra={"compensation": comp_action})
|
||
self._save()
|
||
break
|
||
if failed:
|
||
saga["status"] = "MANUAL_TAKEOVER"
|
||
saga["manualTakeover"] = True
|
||
saga["takeoverReason"] = f"compensation-failed: {reason}"
|
||
else:
|
||
saga["status"] = "COMPENSATED"
|
||
saga["updatedAt"] = _now()
|
||
self._audit_saga(saga, "saga.manual_takeover" if failed else "saga.compensated",
|
||
result=saga["status"],
|
||
extra={"reason": reason, "compensationReason": saga["compensationReason"]})
|
||
self._save()
|
||
|
||
@staticmethod
|
||
def _compensatable(step: dict[str, Any]) -> bool:
|
||
if not step.get("compensationAction"):
|
||
return False
|
||
if (step.get("compensation") or {}).get("status") == "SUCCEEDED":
|
||
return False # 已补偿
|
||
return step["status"] == "SUCCEEDED" or bool(step.get("checkpointId"))
|
||
|
||
# ---------------- 基础设施 ----------------
|
||
def _ctx(self, saga: dict[str, Any], step: dict[str, Any] | None) -> dict[str, Any]:
|
||
return {
|
||
"store": self.store,
|
||
"saga": saga,
|
||
"step": step,
|
||
"steps": {s["name"]: s for s in saga["steps"]},
|
||
"actor": self.actor,
|
||
"context": saga.get("context") or {},
|
||
"coordinator": self,
|
||
}
|
||
|
||
def _snapshot(self, saga: dict[str, Any], step: dict[str, Any]) -> str:
|
||
cps = self._checkpoint_store()
|
||
pair = cps.create(self.store.data,
|
||
label=f"saga:{saga['id']}:{step['name']} 前",
|
||
reason="auto:saga.pre-step",
|
||
conversation_note=f"Saga {saga['name']} 步骤 {step['name']} 执行前基线")
|
||
return str(pair["pairId"])
|
||
|
||
def _checkpoint_store(self):
|
||
cps = getattr(self.store, "checkpoints", None)
|
||
if cps is None:
|
||
from server.state.checkpoints import get_checkpoints
|
||
cps = get_checkpoints()
|
||
return cps
|
||
|
||
def _audit_saga(self, saga: dict[str, Any], action: str, *, result: str = "SUCCESS",
|
||
extra: dict[str, Any] | None = None) -> None:
|
||
from server.agent_core.audit import write_audit
|
||
write_audit(self.store.data, self.store.next_id, actor=self.actor, category="SAGA",
|
||
action=action, target={"type": "SAGA", "id": saga["id"], "name": saga["name"]},
|
||
power="P1",
|
||
rationale={"sagaId": saga["id"], "name": saga["name"],
|
||
"status": saga["status"], **(extra or {})},
|
||
result=result,
|
||
evidence_refs=list(saga.get("auditRefs") or [f"saga:{saga['id']}"]))
|
||
|
||
def _audit_step(self, saga: dict[str, Any], step: dict[str, Any], action: str, *,
|
||
result: str = "SUCCESS", extra: dict[str, Any] | None = None) -> None:
|
||
from server.agent_core.audit import write_audit
|
||
refs = list(saga.get("auditRefs") or [f"saga:{saga['id']}"])
|
||
step_ref = f"saga.step:{saga['id']}:{step['name']}"
|
||
if step_ref not in refs:
|
||
refs.append(step_ref)
|
||
saga["auditRefs"] = refs
|
||
write_audit(self.store.data, self.store.next_id, actor=self.actor, category="SAGA",
|
||
action=action,
|
||
target={"type": "SAGA_STEP", "sagaId": saga["id"], "step": step["name"]},
|
||
power="P1",
|
||
rationale={"sagaId": saga["id"], "step": step["name"], "action": step["action"],
|
||
"idemKey": step.get("idemKey"), "status": step["status"],
|
||
"attempts": step.get("attempts"),
|
||
"compensation": step.get("compensationAction"),
|
||
**(extra or {})},
|
||
result=result, evidence_refs=refs)
|
||
# ------------------------------------------------------------
|
||
# ⑥ WMS 缺料闭环 Saga 动作（动作注册表默认值）
|
||
# ------------------------------------------------------------
|
||
def _act_wms_consume_event(ctx: dict[str, Any]) -> dict[str, Any]:
|
||
"""动作：WMS 事件消费（eventId 幂等）。"""
|
||
from server.aps_domain import wms_events
|
||
event = ctx["context"].get("event") or {}
|
||
r = wms_events.consume_event(ctx["store"], event, actor=ctx["actor"])
|
||
if r.get("duplicate"):
|
||
return {**r, "replayed": True}
|
||
return r
|
||
|
||
|
||
def _act_wms_evaluate_impact(ctx: dict[str, Any]) -> dict[str, Any]:
|
||
"""动作：缺料影响半径（沙盒只读差分）。"""
|
||
from server.aps_domain import wms_events
|
||
event = ctx["context"].get("event") or {}
|
||
return wms_events.evaluate_shortage_impact(ctx["store"].data, event)
|
||
|
||
|
||
def _act_wms_stage_solution(ctx: dict[str, Any]) -> dict[str, Any]:
|
||
"""动作：生成 Explore 方案卡（P2 门禁卡；确认点在下一 gate 步骤）。"""
|
||
from server.aps_domain import wms_events
|
||
store = ctx["store"]
|
||
event = ctx["context"].get("event") or {}
|
||
session_id = (ctx["context"].get("sessionId")) or "wms"
|
||
return wms_events.stage_shortage_solution(store, event, session_id=session_id,
|
||
actor=ctx["actor"])
|
||
|
||
|
||
def _act_flex_reschedule(ctx: dict[str, Any]) -> dict[str, Any]:
|
||
"""执行一次经确认的 L4 重排，并只返回本次确认实际创建的精确版本。"""
|
||
from server.aps_domain.workflow import execute_confirmed
|
||
|
||
store = ctx["store"]
|
||
stage_res = (ctx["steps"].get("stage_solution") or {}).get("result") or {}
|
||
confirm_id = str(stage_res.get("confirmId") or "")
|
||
if not confirm_id:
|
||
raise SagaStepFailed("缺少方案卡 confirmId（stage_solution 未成功）")
|
||
|
||
before_versions = list(store.data.get("flexScheduleVersions") or [])
|
||
before_ids = {str(row.get("id")) for row in before_versions}
|
||
msg = execute_confirmed(store, confirm_id, approve=True, actor=ctx["actor"])
|
||
after_versions = list(store.data.get("flexScheduleVersions") or [])
|
||
new_versions = [row for row in after_versions if str(row.get("id")) not in before_ids]
|
||
if len(after_versions) <= len(before_versions) or len(new_versions) != 1:
|
||
raise SagaStepFailed(
|
||
"本次重排确认未创建唯一的新排产版本，拒绝沿用历史 DRAFT 版本："
|
||
f"confirmId={confirm_id}; message={msg}"
|
||
)
|
||
|
||
version = new_versions[0]
|
||
version_id = version.get("id")
|
||
event_id = str((ctx.get("context", {}).get("event") or {}).get("eventId") or "")
|
||
approval = next(
|
||
(
|
||
event
|
||
for event in reversed(store.data.get("auditEvents") or [])
|
||
if event.get("action") == "flex.reschedule.approve"
|
||
and str((event.get("rationale") or {}).get("confirmId") or "") == confirm_id
|
||
and str((event.get("target") or {}).get("id")) == str(version_id)
|
||
),
|
||
None,
|
||
)
|
||
expected_event_ref = f"wms-event:{event_id}" if event_id else ""
|
||
if approval is None or (
|
||
expected_event_ref
|
||
and expected_event_ref not in (approval.get("evidenceRefs") or [])
|
||
):
|
||
raise SagaStepFailed(
|
||
"新排产版本缺少与当前确认卡/WMS 事件一致的审计证据，拒绝进入发布门禁："
|
||
f"confirmId={confirm_id}; versionId={version_id}; eventId={event_id or '-'}"
|
||
)
|
||
|
||
return {
|
||
"message": msg,
|
||
"versionId": version_id,
|
||
"versionNo": version.get("versionNo"),
|
||
"confirmId": confirm_id,
|
||
"eventId": event_id or None,
|
||
"approvalAuditId": approval.get("id"),
|
||
}
|
||
|
||
def _act_schedule_publish_stage(ctx: dict[str, Any]) -> dict[str, Any]:
|
||
"""Stage an exact flex-version P2 publication card after rescheduling."""
|
||
|
||
from server.aps_domain.workflow import stage_schedule_publish
|
||
|
||
store = ctx["store"]
|
||
reschedule_result = (ctx["steps"].get("reschedule") or {}).get("result") or {}
|
||
version_id = reschedule_result.get("versionId")
|
||
if not version_id:
|
||
raise SagaStepFailed("缺少重排版本 versionId（reschedule 未成功）")
|
||
session_id = ctx["context"].get("sessionId") or "wms"
|
||
reply = stage_schedule_publish(
|
||
store,
|
||
session_id=session_id,
|
||
actor=ctx["actor"],
|
||
track="flex",
|
||
version_id=int(version_id),
|
||
)
|
||
block = next(
|
||
(
|
||
item
|
||
for item in reply.blocks or []
|
||
if getattr(item, "type", None) == "confirm-card"
|
||
),
|
||
None,
|
||
)
|
||
confirm_id = str((getattr(block, "props", None) or {}).get("confirmId") or "")
|
||
if not confirm_id:
|
||
raise SagaStepFailed(f"柔性版本发布暂存失败: {reply.text}")
|
||
return {
|
||
"staged": True,
|
||
"confirmId": confirm_id,
|
||
"versionId": int(version_id),
|
||
"message": reply.text,
|
||
}
|
||
|
||
|
||
def _act_schedule_publish(ctx: dict[str, Any]) -> dict[str, Any]:
|
||
"""Approve the independent P2 publication gate without touching MES."""
|
||
|
||
from server.aps_domain.workflow import execute_confirmed
|
||
|
||
store = ctx["store"]
|
||
stage_result = (ctx["steps"].get("stage_publish") or {}).get("result") or {}
|
||
confirm_id = str(stage_result.get("confirmId") or "")
|
||
version_id = stage_result.get("versionId")
|
||
if not confirm_id or not version_id:
|
||
raise SagaStepFailed("缺少发布确认卡或目标版本（stage_publish 未成功）")
|
||
message = execute_confirmed(
|
||
store,
|
||
confirm_id,
|
||
approve=True,
|
||
actor=ctx["actor"],
|
||
)
|
||
version = next(
|
||
(
|
||
row
|
||
for row in store.data.get("flexScheduleVersions") or []
|
||
if row.get("id") == int(version_id)
|
||
),
|
||
None,
|
||
)
|
||
if version is None or str(version.get("status") or "").upper() != "PUBLISHED":
|
||
raise SagaStepFailed(f"柔性版本发布失败: {message}")
|
||
return {
|
||
"published": True,
|
||
"confirmId": confirm_id,
|
||
"versionId": int(version_id),
|
||
"message": message,
|
||
}
|
||
|
||
|
||
def _act_mes_dispatch_stage(ctx: dict[str, Any]) -> dict[str, Any]:
|
||
"""Stage P3 MES dispatch only for the exact published reschedule version."""
|
||
|
||
from server.aps_domain import wms_events
|
||
|
||
store = ctx["store"]
|
||
event = ctx["context"].get("event") or {}
|
||
event_id = str(event.get("eventId") or "")
|
||
session_id = ctx["context"].get("sessionId") or "wms"
|
||
reschedule_result = (ctx["steps"].get("reschedule") or {}).get("result") or {}
|
||
expected_version_id = reschedule_result.get("versionId")
|
||
versions = store.data.get("flexScheduleVersions") or []
|
||
latest = versions[-1] if versions else None
|
||
if (
|
||
latest is None
|
||
or latest.get("id") != expected_version_id
|
||
or str(latest.get("status") or "").upper() != "PUBLISHED"
|
||
):
|
||
raise SagaStepFailed("MES 下发暂存失败: 重排目标版本不是当前已发布执行基准")
|
||
result = wms_events.stage_dispatch(
|
||
store,
|
||
event_id,
|
||
session_id=session_id,
|
||
actor=ctx["actor"],
|
||
)
|
||
if not result.get("staged"):
|
||
raise SagaStepFailed(f"MES 下发暂存失败: {result.get('message')}")
|
||
staged_version_id = (result.get("validation") or {}).get("versionId")
|
||
if int(staged_version_id or 0) != int(expected_version_id or 0):
|
||
raise SagaStepFailed("MES dispatch staging failed: confirmation card is not bound to the rescheduled version")
|
||
return result
|
||
|
||
|
||
def _act_mes_dispatch(ctx: dict[str, Any]) -> dict[str, Any]:
|
||
"""动作（gate）：P3 双人确认 → MES 下发 → 回执镜像写回审计。
|
||
|
||
失败时保留部分结果（已创建外部工单/执行凭据）供补偿链撤销。
|
||
"""
|
||
from server.aps_domain import wms_events
|
||
store = ctx["store"]
|
||
event = ctx["context"].get("event") or {}
|
||
event_id = str(event.get("eventId") or "")
|
||
stage_res = (ctx["steps"].get("stage_dispatch") or {}).get("result") or {}
|
||
block = stage_res.get("block") or {}
|
||
confirm_id = str((block.get("props") or {}).get("confirmId")
|
||
or stage_res.get("confirmId") or "")
|
||
if not confirm_id:
|
||
raise SagaStepFailed("缺少 MES 下发确认卡 confirmId（stage_dispatch 未成功）")
|
||
step = ctx["step"]
|
||
prior = (step.get("result") or {}).get("executionGrant") if isinstance(step.get("result"), dict) else None
|
||
grant = prior or _p3_double_approve(ctx["coordinator"], confirm_id)
|
||
reschedule_res = (ctx["steps"].get("reschedule") or {}).get("result") or {}
|
||
version_id = reschedule_res.get("versionId")
|
||
validation = stage_res.get("validation") or {}
|
||
dispatch_evidence = [f"schedule-version:{int(version_id or 0)}"]
|
||
if validation.get("evidenceRef"):
|
||
dispatch_evidence.append(str(validation["evidenceRef"]))
|
||
cps = ctx["coordinator"]._checkpoint_store()
|
||
try:
|
||
res = wms_events.execute_dispatch_receipt(
|
||
store, event_id=event_id, confirm_id=confirm_id, execution_grant=grant,
|
||
version_id=int(version_id or 0),
|
||
before_snapshot=str(step.get("checkpointId") or ""),
|
||
checkpoint_store=cps, evidence_refs=dispatch_evidence,
|
||
actor=ctx["actor"])
|
||
except Exception as exc:
|
||
# 保留部分执行信息（已创建的外部工单/凭据）供补偿链撤销
|
||
step["result"] = _jsonable({
|
||
"partial": True, "error": str(exc), "executionGrant": grant,
|
||
"created": _external_wo_ids(store, version_id),
|
||
})
|
||
ctx["coordinator"]._save()
|
||
raise
|
||
return {**res, "executionGrant": grant}
|
||
|
||
|
||
def _p3_double_approve(coordinator: SagaCoordinator, confirm_id: str) -> str:
|
||
"""P3 双人职责分离确认：当前身份第一重 + approver-2 第二重 → executionGrant。"""
|
||
from server.agent_core import harness
|
||
from server.auth.context import IdentityContext, bind_identity, reset_identity
|
||
first = harness.take_confirmation(confirm_id, approve=True)
|
||
if first is None:
|
||
raise SagaStepFailed("MES P3 确认卡不可用（可能已处理）")
|
||
if first.get("needsSecondConfirm"):
|
||
approver = coordinator.second_approver or IdentityContext(
|
||
2002, "approver-2", "Approver 2", "platform", roles=("planner",))
|
||
token = bind_identity(approver)
|
||
try:
|
||
second = harness.take_confirmation(confirm_id, approve=True)
|
||
finally:
|
||
reset_identity(token)
|
||
if second is None:
|
||
raise SagaStepFailed("MES P3 二次确认失败")
|
||
grant = second.get("executionGrant")
|
||
else:
|
||
grant = first.get("executionGrant")
|
||
if not grant:
|
||
raise SagaStepFailed("MES P3 未获得执行凭据")
|
||
return str(grant)
|
||
|
||
|
||
def _external_wo_ids(store, version_id) -> list[str]:
|
||
"""从 mesLinks 反查某版本已下发的外部工单 ID（补偿撤销用）。"""
|
||
out: list[str] = []
|
||
for link in store.data.get("mesLinks", []):
|
||
if link.get("kind") != "dispatch":
|
||
continue
|
||
if version_id is not None and link.get("versionId") != version_id:
|
||
continue
|
||
if link.get("externalWoId") and link["externalWoId"] not in out:
|
||
out.append(link["externalWoId"])
|
||
return out
|
||
|
||
|
||
# ---------------- 补偿动作 ----------------
|
||
def _restore_from_checkpoint(ctx: dict[str, Any]) -> dict[str, Any]:
|
||
"""按步骤写前快照恢复世界（保留审计链与 saga 日志，回滚业务表到步骤前）。"""
|
||
store = ctx["store"]
|
||
step = ctx["step"]
|
||
pair_id = step.get("checkpointId")
|
||
if not pair_id:
|
||
raise SagaCompensationError(f"步骤 {step['name']} 缺少写前快照，无法回滚")
|
||
cps = ctx["coordinator"]._checkpoint_store()
|
||
pair = cps.get(pair_id)
|
||
if pair is None:
|
||
raise SagaCompensationError(f"写前快照 {pair_id} 不存在，无法回滚")
|
||
world = copy.deepcopy(pair["world"])
|
||
current = store.data
|
||
for key in ("auditEvents", "sagas", "sagaIdem", "mesCancellations", "wmsReceiptVoids"):
|
||
if key in current:
|
||
world[key] = current[key]
|
||
_replace_world(store, world)
|
||
return {"pairId": pair_id, "label": pair.get("label")}
|
||
|
||
|
||
def _replace_world(store, world: dict[str, Any]) -> None:
|
||
if hasattr(store, "restore"):
|
||
store.restore(world)
|
||
else:
|
||
store.data.clear()
|
||
store.data.update(world)
|
||
|
||
|
||
def _comp_flex_reschedule_rollback(ctx: dict[str, Any]) -> dict[str, Any]:
|
||
"""补偿：重排失败 → 回滚到步骤执行前版本（写前快照恢复）。"""
|
||
restored = _restore_from_checkpoint(ctx)
|
||
return {"worldRestore": restored, "message": "重排已回滚到步骤执行前版本"}
|
||
|
||
|
||
def _comp_mes_cancel_dispatch(ctx: dict[str, Any]) -> dict[str, Any]:
|
||
"""补偿：MES 下发失败 → 撤销外部工单 + 回执 VOIDED + 世界回滚到下发前。"""
|
||
from server.aps_domain import wms_events
|
||
from server.aps_domain.mes import cancel_dispatch
|
||
store = ctx["store"]
|
||
step = ctx["step"]
|
||
result = step.get("result") or {}
|
||
external_ids = list(result.get("created") or [])
|
||
version_id = (ctx["steps"].get("reschedule") or {}).get("result", {}).get("versionId")
|
||
if not external_ids:
|
||
external_ids = _external_wo_ids(store, version_id)
|
||
cancel_res = cancel_dispatch(store, external_wo_ids=external_ids,
|
||
idem_key=step.get("idemKey"), actor=ctx["actor"],
|
||
reason="saga-compensation")
|
||
rollback: dict[str, Any] = {}
|
||
try:
|
||
rollback = wms_events.rollback_receipt(
|
||
store, event_id=(ctx["context"].get("event") or {}).get("eventId"),
|
||
actor=ctx["actor"], reason="saga-compensation")
|
||
except Exception as exc:
|
||
rollback = {"error": f"{type(exc).__name__}: {exc}"}
|
||
restored = _restore_from_checkpoint(ctx)
|
||
return {"cancel": cancel_res, "receiptRollback": rollback, "worldRestore": restored}
|
||
|
||
|
||
# ------------------------------------------------------------
|
||
# 默认动作注册表（动作 → 补偿动作）
|
||
# ------------------------------------------------------------
|
||
def default_registry() -> dict[str, dict[str, Any]]:
|
||
return {
|
||
"wms.consume_event": {"fn": _act_wms_consume_event, "compensation": None},
|
||
"wms.evaluate_impact": {"fn": _act_wms_evaluate_impact, "compensation": None},
|
||
"wms.stage_solution": {"fn": _act_wms_stage_solution, "compensation": None},
|
||
"flex.reschedule": {
|
||
"fn": _act_flex_reschedule,
|
||
"compensation": "flex.reschedule.rollback",
|
||
},
|
||
"flex.reschedule.rollback": {"fn": _comp_flex_reschedule_rollback},
|
||
"schedule.publish.stage": {"fn": _act_schedule_publish_stage, "compensation": None},
|
||
"schedule.publish": {"fn": _act_schedule_publish, "compensation": None},
|
||
"mes.dispatch.stage": {"fn": _act_mes_dispatch_stage, "compensation": None},
|
||
"mes.dispatch": {"fn": _act_mes_dispatch, "compensation": "mes.cancel_dispatch"},
|
||
"mes.cancel_dispatch": {"fn": _comp_mes_cancel_dispatch},
|
||
}
|
||
|
||
|
||
# ------------------------------------------------------------
|
||
# 便捷入口
|
||
# ------------------------------------------------------------
|
||
def get_coordinator(store, *, actor: str = "saga",
|
||
registry: dict[str, dict[str, Any]] | None = None,
|
||
**kwargs) -> SagaCoordinator:
|
||
"""按世界状态构造（或复用）Saga 编排器。"""
|
||
return SagaCoordinator(store, actor=actor, registry=registry, **kwargs)
|
||
|
||
|
||
def build_wms_shortage_saga(coordinator: SagaCoordinator, event: dict[str, Any], *,
|
||
session_id: str = "wms", actor: str = "saga",
|
||
timeout_sec: float = 30.0, max_retries: int = 2) -> dict[str, Any]:
|
||
"""Build the governed WMS shortage closure Saga.
|
||
|
||
consume -> impact -> solution card -> P2 reschedule -> P2 publish ->
|
||
P3 MES dispatch -> receipt. Publication and dispatch remain separate gates.
|
||
"""
|
||
|
||
event_id = str(event.get("eventId") or "")
|
||
if not event_id:
|
||
raise ValueError("WMS 缺料 Saga 需要事件 eventId")
|
||
steps = [
|
||
SagaStep(
|
||
"consume_event",
|
||
"wms.consume_event",
|
||
idem_key=f"wms-event:{event_id}",
|
||
timeout_sec=timeout_sec,
|
||
max_retries=max_retries,
|
||
),
|
||
SagaStep(
|
||
"evaluate_impact",
|
||
"wms.evaluate_impact",
|
||
idem_key=f"impact:{event_id}",
|
||
timeout_sec=timeout_sec,
|
||
max_retries=max_retries,
|
||
),
|
||
SagaStep(
|
||
"stage_solution",
|
||
"wms.stage_solution",
|
||
idem_key=f"stage:{event_id}",
|
||
timeout_sec=timeout_sec,
|
||
max_retries=max_retries,
|
||
),
|
||
SagaStep(
|
||
"reschedule",
|
||
"flex.reschedule",
|
||
idem_key=f"reschedule:{event_id}",
|
||
timeout_sec=timeout_sec,
|
||
max_retries=max_retries,
|
||
compensation_action="flex.reschedule.rollback",
|
||
gate=True,
|
||
),
|
||
SagaStep(
|
||
"stage_publish",
|
||
"schedule.publish.stage",
|
||
idem_key=f"publish-stage:{event_id}",
|
||
timeout_sec=timeout_sec,
|
||
max_retries=max_retries,
|
||
),
|
||
SagaStep(
|
||
"publish",
|
||
"schedule.publish",
|
||
idem_key=f"publish:{event_id}",
|
||
timeout_sec=timeout_sec,
|
||
max_retries=max_retries,
|
||
gate=True,
|
||
),
|
||
SagaStep(
|
||
"stage_dispatch",
|
||
"mes.dispatch.stage",
|
||
idem_key=f"dispatch-stage:{event_id}",
|
||
timeout_sec=timeout_sec,
|
||
max_retries=max_retries,
|
||
),
|
||
SagaStep(
|
||
"dispatch",
|
||
"mes.dispatch",
|
||
idem_key=f"dispatch:{event_id}",
|
||
timeout_sec=timeout_sec,
|
||
max_retries=max_retries,
|
||
compensation_action="mes.cancel_dispatch",
|
||
gate=True,
|
||
),
|
||
]
|
||
return coordinator.create_saga(
|
||
"wms-shortage-closure",
|
||
steps,
|
||
context={"event": event, "sessionId": session_id},
|
||
dedupe_key=event_id,
|
||
) |