aps-agent/server/engines/solver_process.py

728 lines
26 KiB
Python
Raw Normal View History

"""Fail-closed process boundary for CP-SAT line assignment.
This module is intentionally stdlib-only so callers can construct and supervise the
solver child before importing OR-Tools or any of its native dependencies.
"""
from __future__ import annotations
import dataclasses
import hashlib
import json
import math
import os
import signal
import subprocess
import sys
import tempfile
from collections.abc import Mapping, Sequence
from datetime import date, datetime, time
from enum import Enum
from pathlib import Path
from typing import Any, cast
PROTOCOL_VERSION = "aps.solver-process.v1"
SUCCESS_MARKER = "APS_SOLVER_SUCCESS_V1"
ERROR_MARKER = "APS_SOLVER_ERROR_V1"
ENV_CHILD_COMMAND = "APS_SOLVER_CHILD_COMMAND_JSON"
ENV_CHILD_EXECUTABLE = "APS_SOLVER_CHILD_EXECUTABLE"
ENV_CHILD_ARGUMENTS = "APS_SOLVER_CHILD_ARGUMENTS_JSON"
_DEFAULT_STARTUP_GRACE_SECONDS = 12.0
_MAX_RESPONSE_BYTES = 64 * 1024 * 1024
_CAPTURE_LIMIT = 8_000
_FATAL_MARKERS = (
"windows fatal exception",
"fatal python error",
"0xc0000139",
"winerror 127",
"segmentation fault",
"access violation",
"terminate called",
"check failed",
"check failure",
)
class SolverProcessError(RuntimeError):
"""Structured, stable failure raised by the solver process boundary."""
def __init__(self, code: str, message: str, details: Mapping[str, Any] | None = None) -> None:
self.code = str(code)
self.message = str(message)
self.details = dict(details or {})
super().__init__(f"{self.code}: {self.message}")
def as_dict(self) -> dict[str, Any]:
return {"code": self.code, "message": self.message, "details": self.details}
def _json_value(value: Any) -> Any:
"""Return a detached, deterministic JSON value or fail closed."""
if hasattr(value, "model_dump"):
value = value.model_dump(mode="json")
elif dataclasses.is_dataclass(value) and not isinstance(value, type):
value = dataclasses.asdict(cast(Any, value))
elif isinstance(value, Enum):
value = value.value
elif isinstance(value, (datetime, date, time)):
return value.isoformat()
elif isinstance(value, Path):
return str(value)
if value is None or isinstance(value, (str, bool, int)):
return value
if isinstance(value, float):
if not math.isfinite(value):
raise SolverProcessError(
"SOLVER_REQUEST_INVALID",
"求解请求包含非有限浮点数",
{"value": repr(value)},
)
return value
if isinstance(value, Mapping):
return {
str(key): _json_value(item)
for key, item in sorted(value.items(), key=lambda pair: str(pair[0]))
}
if isinstance(value, (list, tuple)):
return [_json_value(item) for item in value]
if isinstance(value, (set, frozenset)):
normalized = [_json_value(item) for item in value]
return sorted(normalized, key=_canonical_json)
raise SolverProcessError(
"SOLVER_REQUEST_INVALID",
"求解请求包含不可序列化对象",
{"type": type(value).__name__},
)
def _canonical_json(value: Any) -> str:
return json.dumps(
value,
ensure_ascii=False,
allow_nan=False,
sort_keys=True,
separators=(",", ":"),
)
def _request_digest(body: Mapping[str, Any]) -> str:
return hashlib.sha256(_canonical_json(body).encode("utf-8")).hexdigest()
def _parse_string_list(raw: str, *, variable: str) -> list[str]:
try:
value = json.loads(raw)
except json.JSONDecodeError as exc:
raise SolverProcessError(
"SOLVER_COMMAND_INVALID",
f"{variable} 不是合法 JSON",
{"error": str(exc)},
) from exc
if not isinstance(value, list) or not value or not all(
isinstance(item, str) and item for item in value
):
raise SolverProcessError(
"SOLVER_COMMAND_INVALID",
f"{variable} 必须是非空字符串数组",
)
return list(value)
def _child_command() -> tuple[list[str], str]:
raw_command = os.environ.get(ENV_CHILD_COMMAND)
if raw_command:
return _parse_string_list(raw_command, variable=ENV_CHILD_COMMAND), "command-override"
frozen_executable = os.environ.get(ENV_CHILD_EXECUTABLE)
if frozen_executable:
executable = Path(frozen_executable).expanduser()
if not executable.is_absolute() or not executable.exists():
raise SolverProcessError(
"SOLVER_COMMAND_INVALID",
f"{ENV_CHILD_EXECUTABLE} 必须指向存在的绝对路径",
{"executable": str(executable)},
)
extra_args: list[str] = []
raw_args = os.environ.get(ENV_CHILD_ARGUMENTS)
if raw_args:
extra_args = _parse_string_list(raw_args, variable=ENV_CHILD_ARGUMENTS)
return [str(executable), "--solver-child", *extra_args], "frozen-executable"
if getattr(sys, "frozen", False):
executable = Path(sys.executable).resolve()
if not executable.is_file():
raise SolverProcessError(
"SOLVER_COMMAND_INVALID",
"冻结 Sidecar 可执行文件不存在",
{"executable": str(executable)},
)
return [str(executable), "--solver-child"], "frozen-self"
worker = Path(__file__).with_name("solver_worker.py").resolve()
if not worker.is_file():
raise SolverProcessError(
"SOLVER_COMMAND_INVALID",
"求解子进程 worker 不存在",
{"worker": str(worker)},
)
executable = Path(sys.executable).resolve()
if not executable.is_file():
raise SolverProcessError(
"SOLVER_COMMAND_INVALID",
"当前 Python 可执行文件不存在",
{"executable": str(executable)},
)
return [
str(executable),
"-I",
"-B",
"-X",
"utf8",
"-X",
"faulthandler",
str(worker),
], "isolated-python"
def _clean_child_path(raw_path: str) -> str:
"""Remove Conda DLL search roots from the supervised child environment."""
kept: list[str] = []
for segment in raw_path.split(os.pathsep):
lowered = segment.casefold().replace("/", "\\")
if any(token in lowered for token in ("anaconda", "miniconda", "\\conda\\", "\\condabin")):
continue
if segment:
kept.append(segment)
return os.pathsep.join(kept)
def _child_environment() -> dict[str, str]:
env = dict(os.environ)
env.pop("PYTHONHOME", None)
env.pop("PYTHONPATH", None)
for key in list(env):
if key.upper().startswith("CONDA_"):
env.pop(key, None)
env["PYTHONNOUSERSITE"] = "1"
env["PYTHONUTF8"] = "1"
env["PYTHONIOENCODING"] = "utf-8"
env["APS_SOLVER_CHILD_PROTOCOL"] = PROTOCOL_VERSION
env["PATH"] = _clean_child_path(env.get("PATH", ""))
return env
def _is_windows() -> bool:
return os.name == "nt"
def _creation_options() -> tuple[int, bool]:
if _is_windows():
flags = int(getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0))
flags |= int(getattr(subprocess, "CREATE_NO_WINDOW", 0))
return flags, False
return 0, True
def _terminate_process_tree(process: subprocess.Popen[str]) -> None:
"""Best-effort, bounded process-tree termination for timeout handling."""
if process.poll() is not None:
return
if _is_windows():
flags = int(getattr(subprocess, "CREATE_NO_WINDOW", 0))
try:
subprocess.run(
["taskkill", "/PID", str(process.pid), "/T", "/F"],
check=False,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
timeout=5,
creationflags=flags,
)
except (OSError, subprocess.SubprocessError):
pass
else:
try:
getpgid = os.getpgid # type: ignore[attr-defined]
killpg = os.killpg # type: ignore[attr-defined]
killpg(getpgid(process.pid), getattr(signal, "SIGKILL", signal.SIGTERM))
except (OSError, ProcessLookupError):
pass
if process.poll() is None:
try:
process.kill()
except OSError:
pass
try:
process.wait(timeout=5)
except (OSError, subprocess.SubprocessError):
pass
def _captured(text: str | None) -> str:
value = text or ""
if len(value) <= _CAPTURE_LIMIT:
return value
return value[:_CAPTURE_LIMIT] + "\n...[truncated]"
def _fatal_marker(stdout: str, stderr: str) -> str | None:
haystack = f"{stdout}\n{stderr}".casefold()
return next((marker for marker in _FATAL_MARKERS if marker in haystack), None)
def _effective_timeout(params: Mapping[str, Any], explicit: float | None) -> float:
if explicit is not None:
try:
value = float(explicit)
except (TypeError, ValueError) as exc:
raise SolverProcessError(
"SOLVER_REQUEST_INVALID", "timeout_seconds 必须为正数"
) from exc
if not math.isfinite(value) or value <= 0:
raise SolverProcessError("SOLVER_REQUEST_INVALID", "timeout_seconds 必须为正数")
return value
raw_limit = params.get("timeLimitSeconds")
try:
solver_limit = 8.0 if raw_limit is None else max(0.5, float(raw_limit))
except (TypeError, ValueError) as exc:
raise SolverProcessError(
"SOLVER_REQUEST_INVALID", "timeLimitSeconds 必须为数值"
) from exc
return max(15.0, solver_limit + _DEFAULT_STARTUP_GRACE_SECONDS)
def _response_object(stdout: str) -> dict[str, Any]:
if len(stdout.encode("utf-8", errors="replace")) > _MAX_RESPONSE_BYTES:
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID",
"求解子进程响应超过大小上限",
{"limitBytes": _MAX_RESPONSE_BYTES},
)
if not stdout.strip():
raise SolverProcessError("SOLVER_RESPONSE_INVALID", "求解子进程没有返回响应")
try:
payload = json.loads(stdout)
except json.JSONDecodeError as exc:
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID",
"求解子进程返回非法或截断 JSON",
{"error": str(exc), "stdout": _captured(stdout)},
) from exc
if not isinstance(payload, dict):
raise SolverProcessError("SOLVER_RESPONSE_INVALID", "求解子进程响应必须是 JSON 对象")
return payload
def _raise_worker_error(payload: Mapping[str, Any]) -> None:
error = payload.get("error")
if not isinstance(error, Mapping):
raise SolverProcessError("SOLVER_RESPONSE_INVALID", "求解子进程错误响应缺少 error")
code = error.get("code")
message = error.get("message")
details = error.get("details")
if not isinstance(code, str) or not code or not isinstance(message, str) or not message:
raise SolverProcessError("SOLVER_RESPONSE_INVALID", "求解子进程错误响应结构非法")
if details is not None and not isinstance(details, Mapping):
raise SolverProcessError("SOLVER_RESPONSE_INVALID", "求解子进程错误 details 必须是对象")
raise SolverProcessError(code, message, details if isinstance(details, Mapping) else None)
def _validate_response_digest(payload: Mapping[str, Any]) -> str:
provided = payload.get("responseDigest")
if not isinstance(provided, str) or len(provided) != 64:
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID",
"求解子进程响应缺少有效 responseDigest",
)
try:
int(provided, 16)
except ValueError as exc:
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID",
"求解子进程 responseDigest 不是十六进制摘要",
) from exc
unsigned = dict(payload)
unsigned.pop("responseDigest", None)
try:
normalized = _json_value(unsigned)
actual = hashlib.sha256(
_canonical_json(normalized).encode("utf-8")
).hexdigest()
except SolverProcessError as exc:
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID",
"求解子进程响应包含非法摘要载荷",
{"cause": exc.code},
) from exc
if actual != provided.lower():
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID",
"求解子进程响应摘要不匹配",
{"expected": actual, "actual": provided},
)
return actual
def _entry_signature(entry: Mapping[str, Any]) -> str:
try:
detached = _json_value(entry)
except SolverProcessError as exc:
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID",
"排产条目包含非法响应值",
{"cause": exc.code},
) from exc
if not isinstance(detached, dict):
raise SolverProcessError("SOLVER_RESPONSE_INVALID", "排产条目必须是对象")
forced_line_id = detached.pop("forcedLineId", None)
if forced_line_id is not None and (
isinstance(forced_line_id, bool)
or not isinstance(forced_line_id, int)
or forced_line_id < 0
):
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID", "forcedLineId 必须是非负整数"
)
return _canonical_json(detached)
def _validate_result_semantics(
ordered: Sequence[Mapping[str, Any]],
solver_meta: Mapping[str, Any],
*,
expected_entries: Sequence[Mapping[str, Any]],
pipeline_label: str,
) -> None:
expected_signatures = sorted(_entry_signature(entry) for entry in expected_entries)
actual_signatures = sorted(_entry_signature(entry) for entry in ordered)
if actual_signatures != expected_signatures:
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID",
"求解子进程返回条目不是请求条目的完整排列",
{
"expectedCount": len(expected_signatures),
"actualCount": len(actual_signatures),
"expectedDigest": hashlib.sha256(
_canonical_json(expected_signatures).encode("utf-8")
).hexdigest(),
"actualDigest": hashlib.sha256(
_canonical_json(actual_signatures).encode("utf-8")
).hexdigest(),
},
)
status = solver_meta.get("status")
allowed_statuses = {
"TRIVIAL",
"OPTIMAL",
"FEASIBLE",
"INFEASIBLE",
"MODEL_INVALID",
"UNKNOWN",
}
if status not in allowed_statuses:
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID",
"求解子进程返回未知 solve status",
{"status": status},
)
if solver_meta.get("pipeline") != pipeline_label:
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID",
"求解子进程 pipeline 与请求不一致",
{"expected": pipeline_label, "actual": solver_meta.get("pipeline")},
)
if expected_entries and status == "TRIVIAL":
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID", "非空排产请求不能返回 TRIVIAL"
)
if not expected_entries and status != "TRIVIAL":
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID", "空排产请求必须返回 TRIVIAL"
)
if status not in {"OPTIMAL", "FEASIBLE"}:
return
for field in ("objective", "gap"):
value = solver_meta.get(field)
if isinstance(value, bool) or not isinstance(value, (int, float)):
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID",
f"可行解缺少数值 {field}",
{"status": status, "field": field},
)
if not math.isfinite(float(value)):
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID", f"可行解 {field} 不是有限数"
)
slots = solver_meta.get("operationSlots")
if not isinstance(slots, list) or not all(isinstance(slot, dict) for slot in slots):
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID", "可行解缺少 operationSlots"
)
expected_indices = set(range(len(expected_entries)))
covered_indices: set[int] = set()
for slot in slots:
if slot.get("isFrozen") is True:
continue
order_index = slot.get("orderIndex")
if (
isinstance(order_index, bool)
or not isinstance(order_index, int)
or order_index not in expected_indices
):
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID",
"operationSlots 包含越界 orderIndex",
{"orderIndex": order_index, "entryCount": len(expected_entries)},
)
expected_entry = expected_entries[order_index]
expected_so = expected_entry.get("so")
if isinstance(expected_so, Mapping) and slot.get("orderNo") != expected_so.get("orderNo"):
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID",
"operationSlot.orderNo 与请求条目不一致",
{"orderIndex": order_index},
)
expected_item = expected_entry.get("item")
if isinstance(expected_item, Mapping) and slot.get("productId") != expected_item.get("productId"):
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID",
"operationSlot.productId 与请求条目不一致",
{"orderIndex": order_index},
)
covered_indices.add(order_index)
missing_indices = sorted(expected_indices - covered_indices)
if missing_indices:
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID",
"可行解 operationSlots 未覆盖全部排产条目",
{"missingOrderIndices": missing_indices[:50]},
)
def _validate_response(
payload: Mapping[str, Any],
*,
request_id: str,
expected_entries: Sequence[Mapping[str, Any]],
pipeline_label: str,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
if payload.get("protocolVersion") != PROTOCOL_VERSION:
raise SolverProcessError(
"SOLVER_PROTOCOL_MISMATCH",
"求解子进程协议版本不一致",
{"expected": PROTOCOL_VERSION, "actual": payload.get("protocolVersion")},
)
if payload.get("requestId") != request_id:
raise SolverProcessError(
"SOLVER_PROTOCOL_MISMATCH",
"求解子进程响应 requestId 不匹配",
{"expected": request_id, "actual": payload.get("requestId")},
)
marker = payload.get("marker")
if marker == ERROR_MARKER:
_raise_worker_error(payload)
if marker != SUCCESS_MARKER or payload.get("ok") is not True:
raise SolverProcessError(
"SOLVER_RESPONSE_INVALID",
"求解子进程响应缺少 success marker",
{"marker": marker},
)
runtime = payload.get("runtimeIdentity")
if not isinstance(runtime, Mapping):
raise SolverProcessError("SOLVER_RESPONSE_INVALID", "响应缺少 runtimeIdentity")
if runtime.get("safe") is not True or runtime.get("implementation") != "CPython":
raise SolverProcessError(
"SOLVER_RUNTIME_UNSAFE",
"求解子进程运行时身份不安全",
{"runtimeIdentity": dict(runtime)},
)
response_digest = _validate_response_digest(payload)
result = payload.get("result")
if not isinstance(result, Mapping):
raise SolverProcessError("SOLVER_RESPONSE_INVALID", "响应缺少 result 对象")
ordered = result.get("entries")
solver_meta = result.get("solverMeta")
if not isinstance(ordered, list) or not all(isinstance(item, dict) for item in ordered):
raise SolverProcessError("SOLVER_RESPONSE_INVALID", "result.entries 必须是对象数组")
if not isinstance(solver_meta, dict):
raise SolverProcessError("SOLVER_RESPONSE_INVALID", "result.solverMeta 必须是对象")
detached_entries = _json_value(ordered)
detached_meta = _json_value(solver_meta)
detached_runtime = _json_value(runtime)
if (
not isinstance(detached_entries, list)
or not isinstance(detached_meta, dict)
or not isinstance(detached_runtime, dict)
):
raise SolverProcessError("SOLVER_RESPONSE_INVALID", "求解响应无法安全复制")
_validate_result_semantics(
detached_entries,
detached_meta,
expected_entries=expected_entries,
pipeline_label=pipeline_label,
)
detached_meta["solverProcess"] = {
"protocolVersion": PROTOCOL_VERSION,
"requestId": request_id,
"responseDigest": response_digest,
"runtimeIdentity": detached_runtime,
}
return detached_entries, detached_meta
def run_cp_assignment(
world: Mapping[str, Any],
entries: Sequence[Mapping[str, Any]],
params: Any,
*,
warm_start: Sequence[Mapping[str, Any]] | None = None,
pipeline_label: str,
timeout_seconds: float | None = None,
) -> tuple[list[dict[str, Any]], dict[str, Any]]:
"""Run ``optimize_line_assignment`` in a supervised isolated child process.
The caller's ``world`` and ``entries`` are serialized into detached canonical JSON.
The worker cannot mutate the parent objects and has no persistence or integration
handles. A frozen sidecar can later provide ``APS_SOLVER_CHILD_EXECUTABLE``;
tests and controlled launchers may provide ``APS_SOLVER_CHILD_COMMAND_JSON``.
"""
if not isinstance(pipeline_label, str) or not pipeline_label.strip():
raise SolverProcessError("SOLVER_REQUEST_INVALID", "pipeline_label 不能为空")
normalized_world = _json_value(world)
normalized_entries = _json_value(entries)
normalized_params = _json_value(params)
normalized_warm_start = _json_value(warm_start) if warm_start is not None else None
if not isinstance(normalized_world, dict):
raise SolverProcessError("SOLVER_REQUEST_INVALID", "world 必须是对象")
if not isinstance(normalized_entries, list) or not all(
isinstance(item, dict) for item in normalized_entries
):
raise SolverProcessError("SOLVER_REQUEST_INVALID", "entries 必须是对象数组")
if not isinstance(normalized_params, dict):
raise SolverProcessError("SOLVER_REQUEST_INVALID", "params 必须是对象")
if normalized_warm_start is not None and (
not isinstance(normalized_warm_start, list)
or not all(isinstance(item, dict) for item in normalized_warm_start)
):
raise SolverProcessError("SOLVER_REQUEST_INVALID", "warm_start 必须是对象数组")
body: dict[str, Any] = {
"protocolVersion": PROTOCOL_VERSION,
"operation": "optimize_line_assignment",
"world": normalized_world,
"entries": normalized_entries,
"params": normalized_params,
"warmStart": normalized_warm_start,
"pipelineLabel": pipeline_label,
}
request_id = _request_digest(body)
request = {**body, "requestId": request_id}
request_text = _canonical_json(request)
timeout = _effective_timeout(normalized_params, timeout_seconds)
command, command_kind = _child_command()
creationflags, start_new_session = _creation_options()
# Pipe-backed stdin can block the Windows communicate() writer before
# timeout supervision starts when a child never reads a large world.
# Stage the canonical request in an auto-deleting file inherited as stdin.
with tempfile.TemporaryFile(
mode="w+", encoding="utf-8", newline=""
) as request_stream:
request_stream.write(request_text)
request_stream.flush()
request_stream.seek(0)
try:
process = subprocess.Popen(
command,
stdin=request_stream,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
encoding="utf-8",
errors="replace",
env=_child_environment(),
shell=False,
creationflags=creationflags,
start_new_session=start_new_session,
)
except OSError as exc:
raise SolverProcessError(
"SOLVER_PROCESS_EXITED",
"无法启动求解子进程",
{"error": str(exc), "commandKind": command_kind},
) from exc
try:
stdout, stderr = process.communicate(timeout=timeout)
except subprocess.TimeoutExpired as exc:
_terminate_process_tree(process)
try:
stdout, stderr = process.communicate(timeout=1)
except (OSError, subprocess.SubprocessError):
stdout, stderr = "", ""
raise SolverProcessError(
"SOLVER_PROCESS_TIMEOUT",
"求解子进程超过父进程时限并已回收",
{
"timeoutSeconds": timeout,
"pid": process.pid,
"commandKind": command_kind,
"stdout": _captured(stdout),
"stderr": _captured(stderr),
},
) from exc
marker = _fatal_marker(stdout, stderr)
if marker is not None:
raise SolverProcessError(
"SOLVER_NATIVE_FATAL",
"求解子进程输出原生 fatal 标记",
{
"fatalMarker": marker,
"returnCode": process.returncode,
"commandKind": command_kind,
"stdout": _captured(stdout),
"stderr": _captured(stderr),
},
)
if process.returncode != 0:
raise SolverProcessError(
"SOLVER_PROCESS_EXITED",
"求解子进程异常退出",
{
"returnCode": process.returncode,
"commandKind": command_kind,
"stdout": _captured(stdout),
"stderr": _captured(stderr),
},
)
response = _response_object(stdout)
return _validate_response(
response,
request_id=request_id,
expected_entries=normalized_entries,
pipeline_label=pipeline_label,
)
__all__ = [
"ERROR_MARKER",
"PROTOCOL_VERSION",
"SUCCESS_MARKER",
"SolverProcessError",
"run_cp_assignment",
]