aps-agent/tests/golden/test_solver_subprocess.py

788 lines
24 KiB
Python

from __future__ import annotations
import copy
import json
import shutil
import subprocess
import sys
import textwrap
from pathlib import Path
import pytest
from server.engines import solver_process
from server.engines.base import EngineParams
from server.engines.solver_process import (
ENV_CHILD_COMMAND,
PROTOCOL_VERSION,
SolverProcessError,
run_cp_assignment,
)
def _set_child(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
source: str,
*,
name: str = "child_fixture.py",
executable: str | None = None,
) -> Path:
script = tmp_path / name
script.write_text(textwrap.dedent(source), encoding="utf-8")
command = [
executable or sys.executable,
"-I",
"-B",
"-X",
"utf8",
"-X",
"faulthandler",
str(script),
]
monkeypatch.setenv(ENV_CHILD_COMMAND, json.dumps(command))
return script
def _set_semantic_child(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
mode: str,
) -> Path:
return _set_child(
monkeypatch,
tmp_path,
f"""
import hashlib, json, sys
mode = {mode!r}
request = json.loads(sys.stdin.read())
entries = list(request["entries"])
status = "TRIVIAL" if not entries else "OPTIMAL"
slots = []
for index, entry in enumerate(entries):
slot = {{"orderIndex": index, "isFrozen": False}}
so = entry.get("so")
if isinstance(so, dict):
slot["orderNo"] = so.get("orderNo")
item = entry.get("item")
if isinstance(item, dict):
slot["productId"] = item.get("productId")
slots.append(slot)
pipeline = request["pipelineLabel"]
if mode == "wrong-pipeline":
pipeline = "FORGED-PIPELINE"
elif mode == "unknown-status":
status = "FORGED_STATUS"
elif mode == "nonempty-trivial":
status = "TRIVIAL"
elif mode == "empty-optimal":
status = "OPTIMAL"
elif mode == "extra-slot":
slots.append({{"orderIndex": 99, "isFrozen": False}})
elif mode == "wrong-order" and slots:
slots[0]["orderNo"] = "FORGED-ORDER"
elif mode == "wrong-product" and slots:
slots[0]["productId"] = "FORGED-PRODUCT"
meta = {{
"pipeline": pipeline,
"status": status,
"objective": 0.0,
"gap": 0.0,
"operationSlots": slots,
}}
response = {{
"protocolVersion": request["protocolVersion"],
"marker": "APS_SOLVER_SUCCESS_V1",
"ok": True,
"requestId": request["requestId"],
"runtimeIdentity": {{"safe": True, "implementation": "CPython"}},
"result": {{"entries": entries, "solverMeta": meta}},
}}
canonical = json.dumps(
response, ensure_ascii=False, allow_nan=False,
sort_keys=True, separators=(",", ":")
)
response["responseDigest"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
sys.stdout.write(json.dumps(response))
""",
name=f"semantic_{mode}.py",
)
def _call(*, timeout_seconds: float = 3.0):
return run_cp_assignment(
{"sentinel": {"value": 1}},
[],
EngineParams(engineType="CP", timeLimitSeconds=0.5),
warm_start=None,
pipeline_label="W66-PROTOCOL",
timeout_seconds=timeout_seconds,
)
def _assert_error(code: str, *, timeout_seconds: float = 3.0) -> SolverProcessError:
with pytest.raises(SolverProcessError) as caught:
_call(timeout_seconds=timeout_seconds)
assert caught.value.code == code
assert caught.value.message
assert caught.value.as_dict() == {
"code": code,
"message": caught.value.message,
"details": caught.value.details,
}
return caught.value
def test_real_worker_uses_isolated_runtime_and_does_not_mutate_parent_inputs(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
monkeypatch.delenv(ENV_CHILD_COMMAND, raising=False)
monkeypatch.delenv(solver_process.ENV_CHILD_EXECUTABLE, raising=False)
monkeypatch.chdir(tmp_path)
world = {"sentinel": {"nested": [1, 2, 3]}}
entries: list[dict] = []
before_world = copy.deepcopy(world)
before_entries = copy.deepcopy(entries)
ordered, meta = run_cp_assignment(
world,
entries,
EngineParams(engineType="CP", timeLimitSeconds=0.5),
pipeline_label="W66-PROTOCOL",
timeout_seconds=20,
)
assert ordered == []
assert meta["status"] == "TRIVIAL"
assert meta["pipeline"] == "W66-PROTOCOL"
process_meta = meta["solverProcess"]
assert process_meta["protocolVersion"] == PROTOCOL_VERSION
assert len(process_meta["requestId"]) == 64
identity = process_meta["runtimeIdentity"]
assert identity["safe"] is True
assert identity["implementation"] == "CPython"
assert identity["isolated"] is True
assert identity["enableUserSite"] is False
assert set(identity["packages"]) == {"ortools", "numpy", "pandas"}
assert world == before_world
assert entries == before_entries
def test_real_worker_translates_python_failure_without_mutating_parent(
monkeypatch: pytest.MonkeyPatch,
):
monkeypatch.delenv(ENV_CHILD_COMMAND, raising=False)
monkeypatch.delenv(solver_process.ENV_CHILD_EXECUTABLE, raising=False)
world = {"sentinel": {"nested": [1, 2, 3]}}
before = copy.deepcopy(world)
with pytest.raises(SolverProcessError) as caught:
run_cp_assignment(
world,
[],
{"engineType": "CP", "timeLimitSeconds": "not-a-number"},
pipeline_label="W66-PROTOCOL",
timeout_seconds=10,
)
assert caught.value.code == "SOLVER_EXECUTION_FAILED"
assert caught.value.details["exceptionType"] == "ValidationError"
assert world == before
def test_request_is_canonical_detached_and_transmits_warm_start(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
_set_child(
monkeypatch,
tmp_path,
"""
import hashlib, json, sys
request = json.loads(sys.stdin.read())
response = {
"protocolVersion": request["protocolVersion"],
"marker": "APS_SOLVER_SUCCESS_V1",
"ok": True,
"requestId": request["requestId"],
"runtimeIdentity": {"safe": True, "implementation": "CPython"},
"result": {
"entries": request["entries"],
"solverMeta": {
"pipeline": request["pipelineLabel"],
"warmStart": request["warmStart"],
"engineType": request["params"]["engineType"],
"status": "OPTIMAL",
"objective": 0.0,
"gap": 0.0,
"operationSlots": [
{"orderIndex": index, "isFrozen": False}
for index, _entry in enumerate(request["entries"])
],
},
},
}
canonical = json.dumps(
response, ensure_ascii=False, allow_nan=False,
sort_keys=True, separators=(",", ":")
)
response["responseDigest"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
sys.stdout.write(json.dumps(response, separators=(",", ":")))
""",
)
world = {"z": [2, 1], "a": {"stable": True}}
entries = [{"order": 2}, {"order": 1}]
warm_start = [{"lineId": 7, "start": 10, "end": 20}]
before = (copy.deepcopy(world), copy.deepcopy(entries), copy.deepcopy(warm_start))
ordered, meta = run_cp_assignment(
world,
entries,
EngineParams(engineType="CP"),
warm_start=warm_start,
pipeline_label="RULE→CP-SAT",
timeout_seconds=3,
)
assert ordered == entries
assert meta["pipeline"] == "RULE→CP-SAT"
assert meta["warmStart"] == warm_start
assert meta["engineType"] == "CP"
assert meta["solverProcess"]["responseDigest"]
assert (world, entries, warm_start) == before
@pytest.mark.parametrize("digest_mode", ["missing", "wrong"])
def test_response_digest_is_mandatory_and_verified(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
digest_mode: str,
) -> None:
digest_statement = (
"pass"
if digest_mode == "missing"
else 'response["responseDigest"] = "0" * 64'
)
_set_child(
monkeypatch,
tmp_path,
f"""
import json, sys
request = json.loads(sys.stdin.read())
response = {{
"protocolVersion": request["protocolVersion"],
"marker": "APS_SOLVER_SUCCESS_V1",
"ok": True,
"requestId": request["requestId"],
"runtimeIdentity": {{"safe": True, "implementation": "CPython"}},
"result": {{
"entries": [],
"solverMeta": {{
"pipeline": request["pipelineLabel"],
"status": "TRIVIAL",
"objective": 0,
"gap": 0.0,
}},
}},
}}
{digest_statement}
sys.stdout.write(json.dumps(response))
""",
)
_assert_error("SOLVER_RESPONSE_INVALID")
@pytest.mark.parametrize(
"response_entries",
[
"[]",
'request["entries"][:-1]',
'request["entries"] + [request["entries"][0]]',
'[dict(request["entries"][0], quantity=999), request["entries"][1]]',
],
)
def test_response_entries_must_be_complete_unchanged_permutation(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
response_entries: str,
) -> None:
_set_child(
monkeypatch,
tmp_path,
f"""
import hashlib, json, sys
request = json.loads(sys.stdin.read())
response = {{
"protocolVersion": request["protocolVersion"],
"marker": "APS_SOLVER_SUCCESS_V1",
"ok": True,
"requestId": request["requestId"],
"runtimeIdentity": {{"safe": True, "implementation": "CPython"}},
"result": {{
"entries": {response_entries},
"solverMeta": {{
"pipeline": request["pipelineLabel"],
"status": "OPTIMAL",
"objective": 0.0,
"gap": 0.0,
"operationSlots": [
{{"orderIndex": index, "isFrozen": False}}
for index, _entry in enumerate(request["entries"])
],
}},
}},
}}
canonical = json.dumps(
response, ensure_ascii=False, allow_nan=False,
sort_keys=True, separators=(",", ":")
)
response["responseDigest"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
sys.stdout.write(json.dumps(response))
""",
)
entries = [
{"orderNo": "SO-1", "quantity": 1},
{"orderNo": "SO-2", "quantity": 2},
]
with pytest.raises(SolverProcessError) as caught:
run_cp_assignment(
{"sentinel": True},
entries,
EngineParams(engineType="CP"),
pipeline_label="W66-SEMANTIC",
timeout_seconds=3,
)
assert caught.value.code == "SOLVER_RESPONSE_INVALID"
def test_feasible_response_requires_operation_slot_coverage(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
_set_child(
monkeypatch,
tmp_path,
"""
import hashlib, json, sys
request = json.loads(sys.stdin.read())
response = {
"protocolVersion": request["protocolVersion"],
"marker": "APS_SOLVER_SUCCESS_V1",
"ok": True,
"requestId": request["requestId"],
"runtimeIdentity": {"safe": True, "implementation": "CPython"},
"result": {
"entries": request["entries"],
"solverMeta": {
"pipeline": request["pipelineLabel"],
"status": "OPTIMAL",
"objective": 0.0,
"gap": 0.0,
"operationSlots": [{"orderIndex": 0, "isFrozen": False}],
},
},
}
canonical = json.dumps(
response, ensure_ascii=False, allow_nan=False,
sort_keys=True, separators=(",", ":")
)
response["responseDigest"] = hashlib.sha256(canonical.encode("utf-8")).hexdigest()
sys.stdout.write(json.dumps(response))
""",
)
with pytest.raises(SolverProcessError) as caught:
run_cp_assignment(
{"sentinel": True},
[{"orderNo": "SO-1"}, {"orderNo": "SO-2"}],
EngineParams(engineType="CP"),
pipeline_label="W66-SLOTS",
timeout_seconds=3,
)
assert caught.value.code == "SOLVER_RESPONSE_INVALID"
@pytest.mark.parametrize("mode", ["extra-slot", "wrong-order", "wrong-product"])
def test_operation_slots_reject_out_of_range_or_misbound_identity(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
mode: str,
) -> None:
_set_semantic_child(monkeypatch, tmp_path, mode)
entries = [
{"so": {"orderNo": "SO-1"}, "item": {"productId": 101}},
{"so": {"orderNo": "SO-2"}, "item": {"productId": 102}},
]
with pytest.raises(SolverProcessError) as caught:
run_cp_assignment(
{"sentinel": True},
entries,
EngineParams(engineType="CP"),
pipeline_label="W66-SLOT-IDENTITY",
timeout_seconds=3,
)
assert caught.value.code == "SOLVER_RESPONSE_INVALID"
@pytest.mark.parametrize(
("mode", "entries"),
[
("wrong-pipeline", [{"orderNo": "SO-1"}]),
("unknown-status", [{"orderNo": "SO-1"}]),
("nonempty-trivial", [{"orderNo": "SO-1"}]),
("empty-optimal", []),
],
)
def test_pipeline_and_status_inconsistency_is_rejected(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
mode: str,
entries: list[dict],
) -> None:
_set_semantic_child(monkeypatch, tmp_path, mode)
with pytest.raises(SolverProcessError) as caught:
run_cp_assignment(
{"sentinel": True},
entries,
EngineParams(engineType="CP"),
pipeline_label="W66-STATUS",
timeout_seconds=3,
)
assert caught.value.code == "SOLVER_RESPONSE_INVALID"
def test_frozen_runtime_uses_its_own_solver_child_entry(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv(ENV_CHILD_COMMAND, raising=False)
monkeypatch.delenv(solver_process.ENV_CHILD_EXECUTABLE, raising=False)
monkeypatch.setattr(sys, "frozen", True, raising=False)
command, command_kind = solver_process._child_command()
assert command == [str(Path(sys.executable).resolve()), "--solver-child"]
assert command_kind == "frozen-self"
def test_nonzero_child_exit_is_structured(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
_set_child(
monkeypatch,
tmp_path,
"""
import sys
sys.stdin.read()
sys.stderr.write("plain child failure")
raise SystemExit(7)
""",
)
error = _assert_error("SOLVER_PROCESS_EXITED")
assert error.details["returnCode"] == 7
assert "plain child failure" in error.details["stderr"]
def test_zero_exit_with_fatal_marker_is_never_accepted(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
_set_child(
monkeypatch,
tmp_path,
"""
import json, sys
request = json.loads(sys.stdin.read())
response = {
"protocolVersion": request["protocolVersion"],
"marker": "APS_SOLVER_SUCCESS_V1",
"ok": True,
"requestId": request["requestId"],
"runtimeIdentity": {"safe": True, "implementation": "CPython"},
"result": {"entries": [], "solverMeta": {}},
}
sys.stdout.write(json.dumps(response))
sys.stderr.write("Windows fatal exception: code 0xc0000139")
""",
)
error = _assert_error("SOLVER_NATIVE_FATAL")
assert error.details["returnCode"] == 0
assert error.details["fatalMarker"] in {"windows fatal exception", "0xc0000139"}
def test_timeout_calls_tree_termination_and_fails_closed(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
_set_child(
monkeypatch,
tmp_path,
"""
import sys, time
sys.stdin.read()
time.sleep(60)
""",
)
terminated: list[int] = []
original = solver_process._terminate_process_tree
def tracking_terminate(process: subprocess.Popen[str]) -> None:
terminated.append(process.pid)
original(process)
monkeypatch.setattr(solver_process, "_terminate_process_tree", tracking_terminate)
error = _assert_error("SOLVER_PROCESS_TIMEOUT", timeout_seconds=0.1)
assert terminated == [error.details["pid"]]
assert error.details["timeoutSeconds"] == pytest.approx(0.1)
def test_timeout_is_bounded_when_child_never_reads_large_stdin(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
import time
_set_child(
monkeypatch,
tmp_path,
"""
import time
time.sleep(60)
""",
name="never_reads_stdin.py",
)
large_world = {"payload": "x" * (2 * 1024 * 1024)}
started = time.monotonic()
with pytest.raises(SolverProcessError) as caught:
run_cp_assignment(
large_world,
[],
EngineParams(engineType="CP", timeLimitSeconds=0.5),
pipeline_label="W66-NO-STDIN-READ",
timeout_seconds=0.2,
)
elapsed = time.monotonic() - started
assert caught.value.code == "SOLVER_PROCESS_TIMEOUT"
assert elapsed < 12.0, f"timeout supervision was blocked for {elapsed:.3f}s"
assert caught.value.details["timeoutSeconds"] == 0.2
def test_windows_tree_termination_uses_taskkill_t_f(monkeypatch: pytest.MonkeyPatch):
calls: list[list[str]] = []
class FakeProcess:
pid = 43210
alive = True
def poll(self):
return None if self.alive else 1
def kill(self):
self.alive = False
def wait(self, timeout):
self.alive = False
return 1
def fake_run(command, **kwargs):
calls.append(command)
return subprocess.CompletedProcess(command, 0)
monkeypatch.setattr(solver_process, "_is_windows", lambda: True)
monkeypatch.setattr(solver_process.subprocess, "run", fake_run)
solver_process._terminate_process_tree(FakeProcess()) # type: ignore[arg-type]
assert calls == [["taskkill", "/PID", "43210", "/T", "/F"]]
@pytest.mark.parametrize("payload", ["{", "not-json", "[]"])
def test_invalid_or_truncated_json_is_rejected(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, payload: str
):
_set_child(
monkeypatch,
tmp_path,
f"""
import sys
sys.stdin.read()
sys.stdout.write({payload!r})
""",
)
_assert_error("SOLVER_RESPONSE_INVALID")
def test_missing_success_marker_is_rejected(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
_set_child(
monkeypatch,
tmp_path,
"""
import json, sys
request = json.loads(sys.stdin.read())
sys.stdout.write(json.dumps({
"protocolVersion": request["protocolVersion"],
"ok": True,
"requestId": request["requestId"],
"runtimeIdentity": {"safe": True, "implementation": "CPython"},
"result": {"entries": [], "solverMeta": {}},
}))
""",
)
error = _assert_error("SOLVER_RESPONSE_INVALID")
assert error.details["marker"] is None
def test_protocol_mismatch_is_rejected(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
_set_child(
monkeypatch,
tmp_path,
"""
import json, sys
request = json.loads(sys.stdin.read())
sys.stdout.write(json.dumps({
"protocolVersion": "aps.solver-process.v999",
"marker": "APS_SOLVER_SUCCESS_V1",
"ok": True,
"requestId": request["requestId"],
"runtimeIdentity": {"safe": True, "implementation": "CPython"},
"result": {"entries": [], "solverMeta": {}},
}))
""",
)
error = _assert_error("SOLVER_PROTOCOL_MISMATCH")
assert error.details["expected"] == PROTOCOL_VERSION
def test_request_id_mismatch_is_rejected(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
_set_child(
monkeypatch,
tmp_path,
"""
import json, sys
request = json.loads(sys.stdin.read())
sys.stdout.write(json.dumps({
"protocolVersion": request["protocolVersion"],
"marker": "APS_SOLVER_SUCCESS_V1",
"ok": True,
"requestId": "0" * 64,
"runtimeIdentity": {"safe": True, "implementation": "CPython"},
"result": {"entries": [], "solverMeta": {}},
}))
""",
)
_assert_error("SOLVER_PROTOCOL_MISMATCH")
def test_unsafe_runtime_identity_is_rejected(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
_set_child(
monkeypatch,
tmp_path,
"""
import json, sys
request = json.loads(sys.stdin.read())
sys.stdout.write(json.dumps({
"protocolVersion": request["protocolVersion"],
"marker": "APS_SOLVER_SUCCESS_V1",
"ok": True,
"requestId": request["requestId"],
"runtimeIdentity": {
"safe": False,
"implementation": "CPython",
"reasons": ["ortools:user-site"],
},
"result": {"entries": [], "solverMeta": {}},
}))
""",
)
error = _assert_error("SOLVER_RUNTIME_UNSAFE")
assert "ortools:user-site" in error.details["runtimeIdentity"]["reasons"]
def test_structured_worker_error_is_propagated(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
):
_set_child(
monkeypatch,
tmp_path,
"""
import json, sys
request = json.loads(sys.stdin.read())
sys.stdout.write(json.dumps({
"protocolVersion": request["protocolVersion"],
"marker": "APS_SOLVER_ERROR_V1",
"ok": False,
"requestId": request["requestId"],
"error": {
"code": "SOLVER_EXECUTION_FAILED",
"message": "fixture execution failed",
"details": {"exceptionType": "RuntimeError"},
},
}))
""",
)
error = _assert_error("SOLVER_EXECUTION_FAILED")
assert error.message == "fixture execution failed"
assert error.details["exceptionType"] == "RuntimeError"
def test_invalid_command_override_fails_before_spawn(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv(ENV_CHILD_COMMAND, "not-json")
error = _assert_error("SOLVER_COMMAND_INVALID")
assert ENV_CHILD_COMMAND in error.message
def test_request_validation_is_fail_closed():
with pytest.raises(SolverProcessError) as caught:
run_cp_assignment({}, [], {}, pipeline_label="", timeout_seconds=1)
assert caught.value.code == "SOLVER_REQUEST_INVALID"
with pytest.raises(SolverProcessError) as caught:
run_cp_assignment({}, [], {}, pipeline_label="CP", timeout_seconds=0)
assert caught.value.code == "SOLVER_REQUEST_INVALID"
@pytest.mark.skipif(sys.platform != "win32", reason="Windows runtime provenance probe")
def test_default_path_anaconda_mix_is_rejected_when_present(
monkeypatch: pytest.MonkeyPatch,
):
default_python = shutil.which("python")
if default_python is None:
pytest.skip("PATH has no alternate python")
assert default_python is not None
if Path(default_python).resolve() == Path(sys.executable).resolve():
pytest.skip("PATH python is the current approved runtime")
worker = Path(solver_process.__file__).with_name("solver_worker.py").resolve()
command = [
str(Path(default_python).resolve()),
"-I",
"-B",
"-X",
"utf8",
"-X",
"faulthandler",
str(worker),
]
monkeypatch.setenv(ENV_CHILD_COMMAND, json.dumps(command))
with pytest.raises(SolverProcessError) as caught:
_call(timeout_seconds=15)
assert caught.value.code in {
"SOLVER_RUNTIME_UNSAFE",
"SOLVER_NATIVE_FATAL",
"SOLVER_PROCESS_EXITED",
}