321 lines
10 KiB
Python
321 lines
10 KiB
Python
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import copy
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
from fastapi.testclient import TestClient
|
||
|
|
|
||
|
|
from server.agent_core.async_jobs import JobCancelled, JobQueue, JobRecord
|
||
|
|
from server.aps_domain.sensitivity import run_sobol_sensitivity
|
||
|
|
from server.state.seed import build_demo_world, empty_world
|
||
|
|
from tests.auth_provider import install_test_auth
|
||
|
|
|
||
|
|
|
||
|
|
def _analytic_kpis(
|
||
|
|
_world,
|
||
|
|
*,
|
||
|
|
horizon,
|
||
|
|
vip_weight,
|
||
|
|
delivery_buffer,
|
||
|
|
freeze_hours,
|
||
|
|
efficiency_scale,
|
||
|
|
**_kwargs,
|
||
|
|
):
|
||
|
|
tardiness = (
|
||
|
|
float(horizon) * 100.0
|
||
|
|
+ float(vip_weight) * 2.0
|
||
|
|
+ float(delivery_buffer) * 0.1
|
||
|
|
+ float(freeze_hours) * 0.01
|
||
|
|
+ float(efficiency_scale) * 0.001
|
||
|
|
)
|
||
|
|
return {
|
||
|
|
"tardiness": tardiness,
|
||
|
|
"conflicts": float(horizon),
|
||
|
|
"utilization": float(efficiency_scale),
|
||
|
|
"changeoverMin": float(freeze_hours),
|
||
|
|
"poCount": 0.0,
|
||
|
|
"woCount": 0.0,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def _interaction_kpis(
|
||
|
|
_world,
|
||
|
|
*,
|
||
|
|
horizon,
|
||
|
|
vip_weight,
|
||
|
|
delivery_buffer,
|
||
|
|
freeze_hours,
|
||
|
|
efficiency_scale,
|
||
|
|
**_kwargs,
|
||
|
|
):
|
||
|
|
return {
|
||
|
|
"tardiness": (float(horizon) - 31.5) * (float(vip_weight) - 5.5),
|
||
|
|
"conflicts": 1.0,
|
||
|
|
"utilization": float(efficiency_scale),
|
||
|
|
"changeoverMin": float(freeze_hours),
|
||
|
|
"poCount": float(delivery_buffer),
|
||
|
|
"woCount": 0.0,
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def test_sobol_is_deterministic_and_ranks_dominant_factor(monkeypatch):
|
||
|
|
from server.aps_domain import sensitivity
|
||
|
|
|
||
|
|
monkeypatch.setattr(sensitivity, "_run_sandbox", _analytic_kpis)
|
||
|
|
world = empty_world()
|
||
|
|
before = copy.deepcopy(world)
|
||
|
|
options = {
|
||
|
|
"base_samples": 32,
|
||
|
|
"seed": 7788,
|
||
|
|
"metric": "tardiness",
|
||
|
|
"start_date": "2026-08-18",
|
||
|
|
}
|
||
|
|
first = run_sobol_sensitivity(world, **options)
|
||
|
|
second = run_sobol_sensitivity(world, **options)
|
||
|
|
|
||
|
|
assert first == second
|
||
|
|
assert world == before
|
||
|
|
assert first["method"] == "sobol-jansen"
|
||
|
|
assert first["evaluations"] == 32 * 7
|
||
|
|
assert first["startDate"] == "2026-08-18"
|
||
|
|
assert first["startDateSource"] == "request"
|
||
|
|
assert first["factors"][0]["factorId"] == "planningHorizonDays"
|
||
|
|
assert first["factors"][0]["totalOrder"] > 0.8
|
||
|
|
|
||
|
|
|
||
|
|
def test_sobol_uses_world_business_date_without_wall_clock(monkeypatch):
|
||
|
|
from server.aps_domain import sensitivity
|
||
|
|
|
||
|
|
monkeypatch.setattr(sensitivity, "_run_sandbox", _analytic_kpis)
|
||
|
|
world = empty_world()
|
||
|
|
world["businessDate"] = "2026-04-09"
|
||
|
|
result = run_sobol_sensitivity(world, base_samples=8, seed=5)
|
||
|
|
assert result["startDate"] == "2026-04-09"
|
||
|
|
assert result["startDateSource"] == "world.businessDate"
|
||
|
|
|
||
|
|
|
||
|
|
def test_sobol_jansen_indices_capture_interaction_and_ignore_unrelated_factors(monkeypatch):
|
||
|
|
from server.aps_domain import sensitivity
|
||
|
|
|
||
|
|
monkeypatch.setattr(sensitivity, "_run_sandbox", _interaction_kpis)
|
||
|
|
result = run_sobol_sensitivity(
|
||
|
|
empty_world(),
|
||
|
|
base_samples=64,
|
||
|
|
seed=20260818,
|
||
|
|
start_date="2026-08-18",
|
||
|
|
)
|
||
|
|
factors = {row["factorId"]: row for row in result["factors"]}
|
||
|
|
for factor_id in ("planningHorizonDays", "vipWeight"):
|
||
|
|
row = factors[factor_id]
|
||
|
|
assert row["totalOrder"] > row["firstOrder"] + 0.5
|
||
|
|
for factor_id in ("deliveryBufferRatio", "freezeWindowHours", "lineEfficiency"):
|
||
|
|
assert abs(factors[factor_id]["totalOrder"]) < 0.05
|
||
|
|
|
||
|
|
|
||
|
|
def test_sobol_marks_zero_variance_indices_unidentifiable(monkeypatch):
|
||
|
|
from server.aps_domain import sensitivity
|
||
|
|
|
||
|
|
def constant_kpis(*_args, **_kwargs):
|
||
|
|
return {
|
||
|
|
"tardiness": 1.0,
|
||
|
|
"conflicts": 0.0,
|
||
|
|
"utilization": 0.5,
|
||
|
|
"changeoverMin": 0.0,
|
||
|
|
"poCount": 0.0,
|
||
|
|
"woCount": 0.0,
|
||
|
|
}
|
||
|
|
|
||
|
|
monkeypatch.setattr(sensitivity, "_run_sandbox", constant_kpis)
|
||
|
|
result = run_sobol_sensitivity(
|
||
|
|
empty_world(), base_samples=8, start_date="2026-08-18",
|
||
|
|
)
|
||
|
|
assert result["isDegenerate"] is True
|
||
|
|
assert all(row["rank"] is None for row in result["factors"])
|
||
|
|
assert all(row["firstOrder"] is None for row in result["factors"])
|
||
|
|
assert all(row["totalOrder"] is None for row in result["factors"])
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize("base_samples", [0, 7, 12, 65, "8", True])
|
||
|
|
def test_sobol_rejects_invalid_base_samples(base_samples):
|
||
|
|
with pytest.raises(ValueError, match="baseSamples"):
|
||
|
|
run_sobol_sensitivity(
|
||
|
|
empty_world(),
|
||
|
|
base_samples=base_samples,
|
||
|
|
start_date="2026-08-18",
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def test_sobol_requires_a_canonical_start_date():
|
||
|
|
with pytest.raises(ValueError, match="startDate"):
|
||
|
|
run_sobol_sensitivity(empty_world(), base_samples=8)
|
||
|
|
with pytest.raises(ValueError, match="YYYY-MM-DD"):
|
||
|
|
run_sobol_sensitivity(empty_world(), base_samples=8, start_date="20260818")
|
||
|
|
|
||
|
|
|
||
|
|
def test_sobol_cooperatively_cancels_between_evaluations(monkeypatch):
|
||
|
|
from server.aps_domain import sensitivity
|
||
|
|
|
||
|
|
record = JobRecord(job_id="job-cancel", kind="sobol.recompute")
|
||
|
|
calls = 0
|
||
|
|
|
||
|
|
def cancel_after_third(*args, **kwargs):
|
||
|
|
nonlocal calls
|
||
|
|
calls += 1
|
||
|
|
if calls == 3:
|
||
|
|
record.cancel_requested = True
|
||
|
|
return _analytic_kpis(*args, **kwargs)
|
||
|
|
|
||
|
|
monkeypatch.setattr(sensitivity, "_run_sandbox", cancel_after_third)
|
||
|
|
with pytest.raises(JobCancelled):
|
||
|
|
run_sobol_sensitivity(
|
||
|
|
empty_world(),
|
||
|
|
base_samples=8,
|
||
|
|
start_date="2026-08-18",
|
||
|
|
cancel_check=record.raise_if_cancelled,
|
||
|
|
)
|
||
|
|
assert calls == 3
|
||
|
|
|
||
|
|
|
||
|
|
def test_sobol_real_scipy_smoke_executes_56_rule_evaluations():
|
||
|
|
world = build_demo_world()
|
||
|
|
before = copy.deepcopy(world)
|
||
|
|
result = run_sobol_sensitivity(
|
||
|
|
world,
|
||
|
|
base_samples=8,
|
||
|
|
seed=20260818,
|
||
|
|
start_date="2026-08-18",
|
||
|
|
)
|
||
|
|
assert result["sampler"] == "scipy.stats.qmc.Sobol"
|
||
|
|
assert result["evaluations"] == 56
|
||
|
|
assert result["factorCount"] == 5
|
||
|
|
assert len(result["factors"]) == 5
|
||
|
|
assert world == before
|
||
|
|
|
||
|
|
|
||
|
|
class _Store:
|
||
|
|
def __init__(self) -> None:
|
||
|
|
self.data = empty_world()
|
||
|
|
self.data["businessDate"] = "2026-08-18"
|
||
|
|
|
||
|
|
|
||
|
|
class _Projects:
|
||
|
|
def active_world_key(self) -> str:
|
||
|
|
return "project-sobol"
|
||
|
|
|
||
|
|
def require_active_write(self) -> None:
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def sobol_client(monkeypatch):
|
||
|
|
import server.gateway.app as gateway
|
||
|
|
from server.agent_core import async_jobs
|
||
|
|
from server.state import projects
|
||
|
|
|
||
|
|
install_test_auth(monkeypatch, "tenant-sobol")
|
||
|
|
monkeypatch.setattr(gateway, "get_store", lambda: _Store())
|
||
|
|
monkeypatch.setattr(projects, "get_project_store", lambda: _Projects())
|
||
|
|
queue = JobQueue()
|
||
|
|
monkeypatch.setattr(async_jobs, "_queue", queue)
|
||
|
|
client = TestClient(gateway.create_app())
|
||
|
|
assert client.post(
|
||
|
|
"/api/auth/login", json={"username": "planner", "password": "test"},
|
||
|
|
).status_code == 200
|
||
|
|
return client, queue
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.parametrize(
|
||
|
|
"params",
|
||
|
|
[
|
||
|
|
{"startDate": "2026-08-18", "baseSamples": 0},
|
||
|
|
{"startDate": "2026-08-18", "baseSamples": "8"},
|
||
|
|
{"startDate": "2026-08-18", "seed": "7"},
|
||
|
|
{"startDate": "2026-08-18", "metric": "unknown"},
|
||
|
|
{"startDate": "2026/08/18"},
|
||
|
|
],
|
||
|
|
)
|
||
|
|
def test_sobol_gateway_rejects_invalid_params_before_submit(sobol_client, params):
|
||
|
|
client, queue = sobol_client
|
||
|
|
response = client.post("/api/jobs", json={"kind": "sobol.recompute", "params": params})
|
||
|
|
assert response.status_code == 422, response.text
|
||
|
|
assert queue.stats()["total"] == 0
|
||
|
|
|
||
|
|
|
||
|
|
def test_sobol_gateway_submit_and_poll_is_scope_bound(monkeypatch, sobol_client):
|
||
|
|
from server.aps_domain import sensitivity
|
||
|
|
|
||
|
|
client, queue = sobol_client
|
||
|
|
|
||
|
|
def fast_run(_world, **options):
|
||
|
|
options["cancel_check"]()
|
||
|
|
return {
|
||
|
|
"method": "sobol-jansen",
|
||
|
|
"startDate": options["start_date"],
|
||
|
|
"baseSamples": options["base_samples"],
|
||
|
|
"factors": [],
|
||
|
|
}
|
||
|
|
|
||
|
|
monkeypatch.setattr(sensitivity, "run_sobol_sensitivity", fast_run)
|
||
|
|
response = client.post("/api/jobs", json={
|
||
|
|
"kind": "sobol.recompute",
|
||
|
|
"params": {
|
||
|
|
"strategy": "COMPREHENSIVE",
|
||
|
|
"startDate": "2026-08-18",
|
||
|
|
"baseSamples": 8,
|
||
|
|
"seed": 7,
|
||
|
|
"metric": "tardiness",
|
||
|
|
},
|
||
|
|
})
|
||
|
|
assert response.status_code == 200, response.text
|
||
|
|
job_id = response.json()["jobId"]
|
||
|
|
record = queue.wait(job_id, timeout=10)
|
||
|
|
assert record["status"] == "done"
|
||
|
|
assert record["tenant_uuid"] == "tenant-sobol"
|
||
|
|
assert record["project_id"] == "project-sobol"
|
||
|
|
polled = client.get(f"/api/jobs/{job_id}")
|
||
|
|
assert polled.status_code == 200
|
||
|
|
assert polled.json()["job"]["result"]["baseSamples"] == 8
|
||
|
|
|
||
|
|
|
||
|
|
def test_sobol_gateway_preserves_world_business_date_source(monkeypatch, sobol_client):
|
||
|
|
from server.aps_domain import sensitivity
|
||
|
|
|
||
|
|
client, queue = sobol_client
|
||
|
|
monkeypatch.setattr(sensitivity, "_run_sandbox", _analytic_kpis)
|
||
|
|
response = client.post("/api/jobs", json={
|
||
|
|
"kind": "sobol.recompute",
|
||
|
|
"params": {"baseSamples": 8, "seed": 7, "metric": "tardiness"},
|
||
|
|
})
|
||
|
|
assert response.status_code == 200, response.text
|
||
|
|
record = queue.wait(response.json()["jobId"], timeout=10)
|
||
|
|
assert record["status"] == "done"
|
||
|
|
assert record["result"]["startDate"] == "2026-08-18"
|
||
|
|
assert record["result"]["startDateSource"] == "world.businessDate"
|
||
|
|
|
||
|
|
|
||
|
|
def test_scipy_is_synchronized_across_runtime_and_sidecar_dependencies():
|
||
|
|
root = Path(__file__).resolve().parents[2]
|
||
|
|
assert '"scipy>=1.14"' in (root / "pyproject.toml").read_text(encoding="utf-8")
|
||
|
|
assert "scipy>=1.14" in (root / "requirements.txt").read_text(encoding="utf-8")
|
||
|
|
lock = (root / "packaging" / "requirements-sidecar.lock").read_text(encoding="utf-8")
|
||
|
|
assert "scipy==" in lock
|
||
|
|
assert "# via aps-agent-server (pyproject.toml)" in lock
|
||
|
|
|
||
|
|
|
||
|
|
def test_native_sidecar_probe_executes_scipy_sobol(capsys):
|
||
|
|
from server.sidecar import PROBE_OK_PREFIX, probe_native_dependencies
|
||
|
|
|
||
|
|
probe_native_dependencies()
|
||
|
|
output = capsys.readouterr().out
|
||
|
|
assert PROBE_OK_PREFIX in output
|
||
|
|
assert "scipy=" in output
|
||
|
|
assert '"scipy"' in output
|
||
|
|
|
||
|
|
|
||
|
|
def test_sidecar_spec_collects_sobol_direction_numbers():
|
||
|
|
root = Path(__file__).resolve().parents[2]
|
||
|
|
spec = (root / "packaging" / "aps-sidecar.spec").read_text(encoding="utf-8")
|
||
|
|
assert 'collect_data_files("scipy"' in spec
|
||
|
|
assert '"stats/_sobol_direction_numbers.npz"' in spec
|