aps-agent/tests/golden/test_async_jobs.py

# ============================================================
# 方向 F 黄金测试：轻量异步任务队列（矩阵 88 行剩余）
# 覆盖：提交/轮询/完成+结果、失败记录、取消（pending/running）、
#       取消终态返回 False、列表与统计
# ============================================================
from __future__ import annotations

import threading
import time

import pytest

from server.agent_core.async_jobs import JobCancelled, JobQueue, JobQueueFullError


def test_submit_poll_done_with_result():
    """提交 → 轮询 → done，result/progress/时间戳完整，poll 可重复读。"""
    q = JobQueue()
    job_id = q.submit(
        "sensitivity.recompute",
        lambda rec: {"robustness": 0.92},
        params={"strategy": "COMPREHENSIVE"},
        actor="planner",
    )
    record = q.wait(job_id, timeout=10)
    assert record["status"] == "done"
    assert record["result"] == {"robustness": 0.92}
    assert record["progress"] == 100
    assert record["params"]["strategy"] == "COMPREHENSIVE"
    assert record["actor"] == "planner"
    assert record["started_at"] and record["finished_at"]
    again = q.poll(job_id)
    assert again is not None
    assert again["status"] == "done"
    assert again["result"]["robustness"] == 0.92


def test_failed_job_records_error():
    """fn 抛异常 → failed，error 显式记录，result 为空。"""
    q = JobQueue()

    def boom(rec):
        raise RuntimeError("求解器崩溃")

    job_id = q.submit("montecarlo.recompute", boom)
    record = q.wait(job_id, timeout=10)
    assert record["status"] == "failed"
    assert record["error"] == "求解器崩溃"
    assert record["result"] is None


def test_cancel_running_job():
    """运行中任务可取消：置标记 → 终态 cancelled，结果丢弃。"""
    q = JobQueue()
    started = threading.Event()

    def long_run(rec):
        started.set()
        while not rec.cancel_requested:
            time.sleep(0.01)
        raise JobCancelled()

    job_id = q.submit("montecarlo.recompute", long_run)
    assert started.wait(timeout=5), "任务未进入运行"
    assert q.cancel(job_id) is True
    record = q.wait(job_id, timeout=10)
    assert record["status"] == "cancelled"
    assert record["result"] is None
    assert record["cancel_requested"] is True


def test_cancel_unknown_or_terminal_returns_false():
    """不存在或已终态任务取消返回 False。"""
    q = JobQueue()
    assert q.cancel("job-不存在") is False
    job_id = q.submit("t", lambda rec: 1)
    q.wait(job_id, timeout=10)
    assert q.cancel(job_id) is False
    assert q.poll(job_id)["status"] == "done"


def test_list_and_stats():
    """列表可按状态过滤，统计计数与终态一致。"""
    q = JobQueue()

    def boom(rec):
        raise RuntimeError("x")

    q.submit("a", lambda rec: 1)
    q.submit("b", boom)
    q.submit("c", lambda rec: 2)
    for item in q.list():
        q.wait(item["job_id"], timeout=10)
    stats = q.stats()
    assert stats["total"] == 3
    assert stats["done"] == 2
    assert stats["failed"] == 1
    assert stats["pending"] == 0
    failed = q.list(status="failed")
    assert len(failed) == 1 and failed[0]["kind"] == "b"


def test_scope_isolation_applies_to_actor_poll_list_stats_and_cancel():
    q = JobQueue()
    release = threading.Event()
    job_id = q.submit(
        "t", lambda rec: release.wait(5),
        tenant_uuid="t1", project_id="p1", actor="alice",
    )
    assert q.poll(job_id, tenant_uuid="t2", project_id="p1") is None
    assert q.poll(job_id, tenant_uuid="t1", project_id="p2") is None
    assert q.list(tenant_uuid="t2", project_id="p1") == []
    assert q.stats(tenant_uuid="t2", project_id="p1")["total"] == 0
    assert q.cancel(job_id, tenant_uuid="t2", project_id="p1") is False
    assert q.poll(job_id, tenant_uuid="t1", project_id="p1", actor="bob") is None
    assert q.list(tenant_uuid="t1", project_id="p1", actor="bob") == []
    assert q.stats(tenant_uuid="t1", project_id="p1", actor="bob")["total"] == 0
    assert q.cancel(job_id, tenant_uuid="t1", project_id="p1", actor="bob") is False
    assert q.poll(job_id, tenant_uuid="t1", project_id="p1", actor="alice") is not None
    release.set(); q.wait(job_id, timeout=10)


def test_queue_rejects_above_inflight_limit_before_creating_more_threads():
    q = JobQueue(max_concurrent=1, max_inflight=2)
    release = threading.Event()
    ids = [q.submit("t", lambda rec: release.wait(5)) for _ in range(2)]
    with pytest.raises(JobQueueFullError, match="上限"):
        q.submit("overflow", lambda rec: None)
    assert q.cancel(ids[1]) is True
    release.set()
    for job_id in ids:
        q.wait(job_id, timeout=10)


def test_cancelled_pending_worker_still_counts_until_thread_exits():
    q = JobQueue(max_concurrent=1, max_inflight=2)
    release = threading.Event()
    first = q.submit("first", lambda rec: release.wait(5))
    second = q.submit("second", lambda rec: None)
    assert q.cancel(second) is True
    with pytest.raises(JobQueueFullError):
        q.submit("overflow", lambda rec: None)
    release.set(); q.wait(first, timeout=10); q.wait(second, timeout=10)
    third = q.submit("third", lambda rec: 3)
    assert q.wait(third, timeout=10)["status"] == "done"


def test_cancel_success_cannot_race_to_done_for_non_cooperative_job():
    q = JobQueue()
    started = threading.Event(); release = threading.Event()
    def non_cooperative(rec):
        started.set(); release.wait(5); return {"should": "drop"}
    job_id = q.submit("race", non_cooperative)
    assert started.wait(5)
    assert q.cancel(job_id) is True
    release.set()
    record = q.wait(job_id, timeout=10)
    assert record["status"] == "cancelled"
    assert record["result"] is None


def test_cancel_success_cannot_be_overwritten_by_business_exception():
    q = JobQueue()
    started = threading.Event(); release = threading.Event()
    def failing_after_cancel(rec):
        started.set(); release.wait(5); raise RuntimeError("late failure")
    job_id = q.submit("race-error", failing_after_cancel)
    assert started.wait(5)
    assert q.cancel(job_id) is True
    release.set()
    record = q.wait(job_id, timeout=10)
    assert record["status"] == "cancelled"
    assert record["error"] == "cancelled"


def test_thread_start_failure_rolls_back_job_and_inflight_quota(monkeypatch):
    q = JobQueue(max_concurrent=1, max_inflight=1)
    def fail_start(self):
        raise RuntimeError("thread start failed")
    monkeypatch.setattr(threading.Thread, "start", fail_start)
    with pytest.raises(RuntimeError, match="thread start failed"):
        q.submit("start-fail", lambda rec: None)
    assert q._inflight_threads == 0
    assert q.list() == []