185 lines
6.6 KiB
Python
185 lines
6.6 KiB
Python
# ============================================================
|
||
# 方向 F 黄金测试：轻量异步任务队列（矩阵 88 行剩余）
|
||
# 覆盖：提交/轮询/完成+结果、失败记录、取消（pending/running）、
|
||
# 取消终态返回 False、列表与统计
|
||
# ============================================================
|
||
from __future__ import annotations
|
||
|
||
import threading
|
||
import time
|
||
|
||
import pytest
|
||
|
||
from server.agent_core.async_jobs import JobCancelled, JobQueue, JobQueueFullError
|
||
|
||
|
||
def test_submit_poll_done_with_result():
|
||
"""提交 → 轮询 → done，result/progress/时间戳完整，poll 可重复读。"""
|
||
q = JobQueue()
|
||
job_id = q.submit(
|
||
"sensitivity.recompute",
|
||
lambda rec: {"robustness": 0.92},
|
||
params={"strategy": "COMPREHENSIVE"},
|
||
actor="planner",
|
||
)
|
||
record = q.wait(job_id, timeout=10)
|
||
assert record["status"] == "done"
|
||
assert record["result"] == {"robustness": 0.92}
|
||
assert record["progress"] == 100
|
||
assert record["params"]["strategy"] == "COMPREHENSIVE"
|
||
assert record["actor"] == "planner"
|
||
assert record["started_at"] and record["finished_at"]
|
||
again = q.poll(job_id)
|
||
assert again is not None
|
||
assert again["status"] == "done"
|
||
assert again["result"]["robustness"] == 0.92
|
||
|
||
|
||
def test_failed_job_records_error():
|
||
"""fn 抛异常 → failed，error 显式记录，result 为空。"""
|
||
q = JobQueue()
|
||
|
||
def boom(rec):
|
||
raise RuntimeError("求解器崩溃")
|
||
|
||
job_id = q.submit("montecarlo.recompute", boom)
|
||
record = q.wait(job_id, timeout=10)
|
||
assert record["status"] == "failed"
|
||
assert record["error"] == "求解器崩溃"
|
||
assert record["result"] is None
|
||
|
||
|
||
def test_cancel_running_job():
|
||
"""运行中任务可取消：置标记 → 终态 cancelled，结果丢弃。"""
|
||
q = JobQueue()
|
||
started = threading.Event()
|
||
|
||
def long_run(rec):
|
||
started.set()
|
||
while not rec.cancel_requested:
|
||
time.sleep(0.01)
|
||
raise JobCancelled()
|
||
|
||
job_id = q.submit("montecarlo.recompute", long_run)
|
||
assert started.wait(timeout=5), "任务未进入运行"
|
||
assert q.cancel(job_id) is True
|
||
record = q.wait(job_id, timeout=10)
|
||
assert record["status"] == "cancelled"
|
||
assert record["result"] is None
|
||
assert record["cancel_requested"] is True
|
||
|
||
|
||
def test_cancel_unknown_or_terminal_returns_false():
|
||
"""不存在或已终态任务取消返回 False。"""
|
||
q = JobQueue()
|
||
assert q.cancel("job-不存在") is False
|
||
job_id = q.submit("t", lambda rec: 1)
|
||
q.wait(job_id, timeout=10)
|
||
assert q.cancel(job_id) is False
|
||
assert q.poll(job_id)["status"] == "done"
|
||
|
||
|
||
def test_list_and_stats():
|
||
"""列表可按状态过滤，统计计数与终态一致。"""
|
||
q = JobQueue()
|
||
|
||
def boom(rec):
|
||
raise RuntimeError("x")
|
||
|
||
q.submit("a", lambda rec: 1)
|
||
q.submit("b", boom)
|
||
q.submit("c", lambda rec: 2)
|
||
for item in q.list():
|
||
q.wait(item["job_id"], timeout=10)
|
||
stats = q.stats()
|
||
assert stats["total"] == 3
|
||
assert stats["done"] == 2
|
||
assert stats["failed"] == 1
|
||
assert stats["pending"] == 0
|
||
failed = q.list(status="failed")
|
||
assert len(failed) == 1 and failed[0]["kind"] == "b"
|
||
|
||
|
||
def test_scope_isolation_applies_to_actor_poll_list_stats_and_cancel():
|
||
q = JobQueue()
|
||
release = threading.Event()
|
||
job_id = q.submit(
|
||
"t", lambda rec: release.wait(5),
|
||
tenant_uuid="t1", project_id="p1", actor="alice",
|
||
)
|
||
assert q.poll(job_id, tenant_uuid="t2", project_id="p1") is None
|
||
assert q.poll(job_id, tenant_uuid="t1", project_id="p2") is None
|
||
assert q.list(tenant_uuid="t2", project_id="p1") == []
|
||
assert q.stats(tenant_uuid="t2", project_id="p1")["total"] == 0
|
||
assert q.cancel(job_id, tenant_uuid="t2", project_id="p1") is False
|
||
assert q.poll(job_id, tenant_uuid="t1", project_id="p1", actor="bob") is None
|
||
assert q.list(tenant_uuid="t1", project_id="p1", actor="bob") == []
|
||
assert q.stats(tenant_uuid="t1", project_id="p1", actor="bob")["total"] == 0
|
||
assert q.cancel(job_id, tenant_uuid="t1", project_id="p1", actor="bob") is False
|
||
assert q.poll(job_id, tenant_uuid="t1", project_id="p1", actor="alice") is not None
|
||
release.set(); q.wait(job_id, timeout=10)
|
||
|
||
|
||
def test_queue_rejects_above_inflight_limit_before_creating_more_threads():
|
||
q = JobQueue(max_concurrent=1, max_inflight=2)
|
||
release = threading.Event()
|
||
ids = [q.submit("t", lambda rec: release.wait(5)) for _ in range(2)]
|
||
with pytest.raises(JobQueueFullError, match="上限"):
|
||
q.submit("overflow", lambda rec: None)
|
||
assert q.cancel(ids[1]) is True
|
||
release.set()
|
||
for job_id in ids:
|
||
q.wait(job_id, timeout=10)
|
||
|
||
|
||
def test_cancelled_pending_worker_still_counts_until_thread_exits():
|
||
q = JobQueue(max_concurrent=1, max_inflight=2)
|
||
release = threading.Event()
|
||
first = q.submit("first", lambda rec: release.wait(5))
|
||
second = q.submit("second", lambda rec: None)
|
||
assert q.cancel(second) is True
|
||
with pytest.raises(JobQueueFullError):
|
||
q.submit("overflow", lambda rec: None)
|
||
release.set(); q.wait(first, timeout=10); q.wait(second, timeout=10)
|
||
third = q.submit("third", lambda rec: 3)
|
||
assert q.wait(third, timeout=10)["status"] == "done"
|
||
|
||
|
||
def test_cancel_success_cannot_race_to_done_for_non_cooperative_job():
|
||
q = JobQueue()
|
||
started = threading.Event(); release = threading.Event()
|
||
def non_cooperative(rec):
|
||
started.set(); release.wait(5); return {"should": "drop"}
|
||
job_id = q.submit("race", non_cooperative)
|
||
assert started.wait(5)
|
||
assert q.cancel(job_id) is True
|
||
release.set()
|
||
record = q.wait(job_id, timeout=10)
|
||
assert record["status"] == "cancelled"
|
||
assert record["result"] is None
|
||
|
||
|
||
def test_cancel_success_cannot_be_overwritten_by_business_exception():
|
||
q = JobQueue()
|
||
started = threading.Event(); release = threading.Event()
|
||
def failing_after_cancel(rec):
|
||
started.set(); release.wait(5); raise RuntimeError("late failure")
|
||
job_id = q.submit("race-error", failing_after_cancel)
|
||
assert started.wait(5)
|
||
assert q.cancel(job_id) is True
|
||
release.set()
|
||
record = q.wait(job_id, timeout=10)
|
||
assert record["status"] == "cancelled"
|
||
assert record["error"] == "cancelled"
|
||
|
||
|
||
def test_thread_start_failure_rolls_back_job_and_inflight_quota(monkeypatch):
|
||
q = JobQueue(max_concurrent=1, max_inflight=1)
|
||
def fail_start(self):
|
||
raise RuntimeError("thread start failed")
|
||
monkeypatch.setattr(threading.Thread, "start", fail_start)
|
||
with pytest.raises(RuntimeError, match="thread start failed"):
|
||
q.submit("start-fail", lambda rec: None)
|
||
assert q._inflight_threads == 0
|
||
assert q.list() == []
|