aps-agent/tests/golden/test_multimodal.py

481 lines
21 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

# ============================================================
# 矩阵 101(方向 Y):多模态输入与低置信确认框架黄金测试
# 覆盖:stub 提取候选+置信度、低置信出卡不写世界、高置信直通、
# 确认后写世界、拒绝不写、未注册提取器显式报错、审计齐全、
# 阈值可配、扩展提取器注册接口。
# ============================================================
from __future__ import annotations
import base64
import pytest
from fastapi.testclient import TestClient
from server.agent_core import harness
from server.aps_domain.multimodal import (
DEFAULT_CONFIDENCE_THRESHOLD,
ExtractionCandidate,
ExtractorNotRegisteredError,
ExtractorRegistry,
FileImportExtractor,
TextOrderExtractor,
apply_multimodal_candidates,
candidates_to_batches,
default_registry,
ingest_candidates,
)
from server.aps_domain.workflow import execute_confirmed
from server.auth.context import IdentityContext, bind_identity, reset_identity
from server.state.seed import seed_world
from server.timeutil import today0
_PLANNER = IdentityContext(1001, "planner", "计划员", "tenant-multimodal", roles=("planner",))
_THRESHOLD = DEFAULT_CONFIDENCE_THRESHOLD
def _next_id(world):
counters: dict[str, int] = {}
def next_id(kind: str) -> int:
table = "auditEvents" if kind == "audit" else kind + "s"
if kind not in counters:
rows = world.get(table, [])
counters[kind] = max((r.get("id", 0) for r in rows if isinstance(r, dict)), default=0)
counters[kind] += 1
return counters[kind]
return next_id
class _WorldStore:
def __init__(self, world, next_id_fn, *, tenant_uuid="tenant-multimodal", world_key="personal-1001"):
self.data = world
self._next_id_fn = next_id_fn
self.tenant_uuid = tenant_uuid
self.world_key = world_key
def next_id(self, kind: str) -> int:
return self._next_id_fn(kind)
def save(self) -> None:
pass
def _demo_product(world) -> dict:
return next(m for m in world["materials"] if m.get("type") == "FINISHED_PRODUCT")
def _orders_csv(product_code: str, ok: int = 1, bad: int = 0) -> bytes:
head = "订单号,客户,产品编码,数量,交期,优先级\n"
rows = [f"SO-OK-{i},导入客户,{product_code},{10 + i},2026-08-01,3" for i in range(1, ok + 1)]
rows += [f"SO-BAD-{i},客户X,UNKNOWN-{i},5,2026-08-01,3" for i in range(1, bad + 1)]
return (head + "\n".join(rows) + "\n").encode("utf-8")
# ---------------- stub 提取器:结构化候选 + 置信度 ----------------
def test_text_order_stub_sparse_template_is_low_confidence():
cands = TextOrderExtractor().extract("订单 单号 SO-101 数量 5")
assert len(cands) == 1
c = cands[0]
assert c.kind == "text_order" and c.source == "STUB_TEXT"
assert c.complete is True # 必填(单号+数量)齐
assert c.confidence == pytest.approx(0.6) # 缺交期/客户/产品 → 低于默认 0.7
assert "deliveryDate" in c.fields["missing"]
assert c.value["orderNo"] == "SO-101"
assert c.value["quantity"] == 5
assert c.to_dict()["confidence"] == pytest.approx(0.6)
def test_text_order_stub_full_template_is_high_confidence():
cands = TextOrderExtractor().extract(
"订单 单号 SO-102 数量 8 交期 2026-08-20 客户 华东公司 产品编码 CTRL-A")
c = cands[0]
assert c.complete is True
assert c.confidence == pytest.approx(1.0)
assert c.fields["missing"] == []
assert c.value["deliveryDate"] == "2026-08-20"
def test_text_order_stub_no_match_and_incomplete():
extractor = TextOrderExtractor()
assert extractor.extract("今天天气不错") == [] # 无模板命中:不编造
partial = extractor.extract("订单 数量 5") # 缺单号 → 不完整
assert len(partial) == 1
assert partial[0].complete is False
assert partial[0].confidence == pytest.approx(0.3)
def test_file_import_stub_reuses_importers_preview():
world = seed_world()
product = _demo_product(world)
cands = FileImportExtractor(world=world).extract(
{"filename": "orders.csv", "data": _orders_csv(product["code"])})
c = cands[0]
assert c.kind == "file_import" and c.source == "STUB_FILE"
assert c.confidence == pytest.approx(1.0)
assert c.complete is True
assert c.value["canCommit"] is True
assert c.value["batches"][0]["kind"] == "orders"
assert c.fields["batches"]["orders"]["okCount"] == 1
def test_file_import_stub_low_confidence_on_error_rows():
world = seed_world()
product = _demo_product(world)
cands = FileImportExtractor(world=world).extract(
{"filename": "orders.csv", "data": _orders_csv(product["code"], ok=2, bad=5)})
c = cands[0]
assert c.complete is True # 仍有有效行可入库
assert 0 < c.confidence < _THRESHOLD # 2/7 ≈ 0.286 → 低置信
assert c.value["totalOk"] == 2
assert c.value["totalErrors"] == 5
# ---------------- 注册表:未注册显式报错 + 扩展注册接口 ----------------
def test_registry_unregistered_kind_raises_explicit_error():
registry = default_registry(world=seed_world())
with pytest.raises(ExtractorNotRegisteredError) as excinfo:
registry.extract("asr", {"text": "语音识别内容"})
assert "未注册" in str(excinfo.value) and "asr" in str(excinfo.value)
assert "file_import" in str(excinfo.value) # 提示已注册 kinds
with pytest.raises(ExtractorNotRegisteredError):
registry.get("vlm")
assert registry.get("file_import").kind == "file_import"
assert registry.get("text_order").kind == "text_order"
assert set(registry.list_kinds()) == {"file_import", "image_meta", "text_order", "dxf_drawing"}
def test_registry_accepts_extended_asr_vlm_extractor():
class FakeAsrExtractor:
kind = "asr"
source = "ASR_EXT"
def extract(self, raw):
return [ExtractionCandidate(
kind="asr", value={"text": str((raw or {}).get("text") or "")},
confidence=0.9, source="ASR_EXT", fields={"text": {"raw": raw}}, complete=True)]
registry = ExtractorRegistry()
registry.register(FakeAsrExtractor())
cands = registry.extract("asr", {"text": "语音内容"})
assert len(cands) == 1 and cands[0].kind == "asr"
assert cands[0].confidence == pytest.approx(0.9)
with pytest.raises(ValueError): # 重复注册默认拒绝
registry.register(FakeAsrExtractor())
registry.register(FakeAsrExtractor(), replace=True) # 显式覆盖允许(接真实服务入口)
# ---------------- 归一化与 fail-closed 写入 ----------------
def test_candidates_to_batches_text_order_normalizes():
cands = TextOrderExtractor().extract("订单 单号 SO-103 数量 12")
batches = candidates_to_batches(cands)
assert batches == [{
"kind": "orders", "sheet": "multimodal-text",
"okRows": [{"orderNo": "SO-103", "quantity": 12,
"deliveryDate": today0().strftime("%Y-%m-%d"),
"customerName": ""}],
}]
def test_apply_fail_closed_on_incomplete_and_bad_quantity():
world = seed_world()
next_id = _next_id(world)
incomplete = ExtractionCandidate(
kind="text_order", value={"orderNo": "SO-X", "quantity": ""},
confidence=0.3, source="STUB_TEXT", complete=False,
fields={"missing": ["quantity"]})
before = len(world["salesOrders"]) + len(world["flexOrders"])
with pytest.raises(ValueError):
apply_multimodal_candidates(world, next_id, [incomplete])
bad_qty = ExtractionCandidate(
kind="text_order", value={"orderNo": "SO-Y", "quantity": "abc"},
confidence=0.6, source="STUB_TEXT", complete=True)
with pytest.raises(ValueError):
apply_multimodal_candidates(world, next_id, [bad_qty])
assert len(world["salesOrders"]) + len(world["flexOrders"]) == before # 无部分写入
assert not world.get("auditEvents")
# ---------------- 门禁:高置信直通 / 低置信出卡不写世界 ----------------
def test_ingest_high_confidence_direct_apply_no_card():
world = seed_world()
next_id = _next_id(world)
cands = TextOrderExtractor().extract(
"订单 单号 SO-DIR-1 数量 5 交期 2026-08-20 客户 华东公司 产品编码 CTRL-A")
result = ingest_candidates(world, next_id, cands, threshold=_THRESHOLD)
assert result["status"] == "applied"
assert any(o["orderNo"] == "SO-DIR-1" for o in world["flexOrders"])
assert any(e["action"] == "multimodal.apply" for e in world["auditEvents"])
assert not any(e["action"] == "multimodal.stage" for e in world["auditEvents"])
def test_ingest_low_confidence_stages_card_without_world_write():
world = seed_world()
next_id = _next_id(world)
token = bind_identity(_PLANNER)
try:
cands = TextOrderExtractor().extract("订单 单号 SO-LOW-1 数量 5")
result = ingest_candidates(world, next_id, cands, session_id="mm-session",
threshold=_THRESHOLD)
finally:
reset_identity(token)
assert result["status"] == "staged"
confirm_id = result["confirmId"]
assert result["block"]["type"] == "confirm-card"
assert "低置信" in result["block"]["props"]["title"]
assert not any(o["orderNo"] == "SO-LOW-1" for o in world["flexOrders"]) # 确认前绝不写
assert not any(e["action"] == "multimodal.apply" for e in world["auditEvents"])
stages = [e for e in world["auditEvents"] if e["action"] == "multimodal.stage"]
assert len(stages) == 1
assert stages[0]["rationale"]["confirmId"] == confirm_id
assert stages[0]["rationale"]["action"] == "import.commit"
assert stages[0]["result"] == "PENDING"
token = bind_identity(_PLANNER)
try:
assert harness.is_confirmation_pending(confirm_id) is True
finally:
reset_identity(token)
def test_module_confirm_writes_world_reject_does_not():
world = seed_world()
next_id = _next_id(world)
token = bind_identity(_PLANNER)
try:
cands = TextOrderExtractor().extract("订单 单号 SO-LOW-2 数量 7")
staged = ingest_candidates(world, next_id, cands, session_id="mm", threshold=_THRESHOLD)
cid = staged["confirmId"]
store = _WorldStore(world, next_id)
message = execute_confirmed(store, cid, True, actor="planner")
assert "导入完成" in message
assert any(o["orderNo"] == "SO-LOW-2" for o in world["flexOrders"]) # 确认后写世界
assert any(e["action"] == "import.commit" for e in world["auditEvents"])
assert not any(e["action"] == "multimodal.apply" for e in world["auditEvents"]) # 确认通道审计为 import.commit
# 第二条卡:驳回 → 不写世界 + 留痕
cands2 = TextOrderExtractor().extract("订单 单号 SO-LOW-3 数量 3")
staged2 = ingest_candidates(world, next_id, cands2, session_id="mm", threshold=_THRESHOLD)
cid2 = staged2["confirmId"]
before = len(world["flexOrders"])
message2 = execute_confirmed(store, cid2, False, actor="planner")
assert "已驳回" in message2
assert len(world["flexOrders"]) == before
assert any(e["action"] == "import.commit.reject" for e in world["auditEvents"])
finally:
reset_identity(token)
# ---------------- 网关接线:/api/multimodal/* + 确认卡通道 ----------------
class _AuditStore:
def __init__(self, tenant_uuid: str = "tenant-multimodal"):
self.tenant_uuid = tenant_uuid
self.world_key = "personal-1001"
self.data: dict = seed_world()
self._counter = 0
def next_id(self, _kind: str) -> int:
self._counter += 1
return self._counter
def save(self) -> None:
pass
class _CheckpointStore:
def create(self, *_args, **_kwargs) -> dict:
return {"pairId": "mm-pair"}
def get(self, pair_id: str) -> dict | None:
return {"pairId": pair_id}
class _ProjectStore:
def active_world_key(self) -> str:
return "personal-1001"
def require_active_write(self) -> None:
pass
@pytest.fixture
def client_factory(monkeypatch):
import server.gateway.app as gateway_module
import server.state.projects as projects_module
from tests.auth_provider import install_test_auth
provider = install_test_auth(monkeypatch, "tenant-multimodal")
def factory():
store = _AuditStore()
monkeypatch.setattr(gateway_module, "get_store", lambda: store)
monkeypatch.setattr(gateway_module, "get_checkpoints", lambda: _CheckpointStore())
monkeypatch.setattr(projects_module, "get_project_store", lambda: _ProjectStore())
client = TestClient(gateway_module.create_app())
login = client.post("/api/auth/login", json={"username": "planner", "password": "test"})
assert login.status_code == 200, login.text
return client, store, provider
return factory
def test_api_extract_returns_candidates_and_confidence(client_factory):
client, store, _ = client_factory()
resp = client.post("/api/multimodal/extract", json={
"kind": "text_order", "text": "订单 单号 SO-API-1 数量 5"})
assert resp.status_code == 200, resp.text
data = resp.json()
assert data["kind"] == "text_order"
assert data["registeredKinds"] == ["dxf_drawing", "file_import", "image_meta", "text_order"]
assert data["defaultThreshold"] == pytest.approx(0.7)
assert len(data["candidates"]) == 1
cand = data["candidates"][0]
assert cand["confidence"] == pytest.approx(0.6)
assert cand["complete"] is True
assert cand["source"] == "STUB_TEXT"
assert any(e["action"] == "multimodal.extract" for e in store.data["auditEvents"])
def test_api_extract_unregistered_kind_errors_explicitly(client_factory):
client, _, _ = client_factory()
for kind in ("asr", "vlm"):
resp = client.post("/api/multimodal/extract", json={"kind": kind, "text": "外部语音/图片"})
assert resp.status_code == 200, resp.text
assert "未注册" in resp.json()["error"]
assert kind in resp.json()["error"]
def test_api_file_extract_low_confidence_stage_confirm_writes(client_factory):
client, store, _ = client_factory()
product = _demo_product(store.data)
payload = base64.b64encode(_orders_csv(product["code"], ok=2, bad=5)).decode("ascii")
extract = client.post("/api/multimodal/extract", json={
"kind": "file_import", "filename": "orders.csv", "dataBase64": payload})
assert extract.status_code == 200, extract.text
cands = extract.json()["candidates"]
assert cands[0]["confidence"] < _THRESHOLD
assert cands[0]["complete"] is True
before_so = len(store.data["salesOrders"])
before_flex = len(store.data["flexOrders"])
ingest = client.post("/api/multimodal/ingest", json={
"sessionId": "web", "candidates": cands, "threshold": _THRESHOLD})
assert ingest.status_code == 200, ingest.text
body = ingest.json()
assert body["status"] == "staged"
cid = body["confirmId"]
assert len(store.data["salesOrders"]) == before_so # 出卡不写世界
assert len(store.data["flexOrders"]) == before_flex
assert any(e["action"] == "multimodal.stage" for e in store.data["auditEvents"])
confirm = client.post("/api/actions/confirm", json={
"confirmId": cid, "approve": True, "note": "确认低置信文件"})
assert confirm.status_code == 200, confirm.text
assert len(store.data["salesOrders"]) == before_so
assert len(store.data["flexOrders"]) == before_flex + 2 # 确认后写世界
assert any(e["action"] == "import.commit" for e in store.data["auditEvents"])
assert not any(e["action"] == "multimodal.apply" for e in store.data["auditEvents"])
def test_api_high_confidence_direct_apply(client_factory):
client, store, _ = client_factory()
resp = client.post("/api/multimodal/extract", json={
"kind": "text_order",
"text": "订单 单号 SO-HI-1 数量 9 交期 2026-08-20 客户 华东公司 产品编码 CTRL-A"})
cands = resp.json()["candidates"]
assert cands[0]["confidence"] == pytest.approx(1.0)
before = len(store.data["flexOrders"])
ingest = client.post("/api/multimodal/ingest", json={"candidates": cands})
assert ingest.status_code == 200, ingest.text
body = ingest.json()
assert body["status"] == "applied"
assert len(store.data["flexOrders"]) == before + 1
assert any(o["orderNo"] == "SO-HI-1" for o in store.data["flexOrders"])
assert any(e["action"] == "multimodal.apply" for e in store.data["auditEvents"])
assert not any(e["action"] == "multimodal.stage" for e in store.data["auditEvents"])
def test_api_reject_does_not_write(client_factory):
client, store, _ = client_factory()
product = _demo_product(store.data)
payload = base64.b64encode(_orders_csv(product["code"], ok=1, bad=4)).decode("ascii")
extract = client.post("/api/multimodal/extract", json={
"kind": "file_import", "filename": "orders.csv", "dataBase64": payload})
cands = extract.json()["candidates"]
ingest = client.post("/api/multimodal/ingest", json={"candidates": cands})
assert ingest.json()["status"] == "staged"
before_so = len(store.data["salesOrders"])
reject = client.post("/api/actions/confirm", json={
"confirmId": ingest.json()["confirmId"], "approve": False, "note": "数据不对"})
assert reject.status_code == 200, reject.text
assert "已驳回" in reject.json()["message"]
assert len(store.data["salesOrders"]) == before_so # 拒绝不写
assert any(e["action"] == "import.commit.reject" for e in store.data["auditEvents"])
assert not any(e["action"] == "multimodal.apply" for e in store.data["auditEvents"])
def test_api_incomplete_candidate_rejected_without_card(client_factory):
client, store, _ = client_factory()
extract = client.post("/api/multimodal/extract", json={
"kind": "text_order", "text": "订单 单号 SO-INC-1"}) # 缺数量
cands = extract.json()["candidates"]
assert cands[0]["complete"] is False
before = len(store.data["flexOrders"]) + len(store.data["salesOrders"])
ingest = client.post("/api/multimodal/ingest", json={"candidates": cands})
assert ingest.status_code == 200, ingest.text
assert ingest.json()["status"] == "incomplete"
assert len(store.data["flexOrders"]) + len(store.data["salesOrders"]) == before
assert not any(e["action"] == "multimodal.stage" for e in store.data["auditEvents"])
assert not any(e["action"] == "multimodal.apply" for e in store.data["auditEvents"])
def test_api_threshold_override_changes_decision(client_factory):
client, store, _ = client_factory()
resp = client.post("/api/multimodal/extract", json={
"kind": "text_order", "text": "订单 单号 SO-TH-1 数量 4"}) # 0.6 置信
cands = resp.json()["candidates"]
assert cands[0]["confidence"] == pytest.approx(0.6)
before = len(store.data["flexOrders"])
# 默认 0.7 → 低置信出卡
staged = client.post("/api/multimodal/ingest", json={"candidates": cands})
assert staged.json()["status"] == "staged"
assert len(store.data["flexOrders"]) == before
# 阈值放宽 0.5 → 同一候选直通
applied = client.post("/api/multimodal/ingest", json={
"candidates": cands, "threshold": 0.5})
assert applied.json()["status"] == "applied"
assert len(store.data["flexOrders"]) == before + 1
def test_api_audit_trail_complete_for_low_confidence_flow(client_factory):
"""审计齐全:extract → stage → 确认后 apply(import.commit 通道)→ 拒绝留痕。"""
client, store, _ = client_factory()
product = _demo_product(store.data)
payload = base64.b64encode(_orders_csv(product["code"], ok=1, bad=3)).decode("ascii")
extract = client.post("/api/multimodal/extract", json={
"kind": "file_import", "filename": "orders.csv", "dataBase64": payload})
cands = extract.json()["candidates"]
ingest = client.post("/api/multimodal/ingest", json={"candidates": cands})
assert ingest.json()["status"] == "staged"
confirm = client.post("/api/actions/confirm", json={
"confirmId": ingest.json()["confirmId"], "approve": True})
assert confirm.status_code == 200, confirm.text
actions = [e["action"] for e in store.data["auditEvents"]]
assert "multimodal.extract" in actions
assert "multimodal.stage" in actions
assert "import.commit" in actions
stage = next(e for e in store.data["auditEvents"] if e["action"] == "multimodal.stage")
assert stage["rationale"]["confirmId"] == ingest.json()["confirmId"]
assert stage["rationale"]["candidates"][0]["confidence"] < _THRESHOLD
commit = next(e for e in store.data["auditEvents"] if e["action"] == "import.commit")
assert commit["beforeSnapshot"] is not None