aps-agent/tests/golden/test_approval_migration.py

# ============================================================
# 审批文件→DB 存量迁移黄金测试（矩阵 117 行剩余项 / round-67 R67-B1）
# 覆盖：幂等迁移、dry_run、缺文件空源、后端/停写前置、digest 归一化、
# 损坏 fail-closed、身份冲突、单调状态、legacy raw-token 修复、
# canonical event digest、源漂移/后段故障整体回滚、迁移后 P3 grant 消费一次。
# ============================================================
from __future__ import annotations

import json
from pathlib import Path

import pytest
from sqlalchemy import func, select

from server.agent_core import approval_migrate
from server.agent_core.approval_migrate import MigrationError, migrate_file_to_db
from server.agent_core.approval_store import token_digest
from server.db.database import get_session, reset_engine
from server.db.models import (
    ApprovalEventRecord,
    ApprovalGrantRecord,
    ApprovalRequestRecord,
)

_RAW_TOKEN_1 = "c0ffee00" * 4            # 32 位 raw token（文件后端 grant 键形态）
_RAW_TOKEN_2 = "deadbee5" * 4


@pytest.fixture
def database_backend(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
    monkeypatch.setenv("APS_APPROVAL_BACKEND", "database")
    monkeypatch.setenv("APS_APPROVAL_MIGRATION_QUIESCED", "1")
    monkeypatch.delenv("APS_APPROVAL_PATH", raising=False)
    monkeypatch.setenv("APS_DATABASE_URL", f"sqlite:///{(tmp_path / 'approval.db').as_posix()}")
    monkeypatch.setenv("APS_APPROVAL_DATABASE_ALLOW_SQLITE", "1")
    reset_engine()
    from server.agent_core import harness
    store = harness.configure_approval_store()
    yield store
    store.clear()
    reset_engine()


def _file_store() -> dict:
    """合成文件后端数据：2 pending + 2 grants + 3 history 事件。"""
    pending = {
        "abc1111111": {
            "confirmId": "abc1111111", "sessionId": "s1", "action": "mes.dispatch",
            "power": "P3", "params": {"versionId": 1}, "paramsHash": "h1",
            "tenantUuid": "platform", "ownerUserId": 1001, "projectId": "p1",
            "worldKey": "w1", "evidenceRefs": ["schedule-version:1"],
            "createdAtEpoch": 1750000000.0, "expiresAtEpoch": 1750003600.0,
        },
        "abc2222222": {
            "confirmId": "abc2222222", "sessionId": "s1", "action": "schedule.publish",
            "power": "P2", "params": {}, "paramsHash": "h2",
            "tenantUuid": "platform", "ownerUserId": 1002, "projectId": "p1",
            "createdAtEpoch": 1750000100.0, "expiresAtEpoch": 1750003700.0,
        },
    }
    grants = {
        _RAW_TOKEN_1: {
            "confirmId": "abc1111111", "action": "mes.dispatch", "power": "P3",
            "tenantUuid": "platform", "projectId": "p1", "ownerUserId": 1001,
            "paramsHash": "h1", "status": "ACTIVE",
            "approvals": [{"userId": 1001}, {"userId": 1002}],
            "createdAtEpoch": 1750000000.0, "expiresAtEpoch": 1750003600.0,
        },
        _RAW_TOKEN_2: {
            "confirmId": "abc2222222", "action": "schedule.publish", "power": "P2",
            "tenantUuid": "platform", "projectId": "p1", "ownerUserId": 1002,
            "paramsHash": "h2", "status": "CONSUMED",
            "createdAtEpoch": 1750000100.0, "expiresAtEpoch": 1750003700.0,
            "consumedAtEpoch": 1750000200.0,
        },
    }
    history = [
        {"confirmId": "abc1111111", "action": "mes.dispatch", "power": "P3",
         "status": "APPROVED", "tenantUuid": "platform", "projectId": "p1",
         "ownerUserId": 1001, "decidedAtEpoch": 1750000100.0},
        {"confirmId": "abc2222222", "action": "schedule.publish", "power": "P2",
         "status": "APPROVED", "tenantUuid": "platform", "projectId": "p1",
         "ownerUserId": 1002, "decidedAtEpoch": 1750000200.0},
        {"confirmId": "abc2222222", "action": "schedule.publish", "power": "P2",
         "status": "CONSUMED", "tenantUuid": "platform", "projectId": "p1",
         "ownerUserId": 1002, "decidedAtEpoch": 1750000300.0},
    ]
    return {"schemaVersion": 1, "pending": pending, "grants": grants, "history": history}


def _write(tmp_path: Path, data: dict) -> Path:
    path = tmp_path / "approvals.json"
    path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8")
    return path


def _write_file_store(tmp_path: Path) -> Path:
    return _write(tmp_path, _file_store())


def _counts() -> dict:
    with get_session() as session:
        return {
            "pending": session.execute(select(func.count()).select_from(ApprovalRequestRecord)).scalar(),
            "grants": session.execute(select(func.count()).select_from(ApprovalGrantRecord)).scalar(),
            "events": session.execute(select(func.count()).select_from(ApprovalEventRecord)).scalar(),
        }


def test_migrate_file_to_db_idempotent(database_backend, tmp_path: Path):
    """迁移全量 + 幂等：二次运行全部 skipped，不重复插入。"""
    path = _write_file_store(tmp_path)
    first = migrate_file_to_db(str(path))
    assert first["pending"] == {"migrated": 2, "skipped": 0, "rebuilt": 0}
    assert first["grants"] == {"migrated": 2, "skipped": 0, "repaired": 0}
    assert first["events"] == {"migrated": 3, "skipped": 0}
    assert first["dryRun"] is False
    assert first["sourceDigest"]
    assert _counts() == {"pending": 2, "grants": 2, "events": 3}

    second = migrate_file_to_db(str(path))
    assert second["pending"] == {"migrated": 0, "skipped": 2, "rebuilt": 0}
    assert second["grants"] == {"migrated": 0, "skipped": 2, "repaired": 0}
    assert second["events"] == {"migrated": 0, "skipped": 3}
    assert _counts() == {"pending": 2, "grants": 2, "events": 3}


def test_migrate_dry_run_does_not_touch_db(database_backend, tmp_path: Path):
    """dry_run：报告数量但库内零写入。"""
    path = _write_file_store(tmp_path)
    report = migrate_file_to_db(str(path), dry_run=True)
    assert report["dryRun"] is True
    assert report["pending"]["migrated"] == 2
    assert report["grants"]["migrated"] == 2
    assert report["events"]["migrated"] == 3
    assert _counts() == {"pending": 0, "grants": 0, "events": 0}


def test_migrate_missing_file_empty_source(database_backend, tmp_path: Path):
    """缺文件 → 空源，全部 0，不抛错。"""
    report = migrate_file_to_db(str(tmp_path / "missing.json"))
    assert report["pending"] == {"migrated": 0, "skipped": 0, "rebuilt": 0}
    assert report["grants"] == {"migrated": 0, "skipped": 0, "repaired": 0}
    assert report["events"] == {"migrated": 0, "skipped": 0}
    assert report["sourceDigest"] is None


def test_migrate_requires_database_backend(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
    """未配置 database 后端 → 显式失败（fail closed）。"""
    monkeypatch.delenv("APS_APPROVAL_BACKEND", raising=False)
    monkeypatch.delenv("APS_APPROVAL_PATH", raising=False)
    with pytest.raises(MigrationError, match="APS_APPROVAL_BACKEND=database"):
        migrate_file_to_db(str(tmp_path / "approvals.json"))


def test_migrate_requires_quiesced(database_backend, tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
    """未显式停写（QUIESCED!=1）→ 明确拒绝，库表不变。"""
    monkeypatch.delenv("APS_APPROVAL_MIGRATION_QUIESCED")
    path = _write_file_store(tmp_path)
    with pytest.raises(MigrationError, match="QUIESCED"):
        migrate_file_to_db(str(path))
    assert _counts() == {"pending": 0, "grants": 0, "events": 0}


def test_migrated_data_readable_via_db_store(database_backend, tmp_path: Path):
    """迁移后 pending/历史可通过 database 后端读回（数据真实生效）。"""
    path = _write_file_store(tmp_path)
    migrate_file_to_db(str(path))
    now = 1749990000.0                      # 合成数据时间窗口内（expiresAt 1750003600 之前）
    pending_items = database_backend.pending_items(
        tenant_uuid="platform", project_id="p1", allowed=lambda _payload: True, now_epoch=now)
    assert len(pending_items) == 2
    assert any(item["confirmId"] == "abc1111111" for item in pending_items)
    history = database_backend.history_items(
        tenant_uuid="platform", project_id="p1", allowed=lambda _payload: True,
        limit=50, now_epoch=now)
    assert len(history) == 3
    assert all(item["status"] in ("APPROVED", "CONSUMED") for item in history)


def test_db_persists_only_digests(database_backend, tmp_path: Path):
    """DB 仅含 64 位 SHA-256 digest，不包含 raw token。"""
    path = _write_file_store(tmp_path)
    migrate_file_to_db(str(path))
    with get_session() as session:
        rows = list(session.execute(select(ApprovalGrantRecord)).scalars())
    assert len(rows) == 2
    hashes = {row.token_hash for row in rows}
    assert hashes == {token_digest(_RAW_TOKEN_1), token_digest(_RAW_TOKEN_2)}
    assert all(len(row.token_hash) == 64 for row in rows)
    assert _RAW_TOKEN_1 not in hashes and _RAW_TOKEN_2 not in hashes


def test_migrated_p3_grant_consumable_once(database_backend, tmp_path: Path):
    """grant 无 pending 源记录 → 重建终态 APPROVED request；迁移后消费一次成功、重放失败。"""
    store = _file_store()
    grant = store["grants"][_RAW_TOKEN_1]
    store["pending"].pop("abc1111111")      # 真实场景：批准后 pending 已弹出，grant 仍在
    store["history"] = [e for e in store["history"] if e["confirmId"] != "abc1111111"]
    path = _write(tmp_path, store)
    report = migrate_file_to_db(str(path))
    assert report["pending"]["rebuilt"] == 1
    assert report["grants"]["migrated"] == 2
    with get_session() as session:
        rebuilt = session.get(ApprovalRequestRecord, "abc1111111")
        assert rebuilt is not None and rebuilt.status == "APPROVED"
        assert rebuilt.payload.get("migratedTerminalRequest") is True

    now = 1750001000.0
    ok = database_backend.consume_grant(
        _RAW_TOKEN_1, confirm_id=grant["confirmId"], action=grant["action"],
        params_hash=grant["paramsHash"], allowed=lambda _r: True,
        decided_by={"userId": 1001}, now_epoch=now)
    assert ok is True
    replay = database_backend.consume_grant(
        _RAW_TOKEN_1, confirm_id=grant["confirmId"], action=grant["action"],
        params_hash=grant["paramsHash"], allowed=lambda _r: True,
        decided_by={"userId": 1001}, now_epoch=now)
    assert replay is False


def test_corrupt_json_fails_closed(database_backend, tmp_path: Path):
    """损坏 JSON → 明确失败，三张审批表不变。"""
    path = tmp_path / "approvals.json"
    path.write_text("{not-json", encoding="utf-8")
    with pytest.raises(MigrationError, match="审批源损坏"):
        migrate_file_to_db(str(path))
    assert _counts() == {"pending": 0, "grants": 0, "events": 0}


def test_invalid_schema_and_records_fail_closed(database_backend, tmp_path: Path):
    """错误顶层 schema / 非法 pending / 非法 history → 全部 fail closed。"""
    bad_schema = _write(tmp_path, {"schemaVersion": 2, "pending": {}, "grants": {}, "history": []})
    with pytest.raises(MigrationError, match="schema"):
        migrate_file_to_db(str(bad_schema))

    bad_record = _file_store()
    bad_record["pending"]["abc1111111"].pop("expiresAtEpoch")
    with pytest.raises(MigrationError, match="expiresAtEpoch"):
        migrate_file_to_db(str(_write(tmp_path, bad_record)))

    bad_history = _file_store()
    bad_history["history"] = [{"action": "mes.dispatch"}]          # 缺 status/decidedAtEpoch
    with pytest.raises(MigrationError, match="history"):
        migrate_file_to_db(str(_write(tmp_path, bad_history)))
    assert _counts() == {"pending": 0, "grants": 0, "events": 0}


def test_identity_conflict_fails_closed(database_backend, tmp_path: Path):
    """同 confirmId 不同不可变身份字段 → 冲突拒绝，已迁移数据不被覆盖。"""
    path = _write_file_store(tmp_path)
    migrate_file_to_db(str(path))
    tampered = _file_store()
    tampered["pending"]["abc1111111"]["paramsHash"] = "hX"
    with pytest.raises(MigrationError, match="身份字段不一致"):
        migrate_file_to_db(str(_write(tmp_path, tampered)))
    assert _counts() == {"pending": 2, "grants": 2, "events": 3}
    with get_session() as session:
        row = session.get(ApprovalRequestRecord, "abc1111111")
        assert row.params_hash == "h1"                            # 未被覆盖


def test_monotonic_status_no_regression(database_backend, tmp_path: Path):
    """DB 端已前进到 CONSUMED 的 grant 不被源端 ACTIVE 回退。"""
    path = _write_file_store(tmp_path)
    migrate_file_to_db(str(path))
    now = 1750001000.0
    assert database_backend.consume_grant(
        _RAW_TOKEN_1, confirm_id="abc1111111", action="mes.dispatch",
        params_hash="h1", allowed=lambda _r: True,
        decided_by={"userId": 1001}, now_epoch=now) is True
    second = migrate_file_to_db(str(path))                      # 源文件仍是 ACTIVE
    assert second["grants"]["skipped"] == 2
    with get_session() as session:
        row = session.get(ApprovalGrantRecord, token_digest(_RAW_TOKEN_1))
        assert row.status == "CONSUMED"                         # 未回退


def test_source_ahead_advances_db(database_backend, tmp_path: Path):
    """源端 CONSUMED、DB 端 ACTIVE → 迁移单调前进 DB 状态。"""
    store = _file_store()
    store["grants"].pop(_RAW_TOKEN_2)
    path = _write(tmp_path, store)
    migrate_file_to_db(str(path))
    advanced = _file_store()
    advanced["grants"].pop(_RAW_TOKEN_2)
    advanced["grants"][_RAW_TOKEN_1]["status"] = "CONSUMED"
    advanced["grants"][_RAW_TOKEN_1]["consumedAtEpoch"] = 1750000500.0
    advanced["pending"].pop("abc2222222")                       # 避免无关变更
    advanced["grants"].pop("unused", None)
    report = migrate_file_to_db(str(_write(tmp_path, advanced)))
    assert report["grants"]["migrated"] == 1
    with get_session() as session:
        row = session.get(ApprovalGrantRecord, token_digest(_RAW_TOKEN_1))
        assert row.status == "CONSUMED"
        assert row.consumed_at_epoch == 1750000500.0


def _insert_legacy_raw_token_row(record: dict, raw_token: str) -> None:
    """模拟旧版缺陷迁移产物：token_hash 直接落 raw token。"""
    with get_session() as session:
        if session.get(ApprovalRequestRecord, record["confirmId"]) is None:
            session.add(ApprovalRequestRecord(
                confirm_id=record["confirmId"], tenant_uuid=record["tenantUuid"],
                project_id=record["projectId"], owner_user_id=record["ownerUserId"],
                action=record["action"], power=record["power"],
                params_hash=record["paramsHash"], status="APPROVED",
                approval_step=2, required_approvals=2,
                created_at_epoch=record["createdAtEpoch"],
                expires_at_epoch=record["expiresAtEpoch"], revision=0, payload=dict(record),
            ))
        session.add(ApprovalGrantRecord(
            token_hash=raw_token, confirm_id=record["confirmId"],
            tenant_uuid=record["tenantUuid"], project_id=record["projectId"],
            owner_user_id=record["ownerUserId"], action=record["action"],
            params_hash=record["paramsHash"], status="ACTIVE",
            created_at_epoch=record["createdAtEpoch"],
            expires_at_epoch=record["expiresAtEpoch"], consumed_at_epoch=None,
            revision=0, payload=dict(record),
        ))
        session.commit()


def test_legacy_raw_token_row_repaired(database_backend, tmp_path: Path):
    """一致的 legacy raw-token 行在事务内原子修复为 digest，修复后可消费。"""
    store = _file_store()
    grant = store["grants"][_RAW_TOKEN_1]
    store["pending"].pop("abc1111111")      # 批准后 pending 已弹出，grant 仍在源中
    store["history"] = [e for e in store["history"] if e["confirmId"] != "abc1111111"]
    _insert_legacy_raw_token_row(dict(grant), _RAW_TOKEN_1)
    report = migrate_file_to_db(str(_write(tmp_path, store)))
    assert report["grants"]["repaired"] == 1
    with get_session() as session:
        assert session.get(ApprovalGrantRecord, _RAW_TOKEN_1) is None
        repaired = session.get(ApprovalGrantRecord, token_digest(_RAW_TOKEN_1))
        assert repaired is not None and repaired.status == "ACTIVE"
    assert database_backend.consume_grant(
        _RAW_TOKEN_1, confirm_id=grant["confirmId"], action=grant["action"],
        params_hash=grant["paramsHash"], allowed=lambda _r: True,
        decided_by={"userId": 1001}, now_epoch=1750001000.0) is True


def test_legacy_raw_token_row_conflict_fails_closed(database_backend, tmp_path: Path):
    """不一致的 legacy raw-token 行 → fail-closed，legacy 行不被改写。"""
    store = _file_store()
    store["pending"].pop("abc1111111")                          # 聚焦 grant 冲突路径
    grant = dict(store["grants"][_RAW_TOKEN_1])
    _insert_legacy_raw_token_row(grant, _RAW_TOKEN_1)           # request 行身份一致
    with get_session() as session:                              # legacy grant 行身份不一致
        row = session.get(ApprovalGrantRecord, _RAW_TOKEN_1)
        row.action = "mes.dispatch.v2"
        session.commit()
    with pytest.raises(MigrationError, match="legacy"):
        migrate_file_to_db(str(_write(tmp_path, store)))
    with get_session() as session:
        row = session.get(ApprovalGrantRecord, _RAW_TOKEN_1)
        assert row is not None and row.action == "mes.dispatch.v2"   # 原样保留
        assert session.get(ApprovalGrantRecord, token_digest(_RAW_TOKEN_1)) is None


def test_known_digest_key_never_rehashed(database_backend, tmp_path: Path):
    """文件键本身已是 DB 已知 digest（64 位 hex）→ 按已迁移跳过，禁止再哈希。"""
    raw = "f" * 32
    digest = token_digest(raw)                                  # 64 位 hex
    store = _file_store()
    store["pending"] = {}
    store["history"] = []
    grant = {
        "confirmId": "digestkey01", "action": "mes.dispatch", "power": "P3",
        "tenantUuid": "platform", "projectId": "p1", "ownerUserId": 1001,
        "paramsHash": "hd", "status": "ACTIVE",
        "approvals": [{"userId": 1001}, {"userId": 1002}],
        "createdAtEpoch": 1750000000.0, "expiresAtEpoch": 1750003600.0,
    }
    store["grants"] = {digest: grant}                           # 源键已是 digest 形态
    with get_session() as session:                              # DB 已有该 digest 行
        session.add(ApprovalRequestRecord(
            confirm_id="digestkey01", tenant_uuid="platform", project_id="p1",
            owner_user_id=1001, action="mes.dispatch", power="P3",
            params_hash="hd", status="APPROVED", approval_step=2, required_approvals=2,
            created_at_epoch=1750000000.0, expires_at_epoch=1750003600.0,
            revision=0, payload=dict(grant),
        ))
        session.add(ApprovalGrantRecord(
            token_hash=digest, confirm_id="digestkey01", tenant_uuid="platform",
            project_id="p1", owner_user_id=1001, action="mes.dispatch",
            params_hash="hd", status="ACTIVE", created_at_epoch=1750000000.0,
            expires_at_epoch=1750003600.0, consumed_at_epoch=None,
            revision=0, payload=dict(grant),
        ))
        session.commit()
    report = migrate_file_to_db(str(_write(tmp_path, store)))
    assert report["grants"] == {"migrated": 0, "skipped": 1, "repaired": 0}
    with get_session() as session:
        assert session.get(ApprovalGrantRecord, token_digest(digest)) is None   # 未再哈希
        assert session.get(ApprovalGrantRecord, digest) is not None


def test_history_canonical_digest_preserves_near_duplicates(database_backend, tmp_path: Path):
    """同秒同状态但 note 不同的事件全部保留；完全重复事件幂等去重。"""
    store = _file_store()
    base = dict(store["history"][0])
    twin = dict(base, note="second approver comment")           # 同秒同状态不同 note
    store["history"] = [base, twin, dict(base)]                 # 第 3 条与第 1 条完全重复
    path = _write(tmp_path, store)
    first = migrate_file_to_db(str(path))
    assert first["events"] == {"migrated": 2, "skipped": 1}
    second = migrate_file_to_db(str(path))
    assert second["events"] == {"migrated": 0, "skipped": 3}
    assert _counts()["events"] == 2


def test_source_drift_rolls_back(database_backend, tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
    """锁持有期间源文件 digest 漂移 → 整体回滚，三表不变。"""
    path = _write_file_store(tmp_path)
    original = approval_migrate._sha256_path
    calls = {"n": 0}

    def drifting(p: str) -> str:
        calls["n"] += 1
        return original(p) if calls["n"] == 1 else "0" * 64

    monkeypatch.setattr(approval_migrate, "_sha256_path", drifting)
    with pytest.raises(MigrationError, match="源文件发生变化"):
        migrate_file_to_db(str(path))
    assert _counts() == {"pending": 0, "grants": 0, "events": 0}


def test_late_failure_rolls_back_all(database_backend, tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
    """后段事件构造故障注入 → 请求/grant/事件三表整体回滚。"""
    path = _write_file_store(tmp_path)
    original = approval_migrate._canonical_event_digest
    calls = {"n": 0}

    def failing(event: dict) -> str:
        calls["n"] += 1
        if calls["n"] == 2:
            raise RuntimeError("injected digest failure")
        return original(event)

    monkeypatch.setattr(approval_migrate, "_canonical_event_digest", failing)
    with pytest.raises(RuntimeError, match="injected"):
        migrate_file_to_db(str(path))
    assert _counts() == {"pending": 0, "grants": 0, "events": 0}