aps-agent/tests/golden/test_approval_database_clock.py

from __future__ import annotations

import threading
from pathlib import Path
from types import SimpleNamespace
from typing import Any
from unittest.mock import Mock

import pytest
from alembic import command
from alembic.config import Config
from sqlalchemy import create_engine, select
from sqlalchemy.dialects import mysql
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from sqlalchemy.schema import CreateTable

from server.agent_core import harness
from server.agent_core.approval_db_store import DatabaseApprovalStore
from server.agent_core.approval_store import get_approval_store, set_approval_store
from server.db.database import get_session, reset_engine
from server.db.models import (
    ApprovalEventRecord,
    ApprovalGrantRecord,
    ApprovalRequestRecord,
)


def _record(
    confirm_id: str,
    *,
    created_at: float = 1_000.0,
    expires_at: float = 1_010.0,
    required_approvals: int = 1,
) -> dict[str, Any]:
    return {
        "confirmId": confirm_id,
        "tenantUuid": "platform",
        "projectId": "default",
        "ownerUserId": 7001,
        "action": "mes.dispatch",
        "power": "P3",
        "paramsHash": "clock-contract-hash",
        "approvalStep": 0,
        "requiredApprovals": required_approvals,
        "approvals": [],
        "createdAtEpoch": created_at,
        "expiresAtEpoch": expires_at,
    }


def _approver(user_id: int) -> dict[str, Any]:
    return {
        "userId": user_id,
        "username": f"clock-user-{user_id}",
        "displayName": f"Clock User {user_id}",
        "tenantUuid": "platform",
    }


def _allow(_record: dict[str, Any]) -> bool:
    return True


@pytest.fixture
def database_store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch):
    previous_store = get_approval_store()
    monkeypatch.setenv("APS_APPROVAL_BACKEND", "database")
    monkeypatch.delenv("APS_APPROVAL_PATH", raising=False)
    monkeypatch.setenv(
        "APS_DATABASE_URL",
        f"sqlite:///{(tmp_path / 'approval-clock.db').as_posix()}",
    )
    monkeypatch.setenv("APS_APPROVAL_DATABASE_ALLOW_SQLITE", "1")
    reset_engine()
    store = harness.configure_approval_store()
    assert isinstance(store, DatabaseApprovalStore)
    try:
        yield store
    finally:
        store.clear()
        reset_engine()
        set_approval_store(previous_store)


def _fake_session(dialect_name: str, *, scalar: float | None = None) -> Mock:
    session = Mock()
    session.get_bind.return_value = SimpleNamespace(
        dialect=SimpleNamespace(name=dialect_name)
    )
    if scalar is not None:
        result = Mock()
        result.scalar_one.return_value = scalar
        session.execute.return_value = result
    return session


def _issue_grant(
    store: DatabaseApprovalStore,
    *,
    confirm_id: str,
    request_now: float = 2_000.0,
    grant_ttl_seconds: int = 10,
) -> tuple[str, float]:
    assert store.stage(
        _record(confirm_id, expires_at=1_100.0, required_approvals=2),
        now_epoch=request_now,
    )
    first = store.decide(
        confirm_id,
        approve=True,
        approver=_approver(7001),
        allowed=_allow,
        now_epoch=request_now + 1,
        grant_ttl_seconds=grant_ttl_seconds,
    )
    assert first and first["needsSecondConfirm"] is True
    second = store.decide(
        confirm_id,
        approve=True,
        approver=_approver(7002),
        allowed=_allow,
        now_epoch=request_now + 2,
        grant_ttl_seconds=grant_ttl_seconds,
    )
    assert second and second.get("executionGrant")
    grant = str(second["executionGrant"])
    with get_session() as session:
        row = session.scalar(select(ApprovalGrantRecord))
        assert row is not None
        return grant, float(row.expires_at_epoch)


def test_mysql_transaction_clock_ignores_malicious_supplied_epoch() -> None:
    session = _fake_session("mysql", scalar=1_785_840_000.125)

    actual = DatabaseApprovalStore._transaction_now_epoch(
        session,
        supplied_now_epoch=-9_999_999_999.0,
    )

    assert actual == pytest.approx(1_785_840_000.125)
    statement = session.execute.call_args.args[0]
    assert str(statement) == "SELECT UNIX_TIMESTAMP(CURRENT_TIMESTAMP(6))"


def test_sqlite_transaction_clock_preserves_test_injection() -> None:
    session = _fake_session("sqlite")

    actual = DatabaseApprovalStore._transaction_now_epoch(
        session,
        supplied_now_epoch=1234.56789,
    )

    assert actual == pytest.approx(1234.56789)
    session.execute.assert_not_called()


def test_mysql_database_clock_failure_is_fail_closed() -> None:
    session = _fake_session("mysql")
    session.execute.side_effect = OSError("database clock unavailable")

    with pytest.raises(RuntimeError, match="database clock is unavailable"):
        DatabaseApprovalStore._transaction_now_epoch(
            session,
            supplied_now_epoch=9_999_999_999.0,
        )


def test_stage_reanchors_request_ttl_to_transaction_clock(
    database_store: DatabaseApprovalStore,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    monkeypatch.setattr(
        database_store,
        "_transaction_now_epoch",
        lambda _session, supplied_now_epoch=None: 5_000.0,
    )

    assert database_store.stage(
        _record("clock-stage-reanchor", created_at=1_000.0, expires_at=1_010.0),
        now_epoch=9_999_999_999.0,
    )

    with get_session() as session:
        row = session.get(ApprovalRequestRecord, "clock-stage-reanchor")
    assert row is not None
    assert row.created_at_epoch == pytest.approx(5_000.0)
    assert row.expires_at_epoch == pytest.approx(5_010.0)
    assert row.payload["createdAtEpoch"] == pytest.approx(5_000.0)
    assert row.payload["expiresAtEpoch"] == pytest.approx(5_010.0)


def test_decide_uses_transaction_clock_instead_of_fast_application_clock(
    database_store: DatabaseApprovalStore,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    confirm_id = "clock-decide-authority"
    assert database_store.stage(_record(confirm_id), now_epoch=2_000.0)
    monkeypatch.setattr(
        database_store,
        "_transaction_now_epoch",
        lambda _session, supplied_now_epoch=None: 2_005.0,
    )

    approved = database_store.decide(
        confirm_id,
        approve=True,
        approver=_approver(7001),
        allowed=_allow,
        now_epoch=9_999_999_999.0,
        grant_ttl_seconds=60,
    )

    assert approved is not None
    with get_session() as session:
        row = session.get(ApprovalRequestRecord, confirm_id)
    assert row is not None and row.status == "APPROVED"


def test_consume_uses_transaction_clock_instead_of_fast_application_clock(
    database_store: DatabaseApprovalStore,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    confirm_id = "clock-consume-authority"
    grant, _expires_at = _issue_grant(database_store, confirm_id=confirm_id)
    monkeypatch.setattr(
        database_store,
        "_transaction_now_epoch",
        lambda _session, supplied_now_epoch=None: 2_005.0,
    )

    assert database_store.consume_grant(
        grant,
        confirm_id=confirm_id,
        action="mes.dispatch",
        params_hash="clock-contract-hash",
        allowed=_allow,
        decided_by=_approver(7003),
        now_epoch=9_999_999_999.0,
    )


def test_slow_application_clock_cannot_consume_after_transaction_expiry(
    database_store: DatabaseApprovalStore,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    confirm_id = "clock-consume-expired-authority"
    grant, expires_at = _issue_grant(database_store, confirm_id=confirm_id)
    monkeypatch.setattr(
        database_store,
        "_transaction_now_epoch",
        lambda _session, supplied_now_epoch=None: expires_at,
    )

    assert not database_store.consume_grant(
        grant,
        confirm_id=confirm_id,
        action="mes.dispatch",
        params_hash="clock-contract-hash",
        allowed=_allow,
        decided_by=_approver(7003),
        now_epoch=-9_999_999_999.0,
    )


def test_stage_rejects_incomplete_absolute_time_payload(
    database_store: DatabaseApprovalStore,
) -> None:
    missing_created = _record("clock-missing-created")
    missing_created.pop("createdAtEpoch")
    missing_expires = _record("clock-missing-expires")
    missing_expires.pop("expiresAtEpoch")

    assert not database_store.stage(missing_created, now_epoch=9_999_999_999.0)
    assert not database_store.stage(missing_expires, now_epoch=-9_999_999_999.0)


def test_final_approval_uses_one_canonical_transaction_time_across_evidence(
    database_store: DatabaseApprovalStore,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    confirm_id = "clock-canonical-evidence"
    assert database_store.stage(
        _record(confirm_id, required_approvals=2),
        now_epoch=2_000.0,
    )
    first = database_store.decide(
        confirm_id,
        approve=True,
        approver=_approver(7001),
        allowed=_allow,
        now_epoch=2_001.0,
        grant_ttl_seconds=60,
    )
    assert first and first["needsSecondConfirm"] is True
    monkeypatch.setattr(
        database_store,
        "_transaction_now_epoch",
        lambda _session, supplied_now_epoch=None: 2_005.125,
    )
    second = database_store.decide(
        confirm_id,
        approve=True,
        approver=_approver(7002),
        allowed=_allow,
        now_epoch=9_999_999_999.0,
        grant_ttl_seconds=60,
    )
    assert second and second.get("executionGrant")

    with get_session() as session:
        request = session.get(ApprovalRequestRecord, confirm_id)
        grant = session.scalar(
            select(ApprovalGrantRecord).where(
                ApprovalGrantRecord.confirm_id == confirm_id
            )
        )
        event = session.scalar(
            select(ApprovalEventRecord)
            .where(
                ApprovalEventRecord.confirm_id == confirm_id,
                ApprovalEventRecord.status == "APPROVED",
            )
            .order_by(ApprovalEventRecord.id.desc())
        )
    assert request is not None and grant is not None and event is not None
    request_time = request.payload["approvals"][-1]["approvedAt"]
    assert request_time == grant.payload["approvals"][-1]["approvedAt"]
    assert request_time == event.payload["approvals"][-1]["approvedAt"]
    assert grant.created_at_epoch == pytest.approx(event.decided_at_epoch)


def test_integrity_error_is_not_reported_as_cas_conflict(
    database_store: DatabaseApprovalStore,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    confirm_id = "clock-integrity-failure"
    assert database_store.stage(_record(confirm_id), now_epoch=2_000.0)
    monkeypatch.setattr(
        database_store,
        "_event_row",
        lambda *_args, **_kwargs: (_ for _ in ()).throw(
            IntegrityError("event", {}, RuntimeError("fk failure"))
        ),
    )

    with pytest.raises(RuntimeError, match="database integrity failure"):
        database_store.decide(
            confirm_id,
            approve=True,
            approver=_approver(7001),
            allowed=_allow,
            now_epoch=2_001.0,
            grant_ttl_seconds=60,
        )


def test_clock_precision_downgrade_refuses_active_state(
    tmp_path: Path,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    database_path = tmp_path / "approval-clock-downgrade.db"
    monkeypatch.delenv("APS_DATABASE_URL", raising=False)
    monkeypatch.setenv("APS_DB_PATH", str(database_path))
    reset_engine()
    config = Config("alembic.ini")
    command.upgrade(config, "20260804_01")
    engine = create_engine(f"sqlite:///{database_path.as_posix()}")
    with engine.begin() as connection:
        connection.execute(
            ApprovalRequestRecord.__table__.insert().values(
                confirm_id="active-downgrade",
                tenant_uuid="platform",
                project_id="default",
                owner_user_id=7001,
                action="mes.dispatch",
                power="P3",
                params_hash="downgrade-hash",
                status="PENDING",
                approval_step=0,
                required_approvals=2,
                created_at_epoch=2_000.0,
                expires_at_epoch=2_100.0,
                revision=0,
                payload=_record("active-downgrade", required_approvals=2),
            )
        )
    with pytest.raises(RuntimeError, match="active approval state exists"):
        command.downgrade(config, "20260731_03")
    with engine.begin() as connection:
        connection.execute(
            ApprovalRequestRecord.__table__.update()
            .where(ApprovalRequestRecord.confirm_id == "active-downgrade")
            .values(status="EXPIRED")
        )
    command.downgrade(config, "20260731_03")
    engine.dispose()
    reset_engine()


def test_request_expires_when_transaction_clock_equals_boundary(
    database_store: DatabaseApprovalStore,
) -> None:
    confirm_id = "clock-request-boundary"
    assert database_store.stage(_record(confirm_id), now_epoch=2_000.0)

    result = database_store.decide(
        confirm_id,
        approve=True,
        approver=_approver(7001),
        allowed=_allow,
        now_epoch=2_010.0,
        grant_ttl_seconds=60,
    )

    assert result is None
    with get_session() as session:
        row = session.get(ApprovalRequestRecord, confirm_id)
        statuses = list(session.scalars(select(ApprovalEventRecord.status)))
    assert row is not None and row.status == "EXPIRED"
    assert statuses.count("EXPIRED") == 1
    assert "APPROVED" not in statuses


def test_grant_cannot_be_consumed_when_clock_equals_boundary(
    database_store: DatabaseApprovalStore,
) -> None:
    confirm_id = "clock-grant-boundary"
    grant, expires_at = _issue_grant(database_store, confirm_id=confirm_id)

    consumed = database_store.consume_grant(
        grant,
        confirm_id=confirm_id,
        action="mes.dispatch",
        params_hash="clock-contract-hash",
        allowed=_allow,
        decided_by=_approver(7001),
        now_epoch=expires_at,
    )

    assert consumed is False
    with get_session() as session:
        row = session.scalar(select(ApprovalGrantRecord))
        statuses = list(session.scalars(select(ApprovalEventRecord.status)))
    assert row is not None and row.status == "EXPIRED"
    assert statuses.count("GRANT_EXPIRED") == 1
    assert "EXECUTED" not in statuses


def test_approve_and_expire_are_mutually_exclusive(
    database_store: DatabaseApprovalStore,
) -> None:
    confirm_id = "clock-request-race"
    assert database_store.stage(_record(confirm_id), now_epoch=2_000.0)
    barrier = threading.Barrier(2)
    outcomes: list[object] = []

    def approve() -> None:
        barrier.wait()
        outcomes.append(database_store.decide(
            confirm_id,
            approve=True,
            approver=_approver(7001),
            allowed=_allow,
            now_epoch=2_009.0,
            grant_ttl_seconds=60,
        ))

    def expire() -> None:
        barrier.wait()
        outcomes.append(database_store.expire(now_epoch=2_010.0))

    threads = [threading.Thread(target=approve), threading.Thread(target=expire)]
    for thread in threads:
        thread.start()
    for thread in threads:
        thread.join()

    with get_session() as session:
        row = session.get(ApprovalRequestRecord, confirm_id)
        statuses = list(session.scalars(select(ApprovalEventRecord.status)))
    assert len(outcomes) == 2
    assert row is not None and row.status in {"APPROVED", "EXPIRED"}
    assert statuses.count("APPROVED") + statuses.count("EXPIRED") == 1


def test_consume_and_expire_are_mutually_exclusive(
    database_store: DatabaseApprovalStore,
) -> None:
    confirm_id = "clock-grant-race"
    grant, expires_at = _issue_grant(database_store, confirm_id=confirm_id)
    barrier = threading.Barrier(2)
    outcomes: list[bool] = []

    def consume() -> None:
        barrier.wait()
        outcomes.append(database_store.consume_grant(
            grant,
            confirm_id=confirm_id,
            action="mes.dispatch",
            params_hash="clock-contract-hash",
            allowed=_allow,
            decided_by=_approver(7001),
            now_epoch=expires_at - 0.001,
        ))

    def expire() -> None:
        barrier.wait()
        outcomes.append(database_store.expire(now_epoch=expires_at))

    threads = [threading.Thread(target=consume), threading.Thread(target=expire)]
    for thread in threads:
        thread.start()
    for thread in threads:
        thread.join()

    with get_session() as session:
        row = session.scalar(select(ApprovalGrantRecord))
        statuses = list(session.scalars(select(ApprovalEventRecord.status)))
    assert len(outcomes) == 2
    assert row is not None and row.status in {"CONSUMED", "EXPIRED"}
    assert statuses.count("EXECUTED") + statuses.count("GRANT_EXPIRED") == 1


def test_event_failure_rolls_back_request_transition(
    database_store: DatabaseApprovalStore,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    confirm_id = "clock-event-rollback"
    assert database_store.stage(_record(confirm_id), now_epoch=2_000.0)

    def fail_event(*_args: object, **_kwargs: object) -> ApprovalEventRecord:
        raise RuntimeError("event insert failed")

    monkeypatch.setattr(database_store, "_event_row", fail_event)
    with pytest.raises(RuntimeError, match="event insert failed"):
        database_store.decide(
            confirm_id,
            approve=True,
            approver=_approver(7001),
            allowed=_allow,
            now_epoch=2_001.0,
            grant_ttl_seconds=60,
        )

    with get_session() as session:
        row = session.get(ApprovalRequestRecord, confirm_id)
        events = list(session.scalars(select(ApprovalEventRecord)))
    assert row is not None and row.status == "PENDING" and row.revision == 0
    assert events == []


def test_commit_failure_rolls_back_request_transition(
    database_store: DatabaseApprovalStore,
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    confirm_id = "clock-commit-rollback"
    assert database_store.stage(_record(confirm_id), now_epoch=2_000.0)
    original_commit = Session.commit

    def fail_commit(self: Session) -> None:
        raise RuntimeError("commit failed")

    monkeypatch.setattr(Session, "commit", fail_commit)
    with pytest.raises(RuntimeError, match="commit failed"):
        database_store.decide(
            confirm_id,
            approve=True,
            approver=_approver(7001),
            allowed=_allow,
            now_epoch=2_001.0,
            grant_ttl_seconds=60,
        )
    monkeypatch.setattr(Session, "commit", original_commit)

    with get_session() as session:
        row = session.get(ApprovalRequestRecord, confirm_id)
        events = list(session.scalars(select(ApprovalEventRecord)))
    assert row is not None and row.status == "PENDING" and row.revision == 0
    assert events == []


@pytest.mark.parametrize(
    ("table", "columns"),
    [
        (ApprovalRequestRecord.__table__, ("created_at_epoch", "expires_at_epoch")),
        (
            ApprovalGrantRecord.__table__,
            ("created_at_epoch", "expires_at_epoch", "consumed_at_epoch"),
        ),
        (ApprovalEventRecord.__table__, ("decided_at_epoch",)),
    ],
)
def test_mysql_epoch_columns_compile_as_double(table: Any, columns: tuple[str, ...]) -> None:
    ddl = str(CreateTable(table).compile(dialect=mysql.dialect())).upper()

    for column in columns:
        assert f"{column.upper()} DOUBLE" in ddl