from __future__ import annotations import threading from pathlib import Path from types import SimpleNamespace from typing import Any from unittest.mock import Mock import pytest from alembic import command from alembic.config import Config from sqlalchemy import create_engine, select from sqlalchemy.dialects import mysql from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session from sqlalchemy.schema import CreateTable from server.agent_core import harness from server.agent_core.approval_db_store import DatabaseApprovalStore from server.agent_core.approval_store import get_approval_store, set_approval_store from server.db.database import get_session, reset_engine from server.db.models import ( ApprovalEventRecord, ApprovalGrantRecord, ApprovalRequestRecord, ) def _record( confirm_id: str, *, created_at: float = 1_000.0, expires_at: float = 1_010.0, required_approvals: int = 1, ) -> dict[str, Any]: return { "confirmId": confirm_id, "tenantUuid": "platform", "projectId": "default", "ownerUserId": 7001, "action": "mes.dispatch", "power": "P3", "paramsHash": "clock-contract-hash", "approvalStep": 0, "requiredApprovals": required_approvals, "approvals": [], "createdAtEpoch": created_at, "expiresAtEpoch": expires_at, } def _approver(user_id: int) -> dict[str, Any]: return { "userId": user_id, "username": f"clock-user-{user_id}", "displayName": f"Clock User {user_id}", "tenantUuid": "platform", } def _allow(_record: dict[str, Any]) -> bool: return True @pytest.fixture def database_store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): previous_store = get_approval_store() monkeypatch.setenv("APS_APPROVAL_BACKEND", "database") monkeypatch.delenv("APS_APPROVAL_PATH", raising=False) monkeypatch.setenv( "APS_DATABASE_URL", f"sqlite:///{(tmp_path / 'approval-clock.db').as_posix()}", ) monkeypatch.setenv("APS_APPROVAL_DATABASE_ALLOW_SQLITE", "1") reset_engine() store = harness.configure_approval_store() assert isinstance(store, DatabaseApprovalStore) try: yield store finally: store.clear() reset_engine() set_approval_store(previous_store) def _fake_session(dialect_name: str, *, scalar: float | None = None) -> Mock: session = Mock() session.get_bind.return_value = SimpleNamespace( dialect=SimpleNamespace(name=dialect_name) ) if scalar is not None: result = Mock() result.scalar_one.return_value = scalar session.execute.return_value = result return session def _issue_grant( store: DatabaseApprovalStore, *, confirm_id: str, request_now: float = 2_000.0, grant_ttl_seconds: int = 10, ) -> tuple[str, float]: assert store.stage( _record(confirm_id, expires_at=1_100.0, required_approvals=2), now_epoch=request_now, ) first = store.decide( confirm_id, approve=True, approver=_approver(7001), allowed=_allow, now_epoch=request_now + 1, grant_ttl_seconds=grant_ttl_seconds, ) assert first and first["needsSecondConfirm"] is True second = store.decide( confirm_id, approve=True, approver=_approver(7002), allowed=_allow, now_epoch=request_now + 2, grant_ttl_seconds=grant_ttl_seconds, ) assert second and second.get("executionGrant") grant = str(second["executionGrant"]) with get_session() as session: row = session.scalar(select(ApprovalGrantRecord)) assert row is not None return grant, float(row.expires_at_epoch) def test_mysql_transaction_clock_ignores_malicious_supplied_epoch() -> None: session = _fake_session("mysql", scalar=1_785_840_000.125) actual = DatabaseApprovalStore._transaction_now_epoch( session, supplied_now_epoch=-9_999_999_999.0, ) assert actual == pytest.approx(1_785_840_000.125) statement = session.execute.call_args.args[0] assert str(statement) == "SELECT UNIX_TIMESTAMP(CURRENT_TIMESTAMP(6))" def test_sqlite_transaction_clock_preserves_test_injection() -> None: session = _fake_session("sqlite") actual = DatabaseApprovalStore._transaction_now_epoch( session, supplied_now_epoch=1234.56789, ) assert actual == pytest.approx(1234.56789) session.execute.assert_not_called() def test_mysql_database_clock_failure_is_fail_closed() -> None: session = _fake_session("mysql") session.execute.side_effect = OSError("database clock unavailable") with pytest.raises(RuntimeError, match="database clock is unavailable"): DatabaseApprovalStore._transaction_now_epoch( session, supplied_now_epoch=9_999_999_999.0, ) def test_stage_reanchors_request_ttl_to_transaction_clock( database_store: DatabaseApprovalStore, monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr( database_store, "_transaction_now_epoch", lambda _session, supplied_now_epoch=None: 5_000.0, ) assert database_store.stage( _record("clock-stage-reanchor", created_at=1_000.0, expires_at=1_010.0), now_epoch=9_999_999_999.0, ) with get_session() as session: row = session.get(ApprovalRequestRecord, "clock-stage-reanchor") assert row is not None assert row.created_at_epoch == pytest.approx(5_000.0) assert row.expires_at_epoch == pytest.approx(5_010.0) assert row.payload["createdAtEpoch"] == pytest.approx(5_000.0) assert row.payload["expiresAtEpoch"] == pytest.approx(5_010.0) def test_decide_uses_transaction_clock_instead_of_fast_application_clock( database_store: DatabaseApprovalStore, monkeypatch: pytest.MonkeyPatch, ) -> None: confirm_id = "clock-decide-authority" assert database_store.stage(_record(confirm_id), now_epoch=2_000.0) monkeypatch.setattr( database_store, "_transaction_now_epoch", lambda _session, supplied_now_epoch=None: 2_005.0, ) approved = database_store.decide( confirm_id, approve=True, approver=_approver(7001), allowed=_allow, now_epoch=9_999_999_999.0, grant_ttl_seconds=60, ) assert approved is not None with get_session() as session: row = session.get(ApprovalRequestRecord, confirm_id) assert row is not None and row.status == "APPROVED" def test_consume_uses_transaction_clock_instead_of_fast_application_clock( database_store: DatabaseApprovalStore, monkeypatch: pytest.MonkeyPatch, ) -> None: confirm_id = "clock-consume-authority" grant, _expires_at = _issue_grant(database_store, confirm_id=confirm_id) monkeypatch.setattr( database_store, "_transaction_now_epoch", lambda _session, supplied_now_epoch=None: 2_005.0, ) assert database_store.consume_grant( grant, confirm_id=confirm_id, action="mes.dispatch", params_hash="clock-contract-hash", allowed=_allow, decided_by=_approver(7003), now_epoch=9_999_999_999.0, ) def test_slow_application_clock_cannot_consume_after_transaction_expiry( database_store: DatabaseApprovalStore, monkeypatch: pytest.MonkeyPatch, ) -> None: confirm_id = "clock-consume-expired-authority" grant, expires_at = _issue_grant(database_store, confirm_id=confirm_id) monkeypatch.setattr( database_store, "_transaction_now_epoch", lambda _session, supplied_now_epoch=None: expires_at, ) assert not database_store.consume_grant( grant, confirm_id=confirm_id, action="mes.dispatch", params_hash="clock-contract-hash", allowed=_allow, decided_by=_approver(7003), now_epoch=-9_999_999_999.0, ) def test_stage_rejects_incomplete_absolute_time_payload( database_store: DatabaseApprovalStore, ) -> None: missing_created = _record("clock-missing-created") missing_created.pop("createdAtEpoch") missing_expires = _record("clock-missing-expires") missing_expires.pop("expiresAtEpoch") assert not database_store.stage(missing_created, now_epoch=9_999_999_999.0) assert not database_store.stage(missing_expires, now_epoch=-9_999_999_999.0) def test_final_approval_uses_one_canonical_transaction_time_across_evidence( database_store: DatabaseApprovalStore, monkeypatch: pytest.MonkeyPatch, ) -> None: confirm_id = "clock-canonical-evidence" assert database_store.stage( _record(confirm_id, required_approvals=2), now_epoch=2_000.0, ) first = database_store.decide( confirm_id, approve=True, approver=_approver(7001), allowed=_allow, now_epoch=2_001.0, grant_ttl_seconds=60, ) assert first and first["needsSecondConfirm"] is True monkeypatch.setattr( database_store, "_transaction_now_epoch", lambda _session, supplied_now_epoch=None: 2_005.125, ) second = database_store.decide( confirm_id, approve=True, approver=_approver(7002), allowed=_allow, now_epoch=9_999_999_999.0, grant_ttl_seconds=60, ) assert second and second.get("executionGrant") with get_session() as session: request = session.get(ApprovalRequestRecord, confirm_id) grant = session.scalar( select(ApprovalGrantRecord).where( ApprovalGrantRecord.confirm_id == confirm_id ) ) event = session.scalar( select(ApprovalEventRecord) .where( ApprovalEventRecord.confirm_id == confirm_id, ApprovalEventRecord.status == "APPROVED", ) .order_by(ApprovalEventRecord.id.desc()) ) assert request is not None and grant is not None and event is not None request_time = request.payload["approvals"][-1]["approvedAt"] assert request_time == grant.payload["approvals"][-1]["approvedAt"] assert request_time == event.payload["approvals"][-1]["approvedAt"] assert grant.created_at_epoch == pytest.approx(event.decided_at_epoch) def test_integrity_error_is_not_reported_as_cas_conflict( database_store: DatabaseApprovalStore, monkeypatch: pytest.MonkeyPatch, ) -> None: confirm_id = "clock-integrity-failure" assert database_store.stage(_record(confirm_id), now_epoch=2_000.0) monkeypatch.setattr( database_store, "_event_row", lambda *_args, **_kwargs: (_ for _ in ()).throw( IntegrityError("event", {}, RuntimeError("fk failure")) ), ) with pytest.raises(RuntimeError, match="database integrity failure"): database_store.decide( confirm_id, approve=True, approver=_approver(7001), allowed=_allow, now_epoch=2_001.0, grant_ttl_seconds=60, ) def test_clock_precision_downgrade_refuses_active_state( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: database_path = tmp_path / "approval-clock-downgrade.db" monkeypatch.delenv("APS_DATABASE_URL", raising=False) monkeypatch.setenv("APS_DB_PATH", str(database_path)) reset_engine() config = Config("alembic.ini") command.upgrade(config, "20260804_01") engine = create_engine(f"sqlite:///{database_path.as_posix()}") with engine.begin() as connection: connection.execute( ApprovalRequestRecord.__table__.insert().values( confirm_id="active-downgrade", tenant_uuid="platform", project_id="default", owner_user_id=7001, action="mes.dispatch", power="P3", params_hash="downgrade-hash", status="PENDING", approval_step=0, required_approvals=2, created_at_epoch=2_000.0, expires_at_epoch=2_100.0, revision=0, payload=_record("active-downgrade", required_approvals=2), ) ) with pytest.raises(RuntimeError, match="active approval state exists"): command.downgrade(config, "20260731_03") with engine.begin() as connection: connection.execute( ApprovalRequestRecord.__table__.update() .where(ApprovalRequestRecord.confirm_id == "active-downgrade") .values(status="EXPIRED") ) command.downgrade(config, "20260731_03") engine.dispose() reset_engine() def test_request_expires_when_transaction_clock_equals_boundary( database_store: DatabaseApprovalStore, ) -> None: confirm_id = "clock-request-boundary" assert database_store.stage(_record(confirm_id), now_epoch=2_000.0) result = database_store.decide( confirm_id, approve=True, approver=_approver(7001), allowed=_allow, now_epoch=2_010.0, grant_ttl_seconds=60, ) assert result is None with get_session() as session: row = session.get(ApprovalRequestRecord, confirm_id) statuses = list(session.scalars(select(ApprovalEventRecord.status))) assert row is not None and row.status == "EXPIRED" assert statuses.count("EXPIRED") == 1 assert "APPROVED" not in statuses def test_grant_cannot_be_consumed_when_clock_equals_boundary( database_store: DatabaseApprovalStore, ) -> None: confirm_id = "clock-grant-boundary" grant, expires_at = _issue_grant(database_store, confirm_id=confirm_id) consumed = database_store.consume_grant( grant, confirm_id=confirm_id, action="mes.dispatch", params_hash="clock-contract-hash", allowed=_allow, decided_by=_approver(7001), now_epoch=expires_at, ) assert consumed is False with get_session() as session: row = session.scalar(select(ApprovalGrantRecord)) statuses = list(session.scalars(select(ApprovalEventRecord.status))) assert row is not None and row.status == "EXPIRED" assert statuses.count("GRANT_EXPIRED") == 1 assert "EXECUTED" not in statuses def test_approve_and_expire_are_mutually_exclusive( database_store: DatabaseApprovalStore, ) -> None: confirm_id = "clock-request-race" assert database_store.stage(_record(confirm_id), now_epoch=2_000.0) barrier = threading.Barrier(2) outcomes: list[object] = [] def approve() -> None: barrier.wait() outcomes.append(database_store.decide( confirm_id, approve=True, approver=_approver(7001), allowed=_allow, now_epoch=2_009.0, grant_ttl_seconds=60, )) def expire() -> None: barrier.wait() outcomes.append(database_store.expire(now_epoch=2_010.0)) threads = [threading.Thread(target=approve), threading.Thread(target=expire)] for thread in threads: thread.start() for thread in threads: thread.join() with get_session() as session: row = session.get(ApprovalRequestRecord, confirm_id) statuses = list(session.scalars(select(ApprovalEventRecord.status))) assert len(outcomes) == 2 assert row is not None and row.status in {"APPROVED", "EXPIRED"} assert statuses.count("APPROVED") + statuses.count("EXPIRED") == 1 def test_consume_and_expire_are_mutually_exclusive( database_store: DatabaseApprovalStore, ) -> None: confirm_id = "clock-grant-race" grant, expires_at = _issue_grant(database_store, confirm_id=confirm_id) barrier = threading.Barrier(2) outcomes: list[bool] = [] def consume() -> None: barrier.wait() outcomes.append(database_store.consume_grant( grant, confirm_id=confirm_id, action="mes.dispatch", params_hash="clock-contract-hash", allowed=_allow, decided_by=_approver(7001), now_epoch=expires_at - 0.001, )) def expire() -> None: barrier.wait() outcomes.append(database_store.expire(now_epoch=expires_at)) threads = [threading.Thread(target=consume), threading.Thread(target=expire)] for thread in threads: thread.start() for thread in threads: thread.join() with get_session() as session: row = session.scalar(select(ApprovalGrantRecord)) statuses = list(session.scalars(select(ApprovalEventRecord.status))) assert len(outcomes) == 2 assert row is not None and row.status in {"CONSUMED", "EXPIRED"} assert statuses.count("EXECUTED") + statuses.count("GRANT_EXPIRED") == 1 def test_event_failure_rolls_back_request_transition( database_store: DatabaseApprovalStore, monkeypatch: pytest.MonkeyPatch, ) -> None: confirm_id = "clock-event-rollback" assert database_store.stage(_record(confirm_id), now_epoch=2_000.0) def fail_event(*_args: object, **_kwargs: object) -> ApprovalEventRecord: raise RuntimeError("event insert failed") monkeypatch.setattr(database_store, "_event_row", fail_event) with pytest.raises(RuntimeError, match="event insert failed"): database_store.decide( confirm_id, approve=True, approver=_approver(7001), allowed=_allow, now_epoch=2_001.0, grant_ttl_seconds=60, ) with get_session() as session: row = session.get(ApprovalRequestRecord, confirm_id) events = list(session.scalars(select(ApprovalEventRecord))) assert row is not None and row.status == "PENDING" and row.revision == 0 assert events == [] def test_commit_failure_rolls_back_request_transition( database_store: DatabaseApprovalStore, monkeypatch: pytest.MonkeyPatch, ) -> None: confirm_id = "clock-commit-rollback" assert database_store.stage(_record(confirm_id), now_epoch=2_000.0) original_commit = Session.commit def fail_commit(self: Session) -> None: raise RuntimeError("commit failed") monkeypatch.setattr(Session, "commit", fail_commit) with pytest.raises(RuntimeError, match="commit failed"): database_store.decide( confirm_id, approve=True, approver=_approver(7001), allowed=_allow, now_epoch=2_001.0, grant_ttl_seconds=60, ) monkeypatch.setattr(Session, "commit", original_commit) with get_session() as session: row = session.get(ApprovalRequestRecord, confirm_id) events = list(session.scalars(select(ApprovalEventRecord))) assert row is not None and row.status == "PENDING" and row.revision == 0 assert events == [] @pytest.mark.parametrize( ("table", "columns"), [ (ApprovalRequestRecord.__table__, ("created_at_epoch", "expires_at_epoch")), ( ApprovalGrantRecord.__table__, ("created_at_epoch", "expires_at_epoch", "consumed_at_epoch"), ), (ApprovalEventRecord.__table__, ("decided_at_epoch",)), ], ) def test_mysql_epoch_columns_compile_as_double(table: Any, columns: tuple[str, ...]) -> None: ddl = str(CreateTable(table).compile(dialect=mysql.dialect())).upper() for column in columns: assert f"{column.upper()} DOUBLE" in ddl