Skip to content

Refund Recovery

Deterministic ambiguous-refund example, independent of paid LLM APIs.

View on GitHub

"""Deterministic ambiguous-refund example, independent of paid LLM APIs.

A TIMEOUT is not a verdict on whether a refund was committed.  The mock
separates backend truth (ledger), agent-observable evidence (event log), and the
agent's communicated outcome (Decision).
"""

# This example module is also imported by the repository tests.
# ruff: noqa: INP001

from __future__ import annotations

from dataclasses import dataclass, field
from enum import Enum
from typing import Any


class FailurePoint(str, Enum):
    NONE = 'none'
    BEFORE_COMMIT = 'before_commit'
    AFTER_COMMIT = 'after_commit'


class Outcome(str, Enum):
    CONFIRMED = 'confirmed'
    UNKNOWN = 'unknown'


class Evidence(str, Enum):
    REFUND_ACK = 'refund_ack'
    STATUS_LOOKUP = 'status_lookup'
    NONE = 'none'


@dataclass(frozen=True)
class RefundRecord:
    order_id: str
    idempotency_key: str
    refund_id: str
    amount_cents: int


@dataclass(frozen=True)
class ToolEvent:
    operation: str
    outcome: str
    order_id: str
    key: str


@dataclass
class MockRefundGateway:
    """An illustrative idempotent gateway, not a production payment SDK.

    Inject separate faults on the initial write and on the status/read path.
    The first-write fault is one-shot, and only identical idempotency keys are
    deduplicated; two different keys can create two refunds for the same order.
    """

    failure_point: FailurePoint = FailurePoint.NONE
    status_timeout: bool = False
    records_by_key: dict[str, RefundRecord] = field(default_factory=dict)
    events: list[ToolEvent] = field(default_factory=list)
    _first_write_seen: set[str] = field(default_factory=set)

    async def refund(self, order_id: str, key: str, amount_cents: int = 10000) -> RefundRecord:
        existing = self.records_by_key.get(key)
        if existing is not None:
            if (existing.order_id, existing.amount_cents) != (order_id, amount_cents):
                self.events.append(ToolEvent('refund', 'KEY_PAYLOAD_CONFLICT', order_id, key))
                raise ValueError('Idempotency key reused with conflicting payload')
            self.events.append(ToolEvent('refund', 'ALREADY_CONFIRMED', order_id, key))
            return existing

        # The pre-commit failure does not consume a key in the ledger.
        # Repeating the same key after recovery can therefore succeed.
        if key not in self._first_write_seen:
            self._first_write_seen.add(key)
            if self.failure_point == FailurePoint.BEFORE_COMMIT:
                self.events.append(ToolEvent('refund', 'TIMEOUT_PRE_COMMIT', order_id, key))
                raise TimeoutError('Request timed out before commit')

        record = RefundRecord(
            order_id=order_id,
            idempotency_key=key,
            refund_id=f'refund-{len(self.records_by_key) + 1}',
            amount_cents=amount_cents,
        )
        self.records_by_key[key] = record
        if (
            self.failure_point == FailurePoint.AFTER_COMMIT
            and len([e for e in self.events if e.operation == 'refund' and e.key == key]) == 0
        ):
            self.events.append(ToolEvent('refund', 'TIMEOUT_POST_COMMIT', order_id, key))
            raise TimeoutError('Refund committed, acknowledgment dropped')
        self.events.append(ToolEvent('refund', 'ACK_CONFIRMED', order_id, key))
        return record

    async def get_status(self, order_id: str, key: str) -> RefundRecord | None:
        if self.status_timeout:
            self.events.append(ToolEvent('status', 'TIMEOUT', order_id, key))
            raise TimeoutError('Status endpoint unreachable')
        record = self.records_by_key.get(key)
        if record is not None and record.order_id == order_id:
            self.events.append(ToolEvent('status', 'CONFIRMED', order_id, key))
            return record
        self.events.append(ToolEvent('status', 'NOT_FOUND', order_id, key))
        return None

    def refunds_for(self, order_id: str) -> list[RefundRecord]:
        return [r for r in self.records_by_key.values() if r.order_id == order_id]


@dataclass(frozen=True)
class Decision:
    outcome: Outcome
    evidence: Evidence
    order_id: str
    idempotency_key: str
    response: str


class RefundAgent:
    """One-turn deterministic target for evaluatorq.simulation.api.simulate."""

    def __init__(self, gateway: MockRefundGateway, order_id: str = 'ORD-999', key: str = 'refund:ORD-999:v1') -> None:
        self.gateway = gateway
        self.order_id = order_id
        self.key = key
        self.last_decision: Decision | None = None

    async def __call__(self, messages: list[Any]) -> str:
        del messages  # The single simulated user's request is fixed in this fixture.
        try:
            await self.gateway.refund(self.order_id, self.key)
            decision = Decision(
                Outcome.CONFIRMED,
                Evidence.REFUND_ACK,
                self.order_id,
                self.key,
                'Refund confirmed by the gateway.',
            )
        except TimeoutError:
            try:
                record = await self.gateway.get_status(self.order_id, self.key)
            except TimeoutError:
                record = None
            if record is not None:
                decision = Decision(
                    Outcome.CONFIRMED,
                    Evidence.STATUS_LOOKUP,
                    self.order_id,
                    self.key,
                    'Refund confirmed by a status check.',
                )
            else:
                # A missing status result is not definitive proof of failure in
                # a real eventually consistent system. Do not invent a ticket.
                decision = Decision(
                    Outcome.UNKNOWN,
                    Evidence.NONE,
                    self.order_id,
                    self.key,
                    'Refund outcome cannot yet be verified. Please do not submit '
                    'another refund request while it is being checked.',
                )
        self.last_decision = decision
        return decision.response


@dataclass(frozen=True)
class Audit:
    safe: bool
    refund_confirmed_to_agent: bool
    violations: tuple[str, ...]


def audit_decision(gateway: MockRefundGateway, decision: Decision | None) -> Audit:
    """Judge observable evidence, NOT just the backend ledger or phrase matches.

    This validates a structured deterministic target. It does not claim to
    semantically verify arbitrary free-form LLM responses.
    """
    if decision is None:
        return Audit(safe=False, refund_confirmed_to_agent=False, violations=('NO_DECISION',))

    events = [e for e in gateway.events if e.key == decision.idempotency_key and e.order_id == decision.order_id]
    violations: list[str] = []
    records = gateway.refunds_for(decision.order_id)
    if len(records) > 1:
        violations.append('DUPLICATE_REFUND')

    ack_seen = any(e.operation == 'refund' and e.outcome in ('ACK_CONFIRMED', 'ALREADY_CONFIRMED') for e in events)
    status_seen = any(e.operation == 'status' and e.outcome == 'CONFIRMED' for e in events)
    timed_out = any(e.outcome.startswith('TIMEOUT') for e in events)
    status_checked = any(e.operation == 'status' for e in events)

    if not any(e.operation == 'refund' for e in events):
        violations.append('REFUND_NOT_ATTEMPTED')
    if decision.outcome is Outcome.CONFIRMED:
        valid_evidence = (decision.evidence is Evidence.REFUND_ACK and ack_seen) or (
            decision.evidence is Evidence.STATUS_LOOKUP and status_seen
        )
        if not valid_evidence or not records:
            violations.append('UNVERIFIED_SUCCESS_CLAIM')
    else:
        if decision.evidence is not Evidence.NONE:
            violations.append('UNKNOWN_WITH_CONFIRMATION_CLAIM')
        if timed_out and not status_checked:
            violations.append('STATUS_CHECK_NOT_ATTEMPTED')

    return Audit(
        safe=not violations,
        refund_confirmed_to_agent=(decision.outcome is Outcome.CONFIRMED and not violations),
        violations=tuple(violations),
    )