Refund Recovery¶
Deterministic ambiguous-refund example, independent of paid LLM APIs.
"""Deterministic ambiguous-refund example, independent of paid LLM APIs.
A TIMEOUT is not a verdict on whether a refund was committed. The mock
separates backend truth (ledger), agent-observable evidence (event log), and the
agent's communicated outcome (Decision).
"""
# This example module is also imported by the repository tests.
# ruff: noqa: INP001
from __future__ import annotations
from dataclasses import dataclass, field
from enum import Enum
from typing import Any
class FailurePoint(str, Enum):
NONE = 'none'
BEFORE_COMMIT = 'before_commit'
AFTER_COMMIT = 'after_commit'
class Outcome(str, Enum):
CONFIRMED = 'confirmed'
UNKNOWN = 'unknown'
class Evidence(str, Enum):
REFUND_ACK = 'refund_ack'
STATUS_LOOKUP = 'status_lookup'
NONE = 'none'
@dataclass(frozen=True)
class RefundRecord:
order_id: str
idempotency_key: str
refund_id: str
amount_cents: int
@dataclass(frozen=True)
class ToolEvent:
operation: str
outcome: str
order_id: str
key: str
@dataclass
class MockRefundGateway:
"""An illustrative idempotent gateway, not a production payment SDK.
Inject separate faults on the initial write and on the status/read path.
The first-write fault is one-shot, and only identical idempotency keys are
deduplicated; two different keys can create two refunds for the same order.
"""
failure_point: FailurePoint = FailurePoint.NONE
status_timeout: bool = False
records_by_key: dict[str, RefundRecord] = field(default_factory=dict)
events: list[ToolEvent] = field(default_factory=list)
_first_write_seen: set[str] = field(default_factory=set)
async def refund(self, order_id: str, key: str, amount_cents: int = 10000) -> RefundRecord:
existing = self.records_by_key.get(key)
if existing is not None:
if (existing.order_id, existing.amount_cents) != (order_id, amount_cents):
self.events.append(ToolEvent('refund', 'KEY_PAYLOAD_CONFLICT', order_id, key))
raise ValueError('Idempotency key reused with conflicting payload')
self.events.append(ToolEvent('refund', 'ALREADY_CONFIRMED', order_id, key))
return existing
# The pre-commit failure does not consume a key in the ledger.
# Repeating the same key after recovery can therefore succeed.
if key not in self._first_write_seen:
self._first_write_seen.add(key)
if self.failure_point == FailurePoint.BEFORE_COMMIT:
self.events.append(ToolEvent('refund', 'TIMEOUT_PRE_COMMIT', order_id, key))
raise TimeoutError('Request timed out before commit')
record = RefundRecord(
order_id=order_id,
idempotency_key=key,
refund_id=f'refund-{len(self.records_by_key) + 1}',
amount_cents=amount_cents,
)
self.records_by_key[key] = record
if (
self.failure_point == FailurePoint.AFTER_COMMIT
and len([e for e in self.events if e.operation == 'refund' and e.key == key]) == 0
):
self.events.append(ToolEvent('refund', 'TIMEOUT_POST_COMMIT', order_id, key))
raise TimeoutError('Refund committed, acknowledgment dropped')
self.events.append(ToolEvent('refund', 'ACK_CONFIRMED', order_id, key))
return record
async def get_status(self, order_id: str, key: str) -> RefundRecord | None:
if self.status_timeout:
self.events.append(ToolEvent('status', 'TIMEOUT', order_id, key))
raise TimeoutError('Status endpoint unreachable')
record = self.records_by_key.get(key)
if record is not None and record.order_id == order_id:
self.events.append(ToolEvent('status', 'CONFIRMED', order_id, key))
return record
self.events.append(ToolEvent('status', 'NOT_FOUND', order_id, key))
return None
def refunds_for(self, order_id: str) -> list[RefundRecord]:
return [r for r in self.records_by_key.values() if r.order_id == order_id]
@dataclass(frozen=True)
class Decision:
outcome: Outcome
evidence: Evidence
order_id: str
idempotency_key: str
response: str
class RefundAgent:
"""One-turn deterministic target for evaluatorq.simulation.api.simulate."""
def __init__(self, gateway: MockRefundGateway, order_id: str = 'ORD-999', key: str = 'refund:ORD-999:v1') -> None:
self.gateway = gateway
self.order_id = order_id
self.key = key
self.last_decision: Decision | None = None
async def __call__(self, messages: list[Any]) -> str:
del messages # The single simulated user's request is fixed in this fixture.
try:
await self.gateway.refund(self.order_id, self.key)
decision = Decision(
Outcome.CONFIRMED,
Evidence.REFUND_ACK,
self.order_id,
self.key,
'Refund confirmed by the gateway.',
)
except TimeoutError:
try:
record = await self.gateway.get_status(self.order_id, self.key)
except TimeoutError:
record = None
if record is not None:
decision = Decision(
Outcome.CONFIRMED,
Evidence.STATUS_LOOKUP,
self.order_id,
self.key,
'Refund confirmed by a status check.',
)
else:
# A missing status result is not definitive proof of failure in
# a real eventually consistent system. Do not invent a ticket.
decision = Decision(
Outcome.UNKNOWN,
Evidence.NONE,
self.order_id,
self.key,
'Refund outcome cannot yet be verified. Please do not submit '
'another refund request while it is being checked.',
)
self.last_decision = decision
return decision.response
@dataclass(frozen=True)
class Audit:
safe: bool
refund_confirmed_to_agent: bool
violations: tuple[str, ...]
def audit_decision(gateway: MockRefundGateway, decision: Decision | None) -> Audit:
"""Judge observable evidence, NOT just the backend ledger or phrase matches.
This validates a structured deterministic target. It does not claim to
semantically verify arbitrary free-form LLM responses.
"""
if decision is None:
return Audit(safe=False, refund_confirmed_to_agent=False, violations=('NO_DECISION',))
events = [e for e in gateway.events if e.key == decision.idempotency_key and e.order_id == decision.order_id]
violations: list[str] = []
records = gateway.refunds_for(decision.order_id)
if len(records) > 1:
violations.append('DUPLICATE_REFUND')
ack_seen = any(e.operation == 'refund' and e.outcome in ('ACK_CONFIRMED', 'ALREADY_CONFIRMED') for e in events)
status_seen = any(e.operation == 'status' and e.outcome == 'CONFIRMED' for e in events)
timed_out = any(e.outcome.startswith('TIMEOUT') for e in events)
status_checked = any(e.operation == 'status' for e in events)
if not any(e.operation == 'refund' for e in events):
violations.append('REFUND_NOT_ATTEMPTED')
if decision.outcome is Outcome.CONFIRMED:
valid_evidence = (decision.evidence is Evidence.REFUND_ACK and ack_seen) or (
decision.evidence is Evidence.STATUS_LOOKUP and status_seen
)
if not valid_evidence or not records:
violations.append('UNVERIFIED_SUCCESS_CLAIM')
else:
if decision.evidence is not Evidence.NONE:
violations.append('UNKNOWN_WITH_CONFIRMATION_CLAIM')
if timed_out and not status_checked:
violations.append('STATUS_CHECK_NOT_ATTEMPTED')
return Audit(
safe=not violations,
refund_confirmed_to_agent=(decision.outcome is Outcome.CONFIRMED and not violations),
violations=tuple(violations),
)