AI agent A/B testing cost tradeoffs: how experiment design choices multiply your LLM API bill

import uuid
import threading
from dataclasses import dataclass, field
from typing import Callable, Optional
import anthropic
from runguard import guard, BudgetExceededError

# Variant registry: each variant gets its own BudgetTracker via guard()
@dataclass
class VariantConfig:
    name: str
    model: str
    system_prompt: str
    budget_usd: float           # per-variant spend cap for this experiment window
    traffic_weight: float       # relative weight for traffic splitting

VARIANTS = [
    VariantConfig(
        name="control",
        model="claude-haiku-4-5-20251001",
        system_prompt="You are a helpful customer support agent. Be concise.",
        budget_usd=50.0,        # control arm: $50 cap for this week
        traffic_weight=0.5,
    ),
    VariantConfig(
        name="treatment_verbose",
        model="claude-haiku-4-5-20251001",
        system_prompt="You are a helpful customer support agent. Walk through your reasoning step by step.",
        budget_usd=50.0,        # same model, different prompt; equal cap
        traffic_weight=0.5,
    ),
]

# Thread-local RunGuard guard functions — one per variant
# Each guard() call creates a separate BudgetTracker instance
_variant_guards: dict[str, Callable] = {}
_guard_lock = threading.Lock()

def get_variant_guard(variant: VariantConfig) -> Callable:
    """Return (or create) a RunGuard-wrapped LLM caller for this variant."""
    with _guard_lock:
        if variant.name not in _variant_guards:
            # Create a fresh guard for this variant with its own budget cap
            # The guard persists for the experiment window; resetting it
            # resets the spend counter, which you do at week rollover.
            @guard(budget_usd=variant.budget_usd, loop_max_repeats=5)
            def _call_llm(messages: list, model: str, system: str) -> str:
                client = anthropic.Anthropic()
                resp = client.messages.create(
                    model=model,
                    max_tokens=512,
                    system=system,
                    messages=messages,
                )
                return resp.content[0].text
            _variant_guards[variant.name] = _call_llm
    return _variant_guards[variant.name]

def select_variant(request_id: str) -> VariantConfig:
    """Deterministic variant selection based on request_id hash."""
    import hashlib
    total_weight = sum(v.traffic_weight for v in VARIANTS)
    bucket = int(hashlib.md5(request_id.encode()).hexdigest(), 16) % 10000 / 10000.0
    cumulative = 0.0
    for variant in VARIANTS:
        cumulative += variant.traffic_weight / total_weight
        if bucket < cumulative:
            return variant
    return VARIANTS[-1]

def handle_request(user_message: str, request_id: Optional[str] = None) -> dict:
    """Route a request to the selected variant and return result + metadata."""
    if request_id is None:
        request_id = str(uuid.uuid4())

    variant = select_variant(request_id)
    call_llm = get_variant_guard(variant)

    try:
        response = call_llm(
            messages=[{"role": "user", "content": user_message}],
            model=variant.model,
            system=variant.system_prompt,
        )
        return {
            "variant": variant.name,
            "response": response,
            "budget_exhausted": False,
            "request_id": request_id,
        }
    except BudgetExceededError as e:
        # This variant has hit its experiment budget cap.
        # Fall back to the control arm (do not disable the whole experiment).
        return {
            "variant": variant.name,
            "response": None,
            "budget_exhausted": True,
            "error": str(e),
            "request_id": request_id,
        }

import Anthropic from "@anthropic-ai/sdk";
import { BudgetExceededError } from "@runguard/sdk";
import crypto from "node:crypto";

interface VariantConfig {
  name: string;
  model: string;
  systemPrompt: string;
  budgetUsd: number;
  trafficWeight: number;
}

const VARIANTS: VariantConfig[] = [
  {
    name: "control",
    model: "claude-haiku-4-5-20251001",
    systemPrompt: "You are a helpful customer support agent. Be concise.",
    budgetUsd: 50,
    trafficWeight: 0.5,
  },
  {
    name: "treatment_cot",
    model: "claude-haiku-4-5-20251001",
    systemPrompt: "You are a helpful customer support agent. Think step by step before answering.",
    budgetUsd: 50,
    trafficWeight: 0.5,
  },
];

// Per-variant spend tracking — maintained in process memory for the experiment window
const variantSpend: Record<string, number> = {};
const client = new Anthropic();

function selectVariant(requestId: string): VariantConfig {
  const hash = parseInt(crypto.createHash("md5").update(requestId).digest("hex").slice(0, 8), 16);
  const bucket = (hash % 10000) / 10000;
  let cumulative = 0;
  const totalWeight = VARIANTS.reduce((s, v) => s + v.trafficWeight, 0);
  for (const variant of VARIANTS) {
    cumulative += variant.trafficWeight / totalWeight;
    if (bucket < cumulative) return variant;
  }
  return VARIANTS[VARIANTS.length - 1];
}

function checkVariantBudget(variant: VariantConfig, estimatedCost: number): void {
  const current = variantSpend[variant.name] ?? 0;
  if (current + estimatedCost > variant.budgetUsd) {
    throw new BudgetExceededError(
      `Variant '${variant.name}' budget exhausted: ` +
      `$${current.toFixed(4)} spent + $${estimatedCost.toFixed(4)} estimated > $${variant.budgetUsd} cap`
    );
  }
}

function recordVariantSpend(variantName: string, actualCost: number): void {
  variantSpend[variantName] = (variantSpend[variantName] ?? 0) + actualCost;
}

const HAIKU_IN  = 0.25  / 1_000_000;
const HAIKU_OUT = 1.25  / 1_000_000;

async function routeRequest(
  userMessage: string,
  requestId?: string,
): Promise<{ variant: string; response: string | null; budgetExhausted: boolean }> {
  const id = requestId ?? crypto.randomUUID();
  const variant = selectVariant(id);

  // Pre-call budget estimate (rough: 200 input + 512 output tokens)
  const estimatedCost = 200 * HAIKU_IN + 512 * HAIKU_OUT;

  try {
    checkVariantBudget(variant, estimatedCost);

    const resp = await client.messages.create({
      model: variant.model,
      max_tokens: 512,
      system: variant.systemPrompt,
      messages: [{ role: "user", content: userMessage }],
    });

    const usage = resp.usage;
    const actualCost = usage.input_tokens * HAIKU_IN + usage.output_tokens * HAIKU_OUT;
    recordVariantSpend(variant.name, actualCost);

    return { variant: variant.name, response: (resp.content[0] as Anthropic.TextBlock).text, budgetExhausted: false };
  } catch (e) {
    if (e instanceof BudgetExceededError) {
      return { variant: variant.name, response: null, budgetExhausted: true };
    }
    throw e;
  }
}

// Experiment spend summary — call at end of experiment window
export function experimentSpendSummary(): void {
  console.log("Variant spend summary:");
  for (const variant of VARIANTS) {
    const spent = variantSpend[variant.name] ?? 0;
    const pct = (spent / variant.budgetUsd * 100).toFixed(1);
    console.log(`  ${variant.name}: $${spent.toFixed(4)} / $${variant.budgetUsd} (${pct}%)`);
  }
}