Real World Example

• GitHub Copilot, Notion AI: route between models by task, stream every response, and track latency and acceptance rate per model.
• Swiggy / Zomato style support: cheap models classify and route most chats; only complex ones reach a large model or a human.
• Perplexity style answer engines: heavy caching of popular queries and per-route model choice to control cost.

All ShopEase services call LLMs through one small gateway module, so caching, retries, logging and budgets live in one place.

import json, logging
import anthropic

log = logging.getLogger("llm")
client = anthropic.Anthropic(max_retries=4)
PRICES = {"claude-opus-5-5": (4, 20), "claude-sonnet-5-5": (2, 10), "claude-haiku-5-5": (0.10, 0.50)}
ROUTES = {"intent": "claude-haiku-5-5", "support": "claude-sonnet-5-5", "copy": "claude-opus-5-5"}

def complete(route: str, system: str, messages: list, user_id: str, max_tokens: int = 1000) -> str:
    if (hit := semantic_cache.get(route, messages[-1]["content"])) is not None:
        log.info(json.dumps({"route": route, "cache": "hit"}))
        return hit
    budget.check(user_id)                                  # raises if the daily budget is used up
    model = ROUTES[route]
    resp = client.messages.create(
        model=model, max_tokens=max_tokens, messages=messages,
        system=[{"type": "text", "text": system, "cache_control": {"type": "ephemeral"}}],
    )
    u = resp.usage
    pin, pout = PRICES[model]
    usd = (u.input_tokens * pin + u.cache_read_input_tokens * pin * 0.1
           + u.cache_creation_input_tokens * pin * 1.25 + u.output_tokens * pout) / 1e6
    budget.add(user_id, usd)
    log.info(json.dumps({"route": route, "model": model, "request_id": resp._request_id,
                         "in": u.input_tokens, "cached": u.cache_read_input_tokens,
                         "out": u.output_tokens, "usd": round(usd, 6), "stop": resp.stop_reason}))
    return next(b.text for b in resp.content if b.type == "text")

tail -n 2 logs/llm.jsonl

{"route": "support", "model": "claude-sonnet-5-5", "request_id": "req_011CZx8c4Lh2", "in": 61, "cached": 18420, "out": 132, "usd": 0.005126, "stop": "end_turn"}
{"route": "intent", "cache": "hit"}