Real World Example

During a Flipkart Big Billion Days style sale, the ShopEase on-call engineer gets an alert: "order-service p95 latency 2.4 s". With observability in place the investigation takes minutes:

1. Metrics (Grafana): latency is high only for POST /api/orders; hikaricp_connections_pending in payment-service is climbing.
2. Traces (Zipkin/Tempo): a slow trace shows 1.9 s spent in the span payment-service SELECT payments.
3. Logs (Loki, filtered by the trace ID): WARN ... query took 1874 ms: SELECT * FROM payments WHERE order_id = ?.
4. Root cause: a missing index on payments.order_id after a migration. Add the index; latency drops to 180 ms.

spring:
  application:
    name: payment-service
management:
  endpoints:
    web:
      exposure:
        include: health,info,prometheus,loggers
  endpoint:
    health:
      probes:
        enabled: true                 # /actuator/health/liveness and /readiness for Kubernetes
  metrics:
    tags:
      application: payment-service
    distribution:
      percentiles-histogram:
        http.server.requests: true
  tracing:
    sampling:
      probability: 0.1
  zipkin:
    tracing:
      endpoint: http://zipkin.observability:9411/api/v2/spans
logging:
  structured:
    format:
      console: ecs

@Service
@RequiredArgsConstructor
@Slf4j
public class PaymentService {
    private final ObservationRegistry observations;
    private final MeterRegistry meters;
    private final RazorpayGateway razorpay;

    public PaymentResult charge(ChargePayment cmd) {
        return Observation.createNotStarted("payment.charge", observations)   // span + timer metric
                .lowCardinalityKeyValue("method", cmd.method().name())          // UPI, CARD, NETBANKING
                .observe(() -> {
                    PaymentResult result = razorpay.capture(cmd);
                    if (!result.captured()) {
                        meters.counter("shopease.payments.failed", "reason", result.reason()).increment();
                        log.warn("Payment declined for order {}: {}", cmd.orderId(), result.reason());
                    }
                    return result;
                });
    }
}