Health Probes and Graceful Shutdown

Probes

Spring Boot enables the probe groups automatically on Kubernetes; enable them explicitly so they also work locally, and decide which checks belong to readiness:

application.yml

management:
  endpoint:
    health:
      probes:
        enabled: true
      group:
        readiness:
          include: readinessState,db        # not ready if the database is down
server:
  shutdown: graceful                        # finish in-flight requests on SIGTERM
spring:
  lifecycle:
    timeout-per-shutdown-phase: 30s

Example output

$ curl -s localhost:8082/actuator/health/liveness
{"status":"UP"}
$ curl -s localhost:8082/actuator/health/readiness
{"status":"UP","components":{"db":{"status":"UP"},"readinessState":{"status":"UP"}}}

Example

A simulation of what the Deployment controller does with maxSurge: 1 and maxUnavailable: 0. A new Pod becomes ready after two probe periods; only then is an old Pod removed. In the second rollout the new version has a bad config, its readiness probe never passes, and users keep being served by v1:

public class Main {
    public static void main(String[] args) {
        rollout("v2", true);
        System.out.println();
        rollout("v3", false);
    }

    static void rollout(String version, boolean healthy) {
        System.out.println("== rolling update to " + version + " (replicas=3, maxSurge=1, maxUnavailable=0) ==");
        int oldReady = 3, newReady = 0, startingAge = -1;
        for (int t = 1; t <= 10; t++) {
            String event;
            if (startingAge < 0) {
                startingAge = 0;
                event = "create pod " + version + "-" + (newReady + 1) + " (not ready yet)";
            } else {
                startingAge++;
                if (healthy && startingAge >= 2) {
                    newReady++;
                    oldReady--;
                    startingAge = -1;
                    event = "pod " + version + "-" + newReady + " READY -> terminate one v1 pod";
                } else {
                    event = healthy ? "readiness probe: starting" : "readiness probe: 503 DOWN";
                }
            }
            System.out.printf("t=%-2d v1=%d %s=%d serving=%d | %s%n",
                    t, oldReady, version, newReady, oldReady + newReady, event);
            if (oldReady == 0) {
                System.out.println("deployment 'order-service' successfully rolled out");
                return;
            }
            if (!healthy && t == 6) {
                System.out.println("rollout stuck: new pod never became ready, no user saw an error");
                System.out.println("$ kubectl rollout undo deployment/order-service");
                return;
            }
        }
    }
}

Output

== rolling update to v2 (replicas=3, maxSurge=1, maxUnavailable=0) ==
t=1  v1=3 v2=0 serving=3 | create pod v2-1 (not ready yet)
t=2  v1=3 v2=0 serving=3 | readiness probe: starting
t=3  v1=2 v2=1 serving=3 | pod v2-1 READY -> terminate one v1 pod
t=4  v1=2 v2=1 serving=3 | create pod v2-2 (not ready yet)
t=5  v1=2 v2=1 serving=3 | readiness probe: starting
t=6  v1=1 v2=2 serving=3 | pod v2-2 READY -> terminate one v1 pod
t=7  v1=1 v2=2 serving=3 | create pod v2-3 (not ready yet)
t=8  v1=1 v2=2 serving=3 | readiness probe: starting
t=9  v1=0 v2=3 serving=3 | pod v2-3 READY -> terminate one v1 pod
deployment 'order-service' successfully rolled out

== rolling update to v3 (replicas=3, maxSurge=1, maxUnavailable=0) ==
t=1  v1=3 v3=0 serving=3 | create pod v3-1 (not ready yet)
t=2  v1=3 v3=0 serving=3 | readiness probe: 503 DOWN
t=3  v1=3 v3=0 serving=3 | readiness probe: 503 DOWN
t=4  v1=3 v3=0 serving=3 | readiness probe: 503 DOWN
t=5  v1=3 v3=0 serving=3 | readiness probe: 503 DOWN
t=6  v1=3 v3=0 serving=3 | readiness probe: 503 DOWN
rollout stuck: new pod never became ready, no user saw an error
$ kubectl rollout undo deployment/order-service