Internals & debugging

oom.sh# Cap memory and CPU. Exceeding --memory gets the process SIGKILL'd (exit 137).
docker run --memory=256m --cpus=0.5 myapp:1.4.2

# After it dies, read WHY — OOMKilled and the exit code are recorded here:
docker inspect --format '{{.State.OOMKilled}} {{.State.ExitCode}} {{.State.Error}}' <container>
# -> true 137
# See real-time usage to right-size the limit:
docker stats --no-stream <container>
# Verify field paths/flags against current Docker docs.

exit_triage.pydef diagnose(exit_code, oom_killed):
    if oom_killed or exit_code == 137:
        return "OOMKilled — hit the cgroup memory limit; raise limit or cut memory use"
    if exit_code == 0:
        return "clean exit (PID 1 returned 0) — check restart policy if it shouldn't stop"
    if exit_code == 143:
        return "SIGTERM (128+15) — graceful stop requested; fine if shutdown was clean"
    if exit_code == 139:
        return "SIGSEGV (128+11) — native crash / segfault in the process"
    if exit_code == 1:
        return "app error — read the app logs; the program itself returned non-zero"
    if exit_code == 126 or exit_code == 127:
        return "entrypoint not executable / not found — check the CMD path & +x bit"
    return f"exit {exit_code} — read logs; map 128+N to signal N if > 128"

for ec, oom in [(137, True), (143, False), (1, False), (127, False)]:
    print(ec, "->", diagnose(ec, oom))

debug.sh# 1) Logs — and the logs of the PREVIOUS, already-dead instance:
docker logs <container>
kubectl logs <pod> --previous          # the crashed instance's logs, K8s

# 2) Inspect — exit code, OOM flag, last state, mounts, env:
docker inspect <container> --format '{{json .State}}'

# 3) Exec a shell IF the image has one (won't work on distroless):
docker exec -it <container> sh

# 4) No shell / crash-looping? Attach an EPHEMERAL debug container that shares
#    the target's process + network namespaces and brings its own tools:
kubectl debug -it <pod> --image=busybox --target=<container>
# Command surfaces evolve (esp. kubectl debug) — verify against current docs.

health.yml# Dockerfile HEALTHCHECK (Docker/Compose):
HEALTHCHECK --interval=30s --timeout=3s --start-period=20s --retries=3 \
  CMD python -c "import urllib.request;urllib.request.urlopen('http://localhost:8000/healthz')"

# Kubernetes separates the two intents (put in the Pod spec):
# readinessProbe -> remove from load balancer until ready (don't restart)
# livenessProbe  -> restart the container if it stays unhealthy
readinessProbe:
  httpGet: { path: /healthz, port: 8000 }
  periodSeconds: 5
livenessProbe:
  httpGet: { path: /livez, port: 8000 }
  initialDelaySeconds: 20      # give slow starts room before restarting
  periodSeconds: 10
  failureThreshold: 3

Dockerfile.pid1# WRONG: shell-form makes /bin/sh PID 1; it doesn't forward SIGTERM to python.
# CMD python app.py

# RIGHT: exec-form makes YOUR process PID 1 so it receives signals directly.
CMD ["python", "app.py"]

# If your app spawns children (workers, subprocesses), PID 1 must reap zombies.
# Either handle it in-app, or add a tiny init as PID 1:
#   docker run --init myapp        # Docker's built-in init (tini)
# or bake tini into the image and use it as the entrypoint. Verify current syntax.

graceful_shutdown.pyimport signal, sys, time

shutting_down = False

def handle_term(signum, frame):
    global shutting_down
    shutting_down = True
    print(f"received signal {signum} -> draining, then exit", flush=True)

# PID 1 gets NO default handlers — you MUST install them or SIGTERM is ignored.
signal.signal(signal.SIGTERM, handle_term)   # docker stop / k8s sends SIGTERM first
signal.signal(signal.SIGINT, handle_term)    # Ctrl-C

def serve_one_request():
    time.sleep(0.1)   # pretend to handle in-flight work

while not shutting_down:
    serve_one_request()

print("finished in-flight work, exiting 0", flush=True)
sys.exit(0)   # clean exit -> container exit code 0, fast stop (no SIGKILL wait)