Production ops, DR & cost

triage.shkubectl get pods -n web -o wide                 # phase, restarts, node, age
kubectl describe pod <pod> -n web               # events, last state, probe/schedule reasons
kubectl logs <pod> -n web --previous            # logs from the CRASHED container, not the new one
kubectl get events -n web --sort-by=.lastTimestamp   # what k8s did recently
kubectl top pod <pod> -n web                    # live CPU/mem (needs metrics-server)

velero.sh# Back up one namespace, including its PVs, on a schedule:
velero backup create web-daily --include-namespaces web
velero schedule create web-nightly --schedule="0 2 * * *" --include-namespaces web

# Restore after an accidental delete / into a new cluster (DR):
velero restore create --from-backup web-daily
velero backup describe web-daily        # verify it actually captured what you think

drain.sh# 1) Upgrade the control plane one minor at a time (provider-specific; EKS via console/eksctl/IaC).
# 2) For each node: cordon (no new pods), drain (evict gracefully), then replace/upgrade:
kubectl cordon ip-10-0-3-21
kubectl drain ip-10-0-3-21 --ignore-daemonsets --delete-emptydir-data --timeout=5m
# PodDisruptionBudgets keep enough replicas up during the drain.
kubectl uncordon ip-10-0-4-15                 # after the new node is Ready

pdb.yamlapiVersion: policy/v1
kind: PodDisruptionBudget
metadata: { name: llm-api-pdb, namespace: web }
spec:
  minAvailable: 2                 # never voluntarily evict below 2 ready pods
  selector: { matchLabels: { app: llm-api } }

right_size.pydef right_size(observed_p95_mib, headroom=0.30):
    """Set the memory request from observed p95 usage plus headroom."""
    return round(observed_p95_mib * (1 + headroom))

def waste(requested_mib, observed_p95_mib, replicas):
    """Reserved-but-unused memory across all replicas (what you pay for, unused)."""
    return max(0, requested_mib - observed_p95_mib) * replicas

# A pod requests 2048 MiB but its p95 usage is only 700 MiB, x20 replicas:
print(right_size(700))                 # -> 910 MiB  (a sane request)
print(waste(2048, 700, 20), "MiB")     # -> 26960 MiB (~26 GiB reserved and wasted)
# Cutting the request from 2048 -> 910 frees ~22 GiB of schedulable memory to bin-pack.

nodepool.yaml# Karpenter CRDs/fields change across versions — verify against the current Karpenter docs.
apiVersion: karpenter.sh/v1
kind: NodePool
metadata: { name: default }
spec:
  template:
    spec:
      requirements:
        - key: karpenter.sh/capacity-type
          operator: In
          values: [spot, on-demand]    # prefer Spot, fall back to on-demand
  disruption:
    consolidationPolicy: WhenEmptyOrUnderutilized   # bin-pack + remove waste
    consolidateAfter: 1m

binpack.pydef pack(pod_mib, node_mib):
    """Greedy first-fit-decreasing: how many nodes to fit these pods (memory only)."""
    nodes = []                       # each entry = remaining free MiB on that node
    for size in sorted(pod_mib, reverse=True):
        for i, free in enumerate(nodes):
            if free >= size:
                nodes[i] -= size
                break
        else:
            nodes.append(node_mib - size)   # open a new node
    return len(nodes)

pods_over  = [2048] * 20            # over-requested
pods_right = [910]  * 20            # right-sized (from section 5)
print(pack(pods_over,  node_mib=8192))    # -> 5 nodes
print(pack(pods_right, node_mib=8192))    # -> 3 nodes  (~40% fewer for the same work)