k9s says Running while the cluster burns: which pod first?

# payment-api: fills its memory past a 64Mi limit — the kernel kills it (OOMKilled), then CrashLoopBackOff.apiVersion: apps/v1kind: Deploymentmetadata: { name: payment-api, namespace: payments, labels: { team: team-payments } }spec:  replicas: 1  selector: { matchLabels: { app: payment-api } }  template:    metadata: { labels: { app: payment-api, team: team-payments } }    spec:      containers:        - name: api          image: busybox:1.36          command: ["sh", "-c", "sleep 20; tail /dev/zero"]          resources: { requests: { memory: 32Mi, cpu: 10m }, limits: { memory: 64Mi, cpu: 200m } }---# catalog-api: a busy loop under a 100m CPU limit — saturated and throttled, never killed.apiVersion: apps/v1kind: Deploymentmetadata: { name: catalog-api, namespace: catalog, labels: { team: team-catalog } }spec:  replicas: 1  selector: { matchLabels: { app: catalog-api } }  template:    metadata: { labels: { app: catalog-api, team: team-catalog } }    spec:      containers:        - name: api          image: busybox:1.36          command: ["sh", "-c", "while :; do :; done"]          resources: { requests: { memory: 16Mi, cpu: 50m }, limits: { memory: 64Mi, cpu: 100m } }---# checkout-worker: its readiness probe asks a path nginx does not serve — never Ready, Unhealthy events.apiVersion: apps/v1kind: StatefulSetmetadata: { name: checkout-worker, namespace: checkout, labels: { team: team-checkout } }spec:  serviceName: checkout-worker  replicas: 1  selector: { matchLabels: { app: checkout-worker } }  template:    metadata: { labels: { app: checkout-worker, team: team-checkout } }    spec:      containers:        - name: worker          image: nginx:1.27-alpine          readinessProbe: { httpGet: { path: /ready, port: 80 }, periodSeconds: 5 }          resources: { requests: { memory: 16Mi, cpu: 10m }, limits: { memory: 64Mi, cpu: 100m } }---# billing-api: its liveness probe fails — the kubelet restarts it again and again.apiVersion: apps/v1kind: Deploymentmetadata: { name: billing-api, namespace: billing, labels: { team: team-billing } }spec:  replicas: 1  selector: { matchLabels: { app: billing-api } }  template:    metadata: { labels: { app: billing-api, team: team-billing } }    spec:      containers:        - name: api          image: nginx:1.27-alpine          livenessProbe: { httpGet: { path: /health, port: 80 }, periodSeconds: 5, failureThreshold: 2 }          resources: { requests: { memory: 16Mi, cpu: 10m }, limits: { memory: 64Mi, cpu: 100m } }---# storefront: mounts a ConfigMap nobody created — stuck in ContainerCreating, FailedMount events.apiVersion: apps/v1kind: Deploymentmetadata: { name: storefront, namespace: frontend, labels: { team: team-web } }spec:  replicas: 1  selector: { matchLabels: { app: storefront } }  template:    metadata: { labels: { app: storefront, team: team-web } }    spec:      containers:        - name: web          image: nginx:1.27-alpine          volumeMounts: [{ name: config, mountPath: /etc/storefront }]          resources: { requests: { memory: 16Mi, cpu: 10m }, limits: { memory: 64Mi, cpu: 100m } }      volumes: [{ name: config, configMap: { name: storefront-config } }]

# What lumnik may do in the cluster: read pods and events, cluster-wide. Nothing else.apiVersion: v1kind: Namespacemetadata: { name: lumnik-reader }---apiVersion: v1kind: ServiceAccountmetadata: { name: lumnik-reader, namespace: lumnik-reader }---apiVersion: rbac.authorization.k8s.io/v1kind: ClusterRolemetadata: { name: lumnik-reader }rules:  - apiGroups: [""]    resources: [pods, events]    verbs: [get, list]  - apiGroups: [metrics.k8s.io]    resources: [pods]    verbs: [get, list]---apiVersion: rbac.authorization.k8s.io/v1kind: ClusterRoleBindingmetadata: { name: lumnik-reader }roleRef: { apiGroup: rbac.authorization.k8s.io, kind: ClusterRole, name: lumnik-reader }subjects: [{ kind: ServiceAccount, name: lumnik-reader, namespace: lumnik-reader }]---# A long-lived token for the hub's connector secret (projected tokens expire within the hour).apiVersion: v1kind: Secretmetadata:  name: lumnik-reader-token  namespace: lumnik-reader  annotations: { kubernetes.io/service-account.name: lumnik-reader }type: kubernetes.io/service-account-token

# The cluster's own API, read-only: pods, what they use now, and their recent Warning events.# The token and the CA come from the lumnik-reader ServiceAccount (k8s/30-reader.yaml): lm secret set.apiVersion: connectors.lumnik.io/v1kind: Connectormetadata:  name: k8s-cluster  connector_type: rest-generic  scopes: [devops-pod-triage]spec:  base_url: https://kubernetes.default.svc  auth: { kind: bearer, token_env: K8S_READER_TOKEN }  tls: { ca_secret: CLUSTER_CA }  defaults:    incremental: { mode: full }  endpoints:    - id: pods      method: GET      path: /api/v1/pods      pagination: { kind: cursor, cursor_param: continue, cursor_path: $.metadata.continue, response_path: $.items }      target:        kind: hybrid        table: connector.k8s_pods        unique_key: $.metadata.uid        promote:          - { name: namespace,              path: "$.metadata.namespace",                                     type: text }          - { name: pod,                    path: "$.metadata.name",                                          type: text }          - { name: team,                   path: "$.metadata.labels.team",                                   type: text }          - { name: ready,                  path: "$.status.containerStatuses[0].ready",                      type: boolean }          - { name: restarts,               path: "$.status.containerStatuses[0].restartCount",               type: integer }          - { name: waiting_reason,         path: "$.status.containerStatuses[0].state.waiting.reason",       type: text }          - { name: last_terminated_reason, path: "$.status.containerStatuses[0].lastState.terminated.reason", type: text }          - { name: cpu_limit,              path: "$.spec.containers[0].resources.limits.cpu",                type: decimal, transform: parse_quantity }          - { name: memory_limit,           path: "$.spec.containers[0].resources.limits.memory",             type: decimal, transform: parse_quantity }          - { name: pod_key,       type: text }      # filled by pod-triage.kts          - { name: priority,      type: text }          - { name: priority_rank, type: integer }          - { name: signal,        type: text }      mapping:        cell:          - kotlin: { script: hooks:pod-triage.kts }    - id: usage      method: GET      path: /apis/metrics.k8s.io/v1beta1/pods      pagination: { kind: cursor, cursor_param: continue, cursor_path: $.metadata.continue, response_path: $.items }      target:        kind: hybrid        table: connector.k8s_pod_usage        unique_key: "$.concat($.metadata.namespace, $.metadata.name)"        promote:          - { name: namespace, path: "$.metadata.namespace",     type: text }          - { name: pod,       path: "$.metadata.name",          type: text }          - { name: cpu,       path: "$.containers[0].usage.cpu",    type: decimal, transform: parse_quantity }          - { name: memory,    path: "$.containers[0].usage.memory", type: decimal, transform: parse_quantity }          - { name: pod_key,   type: text }          # filled by pod-key.kts      mapping:        cell:          - kotlin: { script: hooks:pod-key.kts }    - id: warnings      method: GET      path: /api/v1/events?fieldSelector=type%3DWarning,involvedObject.kind%3DPod      pagination: { kind: cursor, cursor_param: continue, cursor_path: $.metadata.continue, response_path: $.items }      target:        kind: hybrid        table: connector.k8s_warnings        unique_key: $.metadata.uid        promote:          - { name: namespace, path: "$.involvedObject.namespace", type: text }          - { name: pod,       path: "$.involvedObject.name",      type: text }          - { name: reason,    path: "$.reason",                   type: text }          - { name: count,     path: "$.count",                    type: integer }          - { name: last_seen, path: "$.lastTimestamp",            type: timestamptz }          - { name: message,   path: "$.message",                  type: text }          - { name: pod_key,   type: text }          # filled by pod-key.kts      mapping:        cell:          - kotlin: { script: hooks:pod-key.kts }

// The operator's triage rule, over one pod's own state (the pods API). Deterministic: the model never decides it.//   critical — the kernel killed it for memory, or it loops on restarts (CrashLoopBackOff)//   high     — it is not ready: a probe fails, or it cannot start (ContainerCreating, ImagePullBackOff…)//   medium   — it is ready but has restarted//   none     — nothing in its state// Saturation is not here: usage comes from metrics.k8s.io, another endpoint.@Suppress("UNCHECKED_CAST")val cells = bindings["cells"] as MutableMap<String, Any?>val waiting = cells["waiting_reason"] as String?val terminated = cells["last_terminated_reason"] as String?val restarts = (cells["restarts"] as Number?)?.toInt() ?: 0val ready = cells["ready"] == truecells["pod_key"] = "${cells["namespace"]}/${cells["pod"]}"val (priority, signal) = when {    terminated == "OOMKilled" -> "critical" to "OOMKilled, $restarts restarts"    waiting == "CrashLoopBackOff" -> "critical" to "CrashLoopBackOff, $restarts restarts"    !ready -> "high" to (waiting ?: "not ready")    restarts > 0 -> "medium" to "$restarts restarts"    else -> "none" to ""}cells["priority"] = prioritycells["priority_rank"] = mapOf("critical" to 1, "high" to 2, "medium" to 3, "none" to 4).getValue(priority)cells["signal"] = signal

// The key of the pod a row is about: namespace/pod — the pods, their usage and their events join on it.@Suppress("UNCHECKED_CAST")val cells = bindings["cells"] as MutableMap<String, Any?>cells["pod_key"] = "${cells["namespace"]}/${cells["pod"]}"

kind: Entityname: KubernetesPodlabel: Kubernetes poddescription: "One pod of the cluster, read from the Kubernetes API: its state, the operator's triage priority computed from that state, and the CPU it uses now against its limit. A pod to investigate first has priority critical, then high, then medium; or uses 90% or more of its CPU limit (cpu / cpu_limit)."match: [pod_key]scopes: [devops-pod-triage]columns:  pod_key:       { label: "namespace/pod — the pod's key" }  namespace:     { label: "Kubernetes namespace" }  pod:           { label: "Pod name" }  team:          { label: "Team that owns the pod (its team label)" }  ready:         { label: "Whether the pod's container is ready" }  restarts:      { label: "How many times its container restarted" }  priority:      { label: "Triage priority from the pod's state, by the operator's rule: critical, high or medium are to investigate; none needs no investigation" }  priority_rank: { label: "Order of the priority: 1 critical, 2 high, 3 medium, 4 none — sort by it, ascending" }  signal:        { label: "What in the pod's state set its priority (empty for none)" }  cpu_limit:     { label: "CPU limit, in cores" }  memory_limit:  { label: "Memory limit, in bytes" }  cpu:           { label: "CPU used now, in cores (metrics.k8s.io); saturated when cpu is 90% or more of cpu_limit" }  memory:        { label: "Memory used now, in bytes (metrics.k8s.io)" }sources:  - table: k8s_pods    map:      pod_key: pod_key      namespace: namespace      pod: pod      team: team      ready: ready      restarts: restarts      priority: priority      priority_rank: priority_rank      signal: signal      cpu_limit: cpu_limit      memory_limit: memory_limitjoins:  - table: k8s_pod_usage    from: pod_key    to: pod_key    map:      cpu: cpu      memory: memory

kind: Entityname: KubernetesWarninglabel: Kubernetes Warning event about a poddescription: "One Warning event the cluster recorded about a pod, read from the Kubernetes API: its reason, how many times it repeated, when it was last seen."match: [pod_key, reason]scopes: [devops-pod-triage]columns:  pod_key:   { label: "namespace/pod — the key of the pod the event is about" }  reason:    { label: "Why Kubernetes warned: Unhealthy, BackOff, FailedMount…" }  count:     { label: "How many times this event repeated" }  last_seen: { label: "When it was last seen" }  message:   { label: "Kubernetes' own message" }sources:  - table: k8s_warnings    map:      pod_key: pod_key      reason: reason      count: count      last_seen: last_seen      message: message