194 lines
10 KiB
YAML
194 lines
10 KiB
YAML
apiVersion: v1
|
|
kind: ConfigMap
|
|
metadata:
|
|
name: prometheus-rulefiles
|
|
namespace: monitoring
|
|
data:
|
|
# Awesome Prometheus alerts
|
|
################################################################################
|
|
# Basic resource monitoring prometheus self-monitoring
|
|
################################################################################
|
|
prometheus-self-monitoring.yaml: |
|
|
groups:
|
|
- name: prometheus-self-monitoring
|
|
rules:
|
|
- alert: Watchdog
|
|
annotations:
|
|
description: |
|
|
This is an alert meant to ensure that the entire alerting pipeline is functional.
|
|
This alert is always firing, therefore it should always be firing in Alertmanager
|
|
and always fire against a receiver. There are integrations with various notification
|
|
mechanisms that send a notification when this alert is not firing. For example the
|
|
"DeadMansSnitch" integration in PagerDuty.
|
|
runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/watchdog
|
|
summary: An alert that should always be firing to certify that Alertmanager
|
|
is working properly.
|
|
expr: vector(1)
|
|
labels:
|
|
severity: none
|
|
# Awesome Prometheus alerts
|
|
################################################################################
|
|
# Basic resource monitoring host and hardware (node-export)
|
|
################################################################################
|
|
prometheus-host-hardware.yaml: |
|
|
groups:
|
|
- name: prometheus-host-hardware
|
|
rules:
|
|
- alert: HostOutOfMemory (1.2.01)
|
|
expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Host out of memory (instance {{ $labels.instance }})
|
|
description: "Node memory is filling up (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
- alert: HostOutOfDiskSpace (1.2.07)
|
|
expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Host out of disk space (instance {{ $labels.instance }})
|
|
description: "Disk is almost full (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
# Please add ignored mountpoints in node_exporter parameters like
|
|
# "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|run)($|/)".
|
|
# Same rule using "node_filesystem_free_bytes" will fire when disk fills for non-root users.
|
|
- alert: HostHighCpuLoad (1.2.13)
|
|
expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 80
|
|
for: 0m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Host high CPU load (instance {{ $labels.instance }})
|
|
description: "CPU load is > 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
- alert: HostOomKillDetected (1.2.24)
|
|
expr: increase(node_vmstat_oom_kill[1m]) > 0
|
|
for: 0m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Host OOM kill detected (instance {{ $labels.instance }})
|
|
description: "OOM kill detected\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
# Awesome Prometheus alerts
|
|
################################################################################
|
|
# Basic resource monitoring docker container (google/cAdvisor)
|
|
################################################################################
|
|
Prometheus-docker-containers.yaml: |
|
|
groups:
|
|
- name: prometheus-docker-containers
|
|
rules:
|
|
# This rule can be very noisy in dynamic infra with legitimate container start/stop/deployment.
|
|
- alert: ContainerKilled (1.3.1)
|
|
expr: time() - container_last_seen > 60
|
|
for: 0m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Container killed (instance {{ $labels.instance }})
|
|
description: "A container has disappeared\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
# This rule can be very noisy in dynamic infra with legitimate container start/stop/deployment.
|
|
- alert: ContainerCpuUsage (1.3.3)
|
|
expr: (sum(rate(container_cpu_usage_seconds_total{name!=""}[3m])) BY (instance, name) * 100) > 80
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Container CPU usage (instance {{ $labels.instance }})
|
|
description: "Container CPU usage is above 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
# See https://medium.com/faun/how-much-is-too-much-the-linux-oomkiller-and-used-memory-d32186f29c9d
|
|
- alert: ContainerMemoryUsage (1.3.4)
|
|
expr: (sum(container_memory_working_set_bytes{name!=""}) BY (instance, name) / sum(container_spec_memory_limit_bytes > 0) BY (instance, name) * 100) > 80
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Container Memory usage (instance {{ $labels.instance }})
|
|
description: "Container Memory usage is above 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
# Awesome Prometheus alerts
|
|
################################################################################
|
|
# Orchestrators kubernetes (kube-state-metrics)
|
|
################################################################################
|
|
prometheus-kubernetes.yaml: |
|
|
groups:
|
|
- name: prometheus-kubernetes
|
|
rules:
|
|
- alert: KubernetesNodeReady (5.1.01)
|
|
expr: kube_node_status_condition{condition="Ready",status="true"} == 0
|
|
for: 10m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Kubernetes Node ready (instance {{ $labels.instance }})
|
|
description: "Node {{ $labels.node }} has been unready for a long time\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
- alert: KubernetesOutOfDisk (5.1.04)
|
|
expr: kube_node_status_condition{condition="OutOfDisk",status="true"} == 1
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Kubernetes out of disk (instance {{ $labels.instance }})
|
|
description: "{{ $labels.node }} has OutOfDisk condition\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
- alert: KubernetesContainerOomKiller (5.1.06)
|
|
expr: (kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 10m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[10m]) == 1
|
|
for: 0m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Kubernetes container oom killer (instance {{ $labels.instance }})
|
|
description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 10 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
- alert: KubernetesVolumeOutOfDiskSpace (5.1.10)
|
|
expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 10
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Kubernetes Volume out of disk space (instance {{ $labels.instance }})
|
|
description: "Volume is almost full (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
- alert: KubernetesPersistentvolumeError (5.1.12)
|
|
expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending", job="kube-state-metrics"} > 0
|
|
for: 0m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Kubernetes PersistentVolume error (instance {{ $labels.instance }})
|
|
description: "Persistent volume is in bad state\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
- alert: KubernetesHpaScaleCapability (5.1.16)
|
|
expr: kube_horizontalpodautoscaler_status_desired_replicas >= kube_horizontalpodautoscaler_spec_max_replicas
|
|
for: 2m
|
|
labels:
|
|
severity: info
|
|
annotations:
|
|
summary: Kubernetes HPA scale capability (instance {{ $labels.instance }})
|
|
description: "The maximum number of desired Pods has been hit\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
- alert: KubernetesPodNotHealthy (5.1.17)
|
|
expr: min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"})[15m:1m]) > 0
|
|
for: 0m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Kubernetes Pod not healthy (instance {{ $labels.instance }})
|
|
description: "Pod has been in a non-ready state for longer than 15 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
- alert: KubernetesPodCrashLooping (5.1.18)
|
|
expr: increase(kube_pod_container_status_restarts_total[1m]) > 3
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: Kubernetes pod crash looping (instance {{ $labels.instance }})
|
|
description: "Pod {{ $labels.pod }} is crash looping\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
- alert: KubernetesApiServerErrors (5.1.29)
|
|
expr: sum(rate(apiserver_request_total{job="apiserver",code=~"^(?:5..)$"}[1m])) / sum(rate(apiserver_request_total{job="apiserver"}[1m])) * 100 > 3
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Kubernetes API server errors (instance {{ $labels.instance }})
|
|
description: "Kubernetes API server is experiencing high error rate\n VALUE = {{ $value }}\n LABELS = {{ $labels }}"
|
|
- alert: KubernetesClientCertificateExpiresSoon (5.1.32)
|
|
expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 24*60*60
|
|
for: 0m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: Kubernetes client certificate expires soon (instance {{ $labels.instance }})
|
|
description: "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" |