130 lines
6.3 KiB
YAML
130 lines
6.3 KiB
YAML
{{- if .Values.prometheus.enable | default true }}
|
|
{{- with $rules := default (dict) .Values.prometheus.rules }}
|
|
{{- if $rules.enable | default false }}
|
|
{{- with $predictFilledHours := $rules.predictFilledHours | default 12 }}
|
|
---
|
|
apiVersion: monitoring.coreos.com/v1
|
|
kind: PrometheusRule
|
|
metadata:
|
|
name: k8s-ephemeral-storage-container-limits
|
|
namespace: "{{ $.Release.Namespace }}"
|
|
labels:
|
|
{{- with $.Values.prometheus.additionalLabels }}
|
|
{{- toYaml . | nindent 4 }}
|
|
{{- end }}
|
|
spec:
|
|
groups:
|
|
- name: k8s.rules.container_resource
|
|
rules:
|
|
- alert: ContainerEphemeralStorageUsageAtLimit
|
|
annotations:
|
|
description: >-
|
|
{{ `Ephemeral storage usage of pod/container {{ $labels.pod_name
|
|
}}/{{ $labels.exported_container }} in Namespace {{
|
|
$labels.pod_namespace }} on Node {{ $labels.node_name }} {{ with
|
|
$labels.cluster -}} on Cluster {{ . }} {{- end }} is at {{ $value
|
|
}}% of the limit.` }}
|
|
summary: Container ephemeral storage usage is at the limit.
|
|
expr: |-2
|
|
( max by (node_name, pod_namespace, pod_name, exported_container)
|
|
(avg_over_time(ephemeral_storage_container_limit_percentage{source="container"}[5m]))
|
|
> 85.0)
|
|
# ignore pods that haven't been running for some time (e.g. completed jobs)
|
|
unless on (pod_namespace, pod_name)
|
|
( (label_replace(label_replace(
|
|
max_over_time(kube_pod_status_phase{phase="Running"}[2m]),
|
|
"pod_namespace", "$1", "namespace", "(.*)"),
|
|
"pod_name", "$1", "pod", "(.*)"))
|
|
== 0)
|
|
for: 1m
|
|
labels:
|
|
{{- $.Values.prometheus.rules.labels | toYaml | nindent 12 }}
|
|
|
|
- alert: ContainerEphemeralStorageUsageReachingLimit
|
|
annotations:
|
|
description: >-
|
|
{{ `Based on recent sampling, the ephemeral storage limit of
|
|
pod/container {{ $labels.pod_name }}/{{
|
|
$labels.exported_container }} in Namespace {{
|
|
$labels.pod_namespace }} on Node {{ $labels.node_name }} {{
|
|
with $labels.cluster -}} on Cluster {{ . }} {{- end }} is
|
|
expected to be reached within ` }}{{ $predictFilledHours }}
|
|
{{ ` hours. Currently, {{ $value }}% is used.` }}
|
|
summary: Container ephemeral storage usage is reaching the limit.
|
|
expr: |-2
|
|
( ( max by (node_name, pod_namespace, pod_name, exported_container)
|
|
(ephemeral_storage_container_limit_percentage{source="container"})
|
|
> {{ $rules.predictMinCurrentUsage | float64 }})
|
|
and on (pod_namespace, pod_name, exported_container)
|
|
( predict_linear( ephemeral_storage_container_limit_percentage{source="container"}[2h]
|
|
, {{ $predictFilledHours | float64 }}*3600)
|
|
> 99.0)
|
|
)
|
|
# ignore pods that haven't been running for enough time
|
|
unless on (pod_namespace, pod_name)
|
|
( (label_replace(label_replace(
|
|
min_over_time(kube_pod_status_phase{phase="Running"}[10m]),
|
|
"pod_namespace", "$1", "namespace", "(.*)"),
|
|
"pod_name", "$1", "pod", "(.*)"))
|
|
== 0)
|
|
for: 15m
|
|
labels:
|
|
{{- $.Values.prometheus.rules.labels | toYaml | nindent 12 }}
|
|
|
|
- alert: EphemeralStorageVolumeFilledUp
|
|
annotations:
|
|
description: >-
|
|
{{ `Ephemeral storage volume "{{ $labels.volume_name }}" of pod
|
|
{{ $labels.pod_name }} in Namespace {{ $labels.pod_namespace }}
|
|
{{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is
|
|
filled from {{ $value }}%.` }}
|
|
summary: Ephemeral storage volume is filled up.
|
|
expr: |-2
|
|
( max by (pod_namespace, pod_name, volume_name)
|
|
(avg_over_time(ephemeral_storage_container_volume_limit_percentage[5m]))
|
|
> 85.0)
|
|
# ignore pods that haven't been running for some time (e.g. completed jobs)
|
|
unless on (pod_namespace, pod_name)
|
|
( (label_replace(label_replace(
|
|
max_over_time(kube_pod_status_phase{phase="Running"}[2m]),
|
|
"pod_namespace", "$1", "namespace", "(.*)"),
|
|
"pod_name", "$1", "pod", "(.*)"))
|
|
== 0)
|
|
for: 1m
|
|
labels:
|
|
{{- $.Values.prometheus.rules.labels | toYaml | nindent 12 }}
|
|
|
|
- alert: EphemeralStorageVolumeFillingUp
|
|
annotations:
|
|
description: >-
|
|
{{ `Based on recent sampling, the ephemeral storage volume "{{
|
|
$labels.volume_name }}" of pod {{ $labels.pod_name }} in
|
|
Namespace {{ $labels.pod_namespace }} {{ with $labels.cluster
|
|
-}} on Cluster {{ . }} {{- end }} is expected to be filled up
|
|
within ` }}{{ $predictFilledHours }}{{ ` hours. Currently, {{
|
|
$value }}% is used.` }}
|
|
summary: Ephemeral storage volume is filling up.
|
|
expr: |-2
|
|
( ( max by (pod_namespace, pod_name, volume_name)
|
|
(ephemeral_storage_container_volume_limit_percentage)
|
|
> {{ $rules.predictMinCurrentUsage | float64 }})
|
|
and ( max by (pod_namespace, pod_name, volume_name)
|
|
(predict_linear( ephemeral_storage_container_volume_limit_percentage[2h]
|
|
, {{ $predictFilledHours | float64 }}*3600))
|
|
> 99)
|
|
)
|
|
# ignore pods that haven't been running for enough time
|
|
unless on (pod_namespace, pod_name)
|
|
( (label_replace(label_replace(
|
|
min_over_time(kube_pod_status_phase{phase="Running"}[10m]),
|
|
"pod_namespace", "$1", "namespace", "(.*)"),
|
|
"pod_name", "$1", "pod", "(.*)"))
|
|
== 0)
|
|
for: 15m
|
|
labels:
|
|
{{- $.Values.prometheus.rules.labels | toYaml | nindent 12 }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|
|
{{- end }}
|