ddn root metrics add

ephemeral_usage/pod_ephemeral_storage.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+# 모든 노드 순회
+for n in $(kubectl get node -o name); do
+  # kubelet stats summary API 호출 후 Pod별 ephemeral-storage 사용량 추출
+  kubectl get --raw "/api/v1/nodes/${n##*/}/proxy/stats/summary" \
+  | jq -r '.pods[] | [.podRef.namespace, .podRef.name, ."ephemeral-storage".usedBytes] | @tsv'
+done > pod_ephemeral_storage.tsv
+

ephemeral_usage/pod_ephemeral_storage.tsv

@@ -0,0 +1,833 @@
+gpu-operator	gpu-feature-discovery-zzg9z	14520320
+gpu-operator	nvidia-device-plugin-daemonset-lv98q	69632
+gpu-operator	nvidia-mig-manager-zhzgk	299008
+logging	fluent-bit-25gzg	12288
+calico-system	csi-node-driver-jvp24	8192
+kube-system	kube-proxy-wsrk7	45056
+calico-system	calico-node-xrrwv	40607744
+default	exa-csi-metrics-exporter-nxv55	8192
+gpu-operator	gpu-operator-node-feature-discovery-worker-7cppx	12144640
+dcgm-exporter	nvidia-dcgm-exporter-custom-2zwd7	154165248
+gpu-operator	nvidia-container-toolkit-daemonset-mqq8m	36864
+monitoring	prometheus-prometheus-node-exporter-mgbqt	12288
+default	exascaler-csi-node-qdw2x	176128
+gpu-operator	nvidia-operator-validator-rk9cj	8192
+gpu-operator	nvidia-container-toolkit-daemonset-8hrnw	36864
+default	exa-csi-metrics-exporter-9ml2r	946176
+gpu-operator	gpu-operator-node-feature-discovery-worker-zvns8	12161024
+logging	fluent-bit-wv9ps	12288
+project-0198ef11-c816-7f26-990a-a3c4df550348	workload-019960f2-1db4-7184-aeba-6c0022f8b23f	913309696
+dcgm-exporter	nvidia-dcgm-exporter-custom-5bs6d	155787264
+gpu-operator	gpu-feature-discovery-qt6t5	14503936
+gpu-operator	nvidia-operator-validator-4hmps	8192
+gpu-operator	nvidia-device-plugin-daemonset-22m7l	69632
+gpu-operator	nvidia-mig-manager-djdxg	299008
+project-0198ef25-200f-7317-9e6f-c3e5641f3ca4	workload-019930e7-9d98-7190-b4bb-403f8771d6e1	8883675136
+calico-system	csi-node-driver-6m44k	8192
+default	exascaler-csi-node-kd6qf	9768960
+kube-system	kube-proxy-w7tf8	45056
+calico-system	calico-node-76t24	42262528
+monitoring	prometheus-prometheus-node-exporter-lkfcw	12288
+gpu-operator	gpu-operator-node-feature-discovery-worker-qb8pg	12144640
+default	exa-csi-metrics-exporter-g5hlm	339968
+calico-system	csi-node-driver-5mhnq	8192
+gpu-operator	gpu-feature-discovery-v94dw	14499840
+gpu-operator	nvidia-operator-validator-jrs49	8192
+logging	fluent-bit-jvzjs	12288
+monitoring	prometheus-prometheus-node-exporter-jvdg2	12288
+gpu-operator	nvidia-mig-manager-n9f6s	299008
+project-0198ef22-7f42-73a9-bc72-8b980c0c1e04	workload-01993c10-33fe-716a-9200-5747fe225c93	8172740608
+gpu-operator	nvidia-device-plugin-daemonset-v92qc	69632
+gpu-operator	nvidia-container-toolkit-daemonset-bssgs	36864
+default	exascaler-csi-node-fz5vd	4030464
+gpulive	pvc-inspector-5659b4576d-hgdcm	8192
+calico-system	calico-node-7zgnp	40546304
+dcgm-exporter	nvidia-dcgm-exporter-custom-ghqmd	154165248
+kube-system	kube-proxy-s22rb	61440
+gpu-operator	gpu-feature-discovery-8bjg2	14438400
+logging	fluent-bit-lhl6h	12288
+kube-system	kube-proxy-dpn5d	45056
+calico-system	calico-node-28rzf	41398272
+monitoring	prometheus-prometheus-node-exporter-ct5h9	12288
+gpu-operator	nvidia-device-plugin-daemonset-75scc	69632
+project-0198ef23-5f82-7b2e-9920-55762dd2680d	workload-019950e4-0720-71db-ab95-46a55cae3fa4	13574742016
+project-0198ef21-88fc-70ba-a461-e5a3dafcf38c	workload-019955bf-f637-780e-a4f1-fbd4c282e7a2	13575716864
+default	exascaler-csi-node-846w5	11583488
+dcgm-exporter	nvidia-dcgm-exporter-custom-bs8nx	154423296
+gpu-operator	nvidia-container-toolkit-daemonset-ft79h	36864
+default	exa-csi-metrics-exporter-hgbdl	946176
+gpu-operator	gpu-operator-node-feature-discovery-worker-ngdwf	12156928
+project-0198ef21-88fc-70ba-a461-e5a3dafcf38c	workload-01990a7c-1ef9-79e6-b8ad-15078958c4af	5864775680
+gpu-operator	nvidia-mig-manager-2gvff	299008
+project-0198ef21-88fc-70ba-a461-e5a3dafcf38c	workload-01990a04-99cc-7bbd-b247-f6a34aec754a	1135833088
+calico-system	csi-node-driver-67xxx	8192
+gpu-operator	nvidia-operator-validator-74t92	8192
+calico-system	csi-node-driver-kv8rq	8192
+project-0198ef26-318b-76cd-a2a0-1944f1aa2e4d	workload-019960bc-8b7c-74ab-a623-89769c3e9299	453714366464
+gpu-operator	nvidia-mig-manager-j6zzk	299008
+kube-system	kube-proxy-f42jt	131072
+logging	fluent-bit-2rft7	12288
+monitoring	prometheus-prometheus-node-exporter-2m7g8	12288
+calico-system	calico-node-qkkhs	40796160
+default	exa-csi-metrics-exporter-kp5h4	176128
+gpu-operator	gpu-feature-discovery-z4nv9	14430208
+gpu-operator	nvidia-device-plugin-daemonset-f9xpn	507904
+default	exascaler-csi-node-cnvzl	1249280
+gpu-operator	gpu-operator-node-feature-discovery-worker-6v94s	12152832
+dcgm-exporter	nvidia-dcgm-exporter-custom-f8l77	156045312
+gpu-operator	nvidia-operator-validator-gpnjp	8192
+gpu-operator	nvidia-container-toolkit-daemonset-mjsm9	36864
+project-0198ef11-c816-7f26-990a-a3c4df550348	workload-0199273e-2348-785a-b93e-4420f885e369	61302378496
+gpu-operator	gpu-feature-discovery-vxpwx	14516224
+gpu-operator	nvidia-device-plugin-daemonset-hbx2m	69632
+kube-system	kube-proxy-tzvsc	45056
+dcgm-exporter	nvidia-dcgm-exporter-custom-xm9dc	156045312
+gpu-operator	nvidia-container-toolkit-daemonset-ckfbb	36864
+gpu-operator	gpu-operator-node-feature-discovery-worker-d97nd	12144640
+gpu-operator	nvidia-mig-manager-7r2t9	299008
+calico-system	csi-node-driver-7ljdq	8192
+default	exa-csi-metrics-exporter-7xffg	884736
+gpu-operator	nvidia-operator-validator-vvqqq	8192
+default	exascaler-csi-node-47h99	6352896
+monitoring	prometheus-prometheus-node-exporter-fgxbt	12288
+project-0198ef25-200f-7317-9e6f-c3e5641f3ca4	workload-01993607-0d7f-72ed-9606-a417cb93d576	8341917696
+logging	fluent-bit-87zmf	12288
+calico-system	calico-node-2k8hm	40448000
+gpu-operator	nvidia-operator-validator-n2lnz	8192
+gpu-operator	nvidia-container-toolkit-daemonset-m6tkw	36864
+project-0198ef1f-4abb-76b8-9412-5f3122883138	workload-01994d27-d6af-7ae3-b7e8-885adc9fe981	93488406528
+gpu-operator	nvidia-device-plugin-daemonset-2pvml	69632
+kube-system	kube-proxy-8mv5c	45056
+project-0198ef1f-ffb9-7827-8c60-a6c1d27aa145	workload-01996f01-5fb6-78a1-9d4c-2977a07ecedb	595062784
+monitoring	prometheus-prometheus-node-exporter-bzztb	200704
+default	exascaler-csi-node-87qrv	8425472
+gpu-operator	gpu-feature-discovery-4lqv4	14467072
+gpu-operator	gpu-operator-node-feature-discovery-worker-tb4lf	12152832
+default	exa-csi-metrics-exporter-2hxf6	491520
+calico-system	csi-node-driver-hnb5k	8192
+logging	fluent-bit-htxfw	12288
+gpu-operator	nvidia-mig-manager-q7rk5	299008
+dcgm-exporter	nvidia-dcgm-exporter-custom-6qfj4	156069888
+calico-system	calico-node-nzb9j	41701376
+project-0198ef23-5f82-7b2e-9920-55762dd2680d	workload-0199551d-ff57-7ad0-ac5d-0919feeb4fef	1710649344
+dcgm-exporter	nvidia-dcgm-exporter-custom-8jkg9	159338496
+gpu-operator	nvidia-container-toolkit-daemonset-msntx	36864
+default	exa-csi-metrics-exporter-p77z7	823296
+gpu-operator	nvidia-device-plugin-daemonset-pfvnw	69632
+gpu-operator	nvidia-mig-manager-7hp8n	299008
+default	exascaler-csi-node-89nq4	10043392
+gpu-operator	nvidia-operator-validator-thr4k	8192
+kube-system	kube-proxy-gdlxf	45056
+monitoring	prometheus-prometheus-node-exporter-sj6r9	131072
+project-0198ef24-a4c7-7e77-8f99-ca484eab6873	workload-01996335-60f8-76cd-b967-deb151cef04d	96037683200
+calico-system	csi-node-driver-2phl5	8192
+gpu-operator	gpu-operator-node-feature-discovery-worker-22bc9	12148736
+logging	fluent-bit-mp6zh	12288
+project-0198ef22-f34e-7937-ac04-e3414897d24b	workload-01992dcf-c257-7d1f-a680-cca2bfc03b2b	17039753216
+calico-system	calico-node-xmtnn	41795584
+gpu-operator	gpu-feature-discovery-jbnj4	14426112
+gpu-operator	gpu-operator-node-feature-discovery-worker-l7nd9	12140544
+monitoring	prometheus-prometheus-node-exporter-27qpl	12288
+gpu-operator	nvidia-mig-manager-kpfdc	299008
+logging	fluent-bit-t7tdv	12288
+gpu-operator	nvidia-device-plugin-daemonset-llrr8	69632
+project-0198ef1f-4abb-76b8-9412-5f3122883138	workload-01997413-6593-744a-b1a7-7b675f46778d	217088
+kube-system	kube-proxy-tk5nc	57344
+default	exascaler-csi-node-nb2ps	4141056
+gpu-operator	nvidia-container-toolkit-daemonset-k6w42	36864
+gpu-operator	nvidia-operator-validator-7m8c8	8192
+project-0198ef25-200f-7317-9e6f-c3e5641f3ca4	workload-01994c45-0d9d-721a-a8b6-a76c12e6c6d5	21612929024
+gpu-operator	gpu-feature-discovery-hz2f8	14491648
+calico-system	csi-node-driver-krr2t	8192
+default	exa-csi-metrics-exporter-s9mpv	856064
+project-0198ef1f-4abb-76b8-9412-5f3122883138	workload-019965a6-d0ec-7932-b248-92b8c4138be1	14975057920
+dcgm-exporter	nvidia-dcgm-exporter-custom-qk46k	156049408
+calico-system	calico-node-5jjm6	40542208
+gpu-operator	nvidia-container-toolkit-daemonset-92mf8	36864
+default	exascaler-csi-node-rccfj	1028096
+gpu-operator	gpu-operator-node-feature-discovery-worker-7kns6	12140544
+dcgm-exporter	nvidia-dcgm-exporter-custom-9jtm9	155783168
+calico-system	calico-node-bv69r	40263680
+default	exa-csi-metrics-exporter-784cf	8192
+calico-system	csi-node-driver-6tz8j	8192
+monitoring	prometheus-prometheus-node-exporter-7x2fw	12288
+kube-system	kube-proxy-q4fgl	49152
+gpu-operator	gpu-feature-discovery-hb9q4	14512128
+gpu-operator	nvidia-operator-validator-kpbrg	8192
+logging	fluent-bit-5dnpt	12288
+gpu-operator	nvidia-device-plugin-daemonset-w9xv4	69632
+gpu-operator	nvidia-mig-manager-c64bh	299008
+default	exascaler-csi-node-8s588	2916352
+gpu-operator	nvidia-device-plugin-daemonset-9mx9h	77824
+gpu-operator	nvidia-operator-validator-ftlzn	8192
+calico-system	calico-node-nj7gs	40075264
+logging	fluent-bit-hvtkl	12288
+dcgm-exporter	nvidia-dcgm-exporter-custom-fqlpc	154169344
+gpu-operator	gpu-feature-discovery-xd4mx	14299136
+calico-system	csi-node-driver-6wb4b	8192
+gpu-operator	nvidia-container-toolkit-daemonset-fzz5g	36864
+monitoring	prometheus-prometheus-node-exporter-bl9dr	299008
+kube-system	kube-proxy-5v6sd	81920
+gpu-operator	gpu-operator-node-feature-discovery-worker-j9xqj	12140544
+project-0198ef1d-84c0-72fd-9a84-1e261d5c36fe	workload-01990a0d-e489-7e5e-8057-d8ad4f5c7277	33686519808
+default	exa-csi-metrics-exporter-lq228	155648
+gpu-operator	nvidia-mig-manager-d78hg	299008
+kube-system	kube-proxy-ctrrz	45056
+gpu-operator	nvidia-mig-manager-4gnf6	299008
+dcgm-exporter	nvidia-dcgm-exporter-custom-ph2pt	154161152
+gpu-operator	nvidia-device-plugin-daemonset-x9jx5	69632
+gpu-operator	gpu-feature-discovery-6rkpv	14520320
+gpu-operator	nvidia-container-toolkit-daemonset-9svqf	36864
+gpu-operator	nvidia-operator-validator-tk6x9	8192
+calico-system	csi-node-driver-sxlwk	8192
+calico-system	calico-node-mxwrb	40886272
+default	exascaler-csi-node-sv4ng	614400
+gpu-operator	gpu-operator-node-feature-discovery-worker-m7qv7	12148736
+monitoring	seeker-79d489fdfc-mk5qw	2035712
+default	exa-csi-metrics-exporter-mf6xv	11522048
+monitoring	prometheus-prometheus-node-exporter-7v5g6	12288
+logging	fluent-bit-frcrw	12288
+gpu-operator	gpu-operator-node-feature-discovery-worker-m7skp	12144640
+gpu-operator	nvidia-operator-validator-p4nkr	8192
+default	exa-csi-metrics-exporter-9bz8z	655360
+project-0198ef22-f34e-7937-ac04-e3414897d24b	workload-0199604b-732e-7f98-bd4a-e5e7b62d7aee	97411362816
+dcgm-exporter	nvidia-dcgm-exporter-custom-jk9gf	154423296
+gpu-operator	nvidia-container-toolkit-daemonset-qknng	36864
+project-0198ef24-a4c7-7e77-8f99-ca484eab6873	workload-0199503f-983c-77fe-9a3d-7b37c1691f0a	313804251136
+calico-system	csi-node-driver-d4t8b	8192
+kube-system	kube-proxy-hw6nt	49152
+default	exascaler-csi-node-qzbld	6463488
+gpu-operator	nvidia-mig-manager-8f9r2	299008
+logging	fluent-bit-9rm68	12288
+gpu-operator	nvidia-device-plugin-daemonset-wthd4	69632
+gpu-operator	gpu-feature-discovery-mnfxc	14516224
+monitoring	prometheus-prometheus-node-exporter-shpwv	40960
+project-0198ef21-88fc-70ba-a461-e5a3dafcf38c	workload-0199575f-aaa6-705c-8cb1-cc139dea0693	6773768192
+calico-system	calico-node-jq2gs	41050112
+gpu-operator	nvidia-container-toolkit-daemonset-trxkn	36864
+project-0198ef23-c509-7898-a5b6-3985c17ca892	workload-01992200-2295-7a53-866c-02d67c115d2a	803872768
+monitoring	prometheus-prometheus-node-exporter-pn7hq	12288
+calico-system	calico-node-r87bt	40284160
+gpu-operator	gpu-feature-discovery-pfksx	14512128
+gpu-operator	nvidia-operator-validator-gcnrr	8192
+kube-system	kube-proxy-4rqtb	49152
+default	exascaler-csi-node-h6kc2	2605056
+logging	fluent-bit-hcxvs	12288
+dcgm-exporter	nvidia-dcgm-exporter-custom-nd87q	159092736
+default	exa-csi-metrics-exporter-w2bdc	184320
+gpu-operator	nvidia-mig-manager-b498b	299008
+gpu-operator	nvidia-device-plugin-daemonset-2xjx9	69632
+calico-system	csi-node-driver-24t5q	8192
+gpu-operator	gpu-operator-node-feature-discovery-worker-psdwh	12140544
+gpu-operator	gpu-feature-discovery-vlwzz	14516224
+logging	fluent-bit-4bvsb	12288
+gpu-operator	gpu-operator-node-feature-discovery-worker-9pnzp	12144640
+default	exascaler-csi-node-w5l98	6471680
+gpu-operator	nvidia-device-plugin-daemonset-shzxh	69632
+calico-system	csi-node-driver-2tkn6	8192
+gpu-operator	nvidia-container-toolkit-daemonset-bqq8j	36864
+gpu-operator	nvidia-operator-validator-9tc6q	8192
+kube-system	kube-proxy-8qbfx	45056
+dcgm-exporter	nvidia-dcgm-exporter-custom-5t64g	155795456
+gpu-operator	nvidia-mig-manager-kmnqm	299008
+monitoring	prometheus-prometheus-node-exporter-q4kdm	12288
+project-0198ef22-f34e-7937-ac04-e3414897d24b	workload-019933aa-9a40-7462-87c0-89beddd522a9	14004805632
+default	exa-csi-metrics-exporter-mfwb5	700416
+project-0198ef1f-ffb9-7827-8c60-a6c1d27aa145	workload-0199750d-44be-7aac-88b1-9e481a8bbb06	25213853696
+calico-system	calico-node-pflt9	40751104
+project-0198ef25-200f-7317-9e6f-c3e5641f3ca4	workload-01993693-28ba-79af-86cc-d87777a2fce1	14447038464
+project-0198ef29-b8d5-707e-9399-950abcd5afce	workload-0199094d-30a7-78ce-af75-2ea3f8add516	237568
+gpu-operator	gpu-operator-node-feature-discovery-worker-fp6rh	12132352
+dcgm-exporter	nvidia-dcgm-exporter-custom-xv4l8	155951104
+gpu-operator	nvidia-mig-manager-4wchj	303104
+calico-system	csi-node-driver-466hp	8192
+gpu-operator	gpu-feature-discovery-rqf8s	14610432
+monitoring	prometheus-prometheus-node-exporter-ttfdt	12288
+calico-system	calico-node-whl78	40050688
+project-0198ef2a-e1b4-72c7-937b-f760dce5d7db	workload-01990d2b-a60f-7ce8-816c-09c06e467866	508761153536
+logging	fluent-bit-zswb9	12288
+kube-system	kube-proxy-j4d4p	40960
+gpu-operator	nvidia-operator-validator-vfj6x	8192
+project-0198dee9-7bc5-7a85-a5cc-96bf056ce33d	workload-01990990-cb58-76cc-a64b-ce8eba84fba5	20044255232
+default	exa-csi-metrics-exporter-cfzgd	442368
+gpu-operator	nvidia-device-plugin-daemonset-w89l5	69632
+project-0198ef24-a98e-78b2-b74a-e1c22f89f207	workload-0199090a-e4eb-77be-ac72-71515bef3bf7	548864
+gpu-operator	nvidia-container-toolkit-daemonset-kcg69	36864
+default	exascaler-csi-node-2tp2z	3137536
+project-0198ef2a-e16b-7e7f-93d4-f7f40267930b	workload-01993fa6-d115-77e3-a6d2-1636ba94e34a	19763908608
+gpu-operator	gpu-feature-discovery-7xp2v	14614528
+monitoring	prometheus-prometheus-node-exporter-rc68r	12288
+gpu-operator	nvidia-mig-manager-9t2sb	303104
+gpu-operator	nvidia-device-plugin-daemonset-5lrv8	69632
+calico-system	calico-node-xvt6t	42422272
+calico-system	csi-node-driver-pxh4h	8192
+gpu-operator	nvidia-operator-validator-dr6dk	8192
+gpu-operator	nvidia-container-toolkit-daemonset-x9ctx	36864
+default	exascaler-csi-node-xtr27	8589312
+project-0198ef17-2356-7faf-b292-d1e88044082e	workload-01996c11-d31a-7c82-bc5b-de169f92e7ff	162210209792
+default	exa-csi-metrics-exporter-66z4x	212992
+logging	fluent-bit-v48xc	12288
+gpu-operator	gpu-operator-node-feature-discovery-worker-hvgkq	12156928
+kube-system	kube-proxy-hft5c	45056
+dcgm-exporter	nvidia-dcgm-exporter-custom-pdjct	155959296
+project-0198ef2a-e16b-7e7f-93d4-f7f40267930b	workload-019942e0-913c-7498-96f1-6fb4345e7775	42303004672
+kube-system	kube-proxy-j4j8n	40960
+gpu-operator	gpu-operator-node-feature-discovery-worker-jqc5f	12144640
+gpu-operator	nvidia-device-plugin-daemonset-66xql	69632
+dcgm-exporter	nvidia-dcgm-exporter-custom-n5f6p	155123712
+calico-system	csi-node-driver-ldfjw	8192
+calico-system	calico-node-vxw8r	40845312
+gpu-operator	gpu-feature-discovery-kdhkn	14618624
+default	exa-csi-metrics-exporter-qskjp	8192
+logging	fluent-bit-n5k4n	12288
+gpu-operator	nvidia-mig-manager-zzqp6	303104
+default	exascaler-csi-node-s84s4	6774784
+gpu-operator	nvidia-container-toolkit-daemonset-g4gwz	36864
+gpu-operator	nvidia-operator-validator-dxs94	8192
+monitoring	prometheus-prometheus-node-exporter-plp7l	12288
+monitoring	prometheus-prometheus-node-exporter-6gvcv	20480
+calico-system	csi-node-driver-8qx6p	12288
+dcgm-exporter	nvidia-dcgm-exporter-custom-m7tjf	111685632
+logging	fluent-bit-mlj8j	20480
+gpu-operator	nvidia-device-plugin-daemonset-kcjcs	122880
+gpu-operator	nvidia-container-toolkit-daemonset-kslg6	65536
+calico-system	calico-node-x9xdd	35176448
+default	exa-csi-metrics-exporter-gcvmg	8192
+default	exascaler-csi-node-44fqf	2883584
+gpu-operator	nvidia-mig-manager-2nwgl	315392
+kube-system	kube-proxy-v22ft	45056
+gpu-operator	gpu-operator-node-feature-discovery-worker-xbvxq	21561344
+gpu-operator	gpu-feature-discovery-kfwvl	33406976
+gpu-operator	nvidia-operator-validator-vxj6k	12288
+default	exascaler-csi-node-nfrt5	3932160
+dcgm-exporter	nvidia-dcgm-exporter-custom-glrss	155033600
+gpu-operator	nvidia-operator-validator-wd64t	8192
+project-0198ef29-361c-7e82-96c2-d205afa24e71	workload-01990ac7-0d60-706e-88b8-b1d6d46394ab	56540262400
+gpu-operator	nvidia-mig-manager-6j5c8	303104
+gpu-operator	gpu-feature-discovery-tbfbj	14606336
+gpu-operator	nvidia-device-plugin-daemonset-dkv7x	77824
+logging	fluent-bit-nqbx2	12288
+monitoring	prometheus-prometheus-node-exporter-5lwvp	12288
+calico-system	calico-node-x6skj	39313408
+kube-system	kube-proxy-2hx98	45056
+calico-system	csi-node-driver-s26rp	8192
+gpu-operator	gpu-operator-node-feature-discovery-worker-fgl6z	12132352
+gpu-operator	nvidia-container-toolkit-daemonset-9tdrl	36864
+default	exa-csi-metrics-exporter-htrxb	118784
+dcgm-exporter	nvidia-dcgm-exporter-custom-sjcwv	155947008
+logging	fluent-bit-8cj7t	12288
+default	exascaler-csi-node-22qz6	2293760
+gpu-operator	nvidia-operator-validator-dwzkp	8192
+gpu-operator	nvidia-container-toolkit-daemonset-9s5l2	36864
+gpu-operator	nvidia-mig-manager-h669m	303104
+gpu-operator	nvidia-device-plugin-daemonset-c2s4c	69632
+monitoring	prometheus-prometheus-node-exporter-kp6zf	12288
+project-0198ef27-a340-7ce7-ab2f-cdba3755fc9c	workload-01997061-2ee4-7c63-b893-bcf3f4c6649c	7203377152
+kube-system	kube-proxy-s4hrj	40960
+calico-system	csi-node-driver-w2sbw	8192
+calico-system	calico-node-bcjzb	41271296
+gpu-operator	gpu-operator-node-feature-discovery-worker-rllw2	12144640
+gpu-operator	gpu-feature-discovery-b6xsb	14598144
+default	exa-csi-metrics-exporter-qns5b	483328
+monitoring	prometheus-prometheus-node-exporter-ww9vd	12288
+dcgm-exporter	nvidia-dcgm-exporter-custom-4pvrv	155123712
+gpu-operator	nvidia-operator-validator-fsb2x	8192
+gpu-operator	nvidia-container-toolkit-daemonset-77k6h	36864
+gpu-operator	gpu-operator-node-feature-discovery-worker-8lh7v	12128256
+default	exascaler-csi-node-8stdc	2215936
+gpu-operator	nvidia-device-plugin-daemonset-54r2n	69632
+gpu-operator	nvidia-mig-manager-85trf	303104
+calico-system	calico-node-nnhh9	49225728
+project-0198ef21-86cd-77b5-8560-60f9366d15fa	workload-019974d1-a471-79e5-9a08-a02064688938	455385088
+gpu-operator	gpu-feature-discovery-tztsw	14598144
+logging	fluent-bit-p4lqm	12288
+kube-system	kube-proxy-dpk7w	45056
+calico-system	csi-node-driver-ft4tx	8192
+default	exa-csi-metrics-exporter-b4sm2	851968
+gpu-operator	nvidia-container-toolkit-daemonset-f8nbh	36864
+monitoring	prometheus-prometheus-node-exporter-dkc5n	12288
+dcgm-exporter	nvidia-dcgm-exporter-custom-tnpfv	155344896
+gpu-operator	nvidia-mig-manager-qq2zc	303104
+gpu-operator	gpu-feature-discovery-c9t2l	14622720
+default	exascaler-csi-node-rwwqc	2949120
+project-0198ef28-9a01-7d4f-a08d-b96fc255eee6	workload-01991795-98de-77e7-8db8-e47736b4bb3d	218070331392
+gpu-operator	nvidia-operator-validator-xs9mt	8192
+kube-system	kube-proxy-qgq56	40960
+logging	fluent-bit-jk6n7	12288
+gpu-operator	gpu-operator-node-feature-discovery-worker-l47r8	12132352
+calico-system	csi-node-driver-pq5wl	8192
+default	exa-csi-metrics-exporter-q9xzr	118784
+calico-system	calico-node-cqgz6	39579648
+gpu-operator	nvidia-device-plugin-daemonset-46l5z	69632
+default	exa-csi-metrics-exporter-qr4b8	581632
+gpu-operator	nvidia-device-plugin-daemonset-mq7pb	73728
+logging	fluent-bit-lvpbf	12288
+default	exascaler-csi-node-s442b	5390336
+project-0198ef17-2356-7faf-b292-d1e88044082e	workload-019973ee-865b-76db-a85b-63ad874a8932	6600462336
+calico-system	csi-node-driver-h8twx	8192
+gpu-operator	gpu-operator-node-feature-discovery-worker-jdfs7	12136448
+calico-system	calico-node-2ks6w	40558592
+project-01992d55-abdd-7770-b94b-c1b3c2684060	workload-019960c7-8984-7db5-bb78-1677cf1076e9	61833879552
+gpu-operator	gpu-feature-discovery-6g2lp	14610432
+dcgm-exporter	nvidia-dcgm-exporter-custom-txlvf	155115520
+gpu-operator	nvidia-operator-validator-5whrl	8192
+gpu-operator	nvidia-container-toolkit-daemonset-64wk4	36864
+project-01995015-81df-7777-9702-d45a4ac32098	workload-019974a1-8fa3-72b9-b637-eab6e5570a56	19768442880
+monitoring	prometheus-prometheus-node-exporter-dhk9h	12288
+gpu-operator	nvidia-mig-manager-kjwmx	303104
+kube-system	kube-proxy-79g9c	45056
+default	exa-csi-metrics-exporter-nq6w6	233472
+dcgm-exporter	nvidia-dcgm-exporter-custom-lb99p	155967488
+gpu-operator	gpu-operator-node-feature-discovery-worker-wqwlk	12136448
+logging	fluent-bit-fr5nj	12288
+calico-system	csi-node-driver-tlc7p	8192
+project-0198ef2a-7b36-799f-8f09-770c7c16a684	workload-01991f69-c35f-73ef-b55f-91be89144b5d	18154962944
+monitoring	prometheus-prometheus-node-exporter-69zzt	12288
+calico-system	calico-node-f2zjh	40194048
+gpu-operator	nvidia-operator-validator-q55lm	8192
+gpu-operator	nvidia-device-plugin-daemonset-6w25m	139264
+project-0198eec1-a3c3-7cf9-86a5-3c720847bd02	workload-0199089d-ad45-7cb1-8f00-ea63fa2d21a6	239140864
+gpu-operator	nvidia-mig-manager-n2m6n	303104
+gpu-operator	gpu-feature-discovery-qdrd5	14622720
+default	exascaler-csi-node-rm5b8	2351104
+kube-system	kube-proxy-fwz5b	40960
+gpu-operator	nvidia-container-toolkit-daemonset-q7hb7	36864
+kube-system	kube-proxy-xs5qb	45056
+gpu-operator	nvidia-device-plugin-daemonset-c84rp	69632
+gpu-operator	gpu-feature-discovery-4ppq7	14602240
+gpu-operator	nvidia-operator-validator-g2cdr	8192
+gpu-operator	gpu-operator-node-feature-discovery-worker-7965x	12132352
+monitoring	prometheus-prometheus-node-exporter-5skwl	12288
+default	exascaler-csi-node-49mwn	3358720
+gpu-operator	nvidia-mig-manager-qtfll	303104
+default	exa-csi-metrics-exporter-4tcls	8192
+gpu-operator	nvidia-container-toolkit-daemonset-cjfm5	36864
+logging	fluent-bit-j5n8m	12288
+dcgm-exporter	nvidia-dcgm-exporter-custom-gf56z	155127808
+calico-system	csi-node-driver-t55v6	8192
+calico-system	calico-node-r9jvn	40140800
+default	exa-csi-metrics-exporter-2xsqm	1937408
+project-01992d56-64c5-794a-968f-be6a5a15709c	workload-01997461-57dc-7909-a2c0-74058234b491	448983040
+gpu-operator	gpu-feature-discovery-gcmq6	14606336
+logging	fluent-bit-lnbkq	12288
+calico-system	csi-node-driver-nzvrw	8192
+dcgm-exporter	nvidia-dcgm-exporter-custom-gpr9p	152702976
+gpu-operator	nvidia-mig-manager-hpjqk	303104
+gpu-operator	nvidia-container-toolkit-daemonset-6mxjv	36864
+project-0198ef29-b8d5-707e-9399-950abcd5afce	workload-019974a5-b609-75ba-91cb-0a6ced77f66d	39038976
+default	test-pod	217088
+gpu-operator	nvidia-device-plugin-daemonset-h2fk7	69632
+gpu-operator	gpu-operator-node-feature-discovery-worker-fsv9r	12148736
+gpu-operator	nvidia-operator-validator-gdcnl	8192
+project-0198ef23-0828-7aec-bf3a-51eed7557a46	workload-01996051-b09a-71b4-a6d1-8280a01e9493	8155205632
+calico-system	calico-node-z9pkv	42258432
+kube-system	kube-proxy-vwsjz	40960
+default	exascaler-csi-node-lbn92	7725056
+monitoring	prometheus-prometheus-node-exporter-gnr6m	12288
+project-0198ef17-2356-7faf-b292-d1e88044082e	workload-019973d9-c22c-7907-a528-68af1e223451	154694045696
+dcgm-exporter	nvidia-dcgm-exporter-custom-f78v8	155353088
+gpu-operator	gpu-feature-discovery-94zbh	14618624
+project-0198ef17-2356-7faf-b292-d1e88044082e	workload-019973d8-b4ca-7c27-995c-5cd540f2d880	16961691648
+project-0198ef23-0828-7aec-bf3a-51eed7557a46	workload-01994bef-2494-7cd3-ada4-dc18e9e6104e	28433612800
+gpu-operator	nvidia-container-toolkit-daemonset-ch24q	36864
+kube-system	kube-proxy-wzkdz	40960
+monitoring	prometheus-prometheus-node-exporter-xlznz	12288
+calico-system	csi-node-driver-fw9xs	8192
+gpu-operator	nvidia-device-plugin-daemonset-92pwm	69632
+gpu-operator	nvidia-mig-manager-5w6d6	303104
+default	exa-csi-metrics-exporter-vzvvb	569344
+default	exascaler-csi-node-lwzmv	12664832
+logging	fluent-bit-z2k9x	12288
+project-0198ef23-0828-7aec-bf3a-51eed7557a46	workload-01997213-5720-76f8-b9db-015c61cccbe3	6868795392
+calico-system	calico-node-r9g9x	41582592
+gpu-operator	nvidia-operator-validator-dzbdv	8192
+project-0198ef17-2356-7faf-b292-d1e88044082e	workload-0199717c-7f94-7d49-987d-5f34cca9cbe6	180840767488
+project-0198ef23-0828-7aec-bf3a-51eed7557a46	workload-01996c89-8aee-7d8c-90ff-9c1fb2fef903	6843600896
+gpu-operator	gpu-operator-node-feature-discovery-worker-rz7lh	12152832
+gpu-operator	gpu-feature-discovery-fvnrc	14594048
+gpu-operator	nvidia-operator-validator-qq7fj	8192
+kube-system	kube-proxy-bf8b5	40960
+dcgm-exporter	nvidia-dcgm-exporter-custom-28hn9	155348992
+calico-system	csi-node-driver-bmgpg	8192
+monitoring	prometheus-prometheus-node-exporter-v56b8	12288
+logging	fluent-bit-ghczj	12288
+gpu-operator	gpu-operator-node-feature-discovery-worker-htkxq	12132352
+project-0198ef25-0222-7513-9f07-2e39adca8b06	workload-0199132a-4ee1-71f3-9d3d-75e1820f1745	20704129024
+default	exascaler-csi-node-vdp72	2834432
+gpu-operator	nvidia-container-toolkit-daemonset-8s7j4	36864
+default	exa-csi-metrics-exporter-68plx	155648
+gpu-operator	nvidia-device-plugin-daemonset-55hsq	69632
+gpu-operator	nvidia-mig-manager-n8dr6	303104
+calico-system	calico-node-qjwbm	39317504
+gpu-operator	gpu-feature-discovery-458l2	14589952
+kube-system	kube-proxy-cbgjl	40960
+logging	fluent-bit-bxcfn	12288
+gpu-operator	nvidia-operator-validator-6jp8q	8192
+default	exa-csi-metrics-exporter-892w7	155648
+calico-system	calico-node-ncmr9	39538688
+gpu-operator	nvidia-device-plugin-daemonset-q2ntf	69632
+calico-system	csi-node-driver-rl9z6	8192
+gpu-operator	nvidia-container-toolkit-daemonset-5knhl	36864
+project-0198ef2d-5e2d-7fe7-885a-fc9df37ffa35	workload-01996e78-8caa-7a35-a314-233d76da3b38	2092269568
+gpu-operator	nvidia-mig-manager-95mcc	303104
+monitoring	prometheus-prometheus-node-exporter-7zm2g	12288
+gpu-operator	gpu-operator-node-feature-discovery-worker-42h78	12136448
+dcgm-exporter	nvidia-dcgm-exporter-custom-6wwxc	155357184
+default	exascaler-csi-node-7sg4d	1904640
+logging	fluent-bit-q48sn	12288
+calico-system	calico-node-cpdhb	49311744
+dcgm-exporter	nvidia-dcgm-exporter-custom-4r8hz	155353088
+gpu-operator	nvidia-operator-validator-vp8l2	8192
+monitoring	prometheus-prometheus-node-exporter-m7mrw	12288
+gpu-operator	nvidia-device-plugin-daemonset-zjhw4	102400
+default	exascaler-csi-node-cjzcz	3919872
+gpu-operator	gpu-operator-node-feature-discovery-worker-4tgcc	12132352
+gpu-operator	nvidia-container-toolkit-daemonset-5ldf9	36864
+calico-system	csi-node-driver-l5pr6	8192
+gpu-operator	nvidia-mig-manager-6dth7	303104
+default	exa-csi-metrics-exporter-82gvw	8192
+gpu-operator	gpu-feature-discovery-ntm87	14618624
+kube-system	kube-proxy-s9k4s	40960
+gpu-operator	nvidia-operator-validator-cf592	8192
+gpu-operator	nvidia-container-toolkit-daemonset-bf4bx	36864
+default	exascaler-csi-node-lzc2w	15024128
+logging	fluent-bit-8mgsj	12288
+monitoring	prometheus-prometheus-node-exporter-4w4wp	12288
+project-0198ef21-86cd-77b5-8560-60f9366d15fa	workload-019973f0-db67-71d9-a69e-63fa9072b0a9	225353728
+calico-system	calico-node-8t9td	43462656
+dcgm-exporter	nvidia-dcgm-exporter-custom-722kb	155959296
+calico-system	csi-node-driver-s54r2	8192
+kube-system	kube-proxy-fw6jd	40960
+gpu-operator	gpu-feature-discovery-jkwx6	14569472
+project-0198ef29-b8d5-707e-9399-950abcd5afce	workload-0199094e-217a-797b-8d03-f63c02088ce4	286720
+gpu-operator	nvidia-mig-manager-chc5q	303104
+default	exa-csi-metrics-exporter-l997r	4628480
+project-0198ef17-2356-7faf-b292-d1e88044082e	workload-0199751e-d7e5-7fe5-a260-8a2b5f1afcc3	561152
+gpu-operator	nvidia-device-plugin-daemonset-s5z5h	69632
+project-0198ef29-b8d5-707e-9399-950abcd5afce	workload-01993c51-9f54-7cdd-b4f2-3f5cd207a043	7719927808
+project-0198ef2a-e16b-7e7f-93d4-f7f40267930b	workload-0199316b-7103-7a63-8604-9d556fba64b7	48907386880
+gpu-operator	gpu-operator-node-feature-discovery-worker-jkrrh	12165120
+project-0198ef29-b8d5-707e-9399-950abcd5afce	workload-0199094d-a3cf-72ee-aaa8-c203e2db7565	286720
+gpu-operator	nvidia-device-plugin-daemonset-6knkx	69632
+calico-system	calico-node-n2n6t	39268352
+gpu-operator	nvidia-operator-validator-hkfwj	8192
+logging	fluent-bit-wp6jw	12288
+gpu-operator	gpu-operator-node-feature-discovery-worker-8gswk	12132352
+dcgm-exporter	nvidia-dcgm-exporter-custom-zg28k	155971584
+default	exascaler-csi-node-dqb2x	2158592
+gpu-operator	nvidia-mig-manager-8qmhj	303104
+monitoring	prometheus-prometheus-node-exporter-wkt8h	12288
+project-0198ef1a-e9a3-721f-8f6c-d3c7c0c14a39	workload-01995f58-7188-7abf-bade-745c473cfced	2830323712
+default	exa-csi-metrics-exporter-lbq6d	294912
+calico-system	csi-node-driver-jbzkg	8192
+kube-system	kube-proxy-n5dxz	45056
+gpu-operator	gpu-feature-discovery-qkmpj	14610432
+gpu-operator	nvidia-container-toolkit-daemonset-htq4j	36864
+kube-system	kube-proxy-pvwsm	40960
+gpu-operator	nvidia-container-toolkit-daemonset-lqrfc	36864
+monitoring	prometheus-prometheus-node-exporter-msnh5	12288
+dcgm-exporter	nvidia-dcgm-exporter-custom-f9j8k	156000256
+logging	fluent-bit-8csgs	12288
+default	exascaler-csi-node-2j47n	1183744
+gpu-operator	nvidia-device-plugin-daemonset-2pccc	73728
+default	exa-csi-metrics-exporter-jvzwb	126976
+gpu-operator	gpu-feature-discovery-5zpzz	14557184
+project-0198ef1b-da73-74fd-9e8d-91ae1127bd1c	workload-01996fb7-6e27-70fd-ac13-fb3956a51427	578825527296
+gpu-operator	nvidia-operator-validator-pr7mh	8192
+gpu-operator	nvidia-mig-manager-vf7nj	303104
+calico-system	calico-node-xqbbn	49655808
+gpu-operator	gpu-operator-node-feature-discovery-worker-8khcf	12132352
+calico-system	csi-node-driver-xvx65	8192
+gpu-operator	nvidia-container-toolkit-daemonset-5b58n	36864
+default	exascaler-csi-node-gkcnh	5177344
+gpu-operator	nvidia-mig-manager-jpg8g	303104
+gpu-operator	nvidia-device-plugin-daemonset-dp4dt	69632
+default	exa-csi-metrics-exporter-kzdww	229376
+calico-system	calico-node-vqkqh	49815552
+kube-system	kube-proxy-2z6qp	40960
+dcgm-exporter	nvidia-dcgm-exporter-custom-gj79c	155963392
+calico-system	csi-node-driver-l7wwm	8192
+gpu-operator	gpu-feature-discovery-jf9kk	14540800
+gpu-operator	nvidia-operator-validator-jjh9g	8192
+logging	fluent-bit-bp8hq	12288
+project-0198ef71-c4b6-7e4e-96e0-68c632eb812f	workload-019907c0-4c2b-7e8c-a648-7227ee44eb46	37874429952
+gpu-operator	gpu-operator-node-feature-discovery-worker-z2gpd	12132352
+project-0198ef72-31df-7f66-b5e9-30090023908e	workload-01996f2d-4bbc-7266-86c3-5df8e682a9a4	11280171008
+monitoring	prometheus-prometheus-node-exporter-tnjhj	12288
+gpu-operator	nvidia-mig-manager-h85xl	303104
+monitoring	prometheus-prometheus-node-exporter-8pb7d	12288
+project-0198ef2d-abb2-7927-b7b7-976f14c22d5e	workload-01994ade-d55c-7e92-a088-028a0922b7b1	148443561984
+gpu-operator	nvidia-container-toolkit-daemonset-tbzxc	36864
+calico-system	csi-node-driver-jd8sh	8192
+logging	fluent-bit-pn4wb	12288
+kube-system	kube-proxy-x2m4g	45056
+gpu-operator	nvidia-operator-validator-bwm4k	8192
+calico-system	calico-node-vfjn8	50200576
+gpu-operator	gpu-feature-discovery-v6mwq	14602240
+dcgm-exporter	nvidia-dcgm-exporter-custom-b22nd	155967488
+default	exascaler-csi-node-v5pfk	2646016
+default	exa-csi-metrics-exporter-tr7vq	184320
+gpu-operator	gpu-operator-node-feature-discovery-worker-77dzt	12128256
+gpu-operator	nvidia-device-plugin-daemonset-vmqpk	69632
+project-01995015-81df-7777-9702-d45a4ac32098	workload-01997426-f74e-721e-a44d-f90cd8ae1f76	3126788096
+project-0198ef17-2356-7faf-b292-d1e88044082e	workload-019974f5-6e85-7b29-92ea-8ad37dbe713d	108005171200
+kube-system	kube-proxy-92dpj	40960
+default	exa-csi-metrics-exporter-ds4fm	712704
+gpu-operator	gpu-feature-discovery-m2hct	14602240
+logging	fluent-bit-7vdr6	12288
+project-0198ef29-b8d5-707e-9399-950abcd5afce	workload-01990e1f-401a-7c7c-a4ec-06270d6a32bd	7394775040
+gpu-operator	nvidia-mig-manager-h2t8m	303104
+gpu-operator	nvidia-device-plugin-daemonset-dlhfv	69632
+calico-system	csi-node-driver-47jdv	8192
+gpu-operator	nvidia-operator-validator-s6zp7	8192
+default	exascaler-csi-node-xg2d2	16121856
+gpu-operator	gpu-operator-node-feature-discovery-worker-6nsb4	12156928
+calico-system	calico-node-hwrjg	42819584
+monitoring	prometheus-prometheus-node-exporter-6l4pq	12288
+dcgm-exporter	nvidia-dcgm-exporter-custom-4lrpn	155959296
+gpu-operator	nvidia-container-toolkit-daemonset-dgblr	36864
+logging	fluent-bit-44fg5	20480
+monitoring	prometheus-prometheus-node-exporter-jwz4t	20480
+dcgm-exporter	nvidia-dcgm-exporter-custom-fzph5	118685696
+default	exascaler-csi-node-vwzll	1794048
+gpu-operator	nvidia-mig-manager-pp9r7	315392
+calico-system	calico-node-98blq	22306816
+gpu-operator	nvidia-device-plugin-daemonset-qgkg9	90112
+gpu-operator	gpu-feature-discovery-clps6	8368128
+gpu-operator	nvidia-container-toolkit-daemonset-xc26s	65536
+default	exa-csi-metrics-exporter-mc666	151552
+calico-system	csi-node-driver-s7fpd	12288
+project-0198ef23-61a0-74f6-9913-b1acc4575b98	workload-019963f0-e9ab-71cd-bc89-29a37e27b339	521310208
+kube-system	kube-proxy-26bkk	20480
+gpu-operator	nvidia-operator-validator-wr84b	12288
+gpu-operator	gpu-operator-node-feature-discovery-worker-rh79b	5439488
+logging	fluent-bit-vnj24	12288
+gpu-operator	nvidia-device-plugin-daemonset-4xd4g	69632
+project-0198ef2c-d34d-79d6-b8d3-ece9b570374e	workload-01996f0f-7bc3-7ec3-bad5-993e29bcdbc0	26536923136
+default	exa-csi-metrics-exporter-wfz8f	155648
+monitoring	prometheus-prometheus-node-exporter-nskms	12288
+gpu-operator	gpu-feature-discovery-ngpbf	14589952
+gpu-operator	nvidia-container-toolkit-daemonset-d6g7x	36864
+gpu-operator	nvidia-operator-validator-nxdff	8192
+calico-system	calico-node-nsjq9	39256064
+calico-system	csi-node-driver-55vwj	8192
+gpu-operator	nvidia-mig-manager-27hgf	303104
+gpu-operator	gpu-operator-node-feature-discovery-worker-82rdq	12128256
+default	exascaler-csi-node-qv2zk	2560000
+dcgm-exporter	nvidia-dcgm-exporter-custom-rxk7l	155111424
+kube-system	kube-proxy-lgh25	45056
+gpu-operator	nvidia-mig-manager-wjx8d	303104
+project-0198ef17-2356-7faf-b292-d1e88044082e	workload-01996c8b-906e-7e35-91ce-702bcffe1fe3	86260031488
+gpu-operator	nvidia-device-plugin-daemonset-d5nsm	77824
+gpu-operator	nvidia-container-toolkit-daemonset-2f9wn	36864
+default	exascaler-csi-node-lq6nw	7569408
+calico-system	csi-node-driver-jxqlt	8192
+project-0198ef29-7464-7da7-a601-e998aa0652a7	workload-019931bc-c58a-7fb0-abd5-a1748f5025e8	65136861184
+logging	fluent-bit-7vfpb	12288
+default	exa-csi-metrics-exporter-tpgjm	581632
+gpu-operator	gpu-feature-discovery-fhgnq	14618624
+project-0198ef29-b8d5-707e-9399-950abcd5afce	workload-01993c50-fb7f-72e5-aff1-0d001d8ebb9f	1389203456
+kube-system	kube-proxy-lzk9r	40960
+gpu-operator	gpu-operator-node-feature-discovery-worker-bbx7f	12144640
+calico-system	calico-node-ln2n7	40837120
+gpu-operator	nvidia-operator-validator-qfjd9	8192
+dcgm-exporter	nvidia-dcgm-exporter-custom-hb2f2	155123712
+monitoring	prometheus-prometheus-node-exporter-trvlb	12288
+default	exa-csi-metrics-exporter-c55zb	208896
+calico-system	csi-node-driver-nwxmz	8192
+gpu-operator	gpu-operator-node-feature-discovery-worker-rtqpk	12128256
+gpu-operator	nvidia-mig-manager-mztfh	303104
+logging	fluent-bit-xt2g2	12288
+calico-system	calico-node-d8qzm	48963584
+gpu-operator	gpu-feature-discovery-gf7c8	14618624
+default	exascaler-csi-node-52657	1761280
+gpu-operator	nvidia-container-toolkit-daemonset-g4s58	36864
+kube-system	kube-proxy-87bc6	57344
+gpu-operator	nvidia-device-plugin-daemonset-r2sh4	131072
+gpu-operator	nvidia-operator-validator-p46sf	8192
+dcgm-exporter	nvidia-dcgm-exporter-custom-h657t	155967488
+project-0198ef71-c4b6-7e4e-96e0-68c632eb812f	workload-01994aaa-f49e-71ff-b865-a07ada0c5841	11233189888
+monitoring	prometheus-prometheus-node-exporter-vlrjc	12288
+dcgm-exporter	nvidia-dcgm-exporter-custom-zgrqv	155373568
+kube-system	kube-proxy-xkq6m	45056
+monitoring	prometheus-prometheus-node-exporter-xsv6f	12288
+gpu-operator	nvidia-container-toolkit-daemonset-mxf2g	36864
+gpu-operator	nvidia-operator-validator-hzvbj	8192
+calico-system	csi-node-driver-hbb6f	8192
+gpu-operator	nvidia-mig-manager-msdtt	303104
+logging	fluent-bit-7bgkt	12288
+default	exascaler-csi-node-w6wgx	2863104
+gpu-operator	gpu-operator-node-feature-discovery-worker-n8w5h	12132352
+gpu-operator	nvidia-device-plugin-daemonset-xnjns	73728
+gpu-operator	gpu-feature-discovery-jsvjn	14618624
+calico-system	calico-node-kqtj6	48955392
+default	exa-csi-metrics-exporter-xkcl9	8192
+gpu-operator	gpu-operator-node-feature-discovery-worker-mnkw4	12148736
+gpu-operator	nvidia-mig-manager-x75rl	303104
+kube-system	kube-proxy-p8vtg	45056
+gpu-operator	nvidia-device-plugin-daemonset-f9rmm	73728
+dcgm-exporter	nvidia-dcgm-exporter-custom-8ps8r	155111424
+gpu-operator	nvidia-container-toolkit-daemonset-6tv5g	36864
+calico-system	calico-node-fq48j	53878784
+project-01992d55-abdd-7770-b94b-c1b3c2684060	workload-01995b2c-7aa9-7057-98e3-d1ae81f19801	2901819392
+gpu-operator	gpu-feature-discovery-7psx7	14614528
+project-0198ef29-7464-7da7-a601-e998aa0652a7	workload-019931b9-993d-7090-8e38-e7f40b252034	10417639424
+logging	fluent-bit-2z8s2	12288
+project-0198ef21-e07b-748e-ac4f-7aa838b46237	workload-01996fd1-7481-726d-8720-8509d96a3c7e	3591245824
+monitoring	prometheus-prometheus-node-exporter-4x6wl	12288
+default	exa-csi-metrics-exporter-sdlb5	589824
+gpu-operator	nvidia-operator-validator-p252t	8192
+calico-system	csi-node-driver-7l7h8	8192
+default	exascaler-csi-node-dbtrw	7929856
+calico-system	calico-node-5x46p	72417280
+kube-system	etcd-master-1	11808768
+calico-system	calico-typha-5958c4f945-c7hkh	937984
+calico-system	csi-node-driver-9wngr	102400
+kube-system	kube-scheduler-master-1	20480
+kube-system	kube-proxy-ww6rg	118784
+kube-system	kube-apiserver-master-1	3342336
+monitoring	prometheus-prometheus-node-exporter-q6czj	45056
+gpu-operator	gpu-operator-node-feature-discovery-worker-mwmww	12214272
+kube-system	kube-controller-manager-master-1	135168
+calico-system	calico-node-lqnnb	71753728
+calico-system	calico-typha-5958c4f945-w2nsb	937984
+kube-system	kube-scheduler-master-2	20480
+calico-system	csi-node-driver-mhpxf	102400
+monitoring	prometheus-prometheus-node-exporter-ms29g	45056
+kube-system	kube-apiserver-master-2	3960832
+kube-system	etcd-master-2	12529664
+calico-apiserver	calico-apiserver-77988d4688-97lzn	53248
+kube-system	kube-controller-manager-master-2	81920
+kube-system	kube-proxy-wbdzk	118784
+tigera-operator	tigera-operator-576646c5b6-bvvmw	12931072
+gpu-operator	gpu-operator-node-feature-discovery-worker-mbsrf	12201984
+calico-apiserver	calico-apiserver-77988d4688-ldrgp	53248
+calico-system	calico-typha-5958c4f945-4hbfh	937984
+monitoring	prometheus-prometheus-node-exporter-88gg7	45056
+kube-system	kube-apiserver-master-3	19660800
+kube-system	etcd-master-3	14163968
+gpu-operator	gpu-operator-node-feature-discovery-worker-cvzph	12206080
+kube-system	kube-scheduler-master-3	28672
+kube-system	kube-controller-manager-master-3	21934080
+calico-system	csi-node-driver-f4pwk	102400
+kube-system	kube-proxy-9g8sz	118784
+calico-system	calico-node-9s9sx	71749632
+gpulive	gpulive-mysql-primary-0	233472
+calico-system	csi-node-driver-jktbw	102400
+gpu-operator	gpu-operator-node-feature-discovery-worker-ql6dx	12349440
+gpulive	gpulive-mysql-secondary-0	13058048
+kube-system	kube-proxy-kcz67	114688
+calico-system	calico-node-ggtft	81764352
+default	exa-csi-metrics-exporter-64h2s	393216
+default	exascaler-csi-node-5s2mj	12976128
+logging	fluent-bit-5hgpj	110592
+monitoring	prometheus-prometheus-node-exporter-vxpns	45056
+loki-monitoring	loki-stack-fluent-bit-loki-cxvlg	466944
+monitoring	grafana-7579f9f69-nslnd	13946880
+gpu-operator	gpu-operator-node-feature-discovery-worker-9bx6d	12218368
+harbor	harbor-database-0	466944
+kube-system	metrics-server-79585c68b5-c6rcw	1318912
+gpu-operator	gpu-operator-node-feature-discovery-master-9bcd6758c-qlj8s	81920
+argocd	argocd-server-5cfd668b45-vn4sm	21540864
+argocd	argocd-dex-server-f447f6fd-5gctz	169504768
+argocd	argocd-repo-server-784cfdbdcc-hjgrw	2807861248
+logging	fluent-bit-55jkg	106496
+default	exascaler-csi-node-ffk9l	18522112
+loki-monitoring	loki-stack-0	155648
+harbor	harbor-portal-6fb475965c-b4hgl	14024704
+harbor	harbor-trivy-0	1634304
+default	exascaler-csi-controller-89bbdb79c-4hg45	17420288
+istio-system	istio-ingressgateway-74598bf8-6vvdz	356352
+istio-system	istio-ingressgateway-74598bf8-s4qnj	368640
+gitea	release-name-postgresql-ha-pgpool-59ff9b48f-fqkfl	274432
+loki-monitoring	loki-grafana-postgresql-0	6291456
+ingress-nginx	ingress-nginx-controller-67bb4bbdbb-46kg8	16703488
+harbor	harbor-redis-0	9949184
+kube-system	kube-proxy-hgkn8	118784
+istio-system	istio-ingressgateway-74598bf8-vcws8	356352
+gitea	release-name-postgresql-ha-postgresql-0	6053888
+gitea	release-name-postgresql-ha-postgresql-1	3751936
+default	tmp-shell2	655360
+monitoring	prometheus-prometheus-node-exporter-dtzms	45056
+calico-system	calico-node-m6tgt	82194432
+default	nfs-client-provisioner-7b7c8cc669-lbzxf	331776
+loki-monitoring	loki-stack-fluent-bit-loki-tdhgv	4816896
+argocd	argocd-application-controller-0	22519808
+kube-system	coredns-55cb58b774-lmqkv	57344
+argocd	argocd-notifications-controller-59557595c9-8rqxv	13262848
+loki-monitoring	loki-stack-grafana-5fd87dbb88-w6nwk	18378752
+default	exa-csi-metrics-exporter-n4qxs	6610944
+istio-system	istiod-5b8966df8b-qgzmv	24104960
+argocd	argocd-redis-7b9f8c86b-qpmx9	32768
+gitea	release-name-postgresql-ha-postgresql-2	6053888
+harbor	harbor-registry-76dbf75c5c-df854	40136704
+default	k8s-ephemeral-storage-metrics-7c8744ff5-gnfg5	65536
+calico-system	csi-node-driver-2426d	102400
+gpu-operator	gpu-operator-b448d84c8-w822h	13930496
+harbor	harbor-jobservice-78959756b-wrfvc	4624384
+gpu-operator	gpu-operator-node-feature-discovery-gc-7b78f7fcfd-455lx	225280
+kube-system	coredns-55cb58b774-zmnn7	57344
+gitea	release-name-gitea-5949dfcc55-n7jt9	22081536
+argocd	argocd-applicationset-controller-697cd7769d-cjvwr	1470464
+gitea	release-name-redis-cluster-2	3346432
+calico-system	calico-kube-controllers-7cd6bfd77c-6nxxr	1716224
+harbor	harbor-core-b946f6bdb-pl7km	3440640
+gitea	release-name-redis-cluster-1	3543040
+monitoring	prometheus-alertmanager-0	61440
+gitea	release-name-redis-cluster-0	3375104
+monitoring	prometheus-server-56c5fd8cf7-qvr6c	18841600
+default	exascaler-csi-node-jpmtc	8396800
+loki-monitoring	loki-stack-fluent-bit-loki-hftjp	495616
+calico-system	calico-node-trgj5	63062016
+logging	fluent-bit-rrwv4	110592
+kube-system	kube-proxy-rjmqx	118784
+monitoring	prometheus-prometheus-node-exporter-c5lxz	45056
+calico-system	csi-node-driver-ph8rm	102400
+monitoring	event-exporter-67cbf64456-26rvm	21626880
+monitoring	prometheus-kube-state-metrics-b7557c785-wkp9f	40960
+monitoring	prometheus-prometheus-pushgateway-5b5c44d8f-ct8mg	45056
+gpu-operator	gpu-operator-node-feature-discovery-worker-qsb9h	12201984
+default	exa-csi-metrics-exporter-5xzwj	196608
+default	exascaler-csi-node-h6c5d	118784
+loki-monitoring	loki-stack-fluent-bit-loki-vbqqg	1699840
+kube-system	kube-proxy-7b9jr	114688
+calico-system	calico-node-qff2x	91930624
+gpulive	gpulive-fe-gpulive-fe-real-7d74958499-gkv2q	3305472
+gpu-operator	gpu-operator-node-feature-discovery-worker-4j5jr	12222464
+calico-system	csi-node-driver-4frzv	102400
+monitoring	prometheus-prometheus-node-exporter-fh5tt	45056
+logging	fluent-bit-5cjmb	106496
+default	exa-csi-metrics-exporter-crhhn	40960
+monitoring	prometheus-prometheus-node-exporter-kb44l	45056
+loki-monitoring	loki-stack-fluent-bit-loki-f486j	1769472
+gpulive	gpulive-fe-gpulive-fe-real-7d74958499-sqh2r	3252224
+kube-system	kube-proxy-zx7bw	114688
+calico-system	calico-node-rk27n	85315584
+calico-system	csi-node-driver-2qmhk	102400
+gpulive	gpulive-be-5f5c7dd88-2n98k	21614592
+default	exascaler-csi-node-t49vg	679936
+logging	fluent-bit-st52x	110592
+gpu-operator	gpu-operator-node-feature-discovery-worker-v2ptv	12275712
+default	exa-csi-metrics-exporter-spmtl	40960
+kube-system	kube-proxy-95p7b	114688
+loki-monitoring	loki-stack-fluent-bit-loki-cl5j9	745472
+calico-system	calico-node-57hps	87633920
+calico-system	csi-node-driver-jp6tc	102400
+monitoring	prometheus-prometheus-node-exporter-4gwp6	45056
+default	exa-csi-metrics-exporter-2brpb	40960
+gpulive	gpulive-be-5f5c7dd88-8nvzw	20832256
+default	exascaler-csi-node-rvqlz	102400
+logging	fluent-bit-r86sv	106496
+gpu-operator	gpu-operator-node-feature-discovery-worker-nvsq8	12222464
+gpulive	gpulive-valkey-node-2	231964672
+loki-monitoring	loki-stack-fluent-bit-loki-q5whl	409600
+logging	fluent-bit-gblzb	106496
+kube-system	kube-proxy-59gws	114688
+default	exa-csi-metrics-exporter-b7rsr	40960
+gpu-operator	gpu-operator-node-feature-discovery-worker-fs5jn	12210176
+gpulive	gpulive-valkey-node-0	231936000
+calico-system	csi-node-driver-rg5h6	102400
+monitoring	prometheus-prometheus-node-exporter-clj9v	45056
+calico-system	calico-node-xnbrt	64724992
+default	exascaler-csi-node-442gx	102400
+gpulive	gpulive-valkey-node-1	231948288

k8s-ephemeral-storage-metrics/.helmignore

@@ -0,0 +1,25 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
+README.md.gotmpl
+*.tgz
+test-values.yaml

k8s-ephemeral-storage-metrics/Chart.yaml

@@ -0,0 +1,18 @@
+annotations:
+  artifacthub.io/license: MIT
+  artifacthub.io/links: |
+    - name: Documentation
+      url: https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics
+  artifacthub.io/prerelease: "false"
+apiVersion: v2
+appVersion: 1.18.2
+description: Ephemeral storage metrics for prometheus operator.
+home: https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics
+keywords:
+- kubernetes
+- metrics
+kubeVersion: '>=1.21.0-0'
+name: k8s-ephemeral-storage-metrics
+sources:
+- https://github.com/jmcgrath207/k8s-ephemeral-storage-metrics
+version: 1.18.2

k8s-ephemeral-storage-metrics/README.md

@@ -0,0 +1,129 @@
+## Helm Install
+
+```bash
+helm repo add k8s-ephemeral-storage-metrics https://jmcgrath207.github.io/k8s-ephemeral-storage-metrics/chart
+helm repo update
+helm upgrade --install my-deployment k8s-ephemeral-storage-metrics/k8s-ephemeral-storage-metrics
+```
+
+## Values
+
+| Key | Type | Default | Description |
+|-----|------|---------|-------------|
+| affinity | object | `{}` |  |
+| client_go_burst | int | `10` | Maximum burst for throttle. |
+| client_go_qps | int | `5` | QPS indicates the maximum QPS to the master from this client. |
+| containerSecurityContext.allowPrivilegeEscalation | bool | `false` |  |
+| containerSecurityContext.capabilities.drop[0] | string | `"ALL"` |  |
+| containerSecurityContext.privileged | bool | `false` |  |
+| containerSecurityContext.readOnlyRootFilesystem | bool | `false` |  |
+| containerSecurityContext.runAsNonRoot | bool | `true` |  |
+| deploy_labels | object | `{}` | Set additional labels for the Deployment/Daemonset |
+| deploy_type | string | `"Deployment"` | Set as Deployment for single controller to query all nodes or Daemonset |
+| dev | object | `{"enabled":false,"grow":{"image":"ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:latest","imagePullPolicy":"IfNotPresent"},"shrink":{"image":"ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:latest","imagePullPolicy":"IfNotPresent"}}` | For local development or testing that will deploy grow and shrink pods and debug service |
+| image.imagePullPolicy | string | `"IfNotPresent"` |  |
+| image.imagePullSecrets | list | `[]` |  |
+| image.repository | string | `"ghcr.io/jmcgrath207/k8s-ephemeral-storage-metrics"` |  |
+| image.tag | string | `"1.18.1"` |  |
+| interval | int | `15` | Polling node rate for exporter |
+| kubelet | object | `{"insecure":false,"readOnlyPort":0,"scrape":false}` | Scrape metrics through kubelet instead of kube api |
+| log_level | string | `"info"` |  |
+| max_node_concurrency | int | `10` | Max number of concurrent query requests to the kubernetes API. |
+| metrics | object | `{"adjusted_polling_rate":false,"ephemeral_storage_container_limit_percentage":true,"ephemeral_storage_container_volume_limit_percentage":true,"ephemeral_storage_container_volume_usage":true,"ephemeral_storage_inodes":true,"ephemeral_storage_node_available":true,"ephemeral_storage_node_capacity":true,"ephemeral_storage_node_percentage":true,"ephemeral_storage_pod_usage":true,"port":9100}` | Set metrics you want to enable |
+| metrics.adjusted_polling_rate | bool | `false` | Create the ephemeral_storage_adjusted_polling_rate metrics to report Adjusted Poll Rate in milliseconds. Typically used for testing. |
+| metrics.ephemeral_storage_container_limit_percentage | bool | `true` | Percentage of ephemeral storage used by a container in a pod |
+| metrics.ephemeral_storage_container_volume_limit_percentage | bool | `true` | Percentage of ephemeral storage used by a container's volume in a pod |
+| metrics.ephemeral_storage_container_volume_usage | bool | `true` | Current ephemeral storage used by a container's volume in a pod |
+| metrics.ephemeral_storage_inodes | bool | `true` | Current ephemeral inode usage of pod |
+| metrics.ephemeral_storage_node_available | bool | `true` | Available ephemeral storage for a node |
+| metrics.ephemeral_storage_node_capacity | bool | `true` | Capacity of ephemeral storage for a node |
+| metrics.ephemeral_storage_node_percentage | bool | `true` | Percentage of ephemeral storage used on a node |
+| metrics.ephemeral_storage_pod_usage | bool | `true` | Current ephemeral byte usage of pod |
+| metrics.port | int | `9100` | Adjust the metric port as needed (default 9100) |
+| nodeSelector | object | `{}` |  |
+| podAnnotations | object | `{}` |  |
+| podSecurityContext.runAsNonRoot | bool | `true` |  |
+| podSecurityContext.seccompProfile.type | string | `"RuntimeDefault"` |  |
+| pod_labels | object | `{}` | Set additional labels for the Pods |
+| pprof | bool | `false` | Enable Pprof |
+| priorityClassName | string | `nil` |  |
+| prometheus.additionalLabels | object | `{}` | Add labels to the PrometheusRule.Spec |
+| prometheus.enable | bool | `true` |  |
+| prometheus.release | string | `"kube-prometheus-stack"` |  |
+| prometheus.rules.enable | bool | `false` | Create PrometheusRules firing alerts when out of ephemeral storage |
+| prometheus.rules.labels | object | `{"severity":"warning"}` | What additional labels to set on alerts |
+| prometheus.rules.predictFilledHours | int | `12` | How many hours in the future to predict filling up of a volume |
+| prometheus.rules.predictMinCurrentUsage | float | `33.3` | What percentage of limit must be used right now to predict filling up of a volume |
+| rbac | object | `{"create":true}` | RBAC configuration |
+| resources | object | `{}` | Resource requests and limits for the container |
+| revisionHistoryLimit | int | `10` | Revision history limit for the Deployment |
+| serviceAccount | object | `{"create":true,"name":null}` | Service Account configuration |
+| serviceMonitor | object | `{"additionalLabels":{},"enable":true,"metricRelabelings":[],"podTargetLabels":[],"relabelings":[],"targetLabels":[]}` | Configure the Service Monitor |
+| serviceMonitor.additionalLabels | object | `{}` | Add labels to the ServiceMonitor.Spec |
+| serviceMonitor.metricRelabelings | list | `[]` | Set metricRelabelings as per https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.RelabelConfig |
+| serviceMonitor.podTargetLabels | list | `[]` | Set podTargetLabels as per https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.ServiceMonitorSpec |
+| serviceMonitor.relabelings | list | `[]` | Set relabelings as per https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.RelabelConfig |
+| serviceMonitor.targetLabels | list | `[]` | Set targetLabels as per https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.ServiceMonitorSpec |
+| tolerations | list | `[]` |  |
+
+## Prometheus alert rules
+
+To prevent from multiple kind of alerts being fired for a single container or
+emptyDir volume when both `prometheus.enable` and `prometheus.rules.enable` are
+on, add the following [inhibition
+rules](https://prometheus.io/docs/alerting/latest/configuration/#inhibition-related-settings)
+to your Alert Manager config:
+
+```yaml
+- source_matchers:
+    - alertname="EphemeralStorageVolumeFilledUp"
+  target_matchers:
+    - severity="warning"
+    - alertname="EphemeralStorageVolumeFillingUp"
+  equal:
+    - pod_namespace
+    - pod_name
+    - volume_name
+- source_matchers:
+    - alertname="ContainerEphemeralStorageUsageAtLimit"
+  target_matchers:
+    - severity="warning"
+    - alertname="ContainerEphemeralStorageUsageReachingLimit"
+  equal:
+    - pod_namespace
+    - pod_name
+    - exported_container
+```
+
+## Contribute
+
+### Start minikube
+```bash
+make new_minikube
+```
+
+### Run locally
+```bash
+make deploy_local
+```
+
+### Run locally with Delve Debug
+```bash
+make deploy_debug
+```
+Then connect to `localhost:30002` with [delve](https://github.com/go-delve/delve) or your IDE.
+
+### Run e2e Test
+```bash
+make deploy_e2e
+```
+
+### Debug e2e
+```bash
+make deploy_e2e_debug
+```
+Then run a debug against [deployment_test.go](tests/e2e/deployment_test.go)
+
+## License
+
+This project is licensed under the [MIT License](https://opensource.org/licenses/MIT). See the `LICENSE` file for more details.

k8s-ephemeral-storage-metrics/ephemeral.yaml

@@ -0,0 +1,184 @@
+---
+# Source: k8s-ephemeral-storage-metrics/templates/RBAC.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-app: ephemeral-storage
+    helm.sh/chart: k8s-ephemeral-storage-metrics-1.18.2
+    app.kubernetes.io/name: k8s-ephemeral-storage-metrics
+    app.kubernetes.io/instance: ephemeral-storage
+    app.kubernetes.io/version: "1.18.2"
+    app.kubernetes.io/managed-by: Helm
+  name: k8s-ephemeral-storage-metrics
+  namespace: default
+---
+# Source: k8s-ephemeral-storage-metrics/templates/RBAC.yaml
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: k8s-ephemeral-storage-metrics
+  labels:
+    helm.sh/chart: k8s-ephemeral-storage-metrics-1.18.2
+    app.kubernetes.io/name: k8s-ephemeral-storage-metrics
+    app.kubernetes.io/instance: ephemeral-storage
+    app.kubernetes.io/version: "1.18.2"
+    app.kubernetes.io/managed-by: Helm
+rules:
+  - apiGroups: [""]
+    resources: ["nodes","nodes/proxy", "nodes/stats", "pods"]
+    verbs: ["get","list", "watch"]
+---
+# Source: k8s-ephemeral-storage-metrics/templates/RBAC.yaml
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: k8s-ephemeral-storage-metrics
+  labels:
+    helm.sh/chart: k8s-ephemeral-storage-metrics-1.18.2
+    app.kubernetes.io/name: k8s-ephemeral-storage-metrics
+    app.kubernetes.io/instance: ephemeral-storage
+    app.kubernetes.io/version: "1.18.2"
+    app.kubernetes.io/managed-by: Helm
+subjects:
+  - kind: ServiceAccount
+    name: k8s-ephemeral-storage-metrics
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: k8s-ephemeral-storage-metrics
+  apiGroup: rbac.authorization.k8s.io
+---
+# Source: k8s-ephemeral-storage-metrics/templates/DeployType.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: k8s-ephemeral-storage-metrics
+  namespace: default
+  labels:
+    helm.sh/chart: k8s-ephemeral-storage-metrics-1.18.2
+    app.kubernetes.io/name: k8s-ephemeral-storage-metrics
+    app.kubernetes.io/instance: ephemeral-storage
+    app.kubernetes.io/version: "1.18.2"
+    app.kubernetes.io/managed-by: Helm
+spec:
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: k8s-ephemeral-storage-metrics
+      app.kubernetes.io/instance: ephemeral-storage
+  template:
+    metadata:
+      labels:
+        helm.sh/chart: k8s-ephemeral-storage-metrics-1.18.2
+        app.kubernetes.io/name: k8s-ephemeral-storage-metrics
+        app.kubernetes.io/instance: ephemeral-storage
+        app.kubernetes.io/version: "1.18.2"
+        app.kubernetes.io/managed-by: Helm
+      annotations:
+        {}
+    spec:
+      serviceAccountName: k8s-ephemeral-storage-metrics
+      nodeSelector:
+        nodegroup: nd
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: metrics
+          image: ghcr.io/jmcgrath207/k8s-ephemeral-storage-metrics:1.18.2
+          imagePullPolicy: IfNotPresent
+          resources:
+            requests:
+              cpu: "500m"
+              memory: "256Mi"
+            limits:
+              cpu: "500m"
+              memory: "256Mi"
+          ports:
+            - name: metrics
+              containerPort: 9100
+              protocol: TCP
+            
+            
+          livenessProbe:
+            failureThreshold: 10
+            httpGet:
+              path: /metrics
+              port: 9100
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 30
+          readinessProbe:
+            failureThreshold: 10
+            httpGet:
+              path: /metrics
+              port: 9100
+              scheme: HTTP
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+              - ALL
+            privileged: false
+            readOnlyRootFilesystem: false
+            runAsNonRoot: true
+          env:
+            - name: DEPLOY_TYPE
+              value: "Deployment"
+            - name: SCRAPE_INTERVAL
+              value: "15"
+            - name: MAX_NODE_CONCURRENCY
+              value: "10"
+            - name: CLIENT_GO_QPS
+              value: "5"
+            - name: CLIENT_GO_BURST
+              value: "10"
+            - name: LOG_LEVEL
+              value: "info"
+            - name: EPHEMERAL_STORAGE_POD_USAGE
+              value: "true"
+            - name: EPHEMERAL_STORAGE_NODE_AVAILABLE
+              value: "true"
+            - name: EPHEMERAL_STORAGE_NODE_CAPACITY
+              value: "true"
+            - name: EPHEMERAL_STORAGE_NODE_PERCENTAGE
+              value: "true"
+            - name: EPHEMERAL_STORAGE_CONTAINER_LIMIT_PERCENTAGE
+              value: "true"
+            - name: EPHEMERAL_STORAGE_CONTAINER_VOLUME_USAGE
+              value: "true"
+            - name: EPHEMERAL_STORAGE_CONTAINER_VOLUME_LIMITS_PERCENTAGE
+              value: "true"
+            - name: EPHEMERAL_STORAGE_INODES
+              value: "true"
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: k8s-ephemeral-storage-metrics
+  namespace: default
+  labels:
+    app.kubernetes.io/name: k8s-ephemeral-storage-metrics
+    app.kubernetes.io/instance: ephemeral-storage
+  annotations:
+    prometheus.io/scrape: "true"
+    prometheus.io/port: "9100" 
+spec:
+  selector:
+    app.kubernetes.io/name: k8s-ephemeral-storage-metrics
+    app.kubernetes.io/instance: ephemeral-storage
+  ports:
+    - name: metrics
+      protocol: TCP
+      port: 9100       
+      targetPort: 9100 
+  type: ClusterIP
+

k8s-ephemeral-storage-metrics/templates/DeployType.yaml

@@ -0,0 +1,165 @@
+apiVersion: apps/v1
+kind: {{ .Values.deploy_type  }}
+metadata:
+  name: k8s-ephemeral-storage-metrics
+  namespace: {{ .Release.Namespace }}
+  labels:
+  {{- include "chart.labels" . | nindent 4 }}
+  {{- if .Values.deploy_labels }}
+  {{- toYaml .Values.deploy_labels | nindent 4 }}
+  {{- end }}
+spec:
+  {{- if eq .Values.deploy_type "Deployment" }}
+  replicas: 1
+  revisionHistoryLimit: {{ .Values.revisionHistoryLimit }}
+  {{- end }}
+  selector:
+    matchLabels:
+  {{- include "chart.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      labels:
+      {{- include "chart.labels" . | nindent 8 }}
+      {{- if .Values.pod_labels }}
+      {{- toYaml .Values.pod_labels | nindent 8 }}
+      {{- end }}
+      annotations:
+      {{- toYaml .Values.podAnnotations | nindent 8 }}
+    spec:
+      serviceAccountName: {{ include "chart.serviceAccountName" . }}
+      nodeSelector:
+      {{- toYaml .Values.nodeSelector | nindent 8 }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- if .Values.priorityClassName }}
+      priorityClassName: {{ .Values.priorityClassName }}
+      {{- end }}
+      {{- with .Values.image.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.podSecurityContext }}
+      securityContext:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      containers:
+        - name: metrics
+          image: {{ .Values.image.repository }}:{{ .Values.image.tag }}
+          imagePullPolicy: {{ .Values.image.imagePullPolicy }}
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          ports:
+            - name: metrics
+              containerPort: {{ .Values.metrics.port  }}
+              protocol: TCP
+            {{ if .Values.dev.enabled }}
+            - name: debug
+              containerPort: 30002
+              protocol: TCP
+              {{ end }}
+            {{ if .Values.pprof }}
+            - name: pprof
+              containerPort: 6060
+              protocol: TCP
+              {{ end }}
+          livenessProbe:
+            failureThreshold: 10
+            httpGet:
+              path: /metrics
+              port: {{ .Values.metrics.port  }}
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 30
+          readinessProbe:
+            failureThreshold: 10
+            httpGet:
+              path: /metrics
+              port: {{ .Values.metrics.port  }}
+              scheme: HTTP
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          {{- with .Values.containerSecurityContext }}
+          securityContext:
+            {{- toYaml . | nindent 12 }}
+          {{- end }}
+          env:
+            - name: DEPLOY_TYPE
+              value: "{{ .Values.deploy_type }}"
+            - name: SCRAPE_INTERVAL
+              value: "{{ .Values.interval }}"
+            - name: MAX_NODE_CONCURRENCY
+              value: "{{ .Values.max_node_concurrency }}"
+            - name: CLIENT_GO_QPS
+              value: "{{ .Values.client_go_qps }}"
+            - name: CLIENT_GO_BURST
+              value: "{{ .Values.client_go_burst }}"
+            - name: LOG_LEVEL
+              value: "{{ .Values.log_level }}"
+              {{- if .Values.metrics.ephemeral_storage_pod_usage }}
+            - name: EPHEMERAL_STORAGE_POD_USAGE
+              value: "{{ .Values.metrics.ephemeral_storage_pod_usage }}"
+              {{- end }}
+              {{- if .Values.metrics.ephemeral_storage_node_available }}
+            - name: EPHEMERAL_STORAGE_NODE_AVAILABLE
+              value: "{{ .Values.metrics.ephemeral_storage_node_available }}"
+              {{- end }}
+              {{- if .Values.metrics.ephemeral_storage_node_capacity }}
+            - name: EPHEMERAL_STORAGE_NODE_CAPACITY
+              value: "{{ .Values.metrics.ephemeral_storage_node_capacity }}"
+              {{- end }}
+              {{- if .Values.metrics.ephemeral_storage_node_percentage }}
+            - name: EPHEMERAL_STORAGE_NODE_PERCENTAGE
+              value: "{{ .Values.metrics.ephemeral_storage_node_percentage }}"
+              {{- end }}
+              {{- if .Values.metrics.ephemeral_storage_container_limit_percentage }}
+            - name: EPHEMERAL_STORAGE_CONTAINER_LIMIT_PERCENTAGE
+              value: "{{ .Values.metrics.ephemeral_storage_container_limit_percentage }}"
+              {{- end }}
+              {{- if .Values.metrics.ephemeral_storage_container_volume_usage }}
+            - name: EPHEMERAL_STORAGE_CONTAINER_VOLUME_USAGE
+              value: "{{ .Values.metrics.ephemeral_storage_container_volume_usage }}"
+              {{- end }}
+              {{- if .Values.metrics.ephemeral_storage_container_volume_limit_percentage }}
+            - name: EPHEMERAL_STORAGE_CONTAINER_VOLUME_LIMITS_PERCENTAGE
+              value: "{{ .Values.metrics.ephemeral_storage_container_volume_limit_percentage }}"
+              {{- end }}
+              {{- if .Values.metrics.ephemeral_storage_inodes }}
+            - name: EPHEMERAL_STORAGE_INODES
+              value: "{{ .Values.metrics.ephemeral_storage_inodes }}"
+              {{- end }}
+              {{- if .Values.kubelet.scrape }}
+            - name: SCRAPE_FROM_KUBELET
+              value: "{{ .Values.kubelet.scrape }}"
+              {{- end }}
+              {{- if .Values.kubelet.readOnlyPort }}
+            - name: KUBELET_READONLY_PORT
+              value: "{{ .Values.kubelet.readOnlyPort }}"
+              {{- end }}
+              {{- if .Values.kubelet.insecure }}
+            - name: SCRAPE_FROM_KUBELET_TLS_INSECURE_SKIP_VERIFY
+              value: "{{ .Values.kubelet.insecure }}"
+              {{- end }}
+              {{- if .Values.metrics.adjusted_polling_rate }}
+            - name: ADJUSTED_POLLING_RATE
+              value: "{{ .Values.metrics.adjusted_polling_rate }}"
+              {{- end }}
+              {{- if .Values.pprof }}
+            - name: PPROF
+              value: "{{ .Values.pprof }}"
+              {{- end }}
+              {{- if eq .Values.deploy_type  "DaemonSet" }}
+            - name: CURRENT_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+              {{- end }}
+

k8s-ephemeral-storage-metrics/templates/RBAC.yaml

@@ -0,0 +1,40 @@
+{{- if .Values.serviceAccount.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-app: {{ .Release.Name }}
+  {{- include "chart.labels" . | nindent 4 }}
+  name: {{ include "chart.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+{{- end }}
+
+{{- if .Values.rbac.create }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: k8s-ephemeral-storage-metrics
+  labels:
+  {{- include "chart.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [""]
+    resources: ["nodes","nodes/proxy", "nodes/stats", "pods"]
+    verbs: ["get","list", "watch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: k8s-ephemeral-storage-metrics
+  labels:
+  {{- include "chart.labels" . | nindent 4 }}
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "chart.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: k8s-ephemeral-storage-metrics
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}

k8s-ephemeral-storage-metrics/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "chart.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "chart.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "chart.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "chart.labels" -}}
+helm.sh/chart: {{ include "chart.chart" . }}
+{{ include "chart.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "chart.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "chart.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "chart.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "chart.name" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

k8s-ephemeral-storage-metrics/templates/localDebugService.yaml

@@ -0,0 +1,36 @@
+{{ if .Values.dev.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: debug
+  labels:
+    {{- include "chart.labels" . | nindent 4 }}
+    app.kubernetes.io/name: k8s-ephemeral-storage-metrics-debug
+spec:
+  type: ClusterIP
+  selector:
+    {{- include "chart.selectorLabels" . | nindent 4 }}
+  ports:
+    - protocol: TCP
+      port: 30002
+      targetPort: debug
+      name: debug
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: pprof
+  labels:
+    {{- include "chart.labels" . | nindent 4 }}
+    app.kubernetes.io/name: k8s-ephemeral-storage-metrics-debug
+spec:
+  type: ClusterIP
+  selector:
+    {{- include "chart.selectorLabels" . | nindent 4 }}
+  ports:
+    - protocol: TCP
+      port: 6060
+      targetPort: pprof
+      name: pprof
+ {{ end }}

k8s-ephemeral-storage-metrics/templates/metrics.yaml

@@ -0,0 +1,64 @@
+{{ if .Values.prometheus.enable }}
+{{ if .Values.serviceMonitor.enable }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: k8s-ephemeral-storage-metrics
+  namespace: "{{ $.Release.Namespace }}"
+  labels:
+  {{- include "chart.labels" . | nindent 4 }}
+{{- with .Values.serviceMonitor.additionalLabels }}
+{{- toYaml . | nindent 4 }}
+{{- end }}
+spec:
+  endpoints:
+    - path: /metrics
+      port: metrics
+      scheme: http
+      interval: "{{ .Values.interval }}s"
+{{- if .Values.serviceMonitor.metricRelabelings }}
+      metricRelabelings:
+{{ toYaml .Values.serviceMonitor.metricRelabelings | indent 8 }}
+{{- end }}
+{{- if .Values.serviceMonitor.relabelings }}
+      relabelings:
+{{ toYaml .Values.serviceMonitor.relabelings | indent 8 }}
+{{- end }}
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace }}
+  selector:
+    matchLabels:
+      {{- include "chart.selectorLabels" . | nindent 6 }}
+  {{- if .Values.serviceMonitor.targetLabels }}
+  targetLabels:
+    {{- range .Values.serviceMonitor.targetLabels }}
+    - {{ . }}
+    {{- end }}
+  {{- end }}
+  {{- if .Values.serviceMonitor.podTargetLabels }}
+  podTargetLabels:
+    {{- range .Values.serviceMonitor.podTargetLabels }}
+    - {{ . }}
+    {{- end }}
+  {{- end }}
+{{ end }}
+---
+
+apiVersion: v1
+kind: Service
+metadata:
+  name: k8s-ephemeral-storage-metrics
+  namespace: "{{ $.Release.Namespace }}"
+  labels:
+    {{- include "chart.labels" . | nindent 6 }}
+spec:
+  type: ClusterIP
+  selector:
+  {{- include "chart.selectorLabels" . | nindent 6 }}
+  ports:
+    - name: metrics
+      port: {{ .Values.metrics.port  }}
+      protocol: TCP
+      targetPort: metrics
+{{ end }}

k8s-ephemeral-storage-metrics/templates/prometheusrules.yaml

@@ -0,0 +1,129 @@
+{{- if .Values.prometheus.enable | default true }}
+{{- with $rules := default (dict) .Values.prometheus.rules }}
+{{- if $rules.enable | default false }}
+{{- with $predictFilledHours := $rules.predictFilledHours | default 12 }}
+---
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: k8s-ephemeral-storage-container-limits
+  namespace: "{{ $.Release.Namespace }}"
+  labels:
+{{- with $.Values.prometheus.additionalLabels }}
+{{- toYaml . | nindent 4 }}
+{{- end }}
+spec:
+  groups:
+    - name: k8s.rules.container_resource
+      rules:
+        - alert: ContainerEphemeralStorageUsageAtLimit
+          annotations:
+            description: >-
+              {{ `Ephemeral storage usage of pod/container {{ $labels.pod_name
+              }}/{{ $labels.exported_container }} in Namespace {{
+              $labels.pod_namespace }} on Node {{ $labels.node_name }} {{ with
+              $labels.cluster -}} on Cluster {{ . }} {{- end }} is at {{ $value
+              }}% of the limit.` }}
+            summary: Container ephemeral storage usage is at the limit.
+          expr: |-2
+            ( max by (node_name, pod_namespace, pod_name, exported_container)
+                     (avg_over_time(ephemeral_storage_container_limit_percentage{source="container"}[5m]))
+            > 85.0)
+            # ignore pods that haven't been running for some time (e.g. completed jobs)
+            unless on (pod_namespace, pod_name)
+                      ( (label_replace(label_replace(
+                           max_over_time(kube_pod_status_phase{phase="Running"}[2m]),
+                           "pod_namespace", "$1", "namespace", "(.*)"),
+                           "pod_name", "$1", "pod", "(.*)"))
+                      == 0)
+          for: 1m
+          labels:
+            {{- $.Values.prometheus.rules.labels | toYaml | nindent 12 }}
+
+        - alert: ContainerEphemeralStorageUsageReachingLimit
+          annotations:
+            description: >-
+              {{ `Based on recent sampling, the ephemeral storage limit of
+              pod/container {{ $labels.pod_name }}/{{
+              $labels.exported_container }} in Namespace {{
+              $labels.pod_namespace }} on Node {{ $labels.node_name }} {{
+              with $labels.cluster -}} on Cluster {{ . }} {{- end }} is
+              expected to be reached within ` }}{{ $predictFilledHours }}
+              {{ ` hours. Currently, {{ $value }}% is used.` }}
+            summary: Container ephemeral storage usage is reaching the limit.
+          expr: |-2
+            (   ( max by (node_name, pod_namespace, pod_name, exported_container)
+                         (ephemeral_storage_container_limit_percentage{source="container"})
+                > {{ $rules.predictMinCurrentUsage | float64 }})
+            and on (pod_namespace, pod_name, exported_container)
+                   ( predict_linear( ephemeral_storage_container_limit_percentage{source="container"}[2h]
+                                   , {{ $predictFilledHours | float64 }}*3600)
+                   > 99.0)
+            )
+            # ignore pods that haven't been running for enough time
+            unless on (pod_namespace, pod_name)
+                      ( (label_replace(label_replace(
+                           min_over_time(kube_pod_status_phase{phase="Running"}[10m]),
+                           "pod_namespace", "$1", "namespace", "(.*)"),
+                           "pod_name", "$1", "pod", "(.*)"))
+                      == 0)
+          for: 15m
+          labels:
+            {{- $.Values.prometheus.rules.labels | toYaml | nindent 12 }}
+
+        - alert: EphemeralStorageVolumeFilledUp
+          annotations:
+            description: >-
+              {{ `Ephemeral storage volume "{{ $labels.volume_name }}" of pod
+              {{ $labels.pod_name }} in Namespace {{ $labels.pod_namespace }}
+              {{ with $labels.cluster -}} on Cluster {{ . }} {{- end }} is
+              filled from {{ $value }}%.` }}
+            summary: Ephemeral storage volume is filled up.
+          expr: |-2
+            ( max by (pod_namespace, pod_name, volume_name)
+                     (avg_over_time(ephemeral_storage_container_volume_limit_percentage[5m]))
+            > 85.0)
+            # ignore pods that haven't been running for some time (e.g. completed jobs)
+            unless on (pod_namespace, pod_name)
+                      ( (label_replace(label_replace(
+                           max_over_time(kube_pod_status_phase{phase="Running"}[2m]),
+                           "pod_namespace", "$1", "namespace", "(.*)"),
+                           "pod_name", "$1", "pod", "(.*)"))
+                      == 0)
+          for: 1m
+          labels:
+            {{- $.Values.prometheus.rules.labels | toYaml | nindent 12 }}
+
+        - alert: EphemeralStorageVolumeFillingUp
+          annotations:
+            description: >-
+              {{ `Based on recent sampling, the ephemeral storage volume "{{
+              $labels.volume_name }}" of pod {{ $labels.pod_name }} in
+              Namespace {{ $labels.pod_namespace }} {{ with $labels.cluster
+              -}} on Cluster {{ . }} {{- end }} is expected to be filled up
+              within ` }}{{ $predictFilledHours }}{{ ` hours. Currently, {{
+              $value }}% is used.` }}
+            summary: Ephemeral storage volume is filling up.
+          expr: |-2
+            (   ( max by (pod_namespace, pod_name, volume_name)
+                         (ephemeral_storage_container_volume_limit_percentage)
+                > {{ $rules.predictMinCurrentUsage | float64 }})
+            and ( max by (pod_namespace, pod_name, volume_name)
+                         (predict_linear( ephemeral_storage_container_volume_limit_percentage[2h]
+                                        , {{ $predictFilledHours | float64 }}*3600))
+                > 99)
+            )
+            # ignore pods that haven't been running for enough time
+            unless on (pod_namespace, pod_name)
+                      ( (label_replace(label_replace(
+                            min_over_time(kube_pod_status_phase{phase="Running"}[10m]),
+                           "pod_namespace", "$1", "namespace", "(.*)"),
+                           "pod_name", "$1", "pod", "(.*)"))
+                      == 0)
+          for: 15m
+          labels:
+            {{- $.Values.prometheus.rules.labels | toYaml | nindent 12 }}
+{{- end }}
+{{- end }}
+{{- end }}
+{{- end }}

k8s-ephemeral-storage-metrics/templates/test_deployments.yaml

@@ -0,0 +1,77 @@
+{{ if .Values.dev.enabled }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: shrink-test
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: shrink-test
+  template:
+    metadata:
+      name: shrink-test
+      labels:
+        name: shrink-test
+    spec:
+      containers:
+        - image: "{{ .Values.dev.shrink.image }}"
+          imagePullPolicy: "{{ .Values.dev.shrink.imagePullPolicy }}"
+          name: shrink-test
+          resources:
+            requests:
+              ephemeral-storage: "1Mi"
+            limits:
+              ephemeral-storage: "60Mi"
+          volumeMounts:
+            - mountPath: /cache
+              name: cache-volume-1
+            - mountPath: /cachez
+              name: cache-volume-3
+      volumes:
+        - name: cache-volume-1
+          emptyDir:
+            sizeLimit: 24Mi
+        - name: cache-volume-3
+          emptyDir:
+            sizeLimit: 24Mi
+
+---
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: grow-test
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      name: grow-test
+  template:
+    metadata:
+      name: grow-test
+      labels:
+        name: grow-test
+    spec:
+      containers:
+        - image: "{{ .Values.dev.grow.image }}"
+          imagePullPolicy: "{{ .Values.dev.grow.imagePullPolicy }}"
+          name: grow-test
+          resources:
+            requests:
+              ephemeral-storage: "1Mi"
+            limits:
+              ephemeral-storage: "60Mi"
+          volumeMounts:
+            - mountPath: /cache
+              name: cache-volume-two
+            - mountPath: /cachez
+              name: cache-volume-four
+      volumes:
+        - name: cache-volume-two
+          emptyDir:
+            sizeLimit: 24Mi
+        - name: cache-volume-four
+          emptyDir:
+            sizeLimit: 24Mi
+ {{ end }}

k8s-ephemeral-storage-metrics/test.yaml

@@ -0,0 +1,156 @@
+---
+# Source: k8s-ephemeral-storage-metrics/templates/RBAC.yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-app: ephemeral-storage
+    helm.sh/chart: k8s-ephemeral-storage-metrics-1.18.2
+    app.kubernetes.io/name: k8s-ephemeral-storage-metrics
+    app.kubernetes.io/instance: ephemeral-storage
+    app.kubernetes.io/version: "1.18.2"
+    app.kubernetes.io/managed-by: Helm
+  name: k8s-ephemeral-storage-metrics
+  namespace: default
+---
+# Source: k8s-ephemeral-storage-metrics/templates/RBAC.yaml
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: k8s-ephemeral-storage-metrics
+  labels:
+    helm.sh/chart: k8s-ephemeral-storage-metrics-1.18.2
+    app.kubernetes.io/name: k8s-ephemeral-storage-metrics
+    app.kubernetes.io/instance: ephemeral-storage
+    app.kubernetes.io/version: "1.18.2"
+    app.kubernetes.io/managed-by: Helm
+rules:
+  - apiGroups: [""]
+    resources: ["nodes","nodes/proxy", "nodes/stats", "pods"]
+    verbs: ["get","list", "watch"]
+---
+# Source: k8s-ephemeral-storage-metrics/templates/RBAC.yaml
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: k8s-ephemeral-storage-metrics
+  labels:
+    helm.sh/chart: k8s-ephemeral-storage-metrics-1.18.2
+    app.kubernetes.io/name: k8s-ephemeral-storage-metrics
+    app.kubernetes.io/instance: ephemeral-storage
+    app.kubernetes.io/version: "1.18.2"
+    app.kubernetes.io/managed-by: Helm
+subjects:
+  - kind: ServiceAccount
+    name: k8s-ephemeral-storage-metrics
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: k8s-ephemeral-storage-metrics
+  apiGroup: rbac.authorization.k8s.io
+---
+# Source: k8s-ephemeral-storage-metrics/templates/DeployType.yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: k8s-ephemeral-storage-metrics
+  namespace: default
+  labels:
+    helm.sh/chart: k8s-ephemeral-storage-metrics-1.18.2
+    app.kubernetes.io/name: k8s-ephemeral-storage-metrics
+    app.kubernetes.io/instance: ephemeral-storage
+    app.kubernetes.io/version: "1.18.2"
+    app.kubernetes.io/managed-by: Helm
+spec:
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: k8s-ephemeral-storage-metrics
+      app.kubernetes.io/instance: ephemeral-storage
+  template:
+    metadata:
+      labels:
+        helm.sh/chart: k8s-ephemeral-storage-metrics-1.18.2
+        app.kubernetes.io/name: k8s-ephemeral-storage-metrics
+        app.kubernetes.io/instance: ephemeral-storage
+        app.kubernetes.io/version: "1.18.2"
+        app.kubernetes.io/managed-by: Helm
+      annotations:
+        {}
+    spec:
+      serviceAccountName: k8s-ephemeral-storage-metrics
+      nodeSelector:
+        {}
+      securityContext:
+        runAsNonRoot: true
+        seccompProfile:
+          type: RuntimeDefault
+      containers:
+        - name: metrics
+          image: ghcr.io/jmcgrath207/k8s-ephemeral-storage-metrics:1.18.2
+          imagePullPolicy: IfNotPresent
+          resources:
+            {}
+          ports:
+            - name: metrics
+              containerPort: 9100
+              protocol: TCP
+            
+            
+          livenessProbe:
+            failureThreshold: 10
+            httpGet:
+              path: /metrics
+              port: 9100
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 30
+          readinessProbe:
+            failureThreshold: 10
+            httpGet:
+              path: /metrics
+              port: 9100
+              scheme: HTTP
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 1
+          securityContext:
+            allowPrivilegeEscalation: false
+            capabilities:
+              drop:
+              - ALL
+            privileged: false
+            readOnlyRootFilesystem: false
+            runAsNonRoot: true
+          env:
+            - name: DEPLOY_TYPE
+              value: "Deployment"
+            - name: SCRAPE_INTERVAL
+              value: "15"
+            - name: MAX_NODE_CONCURRENCY
+              value: "10"
+            - name: CLIENT_GO_QPS
+              value: "5"
+            - name: CLIENT_GO_BURST
+              value: "10"
+            - name: LOG_LEVEL
+              value: "info"
+            - name: EPHEMERAL_STORAGE_POD_USAGE
+              value: "true"
+            - name: EPHEMERAL_STORAGE_NODE_AVAILABLE
+              value: "true"
+            - name: EPHEMERAL_STORAGE_NODE_CAPACITY
+              value: "true"
+            - name: EPHEMERAL_STORAGE_NODE_PERCENTAGE
+              value: "true"
+            - name: EPHEMERAL_STORAGE_CONTAINER_LIMIT_PERCENTAGE
+              value: "true"
+            - name: EPHEMERAL_STORAGE_CONTAINER_VOLUME_USAGE
+              value: "true"
+            - name: EPHEMERAL_STORAGE_CONTAINER_VOLUME_LIMITS_PERCENTAGE
+              value: "true"
+            - name: EPHEMERAL_STORAGE_INODES
+              value: "true"

k8s-ephemeral-storage-metrics/values.yaml

@@ -0,0 +1,138 @@
+image:
+  repository: ghcr.io/jmcgrath207/k8s-ephemeral-storage-metrics
+  tag: 1.18.2
+  imagePullPolicy: IfNotPresent
+  imagePullSecrets: []
+
+# -- Configure the Service Monitor
+serviceMonitor:
+  enable: false
+  # -- Add labels to the ServiceMonitor.Spec
+  additionalLabels: {}
+  # -- Set relabelings as per https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.RelabelConfig
+  relabelings: []
+  # -- Set metricRelabelings as per https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.RelabelConfig
+  metricRelabelings: []
+  # -- Set targetLabels as per https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.ServiceMonitorSpec
+  targetLabels: []
+  # -- Set podTargetLabels as per https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/api.md#monitoring.coreos.com/v1.ServiceMonitorSpec
+  podTargetLabels: []
+
+# -- Scrape metrics through kubelet instead of kube api
+kubelet:
+  scrape: false
+  readOnlyPort: 0
+  insecure: false
+
+# -- Set metrics you want to enable
+metrics:
+  # -- Adjust the metric port as needed (default 9100)
+  port: 9100
+  # -- Percentage of ephemeral storage used by a container in a pod
+  ephemeral_storage_container_limit_percentage: true
+  # -- Current ephemeral storage used by a container's volume in a pod
+  ephemeral_storage_container_volume_usage: true
+  # -- Percentage of ephemeral storage used by a container's volume in a pod
+  ephemeral_storage_container_volume_limit_percentage: true
+  # -- Current ephemeral byte usage of pod
+  ephemeral_storage_pod_usage: true
+  # -- Current ephemeral inode usage of pod
+  ephemeral_storage_inodes: true
+  # -- Available ephemeral storage for a node
+  ephemeral_storage_node_available: true
+  # -- Capacity of ephemeral storage for a node
+  ephemeral_storage_node_capacity: true
+  # -- Percentage of ephemeral storage used on a node
+  ephemeral_storage_node_percentage: true
+  # -- Create the ephemeral_storage_adjusted_polling_rate metrics to report Adjusted Poll Rate in milliseconds. Typically used for testing.
+  adjusted_polling_rate: false
+
+log_level: info
+# -- Set as Deployment for single controller to query all nodes or Daemonset
+deploy_type: Deployment
+# -- Set additional labels for the Deployment/Daemonset
+deploy_labels: {}
+# -- Set additional labels for the Pods
+pod_labels: {}
+# Note in testing, Kube API does not refresh faster than 10 seconds
+# -- Polling node rate for exporter
+interval: 15 # Seconds
+# -- Max number of concurrent query requests to the kubernetes API.
+max_node_concurrency: 10
+# -- QPS indicates the maximum QPS to the master from this client.
+client_go_qps: 5
+# --  Maximum burst for throttle.
+client_go_burst: 10
+
+prometheus:
+  enable: false
+  release: kube-prometheus-stack
+  # -- Add labels to the PrometheusRule.Spec
+  additionalLabels: {}
+  rules:
+    # -- Create PrometheusRules firing alerts when out of ephemeral storage
+    enable: false
+    # -- How many hours in the future to predict filling up of a volume
+    predictFilledHours: 12
+    # -- What percentage of limit must be used right now to predict filling up of a volume
+    predictMinCurrentUsage: 33.3
+    # -- What additional labels to set on alerts
+    labels:
+      severity: warning
+
+# -- Enable Pprof
+pprof: false
+
+# -- For local development or testing that will deploy grow and shrink pods and debug service
+dev:
+  enabled: false
+  shrink:
+    image: ghcr.io/jmcgrath207/k8s-ephemeral-storage-shrink-test:latest
+    imagePullPolicy: IfNotPresent
+  grow:
+    image: ghcr.io/jmcgrath207/k8s-ephemeral-storage-grow-test:latest
+    imagePullPolicy: IfNotPresent
+
+# -- Resource requests and limits for the container
+resources: {}
+
+podAnnotations: {}
+
+nodeSelector: {}
+
+affinity: {}
+
+tolerations: []
+
+priorityClassName:
+
+# -- Revision history limit for the Deployment
+revisionHistoryLimit: 10
+
+# -- Service Account configuration
+serviceAccount:
+  create: true
+  # If not set and create is true, a name is generated using the name template
+  name: null
+
+# -- RBAC configuration
+rbac:
+  create: true
+
+# Pod Security Context
+# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+podSecurityContext:
+  runAsNonRoot: true
+  seccompProfile:
+    type: RuntimeDefault
+
+# Container Security Context to be set on the controller component container
+# ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/
+containerSecurityContext:
+  allowPrivilegeEscalation: false
+  capabilities:
+    drop:
+      - ALL
+  privileged: false
+  readOnlyRootFilesystem: false
+  runAsNonRoot: true

template/prd/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/kubernetes/metric-exporter.yaml

@@ -67,15 +67,15 @@ spec:
       labels:
         app: exa-csi-metrics-exporter
     spec:
-      affinity:
+#      affinity:
-        nodeAffinity:
+#        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
+#          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
+#            nodeSelectorTerms:
-            - matchExpressions:
+#            - matchExpressions:
-              - key: kubernetes.io/hostname
+#              - key: kubernetes.io/hostname
-                operator: NotIn
+#                operator: NotIn
-                values:
+#                values:
-                - nd-worker-2
+#                - nd-worker-2
       serviceAccountName: exa-csi-metrics-sa
       automountServiceAccountToken: true    # Ensure token is mounted
       containers: