From d30bc8cf7a30d3c16fad2da8647a8520a3a551b8 Mon Sep 17 00:00:00 2001 From: irteam su-account Date: Mon, 23 Jun 2025 21:57:01 +0900 Subject: [PATCH] pv --- gpulive/cert/gpulive.crt | 22 + gpulive/cert/gpulive.key | 28 + gpulive/cert/vm/gpulive.crt | 22 + gpulive/cert/vm/gpulive.key | 28 + gpulive/cert/vm/rootCA.key | 28 + gpulive/cert/vm/rootCA.pem | 21 + gpulive/cert/vm/server.csr | 16 + gpulive/gitea/vm/1_configure/1_namespace.yaml | 6 + .../vm/1_configure/2_networki_policy.yaml | 79 + .../vm/1_configure/3_PodDisruptionBudget.yaml | 77 + gpulive/gitea/vm/1_configure/4_sa.yaml | 23 + gpulive/gitea/vm/1_configure/5_secret.yaml | 384 +++ gpulive/gitea/vm/1_configure/6_pvc.yaml | 18 + gpulive/gitea/vm/1_configure/README.md | 6 + gpulive/gitea/vm/1_configure/pv.yaml | 125 + gpulive/gitea/vm/2_configmap/default.yaml | 2493 +++++++++++++++++ gpulive/gitea/vm/3_apply/deploy.yaml | 820 ++++++ gpulive/gitea/vm/3_apply/ing.yaml | 26 + gpulive/gitea/vm/3_apply/service.yaml | 172 ++ .../harbor_2.9.1/2_configmap/configmap.yaml | 1 - gpulive/harbor_2.9.1/vm/1_configure/cert.yaml | 29 + gpulive/harbor_2.9.1/vm/1_configure/init.yaml | 83 + gpulive/harbor_2.9.1/vm/1_configure/pv.yaml | 90 + gpulive/harbor_2.9.1/vm/1_configure/pvc.yaml | 43 + .../vm/2_configmap/configmap.yaml | 226 ++ gpulive/harbor_2.9.1/vm/3_apply/deploy.yaml | 804 ++++++ gpulive/harbor_2.9.1/vm/3_apply/gateway.yaml | 65 + gpulive/harbor_2.9.1/vm/ing.yaml | 64 + gpulive/nfs-provisioner/test-claim.yaml | 3 +- gpulive/nfs-provisioner/test-pod.yaml | 2 +- .../base/prometheus/cluster_role.yaml | 0 .../base/prometheus/cm.yaml | 0 .../prometheus/cm_custom_alert_rules.yaml | 0 .../base/prometheus/daemonset.yaml | 0 .../base/prometheus/deploy.yaml | 0 .../base/prometheus/ingress_alertmanager.yaml | 0 .../base/prometheus/ingress_prometheus.yaml | 0 .../base/prometheus/kustomization.yaml | 0 .../base/prometheus/namespace.yaml | 0 .../base/prometheus/pvc.yaml | 0 .../base/prometheus/sa.yaml | 0 .../base/prometheus/service.yaml | 0 .../base/prometheus/state_full_set.yaml | 0 .../overlays/default/auth | 0 .../default/grafana/kustomization.yaml | 0 .../overlays/default/grafana/patch-cm.yaml | 0 .../default/grafana/patch-deploy.yaml | 0 .../grafana/patch-ingress_grafana.yaml | 0 .../default/grafana/patch-secret.yaml | 0 .../overlays/default/kustomization.yaml | 0 .../default/prometheus/kustomization.yaml | 0 .../patch-cm-prometheus-server.yaml | 0 .../default/prometheus/patch-deploy.yaml | 0 .../patch-ingress_alertmanager.yaml | 0 .../prometheus/patch-ingress_prometheus.yaml | 0 .../prometheus/overlays/vm/kustomization.yaml | 16 + .../overlays/vm/prometheus/kustomization.yaml | 34 + .../patch-cm-prometheus-server.yaml | 335 +++ .../overlays/vm/prometheus/patch-deploy.yaml | 19 + .../patch-ingress_alertmanager.yaml | 29 + .../prometheus/patch-ingress_prometheus.yaml | 22 + .../base/grafana/cm.yaml | 0 .../base/grafana/deploy.yaml | 0 .../base/grafana/kustomization.yaml | 0 .../base/grafana/pvc.yaml | 0 .../base/grafana/role.yaml | 0 .../base/grafana/sa.yaml | 0 .../base/grafana/secret.yaml | 0 .../base/grafana/service.yaml | 0 .../base/prometheus/cluster_role.yaml | 248 ++ gpulive/vm-monitoring/base/prometheus/cm.yaml | 375 +++ .../prometheus/cm_custom_alert_rules.yaml | 194 ++ .../base/prometheus/daemonset.yaml | 135 + .../vm-monitoring/base/prometheus/deploy.yaml | 290 ++ .../base/prometheus/ingress_alertmanager.yaml | 29 + .../base/prometheus/ingress_prometheus.yaml | 29 + .../base/prometheus/kustomization.yaml | 19 + .../base/prometheus/namespace.yaml | 6 + .../vm-monitoring/base/prometheus/pvc.yaml | 19 + gpulive/vm-monitoring/base/prometheus/sa.yaml | 76 + .../base/prometheus/service.yaml | 151 + .../base/prometheus/state_full_set.yaml | 90 + gpulive/vm-monitoring/overlays/default/auth | 1 + .../default/grafana/kustomization.yaml | 29 + .../overlays/default/grafana/patch-cm.yaml | 32 + .../default/grafana/patch-deploy.yaml | 20 + .../grafana/patch-ingress_grafana.yaml} | 8 +- .../default/grafana/patch-secret.yaml | 17 + .../overlays/default/kustomization.yaml | 17 + .../default/prometheus/kustomization.yaml | 34 + .../patch-cm-prometheus-server.yaml | 335 +++ .../default/prometheus/patch-deploy.yaml | 19 + .../patch-ingress_alertmanager.yaml | 29 + .../prometheus/patch-ingress_prometheus.yaml | 22 + gpulive/vm-monitoring/overlays/vm/auth | 1 + .../overlays/vm/grafana/kustomization.yaml | 29 + .../overlays/vm/grafana/patch-cm.yaml | 32 + .../overlays/vm/grafana/patch-deploy.yaml | 20 + .../vm/grafana/patch-ingress_grafana.yaml | 24 + .../overlays/vm/grafana/patch-secret.yaml | 17 + .../overlays/vm/kustomization.yaml | 17 + .../overlays/vm/prometheus/kustomization.yaml | 34 + .../patch-cm-prometheus-server.yaml | 335 +++ .../overlays/vm/prometheus/patch-deploy.yaml | 19 + .../patch-ingress_alertmanager.yaml | 29 + .../prometheus/patch-ingress_prometheus.yaml | 22 + nodegroup/make_nodegroup | 4 + .../crds/nvidia.com_clusterpolicies.yaml | 2384 ++++++++++++++++ .../crds/nvidia.com_nvidiadrivers.yaml | 797 ++++++ template-gpu-operator/gpu-operator.yaml | 1512 ++++++++++ template-gpu-operator/highway.yaml | 1512 ++++++++++ template-gpu-operator/mig.yaml | 21 + template-gpu-operator/mig.yaml.bak | 30 + template-gpu-operator/script.sh | 37 + template-gpu-operator/script_nativedeck.sh | 42 + template-gpu-operator/time-slicing.yaml | 16 + 116 files changed, 15410 insertions(+), 7 deletions(-) create mode 100644 gpulive/cert/gpulive.crt create mode 100644 gpulive/cert/gpulive.key create mode 100644 gpulive/cert/vm/gpulive.crt create mode 100644 gpulive/cert/vm/gpulive.key create mode 100644 gpulive/cert/vm/rootCA.key create mode 100644 gpulive/cert/vm/rootCA.pem create mode 100644 gpulive/cert/vm/server.csr create mode 100644 gpulive/gitea/vm/1_configure/1_namespace.yaml create mode 100644 gpulive/gitea/vm/1_configure/2_networki_policy.yaml create mode 100644 gpulive/gitea/vm/1_configure/3_PodDisruptionBudget.yaml create mode 100644 gpulive/gitea/vm/1_configure/4_sa.yaml create mode 100644 gpulive/gitea/vm/1_configure/5_secret.yaml create mode 100644 gpulive/gitea/vm/1_configure/6_pvc.yaml create mode 100644 gpulive/gitea/vm/1_configure/README.md create mode 100644 gpulive/gitea/vm/1_configure/pv.yaml create mode 100644 gpulive/gitea/vm/2_configmap/default.yaml create mode 100644 gpulive/gitea/vm/3_apply/deploy.yaml create mode 100644 gpulive/gitea/vm/3_apply/ing.yaml create mode 100644 gpulive/gitea/vm/3_apply/service.yaml create mode 100644 gpulive/harbor_2.9.1/vm/1_configure/cert.yaml create mode 100644 gpulive/harbor_2.9.1/vm/1_configure/init.yaml create mode 100644 gpulive/harbor_2.9.1/vm/1_configure/pv.yaml create mode 100644 gpulive/harbor_2.9.1/vm/1_configure/pvc.yaml create mode 100644 gpulive/harbor_2.9.1/vm/2_configmap/configmap.yaml create mode 100644 gpulive/harbor_2.9.1/vm/3_apply/deploy.yaml create mode 100644 gpulive/harbor_2.9.1/vm/3_apply/gateway.yaml create mode 100644 gpulive/harbor_2.9.1/vm/ing.yaml rename gpulive/{monitoring => prometheus}/base/prometheus/cluster_role.yaml (100%) rename gpulive/{monitoring => prometheus}/base/prometheus/cm.yaml (100%) rename gpulive/{monitoring => prometheus}/base/prometheus/cm_custom_alert_rules.yaml (100%) rename gpulive/{monitoring => prometheus}/base/prometheus/daemonset.yaml (100%) rename gpulive/{monitoring => prometheus}/base/prometheus/deploy.yaml (100%) rename gpulive/{monitoring => prometheus}/base/prometheus/ingress_alertmanager.yaml (100%) rename gpulive/{monitoring => prometheus}/base/prometheus/ingress_prometheus.yaml (100%) rename gpulive/{monitoring => prometheus}/base/prometheus/kustomization.yaml (100%) rename gpulive/{monitoring => prometheus}/base/prometheus/namespace.yaml (100%) rename gpulive/{monitoring => prometheus}/base/prometheus/pvc.yaml (100%) rename gpulive/{monitoring => prometheus}/base/prometheus/sa.yaml (100%) rename gpulive/{monitoring => prometheus}/base/prometheus/service.yaml (100%) rename gpulive/{monitoring => prometheus}/base/prometheus/state_full_set.yaml (100%) rename gpulive/{monitoring => prometheus}/overlays/default/auth (100%) rename gpulive/{monitoring => prometheus}/overlays/default/grafana/kustomization.yaml (100%) rename gpulive/{monitoring => prometheus}/overlays/default/grafana/patch-cm.yaml (100%) rename gpulive/{monitoring => prometheus}/overlays/default/grafana/patch-deploy.yaml (100%) rename gpulive/{monitoring => prometheus}/overlays/default/grafana/patch-ingress_grafana.yaml (100%) rename gpulive/{monitoring => prometheus}/overlays/default/grafana/patch-secret.yaml (100%) rename gpulive/{monitoring => prometheus}/overlays/default/kustomization.yaml (100%) rename gpulive/{monitoring => prometheus}/overlays/default/prometheus/kustomization.yaml (100%) rename gpulive/{monitoring => prometheus}/overlays/default/prometheus/patch-cm-prometheus-server.yaml (100%) rename gpulive/{monitoring => prometheus}/overlays/default/prometheus/patch-deploy.yaml (100%) rename gpulive/{monitoring => prometheus}/overlays/default/prometheus/patch-ingress_alertmanager.yaml (100%) rename gpulive/{monitoring => prometheus}/overlays/default/prometheus/patch-ingress_prometheus.yaml (100%) create mode 100644 gpulive/prometheus/overlays/vm/kustomization.yaml create mode 100644 gpulive/prometheus/overlays/vm/prometheus/kustomization.yaml create mode 100644 gpulive/prometheus/overlays/vm/prometheus/patch-cm-prometheus-server.yaml create mode 100644 gpulive/prometheus/overlays/vm/prometheus/patch-deploy.yaml create mode 100644 gpulive/prometheus/overlays/vm/prometheus/patch-ingress_alertmanager.yaml create mode 100644 gpulive/prometheus/overlays/vm/prometheus/patch-ingress_prometheus.yaml rename gpulive/{monitoring => vm-monitoring}/base/grafana/cm.yaml (100%) rename gpulive/{monitoring => vm-monitoring}/base/grafana/deploy.yaml (100%) rename gpulive/{monitoring => vm-monitoring}/base/grafana/kustomization.yaml (100%) rename gpulive/{monitoring => vm-monitoring}/base/grafana/pvc.yaml (100%) rename gpulive/{monitoring => vm-monitoring}/base/grafana/role.yaml (100%) rename gpulive/{monitoring => vm-monitoring}/base/grafana/sa.yaml (100%) rename gpulive/{monitoring => vm-monitoring}/base/grafana/secret.yaml (100%) rename gpulive/{monitoring => vm-monitoring}/base/grafana/service.yaml (100%) create mode 100644 gpulive/vm-monitoring/base/prometheus/cluster_role.yaml create mode 100644 gpulive/vm-monitoring/base/prometheus/cm.yaml create mode 100644 gpulive/vm-monitoring/base/prometheus/cm_custom_alert_rules.yaml create mode 100644 gpulive/vm-monitoring/base/prometheus/daemonset.yaml create mode 100644 gpulive/vm-monitoring/base/prometheus/deploy.yaml create mode 100644 gpulive/vm-monitoring/base/prometheus/ingress_alertmanager.yaml create mode 100644 gpulive/vm-monitoring/base/prometheus/ingress_prometheus.yaml create mode 100644 gpulive/vm-monitoring/base/prometheus/kustomization.yaml create mode 100644 gpulive/vm-monitoring/base/prometheus/namespace.yaml create mode 100644 gpulive/vm-monitoring/base/prometheus/pvc.yaml create mode 100644 gpulive/vm-monitoring/base/prometheus/sa.yaml create mode 100644 gpulive/vm-monitoring/base/prometheus/service.yaml create mode 100644 gpulive/vm-monitoring/base/prometheus/state_full_set.yaml create mode 100644 gpulive/vm-monitoring/overlays/default/auth create mode 100644 gpulive/vm-monitoring/overlays/default/grafana/kustomization.yaml create mode 100644 gpulive/vm-monitoring/overlays/default/grafana/patch-cm.yaml create mode 100644 gpulive/vm-monitoring/overlays/default/grafana/patch-deploy.yaml rename gpulive/{monitoring/base/grafana/ingress.yaml => vm-monitoring/overlays/default/grafana/patch-ingress_grafana.yaml} (77%) create mode 100644 gpulive/vm-monitoring/overlays/default/grafana/patch-secret.yaml create mode 100644 gpulive/vm-monitoring/overlays/default/kustomization.yaml create mode 100644 gpulive/vm-monitoring/overlays/default/prometheus/kustomization.yaml create mode 100644 gpulive/vm-monitoring/overlays/default/prometheus/patch-cm-prometheus-server.yaml create mode 100644 gpulive/vm-monitoring/overlays/default/prometheus/patch-deploy.yaml create mode 100644 gpulive/vm-monitoring/overlays/default/prometheus/patch-ingress_alertmanager.yaml create mode 100644 gpulive/vm-monitoring/overlays/default/prometheus/patch-ingress_prometheus.yaml create mode 100644 gpulive/vm-monitoring/overlays/vm/auth create mode 100644 gpulive/vm-monitoring/overlays/vm/grafana/kustomization.yaml create mode 100644 gpulive/vm-monitoring/overlays/vm/grafana/patch-cm.yaml create mode 100644 gpulive/vm-monitoring/overlays/vm/grafana/patch-deploy.yaml create mode 100644 gpulive/vm-monitoring/overlays/vm/grafana/patch-ingress_grafana.yaml create mode 100644 gpulive/vm-monitoring/overlays/vm/grafana/patch-secret.yaml create mode 100644 gpulive/vm-monitoring/overlays/vm/kustomization.yaml create mode 100644 gpulive/vm-monitoring/overlays/vm/prometheus/kustomization.yaml create mode 100644 gpulive/vm-monitoring/overlays/vm/prometheus/patch-cm-prometheus-server.yaml create mode 100644 gpulive/vm-monitoring/overlays/vm/prometheus/patch-deploy.yaml create mode 100644 gpulive/vm-monitoring/overlays/vm/prometheus/patch-ingress_alertmanager.yaml create mode 100644 gpulive/vm-monitoring/overlays/vm/prometheus/patch-ingress_prometheus.yaml create mode 100755 nodegroup/make_nodegroup create mode 100644 template-gpu-operator/crds/nvidia.com_clusterpolicies.yaml create mode 100644 template-gpu-operator/crds/nvidia.com_nvidiadrivers.yaml create mode 100644 template-gpu-operator/gpu-operator.yaml create mode 100644 template-gpu-operator/highway.yaml create mode 100644 template-gpu-operator/mig.yaml create mode 100644 template-gpu-operator/mig.yaml.bak create mode 100644 template-gpu-operator/script.sh create mode 100644 template-gpu-operator/script_nativedeck.sh create mode 100644 template-gpu-operator/time-slicing.yaml diff --git a/gpulive/cert/gpulive.crt b/gpulive/cert/gpulive.crt new file mode 100644 index 0000000..a4ba7f1 --- /dev/null +++ b/gpulive/cert/gpulive.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDljCCAn6gAwIBAgIUagOvFaYyS/yDsKIDiEFndbL94HUwDQYJKoZIhvcNAQEL +BQAwPzELMAkGA1UEBhMCS1IxDjAMBgNVBAgMBVNlb3VsMQ0wCwYDVQQKDARnb29k +MREwDwYDVQQDDAhnb29kLmNvbTAeFw0yNTA2MjMwOTQ2MjlaFw0yNzA2MjMwOTQ2 +MjlaME4xCzAJBgNVBAYTAktSMQ4wDAYDVQQIDAVTZW91bDEOMAwGA1UECgwFaWpp +bmMxHzAdBgNVBAMMFiouZ3B1bGl2ZS5uaG5jbG91ZC5jb20wggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQCefQoIn0JWkKchP4U8DV4dafhJfyy6LExm2sMv +hT8dTa3PuNXve1thXS5l/9UEVGdC7EP1k+biI382ouMfhZeLK5vg29H1BXzUvTx3 +h6aPypUi8FPm1SuM6zL1V9ZVh5z6QiwkzyMxVYGM1YsiWLO0SH24eLVDi3/EFYVx +JrXsbe82jJ0MmZ+ZqWwI8MJSXCef4YE7+BxGrBTo9jaPm+Nb163ZW6ZokWB4WZNt +SVYXcSFULYMPB085LZeuJUtE9APLq7VG8GbEqe+ApENwLpF/HvtHnuFLgftNu9zS +L+gX9s2ekjg8q7IT9f20N59/P3mEMk9VHA9ABnNaLEJ2KQ9RAgMBAAGjezB5MDcG +A1UdEQQwMC6CFGdwdWxpdmUubmhuY2xvdWQuY29tghYqLmdwdWxpdmUubmhuY2xv +dWQuY29tMB0GA1UdDgQWBBR2c6jASvY/IiMw9tEjKXRKUXyUAzAfBgNVHSMEGDAW +gBRGafRI9OGcRnoBgsE1Qhqf5rLl5TANBgkqhkiG9w0BAQsFAAOCAQEATZqNGWj3 +e7c2KDH4h4eCnTeuZinONpd1XUvk/LT8GkrndjjfX74hc2G4Es8v1ojWFOEtCFkI +v0FRQkHwI07RW9hojldUPC2SksZWxYztOivI4y5rW/Q9K5Fl02YTtgcTZIQx1BAj +7qehEp+Fs3bdjLjornPuyG9rqwsWpmwoYNASECcGr5ON5CJ0BljuD7rxbLrT/rR/ +fG8/NzA0EBYbfUNQzc4xR6vsTmJOEaty2H1HsVL+LQSD7Bj6+zNAfORLS2+OHBz7 +qZJzyH4IawJvvRzoh9KKiYTZORCv5lkcpjIVFVqWwh+AIHTyieHo5j8EiAHT4//r +CcX/mCaHlIVa9g== +-----END CERTIFICATE----- diff --git a/gpulive/cert/gpulive.key b/gpulive/cert/gpulive.key new file mode 100644 index 0000000..0d59b41 --- /dev/null +++ b/gpulive/cert/gpulive.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCefQoIn0JWkKch +P4U8DV4dafhJfyy6LExm2sMvhT8dTa3PuNXve1thXS5l/9UEVGdC7EP1k+biI382 +ouMfhZeLK5vg29H1BXzUvTx3h6aPypUi8FPm1SuM6zL1V9ZVh5z6QiwkzyMxVYGM +1YsiWLO0SH24eLVDi3/EFYVxJrXsbe82jJ0MmZ+ZqWwI8MJSXCef4YE7+BxGrBTo +9jaPm+Nb163ZW6ZokWB4WZNtSVYXcSFULYMPB085LZeuJUtE9APLq7VG8GbEqe+A +pENwLpF/HvtHnuFLgftNu9zSL+gX9s2ekjg8q7IT9f20N59/P3mEMk9VHA9ABnNa +LEJ2KQ9RAgMBAAECggEABbYIk1PDcDS5aqYY7WRBH/SeaqEw9QR9PFEykP1pAmu2 +97IRaeMBrmCAa4Yfuig23SwZk5/gxTiPCcj7P9RW9NmwpEmt8/mR2TJFTryOPZNv +P4ypeVTB9phphBQsnqhkX7ncUJDZejQHY98BRIH6kWSLVm2ctCWZIde+SnWHclDC +FGIGXHeXUz+mueqK4d6G7cYpj53NQkboEG6Nq/wenQFVuJ7oS84WlEjza+nXK8TY +iz5bJcDprZTWwe6sP50As1bDeWohUBhGYCRTYxYqsWWI4Y+whpyCyPl1J/P/M1GR +Hq2qcK+4/GhA6WZ2Rd+iYxLCOooBqLVGiLIzUyG2AQKBgQDXfKtTiD7gr/ufJdiQ +hWZcMzC3GecdqLzyoAurVtwFyuKefzQvRtLD4mRFOfk2BClULKqlSWCwMuR0ZJo4 +yyRw8Cj/0zGUXrWGsol+8WI0FrBu/7i05p8NP5QAU5bTY/+yqNr2/b2dCi/UPa0V +aROeFOq/SE2PfiAKdEjOC9xAhQKBgQC8SQt7obbz2gG0m9iKpRbV4plycFuQ/kNY +pNEUfF76D+wXo4nhgSOkp1p0LV2qkUi4rbNuUumhnbAJ7K7/lUCfp7Wn54RsWMZE +KejH2ro4gx3/XueufHLomrQ0czMD9A42Vkf1HC4/bjxkmdFWzFarRXFOyC9YQ+xj +kVe+w5jTXQKBgQDGFw8QLRFQT7bJ6GqbAGbGnzBLQf6Z30JC8CmKCsEcehO1jE4W +n86kz/tJQC/+Hfk4Lg94/mlp0H7/GHRFfUk0oTGvayAKur2442tOTOvv4mOyxlWv +xsmzzhxp6G2gSi8Gt/8CSuQB9xlczI3OPtgP3D6oNPlHzbP6qEc9Ut3YWQKBgCn/ +9ULHyQLOP5ElLoGG3/goCuifLZ3DSgyM/2Kdd6Y6RQTk4w3de7Dv79p4gCtbKyie +/qZ+ckUt4qXkGQlEJt02UOw6VazBhMCRxK5IGUAf52IernmaoxtF3yrQA7I/D6iJ +SJXhimN12JzsPukovbCI1gSn3P/IbwOq2TLheMa1AoGBAMgIkmtSpk3DG1XeM8Cj +auR/HAh0tumCe6qmjnKGXhJyhMnsWtJ/TGF5Y/7Cfj45bHA851TiBkTdNRuI+CHJ +tXm2iwnyHyi32lCqyjVc746QJv43DEXeWk/bx1fmCgDUAO4OrTP/IeHLAeXZvih4 +bwNJTWjdNQPSytvcKv1KMiZj +-----END PRIVATE KEY----- diff --git a/gpulive/cert/vm/gpulive.crt b/gpulive/cert/vm/gpulive.crt new file mode 100644 index 0000000..a4ba7f1 --- /dev/null +++ b/gpulive/cert/vm/gpulive.crt @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDljCCAn6gAwIBAgIUagOvFaYyS/yDsKIDiEFndbL94HUwDQYJKoZIhvcNAQEL +BQAwPzELMAkGA1UEBhMCS1IxDjAMBgNVBAgMBVNlb3VsMQ0wCwYDVQQKDARnb29k +MREwDwYDVQQDDAhnb29kLmNvbTAeFw0yNTA2MjMwOTQ2MjlaFw0yNzA2MjMwOTQ2 +MjlaME4xCzAJBgNVBAYTAktSMQ4wDAYDVQQIDAVTZW91bDEOMAwGA1UECgwFaWpp +bmMxHzAdBgNVBAMMFiouZ3B1bGl2ZS5uaG5jbG91ZC5jb20wggEiMA0GCSqGSIb3 +DQEBAQUAA4IBDwAwggEKAoIBAQCefQoIn0JWkKchP4U8DV4dafhJfyy6LExm2sMv +hT8dTa3PuNXve1thXS5l/9UEVGdC7EP1k+biI382ouMfhZeLK5vg29H1BXzUvTx3 +h6aPypUi8FPm1SuM6zL1V9ZVh5z6QiwkzyMxVYGM1YsiWLO0SH24eLVDi3/EFYVx +JrXsbe82jJ0MmZ+ZqWwI8MJSXCef4YE7+BxGrBTo9jaPm+Nb163ZW6ZokWB4WZNt +SVYXcSFULYMPB085LZeuJUtE9APLq7VG8GbEqe+ApENwLpF/HvtHnuFLgftNu9zS +L+gX9s2ekjg8q7IT9f20N59/P3mEMk9VHA9ABnNaLEJ2KQ9RAgMBAAGjezB5MDcG +A1UdEQQwMC6CFGdwdWxpdmUubmhuY2xvdWQuY29tghYqLmdwdWxpdmUubmhuY2xv +dWQuY29tMB0GA1UdDgQWBBR2c6jASvY/IiMw9tEjKXRKUXyUAzAfBgNVHSMEGDAW +gBRGafRI9OGcRnoBgsE1Qhqf5rLl5TANBgkqhkiG9w0BAQsFAAOCAQEATZqNGWj3 +e7c2KDH4h4eCnTeuZinONpd1XUvk/LT8GkrndjjfX74hc2G4Es8v1ojWFOEtCFkI +v0FRQkHwI07RW9hojldUPC2SksZWxYztOivI4y5rW/Q9K5Fl02YTtgcTZIQx1BAj +7qehEp+Fs3bdjLjornPuyG9rqwsWpmwoYNASECcGr5ON5CJ0BljuD7rxbLrT/rR/ +fG8/NzA0EBYbfUNQzc4xR6vsTmJOEaty2H1HsVL+LQSD7Bj6+zNAfORLS2+OHBz7 +qZJzyH4IawJvvRzoh9KKiYTZORCv5lkcpjIVFVqWwh+AIHTyieHo5j8EiAHT4//r +CcX/mCaHlIVa9g== +-----END CERTIFICATE----- diff --git a/gpulive/cert/vm/gpulive.key b/gpulive/cert/vm/gpulive.key new file mode 100644 index 0000000..0d59b41 --- /dev/null +++ b/gpulive/cert/vm/gpulive.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCefQoIn0JWkKch +P4U8DV4dafhJfyy6LExm2sMvhT8dTa3PuNXve1thXS5l/9UEVGdC7EP1k+biI382 +ouMfhZeLK5vg29H1BXzUvTx3h6aPypUi8FPm1SuM6zL1V9ZVh5z6QiwkzyMxVYGM +1YsiWLO0SH24eLVDi3/EFYVxJrXsbe82jJ0MmZ+ZqWwI8MJSXCef4YE7+BxGrBTo +9jaPm+Nb163ZW6ZokWB4WZNtSVYXcSFULYMPB085LZeuJUtE9APLq7VG8GbEqe+A +pENwLpF/HvtHnuFLgftNu9zSL+gX9s2ekjg8q7IT9f20N59/P3mEMk9VHA9ABnNa +LEJ2KQ9RAgMBAAECggEABbYIk1PDcDS5aqYY7WRBH/SeaqEw9QR9PFEykP1pAmu2 +97IRaeMBrmCAa4Yfuig23SwZk5/gxTiPCcj7P9RW9NmwpEmt8/mR2TJFTryOPZNv +P4ypeVTB9phphBQsnqhkX7ncUJDZejQHY98BRIH6kWSLVm2ctCWZIde+SnWHclDC +FGIGXHeXUz+mueqK4d6G7cYpj53NQkboEG6Nq/wenQFVuJ7oS84WlEjza+nXK8TY +iz5bJcDprZTWwe6sP50As1bDeWohUBhGYCRTYxYqsWWI4Y+whpyCyPl1J/P/M1GR +Hq2qcK+4/GhA6WZ2Rd+iYxLCOooBqLVGiLIzUyG2AQKBgQDXfKtTiD7gr/ufJdiQ +hWZcMzC3GecdqLzyoAurVtwFyuKefzQvRtLD4mRFOfk2BClULKqlSWCwMuR0ZJo4 +yyRw8Cj/0zGUXrWGsol+8WI0FrBu/7i05p8NP5QAU5bTY/+yqNr2/b2dCi/UPa0V +aROeFOq/SE2PfiAKdEjOC9xAhQKBgQC8SQt7obbz2gG0m9iKpRbV4plycFuQ/kNY +pNEUfF76D+wXo4nhgSOkp1p0LV2qkUi4rbNuUumhnbAJ7K7/lUCfp7Wn54RsWMZE +KejH2ro4gx3/XueufHLomrQ0czMD9A42Vkf1HC4/bjxkmdFWzFarRXFOyC9YQ+xj +kVe+w5jTXQKBgQDGFw8QLRFQT7bJ6GqbAGbGnzBLQf6Z30JC8CmKCsEcehO1jE4W +n86kz/tJQC/+Hfk4Lg94/mlp0H7/GHRFfUk0oTGvayAKur2442tOTOvv4mOyxlWv +xsmzzhxp6G2gSi8Gt/8CSuQB9xlczI3OPtgP3D6oNPlHzbP6qEc9Ut3YWQKBgCn/ +9ULHyQLOP5ElLoGG3/goCuifLZ3DSgyM/2Kdd6Y6RQTk4w3de7Dv79p4gCtbKyie +/qZ+ckUt4qXkGQlEJt02UOw6VazBhMCRxK5IGUAf52IernmaoxtF3yrQA7I/D6iJ +SJXhimN12JzsPukovbCI1gSn3P/IbwOq2TLheMa1AoGBAMgIkmtSpk3DG1XeM8Cj +auR/HAh0tumCe6qmjnKGXhJyhMnsWtJ/TGF5Y/7Cfj45bHA851TiBkTdNRuI+CHJ +tXm2iwnyHyi32lCqyjVc746QJv43DEXeWk/bx1fmCgDUAO4OrTP/IeHLAeXZvih4 +bwNJTWjdNQPSytvcKv1KMiZj +-----END PRIVATE KEY----- diff --git a/gpulive/cert/vm/rootCA.key b/gpulive/cert/vm/rootCA.key new file mode 100644 index 0000000..ab6751a --- /dev/null +++ b/gpulive/cert/vm/rootCA.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDOd8XzD5yvrtT4 +Q4yRnyFl/LKaeTsrbK9qcYgbwb/A4iTzak5b5XCf/g72GMwuNokmmOlk2ppEmdJJ +qCMEdXHCR7YCuf81bxdd9xvsxShQ1JTdAdlOBA6OlsqGsk5ls/gTxLWyj+EgvkPf +l/WnIH0s2J+tDwr/QNMHMPE0pCLXef3F2TeeTyrbHv2Vbf4KcAOfA+wDjrWK49rr +1In5U0Q9Vhk5/4tbUU9SrnBwCrF5B7c10fajJ2iIZj6gSjZd1/C7L+LntC3KtJF5 +bhASjvixG92CsysBxnoZsQnhRa2CFVi1jVfWLwxsNRcT2bVGnoEzaS7P2pkavjT3 +A+SxS4QJAgMBAAECggEAGODz2FMERWMixHuXP6DWLGzjtJtxFs104bWmefnBumNF +opMnuAkQpia99MeOKu9lXWJFlWKF/kguv7Nj7NCeEZQEiZWR/AVT4n25PrGPJxZG +jc8AOVdhzaq74rkvmy+xb+hfIJIXxZNHqHSuJCJVEwdTTk4mvBU6mH48QmSY0i/o +aETuMrfB86pZYgf/mND2IlIcfbTDFyn1bUevI+hxh2kU71BDrLyH34ib09OxhYnO +8uS1SOwsPF/atLOmS5efxYLtFwS1yF1hQvYt8rUS+5Y/CsGO3ZO2X6WiJB1+KA5x +adeU2CckeNQVkljJm6C9ZwFpXU/v237gob5cKXOqpQKBgQDV3AIiRCG6b4gU/ZGK +UhAMFIKllVILSF1RyIansu5baM+1jysRDvkqpUoYoc0pljGaYHoWXyDLJ5WnHPMN +8E7SQRkZ2s0h8NsR7TukTqm4LeuGzDH64rjAb6VOun3uKMDmHTjoo5rU/7Tdsxjh +ggY9WkltvniHg0EkdlMHCRhk2wKBgQD3JudPMj2gSXoF6ylTX6mviGwc2FSxZOLn +83PdCV2iQyVJwMUlCl5IZjATERFuj4NV1TE6j2GjOl1pCnoanDgg7603Aw3bOPT3 +xcnQIhE4UmSsIfLjUfZrM9oTE1d6L1R2/ptXYSF4nbmOuAqknp9tpHOrtrv9L+xy +SSkB+Hh96wKBgAHlWmt9WSMy++zbtp+YTKYexG29XiXVdWZVNcRk1LIgwr6kT0hu +bo0PPBHt1UjKGRR6SHrMSxkh97SskAcOm1RH2XYeN3VWkkqdZMr3G5ATcCQYyVSl +D+g81SVWh+lFmTgxCCEf2i6LOcpBXTYalf2TQFVlu+HPqxMsrV3BmeLDAoGBAO1R +FNCN7HDJvPqL6P2eoN/k/fLgospHlj2lHf8DbD5PfTGJgYPkpc2dGdJlFDj2YuT7 +Ni8F2HoUo/cl0UNV9+tgWRG2xApZcZes+vD4lVJggzxpyMEQRa6AdojUjK3NXqx4 +JA9OPAaGZNOu0sFpcqqkO1GJwI8IAmFm8JJs1L4LAoGAS61hq8fUegnvv1HQtfHP +CnHKUXHazQoTyycbrZPnN0jmYNmnBW8jyF+WHg2+Qa6/Bo5Tq2H9242VCktID6M9 +TEAhBkFifbUzFBLvl//C86xUBslDqND4Qa+8HFE4FbSO0RcmTyLatZ0V/ai2cVtj +jAD3OaTPNdv3NJbd8+zetTY= +-----END PRIVATE KEY----- diff --git a/gpulive/cert/vm/rootCA.pem b/gpulive/cert/vm/rootCA.pem new file mode 100644 index 0000000..401dc88 --- /dev/null +++ b/gpulive/cert/vm/rootCA.pem @@ -0,0 +1,21 @@ +-----BEGIN CERTIFICATE----- +MIIDXzCCAkegAwIBAgIUPHDvz3298b47cmBPGk9aukCPmTwwDQYJKoZIhvcNAQEL +BQAwPzELMAkGA1UEBhMCS1IxDjAMBgNVBAgMBVNlb3VsMQ0wCwYDVQQKDARnb29k +MREwDwYDVQQDDAhnb29kLmNvbTAeFw0yNTA2MjMwOTQxMjdaFw0yNzA2MjMwOTQx +MjdaMD8xCzAJBgNVBAYTAktSMQ4wDAYDVQQIDAVTZW91bDENMAsGA1UECgwEZ29v +ZDERMA8GA1UEAwwIZ29vZC5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEK +AoIBAQDOd8XzD5yvrtT4Q4yRnyFl/LKaeTsrbK9qcYgbwb/A4iTzak5b5XCf/g72 +GMwuNokmmOlk2ppEmdJJqCMEdXHCR7YCuf81bxdd9xvsxShQ1JTdAdlOBA6OlsqG +sk5ls/gTxLWyj+EgvkPfl/WnIH0s2J+tDwr/QNMHMPE0pCLXef3F2TeeTyrbHv2V +bf4KcAOfA+wDjrWK49rr1In5U0Q9Vhk5/4tbUU9SrnBwCrF5B7c10fajJ2iIZj6g +SjZd1/C7L+LntC3KtJF5bhASjvixG92CsysBxnoZsQnhRa2CFVi1jVfWLwxsNRcT +2bVGnoEzaS7P2pkavjT3A+SxS4QJAgMBAAGjUzBRMB0GA1UdDgQWBBRGafRI9OGc +RnoBgsE1Qhqf5rLl5TAfBgNVHSMEGDAWgBRGafRI9OGcRnoBgsE1Qhqf5rLl5TAP +BgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAD2qU2QxK43/0tmDTc +mpXjXA58bIdbFXJAgK7xXVoDixG6RxAx6SS5puZtRtlh4n1phcdKbez4bMV6aOOY +2DUsQBwEU7Lj0o3wX3hhJ//RG6IzibwpnlxgE/En44KZ6i1plaNLw5FqZRRuMKeZ +I6iyuDi9Dk2W27NIJHuO2ryNXOCBwVj5hDeQmoVAtZJntlmILG3/fqK2ma1m3xIg +rMNbJ/NuQZa/bHqgJvBQkTiNPtzuA/DJzFovQ6CXQQ4Jv7GPh3prlkHJE3rAAzAg +PZ4RtUZX+LhquM4lwSveYG4fESKHiFJgB+oclBzS/nf8KzkX22sdBwdW0YNtOllS +6BRS +-----END CERTIFICATE----- diff --git a/gpulive/cert/vm/server.csr b/gpulive/cert/vm/server.csr new file mode 100644 index 0000000..5eec72a --- /dev/null +++ b/gpulive/cert/vm/server.csr @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICkzCCAXsCAQAwTjELMAkGA1UEBhMCS1IxDjAMBgNVBAgMBVNlb3VsMQ4wDAYD +VQQKDAVpamluYzEfMB0GA1UEAwwWKi5ncHVsaXZlLm5obmNsb3VkLmNvbTCCASIw +DQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAJ59CgifQlaQpyE/hTwNXh1p+El/ +LLosTGbawy+FPx1Nrc+41e97W2FdLmX/1QRUZ0LsQ/WT5uIjfzai4x+Fl4srm+Db +0fUFfNS9PHeHpo/KlSLwU+bVK4zrMvVX1lWHnPpCLCTPIzFVgYzViyJYs7RIfbh4 +tUOLf8QVhXEmtext7zaMnQyZn5mpbAjwwlJcJ5/hgTv4HEasFOj2No+b41vXrdlb +pmiRYHhZk21JVhdxIVQtgw8HTzktl64lS0T0A8urtUbwZsSp74CkQ3AukX8e+0ee +4UuB+0273NIv6Bf2zZ6SODyrshP1/bQ3n38/eYQyT1UcD0AGc1osQnYpD1ECAwEA +AaAAMA0GCSqGSIb3DQEBCwUAA4IBAQA3zXoZY3NQyRX+e5NmiLKsOTeQOkuJ3z5L +0vXy0UXNaaLPuNMemU3f4PqW2ye2qp3618tXvIeR0dJ/kFNjvHanAz3Iuix0Kgfp +3eXrsZHKrmFbUkzEl2WtcemoQxeZJnqwGyYaIm0kWZio05fErobYPz69xl49gqnT +P5QgVKW6l+WoqawJ35O/IPscRlWV1KCLc/z08KQ0cZQJWNcL91DgSrEJq265CSTT +CMLRF4JI3bOTZ47e9oSD4mdHRgeIPz0rUAMQjnEOMhCvl2bLAENzDRobwC1YNUVY +Fz/4cnLsWPHPoTRd4EmaWyUu32UITxDKcBoZ9muoMpQsu9yEMDiY +-----END CERTIFICATE REQUEST----- diff --git a/gpulive/gitea/vm/1_configure/1_namespace.yaml b/gpulive/gitea/vm/1_configure/1_namespace.yaml new file mode 100644 index 0000000..4ae1fdb --- /dev/null +++ b/gpulive/gitea/vm/1_configure/1_namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: gitea + labels: + kubernetes.io/metadata.name: gitea diff --git a/gpulive/gitea/vm/1_configure/2_networki_policy.yaml b/gpulive/gitea/vm/1_configure/2_networki_policy.yaml new file mode 100644 index 0000000..2792728 --- /dev/null +++ b/gpulive/gitea/vm/1_configure/2_networki_policy.yaml @@ -0,0 +1,79 @@ +# Source: gitea/charts/postgresql-ha/templates/pgpool/networkpolicy.yaml +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: release-name-postgresql-ha-pgpool + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/version: 16.3.0 + app.kubernetes.io/component: pgpool + role: data +spec: + podSelector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/component: pgpool + policyTypes: + - Ingress + - Egress + egress: + - {} + ingress: + - ports: + - port: 5432 +--- +# Source: gitea/charts/postgresql-ha/templates/postgresql/networkpolicy.yaml +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: release-name-postgresql-ha-postgresql + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/version: 16.3.0 + app.kubernetes.io/component: postgresql + role: data +spec: + podSelector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/component: postgresql + policyTypes: + - Ingress + - Egress + egress: + - {} + ingress: + - ports: + - port: 5432 +--- +# Source: gitea/charts/redis-cluster/templates/networkpolicy.yaml +kind: NetworkPolicy +apiVersion: networking.k8s.io/v1 +metadata: + name: release-name-redis-cluster + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: redis-cluster + app.kubernetes.io/version: 7.2.5 +spec: + podSelector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: redis-cluster + policyTypes: + - Ingress + - Egress + egress: + - {} + ingress: + # Allow inbound connections + - ports: + - port: 6379 + - port: 16379 diff --git a/gpulive/gitea/vm/1_configure/3_PodDisruptionBudget.yaml b/gpulive/gitea/vm/1_configure/3_PodDisruptionBudget.yaml new file mode 100644 index 0000000..f01e48b --- /dev/null +++ b/gpulive/gitea/vm/1_configure/3_PodDisruptionBudget.yaml @@ -0,0 +1,77 @@ +# Source: gitea/charts/postgresql-ha/templates/pgpool/pdb.yaml +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: release-name-postgresql-ha-pgpool + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/version: 4.5.2 + app.kubernetes.io/component: pgpool +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/component: pgpool +--- +# Source: gitea/charts/postgresql-ha/templates/postgresql/pdb.yaml +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: release-name-postgresql-ha-postgresql + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/version: 16.3.0 + app.kubernetes.io/component: postgresql +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/component: postgresql +--- +# Source: gitea/charts/postgresql-ha/templates/postgresql/witness-pdb.yaml +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: release-name-postgresql-ha-postgresql-witness + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/version: 16.3.0 + app.kubernetes.io/component: postgresql + role: witness +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/component: postgresql + role: witness +--- +# Source: gitea/charts/redis-cluster/templates/poddisruptionbudget.yaml +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: release-name-redis-cluster + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: redis-cluster + app.kubernetes.io/version: 7.2.5 +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: redis-cluster + matchExpressions: + - {key: job-name, operator: NotIn, values: [release-name-redis-cluster-cluster-update]} diff --git a/gpulive/gitea/vm/1_configure/4_sa.yaml b/gpulive/gitea/vm/1_configure/4_sa.yaml new file mode 100644 index 0000000..4667fb0 --- /dev/null +++ b/gpulive/gitea/vm/1_configure/4_sa.yaml @@ -0,0 +1,23 @@ +# Source: gitea/charts/postgresql-ha/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: release-name-postgresql-ha + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/version: 16.3.0 +automountServiceAccountToken: false +--- +# Source: gitea/charts/redis-cluster/templates/redis-serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: release-name-redis-cluster + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: redis-cluster + app.kubernetes.io/version: 7.2.5 +automountServiceAccountToken: false diff --git a/gpulive/gitea/vm/1_configure/5_secret.yaml b/gpulive/gitea/vm/1_configure/5_secret.yaml new file mode 100644 index 0000000..375e3b0 --- /dev/null +++ b/gpulive/gitea/vm/1_configure/5_secret.yaml @@ -0,0 +1,384 @@ +# Source: gitea/charts/postgresql-ha/templates/pgpool/secrets.yaml +apiVersion: v1 +kind: Secret +metadata: + name: release-name-postgresql-ha-pgpool + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/version: 4.5.2 + app.kubernetes.io/component: pgpool +type: Opaque +data: + admin-password: "aWppbmMxMjMh" +--- +# Source: gitea/charts/postgresql-ha/templates/postgresql/secrets.yaml +apiVersion: v1 +kind: Secret +metadata: + name: release-name-postgresql-ha-postgresql + namespace: "gitea" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/version: 16.3.0 + app.kubernetes.io/component: postgresql +type: Opaque +data: + postgres-password: "aWppbmMxMjMh" + password: "Z2l0ZWE=" + repmgr-password: "aWppbmMxMjMh" +--- +# Source: gitea/templates/gitea/config.yaml +apiVersion: v1 +kind: Secret +metadata: + name: release-name-gitea-inline-config + namespace: gitea + labels: + app: gitea + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "1.22.1" + version: "1.22.1" +type: Opaque +stringData: + _generals_: "" + cache: |- + ADAPTER=redis + HOST=redis+cluster://:@release-name-redis-cluster-headless.gitea.svc.cluster.local:6379/0?pool_size=100&idle_timeout=180s& + database: |- + DB_TYPE=postgres + HOST=release-name-postgresql-ha-pgpool.gitea.svc.cluster.local:5432 + NAME=gitea + PASSWD=gitea + USER=gitea + indexer: ISSUE_INDEXER_TYPE=db + metrics: ENABLED=false + queue: |- + CONN_STR=redis+cluster://:@release-name-redis-cluster-headless.gitea.svc.cluster.local:6379/0?pool_size=100&idle_timeout=180s& + TYPE=redis + repository: ROOT=/data/git/gitea-repositories + security: INSTALL_LOCK=true + server: |- + APP_DATA_PATH=/data + DOMAIN=gitea.nhngpuaas.com + ENABLE_PPROF=false + HTTP_PORT=3000 + PROTOCOL=http + ROOT_URL=https://gitea.nhngpuaas.com + SSH_DOMAIN=gitea.nhngpuaas.com + SSH_LISTEN_PORT=2222 + SSH_PORT=22 + START_SSH_SERVER=true + session: |- + PROVIDER=redis + PROVIDER_CONFIG=redis+cluster://:@release-name-redis-cluster-headless.gitea.svc.cluster.local:6379/0?pool_size=100&idle_timeout=180s& +--- +# Source: gitea/templates/gitea/config.yaml +apiVersion: v1 +kind: Secret +metadata: + name: release-name-gitea + namespace: gitea + labels: + app: gitea + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "1.22.1" + version: "1.22.1" +type: Opaque +stringData: + assertions: | + config_environment.sh: |- + #!/usr/bin/env bash + set -euo pipefail + + function env2ini::log() { + printf "${1}\n" + } + + function env2ini::read_config_to_env() { + local section="${1}" + local line="${2}" + + if [[ -z "${line}" ]]; then + # skip empty line + return + fi + + # 'xargs echo -n' trims all leading/trailing whitespaces and a trailing new line + local setting="$(awk -F '=' '{print $1}' <<< "${line}" | xargs echo -n)" + + if [[ -z "${setting}" ]]; then + env2ini::log ' ! invalid setting' + exit 1 + fi + + local value='' + local regex="^${setting}(\s*)=(\s*)(.*)" + if [[ $line =~ $regex ]]; then + value="${BASH_REMATCH[3]}" + else + env2ini::log ' ! invalid setting' + exit 1 + fi + + env2ini::log " + '${setting}'" + + if [[ -z "${section}" ]]; then + export "GITEA____${setting^^}=${value}" # '^^' makes the variable content uppercase + return + fi + + local masked_section="${section//./_0X2E_}" # '//' instructs to replace all matches + masked_section="${masked_section//-/_0X2D_}" + + export "GITEA__${masked_section^^}__${setting^^}=${value}" # '^^' makes the variable content uppercase + } + + function env2ini::reload_preset_envs() { + env2ini::log "Reloading preset envs..." + + while read -r line; do + if [[ -z "${line}" ]]; then + # skip empty line + return + fi + + # 'xargs echo -n' trims all leading/trailing whitespaces and a trailing new line + local setting="$(awk -F '=' '{print $1}' <<< "${line}" | xargs echo -n)" + + if [[ -z "${setting}" ]]; then + env2ini::log ' ! invalid setting' + exit 1 + fi + + local value='' + local regex="^${setting}(\s*)=(\s*)(.*)" + if [[ $line =~ $regex ]]; then + value="${BASH_REMATCH[3]}" + else + env2ini::log ' ! invalid setting' + exit 1 + fi + + env2ini::log " + '${setting}'" + + export "${setting^^}=${value}" # '^^' makes the variable content uppercase + done < "/tmp/existing-envs" + + rm /tmp/existing-envs + } + + + function env2ini::process_config_file() { + local config_file="${1}" + local section="$(basename "${config_file}")" + + if [[ $section == '_generals_' ]]; then + + section='' + else + env2ini::log " ${section}" + fi + + while read -r line; do + env2ini::read_config_to_env "${section}" "${line}" + done < <(awk 1 "${config_file}") # Helm .toYaml trims the trailing new line which breaks line processing; awk 1 ... adds it back while reading + } + + function env2ini::load_config_sources() { + local path="${1}" + + if [[ -d "${path}" ]]; then + env2ini::log "Processing $(basename "${path}")..." + + while read -d '' configFile; do + env2ini::process_config_file "${configFile}" + done < <(find "${path}" -type l -not -name '..data' -print0) + + env2ini::log "\n" + fi + } + + function env2ini::generate_initial_secrets() { + # These environment variables will either be + # - overwritten with user defined values, + # - initially used to set up Gitea + # Anyway, they won't harm existing app.ini files + + export GITEA__SECURITY__INTERNAL_TOKEN=$(gitea generate secret INTERNAL_TOKEN) + export GITEA__SECURITY__SECRET_KEY=$(gitea generate secret SECRET_KEY) + export GITEA__OAUTH2__JWT_SECRET=$(gitea generate secret JWT_SECRET) + export GITEA__SERVER__LFS_JWT_SECRET=$(gitea generate secret LFS_JWT_SECRET) + + env2ini::log "...Initial secrets generated\n" + } + + # save existing envs prior to script execution. Necessary to keep order of preexisting and custom envs + env | (grep -e '^GITEA__' || [[ $? == 1 ]]) > /tmp/existing-envs + + # MUST BE CALLED BEFORE OTHER CONFIGURATION + env2ini::generate_initial_secrets + + env2ini::load_config_sources '/env-to-ini-mounts/inlines/' + env2ini::load_config_sources '/env-to-ini-mounts/additionals/' + + # load existing envs to override auto generated envs + env2ini::reload_preset_envs + + env2ini::log "=== All configuration sources loaded ===\n" + + # safety to prevent rewrite of secret keys if an app.ini already exists + if [ -f ${GITEA_APP_INI} ]; then + env2ini::log 'An app.ini file already exists. To prevent overwriting secret keys, these settings are dropped and remain unchanged:' + env2ini::log ' - security.INTERNAL_TOKEN' + env2ini::log ' - security.SECRET_KEY' + env2ini::log ' - oauth2.JWT_SECRET' + env2ini::log ' - server.LFS_JWT_SECRET' + + unset GITEA__SECURITY__INTERNAL_TOKEN + unset GITEA__SECURITY__SECRET_KEY + unset GITEA__OAUTH2__JWT_SECRET + unset GITEA__SERVER__LFS_JWT_SECRET + fi + + environment-to-ini -o $GITEA_APP_INI +--- +# Source: gitea/templates/gitea/init.yaml +apiVersion: v1 +kind: Secret +metadata: + name: release-name-gitea-init + namespace: gitea + labels: + app: gitea + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "1.22.1" + version: "1.22.1" +type: Opaque +stringData: + configure_gpg_environment.sh: |- + #!/usr/bin/env bash + set -eu + + gpg --batch --import /raw/private.asc + init_directory_structure.sh: |- + #!/usr/bin/env bash + + set -euo pipefail + + set -x + mkdir -p /data/git/.ssh + chmod -R 700 /data/git/.ssh + [ ! -d /data/gitea/conf ] && mkdir -p /data/gitea/conf + + # prepare temp directory structure + mkdir -p "${GITEA_TEMP}" + chmod ug+rwx "${GITEA_TEMP}" + + + + configure_gitea.sh: |- + #!/usr/bin/env bash + + set -euo pipefail + + echo '==== BEGIN GITEA CONFIGURATION ====' + + { # try + gitea migrate + } || { # catch + echo "Gitea migrate might fail due to database connection...This init-container will try again in a few seconds" + exit 1 + } + function test_redis_connection() { + local RETRY=0 + local MAX=30 + + echo 'Wait for redis to become avialable...' + until [ "${RETRY}" -ge "${MAX}" ]; do + nc -vz -w2 release-name-redis-cluster-headless.gitea.svc.cluster.local 6379 && break + RETRY=$[${RETRY}+1] + echo "...not ready yet (${RETRY}/${MAX})" + done + + if [ "${RETRY}" -ge "${MAX}" ]; then + echo "Redis not reachable after '${MAX}' attempts!" + exit 1 + fi + } + + test_redis_connection + function configure_admin_user() { + local full_admin_list=$(gitea admin user list --admin) + local actual_user_table='' + + # We might have distorted output due to warning logs, so we have to detect the actual user table by its headline and trim output above that line + local regex="(.*)(ID\s+Username\s+Email\s+IsActive.*)" + if [[ "${full_admin_list}" =~ $regex ]]; then + actual_user_table=$(echo "${BASH_REMATCH[2]}" | tail -n+2) # tail'ing to drop the table headline + else + # This code block should never be reached, as long as the output table header remains the same. + # If this code block is reached, the regex doesn't match anymore and we probably have to adjust this script. + + echo "ERROR: 'configure_admin_user' was not able to determine the current list of admin users." + echo " Please review the output of 'gitea admin user list --admin' shown below." + echo " If you think it is an issue with the Helm Chart provisioning, file an issue at https://gitea.com/gitea/helm-chart/issues." + echo "DEBUG: Output of 'gitea admin user list --admin'" + echo "--" + echo "${full_admin_list}" + echo "--" + exit 1 + fi + + local ACCOUNT_ID=$(echo "${actual_user_table}" | grep -E "\s+${GITEA_ADMIN_USERNAME}\s+" | awk -F " " "{printf \$1}") + if [[ -z "${ACCOUNT_ID}" ]]; then + local -a create_args + create_args=(--admin --username "${GITEA_ADMIN_USERNAME}" --password "${GITEA_ADMIN_PASSWORD}" --email "gitea@local.domain") + if [[ "${GITEA_ADMIN_PASSWORD_MODE}" = initialOnlyRequireReset ]]; then + create_args+=(--must-change-password=true) + else + create_args+=(--must-change-password=false) + fi + echo "No admin user '${GITEA_ADMIN_USERNAME}' found. Creating now..." + gitea admin user create "${create_args[@]}" + echo '...created.' + else + if [[ "${GITEA_ADMIN_PASSWORD_MODE}" = keepUpdated ]]; then + echo "Admin account '${GITEA_ADMIN_USERNAME}' already exist. Running update to sync password..." + # See https://gitea.com/gitea/helm-chart/issues/673 + # --must-change-password argument was added to change-password, defaulting to true, counter to the previous behavior + # which acted as if it were provided with =false. If the argument is present in this version of gitea, then we + # should add it to prevent requiring frequent admin password resets. + local -a change_args + change_args=(--username "${GITEA_ADMIN_USERNAME}" --password "${GITEA_ADMIN_PASSWORD}") + if gitea admin user change-password --help | grep -qF -- '--must-change-password'; then + change_args+=(--must-change-password=false) + fi + gitea admin user change-password "${change_args[@]}" + echo '...password sync done.' + else + echo "Admin account '${GITEA_ADMIN_USERNAME}' already exist, but update mode is set to '${GITEA_ADMIN_PASSWORD_MODE}'. Skipping." + fi + fi + } + + configure_admin_user + + function configure_ldap() { + echo 'no ldap configuration... skipping.' + } + + configure_ldap + + function configure_oauth() { + echo 'no oauth configuration... skipping.' + } + + configure_oauth + + echo '==== END GITEA CONFIGURATION ====' diff --git a/gpulive/gitea/vm/1_configure/6_pvc.yaml b/gpulive/gitea/vm/1_configure/6_pvc.yaml new file mode 100644 index 0000000..8d6aec6 --- /dev/null +++ b/gpulive/gitea/vm/1_configure/6_pvc.yaml @@ -0,0 +1,18 @@ +# Source: gitea/templates/gitea/pvc.yaml +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: gitea-shared-storage + namespace: gitea +spec: + accessModes: + - ReadWriteOnce + volumeMode: Filesystem + # storageClassName: sc-monitoring + resources: + requests: + storage: 500Gi # default: 10Gi + selector: + matchLabels: + # to create 1-1 relationship for pod - persistent volume use unique labels + name: gitea-shared-storage-pv diff --git a/gpulive/gitea/vm/1_configure/README.md b/gpulive/gitea/vm/1_configure/README.md new file mode 100644 index 0000000..40cfad7 --- /dev/null +++ b/gpulive/gitea/vm/1_configure/README.md @@ -0,0 +1,6 @@ +# Need To Change +## Secret File +* DOMAIN +* ROOT_URL +* SSH_DOMAIN +* ###-password : aWppbmMxMjMh diff --git a/gpulive/gitea/vm/1_configure/pv.yaml b/gpulive/gitea/vm/1_configure/pv.yaml new file mode 100644 index 0000000..8411e61 --- /dev/null +++ b/gpulive/gitea/vm/1_configure/pv.yaml @@ -0,0 +1,125 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: gitea-shared-storage-pv + labels: + name: gitea-shared-storage-pv +spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 500Gi + csi: + driver: exa.csi.ddn.com + volumeHandle: exa1:/gitea + volumeAttributes: # volumeAttributes are the alternative of storageClass params for static (precreated) volumes. + # mountOptions: ro, noflock # list of options for `mount` command + bindMount: "false" # Determines, whether volume will bind mounted or as a separate lustre mount. +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: redis-data-release-name-redis-cluster-0 + labels: + name: redis-data-release-name-redis-cluster-0 +spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 8Gi + csi: + driver: exa.csi.ddn.com + volumeHandle: exa1:/gitea_redis_0 + volumeAttributes: # volumeAttributes are the alternative of storageClass params for static (precreated) volumes. + # mountOptions: ro, noflock # list of options for `mount` command + bindMount: "false" # Determines, whether volume will bind mounted or as a separate lustre mount. +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: redis-data-release-name-redis-cluster-1 + labels: + name: redis-data-release-name-redis-cluster-1 +spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 8Gi + csi: + driver: exa.csi.ddn.com + volumeHandle: exa1:/gitea_redis_1 + volumeAttributes: # volumeAttributes are the alternative of storageClass params for static (precreated) volumes. + # mountOptions: ro, noflock # list of options for `mount` command + bindMount: "false" # Determines, whether volume will bind mounted or as a separate lustre mount. +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: redis-data-release-name-redis-cluster-2 + labels: + name: redis-data-release-name-redis-cluster-2 +spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 8Gi + csi: + driver: exa.csi.ddn.com + volumeHandle: exa1:/gitea_redis_2 + volumeAttributes: # volumeAttributes are the alternative of storageClass params for static (precreated) volumes. + # mountOptions: ro, noflock # list of options for `mount` command + bindMount: "false" # Determines, whether volume will bind mounted or as a separate lustre mount. +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: data-release-name-postgresql-ha-postgresql-0 + labels: + name: data-release-name-postgresql-ha-postgresql-0 +spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 8Gi + csi: + driver: exa.csi.ddn.com + volumeHandle: exa1:/gitea_post_0 + volumeAttributes: # volumeAttributes are the alternative of storageClass params for static (precreated) volumes. + # mountOptions: ro, noflock # list of options for `mount` command + bindMount: "false" # Determines, whether volume will bind mounted or as a separate lustre mount. +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: data-release-name-postgresql-ha-postgresql-1 + labels: + name: data-release-name-postgresql-ha-postgresql-1 +spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 8Gi + csi: + driver: exa.csi.ddn.com + volumeHandle: exa1:/gitea_post_1 + volumeAttributes: # volumeAttributes are the alternative of storageClass params for static (precreated) volumes. + # mountOptions: ro, noflock # list of options for `mount` command + bindMount: "false" # Determines, whether volume will bind mounted or as a separate lustre mount. +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: data-release-name-postgresql-ha-postgresql-2 + labels: + name: data-release-name-postgresql-ha-postgresql-2 +spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 8Gi + csi: + driver: exa.csi.ddn.com + volumeHandle: exa1:/gitea_post_2 + volumeAttributes: # volumeAttributes are the alternative of storageClass params for static (precreated) volumes. + # mountOptions: ro, noflock # list of options for `mount` command + bindMount: "false" # Determines, whether volume will bind mounted or as a separate lustre mount. diff --git a/gpulive/gitea/vm/2_configmap/default.yaml b/gpulive/gitea/vm/2_configmap/default.yaml new file mode 100644 index 0000000..6df32ca --- /dev/null +++ b/gpulive/gitea/vm/2_configmap/default.yaml @@ -0,0 +1,2493 @@ +# Source: gitea/charts/postgresql-ha/templates/postgresql/hooks-scripts-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-postgresql-ha-postgresql-hooks-scripts + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/version: 16.3.0 + app.kubernetes.io/component: postgresql +data: + pre-stop.sh: |- + #!/bin/bash + set -o errexit + set -o pipefail + set -o nounset + + # Debug section + exec 3>&1 + exec 4>&2 + + # Process input parameters + MIN_DELAY_AFTER_PG_STOP_SECONDS=$1 + + # Load Libraries + . /opt/bitnami/scripts/liblog.sh + . /opt/bitnami/scripts/libpostgresql.sh + . /opt/bitnami/scripts/librepmgr.sh + + # Load PostgreSQL & repmgr environment variables + . /opt/bitnami/scripts/postgresql-env.sh + + # Auxiliary functions + is_new_primary_ready() { + return_value=1 + currenty_primary_node="$(repmgr_get_primary_node)" + currenty_primary_host="$(echo $currenty_primary_node | awk '{print $1}')" + + info "$currenty_primary_host != $REPMGR_NODE_NETWORK_NAME" + if [[ $(echo $currenty_primary_node | wc -w) -eq 2 ]] && [[ "$currenty_primary_host" != "$REPMGR_NODE_NETWORK_NAME" ]]; then + info "New primary detected, leaving the cluster..." + return_value=0 + else + info "Waiting for a new primary to be available..." + fi + return $return_value + } + + export MODULE="pre-stop-hook" + + if [[ "${BITNAMI_DEBUG}" == "true" ]]; then + info "Bash debug is on" + else + info "Bash debug is off" + exec 1>/dev/null + exec 2>/dev/null + fi + + postgresql_enable_nss_wrapper + + # Prepare env vars for managing roles + readarray -t primary_node < <(repmgr_get_upstream_node) + primary_host="${primary_node[0]}" + + # Stop postgresql for graceful exit. + PG_STOP_TIME=$EPOCHSECONDS + postgresql_stop + + if [[ -z "$primary_host" ]] || [[ "$primary_host" == "$REPMGR_NODE_NETWORK_NAME" ]]; then + info "Primary node need to wait for a new primary node before leaving the cluster" + retry_while is_new_primary_ready 10 5 + else + info "Standby node doesn't need to wait for a new primary switchover. Leaving the cluster" + fi + + # Make sure pre-stop hook waits at least 25 seconds after stop of PG to make sure PGPOOL detects node is down. + # default terminationGracePeriodSeconds=30 seconds + PG_STOP_DURATION=$(($EPOCHSECONDS - $PG_STOP_TIME)) + if (( $PG_STOP_DURATION < $MIN_DELAY_AFTER_PG_STOP_SECONDS )); then + WAIT_TO_PG_POOL_TIME=$(($MIN_DELAY_AFTER_PG_STOP_SECONDS - $PG_STOP_DURATION)) + info "PG stopped including primary switchover in $PG_STOP_DURATION. Waiting additional $WAIT_TO_PG_POOL_TIME seconds for PG pool" + sleep $WAIT_TO_PG_POOL_TIME + fi + + readiness-probe.sh: |- + #!/bin/bash + set -o errexit + set -o pipefail + set -o nounset + + # Debug section + exec 3>&1 + exec 4>&2 + + # Load Libraries + . /opt/bitnami/scripts/liblog.sh + . /opt/bitnami/scripts/libpostgresql.sh + + # Load PostgreSQL & repmgr environment variables + . /opt/bitnami/scripts/postgresql-env.sh + + # Process input parameters + MIN_DELAY_AFTER_POD_READY_FIRST_TIME=$1 + TMP_FIRST_READY_FILE_TS="/tmp/ts-first-ready.mark" + TMP_DELAY_APPLIED_FILE="/tmp/delay-applied.mark" + + DB_CHECK_RESULT=$(echo "SELECT 1" | postgresql_execute_print_output "$POSTGRESQL_DATABASE" "$POSTGRESQL_USERNAME" "$POSTGRESQL_PASSWORD" "-h 127.0.0.1 -tA" || echo "command failed") + if [[ "$DB_CHECK_RESULT" == "1" ]]; then + if [[ ! -f "$TMP_DELAY_APPLIED_FILE" ]]; then + # DB up, but initial readiness delay not applied + if [[ -f "$TMP_FIRST_READY_FILE_TS" ]]; then + # calculate delay from the first readiness success + FIRST_READY_TS=$(cat $TMP_FIRST_READY_FILE_TS) + CURRENT_DELAY_SECONDS=$(($EPOCHSECONDS - $FIRST_READY_TS)) + if (( $CURRENT_DELAY_SECONDS > $MIN_DELAY_AFTER_POD_READY_FIRST_TIME )); then + # minimal delay of the first readiness state passed - report success and mark delay as applied + touch "$TMP_DELAY_APPLIED_FILE" + else + # minimal delay of the first readiness state not reached yet - report failure + exit 1 + fi + else + # first ever readiness test success - store timestamp and report failure + echo $EPOCHSECONDS > $TMP_FIRST_READY_FILE_TS + exit 1 + fi + fi + else + # DB test failed - report failure + exit 1 + fi +--- +# Source: gitea/charts/redis-cluster/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-redis-cluster-default + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: redis-cluster + app.kubernetes.io/version: 7.2.5 +data: + redis-default.conf: |- + # Redis configuration file example. + # + # Note that in order to read the configuration file, Redis must be + # started with the file path as first argument: + # + # ./redis-server /path/to/redis.conf + + # Note on units: when memory size is needed, it is possible to specify + # it in the usual form of 1k 5GB 4M and so forth: + # + # 1k => 1000 bytes + # 1kb => 1024 bytes + # 1m => 1000000 bytes + # 1mb => 1024*1024 bytes + # 1g => 1000000000 bytes + # 1gb => 1024*1024*1024 bytes + # + # units are case insensitive so 1GB 1Gb 1gB are all the same. + + ################################## INCLUDES ################################### + + # Include one or more other config files here. This is useful if you + # have a standard template that goes to all Redis servers but also need + # to customize a few per-server settings. Include files can include + # other files, so use this wisely. + # + # Note that option "include" won't be rewritten by command "CONFIG REWRITE" + # from admin or Redis Sentinel. Since Redis always uses the last processed + # line as value of a configuration directive, you'd better put includes + # at the beginning of this file to avoid overwriting config change at runtime. + # + # If instead you are interested in using includes to override configuration + # options, it is better to use include as the last line. + # + # Included paths may contain wildcards. All files matching the wildcards will + # be included in alphabetical order. + # Note that if an include path contains a wildcards but no files match it when + # the server is started, the include statement will be ignored and no error will + # be emitted. It is safe, therefore, to include wildcard files from empty + # directories. + # + # include /path/to/local.conf + # include /path/to/other.conf + # include /path/to/fragments/*.conf + # + + ################################## MODULES ##################################### + + # Load modules at startup. If the server is not able to load modules + # it will abort. It is possible to use multiple loadmodule directives. + # + # loadmodule /path/to/my_module.so + # loadmodule /path/to/other_module.so + + ################################## NETWORK ##################################### + + # By default, if no "bind" configuration directive is specified, Redis listens + # for connections from all available network interfaces on the host machine. + # It is possible to listen to just one or multiple selected interfaces using + # the "bind" configuration directive, followed by one or more IP addresses. + # Each address can be prefixed by "-", which means that redis will not fail to + # start if the address is not available. Being not available only refers to + # addresses that does not correspond to any network interface. Addresses that + # are already in use will always fail, and unsupported protocols will always BE + # silently skipped. + # + # Examples: + # + # bind 192.168.1.100 10.0.0.1 # listens on two specific IPv4 addresses + # bind 127.0.0.1 ::1 # listens on loopback IPv4 and IPv6 + # bind * -::* # like the default, all available interfaces + # + # ~~~ WARNING ~~~ If the computer running Redis is directly exposed to the + # internet, binding to all the interfaces is dangerous and will expose the + # instance to everybody on the internet. So by default we uncomment the + # following bind directive, that will force Redis to listen only on the + # IPv4 and IPv6 (if available) loopback interface addresses (this means Redis + # will only be able to accept client connections from the same host that it is + # running on). + # + # IF YOU ARE SURE YOU WANT YOUR INSTANCE TO LISTEN TO ALL THE INTERFACES + # COMMENT OUT THE FOLLOWING LINE. + # + # You will also need to set a password unless you explicitly disable protected + # mode. + # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + bind 127.0.0.1 -::1 + + # By default, outgoing connections (from replica to master, from Sentinel to + # instances, cluster bus, etc.) are not bound to a specific local address. In + # most cases, this means the operating system will handle that based on routing + # and the interface through which the connection goes out. + # + # Using bind-source-addr it is possible to configure a specific address to bind + # to, which may also affect how the connection gets routed. + # + # Example: + # + # bind-source-addr 10.0.0.1 + + # Protected mode is a layer of security protection, in order to avoid that + # Redis instances left open on the internet are accessed and exploited. + # + # When protected mode is on and the default user has no password, the server + # only accepts local connections from the IPv4 address (127.0.0.1), IPv6 address + # (::1) or Unix domain sockets. + # + # By default protected mode is enabled. You should disable it only if + # you are sure you want clients from other hosts to connect to Redis + # even if no authentication is configured. + protected-mode yes + + # Redis uses default hardened security configuration directives to reduce the + # attack surface on innocent users. Therefore, several sensitive configuration + # directives are immutable, and some potentially-dangerous commands are blocked. + # + # Configuration directives that control files that Redis writes to (e.g., 'dir' + # and 'dbfilename') and that aren't usually modified during runtime + # are protected by making them immutable. + # + # Commands that can increase the attack surface of Redis and that aren't usually + # called by users are blocked by default. + # + # These can be exposed to either all connections or just local ones by setting + # each of the configs listed below to either of these values: + # + # no - Block for any connection (remain immutable) + # yes - Allow for any connection (no protection) + # local - Allow only for local connections. Ones originating from the + # IPv4 address (127.0.0.1), IPv6 address (::1) or Unix domain sockets. + # + # enable-protected-configs no + # enable-debug-command no + # enable-module-command no + + # Accept connections on the specified port, default is 6379 (IANA #815344). + # If port 0 is specified Redis will not listen on a TCP socket. + port 6379 + + # TCP listen() backlog. + # + # In high requests-per-second environments you need a high backlog in order + # to avoid slow clients connection issues. Note that the Linux kernel + # will silently truncate it to the value of /proc/sys/net/core/somaxconn so + # make sure to raise both the value of somaxconn and tcp_max_syn_backlog + # in order to get the desired effect. + tcp-backlog 511 + + # Unix socket. + # + # Specify the path for the Unix socket that will be used to listen for + # incoming connections. There is no default, so Redis will not listen + # on a unix socket when not specified. + # + # unixsocket /run/redis.sock + # unixsocketperm 700 + + # Close the connection after a client is idle for N seconds (0 to disable) + timeout 0 + + # TCP keepalive. + # + # If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence + # of communication. This is useful for two reasons: + # + # 1) Detect dead peers. + # 2) Force network equipment in the middle to consider the connection to be + # alive. + # + # On Linux, the specified value (in seconds) is the period used to send ACKs. + # Note that to close the connection the double of the time is needed. + # On other kernels the period depends on the kernel configuration. + # + # A reasonable value for this option is 300 seconds, which is the new + # Redis default starting with Redis 3.2.1. + tcp-keepalive 300 + + # Apply OS-specific mechanism to mark the listening socket with the specified + # ID, to support advanced routing and filtering capabilities. + # + # On Linux, the ID represents a connection mark. + # On FreeBSD, the ID represents a socket cookie ID. + # On OpenBSD, the ID represents a route table ID. + # + # The default value is 0, which implies no marking is required. + # socket-mark-id 0 + + ################################# TLS/SSL ##################################### + + # By default, TLS/SSL is disabled. To enable it, the "tls-port" configuration + # directive can be used to define TLS-listening ports. To enable TLS on the + # default port, use: + # + # port 0 + # tls-port 6379 + + # Configure a X.509 certificate and private key to use for authenticating the + # server to connected clients, masters or cluster peers. These files should be + # PEM formatted. + # + # tls-cert-file redis.crt + # tls-key-file redis.key + # + # If the key file is encrypted using a passphrase, it can be included here + # as well. + # + # tls-key-file-pass secret + + # Normally Redis uses the same certificate for both server functions (accepting + # connections) and client functions (replicating from a master, establishing + # cluster bus connections, etc.). + # + # Sometimes certificates are issued with attributes that designate them as + # client-only or server-only certificates. In that case it may be desired to use + # different certificates for incoming (server) and outgoing (client) + # connections. To do that, use the following directives: + # + # tls-client-cert-file client.crt + # tls-client-key-file client.key + # + # If the key file is encrypted using a passphrase, it can be included here + # as well. + # + # tls-client-key-file-pass secret + + # Configure a DH parameters file to enable Diffie-Hellman (DH) key exchange, + # required by older versions of OpenSSL (<3.0). Newer versions do not require + # this configuration and recommend against it. + # + # tls-dh-params-file redis.dh + + # Configure a CA certificate(s) bundle or directory to authenticate TLS/SSL + # clients and peers. Redis requires an explicit configuration of at least one + # of these, and will not implicitly use the system wide configuration. + # + # tls-ca-cert-file ca.crt + # tls-ca-cert-dir /etc/ssl/certs + + # By default, clients (including replica servers) on a TLS port are required + # to authenticate using valid client side certificates. + # + # If "no" is specified, client certificates are not required and not accepted. + # If "optional" is specified, client certificates are accepted and must be + # valid if provided, but are not required. + # + # tls-auth-clients no + # tls-auth-clients optional + + # By default, a Redis replica does not attempt to establish a TLS connection + # with its master. + # + # Use the following directive to enable TLS on replication links. + # + # tls-replication yes + + # By default, the Redis Cluster bus uses a plain TCP connection. To enable + # TLS for the bus protocol, use the following directive: + # + # tls-cluster yes + + # By default, only TLSv1.2 and TLSv1.3 are enabled and it is highly recommended + # that older formally deprecated versions are kept disabled to reduce the attack surface. + # You can explicitly specify TLS versions to support. + # Allowed values are case insensitive and include "TLSv1", "TLSv1.1", "TLSv1.2", + # "TLSv1.3" (OpenSSL >= 1.1.1) or any combination. + # To enable only TLSv1.2 and TLSv1.3, use: + # + # tls-protocols "TLSv1.2 TLSv1.3" + + # Configure allowed ciphers. See the ciphers(1ssl) manpage for more information + # about the syntax of this string. + # + # Note: this configuration applies only to <= TLSv1.2. + # + # tls-ciphers DEFAULT:!MEDIUM + + # Configure allowed TLSv1.3 ciphersuites. See the ciphers(1ssl) manpage for more + # information about the syntax of this string, and specifically for TLSv1.3 + # ciphersuites. + # + # tls-ciphersuites TLS_CHACHA20_POLY1305_SHA256 + + # When choosing a cipher, use the server's preference instead of the client + # preference. By default, the server follows the client's preference. + # + # tls-prefer-server-ciphers yes + + # By default, TLS session caching is enabled to allow faster and less expensive + # reconnections by clients that support it. Use the following directive to disable + # caching. + # + # tls-session-caching no + + # Change the default number of TLS sessions cached. A zero value sets the cache + # to unlimited size. The default size is 20480. + # + # tls-session-cache-size 5000 + + # Change the default timeout of cached TLS sessions. The default timeout is 300 + # seconds. + # + # tls-session-cache-timeout 60 + + ################################# GENERAL ##################################### + + # By default Redis does not run as a daemon. Use 'yes' if you need it. + # Note that Redis will write a pid file in /var/run/redis.pid when daemonized. + # When Redis is supervised by upstart or systemd, this parameter has no impact. + daemonize no + + # If you run Redis from upstart or systemd, Redis can interact with your + # supervision tree. Options: + # supervised no - no supervision interaction + # supervised upstart - signal upstart by putting Redis into SIGSTOP mode + # requires "expect stop" in your upstart job config + # supervised systemd - signal systemd by writing READY=1 to $NOTIFY_SOCKET + # on startup, and updating Redis status on a regular + # basis. + # supervised auto - detect upstart or systemd method based on + # UPSTART_JOB or NOTIFY_SOCKET environment variables + # Note: these supervision methods only signal "process is ready." + # They do not enable continuous pings back to your supervisor. + # + # The default is "no". To run under upstart/systemd, you can simply uncomment + # the line below: + # + # supervised auto + + # If a pid file is specified, Redis writes it where specified at startup + # and removes it at exit. + # + # When the server runs non daemonized, no pid file is created if none is + # specified in the configuration. When the server is daemonized, the pid file + # is used even if not specified, defaulting to "/var/run/redis.pid". + # + # Creating a pid file is best effort: if Redis is not able to create it + # nothing bad happens, the server will start and run normally. + # + # Note that on modern Linux systems "/run/redis.pid" is more conforming + # and should be used instead. + pidfile /opt/bitnami/redis/tmp/redis_6379.pid + + # Specify the server verbosity level. + # This can be one of: + # debug (a lot of information, useful for development/testing) + # verbose (many rarely useful info, but not a mess like the debug level) + # notice (moderately verbose, what you want in production probably) + # warning (only very important / critical messages are logged) + loglevel notice + + # Specify the log file name. Also the empty string can be used to force + # Redis to log on the standard output. Note that if you use standard + # output for logging but daemonize, logs will be sent to /dev/null + logfile "" + + # To enable logging to the system logger, just set 'syslog-enabled' to yes, + # and optionally update the other syslog parameters to suit your needs. + # syslog-enabled no + + # Specify the syslog identity. + # syslog-ident redis + + # Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7. + # syslog-facility local0 + + # To disable the built in crash log, which will possibly produce cleaner core + # dumps when they are needed, uncomment the following: + # + # crash-log-enabled no + + # To disable the fast memory check that's run as part of the crash log, which + # will possibly let redis terminate sooner, uncomment the following: + # + # crash-memcheck-enabled no + + # Set the number of databases. The default database is DB 0, you can select + # a different one on a per-connection basis using SELECT where + # dbid is a number between 0 and 'databases'-1 + databases 16 + + # By default Redis shows an ASCII art logo only when started to log to the + # standard output and if the standard output is a TTY and syslog logging is + # disabled. Basically this means that normally a logo is displayed only in + # interactive sessions. + # + # However it is possible to force the pre-4.0 behavior and always show a + # ASCII art logo in startup logs by setting the following option to yes. + always-show-logo yes + + # By default, Redis modifies the process title (as seen in 'top' and 'ps') to + # provide some runtime information. It is possible to disable this and leave + # the process name as executed by setting the following to no. + set-proc-title yes + + # When changing the process title, Redis uses the following template to construct + # the modified title. + # + # Template variables are specified in curly brackets. The following variables are + # supported: + # + # {title} Name of process as executed if parent, or type of child process. + # {listen-addr} Bind address or '*' followed by TCP or TLS port listening on, or + # Unix socket if only that's available. + # {server-mode} Special mode, i.e. "[sentinel]" or "[cluster]". + # {port} TCP port listening on, or 0. + # {tls-port} TLS port listening on, or 0. + # {unixsocket} Unix domain socket listening on, or "". + # {config-file} Name of configuration file used. + # + proc-title-template "{title} {listen-addr} {server-mode}" + + ################################ SNAPSHOTTING ################################ + + # Save the DB to disk. + # + # save [ ...] + # + # Redis will save the DB if the given number of seconds elapsed and it + # surpassed the given number of write operations against the DB. + # + # Snapshotting can be completely disabled with a single empty string argument + # as in following example: + # + # save "" + # + # Unless specified otherwise, by default Redis will save the DB: + # * After 3600 seconds (an hour) if at least 1 change was performed + # * After 300 seconds (5 minutes) if at least 100 changes were performed + # * After 60 seconds if at least 10000 changes were performed + # + # You can set these explicitly by uncommenting the following line. + # + # save 3600 1 300 100 60 10000 + save 900 1 300 10 60 10000 + # By default Redis will stop accepting writes if RDB snapshots are enabled + # (at least one save point) and the latest background save failed. + # This will make the user aware (in a hard way) that data is not persisting + # on disk properly, otherwise chances are that no one will notice and some + # disaster will happen. + # + # If the background saving process will start working again Redis will + # automatically allow writes again. + # + # However if you have setup your proper monitoring of the Redis server + # and persistence, you may want to disable this feature so that Redis will + # continue to work as usual even if there are problems with disk, + # permissions, and so forth. + stop-writes-on-bgsave-error yes + + # Compress string objects using LZF when dump .rdb databases? + # By default compression is enabled as it's almost always a win. + # If you want to save some CPU in the saving child set it to 'no' but + # the dataset will likely be bigger if you have compressible values or keys. + rdbcompression yes + + # Since version 5 of RDB a CRC64 checksum is placed at the end of the file. + # This makes the format more resistant to corruption but there is a performance + # hit to pay (around 10%) when saving and loading RDB files, so you can disable it + # for maximum performances. + # + # RDB files created with checksum disabled have a checksum of zero that will + # tell the loading code to skip the check. + rdbchecksum yes + + # Enables or disables full sanitization checks for ziplist and listpack etc when + # loading an RDB or RESTORE payload. This reduces the chances of a assertion or + # crash later on while processing commands. + # Options: + # no - Never perform full sanitization + # yes - Always perform full sanitization + # clients - Perform full sanitization only for user connections. + # Excludes: RDB files, RESTORE commands received from the master + # connection, and client connections which have the + # skip-sanitize-payload ACL flag. + # The default should be 'clients' but since it currently affects cluster + # resharding via MIGRATE, it is temporarily set to 'no' by default. + # + # sanitize-dump-payload no + + # The filename where to dump the DB + dbfilename dump.rdb + + # Remove RDB files used by replication in instances without persistence + # enabled. By default this option is disabled, however there are environments + # where for regulations or other security concerns, RDB files persisted on + # disk by masters in order to feed replicas, or stored on disk by replicas + # in order to load them for the initial synchronization, should be deleted + # ASAP. Note that this option ONLY WORKS in instances that have both AOF + # and RDB persistence disabled, otherwise is completely ignored. + # + # An alternative (and sometimes better) way to obtain the same effect is + # to use diskless replication on both master and replicas instances. However + # in the case of replicas, diskless is not always an option. + rdb-del-sync-files no + + # The working directory. + # + # The DB will be written inside this directory, with the filename specified + # above using the 'dbfilename' configuration directive. + # + # The Append Only File will also be created inside this directory. + # + # Note that you must specify a directory here, not a file name. + dir /bitnami/redis/data + + ################################# REPLICATION ################################# + + # Master-Replica replication. Use replicaof to make a Redis instance a copy of + # another Redis server. A few things to understand ASAP about Redis replication. + # + # +------------------+ +---------------+ + # | Master | ---> | Replica | + # | (receive writes) | | (exact copy) | + # +------------------+ +---------------+ + # + # 1) Redis replication is asynchronous, but you can configure a master to + # stop accepting writes if it appears to be not connected with at least + # a given number of replicas. + # 2) Redis replicas are able to perform a partial resynchronization with the + # master if the replication link is lost for a relatively small amount of + # time. You may want to configure the replication backlog size (see the next + # sections of this file) with a sensible value depending on your needs. + # 3) Replication is automatic and does not need user intervention. After a + # network partition replicas automatically try to reconnect to masters + # and resynchronize with them. + # + # replicaof + + # If the master is password protected (using the "requirepass" configuration + # directive below) it is possible to tell the replica to authenticate before + # starting the replication synchronization process, otherwise the master will + # refuse the replica request. + # + # masterauth + # + # However this is not enough if you are using Redis ACLs (for Redis version + # 6 or greater), and the default user is not capable of running the PSYNC + # command and/or other commands needed for replication. In this case it's + # better to configure a special user to use with replication, and specify the + # masteruser configuration as such: + # + # masteruser + # + # When masteruser is specified, the replica will authenticate against its + # master using the new AUTH form: AUTH . + + # When a replica loses its connection with the master, or when the replication + # is still in progress, the replica can act in two different ways: + # + # 1) if replica-serve-stale-data is set to 'yes' (the default) the replica will + # still reply to client requests, possibly with out of date data, or the + # data set may just be empty if this is the first synchronization. + # + # 2) If replica-serve-stale-data is set to 'no' the replica will reply with error + # "MASTERDOWN Link with MASTER is down and replica-serve-stale-data is set to 'no'" + # to all data access commands, excluding commands such as: + # INFO, REPLICAOF, AUTH, SHUTDOWN, REPLCONF, ROLE, CONFIG, SUBSCRIBE, + # UNSUBSCRIBE, PSUBSCRIBE, PUNSUBSCRIBE, PUBLISH, PUBSUB, COMMAND, POST, + # HOST and LATENCY. + # + replica-serve-stale-data yes + + # You can configure a replica instance to accept writes or not. Writing against + # a replica instance may be useful to store some ephemeral data (because data + # written on a replica will be easily deleted after resync with the master) but + # may also cause problems if clients are writing to it because of a + # misconfiguration. + # + # Since Redis 2.6 by default replicas are read-only. + # + # Note: read only replicas are not designed to be exposed to untrusted clients + # on the internet. It's just a protection layer against misuse of the instance. + # Still a read only replica exports by default all the administrative commands + # such as CONFIG, DEBUG, and so forth. To a limited extent you can improve + # security of read only replicas using 'rename-command' to shadow all the + # administrative / dangerous commands. + replica-read-only yes + + # Replication SYNC strategy: disk or socket. + # + # New replicas and reconnecting replicas that are not able to continue the + # replication process just receiving differences, need to do what is called a + # "full synchronization". An RDB file is transmitted from the master to the + # replicas. + # + # The transmission can happen in two different ways: + # + # 1) Disk-backed: The Redis master creates a new process that writes the RDB + # file on disk. Later the file is transferred by the parent + # process to the replicas incrementally. + # 2) Diskless: The Redis master creates a new process that directly writes the + # RDB file to replica sockets, without touching the disk at all. + # + # With disk-backed replication, while the RDB file is generated, more replicas + # can be queued and served with the RDB file as soon as the current child + # producing the RDB file finishes its work. With diskless replication instead + # once the transfer starts, new replicas arriving will be queued and a new + # transfer will start when the current one terminates. + # + # When diskless replication is used, the master waits a configurable amount of + # time (in seconds) before starting the transfer in the hope that multiple + # replicas will arrive and the transfer can be parallelized. + # + # With slow disks and fast (large bandwidth) networks, diskless replication + # works better. + repl-diskless-sync no + + # When diskless replication is enabled, it is possible to configure the delay + # the server waits in order to spawn the child that transfers the RDB via socket + # to the replicas. + # + # This is important since once the transfer starts, it is not possible to serve + # new replicas arriving, that will be queued for the next RDB transfer, so the + # server waits a delay in order to let more replicas arrive. + # + # The delay is specified in seconds, and by default is 5 seconds. To disable + # it entirely just set it to 0 seconds and the transfer will start ASAP. + repl-diskless-sync-delay 5 + + # When diskless replication is enabled with a delay, it is possible to let + # the replication start before the maximum delay is reached if the maximum + # number of replicas expected have connected. Default of 0 means that the + # maximum is not defined and Redis will wait the full delay. + repl-diskless-sync-max-replicas 0 + + # ----------------------------------------------------------------------------- + # WARNING: RDB diskless load is experimental. Since in this setup the replica + # does not immediately store an RDB on disk, it may cause data loss during + # failovers. RDB diskless load + Redis modules not handling I/O reads may also + # cause Redis to abort in case of I/O errors during the initial synchronization + # stage with the master. Use only if you know what you are doing. + # ----------------------------------------------------------------------------- + # + # Replica can load the RDB it reads from the replication link directly from the + # socket, or store the RDB to a file and read that file after it was completely + # received from the master. + # + # In many cases the disk is slower than the network, and storing and loading + # the RDB file may increase replication time (and even increase the master's + # Copy on Write memory and replica buffers). + # However, parsing the RDB file directly from the socket may mean that we have + # to flush the contents of the current database before the full rdb was + # received. For this reason we have the following options: + # + # "disabled" - Don't use diskless load (store the rdb file to the disk first) + # "on-empty-db" - Use diskless load only when it is completely safe. + # "swapdb" - Keep current db contents in RAM while parsing the data directly + # from the socket. Replicas in this mode can keep serving current + # data set while replication is in progress, except for cases where + # they can't recognize master as having a data set from same + # replication history. + # Note that this requires sufficient memory, if you don't have it, + # you risk an OOM kill. + repl-diskless-load disabled + + # Master send PINGs to its replicas in a predefined interval. It's possible to + # change this interval with the repl_ping_replica_period option. The default + # value is 10 seconds. + # + # repl-ping-replica-period 10 + + # The following option sets the replication timeout for: + # + # 1) Bulk transfer I/O during SYNC, from the point of view of replica. + # 2) Master timeout from the point of view of replicas (data, pings). + # 3) Replica timeout from the point of view of masters (REPLCONF ACK pings). + # + # It is important to make sure that this value is greater than the value + # specified for repl-ping-replica-period otherwise a timeout will be detected + # every time there is low traffic between the master and the replica. The default + # value is 60 seconds. + # + # repl-timeout 60 + + # Disable TCP_NODELAY on the replica socket after SYNC? + # + # If you select "yes" Redis will use a smaller number of TCP packets and + # less bandwidth to send data to replicas. But this can add a delay for + # the data to appear on the replica side, up to 40 milliseconds with + # Linux kernels using a default configuration. + # + # If you select "no" the delay for data to appear on the replica side will + # be reduced but more bandwidth will be used for replication. + # + # By default we optimize for low latency, but in very high traffic conditions + # or when the master and replicas are many hops away, turning this to "yes" may + # be a good idea. + repl-disable-tcp-nodelay no + + # Set the replication backlog size. The backlog is a buffer that accumulates + # replica data when replicas are disconnected for some time, so that when a + # replica wants to reconnect again, often a full resync is not needed, but a + # partial resync is enough, just passing the portion of data the replica + # missed while disconnected. + # + # The bigger the replication backlog, the longer the replica can endure the + # disconnect and later be able to perform a partial resynchronization. + # + # The backlog is only allocated if there is at least one replica connected. + # + # repl-backlog-size 1mb + + # After a master has no connected replicas for some time, the backlog will be + # freed. The following option configures the amount of seconds that need to + # elapse, starting from the time the last replica disconnected, for the backlog + # buffer to be freed. + # + # Note that replicas never free the backlog for timeout, since they may be + # promoted to masters later, and should be able to correctly "partially + # resynchronize" with other replicas: hence they should always accumulate backlog. + # + # A value of 0 means to never release the backlog. + # + # repl-backlog-ttl 3600 + + # The replica priority is an integer number published by Redis in the INFO + # output. It is used by Redis Sentinel in order to select a replica to promote + # into a master if the master is no longer working correctly. + # + # A replica with a low priority number is considered better for promotion, so + # for instance if there are three replicas with priority 10, 100, 25 Sentinel + # will pick the one with priority 10, that is the lowest. + # + # However a special priority of 0 marks the replica as not able to perform the + # role of master, so a replica with priority of 0 will never be selected by + # Redis Sentinel for promotion. + # + # By default the priority is 100. + replica-priority 100 + + # The propagation error behavior controls how Redis will behave when it is + # unable to handle a command being processed in the replication stream from a master + # or processed while reading from an AOF file. Errors that occur during propagation + # are unexpected, and can cause data inconsistency. However, there are edge cases + # in earlier versions of Redis where it was possible for the server to replicate or persist + # commands that would fail on future versions. For this reason the default behavior + # is to ignore such errors and continue processing commands. + # + # If an application wants to ensure there is no data divergence, this configuration + # should be set to 'panic' instead. The value can also be set to 'panic-on-replicas' + # to only panic when a replica encounters an error on the replication stream. One of + # these two panic values will become the default value in the future once there are + # sufficient safety mechanisms in place to prevent false positive crashes. + # + # propagation-error-behavior ignore + + # Replica ignore disk write errors controls the behavior of a replica when it is + # unable to persist a write command received from its master to disk. By default, + # this configuration is set to 'no' and will crash the replica in this condition. + # It is not recommended to change this default, however in order to be compatible + # with older versions of Redis this config can be toggled to 'yes' which will just + # log a warning and execute the write command it got from the master. + # + # replica-ignore-disk-write-errors no + + # ----------------------------------------------------------------------------- + # By default, Redis Sentinel includes all replicas in its reports. A replica + # can be excluded from Redis Sentinel's announcements. An unannounced replica + # will be ignored by the 'sentinel replicas ' command and won't be + # exposed to Redis Sentinel's clients. + # + # This option does not change the behavior of replica-priority. Even with + # replica-announced set to 'no', the replica can be promoted to master. To + # prevent this behavior, set replica-priority to 0. + # + # replica-announced yes + + # It is possible for a master to stop accepting writes if there are less than + # N replicas connected, having a lag less or equal than M seconds. + # + # The N replicas need to be in "online" state. + # + # The lag in seconds, that must be <= the specified value, is calculated from + # the last ping received from the replica, that is usually sent every second. + # + # This option does not GUARANTEE that N replicas will accept the write, but + # will limit the window of exposure for lost writes in case not enough replicas + # are available, to the specified number of seconds. + # + # For example to require at least 3 replicas with a lag <= 10 seconds use: + # + # min-replicas-to-write 3 + # min-replicas-max-lag 10 + # + # Setting one or the other to 0 disables the feature. + # + # By default min-replicas-to-write is set to 0 (feature disabled) and + # min-replicas-max-lag is set to 10. + + # A Redis master is able to list the address and port of the attached + # replicas in different ways. For example the "INFO replication" section + # offers this information, which is used, among other tools, by + # Redis Sentinel in order to discover replica instances. + # Another place where this info is available is in the output of the + # "ROLE" command of a master. + # + # The listed IP address and port normally reported by a replica is + # obtained in the following way: + # + # IP: The address is auto detected by checking the peer address + # of the socket used by the replica to connect with the master. + # + # Port: The port is communicated by the replica during the replication + # handshake, and is normally the port that the replica is using to + # listen for connections. + # + # However when port forwarding or Network Address Translation (NAT) is + # used, the replica may actually be reachable via different IP and port + # pairs. The following two options can be used by a replica in order to + # report to its master a specific set of IP and port, so that both INFO + # and ROLE will report those values. + # + # There is no need to use both the options if you need to override just + # the port or the IP address. + # + # replica-announce-ip 5.5.5.5 + # replica-announce-port 1234 + + ############################### KEYS TRACKING ################################# + + # Redis implements server assisted support for client side caching of values. + # This is implemented using an invalidation table that remembers, using + # a radix key indexed by key name, what clients have which keys. In turn + # this is used in order to send invalidation messages to clients. Please + # check this page to understand more about the feature: + # + # https://redis.io/topics/client-side-caching + # + # When tracking is enabled for a client, all the read only queries are assumed + # to be cached: this will force Redis to store information in the invalidation + # table. When keys are modified, such information is flushed away, and + # invalidation messages are sent to the clients. However if the workload is + # heavily dominated by reads, Redis could use more and more memory in order + # to track the keys fetched by many clients. + # + # For this reason it is possible to configure a maximum fill value for the + # invalidation table. By default it is set to 1M of keys, and once this limit + # is reached, Redis will start to evict keys in the invalidation table + # even if they were not modified, just to reclaim memory: this will in turn + # force the clients to invalidate the cached values. Basically the table + # maximum size is a trade off between the memory you want to spend server + # side to track information about who cached what, and the ability of clients + # to retain cached objects in memory. + # + # If you set the value to 0, it means there are no limits, and Redis will + # retain as many keys as needed in the invalidation table. + # In the "stats" INFO section, you can find information about the number of + # keys in the invalidation table at every given moment. + # + # Note: when key tracking is used in broadcasting mode, no memory is used + # in the server side so this setting is useless. + # + # tracking-table-max-keys 1000000 + + ################################## SECURITY ################################### + + # Warning: since Redis is pretty fast, an outside user can try up to + # 1 million passwords per second against a modern box. This means that you + # should use very strong passwords, otherwise they will be very easy to break. + # Note that because the password is really a shared secret between the client + # and the server, and should not be memorized by any human, the password + # can be easily a long string from /dev/urandom or whatever, so by using a + # long and unguessable password no brute force attack will be possible. + + # Redis ACL users are defined in the following format: + # + # user ... acl rules ... + # + # For example: + # + # user worker +@list +@connection ~jobs:* on >ffa9203c493aa99 + # + # The special username "default" is used for new connections. If this user + # has the "nopass" rule, then new connections will be immediately authenticated + # as the "default" user without the need of any password provided via the + # AUTH command. Otherwise if the "default" user is not flagged with "nopass" + # the connections will start in not authenticated state, and will require + # AUTH (or the HELLO command AUTH option) in order to be authenticated and + # start to work. + # + # The ACL rules that describe what a user can do are the following: + # + # on Enable the user: it is possible to authenticate as this user. + # off Disable the user: it's no longer possible to authenticate + # with this user, however the already authenticated connections + # will still work. + # skip-sanitize-payload RESTORE dump-payload sanitization is skipped. + # sanitize-payload RESTORE dump-payload is sanitized (default). + # + Allow the execution of that command. + # May be used with `|` for allowing subcommands (e.g "+config|get") + # - Disallow the execution of that command. + # May be used with `|` for blocking subcommands (e.g "-config|set") + # +@ Allow the execution of all the commands in such category + # with valid categories are like @admin, @set, @sortedset, ... + # and so forth, see the full list in the server.c file where + # the Redis command table is described and defined. + # The special category @all means all the commands, but currently + # present in the server, and that will be loaded in the future + # via modules. + # +|first-arg Allow a specific first argument of an otherwise + # disabled command. It is only supported on commands with + # no sub-commands, and is not allowed as negative form + # like -SELECT|1, only additive starting with "+". This + # feature is deprecated and may be removed in the future. + # allcommands Alias for +@all. Note that it implies the ability to execute + # all the future commands loaded via the modules system. + # nocommands Alias for -@all. + # ~ Add a pattern of keys that can be mentioned as part of + # commands. For instance ~* allows all the keys. The pattern + # is a glob-style pattern like the one of KEYS. + # It is possible to specify multiple patterns. + # %R~ Add key read pattern that specifies which keys can be read + # from. + # %W~ Add key write pattern that specifies which keys can be + # written to. + # allkeys Alias for ~* + # resetkeys Flush the list of allowed keys patterns. + # & Add a glob-style pattern of Pub/Sub channels that can be + # accessed by the user. It is possible to specify multiple channel + # patterns. + # allchannels Alias for &* + # resetchannels Flush the list of allowed channel patterns. + # > Add this password to the list of valid password for the user. + # For example >mypass will add "mypass" to the list. + # This directive clears the "nopass" flag (see later). + # < Remove this password from the list of valid passwords. + # nopass All the set passwords of the user are removed, and the user + # is flagged as requiring no password: it means that every + # password will work against this user. If this directive is + # used for the default user, every new connection will be + # immediately authenticated with the default user without + # any explicit AUTH command required. Note that the "resetpass" + # directive will clear this condition. + # resetpass Flush the list of allowed passwords. Moreover removes the + # "nopass" status. After "resetpass" the user has no associated + # passwords and there is no way to authenticate without adding + # some password (or setting it as "nopass" later). + # reset Performs the following actions: resetpass, resetkeys, off, + # -@all. The user returns to the same state it has immediately + # after its creation. + # () Create a new selector with the options specified within the + # parentheses and attach it to the user. Each option should be + # space separated. The first character must be ( and the last + # character must be ). + # clearselectors Remove all of the currently attached selectors. + # Note this does not change the "root" user permissions, + # which are the permissions directly applied onto the + # user (outside the parentheses). + # + # ACL rules can be specified in any order: for instance you can start with + # passwords, then flags, or key patterns. However note that the additive + # and subtractive rules will CHANGE MEANING depending on the ordering. + # For instance see the following example: + # + # user alice on +@all -DEBUG ~* >somepassword + # + # This will allow "alice" to use all the commands with the exception of the + # DEBUG command, since +@all added all the commands to the set of the commands + # alice can use, and later DEBUG was removed. However if we invert the order + # of two ACL rules the result will be different: + # + # user alice on -DEBUG +@all ~* >somepassword + # + # Now DEBUG was removed when alice had yet no commands in the set of allowed + # commands, later all the commands are added, so the user will be able to + # execute everything. + # + # Basically ACL rules are processed left-to-right. + # + # The following is a list of command categories and their meanings: + # * keyspace - Writing or reading from keys, databases, or their metadata + # in a type agnostic way. Includes DEL, RESTORE, DUMP, RENAME, EXISTS, DBSIZE, + # KEYS, EXPIRE, TTL, FLUSHALL, etc. Commands that may modify the keyspace, + # key or metadata will also have `write` category. Commands that only read + # the keyspace, key or metadata will have the `read` category. + # * read - Reading from keys (values or metadata). Note that commands that don't + # interact with keys, will not have either `read` or `write`. + # * write - Writing to keys (values or metadata) + # * admin - Administrative commands. Normal applications will never need to use + # these. Includes REPLICAOF, CONFIG, DEBUG, SAVE, MONITOR, ACL, SHUTDOWN, etc. + # * dangerous - Potentially dangerous (each should be considered with care for + # various reasons). This includes FLUSHALL, MIGRATE, RESTORE, SORT, KEYS, + # CLIENT, DEBUG, INFO, CONFIG, SAVE, REPLICAOF, etc. + # * connection - Commands affecting the connection or other connections. + # This includes AUTH, SELECT, COMMAND, CLIENT, ECHO, PING, etc. + # * blocking - Potentially blocking the connection until released by another + # command. + # * fast - Fast O(1) commands. May loop on the number of arguments, but not the + # number of elements in the key. + # * slow - All commands that are not Fast. + # * pubsub - PUBLISH / SUBSCRIBE related + # * transaction - WATCH / MULTI / EXEC related commands. + # * scripting - Scripting related. + # * set - Data type: sets related. + # * sortedset - Data type: zsets related. + # * list - Data type: lists related. + # * hash - Data type: hashes related. + # * string - Data type: strings related. + # * bitmap - Data type: bitmaps related. + # * hyperloglog - Data type: hyperloglog related. + # * geo - Data type: geo related. + # * stream - Data type: streams related. + # + # For more information about ACL configuration please refer to + # the Redis web site at https://redis.io/topics/acl + + # ACL LOG + # + # The ACL Log tracks failed commands and authentication events associated + # with ACLs. The ACL Log is useful to troubleshoot failed commands blocked + # by ACLs. The ACL Log is stored in memory. You can reclaim memory with + # ACL LOG RESET. Define the maximum entry length of the ACL Log below. + acllog-max-len 128 + + # Using an external ACL file + # + # Instead of configuring users here in this file, it is possible to use + # a stand-alone file just listing users. The two methods cannot be mixed: + # if you configure users here and at the same time you activate the external + # ACL file, the server will refuse to start. + # + # The format of the external ACL user file is exactly the same as the + # format that is used inside redis.conf to describe users. + # + # aclfile /etc/redis/users.acl + + # IMPORTANT NOTE: starting with Redis 6 "requirepass" is just a compatibility + # layer on top of the new ACL system. The option effect will be just setting + # the password for the default user. Clients will still authenticate using + # AUTH as usually, or more explicitly with AUTH default + # if they follow the new protocol: both will work. + # + # The requirepass is not compatible with aclfile option and the ACL LOAD + # command, these will cause requirepass to be ignored. + # + # requirepass foobared + + # New users are initialized with restrictive permissions by default, via the + # equivalent of this ACL rule 'off resetkeys -@all'. Starting with Redis 6.2, it + # is possible to manage access to Pub/Sub channels with ACL rules as well. The + # default Pub/Sub channels permission if new users is controlled by the + # acl-pubsub-default configuration directive, which accepts one of these values: + # + # allchannels: grants access to all Pub/Sub channels + # resetchannels: revokes access to all Pub/Sub channels + # + # From Redis 7.0, acl-pubsub-default defaults to 'resetchannels' permission. + # + # acl-pubsub-default resetchannels + + # Command renaming (DEPRECATED). + # + # ------------------------------------------------------------------------ + # WARNING: avoid using this option if possible. Instead use ACLs to remove + # commands from the default user, and put them only in some admin user you + # create for administrative purposes. + # ------------------------------------------------------------------------ + # + # It is possible to change the name of dangerous commands in a shared + # environment. For instance the CONFIG command may be renamed into something + # hard to guess so that it will still be available for internal-use tools + # but not available for general clients. + # + # Example: + # + # rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52 + # + # It is also possible to completely kill a command by renaming it into + # an empty string: + # + # rename-command CONFIG "" + # + # Please note that changing the name of commands that are logged into the + # AOF file or transmitted to replicas may cause problems. + + ################################### CLIENTS #################################### + + # Set the max number of connected clients at the same time. By default + # this limit is set to 10000 clients, however if the Redis server is not + # able to configure the process file limit to allow for the specified limit + # the max number of allowed clients is set to the current file limit + # minus 32 (as Redis reserves a few file descriptors for internal uses). + # + # Once the limit is reached Redis will close all the new connections sending + # an error 'max number of clients reached'. + # + # IMPORTANT: When Redis Cluster is used, the max number of connections is also + # shared with the cluster bus: every node in the cluster will use two + # connections, one incoming and another outgoing. It is important to size the + # limit accordingly in case of very large clusters. + # + # maxclients 10000 + + ############################## MEMORY MANAGEMENT ################################ + + # Set a memory usage limit to the specified amount of bytes. + # When the memory limit is reached Redis will try to remove keys + # according to the eviction policy selected (see maxmemory-policy). + # + # If Redis can't remove keys according to the policy, or if the policy is + # set to 'noeviction', Redis will start to reply with errors to commands + # that would use more memory, like SET, LPUSH, and so on, and will continue + # to reply to read-only commands like GET. + # + # This option is usually useful when using Redis as an LRU or LFU cache, or to + # set a hard memory limit for an instance (using the 'noeviction' policy). + # + # WARNING: If you have replicas attached to an instance with maxmemory on, + # the size of the output buffers needed to feed the replicas are subtracted + # from the used memory count, so that network problems / resyncs will + # not trigger a loop where keys are evicted, and in turn the output + # buffer of replicas is full with DELs of keys evicted triggering the deletion + # of more keys, and so forth until the database is completely emptied. + # + # In short... if you have replicas attached it is suggested that you set a lower + # limit for maxmemory so that there is some free RAM on the system for replica + # output buffers (but this is not needed if the policy is 'noeviction'). + # + # maxmemory + + # MAXMEMORY POLICY: how Redis will select what to remove when maxmemory + # is reached. You can select one from the following behaviors: + # + # volatile-lru -> Evict using approximated LRU, only keys with an expire set. + # allkeys-lru -> Evict any key using approximated LRU. + # volatile-lfu -> Evict using approximated LFU, only keys with an expire set. + # allkeys-lfu -> Evict any key using approximated LFU. + # volatile-random -> Remove a random key having an expire set. + # allkeys-random -> Remove a random key, any key. + # volatile-ttl -> Remove the key with the nearest expire time (minor TTL) + # noeviction -> Don't evict anything, just return an error on write operations. + # + # LRU means Least Recently Used + # LFU means Least Frequently Used + # + # Both LRU, LFU and volatile-ttl are implemented using approximated + # randomized algorithms. + # + # Note: with any of the above policies, when there are no suitable keys for + # eviction, Redis will return an error on write operations that require + # more memory. These are usually commands that create new keys, add data or + # modify existing keys. A few examples are: SET, INCR, HSET, LPUSH, SUNIONSTORE, + # SORT (due to the STORE argument), and EXEC (if the transaction includes any + # command that requires memory). + # + # The default is: + # + # maxmemory-policy noeviction + + # LRU, LFU and minimal TTL algorithms are not precise algorithms but approximated + # algorithms (in order to save memory), so you can tune it for speed or + # accuracy. By default Redis will check five keys and pick the one that was + # used least recently, you can change the sample size using the following + # configuration directive. + # + # The default of 5 produces good enough results. 10 Approximates very closely + # true LRU but costs more CPU. 3 is faster but not very accurate. + # + # maxmemory-samples 5 + + # Eviction processing is designed to function well with the default setting. + # If there is an unusually large amount of write traffic, this value may need to + # be increased. Decreasing this value may reduce latency at the risk of + # eviction processing effectiveness + # 0 = minimum latency, 10 = default, 100 = process without regard to latency + # + # maxmemory-eviction-tenacity 10 + + # Starting from Redis 5, by default a replica will ignore its maxmemory setting + # (unless it is promoted to master after a failover or manually). It means + # that the eviction of keys will be just handled by the master, sending the + # DEL commands to the replica as keys evict in the master side. + # + # This behavior ensures that masters and replicas stay consistent, and is usually + # what you want, however if your replica is writable, or you want the replica + # to have a different memory setting, and you are sure all the writes performed + # to the replica are idempotent, then you may change this default (but be sure + # to understand what you are doing). + # + # Note that since the replica by default does not evict, it may end using more + # memory than the one set via maxmemory (there are certain buffers that may + # be larger on the replica, or data structures may sometimes take more memory + # and so forth). So make sure you monitor your replicas and make sure they + # have enough memory to never hit a real out-of-memory condition before the + # master hits the configured maxmemory setting. + # + # replica-ignore-maxmemory yes + + # Redis reclaims expired keys in two ways: upon access when those keys are + # found to be expired, and also in background, in what is called the + # "active expire key". The key space is slowly and interactively scanned + # looking for expired keys to reclaim, so that it is possible to free memory + # of keys that are expired and will never be accessed again in a short time. + # + # The default effort of the expire cycle will try to avoid having more than + # ten percent of expired keys still in memory, and will try to avoid consuming + # more than 25% of total memory and to add latency to the system. However + # it is possible to increase the expire "effort" that is normally set to + # "1", to a greater value, up to the value "10". At its maximum value the + # system will use more CPU, longer cycles (and technically may introduce + # more latency), and will tolerate less already expired keys still present + # in the system. It's a tradeoff between memory, CPU and latency. + # + # active-expire-effort 1 + + ############################# LAZY FREEING #################################### + + # Redis has two primitives to delete keys. One is called DEL and is a blocking + # deletion of the object. It means that the server stops processing new commands + # in order to reclaim all the memory associated with an object in a synchronous + # way. If the key deleted is associated with a small object, the time needed + # in order to execute the DEL command is very small and comparable to most other + # O(1) or O(log_N) commands in Redis. However if the key is associated with an + # aggregated value containing millions of elements, the server can block for + # a long time (even seconds) in order to complete the operation. + # + # For the above reasons Redis also offers non blocking deletion primitives + # such as UNLINK (non blocking DEL) and the ASYNC option of FLUSHALL and + # FLUSHDB commands, in order to reclaim memory in background. Those commands + # are executed in constant time. Another thread will incrementally free the + # object in the background as fast as possible. + # + # DEL, UNLINK and ASYNC option of FLUSHALL and FLUSHDB are user-controlled. + # It's up to the design of the application to understand when it is a good + # idea to use one or the other. However the Redis server sometimes has to + # delete keys or flush the whole database as a side effect of other operations. + # Specifically Redis deletes objects independently of a user call in the + # following scenarios: + # + # 1) On eviction, because of the maxmemory and maxmemory policy configurations, + # in order to make room for new data, without going over the specified + # memory limit. + # 2) Because of expire: when a key with an associated time to live (see the + # EXPIRE command) must be deleted from memory. + # 3) Because of a side effect of a command that stores data on a key that may + # already exist. For example the RENAME command may delete the old key + # content when it is replaced with another one. Similarly SUNIONSTORE + # or SORT with STORE option may delete existing keys. The SET command + # itself removes any old content of the specified key in order to replace + # it with the specified string. + # 4) During replication, when a replica performs a full resynchronization with + # its master, the content of the whole database is removed in order to + # load the RDB file just transferred. + # + # In all the above cases the default is to delete objects in a blocking way, + # like if DEL was called. However you can configure each case specifically + # in order to instead release memory in a non-blocking way like if UNLINK + # was called, using the following configuration directives. + + lazyfree-lazy-eviction no + lazyfree-lazy-expire no + lazyfree-lazy-server-del no + replica-lazy-flush no + + # It is also possible, for the case when to replace the user code DEL calls + # with UNLINK calls is not easy, to modify the default behavior of the DEL + # command to act exactly like UNLINK, using the following configuration + # directive: + + lazyfree-lazy-user-del no + + # FLUSHDB, FLUSHALL, SCRIPT FLUSH and FUNCTION FLUSH support both asynchronous and synchronous + # deletion, which can be controlled by passing the [SYNC|ASYNC] flags into the + # commands. When neither flag is passed, this directive will be used to determine + # if the data should be deleted asynchronously. + + lazyfree-lazy-user-flush no + + ################################ THREADED I/O ################################# + + # Redis is mostly single threaded, however there are certain threaded + # operations such as UNLINK, slow I/O accesses and other things that are + # performed on side threads. + # + # Now it is also possible to handle Redis clients socket reads and writes + # in different I/O threads. Since especially writing is so slow, normally + # Redis users use pipelining in order to speed up the Redis performances per + # core, and spawn multiple instances in order to scale more. Using I/O + # threads it is possible to easily speedup two times Redis without resorting + # to pipelining nor sharding of the instance. + # + # By default threading is disabled, we suggest enabling it only in machines + # that have at least 4 or more cores, leaving at least one spare core. + # Using more than 8 threads is unlikely to help much. We also recommend using + # threaded I/O only if you actually have performance problems, with Redis + # instances being able to use a quite big percentage of CPU time, otherwise + # there is no point in using this feature. + # + # So for instance if you have a four cores boxes, try to use 2 or 3 I/O + # threads, if you have a 8 cores, try to use 6 threads. In order to + # enable I/O threads use the following configuration directive: + # + # io-threads 4 + # + # Setting io-threads to 1 will just use the main thread as usual. + # When I/O threads are enabled, we only use threads for writes, that is + # to thread the write(2) syscall and transfer the client buffers to the + # socket. However it is also possible to enable threading of reads and + # protocol parsing using the following configuration directive, by setting + # it to yes: + # + # io-threads-do-reads no + # + # Usually threading reads doesn't help much. + # + # NOTE 1: This configuration directive cannot be changed at runtime via + # CONFIG SET. Also, this feature currently does not work when SSL is + # enabled. + # + # NOTE 2: If you want to test the Redis speedup using redis-benchmark, make + # sure you also run the benchmark itself in threaded mode, using the + # --threads option to match the number of Redis threads, otherwise you'll not + # be able to notice the improvements. + + ############################ KERNEL OOM CONTROL ############################## + + # On Linux, it is possible to hint the kernel OOM killer on what processes + # should be killed first when out of memory. + # + # Enabling this feature makes Redis actively control the oom_score_adj value + # for all its processes, depending on their role. The default scores will + # attempt to have background child processes killed before all others, and + # replicas killed before masters. + # + # Redis supports these options: + # + # no: Don't make changes to oom-score-adj (default). + # yes: Alias to "relative" see below. + # absolute: Values in oom-score-adj-values are written as is to the kernel. + # relative: Values are used relative to the initial value of oom_score_adj when + # the server starts and are then clamped to a range of -1000 to 1000. + # Because typically the initial value is 0, they will often match the + # absolute values. + oom-score-adj no + + # When oom-score-adj is used, this directive controls the specific values used + # for master, replica and background child processes. Values range -2000 to + # 2000 (higher means more likely to be killed). + # + # Unprivileged processes (not root, and without CAP_SYS_RESOURCE capabilities) + # can freely increase their value, but not decrease it below its initial + # settings. This means that setting oom-score-adj to "relative" and setting the + # oom-score-adj-values to positive values will always succeed. + oom-score-adj-values 0 200 800 + + + #################### KERNEL transparent hugepage CONTROL ###################### + + # Usually the kernel Transparent Huge Pages control is set to "madvise" or + # or "never" by default (/sys/kernel/mm/transparent_hugepage/enabled), in which + # case this config has no effect. On systems in which it is set to "always", + # redis will attempt to disable it specifically for the redis process in order + # to avoid latency problems specifically with fork(2) and CoW. + # If for some reason you prefer to keep it enabled, you can set this config to + # "no" and the kernel global to "always". + + disable-thp yes + + ############################## APPEND ONLY MODE ############################### + + # By default Redis asynchronously dumps the dataset on disk. This mode is + # good enough in many applications, but an issue with the Redis process or + # a power outage may result into a few minutes of writes lost (depending on + # the configured save points). + # + # The Append Only File is an alternative persistence mode that provides + # much better durability. For instance using the default data fsync policy + # (see later in the config file) Redis can lose just one second of writes in a + # dramatic event like a server power outage, or a single write if something + # wrong with the Redis process itself happens, but the operating system is + # still running correctly. + # + # AOF and RDB persistence can be enabled at the same time without problems. + # If the AOF is enabled on startup Redis will load the AOF, that is the file + # with the better durability guarantees. + # + # Please check https://redis.io/topics/persistence for more information. + + appendonly no + + # The base name of the append only file. + # + # Redis 7 and newer use a set of append-only files to persist the dataset + # and changes applied to it. There are two basic types of files in use: + # + # - Base files, which are a snapshot representing the complete state of the + # dataset at the time the file was created. Base files can be either in + # the form of RDB (binary serialized) or AOF (textual commands). + # - Incremental files, which contain additional commands that were applied + # to the dataset following the previous file. + # + # In addition, manifest files are used to track the files and the order in + # which they were created and should be applied. + # + # Append-only file names are created by Redis following a specific pattern. + # The file name's prefix is based on the 'appendfilename' configuration + # parameter, followed by additional information about the sequence and type. + # + # For example, if appendfilename is set to appendonly.aof, the following file + # names could be derived: + # + # - appendonly.aof.1.base.rdb as a base file. + # - appendonly.aof.1.incr.aof, appendonly.aof.2.incr.aof as incremental files. + # - appendonly.aof.manifest as a manifest file. + + appendfilename "appendonly.aof" + + # For convenience, Redis stores all persistent append-only files in a dedicated + # directory. The name of the directory is determined by the appenddirname + # configuration parameter. + + appenddirname "appendonlydir" + + # The fsync() call tells the Operating System to actually write data on disk + # instead of waiting for more data in the output buffer. Some OS will really flush + # data on disk, some other OS will just try to do it ASAP. + # + # Redis supports three different modes: + # + # no: don't fsync, just let the OS flush the data when it wants. Faster. + # always: fsync after every write to the append only log. Slow, Safest. + # everysec: fsync only one time every second. Compromise. + # + # The default is "everysec", as that's usually the right compromise between + # speed and data safety. It's up to you to understand if you can relax this to + # "no" that will let the operating system flush the output buffer when + # it wants, for better performances (but if you can live with the idea of + # some data loss consider the default persistence mode that's snapshotting), + # or on the contrary, use "always" that's very slow but a bit safer than + # everysec. + # + # More details please check the following article: + # http://antirez.com/post/redis-persistence-demystified.html + # + # If unsure, use "everysec". + + # appendfsync always + appendfsync everysec + # appendfsync no + + # When the AOF fsync policy is set to always or everysec, and a background + # saving process (a background save or AOF log background rewriting) is + # performing a lot of I/O against the disk, in some Linux configurations + # Redis may block too long on the fsync() call. Note that there is no fix for + # this currently, as even performing fsync in a different thread will block + # our synchronous write(2) call. + # + # In order to mitigate this problem it's possible to use the following option + # that will prevent fsync() from being called in the main process while a + # BGSAVE or BGREWRITEAOF is in progress. + # + # This means that while another child is saving, the durability of Redis is + # the same as "appendfsync no". In practical terms, this means that it is + # possible to lose up to 30 seconds of log in the worst scenario (with the + # default Linux settings). + # + # If you have latency problems turn this to "yes". Otherwise leave it as + # "no" that is the safest pick from the point of view of durability. + + no-appendfsync-on-rewrite no + + # Automatic rewrite of the append only file. + # Redis is able to automatically rewrite the log file implicitly calling + # BGREWRITEAOF when the AOF log size grows by the specified percentage. + # + # This is how it works: Redis remembers the size of the AOF file after the + # latest rewrite (if no rewrite has happened since the restart, the size of + # the AOF at startup is used). + # + # This base size is compared to the current size. If the current size is + # bigger than the specified percentage, the rewrite is triggered. Also + # you need to specify a minimal size for the AOF file to be rewritten, this + # is useful to avoid rewriting the AOF file even if the percentage increase + # is reached but it is still pretty small. + # + # Specify a percentage of zero in order to disable the automatic AOF + # rewrite feature. + + auto-aof-rewrite-percentage 100 + auto-aof-rewrite-min-size 64mb + + # An AOF file may be found to be truncated at the end during the Redis + # startup process, when the AOF data gets loaded back into memory. + # This may happen when the system where Redis is running + # crashes, especially when an ext4 filesystem is mounted without the + # data=ordered option (however this can't happen when Redis itself + # crashes or aborts but the operating system still works correctly). + # + # Redis can either exit with an error when this happens, or load as much + # data as possible (the default now) and start if the AOF file is found + # to be truncated at the end. The following option controls this behavior. + # + # If aof-load-truncated is set to yes, a truncated AOF file is loaded and + # the Redis server starts emitting a log to inform the user of the event. + # Otherwise if the option is set to no, the server aborts with an error + # and refuses to start. When the option is set to no, the user requires + # to fix the AOF file using the "redis-check-aof" utility before to restart + # the server. + # + # Note that if the AOF file will be found to be corrupted in the middle + # the server will still exit with an error. This option only applies when + # Redis will try to read more data from the AOF file but not enough bytes + # will be found. + aof-load-truncated yes + + # Redis can create append-only base files in either RDB or AOF formats. Using + # the RDB format is always faster and more efficient, and disabling it is only + # supported for backward compatibility purposes. + aof-use-rdb-preamble yes + + # Redis supports recording timestamp annotations in the AOF to support restoring + # the data from a specific point-in-time. However, using this capability changes + # the AOF format in a way that may not be compatible with existing AOF parsers. + aof-timestamp-enabled no + + ################################ SHUTDOWN ##################################### + + # Maximum time to wait for replicas when shutting down, in seconds. + # + # During shut down, a grace period allows any lagging replicas to catch up with + # the latest replication offset before the master exists. This period can + # prevent data loss, especially for deployments without configured disk backups. + # + # The 'shutdown-timeout' value is the grace period's duration in seconds. It is + # only applicable when the instance has replicas. To disable the feature, set + # the value to 0. + # + # shutdown-timeout 10 + + # When Redis receives a SIGINT or SIGTERM, shutdown is initiated and by default + # an RDB snapshot is written to disk in a blocking operation if save points are configured. + # The options used on signaled shutdown can include the following values: + # default: Saves RDB snapshot only if save points are configured. + # Waits for lagging replicas to catch up. + # save: Forces a DB saving operation even if no save points are configured. + # nosave: Prevents DB saving operation even if one or more save points are configured. + # now: Skips waiting for lagging replicas. + # force: Ignores any errors that would normally prevent the server from exiting. + # + # Any combination of values is allowed as long as "save" and "nosave" are not set simultaneously. + # Example: "nosave force now" + # + # shutdown-on-sigint default + # shutdown-on-sigterm default + + ################ NON-DETERMINISTIC LONG BLOCKING COMMANDS ##################### + + # Maximum time in milliseconds for EVAL scripts, functions and in some cases + # modules' commands before Redis can start processing or rejecting other clients. + # + # If the maximum execution time is reached Redis will start to reply to most + # commands with a BUSY error. + # + # In this state Redis will only allow a handful of commands to be executed. + # For instance, SCRIPT KILL, FUNCTION KILL, SHUTDOWN NOSAVE and possibly some + # module specific 'allow-busy' commands. + # + # SCRIPT KILL and FUNCTION KILL will only be able to stop a script that did not + # yet call any write commands, so SHUTDOWN NOSAVE may be the only way to stop + # the server in the case a write command was already issued by the script when + # the user doesn't want to wait for the natural termination of the script. + # + # The default is 5 seconds. It is possible to set it to 0 or a negative value + # to disable this mechanism (uninterrupted execution). Note that in the past + # this config had a different name, which is now an alias, so both of these do + # the same: + lua-time-limit 5000 + # busy-reply-threshold 5000 + + ################################ REDIS CLUSTER ############################### + + # Normal Redis instances can't be part of a Redis Cluster; only nodes that are + # started as cluster nodes can. In order to start a Redis instance as a + # cluster node enable the cluster support uncommenting the following: + # + cluster-enabled yes + + # Every cluster node has a cluster configuration file. This file is not + # intended to be edited by hand. It is created and updated by Redis nodes. + # Every Redis Cluster node requires a different cluster configuration file. + # Make sure that instances running in the same system do not have + # overlapping cluster configuration file names. + # + cluster-config-file /bitnami/redis/data/nodes.conf + + # Cluster node timeout is the amount of milliseconds a node must be unreachable + # for it to be considered in failure state. + # Most other internal time limits are a multiple of the node timeout. + # + # cluster-node-timeout 15000 + + # The cluster port is the port that the cluster bus will listen for inbound connections on. When set + # to the default value, 0, it will be bound to the command port + 10000. Setting this value requires + # you to specify the cluster bus port when executing cluster meet. + # cluster-port 0 + + # A replica of a failing master will avoid to start a failover if its data + # looks too old. + # + # There is no simple way for a replica to actually have an exact measure of + # its "data age", so the following two checks are performed: + # + # 1) If there are multiple replicas able to failover, they exchange messages + # in order to try to give an advantage to the replica with the best + # replication offset (more data from the master processed). + # Replicas will try to get their rank by offset, and apply to the start + # of the failover a delay proportional to their rank. + # + # 2) Every single replica computes the time of the last interaction with + # its master. This can be the last ping or command received (if the master + # is still in the "connected" state), or the time that elapsed since the + # disconnection with the master (if the replication link is currently down). + # If the last interaction is too old, the replica will not try to failover + # at all. + # + # The point "2" can be tuned by user. Specifically a replica will not perform + # the failover if, since the last interaction with the master, the time + # elapsed is greater than: + # + # (node-timeout * cluster-replica-validity-factor) + repl-ping-replica-period + # + # So for example if node-timeout is 30 seconds, and the cluster-replica-validity-factor + # is 10, and assuming a default repl-ping-replica-period of 10 seconds, the + # replica will not try to failover if it was not able to talk with the master + # for longer than 310 seconds. + # + # A large cluster-replica-validity-factor may allow replicas with too old data to failover + # a master, while a too small value may prevent the cluster from being able to + # elect a replica at all. + # + # For maximum availability, it is possible to set the cluster-replica-validity-factor + # to a value of 0, which means, that replicas will always try to failover the + # master regardless of the last time they interacted with the master. + # (However they'll always try to apply a delay proportional to their + # offset rank). + # + # Zero is the only value able to guarantee that when all the partitions heal + # the cluster will always be able to continue. + # + # cluster-replica-validity-factor 10 + + # Cluster replicas are able to migrate to orphaned masters, that are masters + # that are left without working replicas. This improves the cluster ability + # to resist to failures as otherwise an orphaned master can't be failed over + # in case of failure if it has no working replicas. + # + # Replicas migrate to orphaned masters only if there are still at least a + # given number of other working replicas for their old master. This number + # is the "migration barrier". A migration barrier of 1 means that a replica + # will migrate only if there is at least 1 other working replica for its master + # and so forth. It usually reflects the number of replicas you want for every + # master in your cluster. + # + # Default is 1 (replicas migrate only if their masters remain with at least + # one replica). To disable migration just set it to a very large value or + # set cluster-allow-replica-migration to 'no'. + # A value of 0 can be set but is useful only for debugging and dangerous + # in production. + # + # cluster-migration-barrier 1 + + # Turning off this option allows to use less automatic cluster configuration. + # It both disables migration to orphaned masters and migration from masters + # that became empty. + # + # Default is 'yes' (allow automatic migrations). + # + # cluster-allow-replica-migration yes + + # By default Redis Cluster nodes stop accepting queries if they detect there + # is at least a hash slot uncovered (no available node is serving it). + # This way if the cluster is partially down (for example a range of hash slots + # are no longer covered) all the cluster becomes, eventually, unavailable. + # It automatically returns available as soon as all the slots are covered again. + # + # However sometimes you want the subset of the cluster which is working, + # to continue to accept queries for the part of the key space that is still + # covered. In order to do so, just set the cluster-require-full-coverage + # option to no. + # + # cluster-require-full-coverage yes + + # This option, when set to yes, prevents replicas from trying to failover its + # master during master failures. However the replica can still perform a + # manual failover, if forced to do so. + # + # This is useful in different scenarios, especially in the case of multiple + # data center operations, where we want one side to never be promoted if not + # in the case of a total DC failure. + # + # cluster-replica-no-failover no + + # This option, when set to yes, allows nodes to serve read traffic while the + # cluster is in a down state, as long as it believes it owns the slots. + # + # This is useful for two cases. The first case is for when an application + # doesn't require consistency of data during node failures or network partitions. + # One example of this is a cache, where as long as the node has the data it + # should be able to serve it. + # + # The second use case is for configurations that don't meet the recommended + # three shards but want to enable cluster mode and scale later. A + # master outage in a 1 or 2 shard configuration causes a read/write outage to the + # entire cluster without this option set, with it set there is only a write outage. + # Without a quorum of masters, slot ownership will not change automatically. + # + # cluster-allow-reads-when-down no + + # This option, when set to yes, allows nodes to serve pubsub shard traffic while + # the cluster is in a down state, as long as it believes it owns the slots. + # + # This is useful if the application would like to use the pubsub feature even when + # the cluster global stable state is not OK. If the application wants to make sure only + # one shard is serving a given channel, this feature should be kept as yes. + # + # cluster-allow-pubsubshard-when-down yes + + # Cluster link send buffer limit is the limit on the memory usage of an individual + # cluster bus link's send buffer in bytes. Cluster links would be freed if they exceed + # this limit. This is to primarily prevent send buffers from growing unbounded on links + # toward slow peers (E.g. PubSub messages being piled up). + # This limit is disabled by default. Enable this limit when 'mem_cluster_links' INFO field + # and/or 'send-buffer-allocated' entries in the 'CLUSTER LINKS` command output continuously increase. + # Minimum limit of 1gb is recommended so that cluster link buffer can fit in at least a single + # PubSub message by default. (client-query-buffer-limit default value is 1gb) + # + # cluster-link-sendbuf-limit 0 + + # Clusters can configure their announced hostname using this config. This is a common use case for + # applications that need to use TLS Server Name Indication (SNI) or dealing with DNS based + # routing. By default this value is only shown as additional metadata in the CLUSTER SLOTS + # command, but can be changed using 'cluster-preferred-endpoint-type' config. This value is + # communicated along the clusterbus to all nodes, setting it to an empty string will remove + # the hostname and also propagate the removal. + # + # cluster-announce-hostname "" + + # Clusters can advertise how clients should connect to them using either their IP address, + # a user defined hostname, or by declaring they have no endpoint. Which endpoint is + # shown as the preferred endpoint is set by using the cluster-preferred-endpoint-type + # config with values 'ip', 'hostname', or 'unknown-endpoint'. This value controls how + # the endpoint returned for MOVED/ASKING requests as well as the first field of CLUSTER SLOTS. + # If the preferred endpoint type is set to hostname, but no announced hostname is set, a '?' + # will be returned instead. + # + # When a cluster advertises itself as having an unknown endpoint, it's indicating that + # the server doesn't know how clients can reach the cluster. This can happen in certain + # networking situations where there are multiple possible routes to the node, and the + # server doesn't know which one the client took. In this case, the server is expecting + # the client to reach out on the same endpoint it used for making the last request, but use + # the port provided in the response. + # + # cluster-preferred-endpoint-type ip + + # In order to setup your cluster make sure to read the documentation + # available at https://redis.io web site. + + ########################## CLUSTER DOCKER/NAT support ######################## + + # In certain deployments, Redis Cluster nodes address discovery fails, because + # addresses are NAT-ted or because ports are forwarded (the typical case is + # Docker and other containers). + # + # In order to make Redis Cluster working in such environments, a static + # configuration where each node knows its public address is needed. The + # following four options are used for this scope, and are: + # + # * cluster-announce-ip + # * cluster-announce-port + # * cluster-announce-tls-port + # * cluster-announce-bus-port + # + # Each instructs the node about its address, client ports (for connections + # without and with TLS) and cluster message bus port. The information is then + # published in the header of the bus packets so that other nodes will be able to + # correctly map the address of the node publishing the information. + # + # If cluster-tls is set to yes and cluster-announce-tls-port is omitted or set + # to zero, then cluster-announce-port refers to the TLS port. Note also that + # cluster-announce-tls-port has no effect if cluster-tls is set to no. + # + # If the above options are not used, the normal Redis Cluster auto-detection + # will be used instead. + # + # Note that when remapped, the bus port may not be at the fixed offset of + # clients port + 10000, so you can specify any port and bus-port depending + # on how they get remapped. If the bus-port is not set, a fixed offset of + # 10000 will be used as usual. + # + # Example: + # + # cluster-announce-ip 10.1.1.5 + # cluster-announce-tls-port 6379 + # cluster-announce-port 0 + # cluster-announce-bus-port 6380 + + ################################## SLOW LOG ################################### + + # The Redis Slow Log is a system to log queries that exceeded a specified + # execution time. The execution time does not include the I/O operations + # like talking with the client, sending the reply and so forth, + # but just the time needed to actually execute the command (this is the only + # stage of command execution where the thread is blocked and can not serve + # other requests in the meantime). + # + # You can configure the slow log with two parameters: one tells Redis + # what is the execution time, in microseconds, to exceed in order for the + # command to get logged, and the other parameter is the length of the + # slow log. When a new command is logged the oldest one is removed from the + # queue of logged commands. + + # The following time is expressed in microseconds, so 1000000 is equivalent + # to one second. Note that a negative number disables the slow log, while + # a value of zero forces the logging of every command. + slowlog-log-slower-than 10000 + + # There is no limit to this length. Just be aware that it will consume memory. + # You can reclaim memory used by the slow log with SLOWLOG RESET. + slowlog-max-len 128 + + ################################ LATENCY MONITOR ############################## + + # The Redis latency monitoring subsystem samples different operations + # at runtime in order to collect data related to possible sources of + # latency of a Redis instance. + # + # Via the LATENCY command this information is available to the user that can + # print graphs and obtain reports. + # + # The system only logs operations that were performed in a time equal or + # greater than the amount of milliseconds specified via the + # latency-monitor-threshold configuration directive. When its value is set + # to zero, the latency monitor is turned off. + # + # By default latency monitoring is disabled since it is mostly not needed + # if you don't have latency issues, and collecting data has a performance + # impact, that while very small, can be measured under big load. Latency + # monitoring can easily be enabled at runtime using the command + # "CONFIG SET latency-monitor-threshold " if needed. + latency-monitor-threshold 0 + + ################################ LATENCY TRACKING ############################## + + # The Redis extended latency monitoring tracks the per command latencies and enables + # exporting the percentile distribution via the INFO latencystats command, + # and cumulative latency distributions (histograms) via the LATENCY command. + # + # By default, the extended latency monitoring is enabled since the overhead + # of keeping track of the command latency is very small. + # latency-tracking yes + + # By default the exported latency percentiles via the INFO latencystats command + # are the p50, p99, and p999. + # latency-tracking-info-percentiles 50 99 99.9 + + ############################# EVENT NOTIFICATION ############################## + + # Redis can notify Pub/Sub clients about events happening in the key space. + # This feature is documented at https://redis.io/topics/notifications + # + # For instance if keyspace events notification is enabled, and a client + # performs a DEL operation on key "foo" stored in the Database 0, two + # messages will be published via Pub/Sub: + # + # PUBLISH __keyspace@0__:foo del + # PUBLISH __keyevent@0__:del foo + # + # It is possible to select the events that Redis will notify among a set + # of classes. Every class is identified by a single character: + # + # K Keyspace events, published with __keyspace@__ prefix. + # E Keyevent events, published with __keyevent@__ prefix. + # g Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ... + # $ String commands + # l List commands + # s Set commands + # h Hash commands + # z Sorted set commands + # x Expired events (events generated every time a key expires) + # e Evicted events (events generated when a key is evicted for maxmemory) + # n New key events (Note: not included in the 'A' class) + # t Stream commands + # d Module key type events + # m Key-miss events (Note: It is not included in the 'A' class) + # A Alias for g$lshzxetd, so that the "AKE" string means all the events + # (Except key-miss events which are excluded from 'A' due to their + # unique nature). + # + # The "notify-keyspace-events" takes as argument a string that is composed + # of zero or multiple characters. The empty string means that notifications + # are disabled. + # + # Example: to enable list and generic events, from the point of view of the + # event name, use: + # + # notify-keyspace-events Elg + # + # Example 2: to get the stream of the expired keys subscribing to channel + # name __keyevent@0__:expired use: + # + # notify-keyspace-events Ex + # + # By default all notifications are disabled because most users don't need + # this feature and the feature has some overhead. Note that if you don't + # specify at least one of K or E, no events will be delivered. + notify-keyspace-events "" + + ############################### ADVANCED CONFIG ############################### + + # Hashes are encoded using a memory efficient data structure when they have a + # small number of entries, and the biggest entry does not exceed a given + # threshold. These thresholds can be configured using the following directives. + hash-max-listpack-entries 512 + hash-max-listpack-value 64 + + # Lists are also encoded in a special way to save a lot of space. + # The number of entries allowed per internal list node can be specified + # as a fixed maximum size or a maximum number of elements. + # For a fixed maximum size, use -5 through -1, meaning: + # -5: max size: 64 Kb <-- not recommended for normal workloads + # -4: max size: 32 Kb <-- not recommended + # -3: max size: 16 Kb <-- probably not recommended + # -2: max size: 8 Kb <-- good + # -1: max size: 4 Kb <-- good + # Positive numbers mean store up to _exactly_ that number of elements + # per list node. + # The highest performing option is usually -2 (8 Kb size) or -1 (4 Kb size), + # but if your use case is unique, adjust the settings as necessary. + list-max-listpack-size -2 + + # Lists may also be compressed. + # Compress depth is the number of quicklist ziplist nodes from *each* side of + # the list to *exclude* from compression. The head and tail of the list + # are always uncompressed for fast push/pop operations. Settings are: + # 0: disable all list compression + # 1: depth 1 means "don't start compressing until after 1 node into the list, + # going from either the head or tail" + # So: [head]->node->node->...->node->[tail] + # [head], [tail] will always be uncompressed; inner nodes will compress. + # 2: [head]->[next]->node->node->...->node->[prev]->[tail] + # 2 here means: don't compress head or head->next or tail->prev or tail, + # but compress all nodes between them. + # 3: [head]->[next]->[next]->node->node->...->node->[prev]->[prev]->[tail] + # etc. + list-compress-depth 0 + + # Sets have a special encoding in just one case: when a set is composed + # of just strings that happen to be integers in radix 10 in the range + # of 64 bit signed integers. + # The following configuration setting sets the limit in the size of the + # set in order to use this special memory saving encoding. + set-max-intset-entries 512 + + # Similarly to hashes and lists, sorted sets are also specially encoded in + # order to save a lot of space. This encoding is only used when the length and + # elements of a sorted set are below the following limits: + zset-max-listpack-entries 128 + zset-max-listpack-value 64 + + # HyperLogLog sparse representation bytes limit. The limit includes the + # 16 bytes header. When an HyperLogLog using the sparse representation crosses + # this limit, it is converted into the dense representation. + # + # A value greater than 16000 is totally useless, since at that point the + # dense representation is more memory efficient. + # + # The suggested value is ~ 3000 in order to have the benefits of + # the space efficient encoding without slowing down too much PFADD, + # which is O(N) with the sparse encoding. The value can be raised to + # ~ 10000 when CPU is not a concern, but space is, and the data set is + # composed of many HyperLogLogs with cardinality in the 0 - 15000 range. + hll-sparse-max-bytes 3000 + + # Streams macro node max size / items. The stream data structure is a radix + # tree of big nodes that encode multiple items inside. Using this configuration + # it is possible to configure how big a single node can be in bytes, and the + # maximum number of items it may contain before switching to a new node when + # appending new stream entries. If any of the following settings are set to + # zero, the limit is ignored, so for instance it is possible to set just a + # max entries limit by setting max-bytes to 0 and max-entries to the desired + # value. + stream-node-max-bytes 4096 + stream-node-max-entries 100 + + # Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in + # order to help rehashing the main Redis hash table (the one mapping top-level + # keys to values). The hash table implementation Redis uses (see dict.c) + # performs a lazy rehashing: the more operation you run into a hash table + # that is rehashing, the more rehashing "steps" are performed, so if the + # server is idle the rehashing is never complete and some more memory is used + # by the hash table. + # + # The default is to use this millisecond 10 times every second in order to + # actively rehash the main dictionaries, freeing memory when possible. + # + # If unsure: + # use "activerehashing no" if you have hard latency requirements and it is + # not a good thing in your environment that Redis can reply from time to time + # to queries with 2 milliseconds delay. + # + # use "activerehashing yes" if you don't have such hard requirements but + # want to free memory asap when possible. + activerehashing yes + + # The client output buffer limits can be used to force disconnection of clients + # that are not reading data from the server fast enough for some reason (a + # common reason is that a Pub/Sub client can't consume messages as fast as the + # publisher can produce them). + # + # The limit can be set differently for the three different classes of clients: + # + # normal -> normal clients including MONITOR clients + # replica -> replica clients + # pubsub -> clients subscribed to at least one pubsub channel or pattern + # + # The syntax of every client-output-buffer-limit directive is the following: + # + # client-output-buffer-limit + # + # A client is immediately disconnected once the hard limit is reached, or if + # the soft limit is reached and remains reached for the specified number of + # seconds (continuously). + # So for instance if the hard limit is 32 megabytes and the soft limit is + # 16 megabytes / 10 seconds, the client will get disconnected immediately + # if the size of the output buffers reach 32 megabytes, but will also get + # disconnected if the client reaches 16 megabytes and continuously overcomes + # the limit for 10 seconds. + # + # By default normal clients are not limited because they don't receive data + # without asking (in a push way), but just after a request, so only + # asynchronous clients may create a scenario where data is requested faster + # than it can read. + # + # Instead there is a default limit for pubsub and replica clients, since + # subscribers and replicas receive data in a push fashion. + # + # Note that it doesn't make sense to set the replica clients output buffer + # limit lower than the repl-backlog-size config (partial sync will succeed + # and then replica will get disconnected). + # Such a configuration is ignored (the size of repl-backlog-size will be used). + # This doesn't have memory consumption implications since the replica client + # will share the backlog buffers memory. + # + # Both the hard or the soft limit can be disabled by setting them to zero. + client-output-buffer-limit normal 0 0 0 + client-output-buffer-limit replica 256mb 64mb 60 + client-output-buffer-limit pubsub 32mb 8mb 60 + + # Client query buffers accumulate new commands. They are limited to a fixed + # amount by default in order to avoid that a protocol desynchronization (for + # instance due to a bug in the client) will lead to unbound memory usage in + # the query buffer. However you can configure it here if you have very special + # needs, such us huge multi/exec requests or alike. + # + # client-query-buffer-limit 1gb + + # In some scenarios client connections can hog up memory leading to OOM + # errors or data eviction. To avoid this we can cap the accumulated memory + # used by all client connections (all pubsub and normal clients). Once we + # reach that limit connections will be dropped by the server freeing up + # memory. The server will attempt to drop the connections using the most + # memory first. We call this mechanism "client eviction". + # + # Client eviction is configured using the maxmemory-clients setting as follows: + # 0 - client eviction is disabled (default) + # + # A memory value can be used for the client eviction threshold, + # for example: + # maxmemory-clients 1g + # + # A percentage value (between 1% and 100%) means the client eviction threshold + # is based on a percentage of the maxmemory setting. For example to set client + # eviction at 5% of maxmemory: + # maxmemory-clients 5% + + # In the Redis protocol, bulk requests, that are, elements representing single + # strings, are normally limited to 512 mb. However you can change this limit + # here, but must be 1mb or greater + # + # proto-max-bulk-len 512mb + + # Redis calls an internal function to perform many background tasks, like + # closing connections of clients in timeout, purging expired keys that are + # never requested, and so forth. + # + # Not all tasks are performed with the same frequency, but Redis checks for + # tasks to perform according to the specified "hz" value. + # + # By default "hz" is set to 10. Raising the value will use more CPU when + # Redis is idle, but at the same time will make Redis more responsive when + # there are many keys expiring at the same time, and timeouts may be + # handled with more precision. + # + # The range is between 1 and 500, however a value over 100 is usually not + # a good idea. Most users should use the default of 10 and raise this up to + # 100 only in environments where very low latency is required. + hz 10 + + # Normally it is useful to have an HZ value which is proportional to the + # number of clients connected. This is useful in order, for instance, to + # avoid too many clients are processed for each background task invocation + # in order to avoid latency spikes. + # + # Since the default HZ value by default is conservatively set to 10, Redis + # offers, and enables by default, the ability to use an adaptive HZ value + # which will temporarily raise when there are many connected clients. + # + # When dynamic HZ is enabled, the actual configured HZ will be used + # as a baseline, but multiples of the configured HZ value will be actually + # used as needed once more clients are connected. In this way an idle + # instance will use very little CPU time while a busy instance will be + # more responsive. + dynamic-hz yes + + # When a child rewrites the AOF file, if the following option is enabled + # the file will be fsync-ed every 4 MB of data generated. This is useful + # in order to commit the file to the disk more incrementally and avoid + # big latency spikes. + aof-rewrite-incremental-fsync yes + + # When redis saves RDB file, if the following option is enabled + # the file will be fsync-ed every 4 MB of data generated. This is useful + # in order to commit the file to the disk more incrementally and avoid + # big latency spikes. + rdb-save-incremental-fsync yes + + # Redis LFU eviction (see maxmemory setting) can be tuned. However it is a good + # idea to start with the default settings and only change them after investigating + # how to improve the performances and how the keys LFU change over time, which + # is possible to inspect via the OBJECT FREQ command. + # + # There are two tunable parameters in the Redis LFU implementation: the + # counter logarithm factor and the counter decay time. It is important to + # understand what the two parameters mean before changing them. + # + # The LFU counter is just 8 bits per key, it's maximum value is 255, so Redis + # uses a probabilistic increment with logarithmic behavior. Given the value + # of the old counter, when a key is accessed, the counter is incremented in + # this way: + # + # 1. A random number R between 0 and 1 is extracted. + # 2. A probability P is calculated as 1/(old_value*lfu_log_factor+1). + # 3. The counter is incremented only if R < P. + # + # The default lfu-log-factor is 10. This is a table of how the frequency + # counter changes with a different number of accesses with different + # logarithmic factors: + # + # +--------+------------+------------+------------+------------+------------+ + # | factor | 100 hits | 1000 hits | 100K hits | 1M hits | 10M hits | + # +--------+------------+------------+------------+------------+------------+ + # | 0 | 104 | 255 | 255 | 255 | 255 | + # +--------+------------+------------+------------+------------+------------+ + # | 1 | 18 | 49 | 255 | 255 | 255 | + # +--------+------------+------------+------------+------------+------------+ + # | 10 | 10 | 18 | 142 | 255 | 255 | + # +--------+------------+------------+------------+------------+------------+ + # | 100 | 8 | 11 | 49 | 143 | 255 | + # +--------+------------+------------+------------+------------+------------+ + # + # NOTE: The above table was obtained by running the following commands: + # + # redis-benchmark -n 1000000 incr foo + # redis-cli object freq foo + # + # NOTE 2: The counter initial value is 5 in order to give new objects a chance + # to accumulate hits. + # + # The counter decay time is the time, in minutes, that must elapse in order + # for the key counter to be divided by two (or decremented if it has a value + # less <= 10). + # + # The default value for the lfu-decay-time is 1. A special value of 0 means to + # decay the counter every time it happens to be scanned. + # + # lfu-log-factor 10 + # lfu-decay-time 1 + + ########################### ACTIVE DEFRAGMENTATION ####################### + # + # What is active defragmentation? + # ------------------------------- + # + # Active (online) defragmentation allows a Redis server to compact the + # spaces left between small allocations and deallocations of data in memory, + # thus allowing to reclaim back memory. + # + # Fragmentation is a natural process that happens with every allocator (but + # less so with Jemalloc, fortunately) and certain workloads. Normally a server + # restart is needed in order to lower the fragmentation, or at least to flush + # away all the data and create it again. However thanks to this feature + # implemented by Oran Agra for Redis 4.0 this process can happen at runtime + # in a "hot" way, while the server is running. + # + # Basically when the fragmentation is over a certain level (see the + # configuration options below) Redis will start to create new copies of the + # values in contiguous memory regions by exploiting certain specific Jemalloc + # features (in order to understand if an allocation is causing fragmentation + # and to allocate it in a better place), and at the same time, will release the + # old copies of the data. This process, repeated incrementally for all the keys + # will cause the fragmentation to drop back to normal values. + # + # Important things to understand: + # + # 1. This feature is disabled by default, and only works if you compiled Redis + # to use the copy of Jemalloc we ship with the source code of Redis. + # This is the default with Linux builds. + # + # 2. You never need to enable this feature if you don't have fragmentation + # issues. + # + # 3. Once you experience fragmentation, you can enable this feature when + # needed with the command "CONFIG SET activedefrag yes". + # + # The configuration parameters are able to fine tune the behavior of the + # defragmentation process. If you are not sure about what they mean it is + # a good idea to leave the defaults untouched. + + # Active defragmentation is disabled by default + # activedefrag no + + # Minimum amount of fragmentation waste to start active defrag + # active-defrag-ignore-bytes 100mb + + # Minimum percentage of fragmentation to start active defrag + # active-defrag-threshold-lower 10 + + # Maximum percentage of fragmentation at which we use maximum effort + # active-defrag-threshold-upper 100 + + # Minimal effort for defrag in CPU percentage, to be used when the lower + # threshold is reached + # active-defrag-cycle-min 1 + + # Maximal effort for defrag in CPU percentage, to be used when the upper + # threshold is reached + # active-defrag-cycle-max 25 + + # Maximum number of set/hash/zset/list fields that will be processed from + # the main dictionary scan + # active-defrag-max-scan-fields 1000 + + # Jemalloc background thread for purging will be enabled by default + jemalloc-bg-thread yes + + # It is possible to pin different threads and processes of Redis to specific + # CPUs in your system, in order to maximize the performances of the server. + # This is useful both in order to pin different Redis threads in different + # CPUs, but also in order to make sure that multiple Redis instances running + # in the same host will be pinned to different CPUs. + # + # Normally you can do this using the "taskset" command, however it is also + # possible to this via Redis configuration directly, both in Linux and FreeBSD. + # + # You can pin the server/IO threads, bio threads, aof rewrite child process, and + # the bgsave child process. The syntax to specify the cpu list is the same as + # the taskset command: + # + # Set redis server/io threads to cpu affinity 0,2,4,6: + # server_cpulist 0-7:2 + # + # Set bio threads to cpu affinity 1,3: + # bio_cpulist 1,3 + # + # Set aof rewrite child process to cpu affinity 8,9,10,11: + # aof_rewrite_cpulist 8-11 + # + # Set bgsave child process to cpu affinity 1,10,11 + # bgsave_cpulist 1,10-11 + + # In some cases redis will emit warnings and even refuse to start if it detects + # that the system is in bad state, it is possible to suppress these warnings + # by setting the following config which takes a space delimited list of warnings + # to suppress + # + # ignore-warnings ARM64-COW-BUG +--- +# Source: gitea/charts/redis-cluster/templates/scripts-configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-redis-cluster-scripts + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: redis-cluster + app.kubernetes.io/version: 7.2.5 +data: + ping_readiness_local.sh: |- + #!/bin/sh + set -e + + REDIS_STATUS_FILE=/tmp/.redis_cluster_check + if [ ! -z "$REDIS_PASSWORD" ]; then export REDISCLI_AUTH=$REDIS_PASSWORD; fi; + response=$( + timeout -s 15 $1 \ + redis-cli \ + -h localhost \ + -p $REDIS_PORT_NUMBER \ + ping + ) + if [ "$?" -eq "124" ]; then + echo "Timed out" + exit 1 + fi + if [ "$response" != "PONG" ]; then + echo "$response" + exit 1 + fi + if [ ! -f "$REDIS_STATUS_FILE" ]; then + response=$( + timeout -s 15 $1 \ + redis-cli \ + -h localhost \ + -p $REDIS_PORT_NUMBER \ + CLUSTER INFO | grep cluster_state | tr -d '[:space:]' + ) + if [ "$?" -eq "124" ]; then + echo "Timed out" + exit 1 + fi + if [ "$response" != "cluster_state:ok" ]; then + echo "$response" + exit 1 + else + touch "$REDIS_STATUS_FILE" + fi + fi + ping_liveness_local.sh: |- + #!/bin/sh + set -e + if [ ! -z "$REDIS_PASSWORD" ]; then export REDISCLI_AUTH=$REDIS_PASSWORD; fi; + response=$( + timeout -s 15 $1 \ + redis-cli \ + -h localhost \ + -p $REDIS_PORT_NUMBER \ + ping + ) + if [ "$?" -eq "124" ]; then + echo "Timed out" + exit 1 + fi + responseFirstWord=$(echo $response | head -n1 | awk '{print $1;}') + if [ "$response" != "PONG" ] && [ "$responseFirstWord" != "LOADING" ] && [ "$responseFirstWord" != "MASTERDOWN" ]; then + echo "$response" + exit 1 + fi diff --git a/gpulive/gitea/vm/3_apply/deploy.yaml b/gpulive/gitea/vm/3_apply/deploy.yaml new file mode 100644 index 0000000..e411cf5 --- /dev/null +++ b/gpulive/gitea/vm/3_apply/deploy.yaml @@ -0,0 +1,820 @@ +# Source: gitea/charts/postgresql-ha/templates/pgpool/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: release-name-postgresql-ha-pgpool + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/version: 4.5.2 + app.kubernetes.io/component: pgpool +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/component: pgpool + template: + metadata: + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/version: 4.5.2 + app.kubernetes.io/component: pgpool + spec: + nodeSelector: + nodegroup: nd + automountServiceAccountToken: false + affinity: + podAffinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/component: pgpool + topologyKey: kubernetes.io/hostname + weight: 1 + nodeAffinity: + + securityContext: + fsGroup: 1001 + fsGroupChangePolicy: Always + supplementalGroups: [] + sysctls: [] + serviceAccountName: release-name-postgresql-ha + # Auxiliary vars to populate environment variables + containers: + - name: pgpool + image: docker.io/bitnami/pgpool:4.5.2-debian-12-r2 + imagePullPolicy: "IfNotPresent" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + env: + - name: BITNAMI_DEBUG + value: "false" + - name: PGPOOL_BACKEND_NODES + value: 0:release-name-postgresql-ha-postgresql-0.release-name-postgresql-ha-postgresql-headless:5432,1:release-name-postgresql-ha-postgresql-1.release-name-postgresql-ha-postgresql-headless:5432,2:release-name-postgresql-ha-postgresql-2.release-name-postgresql-ha-postgresql-headless:5432, + - name: PGPOOL_SR_CHECK_USER + value: "repmgr" + - name: PGPOOL_SR_CHECK_PASSWORD + valueFrom: + secretKeyRef: + name: release-name-postgresql-ha-postgresql + key: repmgr-password + - name: PGPOOL_SR_CHECK_DATABASE + value: "postgres" + - name: PGPOOL_ENABLE_LDAP + value: "no" + - name: PGPOOL_POSTGRES_USERNAME + value: "gitea" + - name: PGPOOL_POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: release-name-postgresql-ha-postgresql + key: password + - name: PGPOOL_ADMIN_USERNAME + value: "admin" + - name: PGPOOL_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: release-name-postgresql-ha-pgpool + key: admin-password + - name: PGPOOL_AUTHENTICATION_METHOD + value: "scram-sha-256" + - name: PGPOOL_ENABLE_LOAD_BALANCING + value: "yes" + - name: PGPOOL_DISABLE_LOAD_BALANCE_ON_WRITE + value: "transaction" + - name: PGPOOL_ENABLE_LOG_CONNECTIONS + value: "no" + - name: PGPOOL_ENABLE_LOG_HOSTNAME + value: "yes" + - name: PGPOOL_ENABLE_LOG_PER_NODE_STATEMENT + value: "no" + - name: PGPOOL_RESERVED_CONNECTIONS + value: '1' + - name: PGPOOL_CHILD_LIFE_TIME + value: "" + - name: PGPOOL_ENABLE_TLS + value: "no" + - name: PGPOOL_HEALTH_CHECK_PSQL_TIMEOUT + value: "6" + envFrom: + ports: + - name: postgresql + containerPort: 5432 + protocol: TCP + livenessProbe: + failureThreshold: 5 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + exec: + command: + - /opt/bitnami/scripts/pgpool/healthcheck.sh + readinessProbe: + failureThreshold: 5 + initialDelaySeconds: 5 + periodSeconds: 5 + successThreshold: 1 + timeoutSeconds: 5 + exec: + command: + - bash + - -ec + - PGPASSWORD=${PGPOOL_POSTGRES_PASSWORD} psql -U "gitea" -d "gitea" -h /opt/bitnami/pgpool/tmp -tA -c "SELECT 1" >/dev/null + resources: + limits: + cpu: 375m + ephemeral-storage: 2Gi + memory: 384Mi + requests: + cpu: 250m + ephemeral-storage: 50Mi + memory: 256Mi + volumeMounts: + - name: empty-dir + mountPath: /tmp + subPath: tmp-dir + - name: empty-dir + mountPath: /opt/bitnami/pgpool/etc + subPath: app-etc-dir + - name: empty-dir + mountPath: /opt/bitnami/pgpool/conf + subPath: app-conf-dir + - name: empty-dir + mountPath: /opt/bitnami/pgpool/tmp + subPath: app-tmp-dir + - name: empty-dir + mountPath: /opt/bitnami/pgpool/logs + subPath: app-logs-dir + volumes: + - name: empty-dir + emptyDir: {} +--- +# Source: gitea/templates/gitea/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: release-name-gitea + namespace: gitea + annotations: + labels: + app: gitea + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "1.22.1" + version: "1.22.1" +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 100% + selector: + matchLabels: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: release-name + template: + metadata: + annotations: + checksum/config: 00fd8064076a446a896870db4974c6590fcf51561cf391547e559011465a57d8 + labels: + app: gitea + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "1.22.1" + version: "1.22.1" + spec: + nodeSelector: + nodegroup: nd + securityContext: + fsGroup: 1000 + initContainers: + - name: init-directories + image: "gitea/gitea:1.22.1-rootless" + imagePullPolicy: IfNotPresent + command: ["/usr/sbin/init_directory_structure.sh"] + env: + - name: GITEA_APP_INI + value: /data/gitea/conf/app.ini + - name: GITEA_CUSTOM + value: /data/gitea + - name: GITEA_WORK_DIR + value: /data + - name: GITEA_TEMP + value: /tmp/gitea + volumeMounts: + - name: init + mountPath: /usr/sbin + - name: temp + mountPath: /tmp + - name: data + mountPath: /data + + securityContext: + {} + resources: + limits: {} + requests: + cpu: 100m + memory: 128Mi + - name: init-app-ini + image: "gitea/gitea:1.22.1-rootless" + imagePullPolicy: IfNotPresent + command: ["/usr/sbin/config_environment.sh"] + env: + - name: GITEA_APP_INI + value: /data/gitea/conf/app.ini + - name: GITEA_CUSTOM + value: /data/gitea + - name: GITEA_WORK_DIR + value: /data + - name: GITEA_TEMP + value: /tmp/gitea + volumeMounts: + - name: config + mountPath: /usr/sbin + - name: temp + mountPath: /tmp + - name: data + mountPath: /data + - name: inline-config-sources + mountPath: /env-to-ini-mounts/inlines/ + + securityContext: + {} + resources: + limits: {} + requests: + cpu: 100m + memory: 128Mi + - name: configure-gitea + image: "gitea/gitea:1.22.1-rootless" + command: ["/usr/sbin/configure_gitea.sh"] + imagePullPolicy: IfNotPresent + securityContext: + runAsUser: 1000 + env: + - name: GITEA_APP_INI + value: /data/gitea/conf/app.ini + - name: GITEA_CUSTOM + value: /data/gitea + - name: GITEA_WORK_DIR + value: /data + - name: GITEA_TEMP + value: /tmp/gitea + - name: HOME + value: /data/gitea/git + - name: GITEA_ADMIN_USERNAME + value: "gitea_admin" + - name: GITEA_ADMIN_PASSWORD + value: "nhn!@#123" # sdjo + - name: GITEA_ADMIN_PASSWORD_MODE + value: keepUpdated + volumeMounts: + - name: init + mountPath: /usr/sbin + - name: temp + mountPath: /tmp + - name: data + mountPath: /data + + resources: + limits: {} + requests: + cpu: 100m + memory: 128Mi + terminationGracePeriodSeconds: 60 + containers: + - name: gitea + image: "gitea/gitea:1.22.1-rootless" + imagePullPolicy: IfNotPresent + env: + # SSH Port values have to be set here as well for openssh configuration + - name: SSH_LISTEN_PORT + value: "2222" + - name: SSH_PORT + value: "22" + - name: GITEA_APP_INI + value: /data/gitea/conf/app.ini + - name: GITEA_CUSTOM + value: /data/gitea + - name: GITEA_WORK_DIR + value: /data + - name: GITEA_TEMP + value: /tmp/gitea + - name: TMPDIR + value: /tmp/gitea + - name: HOME + value: /data/gitea/git + ports: + - name: ssh + containerPort: 2222 + - name: http + containerPort: 3000 + livenessProbe: + failureThreshold: 10 + initialDelaySeconds: 200 + periodSeconds: 10 + successThreshold: 1 + tcpSocket: + port: http + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + tcpSocket: + port: http + timeoutSeconds: 1 + resources: + {} + securityContext: + {} + volumeMounts: + - name: temp + mountPath: /tmp + - name: data + mountPath: /data + + volumes: + - name: init + secret: + secretName: release-name-gitea-init + defaultMode: 110 + - name: config + secret: + secretName: release-name-gitea + defaultMode: 110 + - name: inline-config-sources + secret: + secretName: release-name-gitea-inline-config + - name: temp + emptyDir: {} + - name: data + persistentVolumeClaim: + claimName: gitea-shared-storage +--- +# Source: gitea/charts/postgresql-ha/templates/postgresql/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: release-name-postgresql-ha-postgresql + namespace: "gitea" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/version: 16.3.0 + app.kubernetes.io/component: postgresql + role: data +spec: + replicas: 3 + podManagementPolicy: "Parallel" + serviceName: release-name-postgresql-ha-postgresql-headless + updateStrategy: + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/component: postgresql + role: data + template: + metadata: + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/version: 16.3.0 + app.kubernetes.io/component: postgresql + role: data + spec: + nodeSelector: + nodegroup: nd + automountServiceAccountToken: false + affinity: + podAffinity: + + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/component: postgresql + topologyKey: kubernetes.io/hostname + weight: 1 + nodeAffinity: + + securityContext: + fsGroup: 1001 + fsGroupChangePolicy: Always + supplementalGroups: [] + sysctls: [] + serviceAccountName: release-name-postgresql-ha + hostNetwork: false + hostIPC: false + containers: + - name: postgresql + image: docker.io/bitnami/postgresql-repmgr:16.3.0-debian-12-r15 + imagePullPolicy: "IfNotPresent" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + lifecycle: + preStop: + exec: + command: + - /pre-stop.sh + - "25" + # Auxiliary vars to populate environment variables + env: + - name: BITNAMI_DEBUG + value: "false" + # PostgreSQL configuration + - name: POSTGRESQL_VOLUME_DIR + value: "/bitnami/postgresql" + - name: PGDATA + value: "/bitnami/postgresql/data" + - name: POSTGRES_POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: release-name-postgresql-ha-postgresql + key: postgres-password + - name: POSTGRES_USER + value: "gitea" + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: release-name-postgresql-ha-postgresql + key: password + - name: POSTGRES_DB + value: "gitea" + - name: POSTGRESQL_LOG_HOSTNAME + value: "true" + - name: POSTGRESQL_LOG_CONNECTIONS + value: "false" + - name: POSTGRESQL_LOG_DISCONNECTIONS + value: "false" + - name: POSTGRESQL_PGAUDIT_LOG_CATALOG + value: "off" + - name: POSTGRESQL_CLIENT_MIN_MESSAGES + value: "error" + - name: POSTGRESQL_SHARED_PRELOAD_LIBRARIES + value: "pgaudit, repmgr" + - name: POSTGRESQL_ENABLE_TLS + value: "no" + - name: POSTGRESQL_PORT_NUMBER + value: "5432" + # Repmgr configuration + - name: REPMGR_PORT_NUMBER + value: "5432" + - name: REPMGR_PRIMARY_PORT + value: "5432" + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: REPMGR_UPGRADE_EXTENSION + value: "no" + - name: REPMGR_PGHBA_TRUST_ALL + value: "no" + - name: REPMGR_MOUNTED_CONF_DIR + value: "/bitnami/repmgr/conf" + - name: REPMGR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: REPMGR_PARTNER_NODES + value: release-name-postgresql-ha-postgresql-0.release-name-postgresql-ha-postgresql-headless.$(REPMGR_NAMESPACE).svc.cluster.local,release-name-postgresql-ha-postgresql-1.release-name-postgresql-ha-postgresql-headless.$(REPMGR_NAMESPACE).svc.cluster.local,release-name-postgresql-ha-postgresql-2.release-name-postgresql-ha-postgresql-headless.$(REPMGR_NAMESPACE).svc.cluster.local, + - name: REPMGR_PRIMARY_HOST + value: "release-name-postgresql-ha-postgresql-0.release-name-postgresql-ha-postgresql-headless.$(REPMGR_NAMESPACE).svc.cluster.local" + - name: REPMGR_NODE_NAME + value: "$(MY_POD_NAME)" + - name: REPMGR_NODE_NETWORK_NAME + value: "$(MY_POD_NAME).release-name-postgresql-ha-postgresql-headless.$(REPMGR_NAMESPACE).svc.cluster.local" + - name: REPMGR_NODE_TYPE + value: "data" + - name: REPMGR_LOG_LEVEL + value: "NOTICE" + - name: REPMGR_CONNECT_TIMEOUT + value: "5" + - name: REPMGR_RECONNECT_ATTEMPTS + value: "2" + - name: REPMGR_RECONNECT_INTERVAL + value: "3" + - name: REPMGR_USERNAME + value: "repmgr" + - name: REPMGR_PASSWORD + valueFrom: + secretKeyRef: + name: release-name-postgresql-ha-postgresql + key: repmgr-password + - name: REPMGR_DATABASE + value: "repmgr" + - name: REPMGR_FENCE_OLD_PRIMARY + value: "no" + - name: REPMGR_CHILD_NODES_CHECK_INTERVAL + value: "5" + - name: REPMGR_CHILD_NODES_CONNECTED_MIN_COUNT + value: "1" + - name: REPMGR_CHILD_NODES_DISCONNECT_TIMEOUT + value: "30" + envFrom: + ports: + - name: postgresql + containerPort: 5432 + protocol: TCP + livenessProbe: + failureThreshold: 6 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + exec: + command: + - bash + - -ec + - 'PGPASSWORD=$POSTGRES_PASSWORD psql -w -U "gitea" -d "gitea" -h 127.0.0.1 -p 5432 -c "SELECT 1"' + readinessProbe: + failureThreshold: 6 + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + exec: + command: + - bash + - -ec + - | + exec pg_isready -U "postgres" -h 127.0.0.1 -p 5432 + [ -f /opt/bitnami/postgresql/tmp/.initialized ] || [ -f /bitnami/postgresql/.initialized ] + resources: + limits: + cpu: 375m + ephemeral-storage: 2Gi + memory: 384Mi + requests: + cpu: 250m + ephemeral-storage: 50Mi + memory: 256Mi + volumeMounts: + - name: empty-dir + mountPath: /tmp + subPath: tmp-dir + - name: empty-dir + mountPath: /opt/bitnami/postgresql/conf + subPath: app-conf-dir + - name: empty-dir + mountPath: /opt/bitnami/postgresql/tmp + subPath: app-tmp-dir + - name: empty-dir + mountPath: /opt/bitnami/repmgr/conf + subPath: repmgr-conf-dir + - name: empty-dir + mountPath: /opt/bitnami/repmgr/tmp + subPath: repmgr-tmp-dir + - name: empty-dir + mountPath: /opt/bitnami/repmgr/logs + subPath: repmgr-logs-dir + - name: data + mountPath: /bitnami/postgresql + - name: hooks-scripts + mountPath: /pre-stop.sh + subPath: pre-stop.sh + - name: hooks-scripts + mountPath: /readiness-probe.sh + subPath: readiness-probe.sh + volumes: + - name: empty-dir + emptyDir: {} + - name: hooks-scripts + configMap: + name: release-name-postgresql-ha-postgresql-hooks-scripts + defaultMode: 0755 + volumeClaimTemplates: + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: data + spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "8Gi" +--- +# Source: cdgitea/charts/redis-cluster/templates/redis-statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: release-name-redis-cluster + namespace: "gitea" + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: redis-cluster + app.kubernetes.io/version: 7.2.5 +spec: + updateStrategy: + rollingUpdate: + partition: 0 + type: RollingUpdate + selector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: redis-cluster + replicas: 3 + serviceName: release-name-redis-cluster-headless + podManagementPolicy: Parallel + template: + metadata: + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: redis-cluster + app.kubernetes.io/version: 7.2.5 + annotations: + checksum/scripts: f4443ed238f1b19ab40555e6ba2aea56daba79e8270549f884b9b31cf8e0e2ee + checksum/secret: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + checksum/config: 958b865b26062946819f240fe4e5fd268b7c3203b944549981f1bd7fdc7947eb + spec: + nodeSelector: + nodegroup: nd + hostNetwork: false + enableServiceLinks: false + + securityContext: + fsGroup: 1001 + fsGroupChangePolicy: Always + supplementalGroups: [] + sysctls: [] + serviceAccountName: release-name-redis-cluster + automountServiceAccountToken: false + affinity: + podAffinity: + + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - podAffinityTerm: + labelSelector: + matchLabels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: redis-cluster + topologyKey: kubernetes.io/hostname + weight: 1 + nodeAffinity: + + containers: + - name: release-name-redis-cluster + image: docker.io/bitnami/redis-cluster:7.2.5-debian-12-r2 + imagePullPolicy: "IfNotPresent" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + privileged: false + readOnlyRootFilesystem: true + runAsGroup: 1001 + runAsNonRoot: true + runAsUser: 1001 + seccompProfile: + type: RuntimeDefault + command: ['/bin/bash', '-c'] + args: + - | + # Backwards compatibility change + if ! [[ -f /opt/bitnami/redis/etc/redis.conf ]]; then + echo COPYING FILE + cp /opt/bitnami/redis/etc/redis-default.conf /opt/bitnami/redis/etc/redis.conf + fi + pod_index=($(echo "$POD_NAME" | tr "-" "\n")) + pod_index="${pod_index[-1]}" + if [[ "$pod_index" == "0" ]]; then + export REDIS_CLUSTER_CREATOR="yes" + export REDIS_CLUSTER_REPLICAS="0" + fi + /opt/bitnami/scripts/redis-cluster/entrypoint.sh /opt/bitnami/scripts/redis-cluster/run.sh + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: REDIS_NODES + value: "release-name-redis-cluster-0.release-name-redis-cluster-headless release-name-redis-cluster-1.release-name-redis-cluster-headless release-name-redis-cluster-2.release-name-redis-cluster-headless " + - name: ALLOW_EMPTY_PASSWORD + value: "yes" + - name: REDIS_AOF_ENABLED + value: "yes" + - name: REDIS_TLS_ENABLED + value: "no" + - name: REDIS_PORT_NUMBER + value: "6379" + ports: + - name: tcp-redis + containerPort: 6379 + - name: tcp-redis-bus + containerPort: 16379 + livenessProbe: + initialDelaySeconds: 5 + periodSeconds: 5 + # One second longer than command timeout should prevent generation of zombie processes. + timeoutSeconds: 6 + successThreshold: 1 + failureThreshold: 5 + exec: + command: + - sh + - -c + - /scripts/ping_liveness_local.sh 5 + readinessProbe: + initialDelaySeconds: 5 + periodSeconds: 5 + # One second longer than command timeout should prevent generation of zombie processes. + timeoutSeconds: 2 + successThreshold: 1 + failureThreshold: 5 + exec: + command: + - sh + - -c + - /scripts/ping_readiness_local.sh 1 + resources: + limits: + cpu: 150m + ephemeral-storage: 2Gi + memory: 192Mi + requests: + cpu: 100m + ephemeral-storage: 50Mi + memory: 128Mi + volumeMounts: + - name: scripts + mountPath: /scripts + - name: redis-data + mountPath: /bitnami/redis/data + subPath: + - name: default-config + mountPath: /opt/bitnami/redis/etc/redis-default.conf + subPath: redis-default.conf + - name: empty-dir + mountPath: /opt/bitnami/redis/etc/ + subPath: app-conf-dir + - name: empty-dir + mountPath: /opt/bitnami/redis/tmp + subPath: app-tmp-dir + - name: empty-dir + mountPath: /opt/bitnami/redis/logs + subPath: app-logs-dir + - name: empty-dir + mountPath: /tmp + subPath: tmp-dir + volumes: + - name: scripts + configMap: + name: release-name-redis-cluster-scripts + defaultMode: 0755 + - name: default-config + configMap: + name: release-name-redis-cluster-default + - name: empty-dir + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: redis-data + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: redis-cluster + spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "8Gi" diff --git a/gpulive/gitea/vm/3_apply/ing.yaml b/gpulive/gitea/vm/3_apply/ing.yaml new file mode 100644 index 0000000..edc9e88 --- /dev/null +++ b/gpulive/gitea/vm/3_apply/ing.yaml @@ -0,0 +1,26 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: ing-gitea + namespace: gitea + annotations: + nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + nginx.ingress.kubernetes.io/proxy-body-size: "0" +spec: + ingressClassName: nginx + rules: + - host: gitea.nhngpuaas.com + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: release-name-gitea-http + port: + number: 3000 + tls: + - hosts: + - "gitea.nhngpuaas.com" + secretName: nhngpuaas-ssl + diff --git a/gpulive/gitea/vm/3_apply/service.yaml b/gpulive/gitea/vm/3_apply/service.yaml new file mode 100644 index 0000000..6f43ee8 --- /dev/null +++ b/gpulive/gitea/vm/3_apply/service.yaml @@ -0,0 +1,172 @@ +# Source: gitea/charts/postgresql-ha/templates/pgpool/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-postgresql-ha-pgpool + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/version: 4.5.2 + app.kubernetes.io/component: pgpool +spec: + type: ClusterIP + sessionAffinity: None + ports: + - name: "postgresql" + port: 5432 + targetPort: postgresql + protocol: TCP + # nodePort: null + selector: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/component: pgpool +--- +# Source: gitea/charts/postgresql-ha/templates/postgresql/service-headless.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-postgresql-ha-postgresql-headless + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/version: 16.3.0 +spec: + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: false + ports: + - name: "postgresql" + port: 5432 + targetPort: postgresql + protocol: TCP + selector: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/component: postgresql + role: data +--- +# Source: gitea/charts/postgresql-ha/templates/postgresql/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-postgresql-ha-postgresql + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/version: 16.3.0 + app.kubernetes.io/component: postgresql +spec: + type: ClusterIP + ports: + - name: "postgresql" + port: 5432 + targetPort: postgresql + protocol: TCP + selector: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: postgresql-ha + app.kubernetes.io/component: postgresql + role: data +--- +# Source: gitea/charts/redis-cluster/templates/headless-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-redis-cluster-headless + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: redis-cluster + app.kubernetes.io/version: 7.2.5 +spec: + type: ClusterIP + clusterIP: None + publishNotReadyAddresses: true + ports: + - name: tcp-redis + port: 6379 + targetPort: tcp-redis + - name: tcp-redis-bus + port: 16379 + targetPort: tcp-redis-bus + selector: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: redis-cluster +--- +# Source: gitea/charts/redis-cluster/templates/redis-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-redis-cluster + namespace: gitea + labels: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: redis-cluster + app.kubernetes.io/version: 7.2.5 +spec: + type: ClusterIP + sessionAffinity: None + ports: + - name: tcp-redis + port: 6379 + targetPort: tcp-redis + protocol: TCP + nodePort: null + selector: + app.kubernetes.io/instance: release-name + app.kubernetes.io/name: redis-cluster +--- +# Source: gitea/templates/gitea/http-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-gitea-http + namespace: gitea + labels: + app: gitea + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "1.22.1" + version: "1.22.1" + # annotations: + # {} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: http + port: 3000 + targetPort: + selector: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: release-name +--- +# Source: gitea/templates/gitea/ssh-svc.yaml +apiVersion: v1 +kind: Service +metadata: + name: release-name-gitea-ssh + namespace: gitea + labels: + app: gitea + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "1.22.1" + version: "1.22.1" + # annotations: + # {} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: ssh + port: 22 + targetPort: 2222 + protocol: TCP + selector: + app.kubernetes.io/name: gitea + app.kubernetes.io/instance: release-name diff --git a/gpulive/harbor_2.9.1/2_configmap/configmap.yaml b/gpulive/harbor_2.9.1/2_configmap/configmap.yaml index 7e3df7b..1905080 100644 --- a/gpulive/harbor_2.9.1/2_configmap/configmap.yaml +++ b/gpulive/harbor_2.9.1/2_configmap/configmap.yaml @@ -132,7 +132,6 @@ data: scgi_temp_path /tmp/scgi_temp; server { listen 8080; - listen [::]:8080; server_name localhost; root /usr/share/nginx/html; index index.html index.htm; diff --git a/gpulive/harbor_2.9.1/vm/1_configure/cert.yaml b/gpulive/harbor_2.9.1/vm/1_configure/cert.yaml new file mode 100644 index 0000000..541df26 --- /dev/null +++ b/gpulive/harbor_2.9.1/vm/1_configure/cert.yaml @@ -0,0 +1,29 @@ +apiVersion: v1 +kind: Secret +metadata: + name: harbor-cacerts + namespace: harbor + # harbor-corer pod // /etc/core/ca +data: + cert: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURtekNDQW9PZ0F3SUJBZ0lVTFVnOHBsdGlSTDVqSVhzL1ppQ1pacmtncElRd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1hURUxNQWtHQTFVRUJoTUNTMUl4RGpBTUJnTlZCQWdNQlZObGIzVnNNUTB3Q3dZRFZRUUhEQVJKYm1wbApNUTh3RFFZRFZRUUtEQVpwYm1OdmNuQXhEREFLQmdOVkJBc01BMlJsZGpFUU1BNEdBMVVFQXd3SGFXNXFaV2x1Cll6QWVGdzB5TkRFeU1ESXdORFEwTlRkYUZ3MHlOakV5TURJd05EUTBOVGRhTUYweEN6QUpCZ05WQkFZVEFrdFMKTVE0d0RBWURWUVFJREFWVFpXOTFiREVOTUFzR0ExVUVCd3dFU1c1cVpURVBNQTBHQTFVRUNnd0dhVzVqYjNKdwpNUXd3Q2dZRFZRUUxEQU5rWlhZeEVEQU9CZ05WQkFNTUIybHVhbVZwYm1Nd2dnRWlNQTBHQ1NxR1NJYjNEUUVCCkFRVUFBNElCRHdBd2dnRUtBb0lCQVFERmVDRU1iM3Yrbmt1OUJic1FVdUt5dGJabUN5MTdPcDZSTGxmWEZaSTAKbUxLRjZadTlTczh0cmV0VDA3eHZrY2x2YzF4b2ZJMlFWMVFmSkFDQmhqU0FaWUJTbWZhNE5oWExnNEYvcEFSQQpjbStINnR2TGtSNllvZVMvWXVJUDVQVXVxWmt6Ym9ZQWl5T0NUelYxNEhRb25OU20rOFZvSnNMemEwZEZZZklNCmtGOVVsMklFSERmQ2xrQWt1cDNFYlNyMkNTYXdLV3J4OXd2N2cvbXR3ekloL0MwKy81MXh2dFZSSVVUZk1qQlIKUVVWL1VBWEJhS0UvNHlwN1RZaWpHa21WUEJUVTRid2hmcHdNeklUMWxpSjhwaE1qTGZtbDMxd1Q5R1FWYnhyMgpXOU5NdnFQWklMVmx6c1VUcTJyOU13TU8rQ3VreVNWbWRBOXgwenc1MUNGZkFnTUJBQUdqVXpCUk1CMEdBMVVkCkRnUVdCQlFsTWo0YzFmMjg1bmlHREJRcmNlYUp5MDJtalRBZkJnTlZIU01FR0RBV2dCUWxNajRjMWYyODVuaUcKREJRcmNlYUp5MDJtalRBUEJnTlZIUk1CQWY4RUJUQURBUUgvTUEwR0NTcUdTSWIzRFFFQkN3VUFBNElCQVFBTgpBZ0R1OEY3SGVWakZ4bU11NGZZVWZrN0czU1ZObWdacXo0MzhNanhjS2dJYmR6NnE1Q0xTNW1hSng0RitVc2VPCjZaZHBEQm1GSncrK1E4QVUvQ0tLY3dOZC83eGRzSEhEUFY0MFNNVStLc08xT0FJZjBIeFZXV3lFd0VRaW1obFgKWEJBZVJocEF5NHFTNTI1dE11RnUxeFkxOVJJME1VRlpmV05JUW1OZ0N5QWRINkIyY3Z5dmhSU2crRXdhYnV6RQpkdmN6R3JOZm1tUGphMXRZTnBQWGFpQ0hGc2h2QzlJMzZwOEhuYjNKY3VsVSswL2Y3UzRWZGhSTm1nakY1elhMCkl6Wk1TbXhnMnVmc25DNHFhS3FONU9ISi9NbGJIYTRzRFR0cHNLaWNBWUE3aUw2WjFvcmtIeTc1cUdYVlRtekMKU2xkbkxwQWw2cS9wUXg1bUpVdEwKLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQ== +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + app: harbor + release: harbor + name: harbor-core + namespace: harbor +type: Opaque +data: + CSRF_KEY: Tk9ORjNTYjZoZ2tIdlZmcXNlcTVEdjNhUlBjWDZ6Rjg= + HARBOR_ADMIN_PASSWORD: bmhuIUAjMTIz + # POSTGRES_PASSWORD --> init파일에 있는 것. database에서 사용하는 변수명 + POSTGRESQL_PASSWORD: bmhuIUAjMTIz # harbor-core에서 사용하는 변수명 + REGISTRY_CREDENTIAL_PASSWORD: aGFyYm9yX3JlZ2lzdHJ5X3Bhc3N3b3Jk + secret: RTRQY0FxZ3RPVmdKWTBRRw== + secretKey: bm90LWEtc2VjdXJlLWtleQ== + tls.cert: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSURyekNDQXBlZ0F3SUJBZ0lVVzZZZlZMdkhUVEZ6ZU40R05wbmgxZW1XcXpRd0RRWUpLb1pJaHZjTkFRRUwKQlFBd1hURUxNQWtHQTFVRUJoTUNTMUl4RGpBTUJnTlZCQWdNQlZObGIzVnNNUTB3Q3dZRFZRUUhEQVJKYm1wbApNUTh3RFFZRFZRUUtEQVpwYm1OdmNuQXhEREFLQmdOVkJBc01BMlJsZGpFUU1BNEdBMVVFQXd3SGFXNXFaV2x1Cll6QWVGdzB5TkRFeU1ESXdOakV3TlRkYUZ3MHlOakV5TURJd05qRXdOVGRhTUdVeEN6QUpCZ05WQkFZVEFrdFMKTVE0d0RBWURWUVFJREFWVFpXOTFiREVPTUF3R0ExVUVCd3dGUjBGVFFVNHhEREFLQmdOVkJBb01BMlJsWWpFTgpNQXNHQTFVRUN3d0VkR1Z6ZERFWk1CY0dBMVVFQXd3UUtpNXVZWFJwZG1Wa1pXTnJMbU52YlRDQ0FTSXdEUVlKCktvWklodmNOQVFFQkJRQURnZ0VQQURDQ0FRb0NnZ0VCQU11U1JJVUJBMWhSWmd4M3p4djVJeWljMnd3alZUcmkKa3QrQ01CNndTLzQrVDUxMkxnd29PQU9IdnQ1L0hDbEVSUVdWd280ZDN2cDR3WmQ3MktxVXM2Ym9CclZmeWU4bAo1bC84MkpGQkhqNStGWDhtTS8wYldkMHVhV1NVNDdsNHhBTzFwU3V0cXgxdkJQdlI2aC9DT0k4cEpBakVxd1llCjVLU1RGWllKOG5YWXJrbkFhRnl1dnkwZjhNcHdUbW1GWEZWeVZQbGlhZkhYUjFVWVNQUlFIcWo0TXF3VTM4OXMKSjdOOVlsMzBwMVUzVWVMRkx2Q0lEUU1UVFFzbUNSbnA3UzV1VzZETC90eXNwdnAwRVVzcDRHRXRhY3drZU84cwo0UlZqeGEyR0JiUTViSGJ2MGxZLzZBKzk5Szc5U0ZNcksvd1ptM0dSeDkvblJKWXc2OUpLZ2xVQ0F3RUFBYU5mCk1GMHdHd1lEVlIwUkJCUXdFb0lRS2k1dVlYUnBkbVZrWldOckxtTnZiVEFkQmdOVkhRNEVGZ1FVbitzL2oyMzEKcC8ycnFWWFNTSk5IVWZnNU5PQXdId1lEVlIwakJCZ3dGb0FVSlRJK0hOWDl2T1o0aGd3VUszSG1pY3ROcG8wdwpEUVlKS29aSWh2Y05BUUVMQlFBRGdnRUJBR2haQVNobU05Mm9yVHo5ZndtOG9qZ25xekZ5dmlGa2lrYjU2MDI4CmZEZXV5M2d2aUZqNHk5a3hlQ1lqdjQ4ZWJ1cHlFL1oyZnJ5eVltWXNOOHNlanNLcTY2V0J1TDh0ZG9TS3ZjcFMKQU1TOU5reEhWZlJvRmFGSmgxZVQ3RzhoYVd0dTdpSUxRUzB5dXZWYkFhMGFPVzRpY3p0eXVlK0MyOXRXS3pLcAp4ZndtOEI5SzFzRVUwdEN2d1R6L2wwTlBpYXdOelNnc3UrWFkwbW1xZ0NQbG1zd2cxRkNwWTUzcHBqRE5NTzJDCnJ5bGhKbnowM2JpS2NQb0VCQlNKZFo2WGt2UG93SUV5bGZ3SWlEV3VNRjFaNzViaTBoaGFFTjRDaVJkeSt4MDkKR1NQa04zMlZ4d3dvTXhHd3A0VWJSS0g4YVR6Q2VWT0ZuaE8xbGp3MVdSR2ZRUWc9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0= + tls.key: LS0tLS1CRUdJTiBSU0EgUFJJVkFURSBLRVktLS0tLQpNSUlFb3dJQkFBS0NBUUVBeTVKRWhRRURXRkZtREhmUEcva2pLSnpiRENOVk91S1MzNEl3SHJCTC9qNVBuWFl1CkRDZzRBNGUrM244Y0tVUkZCWlhDamgzZStuakJsM3ZZcXBTenB1Z0d0Vi9KN3lYbVgvellrVUVlUG40VmZ5WXoKL1J0WjNTNXBaSlRqdVhqRUE3V2xLNjJySFc4RSs5SHFIOEk0anlra0NNU3JCaDdrcEpNVmxnbnlkZGl1U2NCbwpYSzYvTFIvd3luQk9hWVZjVlhKVStXSnA4ZGRIVlJoSTlGQWVxUGd5ckJUZnoyd25zMzFpWGZTblZUZFI0c1V1CjhJZ05BeE5OQ3lZSkdlbnRMbTVib012KzNLeW0rblFSU3luZ1lTMXB6Q1I0N3l6aEZXUEZyWVlGdERsc2R1L1MKVmovb0Q3MzBydjFJVXlzci9CbWJjWkhIMytkRWxqRHIwa3FDVlFJREFRQUJBb0lCQUZaMGV5Y2xZLzlKS3BiVwo5eDJCTkY1V0oyMURRRG91NllPRTdkektzS3Q4V1NHZGhEYmppYzV1QXpESk9QT0pxK3FRVEwxQWtHYzlpN0pECkJhU0VYTHZneEtDWTJ0RDIvcHo3YkJwY1ZuVGg1VTFIczBLbU1BY1FZL0EwajNaQWhMYXltVUF3bElCK210WTcKc3p2anAxT2RmRkZBL1FVdmI1azVZaXlEY3ZaNjJwL2VaNXdDOXY4dFhjRVpIN2FpVWNKSGM1ajFrRFRpckErawpjWW5xUTJ2NHovWXJKek9Xa2tyTTNudFpOSzM0QzMrMUQrb1d6QjZHNjJJb0JCZnVxVGNJZHgwMXNvbjNIMXVYClpMNHdNbXRLZ0Z0RFJLdEpQNysxOXJsaU1qL09UWEd3WkZlS2c1RGY3T2Y2MFl1em1kVXFhNHQrMXlicHd2RkEKb0U2OW15MENnWUVBM09wcFV1RmZwZUxiRGsvdHJGY3ByNksxaTdubFA0TXowMXhySG03d3VEaS80TWp6VnlkVworZ0t0emYzRm1rNE5iYyt6aGdXblIzN2RheFlzbzVacC9Uck9GemFuNGpIKzhzNU9LWDI2WjBJZjVnQm01NmwzCmtKckU3UnAxMFF3ZE0xRkVQOWtGaFZyK2YzWm5rQ3QyNjQ3aFVWY2Jubk1CUHhZVkNWZzZmOE1DZ1lFQTYrYXoKNG12Y1dSZm1LSFd1dnovRVZPL3lRT3BxOCtkc2l3dXFRZ2xIWm5SZ0Rmc05uZ3NyK3hBaWJoVm94Q3ZZdm84dApnNVVqdXArVklEVFVqRlp6bkd1N2xVWlRyMVBQS1lCS2dtSVVqRkpYQ1JCUUd5akttUG5LYmhJb2RxUWRCV2o3CkIrOTNDeVBsMHRZVWNNMjdEL0E5TDNDaHBJVktxZ25JeHpZZXJBY0NnWUJqS1JFWHM3UUVPUHpNd3RIVFB0aW0KVHFtZ3kxTjhtNzdpaTZSRlo5VERUQzNNajBhekF5U0dXamQ3dEFKRGplOTNML2pNR3JPSVl3aXVMYWpoMFAxSQpQbUlOaDJuNkhTanNBZkNSclM3RGRLbnV5cFZtUE9vL1RVbEJpRlFEL3FVSXlOeVNzQzVMT3Bucnl0RVhwSHJmCnB5VzZKaGh1bWY2Z082QTdybytya3dLQmdIOXVEam4vWGZaYXJKK3FnbzBaR3l0dXVvUktuUi9JSWFBeTJ2UDEKSXphN0lyQXFyNzV4OW40V3h5S0VPaVhPTWJVdWpyMkhzTmM5SVVkV2dvS1VxTHNVank0aDRVN1NqUk8wRTR1LwpBNjZSUkNiSFF4Nkl1SGhPWndWeWxaMFJKZ1FvSTRLejNyQ0w0Ymcwb0xJVzBEZlo1Sk5kMmxPUEZUZ3BOT1kzCjIvVzlBb0dCQU1HRXJtczVrRWZkdDFveXYxUlBwbENSUG1yVCs4TUV1K3lyanJXSXlxWEo4SHcwd09mTlhsTXQKUzhRbFdDb0xnbmFxZU81ZWd6OGF2L2ZVNDhxbU9tTVFPUDRMbWxSSm1ZM1E3MVpCZUc5akxPUFZCZlhFeDVDTQp2d283dFhEU1JKRVdUcGp3dCtUMy9uTlZhMm9QMGNXTUg1akJ0c1ZXMnhrM3F5K0NJWVBqCi0tLS0tRU5EIFJTQSBQUklWQVRFIEtFWS0tLS0t +--- diff --git a/gpulive/harbor_2.9.1/vm/1_configure/init.yaml b/gpulive/harbor_2.9.1/vm/1_configure/init.yaml new file mode 100644 index 0000000..58708d4 --- /dev/null +++ b/gpulive/harbor_2.9.1/vm/1_configure/init.yaml @@ -0,0 +1,83 @@ +apiVersion: v1 +kind: Namespace +metadata: + labels: + kubernetes.io/metadata.name: harbor + name: harbor +spec: + finalizers: + - kubernetes +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + app: harbor + release: harbor + name: harbor-database + namespace: harbor +type: Opaque +data: + POSTGRES_PASSWORD: bmhuIUAjMTIz +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + app: harbor + release: harbor + name: harbor-jobservice + namespace: harbor +type: Opaque +data: + JOBSERVICE_SECRET: akVWbzlyNFJaU2R1clJqUA== + REGISTRY_CREDENTIAL_PASSWORD: aGFyYm9yX3JlZ2lzdHJ5X3Bhc3N3b3Jk +--- +apiVersion: v1 +data: + REGISTRY_HTTP_SECRET: dHlCdm41MXpIakxLRm5mRw== + REGISTRY_REDIS_PASSWORD: "" +kind: Secret +metadata: + labels: + app: harbor + release: harbor + name: harbor-registry + namespace: harbor +type: Opaque +--- +apiVersion: v1 +data: + REGISTRY_HTPASSWD: aGFyYm9yX3JlZ2lzdHJ5X3VzZXI6JDJhJDEwJG1uRkVSQkVyRmROYnNuZDVHWUxTQXVUOTRaUWxZNGkxU3BtNTk4dWlmV2pLbEJrQnJhbHEu +kind: Secret +metadata: + labels: + app: harbor + release: harbor + name: harbor-registry-htpasswd + namespace: harbor +type: Opaque +--- +apiVersion: v1 +data: null +kind: Secret +metadata: + labels: + app: harbor + release: harbor + name: harbor-registryctl + namespace: harbor +type: Opaque +--- +apiVersion: v1 +data: + gitHubToken: "" + redisURL: cmVkaXM6Ly9oYXJib3ItcmVkaXM6NjM3OS81P2lkbGVfdGltZW91dF9zZWNvbmRzPTMw +kind: Secret +metadata: + labels: + app: harbor + release: harbor + name: harbor-trivy + namespace: harbor +type: Opaque diff --git a/gpulive/harbor_2.9.1/vm/1_configure/pv.yaml b/gpulive/harbor_2.9.1/vm/1_configure/pv.yaml new file mode 100644 index 0000000..0d24046 --- /dev/null +++ b/gpulive/harbor_2.9.1/vm/1_configure/pv.yaml @@ -0,0 +1,90 @@ +apiVersion: v1 +kind: PersistentVolume +metadata: + name: harbor-jobservice-pv + labels: + name: harbor-jobservice-pv +spec: + accessModes: + - ReadWriteMany + capacity: + storage: 50Gi + csi: + driver: exa.csi.ddn.com + volumeHandle: exa1:/harbor-job + volumeAttributes: # volumeAttributes are the alternative of storageClass params for static (precreated) volumes. + # mountOptions: ro, noflock # list of options for `mount` command + bindMount: "false" # Determines, whether volume will bind mounted or as a separate lustre mount. +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: harbor-registry-pv + labels: + name: harbor-registry-pv +spec: + accessModes: + - ReadWriteMany + capacity: + storage: 2000Gi + csi: + driver: exa.csi.ddn.com + volumeHandle: exa1:/harbor-registry + volumeAttributes: # volumeAttributes are the alternative of storageClass params for static (precreated) volumes. + # mountOptions: ro, noflock # list of options for `mount` command + bindMount: "false" # Determines, whether volume will bind mounted or as a separate lustre mount. +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: data-harbor-redis-0 + labels: + name: data-harbor-redis-0 +spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 50Gi + csi: + driver: exa.csi.ddn.com + volumeHandle: exa1:/harbor_redis + volumeAttributes: # volumeAttributes are the alternative of storageClass params for static (precreated) volumes. + # mountOptions: ro, noflock # list of options for `mount` command + bindMount: "false" # Determines, whether volume will bind mounted or as a separate lustre mount. +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: data-harbor-trivy-0 + labels: + name: data-harbor-trivy-0 +spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 200Gi + csi: + driver: exa.csi.ddn.com + volumeHandle: exa1:/harbor_trivy + volumeAttributes: # volumeAttributes are the alternative of storageClass params for static (precreated) volumes. + # mountOptions: ro, noflock # list of options for `mount` command + bindMount: "false" # Determines, whether volume will bind mounted or as a separate lustre mount. +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: database-data-harbor-database-0 + labels: + name: database-data-harbor-database-0 +spec: + accessModes: + - ReadWriteOnce + capacity: + storage: 50Gi + csi: + driver: exa.csi.ddn.com + volumeHandle: exa1:/harbor_database + volumeAttributes: # volumeAttributes are the alternative of storageClass params for static (precreated) volumes. + # mountOptions: ro, noflock # list of options for `mount` command + bindMount: "false" # Determines, whether volume will bind mounted or as a separate lustre mount. + diff --git a/gpulive/harbor_2.9.1/vm/1_configure/pvc.yaml b/gpulive/harbor_2.9.1/vm/1_configure/pvc.yaml new file mode 100644 index 0000000..06de65c --- /dev/null +++ b/gpulive/harbor_2.9.1/vm/1_configure/pvc.yaml @@ -0,0 +1,43 @@ +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app: harbor + component: jobservice + release: harbor + name: harbor-jobservice + namespace: harbor +spec: + accessModes: + - ReadWriteMany + # storageClassName: sc-monitoring + resources: + requests: + storage: 10Gi + selector: + matchLabels: + # to create 1-1 relationship for pod - persistent volume use unique labels + name: harbor-jobservice-pv +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app: harbor + component: registry + release: harbor + name: harbor-registry + namespace: harbor +spec: + # storageClassName: sc-monitoring + accessModes: + - ReadWriteMany + resources: + requests: + storage: 100Gi + selector: + matchLabels: + # to create 1-1 relationship for pod - persistent volume use unique labels + name: harbor-registry-pv + diff --git a/gpulive/harbor_2.9.1/vm/2_configmap/configmap.yaml b/gpulive/harbor_2.9.1/vm/2_configmap/configmap.yaml new file mode 100644 index 0000000..54750d2 --- /dev/null +++ b/gpulive/harbor_2.9.1/vm/2_configmap/configmap.yaml @@ -0,0 +1,226 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: harbor + release: harbor + name: harbor-core + namespace: harbor +data: + _REDIS_URL_CORE: redis://harbor-redis:6379/0?idle_timeout_seconds=30 + _REDIS_URL_REG: redis://harbor-redis:6379/2?idle_timeout_seconds=30 + CHART_CACHE_DRIVER: redis + CONFIG_PATH: /etc/core/app.conf + CORE_LOCAL_URL: http://127.0.0.1:8080 + CORE_URL: http://harbor-core:80 + DATABASE_TYPE: postgresql + EXT_ENDPOINT: https://registry.gpulive.nhncloud.com + HTTP_PROXY: "" + HTTPS_PROXY: "" + JOBSERVICE_URL: http://harbor-jobservice + LOG_LEVEL: info + NO_PROXY: harbor-core,harbor-jobservice,harbor-database,harbor-registry,harbor-portal,harbor-trivy,harbor-exporter,127.0.0.1,localhost,.local,.internal + PERMITTED_REGISTRY_TYPES_FOR_PROXY_CACHE: docker-hub,harbor,azure-acr,aws-ecr,google-gcr,quay,docker-registry,github-ghcr,jfrog-artifactory + PORT: "8080" + PORTAL_URL: http://harbor-portal + POSTGRESQL_DATABASE: registry + POSTGRESQL_HOST: harbor-database + POSTGRESQL_MAX_IDLE_CONNS: "100" + POSTGRESQL_MAX_OPEN_CONNS: "900" + POSTGRESQL_PORT: "5432" + POSTGRESQL_SSLMODE: disable + POSTGRESQL_USERNAME: postgres + QUOTA_UPDATE_PROVIDER: db + REGISTRY_CONTROLLER_URL: http://harbor-registry:8080 + REGISTRY_CREDENTIAL_USERNAME: harbor_registry_user + REGISTRY_STORAGE_PROVIDER_NAME: filesystem + REGISTRY_URL: http://harbor-registry:5000 + TOKEN_SERVICE_URL: http://harbor-core:80/service/token + TRIVY_ADAPTER_URL: http://harbor-trivy:8080 + WITH_TRIVY: "true" + app.conf: | + appname = Harbor + runmode = prod + enablegzip = true + + [prod] + httpport = 8080 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: harbor + release: harbor + name: harbor-jobservice + namespace: harbor +data: + config.yml: | + #Server listening port + protocol: "http" + port: 8080 + worker_pool: + workers: 10 + backend: "redis" + redis_pool: + redis_url: "redis://harbor-redis:6379/1" + namespace: "harbor_job_service_namespace" + idle_timeout_second: 3600 + job_loggers: + - name: "FILE" + level: INFO + settings: # Customized settings of logger + base_dir: "/var/log/jobs" + sweeper: + duration: 14 #days + settings: # Customized settings of sweeper + work_dir: "/var/log/jobs" + metric: + enabled: false + path: /metrics + port: 8001 + #Loggers for the job service + loggers: + - name: "STD_OUTPUT" + level: INFO + reaper: + # the max time to wait for a task to finish, if unfinished after max_update_hours, the task will be mark as error, but the task will continue to run, default value is 24 + max_update_hours: 24 + # the max time for execution in running state without new task created + max_dangling_hours: 168 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: harbor + release: harbor + name: harbor-jobservice-env + namespace: harbor +data: + CORE_URL: http://harbor-core:80 + HTTP_PROXY: "" + HTTPS_PROXY: "" + JOBSERVICE_WEBHOOK_JOB_HTTP_CLIENT_TIMEOUT: "3" + JOBSERVICE_WEBHOOK_JOB_MAX_RETRY: "3" + NO_PROXY: harbor-core,harbor-jobservice,harbor-database,harbor-registry,harbor-portal,harbor-trivy,harbor-exporter,127.0.0.1,localhost,.local,.internal + REGISTRY_CONTROLLER_URL: http://harbor-registry:8080 + REGISTRY_CREDENTIAL_USERNAME: harbor_registry_user + REGISTRY_URL: http://harbor-registry:5000 + TOKEN_SERVICE_URL: http://harbor-core:80/service/token +--- +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: harbor + release: harbor + name: harbor-portal + namespace: harbor +data: + nginx.conf: | + worker_processes auto; + pid /tmp/nginx.pid; + events { + worker_connections 1024; + } + http { + client_body_temp_path /tmp/client_body_temp; + proxy_temp_path /tmp/proxy_temp; + fastcgi_temp_path /tmp/fastcgi_temp; + uwsgi_temp_path /tmp/uwsgi_temp; + scgi_temp_path /tmp/scgi_temp; + server { + listen 8080; + server_name localhost; + root /usr/share/nginx/html; + index index.html index.htm; + include /etc/nginx/mime.types; + gzip on; + gzip_min_length 1000; + gzip_proxied expired no-cache no-store private auth; + gzip_types text/plain text/css application/json application/javascript application/x-javascript text/xml application/xml application/xml+rss text/javascript; + location /devcenter-api-2.0 { + try_files $uri $uri/ /swagger-ui-index.html; + } + location / { + try_files $uri $uri/ /index.html; + } + location = /index.html { + add_header Cache-Control "no-store, no-cache, must-revalidate"; + } + } + } +--- +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: harbor + release: harbor + name: harbor-registry + namespace: harbor +data: + config.yml: | + version: 0.1 + log: + level: info + fields: + service: registry + storage: + filesystem: + rootdirectory: /storage + cache: + layerinfo: redis + maintenance: + uploadpurging: + enabled: true + age: 168h + interval: 24h + dryrun: false + delete: + enabled: true + redirect: + disable: false + redis: + addr: harbor-redis:6379 + db: 2 + readtimeout: 10s + writetimeout: 10s + dialtimeout: 10s + pool: + maxidle: 100 + maxactive: 500 + idletimeout: 60s + http: + addr: :5000 + #relativeurls: false + relativeurls: true + # set via environment variable + # secret: placeholder + debug: + addr: localhost:5001 + auth: + htpasswd: + realm: harbor-registry-basic-realm + path: /etc/registry/passwd + validation: + disabled: true + compatibility: + schema1: + enabled: true + ctl-config.yml: | + --- + protocol: "http" + port: 8080 + log_level: info + registry_config: "/etc/registry/config.yml" +--- +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app: harbor + release: harbor + name: harbor-registryctl + namespace: harbor diff --git a/gpulive/harbor_2.9.1/vm/3_apply/deploy.yaml b/gpulive/harbor_2.9.1/vm/3_apply/deploy.yaml new file mode 100644 index 0000000..0e8e4a3 --- /dev/null +++ b/gpulive/harbor_2.9.1/vm/3_apply/deploy.yaml @@ -0,0 +1,804 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: harbor + release: harbor + name: harbor-core + namespace: harbor +spec: + ports: + - name: http-web + port: 80 + targetPort: 8080 + selector: + app: harbor + component: core + release: harbor +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: harbor + release: harbor + name: harbor-database + namespace: harbor +spec: + ports: + - port: 5432 + selector: + app: harbor + component: database + release: harbor +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: harbor + release: harbor + name: harbor-jobservice + namespace: harbor +spec: + ports: + - name: http-jobservice + port: 80 + targetPort: 8080 + selector: + app: harbor + component: jobservice + release: harbor +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: harbor + release: harbor + name: harbor-portal + namespace: harbor +spec: + ports: + - port: 80 + targetPort: 8080 + selector: + app: harbor + component: portal + release: harbor +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: harbor + release: harbor + name: harbor-redis + namespace: harbor +spec: + ports: + - port: 6379 + selector: + app: harbor + component: redis + release: harbor +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: harbor + release: harbor + name: harbor-registry + namespace: harbor +spec: + ports: + - name: http-registry + port: 5000 + - name: http-controller + port: 8080 + selector: + app: harbor + component: registry + release: harbor +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app: harbor + release: harbor + name: harbor-trivy + namespace: harbor +spec: + ports: + - name: http-trivy + port: 8080 + protocol: TCP + selector: + app: harbor + component: trivy + release: harbor +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: harbor + component: core + release: harbor + name: harbor-core + namespace: harbor +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: harbor + component: core + release: harbor + template: + metadata: + labels: + app: harbor + component: core + release: harbor + spec: + nodeSelector: + nodegroup: nd + automountServiceAccountToken: false + containers: + - env: + - name: CORE_SECRET + valueFrom: + secretKeyRef: + key: secret + name: harbor-core + - name: JOBSERVICE_SECRET + valueFrom: + secretKeyRef: + key: JOBSERVICE_SECRET + name: harbor-jobservice + envFrom: + - configMapRef: + name: harbor-core + - secretRef: + name: harbor-core + image: goharbor/harbor-core:v2.11.2 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 2 + httpGet: + path: /api/v2.0/ping + port: 8080 + scheme: HTTP + periodSeconds: 10 + name: core + ports: + - containerPort: 8080 + readinessProbe: + failureThreshold: 2 + httpGet: + path: /api/v2.0/ping + port: 8080 + scheme: HTTP + periodSeconds: 10 + startupProbe: + failureThreshold: 360 + httpGet: + path: /api/v2.0/ping + port: 8080 + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + volumeMounts: + - mountPath: /etc/core/app.conf + name: config + subPath: app.conf + - mountPath: /etc/core/key + name: secret-key + subPath: key + - mountPath: /etc/core/private_key.pem + name: token-service-private-key + subPath: tls.key + - mountPath: /etc/core/ca + name: ca-download + - mountPath: /etc/core/token + name: psc + securityContext: + fsGroup: 10000 + runAsUser: 10000 + terminationGracePeriodSeconds: 120 + volumes: + - configMap: + items: + - key: app.conf + path: app.conf + name: harbor-core + name: config + - name: secret-key + secret: + items: + - key: secretKey + path: key + secretName: harbor-core + - name: token-service-private-key + secret: + secretName: harbor-core + - name: ca-download + secret: + secretName: harbor-cacerts + - emptyDir: {} + name: psc +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: harbor + component: jobservice + release: harbor + name: harbor-jobservice + namespace: harbor +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: harbor + component: jobservice + release: harbor + strategy: + type: RollingUpdate + template: + metadata: + labels: + app: harbor + component: jobservice + release: harbor + spec: + nodeSelector: + nodegroup: nd + automountServiceAccountToken: false + containers: + - env: + - name: CORE_SECRET + valueFrom: + secretKeyRef: + key: secret + name: harbor-core + envFrom: + - configMapRef: + name: harbor-jobservice-env + - secretRef: + name: harbor-jobservice + image: goharbor/harbor-jobservice:v2.11.2 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /api/v1/stats + port: 8080 + scheme: HTTP + initialDelaySeconds: 300 + periodSeconds: 10 + name: jobservice + ports: + - containerPort: 8080 + readinessProbe: + httpGet: + path: /api/v1/stats + port: 8080 + scheme: HTTP + initialDelaySeconds: 20 + periodSeconds: 10 + volumeMounts: + - mountPath: /etc/jobservice/config.yml + name: jobservice-config + subPath: config.yml + - mountPath: /var/log/jobs + name: job-logs + subPath: null + securityContext: + fsGroup: 10000 + runAsUser: 10000 + terminationGracePeriodSeconds: 120 + volumes: + - configMap: + name: harbor-jobservice + name: jobservice-config + - name: job-logs + persistentVolumeClaim: + claimName: harbor-jobservice +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: harbor + component: portal + release: harbor + name: harbor-portal + namespace: harbor +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: harbor + component: portal + release: harbor + template: + metadata: + labels: + app: harbor + component: portal + release: harbor + spec: + nodeSelector: + nodegroup: nd + automountServiceAccountToken: false + containers: + - image: goharbor/harbor-portal:v2.11.2 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: / + port: 8080 + scheme: HTTP + initialDelaySeconds: 300 + periodSeconds: 10 + name: portal + ports: + - containerPort: 8080 + readinessProbe: + httpGet: + path: / + port: 8080 + scheme: HTTP + initialDelaySeconds: 1 + periodSeconds: 10 + volumeMounts: + - mountPath: /etc/nginx/nginx.conf + name: portal-config + subPath: nginx.conf + securityContext: + fsGroup: 10000 + runAsUser: 10000 + volumes: + - configMap: + name: harbor-portal + name: portal-config +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: harbor + component: registry + release: harbor + name: harbor-registry + namespace: harbor +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app: harbor + component: registry + release: harbor + strategy: + type: RollingUpdate + template: + metadata: + labels: + app: harbor + component: registry + release: harbor + spec: + nodeSelector: + nodegroup: nd + automountServiceAccountToken: false + containers: + - args: + - serve + - /etc/registry/config.yml + env: null + envFrom: + - secretRef: + name: harbor-registry + image: goharbor/registry-photon:v2.11.2 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: / + port: 5000 + scheme: HTTP + initialDelaySeconds: 300 + periodSeconds: 10 + name: registry + ports: + - containerPort: 5000 + - containerPort: 5001 + readinessProbe: + httpGet: + path: / + port: 5000 + scheme: HTTP + initialDelaySeconds: 1 + periodSeconds: 10 + volumeMounts: + - mountPath: /storage + name: registry-data + subPath: null + - mountPath: /etc/registry/passwd + name: registry-htpasswd + subPath: passwd + - mountPath: /etc/registry/config.yml + name: registry-config + subPath: config.yml + - env: + - name: CORE_SECRET + valueFrom: + secretKeyRef: + key: secret + name: harbor-core + - name: JOBSERVICE_SECRET + valueFrom: + secretKeyRef: + key: JOBSERVICE_SECRET + name: harbor-jobservice + envFrom: + - configMapRef: + name: harbor-registryctl + - secretRef: + name: harbor-registry + - secretRef: + name: harbor-registryctl + image: goharbor/harbor-registryctl:v2.11.2 + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /api/health + port: 8080 + scheme: HTTP + initialDelaySeconds: 300 + periodSeconds: 10 + name: registryctl + ports: + - containerPort: 8080 + readinessProbe: + httpGet: + path: /api/health + port: 8080 + scheme: HTTP + initialDelaySeconds: 1 + periodSeconds: 10 + volumeMounts: + - mountPath: /storage + name: registry-data + subPath: null + - mountPath: /etc/registry/config.yml + name: registry-config + subPath: config.yml + - mountPath: /etc/registryctl/config.yml + name: registry-config + subPath: ctl-config.yml + securityContext: + fsGroup: 10000 + fsGroupChangePolicy: OnRootMismatch + runAsUser: 10000 + terminationGracePeriodSeconds: 120 + volumes: + - name: registry-htpasswd + secret: + items: + - key: REGISTRY_HTPASSWD + path: passwd + secretName: harbor-registry-htpasswd + - configMap: + name: harbor-registry + name: registry-config + - name: registry-data + persistentVolumeClaim: + claimName: harbor-registry +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app: harbor + component: database + release: harbor + name: harbor-database + namespace: harbor +spec: + replicas: 1 + selector: + matchLabels: + app: harbor + component: database + release: harbor + serviceName: harbor-database + template: + metadata: + labels: + app: harbor + component: database + release: harbor + spec: + nodeSelector: + nodegroup: nd + automountServiceAccountToken: false + containers: + - env: + - name: PGDATA + value: /var/lib/postgresql/data/pgdata + envFrom: + - secretRef: + name: harbor-database + image: goharbor/harbor-db:v2.11.2 + imagePullPolicy: IfNotPresent + livenessProbe: + exec: + command: + - /docker-healthcheck.sh + initialDelaySeconds: 300 + periodSeconds: 10 + timeoutSeconds: 1 + name: database + readinessProbe: + exec: + command: + - /docker-healthcheck.sh + initialDelaySeconds: 1 + periodSeconds: 10 + timeoutSeconds: 1 + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: database-data + - mountPath: /dev/shm + name: shm-volume + initContainers: + - args: + - -c + - '[ -e /var/lib/postgresql/data/postgresql.conf ] && [ ! -d /var/lib/postgresql/data/pgdata + ] && mkdir -m 0700 /var/lib/postgresql/data/pgdata && mv /var/lib/postgresql/data/* + /var/lib/postgresql/data/pgdata/ || true' + command: + - /bin/sh + image: goharbor/harbor-db:v2.11.2 + imagePullPolicy: IfNotPresent + name: data-migrator + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: database-data + - args: + - -c + - chmod -R 700 /var/lib/postgresql/data/pgdata || true + command: + - /bin/sh + image: goharbor/harbor-db:v2.11.2 + imagePullPolicy: IfNotPresent + name: data-permissions-ensurer + volumeMounts: + - mountPath: /var/lib/postgresql/data + name: database-data + securityContext: + fsGroup: 999 + runAsUser: 999 + terminationGracePeriodSeconds: 120 + volumes: + - emptyDir: + medium: Memory + sizeLimit: 512Mi + name: shm-volume + volumeClaimTemplates: + - metadata: + annotations: null + labels: + app: harbor + release: harbor + name: database-data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app: harbor + component: redis + release: harbor + name: harbor-redis + namespace: harbor +spec: + replicas: 1 + selector: + matchLabels: + app: harbor + component: redis + release: harbor + serviceName: harbor-redis + template: + metadata: + labels: + app: harbor + component: redis + release: harbor + spec: + nodeSelector: + nodegroup: nd + automountServiceAccountToken: false + containers: + - image: goharbor/redis-photon:v2.11.2 + imagePullPolicy: IfNotPresent + livenessProbe: + initialDelaySeconds: 300 + periodSeconds: 10 + tcpSocket: + port: 6379 + name: redis + readinessProbe: + initialDelaySeconds: 1 + periodSeconds: 10 + tcpSocket: + port: 6379 + volumeMounts: + - mountPath: /var/lib/redis + name: data + securityContext: + fsGroup: 999 + runAsUser: 999 + terminationGracePeriodSeconds: 120 + volumeClaimTemplates: + - metadata: + annotations: null + labels: + app: harbor + release: harbor + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 10Gi +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + labels: + app: harbor + component: trivy + release: harbor + name: harbor-trivy + namespace: harbor +spec: + replicas: 1 + selector: + matchLabels: + app: harbor + component: trivy + release: harbor + serviceName: harbor-trivy + template: + metadata: + labels: + app: harbor + component: trivy + release: harbor + spec: + nodeSelector: + nodegroup: nd + automountServiceAccountToken: false + containers: + - env: + - name: HTTP_PROXY + value: "" + - name: HTTPS_PROXY + value: "" + - name: NO_PROXY + value: harbor-core,harbor-jobservice,harbor-database,harbor-registry,harbor-portal,harbor-trivy,harbor-exporter,127.0.0.1,localhost,.local,.internal + - name: SCANNER_LOG_LEVEL + value: info + - name: SCANNER_TRIVY_CACHE_DIR + value: /home/scanner/.cache/trivy + - name: SCANNER_TRIVY_REPORTS_DIR + value: /home/scanner/.cache/reports + - name: SCANNER_TRIVY_DEBUG_MODE + value: "false" + - name: SCANNER_TRIVY_VULN_TYPE + value: os,library + - name: SCANNER_TRIVY_TIMEOUT + value: 5m0s + - name: SCANNER_TRIVY_GITHUB_TOKEN + valueFrom: + secretKeyRef: + key: gitHubToken + name: harbor-trivy + - name: SCANNER_TRIVY_SEVERITY + value: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL + - name: SCANNER_TRIVY_IGNORE_UNFIXED + value: "false" + - name: SCANNER_TRIVY_SKIP_UPDATE + value: "false" + - name: SCANNER_TRIVY_OFFLINE_SCAN + value: "false" + - name: SCANNER_TRIVY_SECURITY_CHECKS + value: vuln + - name: SCANNER_TRIVY_INSECURE + value: "false" + - name: SCANNER_API_SERVER_ADDR + value: :8080 + - name: SCANNER_REDIS_URL + valueFrom: + secretKeyRef: + key: redisURL + name: harbor-trivy + - name: SCANNER_STORE_REDIS_URL + valueFrom: + secretKeyRef: + key: redisURL + name: harbor-trivy + - name: SCANNER_JOB_QUEUE_REDIS_URL + valueFrom: + secretKeyRef: + key: redisURL + name: harbor-trivy + image: goharbor/trivy-adapter-photon:v2.11.2 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 10 + httpGet: + path: /probe/healthy + port: api-server + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + name: trivy + ports: + - containerPort: 8080 + name: api-server + readinessProbe: + failureThreshold: 3 + httpGet: + path: /probe/ready + port: api-server + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + resources: + limits: + cpu: 1 + memory: 1Gi + requests: + cpu: 200m + memory: 512Mi + securityContext: + allowPrivilegeEscalation: false + privileged: false + volumeMounts: + - mountPath: /home/scanner/.cache + name: data + readOnly: false + securityContext: + fsGroup: 10000 + runAsUser: 10000 + volumeClaimTemplates: + - metadata: + labels: + app: harbor + release: harbor + name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 50Gi diff --git a/gpulive/harbor_2.9.1/vm/3_apply/gateway.yaml b/gpulive/harbor_2.9.1/vm/3_apply/gateway.yaml new file mode 100644 index 0000000..c6d4df8 --- /dev/null +++ b/gpulive/harbor_2.9.1/vm/3_apply/gateway.yaml @@ -0,0 +1,65 @@ +apiVersion: networking.istio.io/v1beta1 +kind: Gateway +metadata: + name: harbor-gateway + namespace: harbor +spec: + selector: + istio: ingressgateway + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - registry.gpulive.nhncloud.com + - port: + number: 443 + name: https + protocol: HTTPS + tls: + mode: SIMPLE + credentialName: gpulive-ssl # 반드시 istio-system namespace에 존재해야 함 + hosts: + - registry.gpulive.nhncloud.com +--- +apiVersion: networking.istio.io/v1beta1 +kind: VirtualService +metadata: + name: harbor-virtualservice + namespace: harbor +spec: + hosts: + - registry.gpulive.nhncloud.com + gateways: + - harbor-gateway + http: + - match: + - uri: + exact: /v2/ + - uri: + prefix: /v2/ + - uri: + prefix: /api/ + - uri: + prefix: /service/ + - uri: + prefix: /chartrepo/ + - uri: + prefix: /c/ + route: + - destination: + host: harbor-core.harbor.svc.cluster.local + port: + number: 80 + - match: + - uri: + exact: / + - uri: + prefix: / + route: + - destination: + host: harbor-portal.harbor.svc.cluster.local + port: + number: 80 + diff --git a/gpulive/harbor_2.9.1/vm/ing.yaml b/gpulive/harbor_2.9.1/vm/ing.yaml new file mode 100644 index 0000000..d4757cc --- /dev/null +++ b/gpulive/harbor_2.9.1/vm/ing.yaml @@ -0,0 +1,64 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + ingress.kubernetes.io/proxy-body-size: "0" + nginx.ingress.kubernetes.io/proxy-body-size: "0" + #nginx.ingress.kubernetes.io/force-ssl-redirect: "true" + labels: + app: harbor + release: harbor + name: harbor-ingress + namespace: harbor +spec: + ingressClassName: nginx + tls: + - hosts: + - "registry.gpulive.nhncloud.com" + secretName: nhngpuaas-ssl + rules: + - host: registry.gpulive.nhncloud.com + http: + paths: + - backend: + service: + name: harbor-core + port: + number: 80 + path: /api/ + pathType: Prefix + - backend: + service: + name: harbor-core + port: + number: 80 + path: /service/ + pathType: Prefix + - backend: + service: + name: harbor-core + port: + number: 80 + path: /v2/ + pathType: Prefix + - backend: + service: + name: harbor-core + port: + number: 80 + path: /chartrepo/ + pathType: Prefix + - backend: + service: + name: harbor-core + port: + number: 80 + path: /c/ + pathType: Prefix + - backend: + service: + name: harbor-portal + port: + number: 80 + path: / + pathType: Prefix diff --git a/gpulive/nfs-provisioner/test-claim.yaml b/gpulive/nfs-provisioner/test-claim.yaml index 4dd2867..7a53b70 100644 --- a/gpulive/nfs-provisioner/test-claim.yaml +++ b/gpulive/nfs-provisioner/test-claim.yaml @@ -2,9 +2,8 @@ kind: PersistentVolumeClaim apiVersion: v1 metadata: name: test-claim - namespace: istio-system spec: - storageClassName: user-nas + storageClassName: nfs-client accessModes: - ReadWriteMany resources: diff --git a/gpulive/nfs-provisioner/test-pod.yaml b/gpulive/nfs-provisioner/test-pod.yaml index 877bc2d..b54005e 100644 --- a/gpulive/nfs-provisioner/test-pod.yaml +++ b/gpulive/nfs-provisioner/test-pod.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Pod metadata: name: nhnent01-pod01 - namespace: istio-system + namespace: default labels: app: nginx spec: diff --git a/gpulive/monitoring/base/prometheus/cluster_role.yaml b/gpulive/prometheus/base/prometheus/cluster_role.yaml similarity index 100% rename from gpulive/monitoring/base/prometheus/cluster_role.yaml rename to gpulive/prometheus/base/prometheus/cluster_role.yaml diff --git a/gpulive/monitoring/base/prometheus/cm.yaml b/gpulive/prometheus/base/prometheus/cm.yaml similarity index 100% rename from gpulive/monitoring/base/prometheus/cm.yaml rename to gpulive/prometheus/base/prometheus/cm.yaml diff --git a/gpulive/monitoring/base/prometheus/cm_custom_alert_rules.yaml b/gpulive/prometheus/base/prometheus/cm_custom_alert_rules.yaml similarity index 100% rename from gpulive/monitoring/base/prometheus/cm_custom_alert_rules.yaml rename to gpulive/prometheus/base/prometheus/cm_custom_alert_rules.yaml diff --git a/gpulive/monitoring/base/prometheus/daemonset.yaml b/gpulive/prometheus/base/prometheus/daemonset.yaml similarity index 100% rename from gpulive/monitoring/base/prometheus/daemonset.yaml rename to gpulive/prometheus/base/prometheus/daemonset.yaml diff --git a/gpulive/monitoring/base/prometheus/deploy.yaml b/gpulive/prometheus/base/prometheus/deploy.yaml similarity index 100% rename from gpulive/monitoring/base/prometheus/deploy.yaml rename to gpulive/prometheus/base/prometheus/deploy.yaml diff --git a/gpulive/monitoring/base/prometheus/ingress_alertmanager.yaml b/gpulive/prometheus/base/prometheus/ingress_alertmanager.yaml similarity index 100% rename from gpulive/monitoring/base/prometheus/ingress_alertmanager.yaml rename to gpulive/prometheus/base/prometheus/ingress_alertmanager.yaml diff --git a/gpulive/monitoring/base/prometheus/ingress_prometheus.yaml b/gpulive/prometheus/base/prometheus/ingress_prometheus.yaml similarity index 100% rename from gpulive/monitoring/base/prometheus/ingress_prometheus.yaml rename to gpulive/prometheus/base/prometheus/ingress_prometheus.yaml diff --git a/gpulive/monitoring/base/prometheus/kustomization.yaml b/gpulive/prometheus/base/prometheus/kustomization.yaml similarity index 100% rename from gpulive/monitoring/base/prometheus/kustomization.yaml rename to gpulive/prometheus/base/prometheus/kustomization.yaml diff --git a/gpulive/monitoring/base/prometheus/namespace.yaml b/gpulive/prometheus/base/prometheus/namespace.yaml similarity index 100% rename from gpulive/monitoring/base/prometheus/namespace.yaml rename to gpulive/prometheus/base/prometheus/namespace.yaml diff --git a/gpulive/monitoring/base/prometheus/pvc.yaml b/gpulive/prometheus/base/prometheus/pvc.yaml similarity index 100% rename from gpulive/monitoring/base/prometheus/pvc.yaml rename to gpulive/prometheus/base/prometheus/pvc.yaml diff --git a/gpulive/monitoring/base/prometheus/sa.yaml b/gpulive/prometheus/base/prometheus/sa.yaml similarity index 100% rename from gpulive/monitoring/base/prometheus/sa.yaml rename to gpulive/prometheus/base/prometheus/sa.yaml diff --git a/gpulive/monitoring/base/prometheus/service.yaml b/gpulive/prometheus/base/prometheus/service.yaml similarity index 100% rename from gpulive/monitoring/base/prometheus/service.yaml rename to gpulive/prometheus/base/prometheus/service.yaml diff --git a/gpulive/monitoring/base/prometheus/state_full_set.yaml b/gpulive/prometheus/base/prometheus/state_full_set.yaml similarity index 100% rename from gpulive/monitoring/base/prometheus/state_full_set.yaml rename to gpulive/prometheus/base/prometheus/state_full_set.yaml diff --git a/gpulive/monitoring/overlays/default/auth b/gpulive/prometheus/overlays/default/auth similarity index 100% rename from gpulive/monitoring/overlays/default/auth rename to gpulive/prometheus/overlays/default/auth diff --git a/gpulive/monitoring/overlays/default/grafana/kustomization.yaml b/gpulive/prometheus/overlays/default/grafana/kustomization.yaml similarity index 100% rename from gpulive/monitoring/overlays/default/grafana/kustomization.yaml rename to gpulive/prometheus/overlays/default/grafana/kustomization.yaml diff --git a/gpulive/monitoring/overlays/default/grafana/patch-cm.yaml b/gpulive/prometheus/overlays/default/grafana/patch-cm.yaml similarity index 100% rename from gpulive/monitoring/overlays/default/grafana/patch-cm.yaml rename to gpulive/prometheus/overlays/default/grafana/patch-cm.yaml diff --git a/gpulive/monitoring/overlays/default/grafana/patch-deploy.yaml b/gpulive/prometheus/overlays/default/grafana/patch-deploy.yaml similarity index 100% rename from gpulive/monitoring/overlays/default/grafana/patch-deploy.yaml rename to gpulive/prometheus/overlays/default/grafana/patch-deploy.yaml diff --git a/gpulive/monitoring/overlays/default/grafana/patch-ingress_grafana.yaml b/gpulive/prometheus/overlays/default/grafana/patch-ingress_grafana.yaml similarity index 100% rename from gpulive/monitoring/overlays/default/grafana/patch-ingress_grafana.yaml rename to gpulive/prometheus/overlays/default/grafana/patch-ingress_grafana.yaml diff --git a/gpulive/monitoring/overlays/default/grafana/patch-secret.yaml b/gpulive/prometheus/overlays/default/grafana/patch-secret.yaml similarity index 100% rename from gpulive/monitoring/overlays/default/grafana/patch-secret.yaml rename to gpulive/prometheus/overlays/default/grafana/patch-secret.yaml diff --git a/gpulive/monitoring/overlays/default/kustomization.yaml b/gpulive/prometheus/overlays/default/kustomization.yaml similarity index 100% rename from gpulive/monitoring/overlays/default/kustomization.yaml rename to gpulive/prometheus/overlays/default/kustomization.yaml diff --git a/gpulive/monitoring/overlays/default/prometheus/kustomization.yaml b/gpulive/prometheus/overlays/default/prometheus/kustomization.yaml similarity index 100% rename from gpulive/monitoring/overlays/default/prometheus/kustomization.yaml rename to gpulive/prometheus/overlays/default/prometheus/kustomization.yaml diff --git a/gpulive/monitoring/overlays/default/prometheus/patch-cm-prometheus-server.yaml b/gpulive/prometheus/overlays/default/prometheus/patch-cm-prometheus-server.yaml similarity index 100% rename from gpulive/monitoring/overlays/default/prometheus/patch-cm-prometheus-server.yaml rename to gpulive/prometheus/overlays/default/prometheus/patch-cm-prometheus-server.yaml diff --git a/gpulive/monitoring/overlays/default/prometheus/patch-deploy.yaml b/gpulive/prometheus/overlays/default/prometheus/patch-deploy.yaml similarity index 100% rename from gpulive/monitoring/overlays/default/prometheus/patch-deploy.yaml rename to gpulive/prometheus/overlays/default/prometheus/patch-deploy.yaml diff --git a/gpulive/monitoring/overlays/default/prometheus/patch-ingress_alertmanager.yaml b/gpulive/prometheus/overlays/default/prometheus/patch-ingress_alertmanager.yaml similarity index 100% rename from gpulive/monitoring/overlays/default/prometheus/patch-ingress_alertmanager.yaml rename to gpulive/prometheus/overlays/default/prometheus/patch-ingress_alertmanager.yaml diff --git a/gpulive/monitoring/overlays/default/prometheus/patch-ingress_prometheus.yaml b/gpulive/prometheus/overlays/default/prometheus/patch-ingress_prometheus.yaml similarity index 100% rename from gpulive/monitoring/overlays/default/prometheus/patch-ingress_prometheus.yaml rename to gpulive/prometheus/overlays/default/prometheus/patch-ingress_prometheus.yaml diff --git a/gpulive/prometheus/overlays/vm/kustomization.yaml b/gpulive/prometheus/overlays/vm/kustomization.yaml new file mode 100644 index 0000000..8e3646b --- /dev/null +++ b/gpulive/prometheus/overlays/vm/kustomization.yaml @@ -0,0 +1,16 @@ +# Prometheus에 종속적으로 설치 +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: monitoring + +resources: + - prometheus + +generatorOptions: + disableNameSuffixHash: true +secretGenerator: + - name: basic-auth + files: + - auth + type: Opaque diff --git a/gpulive/prometheus/overlays/vm/prometheus/kustomization.yaml b/gpulive/prometheus/overlays/vm/prometheus/kustomization.yaml new file mode 100644 index 0000000..1d35c34 --- /dev/null +++ b/gpulive/prometheus/overlays/vm/prometheus/kustomization.yaml @@ -0,0 +1,34 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../../base/prometheus + +namespace: monitoring + +patches: + - target: + kind: ConfigMap + name: prometheus-server + namespace: monitoring + path: patch-cm-prometheus-server.yaml + - target: + kind: Deployment + name: prometheus-server + namespace: monitoring + path: patch-deploy.yaml + - target: + kind: Ingress + name: prometheus-ingress + namespace: monitoring + path: patch-ingress_prometheus.yaml + - target: + kind: Ingress + name: prometheus-alertmanager-ingress + namespace: monitoring + path: patch-ingress_alertmanager.yaml + # - target: + # kind: Secret + # name: basic-auth + # namespace: monitoring + # path: patch-basic-auth.yaml diff --git a/gpulive/prometheus/overlays/vm/prometheus/patch-cm-prometheus-server.yaml b/gpulive/prometheus/overlays/vm/prometheus/patch-cm-prometheus-server.yaml new file mode 100644 index 0000000..b684a13 --- /dev/null +++ b/gpulive/prometheus/overlays/vm/prometheus/patch-cm-prometheus-server.yaml @@ -0,0 +1,335 @@ +# - "/etc/config/rules/*.yaml" # 수정대상 +# - "/etc/config/alerts/*.yaml" # 수정대상 +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-server + namespace: monitoring +data: + prometheus.yml: | + global: + evaluation_interval: 1m + scrape_interval: 1m + scrape_timeout: 10s + rule_files: + - /etc/config/recording_rules.yml + - /etc/config/alerting_rules.yml + - "/etc/config/rules/*.yaml" + - "/etc/config/alerts/*.yaml" + scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - localhost:9090 + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-apiservers + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: default;kubernetes;https + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_service_name + - __meta_kubernetes_endpoint_port_name + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes-cadvisor + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - honor_labels: true + job_name: kubernetes-service-endpoints + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-service-endpoints-slow + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + - honor_labels: true + job_name: prometheus-pushgateway + kubernetes_sd_configs: + - role: service + relabel_configs: + - action: keep + regex: pushgateway + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - honor_labels: true + job_name: kubernetes-services + kubernetes_sd_configs: + - role: service + metrics_path: /probe + params: + module: + - http_2xx + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - source_labels: + - __address__ + target_label: __param_target + - replacement: blackbox + target_label: __address__ + - source_labels: + - __param_target + target_label: instance + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - honor_labels: true + job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-pods-slow + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + alerting: + alertmanagers: + - kubernetes_sd_configs: + - role: pod + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_namespace] + regex: monitoring + action: keep + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance] + regex: prometheus + action: keep + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + regex: alertmanager + action: keep + - source_labels: [__meta_kubernetes_pod_container_port_number] + regex: "9093" + action: keep diff --git a/gpulive/prometheus/overlays/vm/prometheus/patch-deploy.yaml b/gpulive/prometheus/overlays/vm/prometheus/patch-deploy.yaml new file mode 100644 index 0000000..1758854 --- /dev/null +++ b/gpulive/prometheus/overlays/vm/prometheus/patch-deploy.yaml @@ -0,0 +1,19 @@ +# Source: prometheus/templates/deploy.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-server + namespace: monitoring +spec: + template: + spec: + containers: + - name: prometheus-server + volumeMounts: + - mountPath: /etc/config/alerts + name: config-volume2 + volumes: + - configMap: + defaultMode: 420 + name: prometheus-rulefiles + name: config-volume2 \ No newline at end of file diff --git a/gpulive/prometheus/overlays/vm/prometheus/patch-ingress_alertmanager.yaml b/gpulive/prometheus/overlays/vm/prometheus/patch-ingress_alertmanager.yaml new file mode 100644 index 0000000..d21488b --- /dev/null +++ b/gpulive/prometheus/overlays/vm/prometheus/patch-ingress_alertmanager.yaml @@ -0,0 +1,29 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prometheus-alertmanager-ingress + namespace: monitoring + annotations: + # 인증 방법 설정: basic auth + nginx.ingress.kubernetes.io/auth-type: basic + # basic auth 사용자가 들어있는 secret 설정 + nginx.ingress.kubernetes.io/auth-secret: basic-auth + # 인증 요청시 나오는 메세지 설정 + nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - admin' +spec: + ingressClassName: nginx + rules: + - host: alert.nhngpuaas.com + http: + paths: + - backend: + service: + name: prometheus-alertmanager + port: + number: 9093 + path: / + pathType: Prefix + tls: + - hosts: + - alert.nhngpuaas.com + secretName: nhngpuaas-ssl \ No newline at end of file diff --git a/gpulive/prometheus/overlays/vm/prometheus/patch-ingress_prometheus.yaml b/gpulive/prometheus/overlays/vm/prometheus/patch-ingress_prometheus.yaml new file mode 100644 index 0000000..c03f487 --- /dev/null +++ b/gpulive/prometheus/overlays/vm/prometheus/patch-ingress_prometheus.yaml @@ -0,0 +1,22 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prometheus-ingress + namespace: monitoring +spec: + ingressClassName: nginx + rules: + - host: prometheus.nhngpuaas.com + http: + paths: + - backend: + service: + name: prometheus-server + port: + number: 80 + path: / + pathType: Prefix + tls: + - hosts: + - prometheus.nhngpuaas.com + secretName: nhngpuaas-ssl diff --git a/gpulive/monitoring/base/grafana/cm.yaml b/gpulive/vm-monitoring/base/grafana/cm.yaml similarity index 100% rename from gpulive/monitoring/base/grafana/cm.yaml rename to gpulive/vm-monitoring/base/grafana/cm.yaml diff --git a/gpulive/monitoring/base/grafana/deploy.yaml b/gpulive/vm-monitoring/base/grafana/deploy.yaml similarity index 100% rename from gpulive/monitoring/base/grafana/deploy.yaml rename to gpulive/vm-monitoring/base/grafana/deploy.yaml diff --git a/gpulive/monitoring/base/grafana/kustomization.yaml b/gpulive/vm-monitoring/base/grafana/kustomization.yaml similarity index 100% rename from gpulive/monitoring/base/grafana/kustomization.yaml rename to gpulive/vm-monitoring/base/grafana/kustomization.yaml diff --git a/gpulive/monitoring/base/grafana/pvc.yaml b/gpulive/vm-monitoring/base/grafana/pvc.yaml similarity index 100% rename from gpulive/monitoring/base/grafana/pvc.yaml rename to gpulive/vm-monitoring/base/grafana/pvc.yaml diff --git a/gpulive/monitoring/base/grafana/role.yaml b/gpulive/vm-monitoring/base/grafana/role.yaml similarity index 100% rename from gpulive/monitoring/base/grafana/role.yaml rename to gpulive/vm-monitoring/base/grafana/role.yaml diff --git a/gpulive/monitoring/base/grafana/sa.yaml b/gpulive/vm-monitoring/base/grafana/sa.yaml similarity index 100% rename from gpulive/monitoring/base/grafana/sa.yaml rename to gpulive/vm-monitoring/base/grafana/sa.yaml diff --git a/gpulive/monitoring/base/grafana/secret.yaml b/gpulive/vm-monitoring/base/grafana/secret.yaml similarity index 100% rename from gpulive/monitoring/base/grafana/secret.yaml rename to gpulive/vm-monitoring/base/grafana/secret.yaml diff --git a/gpulive/monitoring/base/grafana/service.yaml b/gpulive/vm-monitoring/base/grafana/service.yaml similarity index 100% rename from gpulive/monitoring/base/grafana/service.yaml rename to gpulive/vm-monitoring/base/grafana/service.yaml diff --git a/gpulive/vm-monitoring/base/prometheus/cluster_role.yaml b/gpulive/vm-monitoring/base/prometheus/cluster_role.yaml new file mode 100644 index 0000000..abcd2d6 --- /dev/null +++ b/gpulive/vm-monitoring/base/prometheus/cluster_role.yaml @@ -0,0 +1,248 @@ +# Source: prometheus/charts/kube-state-metrics/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + helm.sh/chart: kube-state-metrics-5.28.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: kube-state-metrics + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "2.14.0" + name: prometheus-kube-state-metrics +rules: + +- apiGroups: ["certificates.k8s.io"] + resources: + - certificatesigningrequests + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - configmaps + verbs: ["list", "watch"] + +- apiGroups: ["batch"] + resources: + - cronjobs + verbs: ["list", "watch"] + +- apiGroups: ["extensions", "apps"] + resources: + - daemonsets + verbs: ["list", "watch"] + +- apiGroups: ["extensions", "apps"] + resources: + - deployments + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - endpoints + verbs: ["list", "watch"] + +- apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: ["list", "watch"] + +- apiGroups: ["extensions", "networking.k8s.io"] + resources: + - ingresses + verbs: ["list", "watch"] + +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["list", "watch"] + +- apiGroups: ["coordination.k8s.io"] + resources: + - leases + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - limitranges + verbs: ["list", "watch"] + +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - mutatingwebhookconfigurations + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - namespaces + verbs: ["list", "watch"] + +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - nodes + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - persistentvolumeclaims + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - persistentvolumes + verbs: ["list", "watch"] + +- apiGroups: ["policy"] + resources: + - poddisruptionbudgets + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - pods + verbs: ["list", "watch"] + +- apiGroups: ["extensions", "apps"] + resources: + - replicasets + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - replicationcontrollers + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - resourcequotas + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - secrets + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - services + verbs: ["list", "watch"] + +- apiGroups: ["apps"] + resources: + - statefulsets + verbs: ["list", "watch"] + +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["list", "watch"] + +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - validatingwebhookconfigurations + verbs: ["list", "watch"] + +- apiGroups: ["storage.k8s.io"] + resources: + - volumeattachments + verbs: ["list", "watch"] +--- +# Source: prometheus/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.1.0 + helm.sh/chart: prometheus-27.3.1 + app.kubernetes.io/part-of: prometheus + name: prometheus-server +rules: + - apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - nodes/metrics + - services + - endpoints + - pods + - ingresses + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses/status + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "discovery.k8s.io" + resources: + - endpointslices + verbs: + - get + - list + - watch + - nonResourceURLs: + - "/metrics" + verbs: + - get +--- +# Source: prometheus/charts/kube-state-metrics/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + helm.sh/chart: kube-state-metrics-5.28.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: kube-state-metrics + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "2.14.0" + name: prometheus-kube-state-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-kube-state-metrics +subjects: +- kind: ServiceAccount + name: prometheus-kube-state-metrics + namespace: monitoring +--- +# Source: prometheus/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.1.0 + helm.sh/chart: prometheus-27.3.1 + app.kubernetes.io/part-of: prometheus + name: prometheus-server +subjects: + - kind: ServiceAccount + name: prometheus-server + namespace: monitoring +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-server \ No newline at end of file diff --git a/gpulive/vm-monitoring/base/prometheus/cm.yaml b/gpulive/vm-monitoring/base/prometheus/cm.yaml new file mode 100644 index 0000000..a1b4135 --- /dev/null +++ b/gpulive/vm-monitoring/base/prometheus/cm.yaml @@ -0,0 +1,375 @@ +# Source: prometheus/charts/alertmanager/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-alertmanager + labels: + helm.sh/chart: alertmanager-1.14.0 + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v0.28.0" + app.kubernetes.io/managed-by: Helm + namespace: monitoring +data: + alertmanager.yml: | + global: {} + receivers: + - name: default-receiver + route: + group_interval: 5m + group_wait: 10s + receiver: default-receiver + repeat_interval: 3h + templates: + - /etc/alertmanager/*.tmpl +--- +# Source: prometheus/templates/cm.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.1.0 + helm.sh/chart: prometheus-27.3.1 + app.kubernetes.io/part-of: prometheus + name: prometheus-server + namespace: monitoring +data: + allow-snippet-annotations: "false" + alerting_rules.yml: | + {} + alerts: | + {} + prometheus.yml: | + global: + evaluation_interval: 1m + scrape_interval: 1m + scrape_timeout: 10s + rule_files: + - /etc/config/recording_rules.yml + - /etc/config/alerting_rules.yml + - /etc/config/rules + - /etc/config/alerts + scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - localhost:9090 + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-apiservers + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: default;kubernetes;https + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_service_name + - __meta_kubernetes_endpoint_port_name + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes-cadvisor + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - honor_labels: true + job_name: kubernetes-service-endpoints + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-service-endpoints-slow + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + - honor_labels: true + job_name: prometheus-pushgateway + kubernetes_sd_configs: + - role: service + relabel_configs: + - action: keep + regex: pushgateway + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - honor_labels: true + job_name: kubernetes-services + kubernetes_sd_configs: + - role: service + metrics_path: /probe + params: + module: + - http_2xx + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - source_labels: + - __address__ + target_label: __param_target + - replacement: blackbox + target_label: __address__ + - source_labels: + - __param_target + target_label: instance + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - honor_labels: true + job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-pods-slow + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + alerting: + alertmanagers: + - kubernetes_sd_configs: + - role: pod + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_namespace] + regex: monitoring + action: keep + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance] + regex: prometheus + action: keep + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + regex: alertmanager + action: keep + - source_labels: [__meta_kubernetes_pod_container_port_number] + regex: "9093" + action: keep + recording_rules.yml: | + {} + rules: | + {} \ No newline at end of file diff --git a/gpulive/vm-monitoring/base/prometheus/cm_custom_alert_rules.yaml b/gpulive/vm-monitoring/base/prometheus/cm_custom_alert_rules.yaml new file mode 100644 index 0000000..bab5fee --- /dev/null +++ b/gpulive/vm-monitoring/base/prometheus/cm_custom_alert_rules.yaml @@ -0,0 +1,194 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-rulefiles + namespace: monitoring +data: + # Awesome Prometheus alerts + ################################################################################ + # Basic resource monitoring prometheus self-monitoring + ################################################################################ + prometheus-self-monitoring.yaml: | + groups: + - name: prometheus-self-monitoring + rules: + - alert: Watchdog + annotations: + description: | + This is an alert meant to ensure that the entire alerting pipeline is functional. + This alert is always firing, therefore it should always be firing in Alertmanager + and always fire against a receiver. There are integrations with various notification + mechanisms that send a notification when this alert is not firing. For example the + "DeadMansSnitch" integration in PagerDuty. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/watchdog + summary: An alert that should always be firing to certify that Alertmanager + is working properly. + expr: vector(1) + labels: + severity: none + # Awesome Prometheus alerts + ################################################################################ + # Basic resource monitoring host and hardware (node-export) + ################################################################################ + prometheus-host-hardware.yaml: | + groups: + - name: prometheus-host-hardware + rules: + - alert: HostOutOfMemory (1.2.01) + expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10 + for: 2m + labels: + severity: warning + annotations: + summary: Host out of memory (instance {{ $labels.instance }}) + description: "Node memory is filling up (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostOutOfDiskSpace (1.2.07) + expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0 + for: 2m + labels: + severity: warning + annotations: + summary: Host out of disk space (instance {{ $labels.instance }}) + description: "Disk is almost full (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # Please add ignored mountpoints in node_exporter parameters like + # "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|run)($|/)". + # Same rule using "node_filesystem_free_bytes" will fire when disk fills for non-root users. + - alert: HostHighCpuLoad (1.2.13) + expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 80 + for: 0m + labels: + severity: warning + annotations: + summary: Host high CPU load (instance {{ $labels.instance }}) + description: "CPU load is > 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostOomKillDetected (1.2.24) + expr: increase(node_vmstat_oom_kill[1m]) > 0 + for: 0m + labels: + severity: warning + annotations: + summary: Host OOM kill detected (instance {{ $labels.instance }}) + description: "OOM kill detected\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # Awesome Prometheus alerts + ################################################################################ + # Basic resource monitoring docker container (google/cAdvisor) + ################################################################################ + Prometheus-docker-containers.yaml: | + groups: + - name: prometheus-docker-containers + rules: + # This rule can be very noisy in dynamic infra with legitimate container start/stop/deployment. + - alert: ContainerKilled (1.3.1) + expr: time() - container_last_seen > 60 + for: 0m + labels: + severity: warning + annotations: + summary: Container killed (instance {{ $labels.instance }}) + description: "A container has disappeared\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # This rule can be very noisy in dynamic infra with legitimate container start/stop/deployment. + - alert: ContainerCpuUsage (1.3.3) + expr: (sum(rate(container_cpu_usage_seconds_total{name!=""}[3m])) BY (instance, name) * 100) > 80 + for: 2m + labels: + severity: warning + annotations: + summary: Container CPU usage (instance {{ $labels.instance }}) + description: "Container CPU usage is above 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # See https://medium.com/faun/how-much-is-too-much-the-linux-oomkiller-and-used-memory-d32186f29c9d + - alert: ContainerMemoryUsage (1.3.4) + expr: (sum(container_memory_working_set_bytes{name!=""}) BY (instance, name) / sum(container_spec_memory_limit_bytes > 0) BY (instance, name) * 100) > 80 + for: 2m + labels: + severity: warning + annotations: + summary: Container Memory usage (instance {{ $labels.instance }}) + description: "Container Memory usage is above 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # Awesome Prometheus alerts + ################################################################################ + # Orchestrators kubernetes (kube-state-metrics) + ################################################################################ + prometheus-kubernetes.yaml: | + groups: + - name: prometheus-kubernetes + rules: + - alert: KubernetesNodeReady (5.1.01) + expr: kube_node_status_condition{condition="Ready",status="true"} == 0 + for: 10m + labels: + severity: critical + annotations: + summary: Kubernetes Node ready (instance {{ $labels.instance }}) + description: "Node {{ $labels.node }} has been unready for a long time\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesOutOfDisk (5.1.04) + expr: kube_node_status_condition{condition="OutOfDisk",status="true"} == 1 + for: 2m + labels: + severity: critical + annotations: + summary: Kubernetes out of disk (instance {{ $labels.instance }}) + description: "{{ $labels.node }} has OutOfDisk condition\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesContainerOomKiller (5.1.06) + expr: (kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 10m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[10m]) == 1 + for: 0m + labels: + severity: warning + annotations: + summary: Kubernetes container oom killer (instance {{ $labels.instance }}) + description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 10 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesVolumeOutOfDiskSpace (5.1.10) + expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 10 + for: 2m + labels: + severity: warning + annotations: + summary: Kubernetes Volume out of disk space (instance {{ $labels.instance }}) + description: "Volume is almost full (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesPersistentvolumeError (5.1.12) + expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending", job="kube-state-metrics"} > 0 + for: 0m + labels: + severity: critical + annotations: + summary: Kubernetes PersistentVolume error (instance {{ $labels.instance }}) + description: "Persistent volume is in bad state\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesHpaScaleCapability (5.1.16) + expr: kube_horizontalpodautoscaler_status_desired_replicas >= kube_horizontalpodautoscaler_spec_max_replicas + for: 2m + labels: + severity: info + annotations: + summary: Kubernetes HPA scale capability (instance {{ $labels.instance }}) + description: "The maximum number of desired Pods has been hit\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesPodNotHealthy (5.1.17) + expr: min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"})[15m:1m]) > 0 + for: 0m + labels: + severity: critical + annotations: + summary: Kubernetes Pod not healthy (instance {{ $labels.instance }}) + description: "Pod has been in a non-ready state for longer than 15 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesPodCrashLooping (5.1.18) + expr: increase(kube_pod_container_status_restarts_total[1m]) > 3 + for: 2m + labels: + severity: warning + annotations: + summary: Kubernetes pod crash looping (instance {{ $labels.instance }}) + description: "Pod {{ $labels.pod }} is crash looping\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesApiServerErrors (5.1.29) + expr: sum(rate(apiserver_request_total{job="apiserver",code=~"^(?:5..)$"}[1m])) / sum(rate(apiserver_request_total{job="apiserver"}[1m])) * 100 > 3 + for: 2m + labels: + severity: critical + annotations: + summary: Kubernetes API server errors (instance {{ $labels.instance }}) + description: "Kubernetes API server is experiencing high error rate\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesClientCertificateExpiresSoon (5.1.32) + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 24*60*60 + for: 0m + labels: + severity: critical + annotations: + summary: Kubernetes client certificate expires soon (instance {{ $labels.instance }}) + description: "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" \ No newline at end of file diff --git a/gpulive/vm-monitoring/base/prometheus/daemonset.yaml b/gpulive/vm-monitoring/base/prometheus/daemonset.yaml new file mode 100644 index 0000000..53d3288 --- /dev/null +++ b/gpulive/vm-monitoring/base/prometheus/daemonset.yaml @@ -0,0 +1,135 @@ +# Source: prometheus/charts/prometheus-node-exporter/templates/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: prometheus-prometheus-node-exporter + namespace: monitoring + labels: + helm.sh/chart: prometheus-node-exporter-4.43.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: prometheus-node-exporter + app.kubernetes.io/name: prometheus-node-exporter + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "1.8.2" +spec: + selector: + matchLabels: + app.kubernetes.io/name: prometheus-node-exporter + app.kubernetes.io/instance: prometheus + revisionHistoryLimit: 10 + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + labels: + helm.sh/chart: prometheus-node-exporter-4.43.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: prometheus-node-exporter + app.kubernetes.io/name: prometheus-node-exporter + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "1.8.2" + spec: + automountServiceAccountToken: false + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + serviceAccountName: prometheus-prometheus-node-exporter + containers: + - name: node-exporter + image: quay.io/prometheus/node-exporter:v1.8.2 + imagePullPolicy: Always + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + - --path.rootfs=/host/root + - --path.udev.data=/host/root/run/udev/data + - --web.listen-address=[$(HOST_IP)]:9100 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + env: + - name: HOST_IP + value: 0.0.0.0 + ports: + - name: metrics + containerPort: 9100 + protocol: TCP + livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: + path: / + port: 9100 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: + path: / + port: 9100 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + volumeMounts: + - name: proc + mountPath: /host/proc + readOnly: true + - name: sys + mountPath: /host/sys + readOnly: true + - name: root + mountPath: /host/root + mountPropagation: HostToContainer + readOnly: true + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 100m + memory: 100Mi + hostNetwork: true + hostPID: true + hostIPC: false + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + - key: type + operator: NotIn + values: + - virtual-kubelet + nodeSelector: + kubernetes.io/os: linux + tolerations: + - effect: NoSchedule + operator: Exists + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + - name: root + hostPath: + path: / \ No newline at end of file diff --git a/gpulive/vm-monitoring/base/prometheus/deploy.yaml b/gpulive/vm-monitoring/base/prometheus/deploy.yaml new file mode 100644 index 0000000..89ae67d --- /dev/null +++ b/gpulive/vm-monitoring/base/prometheus/deploy.yaml @@ -0,0 +1,290 @@ +# Source: prometheus/charts/kube-state-metrics/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-kube-state-metrics + namespace: monitoring + labels: + helm.sh/chart: kube-state-metrics-5.28.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: kube-state-metrics + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "2.14.0" +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: prometheus + replicas: 1 + strategy: + type: RollingUpdate + revisionHistoryLimit: 10 + template: + metadata: + labels: + helm.sh/chart: kube-state-metrics-5.28.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: kube-state-metrics + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "2.14.0" + spec: + nodeSelector: + nodegroup: nd + automountServiceAccountToken: true + hostNetwork: false + serviceAccountName: prometheus-kube-state-metrics + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: kube-state-metrics + args: + - --port=8080 + - --resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,horizontalpodautoscalers,ingresses,jobs,leases,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments + imagePullPolicy: Always + image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.14.0 + ports: + - containerPort: 8080 + name: "http" + livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: + path: /livez + port: 8080 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: + path: /readyz + port: 8081 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + resources: + requests: + cpu: 50m + memory: 125Mi + limits: + cpu: 100m + memory: 250Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true +--- +# Source: prometheus/charts/prometheus-pushgateway/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + helm.sh/chart: prometheus-pushgateway-2.17.0 + app.kubernetes.io/name: prometheus-pushgateway + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v1.11.0" + app.kubernetes.io/managed-by: Helm + name: prometheus-prometheus-pushgateway + namespace: monitoring +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: prometheus-pushgateway + app.kubernetes.io/instance: prometheus + template: + metadata: + labels: + helm.sh/chart: prometheus-pushgateway-2.17.0 + app.kubernetes.io/name: prometheus-pushgateway + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v1.11.0" + app.kubernetes.io/managed-by: Helm + spec: + nodeSelector: + nodegroup: nd + serviceAccountName: prometheus-prometheus-pushgateway + automountServiceAccountToken: true + containers: + - name: pushgateway + image: "quay.io/prometheus/pushgateway:v1.11.0" + imagePullPolicy: Always + ports: + - name: metrics + containerPort: 9091 + protocol: TCP + livenessProbe: + httpGet: + path: /-/healthy + port: 9091 + initialDelaySeconds: 10 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: /-/ready + port: 9091 + initialDelaySeconds: 10 + timeoutSeconds: 10 + volumeMounts: + - name: storage-volume + mountPath: "/data" + subPath: "" + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 200m + memory: 512Mi + securityContext: + fsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + volumes: + - name: storage-volume + emptyDir: {} +--- +# Source: prometheus/templates/deploy.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.1.0 + helm.sh/chart: prometheus-27.3.1 + app.kubernetes.io/part-of: prometheus + name: prometheus-server + namespace: monitoring +spec: + selector: + matchLabels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + replicas: 1 + revisionHistoryLimit: 10 + strategy: + type: Recreate + rollingUpdate: null + template: + metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.1.0 + helm.sh/chart: prometheus-27.3.1 + app.kubernetes.io/part-of: prometheus + spec: + nodeSelector: + nodegroup: nd + enableServiceLinks: true + serviceAccountName: prometheus-server + containers: + - name: prometheus-server-configmap-reload + image: "quay.io/prometheus-operator/prometheus-config-reloader:v0.79.2" + imagePullPolicy: "Always" + args: + - --watched-dir=/etc/config + - --listen-address=0.0.0.0:8080 + - --reload-url=http://127.0.0.1:9090/-/reload + ports: + - containerPort: 8080 + name: metrics + livenessProbe: + httpGet: + path: /healthz + port: metrics + scheme: HTTP + initialDelaySeconds: 2 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /healthz + port: metrics + scheme: HTTP + periodSeconds: 10 + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + + - name: prometheus-server + image: "quay.io/prometheus/prometheus:v3.1.0" + imagePullPolicy: "Always" + args: + - --storage.tsdb.retention.time=15d + - --config.file=/etc/config/prometheus.yml + - --storage.tsdb.path=/data + - --web.console.libraries=/etc/prometheus/console_libraries + - --web.console.templates=/etc/prometheus/consoles + - --web.enable-lifecycle + ports: + - containerPort: 9090 + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 4 + failureThreshold: 3 + successThreshold: 1 + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 3 + successThreshold: 1 + volumeMounts: + - name: config-volume + mountPath: /etc/config + - name: storage-volume + mountPath: /data + subPath: "" + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: 1050m + memory: 4Gi + dnsPolicy: ClusterFirst + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + terminationGracePeriodSeconds: 300 + volumes: + - name: config-volume + configMap: + name: prometheus-server + - name: storage-volume + persistentVolumeClaim: + claimName: prometheus-server \ No newline at end of file diff --git a/gpulive/vm-monitoring/base/prometheus/ingress_alertmanager.yaml b/gpulive/vm-monitoring/base/prometheus/ingress_alertmanager.yaml new file mode 100644 index 0000000..87afe8d --- /dev/null +++ b/gpulive/vm-monitoring/base/prometheus/ingress_alertmanager.yaml @@ -0,0 +1,29 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prometheus-alertmanager-ingress + namespace: monitoring + annotations: + # 인증 방법 설정: basic auth + nginx.ingress.kubernetes.io/auth-type: basic + # basic auth 사용자가 들어있는 secret 설정 + nginx.ingress.kubernetes.io/auth-secret: basic-auth + # 인증 요청시 나오는 메세지 설정 + nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - admin' +spec: + ingressClassName: nginx + rules: + - host: alert.cone-chain.com + http: + paths: + - backend: + service: + name: prometheus-alertmanager + port: + number: 9093 + path: / + pathType: Prefix + tls: + - hosts: + - prometheus.cone-chain.com + secretName: ssl-cc \ No newline at end of file diff --git a/gpulive/vm-monitoring/base/prometheus/ingress_prometheus.yaml b/gpulive/vm-monitoring/base/prometheus/ingress_prometheus.yaml new file mode 100644 index 0000000..c56bae2 --- /dev/null +++ b/gpulive/vm-monitoring/base/prometheus/ingress_prometheus.yaml @@ -0,0 +1,29 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prometheus-ingress + namespace: monitoring + annotations: + # 인증 방법 설정: basic auth + nginx.ingress.kubernetes.io/auth-type: basic + # basic auth 사용자가 들어있는 secret 설정 + nginx.ingress.kubernetes.io/auth-secret: basic-auth + # 인증 요청시 나오는 메세지 설정 + nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - admin' +spec: + ingressClassName: nginx + rules: + - host: prometheus.cone-chain.com + http: + paths: + - backend: + service: + name: prometheus-server + port: + number: 80 + path: / + pathType: Prefix + tls: + - hosts: + - prometheus.cone-chain.com + secretName: ssl-cc \ No newline at end of file diff --git a/gpulive/vm-monitoring/base/prometheus/kustomization.yaml b/gpulive/vm-monitoring/base/prometheus/kustomization.yaml new file mode 100644 index 0000000..027f1b7 --- /dev/null +++ b/gpulive/vm-monitoring/base/prometheus/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: monitoring + +resources: + - namespace.yaml + - sa.yaml + - cluster_role.yaml + - cm.yaml + - deploy.yaml + - daemonset.yaml + - pvc.yaml + - service.yaml + - state_full_set.yaml + # - secret.yaml + - ingress_alertmanager.yaml + - ingress_prometheus.yaml + - cm_custom_alert_rules.yaml diff --git a/gpulive/vm-monitoring/base/prometheus/namespace.yaml b/gpulive/vm-monitoring/base/prometheus/namespace.yaml new file mode 100644 index 0000000..07aaec3 --- /dev/null +++ b/gpulive/vm-monitoring/base/prometheus/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring + labels: + kubernetes.io/metadata.name: monitoring \ No newline at end of file diff --git a/gpulive/vm-monitoring/base/prometheus/pvc.yaml b/gpulive/vm-monitoring/base/prometheus/pvc.yaml new file mode 100644 index 0000000..70a8a29 --- /dev/null +++ b/gpulive/vm-monitoring/base/prometheus/pvc.yaml @@ -0,0 +1,19 @@ +# Source: prometheus/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.1.0 + helm.sh/chart: prometheus-27.3.1 + app.kubernetes.io/part-of: prometheus + name: prometheus-server + namespace: monitoring +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "8Gi" \ No newline at end of file diff --git a/gpulive/vm-monitoring/base/prometheus/sa.yaml b/gpulive/vm-monitoring/base/prometheus/sa.yaml new file mode 100644 index 0000000..2bfde38 --- /dev/null +++ b/gpulive/vm-monitoring/base/prometheus/sa.yaml @@ -0,0 +1,76 @@ +--- +# Source: prometheus/charts/alertmanager/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus-alertmanager + labels: + helm.sh/chart: alertmanager-1.14.0 + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v0.28.0" + app.kubernetes.io/managed-by: Helm + namespace: monitoring +automountServiceAccountToken: true +--- +# Source: prometheus/charts/kube-state-metrics/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: true +metadata: + labels: + helm.sh/chart: kube-state-metrics-5.28.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: kube-state-metrics + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "2.14.0" + name: prometheus-kube-state-metrics + namespace: monitoring +--- +# Source: prometheus/charts/prometheus-node-exporter/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus-prometheus-node-exporter + namespace: monitoring + labels: + helm.sh/chart: prometheus-node-exporter-4.43.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: prometheus-node-exporter + app.kubernetes.io/name: prometheus-node-exporter + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "1.8.2" +automountServiceAccountToken: false +--- +# Source: prometheus/charts/prometheus-pushgateway/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + helm.sh/chart: prometheus-pushgateway-2.17.0 + app.kubernetes.io/name: prometheus-pushgateway + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v1.11.0" + app.kubernetes.io/managed-by: Helm + name: prometheus-prometheus-pushgateway + namespace: monitoring +automountServiceAccountToken: true +--- +# Source: prometheus/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.1.0 + helm.sh/chart: prometheus-27.3.1 + app.kubernetes.io/part-of: prometheus + name: prometheus-server + namespace: monitoring + annotations: + {} \ No newline at end of file diff --git a/gpulive/vm-monitoring/base/prometheus/service.yaml b/gpulive/vm-monitoring/base/prometheus/service.yaml new file mode 100644 index 0000000..7c4cace --- /dev/null +++ b/gpulive/vm-monitoring/base/prometheus/service.yaml @@ -0,0 +1,151 @@ +# Source: prometheus/charts/alertmanager/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + name: prometheus-alertmanager + labels: + helm.sh/chart: alertmanager-1.14.0 + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v0.28.0" + app.kubernetes.io/managed-by: Helm + namespace: monitoring +spec: + type: ClusterIP + ports: + - port: 9093 + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus +--- +# Source: prometheus/charts/alertmanager/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + name: prometheus-alertmanager-headless + labels: + helm.sh/chart: alertmanager-1.14.0 + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v0.28.0" + app.kubernetes.io/managed-by: Helm + namespace: monitoring +spec: + clusterIP: None + ports: + - port: 9093 + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus +--- +# Source: prometheus/charts/kube-state-metrics/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: prometheus-kube-state-metrics + namespace: monitoring + labels: + helm.sh/chart: kube-state-metrics-5.28.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: kube-state-metrics + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "2.14.0" + annotations: + prometheus.io/scrape: 'true' +spec: + type: "ClusterIP" + ports: + - name: "http" + protocol: TCP + port: 8080 + targetPort: 8080 + + selector: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: prometheus +--- +# Source: prometheus/charts/prometheus-node-exporter/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: prometheus-prometheus-node-exporter + namespace: monitoring + labels: + helm.sh/chart: prometheus-node-exporter-4.43.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: prometheus-node-exporter + app.kubernetes.io/name: prometheus-node-exporter + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "1.8.2" + annotations: + prometheus.io/scrape: "true" +spec: + type: ClusterIP + ports: + - port: 9100 + targetPort: 9100 + protocol: TCP + name: metrics + selector: + app.kubernetes.io/name: prometheus-node-exporter + app.kubernetes.io/instance: prometheus +--- +# Source: prometheus/charts/prometheus-pushgateway/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/probe: pushgateway + labels: + helm.sh/chart: prometheus-pushgateway-2.17.0 + app.kubernetes.io/name: prometheus-pushgateway + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v1.11.0" + app.kubernetes.io/managed-by: Helm + name: prometheus-prometheus-pushgateway + namespace: monitoring +spec: + type: ClusterIP + ports: + - port: 9091 + targetPort: 9091 + protocol: TCP + name: http + selector: + app.kubernetes.io/name: prometheus-pushgateway + app.kubernetes.io/instance: prometheus +--- +# Source: prometheus/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.1.0 + helm.sh/chart: prometheus-27.3.1 + app.kubernetes.io/part-of: prometheus + name: prometheus-server + namespace: monitoring +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 9090 + selector: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + sessionAffinity: None + type: "ClusterIP" \ No newline at end of file diff --git a/gpulive/vm-monitoring/base/prometheus/state_full_set.yaml b/gpulive/vm-monitoring/base/prometheus/state_full_set.yaml new file mode 100644 index 0000000..f02a1c6 --- /dev/null +++ b/gpulive/vm-monitoring/base/prometheus/state_full_set.yaml @@ -0,0 +1,90 @@ +# Source: prometheus/charts/alertmanager/templates/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: prometheus-alertmanager + labels: + helm.sh/chart: alertmanager-1.14.0 + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v0.28.0" + app.kubernetes.io/managed-by: Helm + namespace: monitoring +spec: + replicas: 1 + minReadySeconds: 0 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus + serviceName: prometheus-alertmanager-headless + template: + metadata: + labels: + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus + spec: + nodeSelector: + nodegroup: nd + automountServiceAccountToken: true + serviceAccountName: prometheus-alertmanager + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + containers: + - name: alertmanager + securityContext: + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + image: "quay.io/prometheus/alertmanager:v0.28.0" + imagePullPolicy: Always + env: + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + args: + - --storage.path=/alertmanager + - --config.file=/etc/alertmanager/alertmanager.yml + ports: + - name: http + containerPort: 9093 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: / + port: http + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 200m + memory: 512Mi + volumeMounts: + - name: config + mountPath: /etc/alertmanager + - name: storage + mountPath: /alertmanager + volumes: + - name: config + configMap: + name: prometheus-alertmanager + volumeClaimTemplates: + - metadata: + name: storage + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi \ No newline at end of file diff --git a/gpulive/vm-monitoring/overlays/default/auth b/gpulive/vm-monitoring/overlays/default/auth new file mode 100644 index 0000000..cb0bcdd --- /dev/null +++ b/gpulive/vm-monitoring/overlays/default/auth @@ -0,0 +1 @@ +admin:$apr1$JIoWDh3p$Mo4E8nEh3beHp9n1IRjmc/ diff --git a/gpulive/vm-monitoring/overlays/default/grafana/kustomization.yaml b/gpulive/vm-monitoring/overlays/default/grafana/kustomization.yaml new file mode 100644 index 0000000..2f05aa9 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/default/grafana/kustomization.yaml @@ -0,0 +1,29 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../base/grafana + +namespace: monitoring + +patches: + - target: + kind: ConfigMap + name: grafana + namespace: monitoring + path: patch-cm.yaml + - target: + kind: Secret + name: grafana + namespace: monitoring + path: patch-secret.yaml + - target: + kind: Deployment + name: grafana + namespace: monitoring + path: patch-deploy.yaml + - target: + kind: Ingress + name: grafana-ingress + namespace: monitoring + path: patch-ingress_grafana.yaml \ No newline at end of file diff --git a/gpulive/vm-monitoring/overlays/default/grafana/patch-cm.yaml b/gpulive/vm-monitoring/overlays/default/grafana/patch-cm.yaml new file mode 100644 index 0000000..2019016 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/default/grafana/patch-cm.yaml @@ -0,0 +1,32 @@ +# 도메인 정보 변경 필요. +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.10.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + app.kubernetes.io/version: "11.5.1" +data: + + grafana.ini: | + [analytics] + check_for_updates = true + [grafana_net] + url = https://grafana.net + [log] + mode = console + [paths] + data = /var/lib/grafana/ + logs = /var/log/grafana + plugins = /var/lib/grafana/plugins + provisioning = /etc/grafana/provisioning + [server] + domain = 'grafana.nhngpuaas.com' + root_url = 'https://grafana.nhngpuaas.com' + [auth.anonymous] + enabled = true + org_name = Main Org. + org_role = Viewer diff --git a/gpulive/vm-monitoring/overlays/default/grafana/patch-deploy.yaml b/gpulive/vm-monitoring/overlays/default/grafana/patch-deploy.yaml new file mode 100644 index 0000000..5d3f468 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/default/grafana/patch-deploy.yaml @@ -0,0 +1,20 @@ +# Source: grafana/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: monitoring +spec: + template: + spec: + containers: + - name: grafana + volumeMounts: + - name: storage + mountPath: "/var/lib/grafana" + volumes: + - name: storage + persistentVolumeClaim: + claimName: grafana-pvc + # - name: storage + # emptyDir: {} \ No newline at end of file diff --git a/gpulive/monitoring/base/grafana/ingress.yaml b/gpulive/vm-monitoring/overlays/default/grafana/patch-ingress_grafana.yaml similarity index 77% rename from gpulive/monitoring/base/grafana/ingress.yaml rename to gpulive/vm-monitoring/overlays/default/grafana/patch-ingress_grafana.yaml index 665ea2a..6839329 100644 --- a/gpulive/monitoring/base/grafana/ingress.yaml +++ b/gpulive/vm-monitoring/overlays/default/grafana/patch-ingress_grafana.yaml @@ -6,7 +6,7 @@ metadata: spec: ingressClassName: nginx rules: - - host: grafana.sample.com + - host: grafana.nhngpuaas.com http: paths: - backend: @@ -18,5 +18,7 @@ spec: pathType: Prefix tls: - hosts: - - grafana.sample.com - secretName: ssl-cc \ No newline at end of file + - grafana.nhngpuaas.com + secretName: nhngpuaas-ssl + + \ No newline at end of file diff --git a/gpulive/vm-monitoring/overlays/default/grafana/patch-secret.yaml b/gpulive/vm-monitoring/overlays/default/grafana/patch-secret.yaml new file mode 100644 index 0000000..c4f40c6 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/default/grafana/patch-secret.yaml @@ -0,0 +1,17 @@ +--- +# Source: grafana/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.10.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + app.kubernetes.io/version: "11.5.1" +type: Opaque +data: + admin-user: "YWRtaW4=" + admin-password: "bmhuIUAjMTIz" + # ldap-toml: "" \ No newline at end of file diff --git a/gpulive/vm-monitoring/overlays/default/kustomization.yaml b/gpulive/vm-monitoring/overlays/default/kustomization.yaml new file mode 100644 index 0000000..09bb902 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/default/kustomization.yaml @@ -0,0 +1,17 @@ +# Prometheus에 종속적으로 설치 +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: monitoring + +resources: + - prometheus + - grafana + +generatorOptions: + disableNameSuffixHash: true +secretGenerator: + - name: basic-auth + files: + - auth + type: Opaque \ No newline at end of file diff --git a/gpulive/vm-monitoring/overlays/default/prometheus/kustomization.yaml b/gpulive/vm-monitoring/overlays/default/prometheus/kustomization.yaml new file mode 100644 index 0000000..1d35c34 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/default/prometheus/kustomization.yaml @@ -0,0 +1,34 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../../base/prometheus + +namespace: monitoring + +patches: + - target: + kind: ConfigMap + name: prometheus-server + namespace: monitoring + path: patch-cm-prometheus-server.yaml + - target: + kind: Deployment + name: prometheus-server + namespace: monitoring + path: patch-deploy.yaml + - target: + kind: Ingress + name: prometheus-ingress + namespace: monitoring + path: patch-ingress_prometheus.yaml + - target: + kind: Ingress + name: prometheus-alertmanager-ingress + namespace: monitoring + path: patch-ingress_alertmanager.yaml + # - target: + # kind: Secret + # name: basic-auth + # namespace: monitoring + # path: patch-basic-auth.yaml diff --git a/gpulive/vm-monitoring/overlays/default/prometheus/patch-cm-prometheus-server.yaml b/gpulive/vm-monitoring/overlays/default/prometheus/patch-cm-prometheus-server.yaml new file mode 100644 index 0000000..b684a13 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/default/prometheus/patch-cm-prometheus-server.yaml @@ -0,0 +1,335 @@ +# - "/etc/config/rules/*.yaml" # 수정대상 +# - "/etc/config/alerts/*.yaml" # 수정대상 +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-server + namespace: monitoring +data: + prometheus.yml: | + global: + evaluation_interval: 1m + scrape_interval: 1m + scrape_timeout: 10s + rule_files: + - /etc/config/recording_rules.yml + - /etc/config/alerting_rules.yml + - "/etc/config/rules/*.yaml" + - "/etc/config/alerts/*.yaml" + scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - localhost:9090 + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-apiservers + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: default;kubernetes;https + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_service_name + - __meta_kubernetes_endpoint_port_name + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes-cadvisor + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - honor_labels: true + job_name: kubernetes-service-endpoints + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-service-endpoints-slow + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + - honor_labels: true + job_name: prometheus-pushgateway + kubernetes_sd_configs: + - role: service + relabel_configs: + - action: keep + regex: pushgateway + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - honor_labels: true + job_name: kubernetes-services + kubernetes_sd_configs: + - role: service + metrics_path: /probe + params: + module: + - http_2xx + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - source_labels: + - __address__ + target_label: __param_target + - replacement: blackbox + target_label: __address__ + - source_labels: + - __param_target + target_label: instance + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - honor_labels: true + job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-pods-slow + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + alerting: + alertmanagers: + - kubernetes_sd_configs: + - role: pod + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_namespace] + regex: monitoring + action: keep + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance] + regex: prometheus + action: keep + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + regex: alertmanager + action: keep + - source_labels: [__meta_kubernetes_pod_container_port_number] + regex: "9093" + action: keep diff --git a/gpulive/vm-monitoring/overlays/default/prometheus/patch-deploy.yaml b/gpulive/vm-monitoring/overlays/default/prometheus/patch-deploy.yaml new file mode 100644 index 0000000..1758854 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/default/prometheus/patch-deploy.yaml @@ -0,0 +1,19 @@ +# Source: prometheus/templates/deploy.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-server + namespace: monitoring +spec: + template: + spec: + containers: + - name: prometheus-server + volumeMounts: + - mountPath: /etc/config/alerts + name: config-volume2 + volumes: + - configMap: + defaultMode: 420 + name: prometheus-rulefiles + name: config-volume2 \ No newline at end of file diff --git a/gpulive/vm-monitoring/overlays/default/prometheus/patch-ingress_alertmanager.yaml b/gpulive/vm-monitoring/overlays/default/prometheus/patch-ingress_alertmanager.yaml new file mode 100644 index 0000000..d21488b --- /dev/null +++ b/gpulive/vm-monitoring/overlays/default/prometheus/patch-ingress_alertmanager.yaml @@ -0,0 +1,29 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prometheus-alertmanager-ingress + namespace: monitoring + annotations: + # 인증 방법 설정: basic auth + nginx.ingress.kubernetes.io/auth-type: basic + # basic auth 사용자가 들어있는 secret 설정 + nginx.ingress.kubernetes.io/auth-secret: basic-auth + # 인증 요청시 나오는 메세지 설정 + nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - admin' +spec: + ingressClassName: nginx + rules: + - host: alert.nhngpuaas.com + http: + paths: + - backend: + service: + name: prometheus-alertmanager + port: + number: 9093 + path: / + pathType: Prefix + tls: + - hosts: + - alert.nhngpuaas.com + secretName: nhngpuaas-ssl \ No newline at end of file diff --git a/gpulive/vm-monitoring/overlays/default/prometheus/patch-ingress_prometheus.yaml b/gpulive/vm-monitoring/overlays/default/prometheus/patch-ingress_prometheus.yaml new file mode 100644 index 0000000..c03f487 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/default/prometheus/patch-ingress_prometheus.yaml @@ -0,0 +1,22 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prometheus-ingress + namespace: monitoring +spec: + ingressClassName: nginx + rules: + - host: prometheus.nhngpuaas.com + http: + paths: + - backend: + service: + name: prometheus-server + port: + number: 80 + path: / + pathType: Prefix + tls: + - hosts: + - prometheus.nhngpuaas.com + secretName: nhngpuaas-ssl diff --git a/gpulive/vm-monitoring/overlays/vm/auth b/gpulive/vm-monitoring/overlays/vm/auth new file mode 100644 index 0000000..cb0bcdd --- /dev/null +++ b/gpulive/vm-monitoring/overlays/vm/auth @@ -0,0 +1 @@ +admin:$apr1$JIoWDh3p$Mo4E8nEh3beHp9n1IRjmc/ diff --git a/gpulive/vm-monitoring/overlays/vm/grafana/kustomization.yaml b/gpulive/vm-monitoring/overlays/vm/grafana/kustomization.yaml new file mode 100644 index 0000000..2f05aa9 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/vm/grafana/kustomization.yaml @@ -0,0 +1,29 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../base/grafana + +namespace: monitoring + +patches: + - target: + kind: ConfigMap + name: grafana + namespace: monitoring + path: patch-cm.yaml + - target: + kind: Secret + name: grafana + namespace: monitoring + path: patch-secret.yaml + - target: + kind: Deployment + name: grafana + namespace: monitoring + path: patch-deploy.yaml + - target: + kind: Ingress + name: grafana-ingress + namespace: monitoring + path: patch-ingress_grafana.yaml \ No newline at end of file diff --git a/gpulive/vm-monitoring/overlays/vm/grafana/patch-cm.yaml b/gpulive/vm-monitoring/overlays/vm/grafana/patch-cm.yaml new file mode 100644 index 0000000..8df51ff --- /dev/null +++ b/gpulive/vm-monitoring/overlays/vm/grafana/patch-cm.yaml @@ -0,0 +1,32 @@ +# 도메인 정보 변경 필요. +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.10.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + app.kubernetes.io/version: "11.5.1" +data: + + grafana.ini: | + [analytics] + check_for_updates = true + [grafana_net] + url = https://grafana.net + [log] + mode = console + [paths] + data = /var/lib/grafana/ + logs = /var/log/grafana + plugins = /var/lib/grafana/plugins + provisioning = /etc/grafana/provisioning + [server] + domain = 'monitoring.gpulive.nhncloud.com' + root_url = 'https://monitoring.gpulive.nhncloud.com' + [auth.anonymous] + enabled = true + org_name = Main Org. + org_role = Viewer diff --git a/gpulive/vm-monitoring/overlays/vm/grafana/patch-deploy.yaml b/gpulive/vm-monitoring/overlays/vm/grafana/patch-deploy.yaml new file mode 100644 index 0000000..5d3f468 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/vm/grafana/patch-deploy.yaml @@ -0,0 +1,20 @@ +# Source: grafana/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: monitoring +spec: + template: + spec: + containers: + - name: grafana + volumeMounts: + - name: storage + mountPath: "/var/lib/grafana" + volumes: + - name: storage + persistentVolumeClaim: + claimName: grafana-pvc + # - name: storage + # emptyDir: {} \ No newline at end of file diff --git a/gpulive/vm-monitoring/overlays/vm/grafana/patch-ingress_grafana.yaml b/gpulive/vm-monitoring/overlays/vm/grafana/patch-ingress_grafana.yaml new file mode 100644 index 0000000..6839329 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/vm/grafana/patch-ingress_grafana.yaml @@ -0,0 +1,24 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: grafana-ingress + namespace: monitoring +spec: + ingressClassName: nginx + rules: + - host: grafana.nhngpuaas.com + http: + paths: + - backend: + service: + name: grafana + port: + number: 80 + path: / + pathType: Prefix + tls: + - hosts: + - grafana.nhngpuaas.com + secretName: nhngpuaas-ssl + + \ No newline at end of file diff --git a/gpulive/vm-monitoring/overlays/vm/grafana/patch-secret.yaml b/gpulive/vm-monitoring/overlays/vm/grafana/patch-secret.yaml new file mode 100644 index 0000000..c4f40c6 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/vm/grafana/patch-secret.yaml @@ -0,0 +1,17 @@ +--- +# Source: grafana/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.10.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + app.kubernetes.io/version: "11.5.1" +type: Opaque +data: + admin-user: "YWRtaW4=" + admin-password: "bmhuIUAjMTIz" + # ldap-toml: "" \ No newline at end of file diff --git a/gpulive/vm-monitoring/overlays/vm/kustomization.yaml b/gpulive/vm-monitoring/overlays/vm/kustomization.yaml new file mode 100644 index 0000000..09bb902 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/vm/kustomization.yaml @@ -0,0 +1,17 @@ +# Prometheus에 종속적으로 설치 +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: monitoring + +resources: + - prometheus + - grafana + +generatorOptions: + disableNameSuffixHash: true +secretGenerator: + - name: basic-auth + files: + - auth + type: Opaque \ No newline at end of file diff --git a/gpulive/vm-monitoring/overlays/vm/prometheus/kustomization.yaml b/gpulive/vm-monitoring/overlays/vm/prometheus/kustomization.yaml new file mode 100644 index 0000000..1d35c34 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/vm/prometheus/kustomization.yaml @@ -0,0 +1,34 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../../base/prometheus + +namespace: monitoring + +patches: + - target: + kind: ConfigMap + name: prometheus-server + namespace: monitoring + path: patch-cm-prometheus-server.yaml + - target: + kind: Deployment + name: prometheus-server + namespace: monitoring + path: patch-deploy.yaml + - target: + kind: Ingress + name: prometheus-ingress + namespace: monitoring + path: patch-ingress_prometheus.yaml + - target: + kind: Ingress + name: prometheus-alertmanager-ingress + namespace: monitoring + path: patch-ingress_alertmanager.yaml + # - target: + # kind: Secret + # name: basic-auth + # namespace: monitoring + # path: patch-basic-auth.yaml diff --git a/gpulive/vm-monitoring/overlays/vm/prometheus/patch-cm-prometheus-server.yaml b/gpulive/vm-monitoring/overlays/vm/prometheus/patch-cm-prometheus-server.yaml new file mode 100644 index 0000000..b684a13 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/vm/prometheus/patch-cm-prometheus-server.yaml @@ -0,0 +1,335 @@ +# - "/etc/config/rules/*.yaml" # 수정대상 +# - "/etc/config/alerts/*.yaml" # 수정대상 +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-server + namespace: monitoring +data: + prometheus.yml: | + global: + evaluation_interval: 1m + scrape_interval: 1m + scrape_timeout: 10s + rule_files: + - /etc/config/recording_rules.yml + - /etc/config/alerting_rules.yml + - "/etc/config/rules/*.yaml" + - "/etc/config/alerts/*.yaml" + scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - localhost:9090 + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-apiservers + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: default;kubernetes;https + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_service_name + - __meta_kubernetes_endpoint_port_name + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes-cadvisor + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - honor_labels: true + job_name: kubernetes-service-endpoints + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-service-endpoints-slow + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + - honor_labels: true + job_name: prometheus-pushgateway + kubernetes_sd_configs: + - role: service + relabel_configs: + - action: keep + regex: pushgateway + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - honor_labels: true + job_name: kubernetes-services + kubernetes_sd_configs: + - role: service + metrics_path: /probe + params: + module: + - http_2xx + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - source_labels: + - __address__ + target_label: __param_target + - replacement: blackbox + target_label: __address__ + - source_labels: + - __param_target + target_label: instance + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - honor_labels: true + job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-pods-slow + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + alerting: + alertmanagers: + - kubernetes_sd_configs: + - role: pod + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_namespace] + regex: monitoring + action: keep + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance] + regex: prometheus + action: keep + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + regex: alertmanager + action: keep + - source_labels: [__meta_kubernetes_pod_container_port_number] + regex: "9093" + action: keep diff --git a/gpulive/vm-monitoring/overlays/vm/prometheus/patch-deploy.yaml b/gpulive/vm-monitoring/overlays/vm/prometheus/patch-deploy.yaml new file mode 100644 index 0000000..1758854 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/vm/prometheus/patch-deploy.yaml @@ -0,0 +1,19 @@ +# Source: prometheus/templates/deploy.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-server + namespace: monitoring +spec: + template: + spec: + containers: + - name: prometheus-server + volumeMounts: + - mountPath: /etc/config/alerts + name: config-volume2 + volumes: + - configMap: + defaultMode: 420 + name: prometheus-rulefiles + name: config-volume2 \ No newline at end of file diff --git a/gpulive/vm-monitoring/overlays/vm/prometheus/patch-ingress_alertmanager.yaml b/gpulive/vm-monitoring/overlays/vm/prometheus/patch-ingress_alertmanager.yaml new file mode 100644 index 0000000..d21488b --- /dev/null +++ b/gpulive/vm-monitoring/overlays/vm/prometheus/patch-ingress_alertmanager.yaml @@ -0,0 +1,29 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prometheus-alertmanager-ingress + namespace: monitoring + annotations: + # 인증 방법 설정: basic auth + nginx.ingress.kubernetes.io/auth-type: basic + # basic auth 사용자가 들어있는 secret 설정 + nginx.ingress.kubernetes.io/auth-secret: basic-auth + # 인증 요청시 나오는 메세지 설정 + nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - admin' +spec: + ingressClassName: nginx + rules: + - host: alert.nhngpuaas.com + http: + paths: + - backend: + service: + name: prometheus-alertmanager + port: + number: 9093 + path: / + pathType: Prefix + tls: + - hosts: + - alert.nhngpuaas.com + secretName: nhngpuaas-ssl \ No newline at end of file diff --git a/gpulive/vm-monitoring/overlays/vm/prometheus/patch-ingress_prometheus.yaml b/gpulive/vm-monitoring/overlays/vm/prometheus/patch-ingress_prometheus.yaml new file mode 100644 index 0000000..c03f487 --- /dev/null +++ b/gpulive/vm-monitoring/overlays/vm/prometheus/patch-ingress_prometheus.yaml @@ -0,0 +1,22 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prometheus-ingress + namespace: monitoring +spec: + ingressClassName: nginx + rules: + - host: prometheus.nhngpuaas.com + http: + paths: + - backend: + service: + name: prometheus-server + port: + number: 80 + path: / + pathType: Prefix + tls: + - hosts: + - prometheus.nhngpuaas.com + secretName: nhngpuaas-ssl diff --git a/nodegroup/make_nodegroup b/nodegroup/make_nodegroup new file mode 100755 index 0000000..77871e6 --- /dev/null +++ b/nodegroup/make_nodegroup @@ -0,0 +1,4 @@ +for i in $(seq 1 24); do + kubectl label node gpu-$i nodegroup=gpu +done + diff --git a/template-gpu-operator/crds/nvidia.com_clusterpolicies.yaml b/template-gpu-operator/crds/nvidia.com_clusterpolicies.yaml new file mode 100644 index 0000000..8ee8e9a --- /dev/null +++ b/template-gpu-operator/crds/nvidia.com_clusterpolicies.yaml @@ -0,0 +1,2384 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: clusterpolicies.nvidia.com +spec: + group: nvidia.com + names: + kind: ClusterPolicy + listKind: ClusterPolicyList + plural: clusterpolicies + singular: clusterpolicy + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: string + name: v1 + schema: + openAPIV3Schema: + description: ClusterPolicy is the Schema for the clusterpolicies API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: ClusterPolicySpec defines the desired state of ClusterPolicy + properties: + ccManager: + description: CCManager component spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + defaultMode: + description: Default CC mode setting for compatible GPUs on the + node + enum: + - "on" + - "off" + - devtools + type: string + enabled: + description: Enabled indicates if deployment of CC Manager is + enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: CC Manager image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: CC Manager image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + version: + description: CC Manager image tag + type: string + type: object + cdi: + description: CDI configures how the Container Device Interface is + used in the cluster + properties: + default: + default: false + description: Default indicates whether to use CDI as the default + mechanism for providing GPU access to containers. + type: boolean + enabled: + default: false + description: Enabled indicates whether CDI can be used to make + GPUs accessible to containers. + type: boolean + type: object + daemonsets: + description: Daemonset defines common configuration for all Daemonsets + properties: + annotations: + additionalProperties: + type: string + description: |- + Optional: Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + type: object + labels: + additionalProperties: + type: string + description: |- + Optional: Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + type: object + priorityClassName: + type: string + rollingUpdate: + description: 'Optional: Configuration for rolling update of all + DaemonSet pods' + properties: + maxUnavailable: + type: string + type: object + tolerations: + description: 'Optional: Set tolerations' + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + updateStrategy: + default: RollingUpdate + enum: + - RollingUpdate + - OnDelete + type: string + type: object + dcgm: + description: DCGM component spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + enabled: + description: Enabled indicates if deployment of NVIDIA DCGM Hostengine + as a separate pod is enabled. + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + hostPort: + description: 'Deprecated: HostPort represents host port that needs + to be bound for DCGM engine (Default: 5555)' + format: int32 + type: integer + image: + description: NVIDIA DCGM image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA DCGM image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + version: + description: NVIDIA DCGM image tag + type: string + type: object + dcgmExporter: + description: DCGMExporter spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + config: + description: 'Optional: Custom metrics configuration for NVIDIA + DCGM Exporter' + properties: + name: + description: ConfigMap name with file dcgm-metrics.csv for + metrics to be collected by NVIDIA DCGM Exporter + type: string + type: object + enabled: + description: Enabled indicates if deployment of NVIDIA DCGM Exporter + through operator is enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: NVIDIA DCGM Exporter image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA DCGM Exporter image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + serviceMonitor: + description: 'Optional: ServiceMonitor configuration for NVIDIA + DCGM Exporter' + properties: + additionalLabels: + additionalProperties: + type: string + description: AdditionalLabels to add to ServiceMonitor instance + for NVIDIA DCGM Exporter + type: object + enabled: + description: Enabled indicates if ServiceMonitor is deployed + for NVIDIA DCGM Exporter + type: boolean + honorLabels: + description: HonorLabels chooses the metric’s labels on collisions + with target labels. + type: boolean + interval: + description: |- + Interval which metrics should be scraped from NVIDIA DCGM Exporter. If not specified Prometheus’ global scrape interval is used. + Supported units: y, w, d, h, m, s, ms + pattern: ^(0|(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?)$ + type: string + relabelings: + description: Relabelings allows to rewrite labels on metric + sets for NVIDIA DCGM Exporter + items: + description: |- + RelabelConfig allows dynamic rewriting of the label set for targets, alerts, + scraped samples and remote write samples. + + More info: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#relabel_config + properties: + action: + default: replace + description: |- + Action to perform based on the regex matching. + + `Uppercase` and `Lowercase` actions require Prometheus >= v2.36.0. + `DropEqual` and `KeepEqual` actions require Prometheus >= v2.41.0. + + Default: "Replace" + enum: + - replace + - Replace + - keep + - Keep + - drop + - Drop + - hashmod + - HashMod + - labelmap + - LabelMap + - labeldrop + - LabelDrop + - labelkeep + - LabelKeep + - lowercase + - Lowercase + - uppercase + - Uppercase + - keepequal + - KeepEqual + - dropequal + - DropEqual + type: string + modulus: + description: |- + Modulus to take of the hash of the source label values. + + Only applicable when the action is `HashMod`. + format: int64 + type: integer + regex: + description: Regular expression against which the extracted + value is matched. + type: string + replacement: + description: |- + Replacement value against which a Replace action is performed if the + regular expression matches. + + Regex capture groups are available. + type: string + separator: + description: Separator is the string between concatenated + SourceLabels. + type: string + sourceLabels: + description: |- + The source labels select values from existing labels. Their content is + concatenated using the configured Separator and matched against the + configured regular expression. + items: + description: |- + LabelName is a valid Prometheus label name which may only contain ASCII + letters, numbers, as well as underscores. + pattern: ^[a-zA-Z_][a-zA-Z0-9_]*$ + type: string + type: array + targetLabel: + description: |- + Label to which the resulting string is written in a replacement. + + It is mandatory for `Replace`, `HashMod`, `Lowercase`, `Uppercase`, + `KeepEqual` and `DropEqual` actions. + + Regex capture groups are available. + type: string + type: object + type: array + type: object + version: + description: NVIDIA DCGM Exporter image tag + type: string + type: object + devicePlugin: + description: DevicePlugin component spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + config: + description: 'Optional: Configuration for the NVIDIA Device Plugin + via the ConfigMap' + properties: + default: + description: Default config name within the ConfigMap for + the NVIDIA Device Plugin config + type: string + name: + description: ConfigMap name for NVIDIA Device Plugin config + including shared config between plugin and GFD + type: string + type: object + enabled: + description: Enabled indicates if deployment of NVIDIA Device + Plugin through operator is enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: NVIDIA Device Plugin image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + mps: + description: 'Optional: MPS related configuration for the NVIDIA + Device Plugin' + properties: + root: + default: /run/nvidia/mps + description: Root defines the MPS root path on the host + type: string + type: object + repository: + description: NVIDIA Device Plugin image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + version: + description: NVIDIA Device Plugin image tag + type: string + type: object + driver: + description: Driver component spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + certConfig: + description: 'Optional: Custom certificates configuration for + NVIDIA Driver container' + properties: + name: + type: string + type: object + enabled: + description: Enabled indicates if deployment of NVIDIA Driver + through operator is enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: NVIDIA Driver image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + kernelModuleConfig: + description: 'Optional: Kernel module configuration parameters + for the NVIDIA Driver' + properties: + name: + type: string + type: object + licensingConfig: + description: 'Optional: Licensing configuration for NVIDIA vGPU + licensing' + properties: + configMapName: + type: string + nlsEnabled: + description: NLSEnabled indicates if NVIDIA Licensing System + is used for licensing. + type: boolean + type: object + livenessProbe: + description: NVIDIA Driver container liveness probe settings + properties: + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + minimum: 1 + type: integer + type: object + manager: + description: Manager represents configuration for NVIDIA Driver + Manager initContainer + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: Image represents NVIDIA Driver Manager image + name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: Repository represents Driver Managerrepository + path + type: string + version: + description: Version represents NVIDIA Driver Manager image + tag(version) + type: string + type: object + rdma: + description: GPUDirectRDMASpec defines the properties for nvidia-peermem + deployment + properties: + enabled: + description: Enabled indicates if GPUDirect RDMA is enabled + through GPU operator + type: boolean + useHostMofed: + description: UseHostMOFED indicates to use MOFED drivers directly + installed on the host to enable GPUDirect RDMA + type: boolean + type: object + readinessProbe: + description: NVIDIA Driver container readiness probe settings + properties: + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + minimum: 1 + type: integer + type: object + repoConfig: + description: 'Optional: Custom repo configuration for NVIDIA Driver + container' + properties: + configMapName: + type: string + type: object + repository: + description: NVIDIA Driver image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + startupProbe: + description: NVIDIA Driver container startup probe settings + properties: + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + minimum: 1 + type: integer + type: object + upgradePolicy: + description: Driver auto-upgrade settings + properties: + autoUpgrade: + default: false + description: |- + AutoUpgrade is a global switch for automatic upgrade feature + if set to false all other options are ignored + type: boolean + drain: + description: DrainSpec describes configuration for node drain + during automatic upgrade + properties: + deleteEmptyDir: + default: false + description: |- + DeleteEmptyDir indicates if should continue even if there are pods using emptyDir + (local data that will be deleted when the node is drained) + type: boolean + enable: + default: false + description: Enable indicates if node draining is allowed + during upgrade + type: boolean + force: + default: false + description: Force indicates if force draining is allowed + type: boolean + podSelector: + description: |- + PodSelector specifies a label selector to filter pods on the node that need to be drained + For more details on label selectors, see: + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + type: string + timeoutSeconds: + default: 300 + description: TimeoutSecond specifies the length of time + in seconds to wait before giving up drain, zero means + infinite + minimum: 0 + type: integer + type: object + maxParallelUpgrades: + default: 1 + description: |- + MaxParallelUpgrades indicates how many nodes can be upgraded in parallel + 0 means no limit, all nodes will be upgraded in parallel + minimum: 0 + type: integer + maxUnavailable: + anyOf: + - type: integer + - type: string + default: 25% + description: |- + MaxUnavailable is the maximum number of nodes with the driver installed, that can be unavailable during the upgrade. + Value can be an absolute number (ex: 5) or a percentage of total nodes at the start of upgrade (ex: 10%). + Absolute number is calculated from percentage by rounding up. + By default, a fixed value of 25% is used. + x-kubernetes-int-or-string: true + podDeletion: + description: PodDeletionSpec describes configuration for deletion + of pods using special resources during automatic upgrade + properties: + deleteEmptyDir: + default: false + description: |- + DeleteEmptyDir indicates if should continue even if there are pods using emptyDir + (local data that will be deleted when the pod is deleted) + type: boolean + force: + default: false + description: Force indicates if force deletion is allowed + type: boolean + timeoutSeconds: + default: 300 + description: |- + TimeoutSecond specifies the length of time in seconds to wait before giving up on pod termination, zero means + infinite + minimum: 0 + type: integer + type: object + waitForCompletion: + description: WaitForCompletionSpec describes the configuration + for waiting on job completions + properties: + podSelector: + description: |- + PodSelector specifies a label selector for the pods to wait for completion + For more details on label selectors, see: + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + type: string + timeoutSeconds: + default: 0 + description: |- + TimeoutSecond specifies the length of time in seconds to wait before giving up on pod termination, zero means + infinite + minimum: 0 + type: integer + type: object + type: object + useNvidiaDriverCRD: + description: UseNvidiaDriverCRD indicates if the deployment of + NVIDIA Driver is managed by the NVIDIADriver CRD type + type: boolean + useOpenKernelModules: + description: UseOpenKernelModules indicates if the open GPU kernel + modules should be used + type: boolean + usePrecompiled: + description: UsePrecompiled indicates if deployment of NVIDIA + Driver using pre-compiled modules is enabled + type: boolean + version: + description: NVIDIA Driver image tag + type: string + virtualTopology: + description: 'Optional: Virtual Topology Daemon configuration + for NVIDIA vGPU drivers' + properties: + config: + description: 'Optional: Config name representing virtual topology + daemon configuration file nvidia-topologyd.conf' + type: string + type: object + type: object + gdrcopy: + description: GDRCopy component spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + enabled: + description: Enabled indicates if GDRCopy is enabled through GPU + Operator + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: NVIDIA GDRCopy driver image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA GDRCopy driver image repository + type: string + version: + description: NVIDIA GDRCopy driver image tag + type: string + type: object + gds: + description: GPUDirectStorage defines the spec for GDS components(Experimental) + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + enabled: + description: Enabled indicates if GPUDirect Storage is enabled + through GPU operator + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: NVIDIA GPUDirect Storage Driver image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA GPUDirect Storage Driver image repository + type: string + version: + description: NVIDIA GPUDirect Storage Driver image tag + type: string + type: object + gfd: + description: GPUFeatureDiscovery spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + enabled: + description: Enabled indicates if deployment of GPU Feature Discovery + Plugin is enabled. + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: GFD image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: GFD image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + version: + description: GFD image tag + type: string + type: object + hostPaths: + description: HostPaths defines various paths on the host needed by + GPU Operator components + properties: + driverInstallDir: + description: |- + DriverInstallDir represents the root at which driver files including libraries, + config files, and executables can be found. + type: string + rootFS: + description: |- + RootFS represents the path to the root filesystem of the host. + This is used by components that need to interact with the host filesystem + and as such this must be a chroot-able filesystem. + Examples include the MIG Manager and Toolkit Container which may need to + stop, start, or restart systemd services. + type: string + type: object + kataManager: + description: KataManager component spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + config: + description: Kata Manager config + properties: + artifactsDir: + default: /opt/nvidia-gpu-operator/artifacts/runtimeclasses + description: |- + ArtifactsDir is the directory where kata artifacts (e.g. kernel / guest images, configuration, etc.) + are placed on the local filesystem. + type: string + runtimeClasses: + description: RuntimeClasses is a list of kata runtime classes + to configure. + items: + description: RuntimeClass defines the configuration for + a kata RuntimeClass + properties: + artifacts: + description: Artifacts are the kata artifacts associated + with the runtime class. + properties: + pullSecret: + description: PullSecret is the secret used to pull + the OCI artifact. + type: string + url: + description: |- + URL is the path to the OCI artifact (payload) containing all artifacts + associated with a kata runtime class. + type: string + required: + - url + type: object + name: + description: Name is the name of the kata runtime class. + type: string + nodeSelector: + additionalProperties: + type: string + description: |- + NodeSelector specifies the nodeSelector for the RuntimeClass object. + This ensures pods running with the RuntimeClass only get scheduled + onto nodes which support it. + type: object + required: + - artifacts + - name + type: object + type: array + type: object + enabled: + description: Enabled indicates if deployment of Kata Manager is + enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: Kata Manager image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: Kata Manager image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + version: + description: Kata Manager image tag + type: string + type: object + mig: + description: MIG spec + properties: + strategy: + description: 'Optional: MIGStrategy to apply for GFD and NVIDIA + Device Plugin' + enum: + - none + - single + - mixed + type: string + type: object + migManager: + description: MIGManager for configuration to deploy MIG Manager + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + config: + description: 'Optional: Custom mig-parted configuration for NVIDIA + MIG Manager container' + properties: + default: + default: all-disabled + description: Default MIG config to be applied on the node, + when there is no config specified with the node label nvidia.com/mig.config + enum: + - all-disabled + - "" + type: string + name: + default: default-mig-parted-config + description: ConfigMap name + type: string + type: object + enabled: + description: Enabled indicates if deployment of NVIDIA MIG Manager + is enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + gpuClientsConfig: + description: 'Optional: Custom gpu-clients configuration for NVIDIA + MIG Manager container' + properties: + name: + description: ConfigMap name + type: string + type: object + image: + description: NVIDIA MIG Manager image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA MIG Manager image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + version: + description: NVIDIA MIG Manager image tag + type: string + type: object + nodeStatusExporter: + description: NodeStatusExporter spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + enabled: + description: Enabled indicates if deployment of Node Status Exporter + is enabled. + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: Node Status Exporter image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: Node Status Exporterimage repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + version: + description: Node Status Exporterimage tag + type: string + type: object + operator: + description: Operator component spec + properties: + annotations: + additionalProperties: + type: string + description: |- + Optional: Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + type: object + defaultRuntime: + default: docker + description: Runtime defines container runtime type + enum: + - docker + - crio + - containerd + type: string + initContainer: + description: InitContainerSpec describes configuration for initContainer + image used with all components + properties: + image: + description: Image represents image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: Repository represents image repository path + type: string + version: + description: Version represents image tag(version) + type: string + type: object + labels: + additionalProperties: + type: string + description: |- + Optional: Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + type: object + runtimeClass: + default: nvidia + type: string + use_ocp_driver_toolkit: + description: UseOpenShiftDriverToolkit indicates if DriverToolkit + image should be used on OpenShift to build and install driver + modules + type: boolean + required: + - defaultRuntime + type: object + psa: + description: PSA defines spec for PodSecurityAdmission configuration + properties: + enabled: + description: Enabled indicates if PodSecurityAdmission configuration + needs to be enabled for all Pods + type: boolean + type: object + psp: + description: |- + Deprecated: Pod Security Policies are no longer supported. Please use PodSecurityAdmission instead + PSP defines spec for handling PodSecurityPolicies + properties: + enabled: + description: Enabled indicates if PodSecurityPolicies needs to + be enabled for all Pods + type: boolean + type: object + sandboxDevicePlugin: + description: SandboxDevicePlugin component spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + enabled: + description: Enabled indicates if deployment of NVIDIA Sandbox + Device Plugin through operator is enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: NVIDIA Sandbox Device Plugin image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA Sandbox Device Plugin image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + version: + description: NVIDIA Sandbox Device Plugin image tag + type: string + type: object + sandboxWorkloads: + description: SandboxWorkloads defines the spec for handling sandbox + workloads (i.e. Virtual Machines) + properties: + defaultWorkload: + default: container + description: |- + DefaultWorkload indicates the default GPU workload type to configure + worker nodes in the cluster for + enum: + - container + - vm-passthrough + - vm-vgpu + type: string + enabled: + description: |- + Enabled indicates if the GPU Operator should manage additional operands required + for sandbox workloads (i.e. VFIO Manager, vGPU Manager, and additional device plugins) + type: boolean + type: object + toolkit: + description: Toolkit component spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + enabled: + description: Enabled indicates if deployment of NVIDIA Container + Toolkit through operator is enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: NVIDIA Container Toolkit image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + installDir: + default: /usr/local/nvidia + description: Toolkit install directory on the host + type: string + repository: + description: NVIDIA Container Toolkit image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + version: + description: NVIDIA Container Toolkit image tag + type: string + type: object + validator: + description: Validator defines the spec for operator-validator daemonset + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + cuda: + description: CUDA validator spec + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + type: object + driver: + description: Toolkit validator spec + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + type: object + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: Validator image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + plugin: + description: Plugin validator spec + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + type: object + repository: + description: Validator image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + toolkit: + description: Toolkit validator spec + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + type: object + version: + description: Validator image tag + type: string + vfioPCI: + description: VfioPCI validator spec + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + type: object + vgpuDevices: + description: VGPUDevices validator spec + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + type: object + vgpuManager: + description: VGPUManager validator spec + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + type: object + type: object + vfioManager: + description: VFIOManager for configuration to deploy VFIO-PCI Manager + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + driverManager: + description: DriverManager represents configuration for NVIDIA + Driver Manager + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: Image represents NVIDIA Driver Manager image + name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: Repository represents Driver Managerrepository + path + type: string + version: + description: Version represents NVIDIA Driver Manager image + tag(version) + type: string + type: object + enabled: + description: Enabled indicates if deployment of VFIO Manager is + enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: VFIO Manager image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: VFIO Manager image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + version: + description: VFIO Manager image tag + type: string + type: object + vgpuDeviceManager: + description: VGPUDeviceManager spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + config: + description: NVIDIA vGPU devices configuration for NVIDIA vGPU + Device Manager container + properties: + default: + default: default + description: Default config name within the ConfigMap + type: string + name: + description: ConfigMap name + type: string + type: object + enabled: + description: Enabled indicates if deployment of NVIDIA vGPU Device + Manager is enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: NVIDIA vGPU Device Manager image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA vGPU Device Manager image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + version: + description: NVIDIA vGPU Device Manager image tag + type: string + type: object + vgpuManager: + description: VGPUManager component spec + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + driverManager: + description: DriverManager represents configuration for NVIDIA + Driver Manager initContainer + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: Image represents NVIDIA Driver Manager image + name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: Repository represents Driver Managerrepository + path + type: string + version: + description: Version represents NVIDIA Driver Manager image + tag(version) + type: string + type: object + enabled: + description: Enabled indicates if deployment of NVIDIA vGPU Manager + through operator is enabled + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: NVIDIA vGPU Manager image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA vGPU Manager image repository + type: string + resources: + description: 'Optional: Define resources requests and limits for + each pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + version: + description: NVIDIA vGPU Manager image tag + type: string + type: object + required: + - daemonsets + - dcgm + - dcgmExporter + - devicePlugin + - driver + - gfd + - nodeStatusExporter + - operator + - toolkit + type: object + status: + description: ClusterPolicyStatus defines the observed state of ClusterPolicy + properties: + conditions: + description: Conditions is a list of conditions representing the ClusterPolicy's + current state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + namespace: + description: Namespace indicates a namespace in which the operator + is installed + type: string + state: + description: State indicates status of ClusterPolicy + enum: + - ignored + - ready + - notReady + type: string + required: + - state + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/template-gpu-operator/crds/nvidia.com_nvidiadrivers.yaml b/template-gpu-operator/crds/nvidia.com_nvidiadrivers.yaml new file mode 100644 index 0000000..0721557 --- /dev/null +++ b/template-gpu-operator/crds/nvidia.com_nvidiadrivers.yaml @@ -0,0 +1,797 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: nvidiadrivers.nvidia.com +spec: + group: nvidia.com + names: + kind: NVIDIADriver + listKind: NVIDIADriverList + plural: nvidiadrivers + shortNames: + - nvd + - nvdriver + - nvdrivers + singular: nvidiadriver + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: NVIDIADriver is the Schema for the nvidiadrivers API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NVIDIADriverSpec defines the desired state of NVIDIADriver + properties: + annotations: + additionalProperties: + type: string + description: |- + Optional: Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + type: object + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + certConfig: + description: 'Optional: Custom certificates configuration for NVIDIA + Driver container' + properties: + name: + type: string + type: object + driverType: + default: gpu + description: DriverType defines NVIDIA driver type + enum: + - gpu + - vgpu + - vgpu-host-manager + type: string + x-kubernetes-validations: + - message: driverType is an immutable field. Please create a new NvidiaDriver + resource instead when you want to change this setting. + rule: self == oldSelf + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present in + a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + gdrcopy: + description: GDRCopy defines the spec for GDRCopy driver + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + enabled: + description: Enabled indicates if GDRCopy is enabled through GPU + operator + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: GDRCopy driver image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: GDRCopy diver image repository + type: string + version: + description: GDRCopy driver image tag + type: string + type: object + gds: + description: GPUDirectStorage defines the spec for GDS driver + properties: + args: + description: 'Optional: List of arguments' + items: + type: string + type: array + enabled: + description: Enabled indicates if GPUDirect Storage is enabled + through GPU operator + type: boolean + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: NVIDIA GPUDirect Storage Driver image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: NVIDIA GPUDirect Storage Driver image repository + type: string + version: + description: NVIDIA GPUDirect Storage Driver image tag + type: string + type: object + image: + default: nvcr.io/nvidia/driver + description: NVIDIA Driver container image name + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + kernelModuleConfig: + description: 'Optional: Kernel module configuration parameters for + the NVIDIA Driver' + properties: + name: + type: string + type: object + labels: + additionalProperties: + type: string + description: |- + Optional: Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + type: object + licensingConfig: + description: 'Optional: Licensing configuration for NVIDIA vGPU licensing' + properties: + name: + type: string + nlsEnabled: + description: NLSEnabled indicates if NVIDIA Licensing System is + used for licensing. + type: boolean + type: object + livenessProbe: + description: NVIDIA Driver container liveness probe settings + properties: + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + minimum: 1 + type: integer + type: object + manager: + description: Manager represents configuration for NVIDIA Driver Manager + initContainer + properties: + env: + description: 'Optional: List of environment variables' + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. + type: string + value: + description: Value of the environment variable. + type: string + required: + - name + type: object + type: array + image: + description: Image represents NVIDIA Driver Manager image name + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullPolicy: + description: Image pull policy + type: string + imagePullSecrets: + description: Image pull secrets + items: + type: string + type: array + repository: + description: Repository represents Driver Managerrepository path + type: string + version: + description: Version represents NVIDIA Driver Manager image tag(version) + type: string + type: object + nodeAffinity: + description: Affinity specifies node affinity rules for driver pods + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding + weight. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The + terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector specifies a selector for installation of + NVIDIA driver + type: object + priorityClassName: + description: 'Optional: Set priorityClassName' + type: string + rdma: + description: GPUDirectRDMA defines the spec for NVIDIA Peer Memory + driver + properties: + enabled: + description: Enabled indicates if GPUDirect RDMA is enabled through + GPU operator + type: boolean + useHostMofed: + description: UseHostMOFED indicates to use MOFED drivers directly + installed on the host to enable GPUDirect RDMA + type: boolean + type: object + readinessProbe: + description: NVIDIA Driver container readiness probe settings + properties: + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + minimum: 1 + type: integer + type: object + repoConfig: + description: 'Optional: Custom repo configuration for NVIDIA Driver + container' + properties: + name: + type: string + type: object + repository: + description: NVIDIA Driver repository + type: string + resources: + description: 'Optional: Define resources requests and limits for each + pod' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + type: object + startupProbe: + description: NVIDIA Driver container startup probe settings + properties: + failureThreshold: + description: |- + Minimum consecutive failures for the probe to be considered failed after having succeeded. + Defaults to 3. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + initialDelaySeconds: + description: |- + Number of seconds after the container has started before liveness probes are initiated. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + successThreshold: + description: |- + Minimum consecutive successes for the probe to be considered successful after having failed. + Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + minimum: 1 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes + format: int32 + minimum: 1 + type: integer + type: object + tolerations: + description: 'Optional: Set tolerations' + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + useOpenKernelModules: + description: UseOpenKernelModules indicates if the open GPU kernel + modules should be used + type: boolean + usePrecompiled: + description: UsePrecompiled indicates if deployment of NVIDIA Driver + using pre-compiled modules is enabled + type: boolean + x-kubernetes-validations: + - message: usePrecompiled is an immutable field. Please create a new + NvidiaDriver resource instead when you want to change this setting. + rule: self == oldSelf + version: + description: NVIDIA Driver version (or just branch for precompiled + drivers) + type: string + virtualTopologyConfig: + description: 'Optional: Virtual Topology Daemon configuration for + NVIDIA vGPU drivers' + properties: + name: + description: 'Optional: Config name representing virtual topology + daemon configuration file nvidia-topologyd.conf' + type: string + type: object + required: + - driverType + - image + type: object + status: + description: NVIDIADriverStatus defines the observed state of NVIDIADriver + properties: + conditions: + description: Conditions is a list of conditions representing the NVIDIADriver's + current state. + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + namespace: + description: Namespace indicates a namespace in which the operator + and driver are installed + type: string + state: + description: |- + INSERT ADDITIONAL STATUS FIELD - define observed state of cluster + Important: Run "make" to regenerate code after modifying this file + State indicates status of NVIDIADriver instance + enum: + - ignored + - ready + - notReady + type: string + required: + - state + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/template-gpu-operator/gpu-operator.yaml b/template-gpu-operator/gpu-operator.yaml new file mode 100644 index 0000000..43832f2 --- /dev/null +++ b/template-gpu-operator/gpu-operator.yaml @@ -0,0 +1,1512 @@ +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-feature-discovery + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +--- +# Source: gpu-operator/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gpu-operator + namespace: gpu-operator + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/nfd-master-conf.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-node-feature-discovery-master-conf + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +data: + nfd-master.conf: |- + extraLabelNs: + - nvidia.com +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/nfd-worker-conf.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-node-feature-discovery-worker-conf + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +data: + nfd-worker.conf: |- + sources: + pci: + deviceClassWhitelist: + - "02" + - "0200" + - "0207" + - "0300" + - "0302" + deviceLabelFields: + - vendor +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: release-name-node-feature-discovery + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - "" + resources: + - nodes + - nodes/status + verbs: + - get + - patch + - update + - list +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeatures + - nodefeaturerules + - nodefeaturegroups + verbs: + - get + - list + - watch +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeaturegroups/status + verbs: + - patch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - coordination.k8s.io + resources: + - leases + resourceNames: + - "nfd-master.nfd.kubernetes.io" + verbs: + - get + - update +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: release-name-node-feature-discovery-gc + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes/proxy + verbs: + - get +- apiGroups: + - topology.node.k8s.io + resources: + - noderesourcetopologies + verbs: + - delete + - list +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeatures + verbs: + - delete + - list +--- +# Source: gpu-operator/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gpu-operator + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" +rules: +- apiGroups: + - config.openshift.io + resources: + - clusterversions + - proxies + verbs: + - get + - list + - watch +- apiGroups: + - image.openshift.io + resources: + - imagestreams + verbs: + - get + - list + - watch +- apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - create + - get + - list + - watch + - update + - patch + - delete + - use +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - clusterrolebindings + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - create + - watch + - update + - patch +- apiGroups: + - "" + resources: + - events + - pods + - pods/eviction + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - get + - list + - watch +- apiGroups: + - nvidia.com + resources: + - clusterpolicies + - clusterpolicies/finalizers + - clusterpolicies/status + - nvidiadrivers + - nvidiadrivers/finalizers + - nvidiadrivers/status + verbs: + - create + - get + - list + - watch + - update + - patch + - delete + - deletecollection +- apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses + verbs: + - get + - list + - watch + - create +- apiGroups: + - node.k8s.io + resources: + - runtimeclasses + verbs: + - get + - list + - create + - update + - watch + - delete +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - update + - patch + - create +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: release-name-node-feature-discovery + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: release-name-node-feature-discovery +subjects: +- kind: ServiceAccount + name: node-feature-discovery + namespace: gpu-operator +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: release-name-node-feature-discovery-gc + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: release-name-node-feature-discovery-gc +subjects: +- kind: ServiceAccount + name: node-feature-discovery + namespace: gpu-operator +--- +# Source: gpu-operator/templates/clusterrolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: gpu-operator + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" +subjects: +- kind: ServiceAccount + name: gpu-operator + namespace: gpu-operator +- kind: ServiceAccount + name: node-feature-discovery + namespace: gpu-operator +roleRef: + kind: ClusterRole + name: gpu-operator + apiGroup: rbac.authorization.k8s.io +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: release-name-node-feature-discovery-worker + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeatures + verbs: + - create + - get + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - get +--- +# Source: gpu-operator/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: gpu-operator + namespace: gpu-operator + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" +rules: +- apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - apps + resources: + - controllerrevisions + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - pods + - pods/eviction + - secrets + - services + - services/finalizers + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + - prometheusrules + verbs: + - get + - list + - create + - watch + - update + - delete +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: release-name-node-feature-discovery-worker + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: release-name-node-feature-discovery-worker +subjects: +- kind: ServiceAccount + name: node-feature-discovery + namespace: gpu-operator +--- +# Source: gpu-operator/templates/rolebinding.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: gpu-operator + namespace: gpu-operator + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" +subjects: +- kind: ServiceAccount + name: gpu-operator + namespace: gpu-operator +roleRef: + kind: Role + name: gpu-operator + apiGroup: rbac.authorization.k8s.io +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/worker.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: release-name-node-feature-discovery-worker + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm + role: worker +spec: + revisionHistoryLimit: + selector: + matchLabels: + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + role: worker + template: + metadata: + labels: + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + role: worker + spec: + dnsPolicy: ClusterFirstWithHostNet + priorityClassName: system-node-critical + serviceAccountName: node-feature-discovery + securityContext: + {} + hostNetwork: false + containers: + - name: worker + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + image: "registry.k8s.io/nfd/node-feature-discovery:v0.16.6" + imagePullPolicy: IfNotPresent + livenessProbe: + grpc: + port: 8082 + initialDelaySeconds: 10 + readinessProbe: + failureThreshold: 10 + grpc: + port: 8082 + initialDelaySeconds: 5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + resources: + limits: + memory: 512Mi + requests: + cpu: 5m + memory: 64Mi + command: + - "nfd-worker" + args: +# Go over featureGate and add the feature-gate flag + - "-feature-gates=NodeFeatureAPI=true" + - "-feature-gates=NodeFeatureGroupAPI=false" + - "-metrics=8081" + - "-grpc-health=8082" + ports: + - containerPort: 8081 + name: metrics + - containerPort: 8082 + name: health + volumeMounts: + - name: host-boot + mountPath: "/host-boot" + readOnly: true + - name: host-os-release + mountPath: "/host-etc/os-release" + readOnly: true + - name: host-sys + mountPath: "/host-sys" + readOnly: true + - name: host-usr-lib + mountPath: "/host-usr/lib" + readOnly: true + - name: host-lib + mountPath: "/host-lib" + readOnly: true + - name: host-proc-swaps + mountPath: "/host-proc/swaps" + readOnly: true + - name: source-d + mountPath: "/etc/kubernetes/node-feature-discovery/source.d/" + readOnly: true + - name: features-d + mountPath: "/etc/kubernetes/node-feature-discovery/features.d/" + readOnly: true + - name: nfd-worker-conf + mountPath: "/etc/kubernetes/node-feature-discovery" + readOnly: true + volumes: + - name: host-boot + hostPath: + path: "/boot" + - name: host-os-release + hostPath: + path: "/etc/os-release" + - name: host-sys + hostPath: + path: "/sys" + - name: host-usr-lib + hostPath: + path: "/usr/lib" + - name: host-lib + hostPath: + path: "/lib" + - name: host-proc-swaps + hostPath: + path: "/proc/swaps" + - name: source-d + hostPath: + path: "/etc/kubernetes/node-feature-discovery/source.d/" + - name: features-d + hostPath: + path: "/etc/kubernetes/node-feature-discovery/features.d/" + - name: nfd-worker-conf + configMap: + name: release-name-node-feature-discovery-worker-conf + items: + - key: nfd-worker.conf + path: nfd-worker.conf + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal + value: "" + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Equal + value: "" + - effect: NoSchedule + key: nvidia.com/gpu + operator: Exists +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/master.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: release-name-node-feature-discovery-master + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm + role: master +spec: + replicas: 1 + revisionHistoryLimit: + selector: + matchLabels: + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + role: master + template: + metadata: + labels: + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + role: master + spec: + priorityClassName: system-node-critical + serviceAccountName: node-feature-discovery + enableServiceLinks: false + securityContext: + {} + hostNetwork: false + containers: + - name: master + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + image: "registry.k8s.io/nfd/node-feature-discovery:v0.16.6" + imagePullPolicy: IfNotPresent + livenessProbe: + grpc: + port: 8082 + initialDelaySeconds: 10 + readinessProbe: + failureThreshold: 10 + grpc: + port: 8082 + initialDelaySeconds: 5 + ports: + - containerPort: 8080 + name: grpc + - containerPort: 8081 + name: metrics + - containerPort: 8082 + name: health + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - "nfd-master" + resources: + limits: + memory: 4Gi + requests: + cpu: 100m + memory: 128Mi + args: + ## By default, disable crd controller for other than the default instances + - "-crd-controller=true" + # Go over featureGates and add the feature-gate flag + - "-feature-gates=NodeFeatureAPI=true" + - "-feature-gates=NodeFeatureGroupAPI=false" + - "-metrics=8081" + - "-grpc-health=8082" + volumeMounts: + - name: nfd-master-conf + mountPath: "/etc/kubernetes/node-feature-discovery" + readOnly: true + volumes: + - name: nfd-master-conf + configMap: + name: release-name-node-feature-discovery-master-conf + items: + - key: nfd-master.conf + path: nfd-master.conf + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "" + weight: 1 + - preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: In + values: + - "" + weight: 1 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal + value: "" + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Equal + value: "" +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/nfd-gc.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: release-name-node-feature-discovery-gc + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm + role: gc +spec: + replicas: 1 + revisionHistoryLimit: + selector: + matchLabels: + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + role: gc + template: + metadata: + labels: + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + role: gc + spec: + serviceAccountName: node-feature-discovery + dnsPolicy: ClusterFirstWithHostNet + priorityClassName: system-node-critical + securityContext: + {} + hostNetwork: false + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: gc + image: "registry.k8s.io/nfd/node-feature-discovery:v0.16.6" + imagePullPolicy: "IfNotPresent" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - "nfd-gc" + args: + - "-gc-interval=1h" + resources: + limits: + memory: 1Gi + requests: + cpu: 10m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ "ALL" ] + readOnlyRootFilesystem: true + runAsNonRoot: true + ports: + - name: metrics + containerPort: 8081 +--- +# Source: gpu-operator/templates/operator.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gpu-operator + namespace: gpu-operator + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" + nvidia.com/gpu-driver-upgrade-drain.skip: "true" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: "gpu-operator" + app: "gpu-operator" + template: + metadata: + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" + app: "gpu-operator" + nvidia.com/gpu-driver-upgrade-drain.skip: "true" + annotations: + openshift.io/scc: restricted-readonly + spec: + serviceAccountName: gpu-operator + priorityClassName: system-node-critical + containers: + - name: gpu-operator + image: nvcr.io/nvidia/gpu-operator:v24.9.2 + imagePullPolicy: IfNotPresent + command: ["gpu-operator"] + args: + - --leader-elect + - --zap-time-encoding=epoch + - --zap-log-level=info + env: + - name: WATCH_NAMESPACE + value: "" + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: "DRIVER_MANAGER_IMAGE" + value: "nvcr.io/nvidia/cloud-native/k8s-driver-manager:v0.7.0" + volumeMounts: + - name: host-os-release + mountPath: "/host-etc/os-release" + readOnly: true + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 350Mi + requests: + cpu: 200m + memory: 100Mi + ports: + - name: metrics + containerPort: 8080 + volumes: + - name: host-os-release + hostPath: + path: "/etc/os-release" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "" + weight: 1 + - preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: In + values: + - "" + weight: 1 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal + value: "" + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Equal + value: "" +--- +# Source: gpu-operator/templates/clusterpolicy.yaml +apiVersion: nvidia.com/v1 +kind: ClusterPolicy +metadata: + name: cluster-policy + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" +spec: + hostPaths: + rootFS: / + driverInstallDir: /run/nvidia/driver + operator: + defaultRuntime: docker + runtimeClass: nvidia + initContainer: + repository: nvcr.io/nvidia + image: cuda + version: "12.6.3-base-ubi9" + imagePullPolicy: IfNotPresent + daemonsets: + labels: + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/managed-by: gpu-operator + tolerations: + - effect: NoSchedule + key: nvidia.com/gpu + operator: Exists + priorityClassName: system-node-critical + updateStrategy: RollingUpdate + rollingUpdate: + maxUnavailable: "1" + validator: + repository: nvcr.io/nvidia/cloud-native + image: gpu-operator-validator + version: "v24.9.2" + imagePullPolicy: IfNotPresent + plugin: + env: + - name: WITH_WORKLOAD + value: "false" + + mig: + strategy: single + psa: + enabled: false + cdi: + enabled: true + default: true + driver: + enabled: true + useNvidiaDriverCRD: false + useOpenKernelModules: false + usePrecompiled: false + repository: nvcr.io/nvidia + image: driver + version: "550.144.03" + imagePullPolicy: IfNotPresent + startupProbe: + failureThreshold: 120 + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 60 + rdma: + enabled: false + useHostMofed: false + manager: + repository: nvcr.io/nvidia/cloud-native + image: k8s-driver-manager + version: "v0.7.0" + imagePullPolicy: IfNotPresent + env: + - name: ENABLE_GPU_POD_EVICTION + value: "true" + - name: ENABLE_AUTO_DRAIN + value: "false" + - name: DRAIN_USE_FORCE + value: "false" + - name: DRAIN_POD_SELECTOR_LABEL + value: "" + - name: DRAIN_TIMEOUT_SECONDS + value: 0s + - name: DRAIN_DELETE_EMPTYDIR_DATA + value: "false" + repoConfig: + configMapName: "" + certConfig: + name: "" + licensingConfig: + configMapName: "" + nlsEnabled: true + virtualTopology: + config: "" + kernelModuleConfig: + name: "" + upgradePolicy: + autoUpgrade: true + maxParallelUpgrades: 1 + maxUnavailable : 25% + waitForCompletion: + timeoutSeconds: 0 + podDeletion: + force: false + timeoutSeconds: 300 + deleteEmptyDir: false + drain: + enable: false + force: false + timeoutSeconds: 300 + deleteEmptyDir: false + vgpuManager: + enabled: false + image: vgpu-manager + imagePullPolicy: IfNotPresent + driverManager: + repository: nvcr.io/nvidia/cloud-native + image: k8s-driver-manager + version: "v0.7.0" + imagePullPolicy: IfNotPresent + env: + - name: ENABLE_GPU_POD_EVICTION + value: "false" + - name: ENABLE_AUTO_DRAIN + value: "false" + kataManager: + enabled: false + config: + artifactsDir: /opt/nvidia-gpu-operator/artifacts/runtimeclasses + runtimeClasses: + - artifacts: + pullSecret: "" + url: nvcr.io/nvidia/cloud-native/kata-gpu-artifacts:ubuntu22.04-535.54.03 + name: kata-nvidia-gpu + nodeSelector: {} + - artifacts: + pullSecret: "" + url: nvcr.io/nvidia/cloud-native/kata-gpu-artifacts:ubuntu22.04-535.86.10-snp + name: kata-nvidia-gpu-snp + nodeSelector: + nvidia.com/cc.capable: "true" + repository: nvcr.io/nvidia/cloud-native + image: k8s-kata-manager + version: "v0.2.2" + imagePullPolicy: IfNotPresent + vfioManager: + enabled: true + repository: nvcr.io/nvidia + image: cuda + version: "12.6.3-base-ubi9" + imagePullPolicy: IfNotPresent + driverManager: + repository: nvcr.io/nvidia/cloud-native + image: k8s-driver-manager + version: "v0.7.0" + imagePullPolicy: IfNotPresent + env: + - name: ENABLE_GPU_POD_EVICTION + value: "false" + - name: ENABLE_AUTO_DRAIN + value: "false" + vgpuDeviceManager: + enabled: true + repository: nvcr.io/nvidia/cloud-native + image: vgpu-device-manager + version: "v0.2.8" + imagePullPolicy: IfNotPresent + config: + default: default + name: "" + ccManager: + enabled: false + defaultMode: "off" + repository: nvcr.io/nvidia/cloud-native + image: k8s-cc-manager + version: "v0.1.1" + imagePullPolicy: IfNotPresent + env: + [] + toolkit: + enabled: true + repository: nvcr.io/nvidia/k8s + image: container-toolkit + version: "v1.17.4-ubuntu20.04" + imagePullPolicy: IfNotPresent + installDir: /usr/local/nvidia + devicePlugin: + enabled: true + repository: nvcr.io/nvidia + image: k8s-device-plugin + version: "v0.17.0" + imagePullPolicy: IfNotPresent + env: + - name: PASS_DEVICE_SPECS + value: "true" + - name: FAIL_ON_INIT_ERROR + value: "true" + - name: DEVICE_LIST_STRATEGY + value: envvar + - name: DEVICE_ID_STRATEGY + value: uuid + - name: NVIDIA_VISIBLE_DEVICES + value: all + - name: NVIDIA_DRIVER_CAPABILITIES + value: all + dcgm: + enabled: false + repository: nvcr.io/nvidia/cloud-native + image: dcgm + version: "3.3.9-1-ubuntu22.04" + imagePullPolicy: IfNotPresent + dcgmExporter: + enabled: true + repository: nvcr.io/nvidia/k8s + image: dcgm-exporter + version: "3.3.9-3.6.1-ubuntu22.04" + imagePullPolicy: IfNotPresent + env: + - name: DCGM_EXPORTER_LISTEN + value: :9400 + - name: DCGM_EXPORTER_KUBERNETES + value: "true" + - name: DCGM_EXPORTER_COLLECTORS + value: /etc/dcgm-exporter/dcp-metrics-included.csv + serviceMonitor: + additionalLabels: {} + enabled: false + honorLabels: false + interval: 15s + relabelings: [] + gfd: + enabled: true + repository: nvcr.io/nvidia + image: k8s-device-plugin + version: "v0.17.0" + imagePullPolicy: IfNotPresent + env: + - name: GFD_SLEEP_INTERVAL + value: 60s + - name: GFD_FAIL_ON_INIT_ERROR + value: "true" + migManager: + enabled: true + repository: nvcr.io/nvidia/cloud-native + image: k8s-mig-manager + version: "v0.10.0-ubuntu20.04" + imagePullPolicy: IfNotPresent + env: + - name: WITH_REBOOT + value: "false" + config: + name: + default: all-disabled + gpuClientsConfig: + name: "" + nodeStatusExporter: + enabled: false + repository: nvcr.io/nvidia/cloud-native + image: gpu-operator-validator + version: "v24.9.2" + imagePullPolicy: IfNotPresent + gdrcopy: + enabled: false + repository: nvcr.io/nvidia/cloud-native + image: gdrdrv + version: "v2.4.1-2" + imagePullPolicy: IfNotPresent + sandboxWorkloads: + enabled: false + defaultWorkload: container + sandboxDevicePlugin: + enabled: true + repository: nvcr.io/nvidia + image: kubevirt-gpu-device-plugin + version: "v1.2.10" + imagePullPolicy: IfNotPresent +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/post-delete-job.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: release-name-node-feature-discovery-prune + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +--- +# Source: gpu-operator/templates/upgrade_crd.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gpu-operator-upgrade-crd-hook-sa + namespace: gpu-operator + annotations: + helm.sh/hook: pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/post-delete-job.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: release-name-node-feature-discovery-prune + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +rules: +- apiGroups: + - "" + resources: + - nodes + - nodes/status + verbs: + - get + - patch + - update + - list +--- +# Source: gpu-operator/templates/upgrade_crd.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gpu-operator-upgrade-crd-hook-role + annotations: + helm.sh/hook: pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +rules: + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch + - patch + - update +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/post-delete-job.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: release-name-node-feature-discovery-prune + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: release-name-node-feature-discovery-prune +subjects: +- kind: ServiceAccount + name: release-name-node-feature-discovery-prune + namespace: gpu-operator +--- +# Source: gpu-operator/templates/upgrade_crd.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: gpu-operator-upgrade-crd-hook-binding + annotations: + helm.sh/hook: pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +subjects: + - kind: ServiceAccount + name: gpu-operator-upgrade-crd-hook-sa + namespace: gpu-operator +roleRef: + kind: ClusterRole + name: gpu-operator-upgrade-crd-hook-role + apiGroup: rbac.authorization.k8s.io +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/post-delete-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: release-name-node-feature-discovery-prune + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + template: + metadata: + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm + role: prune + spec: + serviceAccountName: release-name-node-feature-discovery-prune + containers: + - name: nfd-master + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + image: "registry.k8s.io/nfd/node-feature-discovery:v0.16.6" + imagePullPolicy: IfNotPresent + command: + - "nfd-master" + args: + - "-prune" + restartPolicy: Never + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "" + weight: 1 + - preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: In + values: + - "" + weight: 1 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal + value: "" + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Equal + value: "" +--- +# Source: gpu-operator/templates/upgrade_crd.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: gpu-operator-upgrade-crd + namespace: gpu-operator + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" +spec: + template: + metadata: + name: gpu-operator-upgrade-crd + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" + spec: + serviceAccountName: gpu-operator-upgrade-crd-hook-sa + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal + value: "" + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Equal + value: "" + containers: + - name: upgrade-crd + image: nvcr.io/nvidia/gpu-operator:v24.9.2 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + kubectl apply -f /opt/gpu-operator/nvidia.com_clusterpolicies.yaml; + kubectl apply -f /opt/gpu-operator/nvidia.com_nvidiadrivers.yaml; + kubectl apply -f /opt/gpu-operator/nfd-api-crds.yaml; + restartPolicy: OnFailure diff --git a/template-gpu-operator/highway.yaml b/template-gpu-operator/highway.yaml new file mode 100644 index 0000000..8a4e63b --- /dev/null +++ b/template-gpu-operator/highway.yaml @@ -0,0 +1,1512 @@ +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-feature-discovery + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +--- +# Source: gpu-operator/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gpu-operator + namespace: gpu-operator + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/nfd-master-conf.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-node-feature-discovery-master-conf + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +data: + nfd-master.conf: |- + extraLabelNs: + - nvidia.com +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/nfd-worker-conf.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: release-name-node-feature-discovery-worker-conf + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +data: + nfd-worker.conf: |- + sources: + pci: + deviceClassWhitelist: + - "02" + - "0200" + - "0207" + - "0300" + - "0302" + deviceLabelFields: + - vendor +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: release-name-node-feature-discovery + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - "" + resources: + - nodes + - nodes/status + verbs: + - get + - patch + - update + - list +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeatures + - nodefeaturerules + - nodefeaturegroups + verbs: + - get + - list + - watch +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeaturegroups/status + verbs: + - patch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - coordination.k8s.io + resources: + - leases + resourceNames: + - "nfd-master.nfd.kubernetes.io" + verbs: + - get + - update +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: release-name-node-feature-discovery-gc + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes/proxy + verbs: + - get +- apiGroups: + - topology.node.k8s.io + resources: + - noderesourcetopologies + verbs: + - delete + - list +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeatures + verbs: + - delete + - list +--- +# Source: gpu-operator/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gpu-operator + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" +rules: +- apiGroups: + - config.openshift.io + resources: + - clusterversions + - proxies + verbs: + - get + - list + - watch +- apiGroups: + - image.openshift.io + resources: + - imagestreams + verbs: + - get + - list + - watch +- apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + verbs: + - create + - get + - list + - watch + - update + - patch + - delete + - use +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterroles + - clusterrolebindings + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - watch + - update + - patch +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get + - list + - create + - watch + - update + - patch +- apiGroups: + - "" + resources: + - events + - pods + - pods/eviction + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - get + - list + - watch +- apiGroups: + - nvidia.com + resources: + - clusterpolicies + - clusterpolicies/finalizers + - clusterpolicies/status + - nvidiadrivers + - nvidiadrivers/finalizers + - nvidiadrivers/status + verbs: + - create + - get + - list + - watch + - update + - patch + - delete + - deletecollection +- apiGroups: + - scheduling.k8s.io + resources: + - priorityclasses + verbs: + - get + - list + - watch + - create +- apiGroups: + - node.k8s.io + resources: + - runtimeclasses + verbs: + - get + - list + - create + - update + - watch + - delete +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + - watch + - update + - patch + - create +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: release-name-node-feature-discovery + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: release-name-node-feature-discovery +subjects: +- kind: ServiceAccount + name: node-feature-discovery + namespace: gpu-operator +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: release-name-node-feature-discovery-gc + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: release-name-node-feature-discovery-gc +subjects: +- kind: ServiceAccount + name: node-feature-discovery + namespace: gpu-operator +--- +# Source: gpu-operator/templates/clusterrolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: gpu-operator + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" +subjects: +- kind: ServiceAccount + name: gpu-operator + namespace: gpu-operator +- kind: ServiceAccount + name: node-feature-discovery + namespace: gpu-operator +roleRef: + kind: ClusterRole + name: gpu-operator + apiGroup: rbac.authorization.k8s.io +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: release-name-node-feature-discovery-worker + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +rules: +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeatures + verbs: + - create + - get + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - get +--- +# Source: gpu-operator/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: gpu-operator + namespace: gpu-operator + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" +rules: +- apiGroups: + - rbac.authorization.k8s.io + resources: + - roles + - rolebindings + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - apps + resources: + - controllerrevisions + verbs: + - get + - list + - watch +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - "" + resources: + - configmaps + - endpoints + - pods + - pods/eviction + - secrets + - services + - services/finalizers + - serviceaccounts + verbs: + - create + - get + - list + - watch + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + - prometheusrules + verbs: + - get + - list + - create + - watch + - update + - delete +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: release-name-node-feature-discovery-worker + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: release-name-node-feature-discovery-worker +subjects: +- kind: ServiceAccount + name: node-feature-discovery + namespace: gpu-operator +--- +# Source: gpu-operator/templates/rolebinding.yaml +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: gpu-operator + namespace: gpu-operator + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" +subjects: +- kind: ServiceAccount + name: gpu-operator + namespace: gpu-operator +roleRef: + kind: Role + name: gpu-operator + apiGroup: rbac.authorization.k8s.io +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/worker.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: release-name-node-feature-discovery-worker + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm + role: worker +spec: + revisionHistoryLimit: + selector: + matchLabels: + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + role: worker + template: + metadata: + labels: + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + role: worker + spec: + dnsPolicy: ClusterFirstWithHostNet + priorityClassName: system-node-critical + serviceAccountName: node-feature-discovery + securityContext: + {} + hostNetwork: false + containers: + - name: worker + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + image: "harbor.cone-chain.com/highway/nfd/node-feature-discovery:v0.16.6" + imagePullPolicy: IfNotPresent + livenessProbe: + grpc: + port: 8082 + initialDelaySeconds: 10 + readinessProbe: + failureThreshold: 10 + grpc: + port: 8082 + initialDelaySeconds: 5 + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + resources: + limits: + memory: 512Mi + requests: + cpu: 5m + memory: 64Mi + command: + - "nfd-worker" + args: +# Go over featureGate and add the feature-gate flag + - "-feature-gates=NodeFeatureAPI=true" + - "-feature-gates=NodeFeatureGroupAPI=false" + - "-metrics=8081" + - "-grpc-health=8082" + ports: + - containerPort: 8081 + name: metrics + - containerPort: 8082 + name: health + volumeMounts: + - name: host-boot + mountPath: "/host-boot" + readOnly: true + - name: host-os-release + mountPath: "/host-etc/os-release" + readOnly: true + - name: host-sys + mountPath: "/host-sys" + readOnly: true + - name: host-usr-lib + mountPath: "/host-usr/lib" + readOnly: true + - name: host-lib + mountPath: "/host-lib" + readOnly: true + - name: host-proc-swaps + mountPath: "/host-proc/swaps" + readOnly: true + - name: source-d + mountPath: "/etc/kubernetes/node-feature-discovery/source.d/" + readOnly: true + - name: features-d + mountPath: "/etc/kubernetes/node-feature-discovery/features.d/" + readOnly: true + - name: nfd-worker-conf + mountPath: "/etc/kubernetes/node-feature-discovery" + readOnly: true + volumes: + - name: host-boot + hostPath: + path: "/boot" + - name: host-os-release + hostPath: + path: "/etc/os-release" + - name: host-sys + hostPath: + path: "/sys" + - name: host-usr-lib + hostPath: + path: "/usr/lib" + - name: host-lib + hostPath: + path: "/lib" + - name: host-proc-swaps + hostPath: + path: "/proc/swaps" + - name: source-d + hostPath: + path: "/etc/kubernetes/node-feature-discovery/source.d/" + - name: features-d + hostPath: + path: "/etc/kubernetes/node-feature-discovery/features.d/" + - name: nfd-worker-conf + configMap: + name: release-name-node-feature-discovery-worker-conf + items: + - key: nfd-worker.conf + path: nfd-worker.conf + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal + value: "" + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Equal + value: "" + - effect: NoSchedule + key: nvidia.com/gpu + operator: Exists +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/master.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: release-name-node-feature-discovery-master + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm + role: master +spec: + replicas: 1 + revisionHistoryLimit: + selector: + matchLabels: + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + role: master + template: + metadata: + labels: + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + role: master + spec: + priorityClassName: system-node-critical + serviceAccountName: node-feature-discovery + enableServiceLinks: false + securityContext: + {} + hostNetwork: false + containers: + - name: master + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + image: "harbor.cone-chain.com/highway/nfd/node-feature-discovery:v0.16.6" + imagePullPolicy: IfNotPresent + livenessProbe: + grpc: + port: 8082 + initialDelaySeconds: 10 + readinessProbe: + failureThreshold: 10 + grpc: + port: 8082 + initialDelaySeconds: 5 + ports: + - containerPort: 8080 + name: grpc + - containerPort: 8081 + name: metrics + - containerPort: 8082 + name: health + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - "nfd-master" + resources: + limits: + memory: 4Gi + requests: + cpu: 100m + memory: 128Mi + args: + ## By default, disable crd controller for other than the default instances + - "-crd-controller=true" + # Go over featureGates and add the feature-gate flag + - "-feature-gates=NodeFeatureAPI=true" + - "-feature-gates=NodeFeatureGroupAPI=false" + - "-metrics=8081" + - "-grpc-health=8082" + volumeMounts: + - name: nfd-master-conf + mountPath: "/etc/kubernetes/node-feature-discovery" + readOnly: true + volumes: + - name: nfd-master-conf + configMap: + name: release-name-node-feature-discovery-master-conf + items: + - key: nfd-master.conf + path: nfd-master.conf + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "" + weight: 1 + - preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: In + values: + - "" + weight: 1 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal + value: "" + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Equal + value: "" +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/nfd-gc.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: release-name-node-feature-discovery-gc + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm + role: gc +spec: + replicas: 1 + revisionHistoryLimit: + selector: + matchLabels: + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + role: gc + template: + metadata: + labels: + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + role: gc + spec: + serviceAccountName: node-feature-discovery + dnsPolicy: ClusterFirstWithHostNet + priorityClassName: system-node-critical + securityContext: + {} + hostNetwork: false + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: gc + image: "harbor.cone-chain.com/highway/nfd/node-feature-discovery:v0.16.6" + imagePullPolicy: "IfNotPresent" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - "nfd-gc" + args: + - "-gc-interval=1h" + resources: + limits: + memory: 1Gi + requests: + cpu: 10m + memory: 128Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ "ALL" ] + readOnlyRootFilesystem: true + runAsNonRoot: true + ports: + - name: metrics + containerPort: 8081 +--- +# Source: gpu-operator/templates/operator.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: gpu-operator + namespace: gpu-operator + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" + nvidia.com/gpu-driver-upgrade-drain.skip: "true" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/component: "gpu-operator" + app: "gpu-operator" + template: + metadata: + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" + app: "gpu-operator" + nvidia.com/gpu-driver-upgrade-drain.skip: "true" + annotations: + openshift.io/scc: restricted-readonly + spec: + serviceAccountName: gpu-operator + priorityClassName: system-node-critical + containers: + - name: gpu-operator + image: harbor.cone-chain.com/highway/nvidia/gpu-operator:v24.9.2 + imagePullPolicy: IfNotPresent + command: ["gpu-operator"] + args: + - --leader-elect + - --zap-time-encoding=epoch + - --zap-log-level=info + env: + - name: WATCH_NAMESPACE + value: "" + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: "DRIVER_MANAGER_IMAGE" + value: "harbor.cone-chain.com/highway/nvidia/cloud-native/k8s-driver-manager:v0.7.0" + volumeMounts: + - name: host-os-release + mountPath: "/host-etc/os-release" + readOnly: true + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + limits: + cpu: 500m + memory: 350Mi + requests: + cpu: 200m + memory: 100Mi + ports: + - name: metrics + containerPort: 8080 + volumes: + - name: host-os-release + hostPath: + path: "/etc/os-release" + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "" + weight: 1 + - preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: In + values: + - "" + weight: 1 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal + value: "" + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Equal + value: "" +--- +# Source: gpu-operator/templates/clusterpolicy.yaml +apiVersion: nvidia.com/v1 +kind: ClusterPolicy +metadata: + name: cluster-policy + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" +spec: + hostPaths: + rootFS: / + driverInstallDir: /run/nvidia/driver + operator: + defaultRuntime: docker + runtimeClass: nvidia + initContainer: + repository: harbor.cone-chain.com/highway/nvidia + image: cuda + version: "12.6.3-base-ubi9" + imagePullPolicy: IfNotPresent + daemonsets: + labels: + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/managed-by: gpu-operator + tolerations: + - effect: NoSchedule + key: nvidia.com/gpu + operator: Exists + priorityClassName: system-node-critical + updateStrategy: RollingUpdate + rollingUpdate: + maxUnavailable: "1" + validator: + repository: harbor.cone-chain.com/highway/nvidia/cloud-native + image: gpu-operator-validator + version: "v24.9.2" + imagePullPolicy: IfNotPresent + plugin: + env: + - name: WITH_WORKLOAD + value: "false" + + mig: + strategy: single + psa: + enabled: false + cdi: + enabled: true + default: true + driver: + enabled: true + useNvidiaDriverCRD: false + useOpenKernelModules: false + usePrecompiled: false + repository: harbor.cone-chain.com/highway/nvidia + image: driver + version: "550.144.03" + imagePullPolicy: IfNotPresent + startupProbe: + failureThreshold: 120 + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 60 + rdma: + enabled: false + useHostMofed: false + manager: + repository: harbor.cone-chain.com/highway/nvidia/cloud-native + image: k8s-driver-manager + version: "v0.7.0" + imagePullPolicy: IfNotPresent + env: + - name: ENABLE_GPU_POD_EVICTION + value: "true" + - name: ENABLE_AUTO_DRAIN + value: "false" + - name: DRAIN_USE_FORCE + value: "false" + - name: DRAIN_POD_SELECTOR_LABEL + value: "" + - name: DRAIN_TIMEOUT_SECONDS + value: 0s + - name: DRAIN_DELETE_EMPTYDIR_DATA + value: "false" + repoConfig: + configMapName: "" + certConfig: + name: "" + licensingConfig: + configMapName: "" + nlsEnabled: true + virtualTopology: + config: "" + kernelModuleConfig: + name: "" + upgradePolicy: + autoUpgrade: true + maxParallelUpgrades: 1 + maxUnavailable : 25% + waitForCompletion: + timeoutSeconds: 0 + podDeletion: + force: false + timeoutSeconds: 300 + deleteEmptyDir: false + drain: + enable: false + force: false + timeoutSeconds: 300 + deleteEmptyDir: false + vgpuManager: + enabled: false + image: vgpu-manager + imagePullPolicy: IfNotPresent + driverManager: + repository: harbor.cone-chain.com/highway/nvidia/cloud-native + image: k8s-driver-manager + version: "v0.7.0" + imagePullPolicy: IfNotPresent + env: + - name: ENABLE_GPU_POD_EVICTION + value: "false" + - name: ENABLE_AUTO_DRAIN + value: "false" + kataManager: + enabled: false + config: + artifactsDir: /opt/nvidia-gpu-operator/artifacts/runtimeclasses + runtimeClasses: + - artifacts: + pullSecret: "" + url: harbor.cone-chain.com/highway/nvidia/cloud-native/kata-gpu-artifacts:ubuntu22.04-535.54.03 + name: kata-nvidia-gpu + nodeSelector: {} + - artifacts: + pullSecret: "" + url: harbor.cone-chain.com/highway/nvidia/cloud-native/kata-gpu-artifacts:ubuntu22.04-535.86.10-snp + name: kata-nvidia-gpu-snp + nodeSelector: + nvidia.com/cc.capable: "true" + repository: harbor.cone-chain.com/highway/nvidia/cloud-native + image: k8s-kata-manager + version: "v0.2.2" + imagePullPolicy: IfNotPresent + vfioManager: + enabled: true + repository: harbor.cone-chain.com/highway/nvidia + image: cuda + version: "12.6.3-base-ubi9" + imagePullPolicy: IfNotPresent + driverManager: + repository: harbor.cone-chain.com/highway/nvidia/cloud-native + image: k8s-driver-manager + version: "v0.7.0" + imagePullPolicy: IfNotPresent + env: + - name: ENABLE_GPU_POD_EVICTION + value: "false" + - name: ENABLE_AUTO_DRAIN + value: "false" + vgpuDeviceManager: + enabled: true + repository: harbor.cone-chain.com/highway/nvidia/cloud-native + image: vgpu-device-manager + version: "v0.2.8" + imagePullPolicy: IfNotPresent + config: + default: default + name: "" + ccManager: + enabled: false + defaultMode: "off" + repository: harbor.cone-chain.com/highway/nvidia/cloud-native + image: k8s-cc-manager + version: "v0.1.1" + imagePullPolicy: IfNotPresent + env: + [] + toolkit: + enabled: true + repository: harbor.cone-chain.com/highway/nvidia/k8s + image: container-toolkit + version: "v1.17.4-ubuntu20.04" + imagePullPolicy: IfNotPresent + installDir: /usr/local/nvidia + devicePlugin: + enabled: true + repository: harbor.cone-chain.com/highway/nvidia + image: k8s-device-plugin + version: "v0.17.0" + imagePullPolicy: IfNotPresent + env: + - name: PASS_DEVICE_SPECS + value: "true" + - name: FAIL_ON_INIT_ERROR + value: "true" + - name: DEVICE_LIST_STRATEGY + value: envvar + - name: DEVICE_ID_STRATEGY + value: uuid + - name: NVIDIA_VISIBLE_DEVICES + value: all + - name: NVIDIA_DRIVER_CAPABILITIES + value: all + dcgm: + enabled: false + repository: harbor.cone-chain.com/highway/nvidia/cloud-native + image: dcgm + version: "3.3.9-1-ubuntu22.04" + imagePullPolicy: IfNotPresent + dcgmExporter: + enabled: true + repository: harbor.cone-chain.com/highway/nvidia/k8s + image: dcgm-exporter + version: "3.3.9-3.6.1-ubuntu22.04" + imagePullPolicy: IfNotPresent + env: + - name: DCGM_EXPORTER_LISTEN + value: :9400 + - name: DCGM_EXPORTER_KUBERNETES + value: "true" + - name: DCGM_EXPORTER_COLLECTORS + value: /etc/dcgm-exporter/dcp-metrics-included.csv + serviceMonitor: + additionalLabels: {} + enabled: false + honorLabels: false + interval: 15s + relabelings: [] + gfd: + enabled: true + repository: harbor.cone-chain.com/highway/nvidia + image: k8s-device-plugin + version: "v0.17.0" + imagePullPolicy: IfNotPresent + env: + - name: GFD_SLEEP_INTERVAL + value: 60s + - name: GFD_FAIL_ON_INIT_ERROR + value: "true" + migManager: + enabled: true + repository: harbor.cone-chain.com/highway/nvidia/cloud-native + image: k8s-mig-manager + version: "v0.10.0-ubuntu20.04" + imagePullPolicy: IfNotPresent + env: + - name: WITH_REBOOT + value: "false" + config: + name: + default: all-disabled + gpuClientsConfig: + name: "" + nodeStatusExporter: + enabled: false + repository: harbor.cone-chain.com/highway/nvidia/cloud-native + image: gpu-operator-validator + version: "v24.9.2" + imagePullPolicy: IfNotPresent + gdrcopy: + enabled: false + repository: harbor.cone-chain.com/highway/nvidia/cloud-native + image: gdrdrv + version: "v2.4.1-2" + imagePullPolicy: IfNotPresent + sandboxWorkloads: + enabled: false + defaultWorkload: container + sandboxDevicePlugin: + enabled: true + repository: harbor.cone-chain.com/highway/nvidia + image: kubevirt-gpu-device-plugin + version: "v1.2.10" + imagePullPolicy: IfNotPresent +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/post-delete-job.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: release-name-node-feature-discovery-prune + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +--- +# Source: gpu-operator/templates/upgrade_crd.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gpu-operator-upgrade-crd-hook-sa + namespace: gpu-operator + annotations: + helm.sh/hook: pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/post-delete-job.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: release-name-node-feature-discovery-prune + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +rules: +- apiGroups: + - "" + resources: + - nodes + - nodes/status + verbs: + - get + - patch + - update + - list +--- +# Source: gpu-operator/templates/upgrade_crd.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gpu-operator-upgrade-crd-hook-role + annotations: + helm.sh/hook: pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +rules: + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - get + - list + - watch + - patch + - update +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/post-delete-job.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: release-name-node-feature-discovery-prune + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: release-name-node-feature-discovery-prune +subjects: +- kind: ServiceAccount + name: release-name-node-feature-discovery-prune + namespace: gpu-operator +--- +# Source: gpu-operator/templates/upgrade_crd.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: gpu-operator-upgrade-crd-hook-binding + annotations: + helm.sh/hook: pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +subjects: + - kind: ServiceAccount + name: gpu-operator-upgrade-crd-hook-sa + namespace: gpu-operator +roleRef: + kind: ClusterRole + name: gpu-operator-upgrade-crd-hook-role + apiGroup: rbac.authorization.k8s.io +--- +# Source: gpu-operator/charts/node-feature-discovery/templates/post-delete-job.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: release-name-node-feature-discovery-prune + namespace: gpu-operator + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + template: + metadata: + labels: + helm.sh/chart: node-feature-discovery-0.16.6 + app.kubernetes.io/name: node-feature-discovery + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v0.16.6" + app.kubernetes.io/managed-by: Helm + role: prune + spec: + serviceAccountName: release-name-node-feature-discovery-prune + containers: + - name: nfd-master + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true + runAsNonRoot: true + image: "harbor.cone-chain.com/highway/nfd/node-feature-discovery:v0.16.6" + imagePullPolicy: IfNotPresent + command: + - "nfd-master" + args: + - "-prune" + restartPolicy: Never + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - preference: + matchExpressions: + - key: node-role.kubernetes.io/master + operator: In + values: + - "" + weight: 1 + - preference: + matchExpressions: + - key: node-role.kubernetes.io/control-plane + operator: In + values: + - "" + weight: 1 + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal + value: "" + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Equal + value: "" +--- +# Source: gpu-operator/templates/upgrade_crd.yaml +apiVersion: batch/v1 +kind: Job +metadata: + name: gpu-operator-upgrade-crd + namespace: gpu-operator + annotations: + "helm.sh/hook": pre-upgrade + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" +spec: + template: + metadata: + name: gpu-operator-upgrade-crd + labels: + app.kubernetes.io/name: gpu-operator + helm.sh/chart: gpu-operator-v24.9.2 + app.kubernetes.io/instance: release-name + app.kubernetes.io/version: "v24.9.2" + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: "gpu-operator" + spec: + serviceAccountName: gpu-operator-upgrade-crd-hook-sa + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Equal + value: "" + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Equal + value: "" + containers: + - name: upgrade-crd + image: harbor.cone-chain.com/highway/nvidia/gpu-operator:v24.9.2 + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + kubectl apply -f /opt/gpu-operator/nvidia.com_clusterpolicies.yaml; + kubectl apply -f /opt/gpu-operator/nvidia.com_nvidiadrivers.yaml; + kubectl apply -f /opt/gpu-operator/nfd-api-crds.yaml; + restartPolicy: OnFailure diff --git a/template-gpu-operator/mig.yaml b/template-gpu-operator/mig.yaml new file mode 100644 index 0000000..59534df --- /dev/null +++ b/template-gpu-operator/mig.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: custom-mig-config + namespace: gpu-operator +data: + config.yaml: | + version: v1 + mig-configs: + all-disabled: + - devices: all + mig-enabled: false + + all-1g: + - devices: all + mig-enabled: true + mig-devices: + "1g.12gb": 7 + + + diff --git a/template-gpu-operator/mig.yaml.bak b/template-gpu-operator/mig.yaml.bak new file mode 100644 index 0000000..7c9bc55 --- /dev/null +++ b/template-gpu-operator/mig.yaml.bak @@ -0,0 +1,30 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: custom-mig-config + namespace: gpu-operator +data: + config.yaml: | + version: v1 + mig-configs: + all-disabled: + - devices: all + mig-enabled: false + + all-1g: + - devices: all + mig-enabled: true + mig-devices: + "1g.10gb": 7 + + two-1g: + - devices: [0] + mig-enabled: true + mig-devices: + "1g.10gb": 7 + - devices: [1] + mig-enabled: true + mig-devices: + "1g.10gb": 7 + + diff --git a/template-gpu-operator/script.sh b/template-gpu-operator/script.sh new file mode 100644 index 0000000..b5490fd --- /dev/null +++ b/template-gpu-operator/script.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +# 프라이빗 레지스트리 주소 +PRIVATE_REGISTRY="sooeon.com:5000" + +# 이미지 목록 (원본 이미지와 태그) +IMAGES=( + "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v24.9.2" + "nvcr.io/nvidia/cloud-native/k8s-driver-manager:v0.7.0" + "nvcr.io/nvidia/k8s-device-plugin:v0.17.0" + "nvcr.io/nvidia/k8s/container-toolkit:v1.17.4-ubuntu20.04" + "nvcr.io/nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04" +) + +# 1. 모든 이미지 pull, tag, push 실행 +for IMAGE in "${IMAGES[@]}"; do + # 원본 이미지에서 태그 분리 + IMAGE_NAME=$(echo "$IMAGE" | awk -F':' '{print $1}') + IMAGE_TAG=$(echo "$IMAGE" | awk -F':' '{print $2}') + + # 새로운 이미지 경로 설정 + NEW_IMAGE="$PRIVATE_REGISTRY/${IMAGE_NAME#nvcr.io/}" + + echo " Pulling image: $IMAGE" + docker pull "$IMAGE" + + echo " Tagging image: $IMAGE -> $NEW_IMAGE:$IMAGE_TAG" + docker tag "$IMAGE" "$NEW_IMAGE:$IMAGE_TAG" + + echo " Pushing image: $NEW_IMAGE:$IMAGE_TAG" + docker push "$NEW_IMAGE:$IMAGE_TAG" + + echo "✅ Successfully pushed $NEW_IMAGE:$IMAGE_TAG" +done + +echo " 모든 이미지가 프라이빗 레지스트리로 푸시 완료되었습니다!" + diff --git a/template-gpu-operator/script_nativedeck.sh b/template-gpu-operator/script_nativedeck.sh new file mode 100644 index 0000000..f150c56 --- /dev/null +++ b/template-gpu-operator/script_nativedeck.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# 주의 +## 해당스크리ㅂ트에는 k8s.registry.io/nfd/node-feature-discovery는 빠져있음 +## k8s.registry.io/nfd/node-feature-discovery:v0.16.6 +# 프라이빗 레지스트리 주소 +PRIVATE_REGISTRY="harbor.nativedeck.com/nd002-rc.1" + +# 이미지 목록 (원본 이미지와 태그) +IMAGES=( + "nvcr.io/nvidia/cloud-native/k8s-mig-manager:v0.10.0-ubuntu20.04" + "nvcr.io/nvidia/gpu-operator:v24.9.2" + "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v24.9.2" + "nvcr.io/nvidia/cloud-native/k8s-driver-manager:v0.7.0" + "nvcr.io/nvidia/k8s-device-plugin:v0.17.0" + "nvcr.io/nvidia/k8s/container-toolkit:v1.17.4-ubuntu20.04" + "nvcr.io/nvidia/k8s/dcgm-exporter:3.3.9-3.6.1-ubuntu22.04" +) + +# 1. 모든 이미지 pull, tag, push 실행 +for IMAGE in "${IMAGES[@]}"; do + # 원본 이미지에서 태그 분리 + IMAGE_NAME=$(echo "$IMAGE" | awk -F':' '{print $1}') + IMAGE_TAG=$(echo "$IMAGE" | awk -F':' '{print $2}') + + # 새로운 이미지 경로 설정 + NEW_IMAGE="$PRIVATE_REGISTRY/${IMAGE_NAME#nvcr.io/}" + + echo " Pulling image: $IMAGE" + docker pull "$IMAGE" + + echo " Tagging image: $IMAGE -> $NEW_IMAGE:$IMAGE_TAG" + docker tag "$IMAGE" "$NEW_IMAGE:$IMAGE_TAG" + + echo " Pushing image: $NEW_IMAGE:$IMAGE_TAG" + docker push "$NEW_IMAGE:$IMAGE_TAG" + + echo "✅ Successfully pushed $NEW_IMAGE:$IMAGE_TAG" +done + +echo " 모든 이미지가 프라이빗 레지스트리로 푸시 완료되었습니다!" + diff --git a/template-gpu-operator/time-slicing.yaml b/template-gpu-operator/time-slicing.yaml new file mode 100644 index 0000000..cf1eab3 --- /dev/null +++ b/template-gpu-operator/time-slicing.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: time-slicing-config-all + namespace: gpu-operator +data: + any: |- + version: v1 + flags: + migStrategy: none + sharing: + timeSlicing: + resources: + - name: nvidia.com/gpu + replicas: 4 +