From f4de947371135617fb427d84825f5362bbbe80f4 Mon Sep 17 00:00:00 2001 From: root Date: Mon, 23 Jun 2025 21:58:06 +0900 Subject: [PATCH] monitoring --- gpulive/monitoring/base/grafana/cm.yaml | 27 ++ gpulive/monitoring/base/grafana/deploy.yaml | 121 ++++++ gpulive/monitoring/base/grafana/ingress.yaml | 22 + .../base/grafana/kustomization.yaml | 17 + gpulive/monitoring/base/grafana/pvc.yaml | 13 + gpulive/monitoring/base/grafana/role.yaml | 63 +++ gpulive/monitoring/base/grafana/sa.yaml | 12 + gpulive/monitoring/base/grafana/secret.yaml | 17 + gpulive/monitoring/base/grafana/service.yaml | 21 + .../base/prometheus/cluster_role.yaml | 248 ++++++++++++ gpulive/monitoring/base/prometheus/cm.yaml | 375 ++++++++++++++++++ .../prometheus/cm_custom_alert_rules.yaml | 194 +++++++++ .../monitoring/base/prometheus/daemonset.yaml | 135 +++++++ .../monitoring/base/prometheus/deploy.yaml | 290 ++++++++++++++ .../base/prometheus/ingress_alertmanager.yaml | 29 ++ .../base/prometheus/ingress_prometheus.yaml | 29 ++ .../base/prometheus/kustomization.yaml | 19 + .../monitoring/base/prometheus/namespace.yaml | 6 + gpulive/monitoring/base/prometheus/pvc.yaml | 19 + gpulive/monitoring/base/prometheus/sa.yaml | 76 ++++ .../monitoring/base/prometheus/service.yaml | 151 +++++++ .../base/prometheus/state_full_set.yaml | 90 +++++ gpulive/monitoring/overlays/default/auth | 1 + .../default/grafana/kustomization.yaml | 29 ++ .../overlays/default/grafana/patch-cm.yaml | 32 ++ .../default/grafana/patch-deploy.yaml | 20 + .../grafana/patch-ingress_grafana.yaml | 24 ++ .../default/grafana/patch-secret.yaml | 17 + .../overlays/default/kustomization.yaml | 17 + .../default/prometheus/kustomization.yaml | 34 ++ .../patch-cm-prometheus-server.yaml | 335 ++++++++++++++++ .../default/prometheus/patch-deploy.yaml | 19 + .../patch-ingress_alertmanager.yaml | 29 ++ .../prometheus/patch-ingress_prometheus.yaml | 22 + 34 files changed, 2553 insertions(+) create mode 100644 gpulive/monitoring/base/grafana/cm.yaml create mode 100644 gpulive/monitoring/base/grafana/deploy.yaml create mode 100644 gpulive/monitoring/base/grafana/ingress.yaml create mode 100644 gpulive/monitoring/base/grafana/kustomization.yaml create mode 100644 gpulive/monitoring/base/grafana/pvc.yaml create mode 100644 gpulive/monitoring/base/grafana/role.yaml create mode 100644 gpulive/monitoring/base/grafana/sa.yaml create mode 100644 gpulive/monitoring/base/grafana/secret.yaml create mode 100644 gpulive/monitoring/base/grafana/service.yaml create mode 100644 gpulive/monitoring/base/prometheus/cluster_role.yaml create mode 100644 gpulive/monitoring/base/prometheus/cm.yaml create mode 100644 gpulive/monitoring/base/prometheus/cm_custom_alert_rules.yaml create mode 100644 gpulive/monitoring/base/prometheus/daemonset.yaml create mode 100644 gpulive/monitoring/base/prometheus/deploy.yaml create mode 100644 gpulive/monitoring/base/prometheus/ingress_alertmanager.yaml create mode 100644 gpulive/monitoring/base/prometheus/ingress_prometheus.yaml create mode 100644 gpulive/monitoring/base/prometheus/kustomization.yaml create mode 100644 gpulive/monitoring/base/prometheus/namespace.yaml create mode 100644 gpulive/monitoring/base/prometheus/pvc.yaml create mode 100644 gpulive/monitoring/base/prometheus/sa.yaml create mode 100644 gpulive/monitoring/base/prometheus/service.yaml create mode 100644 gpulive/monitoring/base/prometheus/state_full_set.yaml create mode 100644 gpulive/monitoring/overlays/default/auth create mode 100644 gpulive/monitoring/overlays/default/grafana/kustomization.yaml create mode 100644 gpulive/monitoring/overlays/default/grafana/patch-cm.yaml create mode 100644 gpulive/monitoring/overlays/default/grafana/patch-deploy.yaml create mode 100644 gpulive/monitoring/overlays/default/grafana/patch-ingress_grafana.yaml create mode 100644 gpulive/monitoring/overlays/default/grafana/patch-secret.yaml create mode 100644 gpulive/monitoring/overlays/default/kustomization.yaml create mode 100644 gpulive/monitoring/overlays/default/prometheus/kustomization.yaml create mode 100644 gpulive/monitoring/overlays/default/prometheus/patch-cm-prometheus-server.yaml create mode 100644 gpulive/monitoring/overlays/default/prometheus/patch-deploy.yaml create mode 100644 gpulive/monitoring/overlays/default/prometheus/patch-ingress_alertmanager.yaml create mode 100644 gpulive/monitoring/overlays/default/prometheus/patch-ingress_prometheus.yaml diff --git a/gpulive/monitoring/base/grafana/cm.yaml b/gpulive/monitoring/base/grafana/cm.yaml new file mode 100644 index 0000000..b787a2f --- /dev/null +++ b/gpulive/monitoring/base/grafana/cm.yaml @@ -0,0 +1,27 @@ +# Source: grafana/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.10.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + app.kubernetes.io/version: "11.5.1" +data: + + grafana.ini: | + [analytics] + check_for_updates = true + [grafana_net] + url = https://grafana.net + [log] + mode = console + [paths] + data = /var/lib/grafana/ + logs = /var/log/grafana + plugins = /var/lib/grafana/plugins + provisioning = /etc/grafana/provisioning + [server] + domain = '' \ No newline at end of file diff --git a/gpulive/monitoring/base/grafana/deploy.yaml b/gpulive/monitoring/base/grafana/deploy.yaml new file mode 100644 index 0000000..04a5feb --- /dev/null +++ b/gpulive/monitoring/base/grafana/deploy.yaml @@ -0,0 +1,121 @@ +# Source: grafana/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.10.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + app.kubernetes.io/version: "11.5.1" +spec: + replicas: 1 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + strategy: + type: RollingUpdate + template: + metadata: + labels: + helm.sh/chart: grafana-8.10.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + app.kubernetes.io/version: "11.5.1" + annotations: + kubectl.kubernetes.io/default-container: grafana + spec: + nodeSelector: + nodegroup: nd + serviceAccountName: grafana + automountServiceAccountToken: true + shareProcessNamespace: false + securityContext: + fsGroup: 472 + runAsGroup: 472 + runAsNonRoot: true + runAsUser: 472 + enableServiceLinks: true + containers: + - name: grafana + image: "docker.io/grafana/grafana:11.5.1" + imagePullPolicy: Always + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + volumeMounts: + - name: config + mountPath: "/etc/grafana/grafana.ini" + subPath: grafana.ini + - name: storage + mountPath: "/var/lib/grafana" + ports: + - name: grafana + containerPort: 3000 + protocol: TCP + - name: gossip-tcp + containerPort: 9094 + protocol: TCP + - name: gossip-udp + containerPort: 9094 + protocol: UDP + - name: profiling + containerPort: 6060 + protocol: TCP + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: grafana + key: admin-user + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: grafana + key: admin-password + - name: GF_PATHS_DATA + value: /var/lib/grafana/ + - name: GF_PATHS_LOGS + value: /var/log/grafana + - name: GF_PATHS_PLUGINS + value: /var/lib/grafana/plugins + - name: GF_PATHS_PROVISIONING + value: /etc/grafana/provisioning + livenessProbe: + failureThreshold: 10 + httpGet: + path: /api/health + port: 3000 + initialDelaySeconds: 60 + timeoutSeconds: 30 + readinessProbe: + httpGet: + path: /api/health + port: 3000 + resources: + requests: + cpu: 250m + memory: 256Mi + limits: + cpu: 500m + memory: 512Mi + volumes: + - name: config + configMap: + name: grafana + - name: storage + persistentVolumeClaim: + claimName: grafana-pvc + # - name: storage + # emptyDir: {} \ No newline at end of file diff --git a/gpulive/monitoring/base/grafana/ingress.yaml b/gpulive/monitoring/base/grafana/ingress.yaml new file mode 100644 index 0000000..665ea2a --- /dev/null +++ b/gpulive/monitoring/base/grafana/ingress.yaml @@ -0,0 +1,22 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: grafana-ingress + namespace: monitoring +spec: + ingressClassName: nginx + rules: + - host: grafana.sample.com + http: + paths: + - backend: + service: + name: grafana + port: + number: 80 + path: / + pathType: Prefix + tls: + - hosts: + - grafana.sample.com + secretName: ssl-cc \ No newline at end of file diff --git a/gpulive/monitoring/base/grafana/kustomization.yaml b/gpulive/monitoring/base/grafana/kustomization.yaml new file mode 100644 index 0000000..815106d --- /dev/null +++ b/gpulive/monitoring/base/grafana/kustomization.yaml @@ -0,0 +1,17 @@ +# Prometheus에 종속적으로 설치 +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: monitoring + +resources: + - cm.yaml + - deploy.yaml + # - namespace.yaml + - role.yaml + - sa.yaml + - secret.yaml + - service.yaml + - ingress.yaml + - pvc.yaml + \ No newline at end of file diff --git a/gpulive/monitoring/base/grafana/pvc.yaml b/gpulive/monitoring/base/grafana/pvc.yaml new file mode 100644 index 0000000..9a0cc28 --- /dev/null +++ b/gpulive/monitoring/base/grafana/pvc.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: grafana-pvc + namespace: monitoring + labels: + app.kubernetes.io/name: grafana +spec: + accessModes: + - "ReadWriteOnce" + resources: + requests: + storage: "10Gi" \ No newline at end of file diff --git a/gpulive/monitoring/base/grafana/role.yaml b/gpulive/monitoring/base/grafana/role.yaml new file mode 100644 index 0000000..7515fb3 --- /dev/null +++ b/gpulive/monitoring/base/grafana/role.yaml @@ -0,0 +1,63 @@ +# Source: grafana/templates/clusterrole.yaml +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + labels: + helm.sh/chart: grafana-8.10.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + app.kubernetes.io/version: "11.5.1" + name: grafana-clusterrole +rules: [] +--- +# Source: grafana/templates/clusterrolebinding.yaml +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: grafana-clusterrolebinding + labels: + helm.sh/chart: grafana-8.10.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + app.kubernetes.io/version: "11.5.1" +subjects: + - kind: ServiceAccount + name: grafana + namespace: monitoring +roleRef: + kind: ClusterRole + name: grafana-clusterrole + apiGroup: rbac.authorization.k8s.io +--- +# Source: grafana/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.10.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + app.kubernetes.io/version: "11.5.1" +rules: [] +--- +# Source: grafana/templates/rolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.10.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + app.kubernetes.io/version: "11.5.1" +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: grafana +subjects: +- kind: ServiceAccount + name: grafana + namespace: monitoring \ No newline at end of file diff --git a/gpulive/monitoring/base/grafana/sa.yaml b/gpulive/monitoring/base/grafana/sa.yaml new file mode 100644 index 0000000..b32ee04 --- /dev/null +++ b/gpulive/monitoring/base/grafana/sa.yaml @@ -0,0 +1,12 @@ +# Source: grafana/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: false +metadata: + labels: + helm.sh/chart: grafana-8.10.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + app.kubernetes.io/version: "11.5.1" + name: grafana + namespace: monitoring \ No newline at end of file diff --git a/gpulive/monitoring/base/grafana/secret.yaml b/gpulive/monitoring/base/grafana/secret.yaml new file mode 100644 index 0000000..f8e0843 --- /dev/null +++ b/gpulive/monitoring/base/grafana/secret.yaml @@ -0,0 +1,17 @@ +--- +# Source: grafana/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.10.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + app.kubernetes.io/version: "11.5.1" +type: Opaque +data: + admin-user: "YWRtaW4=" + admin-password: "cW9TV0c3T0xyOFB5UGJEaWt5VkYxTE9LUzJOUVhreEh2c3lWMXNHcw==" #qoSWG7OLr8PyPbDikyVF1LOKS2NQXkxHvsyV1sGs + ldap-toml: "" \ No newline at end of file diff --git a/gpulive/monitoring/base/grafana/service.yaml b/gpulive/monitoring/base/grafana/service.yaml new file mode 100644 index 0000000..acd31a1 --- /dev/null +++ b/gpulive/monitoring/base/grafana/service.yaml @@ -0,0 +1,21 @@ +# Source: grafana/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.10.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + app.kubernetes.io/version: "11.5.1" +spec: + type: ClusterIP + ports: + - name: service + port: 80 + protocol: TCP + targetPort: 3000 + selector: + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana \ No newline at end of file diff --git a/gpulive/monitoring/base/prometheus/cluster_role.yaml b/gpulive/monitoring/base/prometheus/cluster_role.yaml new file mode 100644 index 0000000..abcd2d6 --- /dev/null +++ b/gpulive/monitoring/base/prometheus/cluster_role.yaml @@ -0,0 +1,248 @@ +# Source: prometheus/charts/kube-state-metrics/templates/role.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + helm.sh/chart: kube-state-metrics-5.28.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: kube-state-metrics + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "2.14.0" + name: prometheus-kube-state-metrics +rules: + +- apiGroups: ["certificates.k8s.io"] + resources: + - certificatesigningrequests + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - configmaps + verbs: ["list", "watch"] + +- apiGroups: ["batch"] + resources: + - cronjobs + verbs: ["list", "watch"] + +- apiGroups: ["extensions", "apps"] + resources: + - daemonsets + verbs: ["list", "watch"] + +- apiGroups: ["extensions", "apps"] + resources: + - deployments + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - endpoints + verbs: ["list", "watch"] + +- apiGroups: ["autoscaling"] + resources: + - horizontalpodautoscalers + verbs: ["list", "watch"] + +- apiGroups: ["extensions", "networking.k8s.io"] + resources: + - ingresses + verbs: ["list", "watch"] + +- apiGroups: ["batch"] + resources: + - jobs + verbs: ["list", "watch"] + +- apiGroups: ["coordination.k8s.io"] + resources: + - leases + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - limitranges + verbs: ["list", "watch"] + +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - mutatingwebhookconfigurations + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - namespaces + verbs: ["list", "watch"] + +- apiGroups: ["networking.k8s.io"] + resources: + - networkpolicies + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - nodes + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - persistentvolumeclaims + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - persistentvolumes + verbs: ["list", "watch"] + +- apiGroups: ["policy"] + resources: + - poddisruptionbudgets + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - pods + verbs: ["list", "watch"] + +- apiGroups: ["extensions", "apps"] + resources: + - replicasets + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - replicationcontrollers + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - resourcequotas + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - secrets + verbs: ["list", "watch"] + +- apiGroups: [""] + resources: + - services + verbs: ["list", "watch"] + +- apiGroups: ["apps"] + resources: + - statefulsets + verbs: ["list", "watch"] + +- apiGroups: ["storage.k8s.io"] + resources: + - storageclasses + verbs: ["list", "watch"] + +- apiGroups: ["admissionregistration.k8s.io"] + resources: + - validatingwebhookconfigurations + verbs: ["list", "watch"] + +- apiGroups: ["storage.k8s.io"] + resources: + - volumeattachments + verbs: ["list", "watch"] +--- +# Source: prometheus/templates/clusterrole.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.1.0 + helm.sh/chart: prometheus-27.3.1 + app.kubernetes.io/part-of: prometheus + name: prometheus-server +rules: + - apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - nodes/metrics + - services + - endpoints + - pods + - ingresses + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - "extensions" + - "networking.k8s.io" + resources: + - ingresses/status + - ingresses + verbs: + - get + - list + - watch + - apiGroups: + - "discovery.k8s.io" + resources: + - endpointslices + verbs: + - get + - list + - watch + - nonResourceURLs: + - "/metrics" + verbs: + - get +--- +# Source: prometheus/charts/kube-state-metrics/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + helm.sh/chart: kube-state-metrics-5.28.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: kube-state-metrics + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "2.14.0" + name: prometheus-kube-state-metrics +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-kube-state-metrics +subjects: +- kind: ServiceAccount + name: prometheus-kube-state-metrics + namespace: monitoring +--- +# Source: prometheus/templates/clusterrolebinding.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.1.0 + helm.sh/chart: prometheus-27.3.1 + app.kubernetes.io/part-of: prometheus + name: prometheus-server +subjects: + - kind: ServiceAccount + name: prometheus-server + namespace: monitoring +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: prometheus-server \ No newline at end of file diff --git a/gpulive/monitoring/base/prometheus/cm.yaml b/gpulive/monitoring/base/prometheus/cm.yaml new file mode 100644 index 0000000..a1b4135 --- /dev/null +++ b/gpulive/monitoring/base/prometheus/cm.yaml @@ -0,0 +1,375 @@ +# Source: prometheus/charts/alertmanager/templates/configmap.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-alertmanager + labels: + helm.sh/chart: alertmanager-1.14.0 + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v0.28.0" + app.kubernetes.io/managed-by: Helm + namespace: monitoring +data: + alertmanager.yml: | + global: {} + receivers: + - name: default-receiver + route: + group_interval: 5m + group_wait: 10s + receiver: default-receiver + repeat_interval: 3h + templates: + - /etc/alertmanager/*.tmpl +--- +# Source: prometheus/templates/cm.yaml +apiVersion: v1 +kind: ConfigMap +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.1.0 + helm.sh/chart: prometheus-27.3.1 + app.kubernetes.io/part-of: prometheus + name: prometheus-server + namespace: monitoring +data: + allow-snippet-annotations: "false" + alerting_rules.yml: | + {} + alerts: | + {} + prometheus.yml: | + global: + evaluation_interval: 1m + scrape_interval: 1m + scrape_timeout: 10s + rule_files: + - /etc/config/recording_rules.yml + - /etc/config/alerting_rules.yml + - /etc/config/rules + - /etc/config/alerts + scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - localhost:9090 + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-apiservers + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: default;kubernetes;https + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_service_name + - __meta_kubernetes_endpoint_port_name + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes-cadvisor + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - honor_labels: true + job_name: kubernetes-service-endpoints + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-service-endpoints-slow + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + - honor_labels: true + job_name: prometheus-pushgateway + kubernetes_sd_configs: + - role: service + relabel_configs: + - action: keep + regex: pushgateway + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - honor_labels: true + job_name: kubernetes-services + kubernetes_sd_configs: + - role: service + metrics_path: /probe + params: + module: + - http_2xx + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - source_labels: + - __address__ + target_label: __param_target + - replacement: blackbox + target_label: __address__ + - source_labels: + - __param_target + target_label: instance + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - honor_labels: true + job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-pods-slow + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + alerting: + alertmanagers: + - kubernetes_sd_configs: + - role: pod + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_namespace] + regex: monitoring + action: keep + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance] + regex: prometheus + action: keep + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + regex: alertmanager + action: keep + - source_labels: [__meta_kubernetes_pod_container_port_number] + regex: "9093" + action: keep + recording_rules.yml: | + {} + rules: | + {} \ No newline at end of file diff --git a/gpulive/monitoring/base/prometheus/cm_custom_alert_rules.yaml b/gpulive/monitoring/base/prometheus/cm_custom_alert_rules.yaml new file mode 100644 index 0000000..bab5fee --- /dev/null +++ b/gpulive/monitoring/base/prometheus/cm_custom_alert_rules.yaml @@ -0,0 +1,194 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-rulefiles + namespace: monitoring +data: + # Awesome Prometheus alerts + ################################################################################ + # Basic resource monitoring prometheus self-monitoring + ################################################################################ + prometheus-self-monitoring.yaml: | + groups: + - name: prometheus-self-monitoring + rules: + - alert: Watchdog + annotations: + description: | + This is an alert meant to ensure that the entire alerting pipeline is functional. + This alert is always firing, therefore it should always be firing in Alertmanager + and always fire against a receiver. There are integrations with various notification + mechanisms that send a notification when this alert is not firing. For example the + "DeadMansSnitch" integration in PagerDuty. + runbook_url: https://runbooks.prometheus-operator.dev/runbooks/general/watchdog + summary: An alert that should always be firing to certify that Alertmanager + is working properly. + expr: vector(1) + labels: + severity: none + # Awesome Prometheus alerts + ################################################################################ + # Basic resource monitoring host and hardware (node-export) + ################################################################################ + prometheus-host-hardware.yaml: | + groups: + - name: prometheus-host-hardware + rules: + - alert: HostOutOfMemory (1.2.01) + expr: node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes * 100 < 10 + for: 2m + labels: + severity: warning + annotations: + summary: Host out of memory (instance {{ $labels.instance }}) + description: "Node memory is filling up (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostOutOfDiskSpace (1.2.07) + expr: (node_filesystem_avail_bytes * 100) / node_filesystem_size_bytes < 10 and ON (instance, device, mountpoint) node_filesystem_readonly == 0 + for: 2m + labels: + severity: warning + annotations: + summary: Host out of disk space (instance {{ $labels.instance }}) + description: "Disk is almost full (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # Please add ignored mountpoints in node_exporter parameters like + # "--collector.filesystem.ignored-mount-points=^/(sys|proc|dev|run)($|/)". + # Same rule using "node_filesystem_free_bytes" will fire when disk fills for non-root users. + - alert: HostHighCpuLoad (1.2.13) + expr: 100 - (avg by(instance) (rate(node_cpu_seconds_total{mode="idle"}[2m])) * 100) > 80 + for: 0m + labels: + severity: warning + annotations: + summary: Host high CPU load (instance {{ $labels.instance }}) + description: "CPU load is > 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: HostOomKillDetected (1.2.24) + expr: increase(node_vmstat_oom_kill[1m]) > 0 + for: 0m + labels: + severity: warning + annotations: + summary: Host OOM kill detected (instance {{ $labels.instance }}) + description: "OOM kill detected\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # Awesome Prometheus alerts + ################################################################################ + # Basic resource monitoring docker container (google/cAdvisor) + ################################################################################ + Prometheus-docker-containers.yaml: | + groups: + - name: prometheus-docker-containers + rules: + # This rule can be very noisy in dynamic infra with legitimate container start/stop/deployment. + - alert: ContainerKilled (1.3.1) + expr: time() - container_last_seen > 60 + for: 0m + labels: + severity: warning + annotations: + summary: Container killed (instance {{ $labels.instance }}) + description: "A container has disappeared\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # This rule can be very noisy in dynamic infra with legitimate container start/stop/deployment. + - alert: ContainerCpuUsage (1.3.3) + expr: (sum(rate(container_cpu_usage_seconds_total{name!=""}[3m])) BY (instance, name) * 100) > 80 + for: 2m + labels: + severity: warning + annotations: + summary: Container CPU usage (instance {{ $labels.instance }}) + description: "Container CPU usage is above 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # See https://medium.com/faun/how-much-is-too-much-the-linux-oomkiller-and-used-memory-d32186f29c9d + - alert: ContainerMemoryUsage (1.3.4) + expr: (sum(container_memory_working_set_bytes{name!=""}) BY (instance, name) / sum(container_spec_memory_limit_bytes > 0) BY (instance, name) * 100) > 80 + for: 2m + labels: + severity: warning + annotations: + summary: Container Memory usage (instance {{ $labels.instance }}) + description: "Container Memory usage is above 80%\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + # Awesome Prometheus alerts + ################################################################################ + # Orchestrators kubernetes (kube-state-metrics) + ################################################################################ + prometheus-kubernetes.yaml: | + groups: + - name: prometheus-kubernetes + rules: + - alert: KubernetesNodeReady (5.1.01) + expr: kube_node_status_condition{condition="Ready",status="true"} == 0 + for: 10m + labels: + severity: critical + annotations: + summary: Kubernetes Node ready (instance {{ $labels.instance }}) + description: "Node {{ $labels.node }} has been unready for a long time\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesOutOfDisk (5.1.04) + expr: kube_node_status_condition{condition="OutOfDisk",status="true"} == 1 + for: 2m + labels: + severity: critical + annotations: + summary: Kubernetes out of disk (instance {{ $labels.instance }}) + description: "{{ $labels.node }} has OutOfDisk condition\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesContainerOomKiller (5.1.06) + expr: (kube_pod_container_status_restarts_total - kube_pod_container_status_restarts_total offset 10m >= 1) and ignoring (reason) min_over_time(kube_pod_container_status_last_terminated_reason{reason="OOMKilled"}[10m]) == 1 + for: 0m + labels: + severity: warning + annotations: + summary: Kubernetes container oom killer (instance {{ $labels.instance }}) + description: "Container {{ $labels.container }} in pod {{ $labels.namespace }}/{{ $labels.pod }} has been OOMKilled {{ $value }} times in the last 10 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesVolumeOutOfDiskSpace (5.1.10) + expr: kubelet_volume_stats_available_bytes / kubelet_volume_stats_capacity_bytes * 100 < 10 + for: 2m + labels: + severity: warning + annotations: + summary: Kubernetes Volume out of disk space (instance {{ $labels.instance }}) + description: "Volume is almost full (< 10% left)\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesPersistentvolumeError (5.1.12) + expr: kube_persistentvolume_status_phase{phase=~"Failed|Pending", job="kube-state-metrics"} > 0 + for: 0m + labels: + severity: critical + annotations: + summary: Kubernetes PersistentVolume error (instance {{ $labels.instance }}) + description: "Persistent volume is in bad state\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesHpaScaleCapability (5.1.16) + expr: kube_horizontalpodautoscaler_status_desired_replicas >= kube_horizontalpodautoscaler_spec_max_replicas + for: 2m + labels: + severity: info + annotations: + summary: Kubernetes HPA scale capability (instance {{ $labels.instance }}) + description: "The maximum number of desired Pods has been hit\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesPodNotHealthy (5.1.17) + expr: min_over_time(sum by (namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown|Failed"})[15m:1m]) > 0 + for: 0m + labels: + severity: critical + annotations: + summary: Kubernetes Pod not healthy (instance {{ $labels.instance }}) + description: "Pod has been in a non-ready state for longer than 15 minutes.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesPodCrashLooping (5.1.18) + expr: increase(kube_pod_container_status_restarts_total[1m]) > 3 + for: 2m + labels: + severity: warning + annotations: + summary: Kubernetes pod crash looping (instance {{ $labels.instance }}) + description: "Pod {{ $labels.pod }} is crash looping\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesApiServerErrors (5.1.29) + expr: sum(rate(apiserver_request_total{job="apiserver",code=~"^(?:5..)$"}[1m])) / sum(rate(apiserver_request_total{job="apiserver"}[1m])) * 100 > 3 + for: 2m + labels: + severity: critical + annotations: + summary: Kubernetes API server errors (instance {{ $labels.instance }}) + description: "Kubernetes API server is experiencing high error rate\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" + - alert: KubernetesClientCertificateExpiresSoon (5.1.32) + expr: apiserver_client_certificate_expiration_seconds_count{job="apiserver"} > 0 and histogram_quantile(0.01, sum by (job, le) (rate(apiserver_client_certificate_expiration_seconds_bucket{job="apiserver"}[5m]))) < 24*60*60 + for: 0m + labels: + severity: critical + annotations: + summary: Kubernetes client certificate expires soon (instance {{ $labels.instance }}) + description: "A client certificate used to authenticate to the apiserver is expiring in less than 24.0 hours.\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" \ No newline at end of file diff --git a/gpulive/monitoring/base/prometheus/daemonset.yaml b/gpulive/monitoring/base/prometheus/daemonset.yaml new file mode 100644 index 0000000..53d3288 --- /dev/null +++ b/gpulive/monitoring/base/prometheus/daemonset.yaml @@ -0,0 +1,135 @@ +# Source: prometheus/charts/prometheus-node-exporter/templates/daemonset.yaml +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: prometheus-prometheus-node-exporter + namespace: monitoring + labels: + helm.sh/chart: prometheus-node-exporter-4.43.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: prometheus-node-exporter + app.kubernetes.io/name: prometheus-node-exporter + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "1.8.2" +spec: + selector: + matchLabels: + app.kubernetes.io/name: prometheus-node-exporter + app.kubernetes.io/instance: prometheus + revisionHistoryLimit: 10 + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + template: + metadata: + annotations: + cluster-autoscaler.kubernetes.io/safe-to-evict: "true" + labels: + helm.sh/chart: prometheus-node-exporter-4.43.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: prometheus-node-exporter + app.kubernetes.io/name: prometheus-node-exporter + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "1.8.2" + spec: + automountServiceAccountToken: false + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + serviceAccountName: prometheus-prometheus-node-exporter + containers: + - name: node-exporter + image: quay.io/prometheus/node-exporter:v1.8.2 + imagePullPolicy: Always + args: + - --path.procfs=/host/proc + - --path.sysfs=/host/sys + - --path.rootfs=/host/root + - --path.udev.data=/host/root/run/udev/data + - --web.listen-address=[$(HOST_IP)]:9100 + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + env: + - name: HOST_IP + value: 0.0.0.0 + ports: + - name: metrics + containerPort: 9100 + protocol: TCP + livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: + path: / + port: 9100 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: + path: / + port: 9100 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + volumeMounts: + - name: proc + mountPath: /host/proc + readOnly: true + - name: sys + mountPath: /host/sys + readOnly: true + - name: root + mountPath: /host/root + mountPropagation: HostToContainer + readOnly: true + resources: + requests: + cpu: 50m + memory: 50Mi + limits: + cpu: 100m + memory: 100Mi + hostNetwork: true + hostPID: true + hostIPC: false + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: eks.amazonaws.com/compute-type + operator: NotIn + values: + - fargate + - key: type + operator: NotIn + values: + - virtual-kubelet + nodeSelector: + kubernetes.io/os: linux + tolerations: + - effect: NoSchedule + operator: Exists + volumes: + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys + - name: root + hostPath: + path: / \ No newline at end of file diff --git a/gpulive/monitoring/base/prometheus/deploy.yaml b/gpulive/monitoring/base/prometheus/deploy.yaml new file mode 100644 index 0000000..89ae67d --- /dev/null +++ b/gpulive/monitoring/base/prometheus/deploy.yaml @@ -0,0 +1,290 @@ +# Source: prometheus/charts/kube-state-metrics/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-kube-state-metrics + namespace: monitoring + labels: + helm.sh/chart: kube-state-metrics-5.28.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: kube-state-metrics + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "2.14.0" +spec: + selector: + matchLabels: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: prometheus + replicas: 1 + strategy: + type: RollingUpdate + revisionHistoryLimit: 10 + template: + metadata: + labels: + helm.sh/chart: kube-state-metrics-5.28.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: kube-state-metrics + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "2.14.0" + spec: + nodeSelector: + nodegroup: nd + automountServiceAccountToken: true + hostNetwork: false + serviceAccountName: prometheus-kube-state-metrics + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + seccompProfile: + type: RuntimeDefault + containers: + - name: kube-state-metrics + args: + - --port=8080 + - --resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,horizontalpodautoscalers,ingresses,jobs,leases,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments + imagePullPolicy: Always + image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.14.0 + ports: + - containerPort: 8080 + name: "http" + livenessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: + path: /livez + port: 8080 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + readinessProbe: + failureThreshold: 3 + httpGet: + httpHeaders: + path: /readyz + port: 8081 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + resources: + requests: + cpu: 50m + memory: 125Mi + limits: + cpu: 100m + memory: 250Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + readOnlyRootFilesystem: true +--- +# Source: prometheus/charts/prometheus-pushgateway/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + helm.sh/chart: prometheus-pushgateway-2.17.0 + app.kubernetes.io/name: prometheus-pushgateway + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v1.11.0" + app.kubernetes.io/managed-by: Helm + name: prometheus-prometheus-pushgateway + namespace: monitoring +spec: + replicas: 1 + strategy: + type: Recreate + selector: + matchLabels: + app.kubernetes.io/name: prometheus-pushgateway + app.kubernetes.io/instance: prometheus + template: + metadata: + labels: + helm.sh/chart: prometheus-pushgateway-2.17.0 + app.kubernetes.io/name: prometheus-pushgateway + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v1.11.0" + app.kubernetes.io/managed-by: Helm + spec: + nodeSelector: + nodegroup: nd + serviceAccountName: prometheus-prometheus-pushgateway + automountServiceAccountToken: true + containers: + - name: pushgateway + image: "quay.io/prometheus/pushgateway:v1.11.0" + imagePullPolicy: Always + ports: + - name: metrics + containerPort: 9091 + protocol: TCP + livenessProbe: + httpGet: + path: /-/healthy + port: 9091 + initialDelaySeconds: 10 + timeoutSeconds: 10 + readinessProbe: + httpGet: + path: /-/ready + port: 9091 + initialDelaySeconds: 10 + timeoutSeconds: 10 + volumeMounts: + - name: storage-volume + mountPath: "/data" + subPath: "" + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 200m + memory: 512Mi + securityContext: + fsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + volumes: + - name: storage-volume + emptyDir: {} +--- +# Source: prometheus/templates/deploy.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.1.0 + helm.sh/chart: prometheus-27.3.1 + app.kubernetes.io/part-of: prometheus + name: prometheus-server + namespace: monitoring +spec: + selector: + matchLabels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + replicas: 1 + revisionHistoryLimit: 10 + strategy: + type: Recreate + rollingUpdate: null + template: + metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.1.0 + helm.sh/chart: prometheus-27.3.1 + app.kubernetes.io/part-of: prometheus + spec: + nodeSelector: + nodegroup: nd + enableServiceLinks: true + serviceAccountName: prometheus-server + containers: + - name: prometheus-server-configmap-reload + image: "quay.io/prometheus-operator/prometheus-config-reloader:v0.79.2" + imagePullPolicy: "Always" + args: + - --watched-dir=/etc/config + - --listen-address=0.0.0.0:8080 + - --reload-url=http://127.0.0.1:9090/-/reload + ports: + - containerPort: 8080 + name: metrics + livenessProbe: + httpGet: + path: /healthz + port: metrics + scheme: HTTP + initialDelaySeconds: 2 + periodSeconds: 10 + readinessProbe: + httpGet: + path: /healthz + port: metrics + scheme: HTTP + periodSeconds: 10 + volumeMounts: + - name: config-volume + mountPath: /etc/config + readOnly: true + + - name: prometheus-server + image: "quay.io/prometheus/prometheus:v3.1.0" + imagePullPolicy: "Always" + args: + - --storage.tsdb.retention.time=15d + - --config.file=/etc/config/prometheus.yml + - --storage.tsdb.path=/data + - --web.console.libraries=/etc/prometheus/console_libraries + - --web.console.templates=/etc/prometheus/consoles + - --web.enable-lifecycle + ports: + - containerPort: 9090 + readinessProbe: + httpGet: + path: /-/ready + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 5 + timeoutSeconds: 4 + failureThreshold: 3 + successThreshold: 1 + livenessProbe: + httpGet: + path: /-/healthy + port: 9090 + scheme: HTTP + initialDelaySeconds: 30 + periodSeconds: 15 + timeoutSeconds: 10 + failureThreshold: 3 + successThreshold: 1 + volumeMounts: + - name: config-volume + mountPath: /etc/config + - name: storage-volume + mountPath: /data + subPath: "" + resources: + requests: + cpu: 500m + memory: 2Gi + limits: + cpu: 1050m + memory: 4Gi + dnsPolicy: ClusterFirst + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + terminationGracePeriodSeconds: 300 + volumes: + - name: config-volume + configMap: + name: prometheus-server + - name: storage-volume + persistentVolumeClaim: + claimName: prometheus-server \ No newline at end of file diff --git a/gpulive/monitoring/base/prometheus/ingress_alertmanager.yaml b/gpulive/monitoring/base/prometheus/ingress_alertmanager.yaml new file mode 100644 index 0000000..87afe8d --- /dev/null +++ b/gpulive/monitoring/base/prometheus/ingress_alertmanager.yaml @@ -0,0 +1,29 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prometheus-alertmanager-ingress + namespace: monitoring + annotations: + # 인증 방법 설정: basic auth + nginx.ingress.kubernetes.io/auth-type: basic + # basic auth 사용자가 들어있는 secret 설정 + nginx.ingress.kubernetes.io/auth-secret: basic-auth + # 인증 요청시 나오는 메세지 설정 + nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - admin' +spec: + ingressClassName: nginx + rules: + - host: alert.cone-chain.com + http: + paths: + - backend: + service: + name: prometheus-alertmanager + port: + number: 9093 + path: / + pathType: Prefix + tls: + - hosts: + - prometheus.cone-chain.com + secretName: ssl-cc \ No newline at end of file diff --git a/gpulive/monitoring/base/prometheus/ingress_prometheus.yaml b/gpulive/monitoring/base/prometheus/ingress_prometheus.yaml new file mode 100644 index 0000000..c56bae2 --- /dev/null +++ b/gpulive/monitoring/base/prometheus/ingress_prometheus.yaml @@ -0,0 +1,29 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prometheus-ingress + namespace: monitoring + annotations: + # 인증 방법 설정: basic auth + nginx.ingress.kubernetes.io/auth-type: basic + # basic auth 사용자가 들어있는 secret 설정 + nginx.ingress.kubernetes.io/auth-secret: basic-auth + # 인증 요청시 나오는 메세지 설정 + nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - admin' +spec: + ingressClassName: nginx + rules: + - host: prometheus.cone-chain.com + http: + paths: + - backend: + service: + name: prometheus-server + port: + number: 80 + path: / + pathType: Prefix + tls: + - hosts: + - prometheus.cone-chain.com + secretName: ssl-cc \ No newline at end of file diff --git a/gpulive/monitoring/base/prometheus/kustomization.yaml b/gpulive/monitoring/base/prometheus/kustomization.yaml new file mode 100644 index 0000000..027f1b7 --- /dev/null +++ b/gpulive/monitoring/base/prometheus/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: monitoring + +resources: + - namespace.yaml + - sa.yaml + - cluster_role.yaml + - cm.yaml + - deploy.yaml + - daemonset.yaml + - pvc.yaml + - service.yaml + - state_full_set.yaml + # - secret.yaml + - ingress_alertmanager.yaml + - ingress_prometheus.yaml + - cm_custom_alert_rules.yaml diff --git a/gpulive/monitoring/base/prometheus/namespace.yaml b/gpulive/monitoring/base/prometheus/namespace.yaml new file mode 100644 index 0000000..07aaec3 --- /dev/null +++ b/gpulive/monitoring/base/prometheus/namespace.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring + labels: + kubernetes.io/metadata.name: monitoring \ No newline at end of file diff --git a/gpulive/monitoring/base/prometheus/pvc.yaml b/gpulive/monitoring/base/prometheus/pvc.yaml new file mode 100644 index 0000000..70a8a29 --- /dev/null +++ b/gpulive/monitoring/base/prometheus/pvc.yaml @@ -0,0 +1,19 @@ +# Source: prometheus/templates/pvc.yaml +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.1.0 + helm.sh/chart: prometheus-27.3.1 + app.kubernetes.io/part-of: prometheus + name: prometheus-server + namespace: monitoring +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "8Gi" \ No newline at end of file diff --git a/gpulive/monitoring/base/prometheus/sa.yaml b/gpulive/monitoring/base/prometheus/sa.yaml new file mode 100644 index 0000000..2bfde38 --- /dev/null +++ b/gpulive/monitoring/base/prometheus/sa.yaml @@ -0,0 +1,76 @@ +--- +# Source: prometheus/charts/alertmanager/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus-alertmanager + labels: + helm.sh/chart: alertmanager-1.14.0 + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v0.28.0" + app.kubernetes.io/managed-by: Helm + namespace: monitoring +automountServiceAccountToken: true +--- +# Source: prometheus/charts/kube-state-metrics/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: true +metadata: + labels: + helm.sh/chart: kube-state-metrics-5.28.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: kube-state-metrics + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "2.14.0" + name: prometheus-kube-state-metrics + namespace: monitoring +--- +# Source: prometheus/charts/prometheus-node-exporter/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus-prometheus-node-exporter + namespace: monitoring + labels: + helm.sh/chart: prometheus-node-exporter-4.43.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: prometheus-node-exporter + app.kubernetes.io/name: prometheus-node-exporter + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "1.8.2" +automountServiceAccountToken: false +--- +# Source: prometheus/charts/prometheus-pushgateway/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + helm.sh/chart: prometheus-pushgateway-2.17.0 + app.kubernetes.io/name: prometheus-pushgateway + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v1.11.0" + app.kubernetes.io/managed-by: Helm + name: prometheus-prometheus-pushgateway + namespace: monitoring +automountServiceAccountToken: true +--- +# Source: prometheus/templates/serviceaccount.yaml +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.1.0 + helm.sh/chart: prometheus-27.3.1 + app.kubernetes.io/part-of: prometheus + name: prometheus-server + namespace: monitoring + annotations: + {} \ No newline at end of file diff --git a/gpulive/monitoring/base/prometheus/service.yaml b/gpulive/monitoring/base/prometheus/service.yaml new file mode 100644 index 0000000..7c4cace --- /dev/null +++ b/gpulive/monitoring/base/prometheus/service.yaml @@ -0,0 +1,151 @@ +# Source: prometheus/charts/alertmanager/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + name: prometheus-alertmanager + labels: + helm.sh/chart: alertmanager-1.14.0 + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v0.28.0" + app.kubernetes.io/managed-by: Helm + namespace: monitoring +spec: + type: ClusterIP + ports: + - port: 9093 + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus +--- +# Source: prometheus/charts/alertmanager/templates/services.yaml +apiVersion: v1 +kind: Service +metadata: + name: prometheus-alertmanager-headless + labels: + helm.sh/chart: alertmanager-1.14.0 + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v0.28.0" + app.kubernetes.io/managed-by: Helm + namespace: monitoring +spec: + clusterIP: None + ports: + - port: 9093 + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus +--- +# Source: prometheus/charts/kube-state-metrics/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: prometheus-kube-state-metrics + namespace: monitoring + labels: + helm.sh/chart: kube-state-metrics-5.28.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: kube-state-metrics + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "2.14.0" + annotations: + prometheus.io/scrape: 'true' +spec: + type: "ClusterIP" + ports: + - name: "http" + protocol: TCP + port: 8080 + targetPort: 8080 + + selector: + app.kubernetes.io/name: kube-state-metrics + app.kubernetes.io/instance: prometheus +--- +# Source: prometheus/charts/prometheus-node-exporter/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + name: prometheus-prometheus-node-exporter + namespace: monitoring + labels: + helm.sh/chart: prometheus-node-exporter-4.43.1 + app.kubernetes.io/managed-by: Helm + app.kubernetes.io/component: metrics + app.kubernetes.io/part-of: prometheus-node-exporter + app.kubernetes.io/name: prometheus-node-exporter + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "1.8.2" + annotations: + prometheus.io/scrape: "true" +spec: + type: ClusterIP + ports: + - port: 9100 + targetPort: 9100 + protocol: TCP + name: metrics + selector: + app.kubernetes.io/name: prometheus-node-exporter + app.kubernetes.io/instance: prometheus +--- +# Source: prometheus/charts/prometheus-pushgateway/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/probe: pushgateway + labels: + helm.sh/chart: prometheus-pushgateway-2.17.0 + app.kubernetes.io/name: prometheus-pushgateway + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v1.11.0" + app.kubernetes.io/managed-by: Helm + name: prometheus-prometheus-pushgateway + namespace: monitoring +spec: + type: ClusterIP + ports: + - port: 9091 + targetPort: 9091 + protocol: TCP + name: http + selector: + app.kubernetes.io/name: prometheus-pushgateway + app.kubernetes.io/instance: prometheus +--- +# Source: prometheus/templates/service.yaml +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: v3.1.0 + helm.sh/chart: prometheus-27.3.1 + app.kubernetes.io/part-of: prometheus + name: prometheus-server + namespace: monitoring +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: 9090 + selector: + app.kubernetes.io/component: server + app.kubernetes.io/name: prometheus + app.kubernetes.io/instance: prometheus + sessionAffinity: None + type: "ClusterIP" \ No newline at end of file diff --git a/gpulive/monitoring/base/prometheus/state_full_set.yaml b/gpulive/monitoring/base/prometheus/state_full_set.yaml new file mode 100644 index 0000000..f02a1c6 --- /dev/null +++ b/gpulive/monitoring/base/prometheus/state_full_set.yaml @@ -0,0 +1,90 @@ +# Source: prometheus/charts/alertmanager/templates/statefulset.yaml +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: prometheus-alertmanager + labels: + helm.sh/chart: alertmanager-1.14.0 + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus + app.kubernetes.io/version: "v0.28.0" + app.kubernetes.io/managed-by: Helm + namespace: monitoring +spec: + replicas: 1 + minReadySeconds: 0 + revisionHistoryLimit: 10 + selector: + matchLabels: + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus + serviceName: prometheus-alertmanager-headless + template: + metadata: + labels: + app.kubernetes.io/name: alertmanager + app.kubernetes.io/instance: prometheus + spec: + nodeSelector: + nodegroup: nd + automountServiceAccountToken: true + serviceAccountName: prometheus-alertmanager + securityContext: + fsGroup: 65534 + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + containers: + - name: alertmanager + securityContext: + runAsGroup: 65534 + runAsNonRoot: true + runAsUser: 65534 + image: "quay.io/prometheus/alertmanager:v0.28.0" + imagePullPolicy: Always + env: + - name: POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + args: + - --storage.path=/alertmanager + - --config.file=/etc/alertmanager/alertmanager.yml + ports: + - name: http + containerPort: 9093 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: / + port: http + resources: + requests: + cpu: 200m + memory: 512Mi + limits: + cpu: 200m + memory: 512Mi + volumeMounts: + - name: config + mountPath: /etc/alertmanager + - name: storage + mountPath: /alertmanager + volumes: + - name: config + configMap: + name: prometheus-alertmanager + volumeClaimTemplates: + - metadata: + name: storage + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 2Gi \ No newline at end of file diff --git a/gpulive/monitoring/overlays/default/auth b/gpulive/monitoring/overlays/default/auth new file mode 100644 index 0000000..cb0bcdd --- /dev/null +++ b/gpulive/monitoring/overlays/default/auth @@ -0,0 +1 @@ +admin:$apr1$JIoWDh3p$Mo4E8nEh3beHp9n1IRjmc/ diff --git a/gpulive/monitoring/overlays/default/grafana/kustomization.yaml b/gpulive/monitoring/overlays/default/grafana/kustomization.yaml new file mode 100644 index 0000000..2f05aa9 --- /dev/null +++ b/gpulive/monitoring/overlays/default/grafana/kustomization.yaml @@ -0,0 +1,29 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: + - ../../../base/grafana + +namespace: monitoring + +patches: + - target: + kind: ConfigMap + name: grafana + namespace: monitoring + path: patch-cm.yaml + - target: + kind: Secret + name: grafana + namespace: monitoring + path: patch-secret.yaml + - target: + kind: Deployment + name: grafana + namespace: monitoring + path: patch-deploy.yaml + - target: + kind: Ingress + name: grafana-ingress + namespace: monitoring + path: patch-ingress_grafana.yaml \ No newline at end of file diff --git a/gpulive/monitoring/overlays/default/grafana/patch-cm.yaml b/gpulive/monitoring/overlays/default/grafana/patch-cm.yaml new file mode 100644 index 0000000..2019016 --- /dev/null +++ b/gpulive/monitoring/overlays/default/grafana/patch-cm.yaml @@ -0,0 +1,32 @@ +# 도메인 정보 변경 필요. +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.10.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + app.kubernetes.io/version: "11.5.1" +data: + + grafana.ini: | + [analytics] + check_for_updates = true + [grafana_net] + url = https://grafana.net + [log] + mode = console + [paths] + data = /var/lib/grafana/ + logs = /var/log/grafana + plugins = /var/lib/grafana/plugins + provisioning = /etc/grafana/provisioning + [server] + domain = 'grafana.nhngpuaas.com' + root_url = 'https://grafana.nhngpuaas.com' + [auth.anonymous] + enabled = true + org_name = Main Org. + org_role = Viewer diff --git a/gpulive/monitoring/overlays/default/grafana/patch-deploy.yaml b/gpulive/monitoring/overlays/default/grafana/patch-deploy.yaml new file mode 100644 index 0000000..5d3f468 --- /dev/null +++ b/gpulive/monitoring/overlays/default/grafana/patch-deploy.yaml @@ -0,0 +1,20 @@ +# Source: grafana/templates/deployment.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + namespace: monitoring +spec: + template: + spec: + containers: + - name: grafana + volumeMounts: + - name: storage + mountPath: "/var/lib/grafana" + volumes: + - name: storage + persistentVolumeClaim: + claimName: grafana-pvc + # - name: storage + # emptyDir: {} \ No newline at end of file diff --git a/gpulive/monitoring/overlays/default/grafana/patch-ingress_grafana.yaml b/gpulive/monitoring/overlays/default/grafana/patch-ingress_grafana.yaml new file mode 100644 index 0000000..6839329 --- /dev/null +++ b/gpulive/monitoring/overlays/default/grafana/patch-ingress_grafana.yaml @@ -0,0 +1,24 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: grafana-ingress + namespace: monitoring +spec: + ingressClassName: nginx + rules: + - host: grafana.nhngpuaas.com + http: + paths: + - backend: + service: + name: grafana + port: + number: 80 + path: / + pathType: Prefix + tls: + - hosts: + - grafana.nhngpuaas.com + secretName: nhngpuaas-ssl + + \ No newline at end of file diff --git a/gpulive/monitoring/overlays/default/grafana/patch-secret.yaml b/gpulive/monitoring/overlays/default/grafana/patch-secret.yaml new file mode 100644 index 0000000..c4f40c6 --- /dev/null +++ b/gpulive/monitoring/overlays/default/grafana/patch-secret.yaml @@ -0,0 +1,17 @@ +--- +# Source: grafana/templates/secret.yaml +apiVersion: v1 +kind: Secret +metadata: + name: grafana + namespace: monitoring + labels: + helm.sh/chart: grafana-8.10.0 + app.kubernetes.io/name: grafana + app.kubernetes.io/instance: grafana + app.kubernetes.io/version: "11.5.1" +type: Opaque +data: + admin-user: "YWRtaW4=" + admin-password: "bmhuIUAjMTIz" + # ldap-toml: "" \ No newline at end of file diff --git a/gpulive/monitoring/overlays/default/kustomization.yaml b/gpulive/monitoring/overlays/default/kustomization.yaml new file mode 100644 index 0000000..09bb902 --- /dev/null +++ b/gpulive/monitoring/overlays/default/kustomization.yaml @@ -0,0 +1,17 @@ +# Prometheus에 종속적으로 설치 +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: monitoring + +resources: + - prometheus + - grafana + +generatorOptions: + disableNameSuffixHash: true +secretGenerator: + - name: basic-auth + files: + - auth + type: Opaque \ No newline at end of file diff --git a/gpulive/monitoring/overlays/default/prometheus/kustomization.yaml b/gpulive/monitoring/overlays/default/prometheus/kustomization.yaml new file mode 100644 index 0000000..1d35c34 --- /dev/null +++ b/gpulive/monitoring/overlays/default/prometheus/kustomization.yaml @@ -0,0 +1,34 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../../base/prometheus + +namespace: monitoring + +patches: + - target: + kind: ConfigMap + name: prometheus-server + namespace: monitoring + path: patch-cm-prometheus-server.yaml + - target: + kind: Deployment + name: prometheus-server + namespace: monitoring + path: patch-deploy.yaml + - target: + kind: Ingress + name: prometheus-ingress + namespace: monitoring + path: patch-ingress_prometheus.yaml + - target: + kind: Ingress + name: prometheus-alertmanager-ingress + namespace: monitoring + path: patch-ingress_alertmanager.yaml + # - target: + # kind: Secret + # name: basic-auth + # namespace: monitoring + # path: patch-basic-auth.yaml diff --git a/gpulive/monitoring/overlays/default/prometheus/patch-cm-prometheus-server.yaml b/gpulive/monitoring/overlays/default/prometheus/patch-cm-prometheus-server.yaml new file mode 100644 index 0000000..b684a13 --- /dev/null +++ b/gpulive/monitoring/overlays/default/prometheus/patch-cm-prometheus-server.yaml @@ -0,0 +1,335 @@ +# - "/etc/config/rules/*.yaml" # 수정대상 +# - "/etc/config/alerts/*.yaml" # 수정대상 +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-server + namespace: monitoring +data: + prometheus.yml: | + global: + evaluation_interval: 1m + scrape_interval: 1m + scrape_timeout: 10s + rule_files: + - /etc/config/recording_rules.yml + - /etc/config/alerting_rules.yml + - "/etc/config/rules/*.yaml" + - "/etc/config/alerts/*.yaml" + scrape_configs: + - job_name: prometheus + static_configs: + - targets: + - localhost:9090 + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-apiservers + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: default;kubernetes;https + source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_service_name + - __meta_kubernetes_endpoint_port_name + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + job_name: kubernetes-nodes-cadvisor + kubernetes_sd_configs: + - role: node + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - replacement: kubernetes.default.svc:443 + target_label: __address__ + - regex: (.+) + replacement: /api/v1/nodes/$1/proxy/metrics/cadvisor + source_labels: + - __meta_kubernetes_node_name + target_label: __metrics_path__ + scheme: https + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - honor_labels: true + job_name: kubernetes-service-endpoints + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-service-endpoints-slow + kubernetes_sd_configs: + - role: endpoints + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (.+?)(?::\d+)?;(\d+) + replacement: $1:$2 + source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_service_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_service_name + target_label: service + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + - honor_labels: true + job_name: prometheus-pushgateway + kubernetes_sd_configs: + - role: service + relabel_configs: + - action: keep + regex: pushgateway + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - honor_labels: true + job_name: kubernetes-services + kubernetes_sd_configs: + - role: service + metrics_path: /probe + params: + module: + - http_2xx + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_probe + - source_labels: + - __address__ + target_label: __param_target + - replacement: blackbox + target_label: __address__ + - source_labels: + - __param_target + target_label: instance + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - honor_labels: true + job_name: kubernetes-pods + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape + - action: drop + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + - honor_labels: true + job_name: kubernetes-pods-slow + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: keep + regex: true + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scrape_slow + - action: replace + regex: (https?) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_scheme + target_label: __scheme__ + - action: replace + regex: (.+) + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_path + target_label: __metrics_path__ + - action: replace + regex: (\d+);(([A-Fa-f0-9]{1,4}::?){1,7}[A-Fa-f0-9]{1,4}) + replacement: '[$2]:$1' + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: replace + regex: (\d+);((([0-9]+?)(\.|$)){4}) + replacement: $2:$1 + source_labels: + - __meta_kubernetes_pod_annotation_prometheus_io_port + - __meta_kubernetes_pod_ip + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_annotation_prometheus_io_param_(.+) + replacement: __param_$1 + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - action: drop + regex: Pending|Succeeded|Failed|Completed + source_labels: + - __meta_kubernetes_pod_phase + - action: replace + source_labels: + - __meta_kubernetes_pod_node_name + target_label: node + scrape_interval: 5m + scrape_timeout: 30s + alerting: + alertmanagers: + - kubernetes_sd_configs: + - role: pod + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_namespace] + regex: monitoring + action: keep + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance] + regex: prometheus + action: keep + - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name] + regex: alertmanager + action: keep + - source_labels: [__meta_kubernetes_pod_container_port_number] + regex: "9093" + action: keep diff --git a/gpulive/monitoring/overlays/default/prometheus/patch-deploy.yaml b/gpulive/monitoring/overlays/default/prometheus/patch-deploy.yaml new file mode 100644 index 0000000..1758854 --- /dev/null +++ b/gpulive/monitoring/overlays/default/prometheus/patch-deploy.yaml @@ -0,0 +1,19 @@ +# Source: prometheus/templates/deploy.yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-server + namespace: monitoring +spec: + template: + spec: + containers: + - name: prometheus-server + volumeMounts: + - mountPath: /etc/config/alerts + name: config-volume2 + volumes: + - configMap: + defaultMode: 420 + name: prometheus-rulefiles + name: config-volume2 \ No newline at end of file diff --git a/gpulive/monitoring/overlays/default/prometheus/patch-ingress_alertmanager.yaml b/gpulive/monitoring/overlays/default/prometheus/patch-ingress_alertmanager.yaml new file mode 100644 index 0000000..d21488b --- /dev/null +++ b/gpulive/monitoring/overlays/default/prometheus/patch-ingress_alertmanager.yaml @@ -0,0 +1,29 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prometheus-alertmanager-ingress + namespace: monitoring + annotations: + # 인증 방법 설정: basic auth + nginx.ingress.kubernetes.io/auth-type: basic + # basic auth 사용자가 들어있는 secret 설정 + nginx.ingress.kubernetes.io/auth-secret: basic-auth + # 인증 요청시 나오는 메세지 설정 + nginx.ingress.kubernetes.io/auth-realm: 'Authentication Required - admin' +spec: + ingressClassName: nginx + rules: + - host: alert.nhngpuaas.com + http: + paths: + - backend: + service: + name: prometheus-alertmanager + port: + number: 9093 + path: / + pathType: Prefix + tls: + - hosts: + - alert.nhngpuaas.com + secretName: nhngpuaas-ssl \ No newline at end of file diff --git a/gpulive/monitoring/overlays/default/prometheus/patch-ingress_prometheus.yaml b/gpulive/monitoring/overlays/default/prometheus/patch-ingress_prometheus.yaml new file mode 100644 index 0000000..c03f487 --- /dev/null +++ b/gpulive/monitoring/overlays/default/prometheus/patch-ingress_prometheus.yaml @@ -0,0 +1,22 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: prometheus-ingress + namespace: monitoring +spec: + ingressClassName: nginx + rules: + - host: prometheus.nhngpuaas.com + http: + paths: + - backend: + service: + name: prometheus-server + port: + number: 80 + path: / + pathType: Prefix + tls: + - hosts: + - prometheus.nhngpuaas.com + secretName: nhngpuaas-ssl