feat: Initial NFS PV usage exporter implementation
- Flask app with /usage/<pv_name> and /metrics endpoints - 300s cache TTL with threading lock for du command - ionice/nice for low I/O priority - Dockerfile with gunicorn - Helm chart with deployment, service, networkpolicy - NetworkPolicy allows only gpulive and monitoring namespaces Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
commit
6db99ea0ef
|
|
@ -0,0 +1,4 @@
|
|||
*.tgz
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.env
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
coreutils \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY app.py .
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["gunicorn", "--bind", "0.0.0.0:8080", "--workers", "1", "--threads", "4", "app:app"]
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
from flask import Flask
|
||||
from prometheus_client import Gauge, generate_latest, CONTENT_TYPE_LATEST
|
||||
import subprocess
|
||||
import time
|
||||
import os
|
||||
import threading
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
pv_usage_bytes = Gauge('nfs_pv_usage_bytes', 'PV usage in bytes', ['pv_name'])
|
||||
pv_last_check = Gauge('nfs_pv_last_check_timestamp', 'Last check timestamp', ['pv_name'])
|
||||
|
||||
cache = {}
|
||||
CACHE_TTL = 300
|
||||
du_lock = threading.Lock() # 동시 du 방지
|
||||
|
||||
def get_pv_usage(pv_name):
|
||||
now = time.time()
|
||||
|
||||
# 캐시 확인 (락 없이)
|
||||
if pv_name in cache and now - cache[pv_name]['time'] < CACHE_TTL:
|
||||
return cache[pv_name]['bytes']
|
||||
|
||||
# du는 한 번에 하나만
|
||||
with du_lock:
|
||||
# 락 획득 후 다시 캐시 확인
|
||||
if pv_name in cache and time.time() - cache[pv_name]['time'] < CACHE_TTL:
|
||||
return cache[pv_name]['bytes']
|
||||
|
||||
path = f"/mnt/pvs/{pv_name}"
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
|
||||
result = subprocess.run(
|
||||
['ionice', '-c', '3', 'nice', '-n', '19', 'du', '-sb', path],
|
||||
capture_output=True, text=True, timeout=300
|
||||
)
|
||||
|
||||
if result.returncode != 0:
|
||||
return None
|
||||
|
||||
usage = int(result.stdout.split()[0])
|
||||
cache[pv_name] = {'bytes': usage, 'time': time.time()}
|
||||
pv_usage_bytes.labels(pv_name=pv_name).set(usage)
|
||||
pv_last_check.labels(pv_name=pv_name).set(time.time())
|
||||
|
||||
return usage
|
||||
|
||||
@app.route('/usage/<pv_name>')
|
||||
def usage(pv_name):
|
||||
result = get_pv_usage(pv_name)
|
||||
if result is None:
|
||||
return {'error': 'PV not found or timeout'}, 404
|
||||
return {'pv_name': pv_name, 'bytes': result}
|
||||
|
||||
@app.route('/metrics')
|
||||
def metrics():
|
||||
return generate_latest(), 200, {'Content-Type': CONTENT_TYPE_LATEST}
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(host='0.0.0.0', port=8080, threaded=True)
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
apiVersion: v2
|
||||
name: nfs-usage-exporter
|
||||
description: NFS PV usage metrics exporter for Prometheus
|
||||
type: application
|
||||
version: 1.0.0
|
||||
appVersion: "1.0.0"
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ .Chart.Name }}
|
||||
labels:
|
||||
app: {{ .Chart.Name }}
|
||||
spec:
|
||||
replicas: {{ .Values.replicaCount }}
|
||||
selector:
|
||||
matchLabels:
|
||||
app: {{ .Chart.Name }}
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: {{ .Chart.Name }}
|
||||
spec:
|
||||
nodeSelector:
|
||||
{{- toYaml .Values.nodeSelector | nindent 8 }}
|
||||
containers:
|
||||
- name: exporter
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy }}
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
name: metrics
|
||||
volumeMounts:
|
||||
- name: nfs-root
|
||||
mountPath: /mnt/pvs
|
||||
readOnly: true
|
||||
resources:
|
||||
{{- toYaml .Values.resources | nindent 10 }}
|
||||
volumes:
|
||||
- name: nfs-root
|
||||
nfs:
|
||||
server: {{ .Values.nfs.server }}
|
||||
path: {{ .Values.nfs.path }}
|
||||
readOnly: true
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
{{- if .Values.networkPolicy.enabled }}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: NetworkPolicy
|
||||
metadata:
|
||||
name: {{ .Chart.Name }}
|
||||
labels:
|
||||
app: {{ .Chart.Name }}
|
||||
spec:
|
||||
podSelector:
|
||||
matchLabels:
|
||||
app: {{ .Chart.Name }}
|
||||
policyTypes:
|
||||
- Ingress
|
||||
ingress:
|
||||
- from:
|
||||
{{- range .Values.networkPolicy.allowedNamespaces }}
|
||||
- namespaceSelector:
|
||||
matchLabels:
|
||||
kubernetes.io/metadata.name: {{ . }}
|
||||
{{- end }}
|
||||
ports:
|
||||
- protocol: TCP
|
||||
port: 8080
|
||||
{{- end }}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: {{ .Chart.Name }}
|
||||
labels:
|
||||
app: {{ .Chart.Name }}
|
||||
annotations:
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "8080"
|
||||
prometheus.io/path: "/metrics"
|
||||
spec:
|
||||
type: {{ .Values.service.type }}
|
||||
ports:
|
||||
- port: {{ .Values.service.port }}
|
||||
targetPort: 8080
|
||||
protocol: TCP
|
||||
name: metrics
|
||||
selector:
|
||||
app: {{ .Chart.Name }}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
replicaCount: 1
|
||||
|
||||
image:
|
||||
repository: harbor.inje-private.com/infra/nfs-usage-exporter
|
||||
tag: "v1.0.0"
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 8080
|
||||
|
||||
resources:
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 256Mi
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
|
||||
nodeSelector:
|
||||
nodegroup: nd
|
||||
|
||||
nfs:
|
||||
server: "192.168.0.54"
|
||||
path: "/GJ_SHARE_FS6/39197c35-6247-4ece-bc74-d9460a757a04"
|
||||
|
||||
networkPolicy:
|
||||
enabled: true
|
||||
allowedNamespaces:
|
||||
- gpulive
|
||||
- monitoring
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
flask==3.0.0
|
||||
prometheus_client==0.19.0
|
||||
gunicorn==21.2.0
|
||||
Loading…
Reference in New Issue