From 2b596c4cb46ca682fc4578a5a65c1d959b13d1ae Mon Sep 17 00:00:00 2001 From: Cloud User Date: Tue, 17 Mar 2026 14:34:19 +0900 Subject: [PATCH] fix: per-metric z-score thresholds, min stddev guard, nodeSelector - Disk/NAS: z-score threshold 2 (was 5, too insensitive for storage) - CPU/Memory/Ingress/Istio: z-score threshold 5 (unchanged) - Add MIN_STDDEV=0.5 guard: skip peak detection when data has near-zero variance (prevents false peaks on stable disk data) - Add nodeSelector: nodegroup=nd to deployment - Apply thresholds in both /api/report and /api/metrics/[type] Co-Authored-By: Claude Opus 4.6 --- charts/infra-report/templates/deployment.yaml | 4 ++++ charts/infra-report/values.yaml | 3 +++ src/app/api/metrics/[type]/route.ts | 15 +++++++++++++-- src/app/api/report/route.ts | 12 ++++++------ src/lib/analytics.ts | 9 ++++++++- 5 files changed, 34 insertions(+), 9 deletions(-) diff --git a/charts/infra-report/templates/deployment.yaml b/charts/infra-report/templates/deployment.yaml index eaa0e82..9ee54a1 100644 --- a/charts/infra-report/templates/deployment.yaml +++ b/charts/infra-report/templates/deployment.yaml @@ -20,6 +20,10 @@ spec: imagePullSecrets: {{- toYaml . | nindent 8 }} {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: {{ .Chart.Name }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" diff --git a/charts/infra-report/values.yaml b/charts/infra-report/values.yaml index efc885a..6324dad 100644 --- a/charts/infra-report/values.yaml +++ b/charts/infra-report/values.yaml @@ -40,6 +40,9 @@ config: QUERY_ISTIO_RX: 'sum(rate(container_network_receive_bytes_total{namespace="istio-system", pod=~"istio-ingressgateway.*"}[5m]))' QUERY_ISTIO_TX: 'sum(rate(container_network_transmit_bytes_total{namespace="istio-system", pod=~"istio-ingressgateway.*"}[5m]))' +nodeSelector: + nodegroup: nd + resources: requests: cpu: "100m" diff --git a/src/app/api/metrics/[type]/route.ts b/src/app/api/metrics/[type]/route.ts index fa38983..6ab7e8e 100644 --- a/src/app/api/metrics/[type]/route.ts +++ b/src/app/api/metrics/[type]/route.ts @@ -14,6 +14,15 @@ const NETWORK_MAP: Record = { istio: { rx: QUERIES.istioReceive, tx: QUERIES.istioTransmit }, }; +const PEAK_THRESHOLD: Record = { + cpu: 5, + memory: 5, + disk: 2, + nas: 2, + ingress: 5, + istio: 5, +}; + function parsePrometheusValues(values: [number, string][]): DataPoint[] { return values.map(([ts, val]) => ({ timestamp: ts, value: parseFloat(val) })); } @@ -38,13 +47,14 @@ export async function GET( queryRange(netQuery.tx, start, end, step), ]); + const threshold = PEAK_THRESHOLD[type] || 5; const process = (results: typeof rxResults) => results.map((r) => { const dataPoints = parsePrometheusValues(r.values); return { instance: r.metric.node || r.metric.instance || '', dataPoints, stats: calculateStats(dataPoints), - peaks: detectPeaks(dataPoints), + peaks: detectPeaks(dataPoints, threshold), }; }); @@ -57,6 +67,7 @@ export async function GET( } const results = await queryRange(query, start, end, step); + const threshold = PEAK_THRESHOLD[type] || 5; const instances = results.map((r) => { const dataPoints = parsePrometheusValues(r.values); return { @@ -64,7 +75,7 @@ export async function GET( metric: r.metric, dataPoints, stats: calculateStats(dataPoints), - peaks: detectPeaks(dataPoints), + peaks: detectPeaks(dataPoints, threshold), }; }); diff --git a/src/app/api/report/route.ts b/src/app/api/report/route.ts index d319ae4..915ddd3 100644 --- a/src/app/api/report/route.ts +++ b/src/app/api/report/route.ts @@ -7,7 +7,7 @@ function parseValues(values: [number, string][]): DataPoint[] { return values.map(([ts, val]) => ({ timestamp: ts, value: parseFloat(val) })); } -function processResults(results: { metric: Record; values: [number, string][] }[], labelKey?: string) { +function processResults(results: { metric: Record; values: [number, string][] }[], labelKey?: string, peakThreshold?: number) { return results.map((r) => { const dataPoints = parseValues(r.values); return { @@ -15,7 +15,7 @@ function processResults(results: { metric: Record; values: [numb metric: r.metric, dataPoints, stats: calculateStats(dataPoints), - peaks: detectPeaks(dataPoints), + peaks: detectPeaks(dataPoints, peakThreshold), }; }); } @@ -60,10 +60,10 @@ export async function GET(request: NextRequest) { period: { start, end, step }, nodes, metrics: { - cpu: processResults(cpuData), - memory: processResults(memData), - disk: processResults(diskData), - nas: processResults(nasData, 'mountpoint'), + cpu: processResults(cpuData, undefined, 5), + memory: processResults(memData, undefined, 5), + disk: processResults(diskData, undefined, 2), + nas: processResults(nasData, 'mountpoint', 2), ingress: { receive: processResults(ingressRxData), transmit: processResults(ingressTxData), diff --git a/src/lib/analytics.ts b/src/lib/analytics.ts index 0130871..4099b51 100644 --- a/src/lib/analytics.ts +++ b/src/lib/analytics.ts @@ -31,6 +31,10 @@ export function calculateStats(dataPoints: DataPoint[]): Stats { }; } +// minStddev: stddev가 이 값 미만이면 데이터 변동이 무의미하므로 peak 감지 스킵 +// Disk/NAS 같은 안정적 메트릭에서 미세 변동이 peak으로 잡히는 문제 방지 +const MIN_STDDEV = 0.5; + export function detectPeaks( dataPoints: DataPoint[], threshold: number = 5 @@ -41,8 +45,11 @@ export function detectPeaks( values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / values.length ); + // stddev가 너무 작으면 데이터가 거의 일정한 것이므로 peak 없음 + const effectiveStddev = stddev < MIN_STDDEV ? 0 : stddev; + const points = dataPoints.map((d) => { - const zScore = stddev === 0 ? 0 : (d.value - mean) / stddev; + const zScore = effectiveStddev === 0 ? 0 : (d.value - mean) / effectiveStddev; return { ...d, zScore, isPeak: zScore > threshold }; });