fix: per-metric z-score thresholds, min stddev guard, nodeSelector
CI/CD / build-and-push (push) Successful in 1m27s Details

- Disk/NAS: z-score threshold 2 (was 5, too insensitive for storage)
- CPU/Memory/Ingress/Istio: z-score threshold 5 (unchanged)
- Add MIN_STDDEV=0.5 guard: skip peak detection when data has
  near-zero variance (prevents false peaks on stable disk data)
- Add nodeSelector: nodegroup=nd to deployment
- Apply thresholds in both /api/report and /api/metrics/[type]

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Cloud User 2026-03-17 14:34:19 +09:00
parent 7a228a4123
commit 2b596c4cb4
5 changed files with 34 additions and 9 deletions

View File

@ -20,6 +20,10 @@ spec:
imagePullSecrets: imagePullSecrets:
{{- toYaml . | nindent 8 }} {{- toYaml . | nindent 8 }}
{{- end }} {{- end }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
containers: containers:
- name: {{ .Chart.Name }} - name: {{ .Chart.Name }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"

View File

@ -40,6 +40,9 @@ config:
QUERY_ISTIO_RX: 'sum(rate(container_network_receive_bytes_total{namespace="istio-system", pod=~"istio-ingressgateway.*"}[5m]))' QUERY_ISTIO_RX: 'sum(rate(container_network_receive_bytes_total{namespace="istio-system", pod=~"istio-ingressgateway.*"}[5m]))'
QUERY_ISTIO_TX: 'sum(rate(container_network_transmit_bytes_total{namespace="istio-system", pod=~"istio-ingressgateway.*"}[5m]))' QUERY_ISTIO_TX: 'sum(rate(container_network_transmit_bytes_total{namespace="istio-system", pod=~"istio-ingressgateway.*"}[5m]))'
nodeSelector:
nodegroup: nd
resources: resources:
requests: requests:
cpu: "100m" cpu: "100m"

View File

@ -14,6 +14,15 @@ const NETWORK_MAP: Record<string, { rx: string; tx: string }> = {
istio: { rx: QUERIES.istioReceive, tx: QUERIES.istioTransmit }, istio: { rx: QUERIES.istioReceive, tx: QUERIES.istioTransmit },
}; };
const PEAK_THRESHOLD: Record<string, number> = {
cpu: 5,
memory: 5,
disk: 2,
nas: 2,
ingress: 5,
istio: 5,
};
function parsePrometheusValues(values: [number, string][]): DataPoint[] { function parsePrometheusValues(values: [number, string][]): DataPoint[] {
return values.map(([ts, val]) => ({ timestamp: ts, value: parseFloat(val) })); return values.map(([ts, val]) => ({ timestamp: ts, value: parseFloat(val) }));
} }
@ -38,13 +47,14 @@ export async function GET(
queryRange(netQuery.tx, start, end, step), queryRange(netQuery.tx, start, end, step),
]); ]);
const threshold = PEAK_THRESHOLD[type] || 5;
const process = (results: typeof rxResults) => results.map((r) => { const process = (results: typeof rxResults) => results.map((r) => {
const dataPoints = parsePrometheusValues(r.values); const dataPoints = parsePrometheusValues(r.values);
return { return {
instance: r.metric.node || r.metric.instance || '', instance: r.metric.node || r.metric.instance || '',
dataPoints, dataPoints,
stats: calculateStats(dataPoints), stats: calculateStats(dataPoints),
peaks: detectPeaks(dataPoints), peaks: detectPeaks(dataPoints, threshold),
}; };
}); });
@ -57,6 +67,7 @@ export async function GET(
} }
const results = await queryRange(query, start, end, step); const results = await queryRange(query, start, end, step);
const threshold = PEAK_THRESHOLD[type] || 5;
const instances = results.map((r) => { const instances = results.map((r) => {
const dataPoints = parsePrometheusValues(r.values); const dataPoints = parsePrometheusValues(r.values);
return { return {
@ -64,7 +75,7 @@ export async function GET(
metric: r.metric, metric: r.metric,
dataPoints, dataPoints,
stats: calculateStats(dataPoints), stats: calculateStats(dataPoints),
peaks: detectPeaks(dataPoints), peaks: detectPeaks(dataPoints, threshold),
}; };
}); });

View File

@ -7,7 +7,7 @@ function parseValues(values: [number, string][]): DataPoint[] {
return values.map(([ts, val]) => ({ timestamp: ts, value: parseFloat(val) })); return values.map(([ts, val]) => ({ timestamp: ts, value: parseFloat(val) }));
} }
function processResults(results: { metric: Record<string, string>; values: [number, string][] }[], labelKey?: string) { function processResults(results: { metric: Record<string, string>; values: [number, string][] }[], labelKey?: string, peakThreshold?: number) {
return results.map((r) => { return results.map((r) => {
const dataPoints = parseValues(r.values); const dataPoints = parseValues(r.values);
return { return {
@ -15,7 +15,7 @@ function processResults(results: { metric: Record<string, string>; values: [numb
metric: r.metric, metric: r.metric,
dataPoints, dataPoints,
stats: calculateStats(dataPoints), stats: calculateStats(dataPoints),
peaks: detectPeaks(dataPoints), peaks: detectPeaks(dataPoints, peakThreshold),
}; };
}); });
} }
@ -60,10 +60,10 @@ export async function GET(request: NextRequest) {
period: { start, end, step }, period: { start, end, step },
nodes, nodes,
metrics: { metrics: {
cpu: processResults(cpuData), cpu: processResults(cpuData, undefined, 5),
memory: processResults(memData), memory: processResults(memData, undefined, 5),
disk: processResults(diskData), disk: processResults(diskData, undefined, 2),
nas: processResults(nasData, 'mountpoint'), nas: processResults(nasData, 'mountpoint', 2),
ingress: { ingress: {
receive: processResults(ingressRxData), receive: processResults(ingressRxData),
transmit: processResults(ingressTxData), transmit: processResults(ingressTxData),

View File

@ -31,6 +31,10 @@ export function calculateStats(dataPoints: DataPoint[]): Stats {
}; };
} }
// minStddev: stddev가 이 값 미만이면 데이터 변동이 무의미하므로 peak 감지 스킵
// Disk/NAS 같은 안정적 메트릭에서 미세 변동이 peak으로 잡히는 문제 방지
const MIN_STDDEV = 0.5;
export function detectPeaks( export function detectPeaks(
dataPoints: DataPoint[], dataPoints: DataPoint[],
threshold: number = 5 threshold: number = 5
@ -41,8 +45,11 @@ export function detectPeaks(
values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / values.length values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / values.length
); );
// stddev가 너무 작으면 데이터가 거의 일정한 것이므로 peak 없음
const effectiveStddev = stddev < MIN_STDDEV ? 0 : stddev;
const points = dataPoints.map((d) => { const points = dataPoints.map((d) => {
const zScore = stddev === 0 ? 0 : (d.value - mean) / stddev; const zScore = effectiveStddev === 0 ? 0 : (d.value - mean) / effectiveStddev;
return { ...d, zScore, isPeak: zScore > threshold }; return { ...d, zScore, isPeak: zScore > threshold };
}); });