fix: per-metric z-score thresholds, min stddev guard, nodeSelector
CI/CD / build-and-push (push) Successful in 1m27s
Details
CI/CD / build-and-push (push) Successful in 1m27s
Details
- Disk/NAS: z-score threshold 2 (was 5, too insensitive for storage) - CPU/Memory/Ingress/Istio: z-score threshold 5 (unchanged) - Add MIN_STDDEV=0.5 guard: skip peak detection when data has near-zero variance (prevents false peaks on stable disk data) - Add nodeSelector: nodegroup=nd to deployment - Apply thresholds in both /api/report and /api/metrics/[type] Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
7a228a4123
commit
2b596c4cb4
|
|
@ -20,6 +20,10 @@ spec:
|
|||
imagePullSecrets:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
{{- with .Values.nodeSelector }}
|
||||
nodeSelector:
|
||||
{{- toYaml . | nindent 8 }}
|
||||
{{- end }}
|
||||
containers:
|
||||
- name: {{ .Chart.Name }}
|
||||
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
|
||||
|
|
|
|||
|
|
@ -40,6 +40,9 @@ config:
|
|||
QUERY_ISTIO_RX: 'sum(rate(container_network_receive_bytes_total{namespace="istio-system", pod=~"istio-ingressgateway.*"}[5m]))'
|
||||
QUERY_ISTIO_TX: 'sum(rate(container_network_transmit_bytes_total{namespace="istio-system", pod=~"istio-ingressgateway.*"}[5m]))'
|
||||
|
||||
nodeSelector:
|
||||
nodegroup: nd
|
||||
|
||||
resources:
|
||||
requests:
|
||||
cpu: "100m"
|
||||
|
|
|
|||
|
|
@ -14,6 +14,15 @@ const NETWORK_MAP: Record<string, { rx: string; tx: string }> = {
|
|||
istio: { rx: QUERIES.istioReceive, tx: QUERIES.istioTransmit },
|
||||
};
|
||||
|
||||
const PEAK_THRESHOLD: Record<string, number> = {
|
||||
cpu: 5,
|
||||
memory: 5,
|
||||
disk: 2,
|
||||
nas: 2,
|
||||
ingress: 5,
|
||||
istio: 5,
|
||||
};
|
||||
|
||||
function parsePrometheusValues(values: [number, string][]): DataPoint[] {
|
||||
return values.map(([ts, val]) => ({ timestamp: ts, value: parseFloat(val) }));
|
||||
}
|
||||
|
|
@ -38,13 +47,14 @@ export async function GET(
|
|||
queryRange(netQuery.tx, start, end, step),
|
||||
]);
|
||||
|
||||
const threshold = PEAK_THRESHOLD[type] || 5;
|
||||
const process = (results: typeof rxResults) => results.map((r) => {
|
||||
const dataPoints = parsePrometheusValues(r.values);
|
||||
return {
|
||||
instance: r.metric.node || r.metric.instance || '',
|
||||
dataPoints,
|
||||
stats: calculateStats(dataPoints),
|
||||
peaks: detectPeaks(dataPoints),
|
||||
peaks: detectPeaks(dataPoints, threshold),
|
||||
};
|
||||
});
|
||||
|
||||
|
|
@ -57,6 +67,7 @@ export async function GET(
|
|||
}
|
||||
|
||||
const results = await queryRange(query, start, end, step);
|
||||
const threshold = PEAK_THRESHOLD[type] || 5;
|
||||
const instances = results.map((r) => {
|
||||
const dataPoints = parsePrometheusValues(r.values);
|
||||
return {
|
||||
|
|
@ -64,7 +75,7 @@ export async function GET(
|
|||
metric: r.metric,
|
||||
dataPoints,
|
||||
stats: calculateStats(dataPoints),
|
||||
peaks: detectPeaks(dataPoints),
|
||||
peaks: detectPeaks(dataPoints, threshold),
|
||||
};
|
||||
});
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ function parseValues(values: [number, string][]): DataPoint[] {
|
|||
return values.map(([ts, val]) => ({ timestamp: ts, value: parseFloat(val) }));
|
||||
}
|
||||
|
||||
function processResults(results: { metric: Record<string, string>; values: [number, string][] }[], labelKey?: string) {
|
||||
function processResults(results: { metric: Record<string, string>; values: [number, string][] }[], labelKey?: string, peakThreshold?: number) {
|
||||
return results.map((r) => {
|
||||
const dataPoints = parseValues(r.values);
|
||||
return {
|
||||
|
|
@ -15,7 +15,7 @@ function processResults(results: { metric: Record<string, string>; values: [numb
|
|||
metric: r.metric,
|
||||
dataPoints,
|
||||
stats: calculateStats(dataPoints),
|
||||
peaks: detectPeaks(dataPoints),
|
||||
peaks: detectPeaks(dataPoints, peakThreshold),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
|
@ -60,10 +60,10 @@ export async function GET(request: NextRequest) {
|
|||
period: { start, end, step },
|
||||
nodes,
|
||||
metrics: {
|
||||
cpu: processResults(cpuData),
|
||||
memory: processResults(memData),
|
||||
disk: processResults(diskData),
|
||||
nas: processResults(nasData, 'mountpoint'),
|
||||
cpu: processResults(cpuData, undefined, 5),
|
||||
memory: processResults(memData, undefined, 5),
|
||||
disk: processResults(diskData, undefined, 2),
|
||||
nas: processResults(nasData, 'mountpoint', 2),
|
||||
ingress: {
|
||||
receive: processResults(ingressRxData),
|
||||
transmit: processResults(ingressTxData),
|
||||
|
|
|
|||
|
|
@ -31,6 +31,10 @@ export function calculateStats(dataPoints: DataPoint[]): Stats {
|
|||
};
|
||||
}
|
||||
|
||||
// minStddev: stddev가 이 값 미만이면 데이터 변동이 무의미하므로 peak 감지 스킵
|
||||
// Disk/NAS 같은 안정적 메트릭에서 미세 변동이 peak으로 잡히는 문제 방지
|
||||
const MIN_STDDEV = 0.5;
|
||||
|
||||
export function detectPeaks(
|
||||
dataPoints: DataPoint[],
|
||||
threshold: number = 5
|
||||
|
|
@ -41,8 +45,11 @@ export function detectPeaks(
|
|||
values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / values.length
|
||||
);
|
||||
|
||||
// stddev가 너무 작으면 데이터가 거의 일정한 것이므로 peak 없음
|
||||
const effectiveStddev = stddev < MIN_STDDEV ? 0 : stddev;
|
||||
|
||||
const points = dataPoints.map((d) => {
|
||||
const zScore = stddev === 0 ? 0 : (d.value - mean) / stddev;
|
||||
const zScore = effectiveStddev === 0 ? 0 : (d.value - mean) / effectiveStddev;
|
||||
return { ...d, zScore, isPeak: zScore > threshold };
|
||||
});
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue