42 lines
1.6 KiB
Bash
Executable File
42 lines
1.6 KiB
Bash
Executable File
#!/bin/bash
|
|
# gpu-4 노드에서 root로 실행 — 노드 레벨 GPU 관찰
|
|
echo "=========================================="
|
|
echo "노드 레벨 GPU 관찰: $(date)"
|
|
echo "=========================================="
|
|
echo ""
|
|
echo "[1] nvidia-smi 전체"
|
|
nvidia-smi
|
|
echo ""
|
|
echo "[2] GPU 프로세스 목록"
|
|
nvidia-smi --query-compute-apps=pid,used_gpu_memory,name --format=csv
|
|
echo ""
|
|
echo "[3] /dev/nvidia* 사용 프로세스 (fuser)"
|
|
for dev in /dev/nvidia[0-9]* /dev/nvidia-uvm /dev/nvidiactl; do
|
|
[ -e "$dev" ] || continue
|
|
PIDS=$(fuser "$dev" 2>/dev/null)
|
|
if [ -n "$PIDS" ]; then
|
|
echo " $dev →$PIDS"
|
|
fi
|
|
done
|
|
echo ""
|
|
echo "[4] nvidia-smi 미등록 but /dev/nvidia fd 보유 프로세스"
|
|
NVIDIA_PIDS=$(nvidia-smi --query-compute-apps=pid --format=csv,noheader,nounits 2>/dev/null | sort -u)
|
|
for dev in /dev/nvidia[0-9]* /dev/nvidia-uvm /dev/nvidiactl; do
|
|
[ -e "$dev" ] || continue
|
|
for pid in $(fuser "$dev" 2>/dev/null); do
|
|
if ! echo "$NVIDIA_PIDS" | grep -q "^${pid}$"; then
|
|
CMD=$(cat /proc/$pid/cmdline 2>/dev/null | tr '\0' ' ' | head -c 60)
|
|
echo " ⚠️ PID $pid ($dev): nvidia-smi 미등록, CMD: $CMD"
|
|
fi
|
|
done
|
|
done
|
|
echo ""
|
|
echo "[5] Ghost context 체크 (프로세스 0 but 메모리 점유)"
|
|
PROC_COUNT=$(nvidia-smi --query-compute-apps=pid --format=csv,noheader,nounits 2>/dev/null | grep -v '^$' | wc -l)
|
|
nvidia-smi --query-gpu=index,memory.used --format=csv,noheader,nounits 2>/dev/null | while IFS=', ' read -r idx mem; do
|
|
if [ "$PROC_COUNT" -eq 0 ] && [ "$mem" -gt 100 ]; then
|
|
echo " ⚠️ GPU $idx: 프로세스 0개인데 ${mem}MiB 점유 → GHOST CONTEXT"
|
|
fi
|
|
done
|
|
echo "=========================================="
|