From f62425843b5b91a879e8d4622eeebcb6a06326b9 Mon Sep 17 00:00:00 2001
From: selee <selee@inje-private.com>
Date: Tue, 10 Feb 2026 15:57:03 +0900
Subject: [PATCH] Update integration config for InfiniBand GPU environment

- Add IB CNI annotation (hostdevice-net), /dev/shm, /dev/infiniband mounts
- Add nvidia.com/hostdev: 2 resource for IB NICs
- Add privileged securityContext with IPC_LOCK for NCCL
- Add NCCL env vars (NCCL_DEBUG, NCCL_IB_DISABLE=0, NCCL_SOCKET_IFNAME)
- Set image to nvcr.io/nvidia/pytorch:24.10-py3
- Set numNodes: 2 for multi-node distributed training

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
---
 configs/volcano-trainjob-integration.yaml | 46 ++++++++++++++++++++---
 1 file changed, 40 insertions(+), 6 deletions(-)

diff --git a/configs/volcano-trainjob-integration.yaml b/configs/volcano-trainjob-integration.yaml
index d863179..416eae4 100644
--- a/configs/volcano-trainjob-integration.yaml
+++ b/configs/volcano-trainjob-integration.yaml
@@ -12,10 +12,11 @@ spec:
   weight: 1
   reclaimable: true
   capability:
-    # TODO: Adjust based on your cluster capacity
+    # 3노드 x GPU 8장 = 24, hostdev 2개 = 6
     cpu: "100"
     memory: "512Gi"
     nvidia.com/gpu: "24"
+    nvidia.com/hostdev: "6"
 
 ---
 # 2. ClusterTrainingRuntime with Volcano gang scheduling + topology-aware scheduling
@@ -30,7 +31,7 @@ spec:
   mlPolicy:
     torch:
       numProcPerNode: 8
-    numNodes: 1
+    numNodes: 2
   # Volcano gang scheduling 활성화 - PodGroup 자동 생성
   podGroupPolicy:
     volcano:
@@ -48,15 +49,47 @@ spec:
         template:
           spec:
             template:
+              metadata:
+                annotations:
+                  # InfiniBand CNI
+                  k8s.v1.cni.cncf.io/networks: hostdevice-net
               spec:
                 containers:
                 - name: node
-                  image: ghcr.io/kubeflow/trainer/torch-runtime:latest
+                  image: nvcr.io/nvidia/pytorch:24.10-py3
+                  securityContext:
+                    privileged: true
+                    capabilities:
+                      add:
+                        - IPC_LOCK
+                  env:
+                    - name: NCCL_DEBUG
+                      value: "INFO"
+                    - name: NCCL_IB_DISABLE
+                      value: "0"
+                    - name: NCCL_SOCKET_IFNAME
+                      value: "net"
                   resources:
                     requests:
                       nvidia.com/gpu: "8"
+                      nvidia.com/hostdev: "2"
                     limits:
                       nvidia.com/gpu: "8"
+                      nvidia.com/hostdev: "2"
+                  volumeMounts:
+                    - name: shared-memory
+                      mountPath: /dev/shm
+                    - name: infiniband
+                      mountPath: /dev/infiniband
+                volumes:
+                  - name: shared-memory
+                    emptyDir:
+                      medium: Memory
+                      sizeLimit: 128Gi
+                  - name: infiniband
+                    hostPath:
+                      path: /dev/infiniband
+                      type: Directory
 
 ---
 # 3. Example TrainJob (설치 테스트용 - 단일노드)
@@ -72,11 +105,12 @@ spec:
     name: torch-distributed-volcano
   trainer:
     # TODO: Replace with your actual training image
-    image: docker.io/your-org/your-training-image:latest
-    numNodes: 1
-    numProcPerNode: "8"
+    image: nvcr.io/nvidia/pytorch:24.10-py3
+    numNodes: 2
     resourcesPerNode:
       requests:
         nvidia.com/gpu: "8"
+        nvidia.com/hostdev: "2"
       limits:
         nvidia.com/gpu: "8"
+        nvidia.com/hostdev: "2"