# Volcano <-> Kubeflow Trainer Integration Configuration Sample
# Reference: https://www.kubeflow.org/docs/components/trainer/gang-scheduling/volcano/

---
# 1. Volcano Queue for training workloads
# Queue는 리소스 할당 단위. capability는 클러스터 실제 용량에 맞게 조정 필요.
apiVersion: scheduling.volcano.sh/v1beta1
kind: Queue
metadata:
  name: training-queue
spec:
  weight: 1
  reclaimable: true
  capability:
    # 3노드 x GPU 8장 = 24, hostdev 2개 = 6
    cpu: "100"
    memory: "512Gi"
    nvidia.com/gpu: "24"
    nvidia.com/hostdev: "6"

---
# 2. ClusterTrainingRuntime with Volcano gang scheduling + topology-aware scheduling
# podGroupPolicy.volcano를 사용하면 PodGroup이 자동 생성됨 (수동 생성 불필요)
apiVersion: trainer.kubeflow.org/v1alpha1
kind: ClusterTrainingRuntime
metadata:
  name: torch-distributed-volcano
  labels:
    trainer.kubeflow.org/framework: torch
spec:
  mlPolicy:
    torch:
      numProcPerNode: 8
    numNodes: 2
  # Volcano gang scheduling 활성화 - PodGroup 자동 생성
  podGroupPolicy:
    volcano:
      networkTopology:
        mode: hard
        highestTierAllowed: 1
  template:
    metadata:
      annotations:
        # Queue 지정 (runtime level)
        scheduling.volcano.sh/queue-name: training-queue
    spec:
      replicatedJobs:
      - name: node
        template:
          spec:
            template:
              metadata:
                annotations:
                  # InfiniBand CNI
                  k8s.v1.cni.cncf.io/networks: hostdevice-net
              spec:
                containers:
                - name: node
                  image: nvcr.io/nvidia/pytorch:24.10-py3
                  securityContext:
                    privileged: true
                    capabilities:
                      add:
                        - IPC_LOCK
                  env:
                    - name: NCCL_DEBUG
                      value: "INFO"
                    - name: NCCL_IB_DISABLE
                      value: "0"
                    - name: NCCL_SOCKET_IFNAME
                      value: "net"
                  resources:
                    requests:
                      nvidia.com/gpu: "8"
                      nvidia.com/hostdev: "2"
                    limits:
                      nvidia.com/gpu: "8"
                      nvidia.com/hostdev: "2"
                  volumeMounts:
                    - name: shared-memory
                      mountPath: /dev/shm
                    - name: infiniband
                      mountPath: /dev/infiniband
                volumes:
                  - name: shared-memory
                    emptyDir:
                      medium: Memory
                      sizeLimit: 128Gi
                  - name: infiniband
                    hostPath:
                      path: /dev/infiniband
                      type: Directory

---
# 3. Example TrainJob (설치 테스트용 - 단일노드)
# podGroupPolicy는 runtime에서 상속되므로 TrainJob에서는 별도 설정 불필요
# 설치 테스트는 numNodes: 1로 단일노드에서 진행. 멀티노드는 검증 후 numNodes를 늘리면 됨.
apiVersion: trainer.kubeflow.org/v1alpha1
kind: TrainJob
metadata:
  name: example-distributed-training
  namespace: default
spec:
  runtimeRef:
    name: torch-distributed-volcano
  trainer:
    # TODO: Replace with your actual training image
    image: nvcr.io/nvidia/pytorch:24.10-py3
    numNodes: 2
    resourcesPerNode:
      requests:
        nvidia.com/gpu: "8"
        nvidia.com/hostdev: "2"
      limits:
        nvidia.com/gpu: "8"
        nvidia.com/hostdev: "2"