# Volcano <-> Kubeflow Trainer Integration Configuration Sample
# Reference: https://www.kubeflow.org/docs/components/trainer/gang-scheduling/volcano/

---
# 1. Volcano Queue for training workloads
# Queue는 리소스 할당 단위. capability는 클러스터 실제 용량에 맞게 조정 필요.
apiVersion: scheduling.volcano.sh/v1beta1
kind: Queue
metadata:
  name: training-queue
spec:
  weight: 1
  reclaimable: true
  capability:
    # TODO: Adjust based on your cluster capacity
    cpu: "100"
    memory: "512Gi"
    nvidia.com/gpu: "24"

---
# 2. ClusterTrainingRuntime with Volcano gang scheduling + topology-aware scheduling
# podGroupPolicy.volcano를 사용하면 PodGroup이 자동 생성됨 (수동 생성 불필요)
apiVersion: trainer.kubeflow.org/v1alpha1
kind: ClusterTrainingRuntime
metadata:
  name: torch-distributed-volcano
  labels:
    trainer.kubeflow.org/framework: torch
spec:
  mlPolicy:
    torch:
      numProcPerNode: 8
    numNodes: 1
  # Volcano gang scheduling 활성화 - PodGroup 자동 생성
  podGroupPolicy:
    volcano:
      networkTopology:
        mode: hard
        highestTierAllowed: 1
  template:
    metadata:
      annotations:
        # Queue 지정 (runtime level)
        scheduling.volcano.sh/queue-name: training-queue
    spec:
      replicatedJobs:
      - name: node
        template:
          spec:
            template:
              spec:
                containers:
                - name: node
                  image: ghcr.io/kubeflow/trainer/torch-runtime:latest
                  resources:
                    requests:
                      nvidia.com/gpu: "8"
                    limits:
                      nvidia.com/gpu: "8"

---
# 3. Example TrainJob (설치 테스트용 - 단일노드)
# podGroupPolicy는 runtime에서 상속되므로 TrainJob에서는 별도 설정 불필요
# 설치 테스트는 numNodes: 1로 단일노드에서 진행. 멀티노드는 검증 후 numNodes를 늘리면 됨.
apiVersion: trainer.kubeflow.org/v1alpha1
kind: TrainJob
metadata:
  name: example-distributed-training
  namespace: default
spec:
  runtimeRef:
    name: torch-distributed-volcano
  trainer:
    # TODO: Replace with your actual training image
    image: docker.io/your-org/your-training-image:latest
    numNodes: 1
    numProcPerNode: "8"
    resourcesPerNode:
      requests:
        nvidia.com/gpu: "8"
      limits:
        nvidia.com/gpu: "8"