# Volcano <-> Kubeflow Trainer Integration Configuration Sample
# Reference: https://www.kubeflow.org/docs/components/trainer/gang-scheduling/volcano/

---
# 1. Volcano Queue for training workloads
# Queue는 리소스 할당 단위. capability는 클러스터 실제 용량에 맞게 조정 필요.
apiVersion: scheduling.volcano.sh/v1beta1
kind: Queue
metadata:
  name: training-queue
spec:
  weight: 1
  reclaimable: true
  capability:
    # TODO: Adjust based on your cluster capacity
    cpu: "100"
    memory: "512Gi"
    nvidia.com/gpu: "16"

---
# 3. ClusterTrainingRuntime with Volcano gang scheduling + topology-aware scheduling
# podGroupPolicy.volcano를 사용하면 PodGroup이 자동 생성됨 (수동 생성 불필요)
apiVersion: trainer.kubeflow.org/v1alpha1
kind: ClusterTrainingRuntime
metadata:
  name: torch-distributed-volcano
  labels:
    trainer.kubeflow.org/framework: torch
spec:
  mlPolicy:
    torch:
      numProcPerNode: 1
    numNodes: 1
  # Volcano gang scheduling 활성화 - PodGroup 자동 생성
  podGroupPolicy:
    volcano:
      networkTopology:
        mode: hard
        highestTierAllowed: 1
  template:
    metadata:
      annotations:
        # Queue 지정 (runtime level)
        scheduling.volcano.sh/queue-name: training-queue
    spec:
      replicatedJobs:
      - name: node
        template:
          spec:
            template:
              spec:
                containers:
                - name: node
                  image: ghcr.io/kubeflow/trainer/torch-runtime:latest
                  resources:
                    requests:
                      nvidia.com/gpu: "1"
                    limits:
                      nvidia.com/gpu: "1"

---
# 4. Example TrainJob using Volcano-enabled runtime
# podGroupPolicy는 runtime에서 상속되므로 TrainJob에서는 별도 설정 불필요
apiVersion: trainer.kubeflow.org/v1alpha1
kind: TrainJob
metadata:
  name: example-distributed-training
  namespace: default
spec:
  runtimeRef:
    name: torch-distributed-volcano
  trainer:
    # TODO: Replace with your actual training image
    image: docker.io/your-org/your-training-image:latest
    numNodes: 2
    numProcPerNode: "auto"
    resourcesPerNode:
      requests:
        nvidia.com/gpu: "4"
        cpu: "16"
        memory: "64Gi"
      limits:
        nvidia.com/gpu: "4"
        cpu: "16"
        memory: "64Gi"

---
# 5. Example TrainJob with queue override at TrainJob level
# runtime 기본 Queue 대신 다른 Queue를 사용하고 싶을 때
apiVersion: trainer.kubeflow.org/v1alpha1
kind: TrainJob
metadata:
  name: example-high-priority-training
  namespace: default
spec:
  runtimeRef:
    name: torch-distributed-volcano
  annotations:
    scheduling.volcano.sh/queue-name: "high-priority-queue"
  trainer:
    image: docker.io/your-org/your-training-image:latest
    numNodes: 4
    numProcPerNode: "auto"
    resourcesPerNode:
      requests:
        nvidia.com/gpu: "8"
      limits:
        nvidia.com/gpu: "8"