109 lines
2.9 KiB
YAML
109 lines
2.9 KiB
YAML
# Volcano <-> Kubeflow Trainer Integration Configuration Sample
|
|
# Reference: https://www.kubeflow.org/docs/components/trainer/gang-scheduling/volcano/
|
|
|
|
---
|
|
# 1. Volcano Queue for training workloads
|
|
# Queue는 리소스 할당 단위. capability는 클러스터 실제 용량에 맞게 조정 필요.
|
|
apiVersion: scheduling.volcano.sh/v1beta1
|
|
kind: Queue
|
|
metadata:
|
|
name: training-queue
|
|
spec:
|
|
weight: 1
|
|
reclaimable: true
|
|
capability:
|
|
# TODO: Adjust based on your cluster capacity
|
|
cpu: "100"
|
|
memory: "512Gi"
|
|
nvidia.com/gpu: "16"
|
|
|
|
---
|
|
# 3. ClusterTrainingRuntime with Volcano gang scheduling + topology-aware scheduling
|
|
# podGroupPolicy.volcano를 사용하면 PodGroup이 자동 생성됨 (수동 생성 불필요)
|
|
apiVersion: trainer.kubeflow.org/v1alpha1
|
|
kind: ClusterTrainingRuntime
|
|
metadata:
|
|
name: torch-distributed-volcano
|
|
labels:
|
|
trainer.kubeflow.org/framework: torch
|
|
spec:
|
|
mlPolicy:
|
|
torch:
|
|
numProcPerNode: 1
|
|
numNodes: 1
|
|
# Volcano gang scheduling 활성화 - PodGroup 자동 생성
|
|
podGroupPolicy:
|
|
volcano:
|
|
networkTopology:
|
|
mode: hard
|
|
highestTierAllowed: 1
|
|
template:
|
|
metadata:
|
|
annotations:
|
|
# Queue 지정 (runtime level)
|
|
scheduling.volcano.sh/queue-name: training-queue
|
|
spec:
|
|
replicatedJobs:
|
|
- name: node
|
|
template:
|
|
spec:
|
|
template:
|
|
spec:
|
|
containers:
|
|
- name: node
|
|
image: ghcr.io/kubeflow/trainer/torch-runtime:latest
|
|
resources:
|
|
requests:
|
|
nvidia.com/gpu: "1"
|
|
limits:
|
|
nvidia.com/gpu: "1"
|
|
|
|
---
|
|
# 4. Example TrainJob using Volcano-enabled runtime
|
|
# podGroupPolicy는 runtime에서 상속되므로 TrainJob에서는 별도 설정 불필요
|
|
apiVersion: trainer.kubeflow.org/v1alpha1
|
|
kind: TrainJob
|
|
metadata:
|
|
name: example-distributed-training
|
|
namespace: default
|
|
spec:
|
|
runtimeRef:
|
|
name: torch-distributed-volcano
|
|
trainer:
|
|
# TODO: Replace with your actual training image
|
|
image: docker.io/your-org/your-training-image:latest
|
|
numNodes: 2
|
|
numProcPerNode: "auto"
|
|
resourcesPerNode:
|
|
requests:
|
|
nvidia.com/gpu: "4"
|
|
cpu: "16"
|
|
memory: "64Gi"
|
|
limits:
|
|
nvidia.com/gpu: "4"
|
|
cpu: "16"
|
|
memory: "64Gi"
|
|
|
|
---
|
|
# 5. Example TrainJob with queue override at TrainJob level
|
|
# runtime 기본 Queue 대신 다른 Queue를 사용하고 싶을 때
|
|
apiVersion: trainer.kubeflow.org/v1alpha1
|
|
kind: TrainJob
|
|
metadata:
|
|
name: example-high-priority-training
|
|
namespace: default
|
|
spec:
|
|
runtimeRef:
|
|
name: torch-distributed-volcano
|
|
annotations:
|
|
scheduling.volcano.sh/queue-name: "high-priority-queue"
|
|
trainer:
|
|
image: docker.io/your-org/your-training-image:latest
|
|
numNodes: 4
|
|
numProcPerNode: "auto"
|
|
resourcesPerNode:
|
|
requests:
|
|
nvidia.com/gpu: "8"
|
|
limits:
|
|
nvidia.com/gpu: "8"
|