117 lines
3.6 KiB
YAML
117 lines
3.6 KiB
YAML
# Volcano <-> Kubeflow Trainer Integration Configuration Sample
|
|
# Reference: https://www.kubeflow.org/docs/components/trainer/gang-scheduling/volcano/
|
|
|
|
---
|
|
# 1. Volcano Queue for training workloads
|
|
# Queue는 리소스 할당 단위. capability는 클러스터 실제 용량에 맞게 조정 필요.
|
|
apiVersion: scheduling.volcano.sh/v1beta1
|
|
kind: Queue
|
|
metadata:
|
|
name: training-queue
|
|
spec:
|
|
weight: 1
|
|
reclaimable: true
|
|
capability:
|
|
# 3노드 x GPU 8장 = 24, hostdev 2개 = 6
|
|
cpu: "100"
|
|
memory: "512Gi"
|
|
nvidia.com/gpu: "24"
|
|
nvidia.com/hostdev: "6"
|
|
|
|
---
|
|
# 2. ClusterTrainingRuntime with Volcano gang scheduling + topology-aware scheduling
|
|
# podGroupPolicy.volcano를 사용하면 PodGroup이 자동 생성됨 (수동 생성 불필요)
|
|
apiVersion: trainer.kubeflow.org/v1alpha1
|
|
kind: ClusterTrainingRuntime
|
|
metadata:
|
|
name: torch-distributed-volcano
|
|
labels:
|
|
trainer.kubeflow.org/framework: torch
|
|
spec:
|
|
mlPolicy:
|
|
torch:
|
|
numProcPerNode: 8
|
|
numNodes: 2
|
|
# Volcano gang scheduling 활성화 - PodGroup 자동 생성
|
|
podGroupPolicy:
|
|
volcano:
|
|
networkTopology:
|
|
mode: hard
|
|
highestTierAllowed: 1
|
|
template:
|
|
metadata:
|
|
annotations:
|
|
# Queue 지정 (runtime level)
|
|
scheduling.volcano.sh/queue-name: training-queue
|
|
spec:
|
|
replicatedJobs:
|
|
- name: node
|
|
template:
|
|
spec:
|
|
template:
|
|
metadata:
|
|
annotations:
|
|
# InfiniBand CNI
|
|
k8s.v1.cni.cncf.io/networks: hostdevice-net
|
|
spec:
|
|
containers:
|
|
- name: node
|
|
image: nvcr.io/nvidia/pytorch:24.10-py3
|
|
securityContext:
|
|
privileged: true
|
|
capabilities:
|
|
add:
|
|
- IPC_LOCK
|
|
env:
|
|
- name: NCCL_DEBUG
|
|
value: "INFO"
|
|
- name: NCCL_IB_DISABLE
|
|
value: "0"
|
|
- name: NCCL_SOCKET_IFNAME
|
|
value: "net"
|
|
resources:
|
|
requests:
|
|
nvidia.com/gpu: "8"
|
|
nvidia.com/hostdev: "2"
|
|
limits:
|
|
nvidia.com/gpu: "8"
|
|
nvidia.com/hostdev: "2"
|
|
volumeMounts:
|
|
- name: shared-memory
|
|
mountPath: /dev/shm
|
|
- name: infiniband
|
|
mountPath: /dev/infiniband
|
|
volumes:
|
|
- name: shared-memory
|
|
emptyDir:
|
|
medium: Memory
|
|
sizeLimit: 128Gi
|
|
- name: infiniband
|
|
hostPath:
|
|
path: /dev/infiniband
|
|
type: Directory
|
|
|
|
---
|
|
# 3. Example TrainJob (설치 테스트용 - 단일노드)
|
|
# podGroupPolicy는 runtime에서 상속되므로 TrainJob에서는 별도 설정 불필요
|
|
# 설치 테스트는 numNodes: 1로 단일노드에서 진행. 멀티노드는 검증 후 numNodes를 늘리면 됨.
|
|
apiVersion: trainer.kubeflow.org/v1alpha1
|
|
kind: TrainJob
|
|
metadata:
|
|
name: example-distributed-training
|
|
namespace: default
|
|
spec:
|
|
runtimeRef:
|
|
name: torch-distributed-volcano
|
|
trainer:
|
|
# TODO: Replace with your actual training image
|
|
image: nvcr.io/nvidia/pytorch:24.10-py3
|
|
numNodes: 2
|
|
resourcesPerNode:
|
|
requests:
|
|
nvidia.com/gpu: "8"
|
|
nvidia.com/hostdev: "2"
|
|
limits:
|
|
nvidia.com/gpu: "8"
|
|
nvidia.com/hostdev: "2"
|