108 lines
2.8 KiB
YAML
108 lines
2.8 KiB
YAML
# Volcano <-> Kubeflow Trainer Integration Configuration Sample
|
|
# This file contains examples for integrating Volcano scheduler with Kubeflow TrainJob.
|
|
|
|
---
|
|
# 1. Volcano Queue for training workloads
|
|
apiVersion: scheduling.volcano.sh/v1beta1
|
|
kind: Queue
|
|
metadata:
|
|
name: training-queue
|
|
spec:
|
|
weight: 1
|
|
reclaimable: true
|
|
capability:
|
|
# Adjust based on your cluster capacity
|
|
cpu: "100"
|
|
memory: "512Gi"
|
|
nvidia.com/gpu: "16"
|
|
|
|
---
|
|
# 2. Example TrainJob with Volcano PodGroup scheduling
|
|
# TrainJob that uses Volcano as the scheduler via podGroupPolicy
|
|
apiVersion: trainer.kubeflow.org/v1alpha1
|
|
kind: TrainJob
|
|
metadata:
|
|
name: example-distributed-training
|
|
namespace: default
|
|
spec:
|
|
runtimeRef:
|
|
name: torch-distributed
|
|
trainer:
|
|
image: docker.io/your-org/your-training-image:latest
|
|
numNodes: 2
|
|
numProcPerNode: "auto"
|
|
resourcesPerNode:
|
|
requests:
|
|
nvidia.com/gpu: "4"
|
|
cpu: "16"
|
|
memory: "64Gi"
|
|
limits:
|
|
nvidia.com/gpu: "4"
|
|
cpu: "16"
|
|
memory: "64Gi"
|
|
|
|
---
|
|
# 3. Volcano PodGroup for gang scheduling (auto-created by integration)
|
|
# This is a reference example - Volcano controller auto-creates PodGroups
|
|
# when scheduler annotation is set.
|
|
apiVersion: scheduling.volcano.sh/v1beta1
|
|
kind: PodGroup
|
|
metadata:
|
|
name: example-distributed-training
|
|
namespace: default
|
|
spec:
|
|
# MinMember ensures all pods are scheduled together (gang scheduling)
|
|
minMember: 2
|
|
queue: training-queue
|
|
priorityClassName: training-priority
|
|
minResources:
|
|
cpu: "32"
|
|
memory: "128Gi"
|
|
nvidia.com/gpu: "8"
|
|
|
|
---
|
|
# 4. PriorityClass for training workloads
|
|
apiVersion: scheduling.k8s.io/v1
|
|
kind: PriorityClass
|
|
metadata:
|
|
name: training-priority
|
|
value: 100
|
|
globalDefault: false
|
|
description: "Priority class for distributed training workloads"
|
|
|
|
---
|
|
# 5. ClusterTrainingRuntime with Volcano scheduler annotation
|
|
# Use this runtime to automatically schedule TrainJobs with Volcano
|
|
apiVersion: trainer.kubeflow.org/v1alpha1
|
|
kind: ClusterTrainingRuntime
|
|
metadata:
|
|
name: torch-distributed-volcano
|
|
labels:
|
|
trainer.kubeflow.org/framework: torch
|
|
spec:
|
|
mlPolicy:
|
|
torch:
|
|
numProcPerNode: 1
|
|
numNodes: 1
|
|
template:
|
|
spec:
|
|
replicatedJobs:
|
|
- name: node
|
|
template:
|
|
spec:
|
|
template:
|
|
metadata:
|
|
annotations:
|
|
# Use Volcano as the scheduler
|
|
scheduling.volcano.sh/queue-name: training-queue
|
|
spec:
|
|
schedulerName: volcano
|
|
containers:
|
|
- name: node
|
|
image: ghcr.io/kubeflow/trainer/torch-runtime:latest
|
|
resources:
|
|
requests:
|
|
nvidia.com/gpu: "1"
|
|
limits:
|
|
nvidia.com/gpu: "1"
|