From dafa90e6d012f9b16c9104ea408b7e28e0becdde Mon Sep 17 00:00:00 2001 From: selee Date: Mon, 9 Feb 2026 16:15:24 +0900 Subject: [PATCH] Update README and integration config to match official docs - README: reflect podGroupPolicy.volcano usage (not manual schedulerName) - README: remove PriorityClass section (not needed for now, FIFO is default) - README: add component placement summary table (all on nodegroup: nd) - README: add nodegroup: nd prerequisite and verification commands - README: update troubleshooting for podGroupPolicy-based integration - Integration config: remove PriorityClass, renumber sections Co-Authored-By: Claude Opus 4.6 --- README.md | 120 +++++++++++++------- configs/volcano-trainjob-integration.yaml | 131 +++++++++++----------- 2 files changed, 145 insertions(+), 106 deletions(-) diff --git a/README.md b/README.md index 042b246..3e3e1da 100644 --- a/README.md +++ b/README.md @@ -41,13 +41,15 @@ kubeflow_trainer_with_volcano/ - `kubectl` 클러스터 접근 설정 완료 - GPU 노드에 NVIDIA device plugin 설치 완료 - 클러스터에 기존 Volcano/Kubeflow CRD가 없는 상태 +- 컨트롤 플레인 컴포넌트 배치용 노드에 `nodegroup: nd` 라벨 설정 완료 ```bash # 사전 확인 kubectl version helm version kubectl get nodes -nvidia-smi # GPU 노드에서 확인 +kubectl get nodes -l nodegroup=nd # 컨트롤 컴포넌트 배치 노드 확인 +nvidia-smi # GPU 노드에서 확인 ``` --- @@ -64,6 +66,8 @@ kubectl create namespace volcano-system ### 2.2 Helm으로 Volcano 설치 (로컬 chart 사용) +모든 Volcano 컴포넌트는 `nodegroup: nd` 노드에 배치됩니다 (`configs/volcano-values.yaml`의 `default_ns` 설정). + ```bash helm install volcano charts/volcano/volcano-1.14.0.tgz \ -n volcano-system \ @@ -71,11 +75,16 @@ helm install volcano charts/volcano/volcano-1.14.0.tgz \ --wait --timeout 5m ``` +주요 커스텀 설정 (`configs/volcano-values.yaml`): +- **scheduler_config_override**: gang scheduling + binpack (GPU 노드 집약 배치) 활성화 +- **default_ns**: `nodegroup: nd` — Volcano 컴포넌트를 non-GPU 노드에 배치 +- **API rate limits**: 대규모 클러스터용 QPS/Burst 설정 + ### 2.3 설치 확인 ```bash -# Pod 상태 확인 -kubectl get pods -n volcano-system +# Pod 상태 확인 (모두 nodegroup=nd 노드에 배치되어야 함) +kubectl get pods -n volcano-system -o wide # 기대 출력: # volcano-admission-xxxxx 1/1 Running @@ -90,9 +99,6 @@ kubectl get crd | grep volcano # jobs.batch.volcano.sh # podgroups.scheduling.volcano.sh # queues.scheduling.volcano.sh - -# 기본 Queue 확인 -kubectl get queue -n volcano-system ``` ### 2.4 Volcano 삭제 (필요시) @@ -116,6 +122,8 @@ kubectl create namespace kubeflow-trainer ### 3.2 Helm으로 Kubeflow Trainer 설치 (로컬 chart 사용) +Trainer controller와 JobSet controller 모두 `nodegroup: nd` 노드에 배치됩니다 (`configs/kubeflow-trainer-values.yaml` 설정). + ```bash helm install kubeflow-trainer charts/kubeflow-trainer-0.0.0-sha-48e7a93.tgz \ -n kubeflow-trainer \ @@ -123,6 +131,11 @@ helm install kubeflow-trainer charts/kubeflow-trainer-0.0.0-sha-48e7a93.tgz \ --wait --timeout 5m ``` +주요 커스텀 설정 (`configs/kubeflow-trainer-values.yaml`): +- **manager.nodeSelector**: `nodegroup: nd` — Trainer controller를 non-GPU 노드에 배치 +- **jobset.controller.nodeSelector**: `nodegroup: nd` — JobSet controller를 non-GPU 노드에 배치 +- **jobset.install**: `true` — JobSet을 함께 설치 (이미 설치되어 있으면 `false`로 변경) + ### 3.3 ClusterTrainingRuntime 설치 ```bash @@ -132,11 +145,12 @@ kubectl apply -f manifests/kubeflow-trainer/runtimes/runtimes.yaml ### 3.4 설치 확인 ```bash -# Trainer controller 확인 -kubectl get pods -n kubeflow-trainer +# Trainer controller 확인 (nodegroup=nd 노드에 배치되어야 함) +kubectl get pods -n kubeflow-trainer -o wide # 기대 출력: # kubeflow-trainer-controller-manager-xxxxx 1/1 Running +# jobset-controller-manager-xxxxx 1/1 Running # CRD 확인 kubectl get crd | grep trainer @@ -167,10 +181,15 @@ kubectl delete namespace kubeflow-trainer ## Task 4: Volcano ↔ Kubeflow Trainer 연동 설정 -Volcano 스케줄러를 Kubeflow TrainJob과 연동하여 gang scheduling, queue 기반 리소스 관리를 활성화합니다. +Volcano 스케줄러를 Kubeflow TrainJob과 연동하여 gang scheduling, queue 기반 리소스 관리, topology-aware scheduling을 활성화합니다. + +> Reference: https://www.kubeflow.org/docs/components/trainer/gang-scheduling/volcano/ ### 4.1 Training Queue 생성 +Queue는 논리적인 리소스 풀입니다. 이 Queue에 속한 모든 TrainJob이 합산해서 사용할 수 있는 리소스 상한을 정합니다. +`capability`는 실제 클러스터 GPU/CPU/Memory 용량에 맞게 조정해야 합니다. + ```bash kubectl apply -f - < Kubeflow Trainer Integration Configuration Sample -# This file contains examples for integrating Volcano scheduler with Kubeflow TrainJob. +# Reference: https://www.kubeflow.org/docs/components/trainer/gang-scheduling/volcano/ --- # 1. Volcano Queue for training workloads +# Queue는 리소스 할당 단위. capability는 클러스터 실제 용량에 맞게 조정 필요. apiVersion: scheduling.volcano.sh/v1beta1 kind: Queue metadata: @@ -11,14 +12,55 @@ spec: weight: 1 reclaimable: true capability: - # Adjust based on your cluster capacity + # TODO: Adjust based on your cluster capacity cpu: "100" memory: "512Gi" nvidia.com/gpu: "16" --- -# 2. Example TrainJob with Volcano PodGroup scheduling -# TrainJob that uses Volcano as the scheduler via podGroupPolicy +# 3. ClusterTrainingRuntime with Volcano gang scheduling + topology-aware scheduling +# podGroupPolicy.volcano를 사용하면 PodGroup이 자동 생성됨 (수동 생성 불필요) +apiVersion: trainer.kubeflow.org/v1alpha1 +kind: ClusterTrainingRuntime +metadata: + name: torch-distributed-volcano + labels: + trainer.kubeflow.org/framework: torch +spec: + mlPolicy: + torch: + numProcPerNode: 1 + numNodes: 1 + # Volcano gang scheduling 활성화 - PodGroup 자동 생성 + podGroupPolicy: + volcano: + networkTopology: + mode: hard + highestTierAllowed: 1 + template: + metadata: + annotations: + # Queue 지정 (runtime level) + scheduling.volcano.sh/queue-name: training-queue + spec: + replicatedJobs: + - name: node + template: + spec: + template: + spec: + containers: + - name: node + image: ghcr.io/kubeflow/trainer/torch-runtime:latest + resources: + requests: + nvidia.com/gpu: "1" + limits: + nvidia.com/gpu: "1" + +--- +# 4. Example TrainJob using Volcano-enabled runtime +# podGroupPolicy는 runtime에서 상속되므로 TrainJob에서는 별도 설정 불필요 apiVersion: trainer.kubeflow.org/v1alpha1 kind: TrainJob metadata: @@ -26,8 +68,9 @@ metadata: namespace: default spec: runtimeRef: - name: torch-distributed + name: torch-distributed-volcano trainer: + # TODO: Replace with your actual training image image: docker.io/your-org/your-training-image:latest numNodes: 2 numProcPerNode: "auto" @@ -42,66 +85,24 @@ spec: memory: "64Gi" --- -# 3. Volcano PodGroup for gang scheduling (auto-created by integration) -# This is a reference example - Volcano controller auto-creates PodGroups -# when scheduler annotation is set. -apiVersion: scheduling.volcano.sh/v1beta1 -kind: PodGroup +# 5. Example TrainJob with queue override at TrainJob level +# runtime 기본 Queue 대신 다른 Queue를 사용하고 싶을 때 +apiVersion: trainer.kubeflow.org/v1alpha1 +kind: TrainJob metadata: - name: example-distributed-training + name: example-high-priority-training namespace: default spec: - # MinMember ensures all pods are scheduled together (gang scheduling) - minMember: 2 - queue: training-queue - priorityClassName: training-priority - minResources: - cpu: "32" - memory: "128Gi" - nvidia.com/gpu: "8" - ---- -# 4. PriorityClass for training workloads -apiVersion: scheduling.k8s.io/v1 -kind: PriorityClass -metadata: - name: training-priority -value: 100 -globalDefault: false -description: "Priority class for distributed training workloads" - ---- -# 5. ClusterTrainingRuntime with Volcano scheduler annotation -# Use this runtime to automatically schedule TrainJobs with Volcano -apiVersion: trainer.kubeflow.org/v1alpha1 -kind: ClusterTrainingRuntime -metadata: - name: torch-distributed-volcano - labels: - trainer.kubeflow.org/framework: torch -spec: - mlPolicy: - torch: - numProcPerNode: 1 - numNodes: 1 - template: - spec: - replicatedJobs: - - name: node - template: - spec: - template: - metadata: - annotations: - # Use Volcano as the scheduler - scheduling.volcano.sh/queue-name: training-queue - spec: - schedulerName: volcano - containers: - - name: node - image: ghcr.io/kubeflow/trainer/torch-runtime:latest - resources: - requests: - nvidia.com/gpu: "1" - limits: - nvidia.com/gpu: "1" + runtimeRef: + name: torch-distributed-volcano + annotations: + scheduling.volcano.sh/queue-name: "high-priority-queue" + trainer: + image: docker.io/your-org/your-training-image:latest + numNodes: 4 + numProcPerNode: "auto" + resourcesPerNode: + requests: + nvidia.com/gpu: "8" + limits: + nvidia.com/gpu: "8"