From 2b1ad10c2c40d5647477ba577510af4da85e5816 Mon Sep 17 00:00:00 2001 From: selee Date: Tue, 10 Feb 2026 16:26:30 +0900 Subject: [PATCH] Remove command from Runtime, move to TrainJob trainer.command Kubeflow Trainer torch runtime auto-injects torchrun only when command is not set in the container spec. Training script is now specified at TrainJob level via trainer.command instead. Co-Authored-By: Claude Opus 4.6 --- README.md | 6 +++--- configs/volcano-trainjob-integration.yaml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 303fc6e..6c3f77e 100644 --- a/README.md +++ b/README.md @@ -278,11 +278,11 @@ kubectl get pods -l trainer.kubeflow.org/trainjob-name= \ ## 알려진 이슈 -### Runtime에서 `command` 설정 시 torchrun 미주입 +### Runtime에 `command`를 설정하면 torchrun이 주입되지 않음 -Kubeflow Trainer torch runtime은 컨테이너에 `command`가 없을 때 자동으로 torchrun을 주입하여 `RANK`, `LOCAL_RANK`, `WORLD_SIZE` 등 환경변수를 설정합니다. Runtime에서 `command`를 직접 지정하면 torchrun 주입이 우회되어 분산 학습 환경변수가 설정되지 않습니다. +Kubeflow Trainer torch runtime은 컨테이너에 `command`가 없을 때 자동으로 torchrun을 주입하여 `RANK`, `LOCAL_RANK`, `WORLD_SIZE` 등 환경변수를 설정합니다. Runtime에서 `command`를 직접 지정하면 torchrun 주입이 우회됩니다. -현재 `volcano-trainjob-integration.yaml`의 Runtime에는 `command`가 설정되어 있으므로, 학습 스크립트 실행 방식을 조정해야 합니다. +따라서 **Runtime에는 `command`를 넣지 않고**, TrainJob의 `trainer.command`에서 학습 스크립트를 지정해야 합니다. ### JAX ClusterTrainingRuntime 미지원 diff --git a/configs/volcano-trainjob-integration.yaml b/configs/volcano-trainjob-integration.yaml index 7279d3c..77fb7a9 100644 --- a/configs/volcano-trainjob-integration.yaml +++ b/configs/volcano-trainjob-integration.yaml @@ -320,9 +320,6 @@ spec: containers: - name: node image: nvcr.io/nvidia/pytorch:24.10-py3 - command: - - python - - /workspace/scripts/train_nccl.py securityContext: privileged: true capabilities: @@ -378,6 +375,9 @@ spec: name: torch-distributed-volcano trainer: image: nvcr.io/nvidia/pytorch:24.10-py3 + command: + - python + - /workspace/scripts/train_nccl.py numNodes: 2 resourcesPerNode: requests: