From 7cd7c1af7ee1795e11f14f374e67c7824c06229e Mon Sep 17 00:00:00 2001 From: selee Date: Tue, 10 Feb 2026 16:36:44 +0900 Subject: [PATCH] Fix torchrun injection: add ancestor-step label, use torchrun command - Add trainer.kubeflow.org/trainjob-ancestor-step label to replicatedJob template so Trainer controller injects PET_* env vars - Change TrainJob command from python to torchrun so PET_* env vars are used for distributed training setup - Remove command from Runtime (keep it in TrainJob only) Verified: 2-node 16-GPU distributed training running successfully Co-Authored-By: Claude Opus 4.6 --- configs/volcano-trainjob-integration.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/configs/volcano-trainjob-integration.yaml b/configs/volcano-trainjob-integration.yaml index 77fb7a9..0b14d7b 100644 --- a/configs/volcano-trainjob-integration.yaml +++ b/configs/volcano-trainjob-integration.yaml @@ -310,6 +310,9 @@ spec: replicatedJobs: - name: node template: + metadata: + labels: + trainer.kubeflow.org/trainjob-ancestor-step: trainer spec: template: metadata: @@ -376,7 +379,7 @@ spec: trainer: image: nvcr.io/nvidia/pytorch:24.10-py3 command: - - python + - torchrun - /workspace/scripts/train_nccl.py numNodes: 2 resourcesPerNode: