From a80f3bd97e54e57e5fff351cb4707225a613188c Mon Sep 17 00:00:00 2001 From: Cloud User Date: Tue, 3 Mar 2026 15:17:34 +0900 Subject: [PATCH] Fix: all workers must call model.evaluate() together MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit MultiWorkerMirroredStrategy에서 evaluate도 collective op을 수행하므로 모든 워커가 함께 호출해야 한다. Worker 0만 호출하면 나머지 워커가 종료되어 heartbeat timeout 발생. Co-Authored-By: Claude Opus 4.6 --- tensorflow-volcano-trainjob-integration.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tensorflow-volcano-trainjob-integration.yaml b/tensorflow-volcano-trainjob-integration.yaml index 7122669..1b6f76f 100644 --- a/tensorflow-volcano-trainjob-integration.yaml +++ b/tensorflow-volcano-trainjob-integration.yaml @@ -253,9 +253,13 @@ data: verbose=verbose, ) - # 평가 (worker 0만 출력) + # 평가 (모든 워커가 참여해야 함 - MultiWorkerMirroredStrategy 요구사항) + eval_verbose = 2 if worker_index == 0 else 0 + loss, accuracy = model.evaluate( + test_dataset, verbose=eval_verbose + ) + if worker_index == 0: - loss, accuracy = model.evaluate(test_dataset, verbose=2) print(f"\n===== Final Results =====") print(f"Test Loss: {loss:.4f}") print(f"Test Accuracy: {accuracy:.4f}")