diff --git a/nccl_test/simple-test/simple_nccl.py b/nccl_test/simple-test/simple_nccl.py new file mode 100644 index 0000000..a11737c --- /dev/null +++ b/nccl_test/simple-test/simple_nccl.py @@ -0,0 +1,17 @@ +import os +import torch +import torch.distributed as dist + +def main(): + dist.init_process_group("nccl") + local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local_rank) + + print(f"Rank {dist.get_rank()} initialized on GPU {local_rank}") + + x = torch.ones(10).cuda() + dist.all_reduce(x) + print(f"Rank {dist.get_rank()} result: {x[0].item()}") + +if __name__ == "__main__": + main() diff --git a/nccl_test/simple-test/template.yaml b/nccl_test/simple-test/template.yaml index d4f3abd..4d5668d 100644 --- a/nccl_test/simple-test/template.yaml +++ b/nccl_test/simple-test/template.yaml @@ -365,8 +365,8 @@ spec: template: metadata: labels: { app: nccl } - annotations: - k8s.v1.cni.cncf.io/networks: hostdevice-net # HostDeviceNetwork (RDMA) +# annotations: +# k8s.v1.cni.cncf.io/networks: hostdevice-net # HostDeviceNetwork (RDMA) spec: # hostNetwork: true # dnsPolicy: ClusterFirstWithHostNet @@ -392,15 +392,17 @@ spec: resources: limits: nvidia.com/gpu: "8" - nvidia.com/hostdev: "8" # SR-IOV Device Plugin이 광고한 RDMA NIC 리소스 +# nvidia.com/hostdev: "8" # SR-IOV Device Plugin이 광고한 RDMA NIC 리소스 requests: nvidia.com/gpu: "8" - nvidia.com/hostdev: "8" +# nvidia.com/hostdev: "8" volumeMounts: - name: scripts mountPath: /config_scripts - name: shared-memory mountPath: /dev/shm + - name: infiniband + mountPath: /dev/infiniband volumes: - name: scripts configMap: @@ -410,5 +412,9 @@ spec: emptyDir: medium: Memory sizeLimit: 118541097369600m + - name: infiniband + hostPath: + path: /dev/infiniband + type: Directory