From 18d9630d08fd5052c647b8aed28cff2e43ece069 Mon Sep 17 00:00:00 2001 From: Cloud User Date: Tue, 3 Mar 2026 17:57:41 +0900 Subject: [PATCH] Initial commit: nccl-perftest Helm chart - Original K8s manifest (hostdev-node1-pod1.yaml) - Helm chart with templated ConfigMap, Service, StatefulSet - README with install guide and values reference --- .gitignore | 2 + README.md | 119 +++++++ hostdev-node1-pod1.yaml | 376 +++++++++++++++++++++++ nccl-perftest/Chart.yaml | 6 + nccl-perftest/templates/_helpers.tpl | 33 ++ nccl-perftest/templates/configmap.yaml | 294 ++++++++++++++++++ nccl-perftest/templates/service.yaml | 13 + nccl-perftest/templates/statefulset.yaml | 78 +++++ nccl-perftest/values.yaml | 37 +++ 9 files changed, 958 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 hostdev-node1-pod1.yaml create mode 100644 nccl-perftest/Chart.yaml create mode 100644 nccl-perftest/templates/_helpers.tpl create mode 100644 nccl-perftest/templates/configmap.yaml create mode 100644 nccl-perftest/templates/service.yaml create mode 100644 nccl-perftest/templates/statefulset.yaml create mode 100644 nccl-perftest/values.yaml diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..123b049 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.env +*.tgz diff --git a/README.md b/README.md new file mode 100644 index 0000000..5dfa4e3 --- /dev/null +++ b/README.md @@ -0,0 +1,119 @@ +# nccl-perftest Helm Chart + +Multi-node NCCL 분산 학습 성능 테스트를 위한 Helm chart. +VGG11 + CIFAR-10 기반 torchrun 분산 학습을 StatefulSet으로 배포한다. + +## Prerequisites + +- Kubernetes 1.24+ +- Helm 3.x +- NVIDIA GPU Operator 설치 +- Multus CNI + hostdevice NAD 구성 +- InfiniBand 장치 (`/dev/infiniband`) + +## Helm Repo (Harbor OCI) + +```bash +# Harbor 로그인 +helm registry login harbor.inje-private.com -u admin + +# Chart pull +helm pull oci://harbor.inje-private.com/nccl-perftest/nccl-perftest --version 0.1.0 +``` + +## Install + +```bash +# 기본 설치 (values.yaml 기본값 사용) +helm install nccl-perftest ./nccl-perftest + +# 릴리스 이름 지정 +helm install my-release ./nccl-perftest + +# OCI 레지스트리에서 직접 설치 +helm install nccl-perftest oci://harbor.inje-private.com/nccl-perftest/nccl-perftest --version 0.1.0 +``` + +## Uninstall + +```bash +helm uninstall nccl-perftest +``` + +## Values 설정 + +### 커스텀 values 파일 사용 + +```bash +helm install nccl-perftest ./nccl-perftest -f my-values.yaml +``` + +### CLI로 개별 값 오버라이드 + +```bash +# 노드 수 변경 (3노드) +helm install nccl-perftest ./nccl-perftest --set replicaCount=3 + +# GPU 수 + 노드 수 동시 변경 +helm install nccl-perftest ./nccl-perftest \ + --set replicaCount=4 \ + --set resources.gpu=4 \ + --set training.nprocPerNode=4 + +# 이미지 태그 변경 +helm install nccl-perftest ./nccl-perftest --set image.tag="24.12-py3" + +# NCCL 디버그 옵션 +helm install nccl-perftest ./nccl-perftest \ + --set nccl.ibHCA="mlx5_0\,mlx5_1\,mlx5_2\,mlx5_3" \ + --set nccl.debugFile="/tmp/nccl-%h-%p.log" +``` + +## Values Reference + +| Key | Default | Description | +|-----|---------|-------------| +| `nameOverride` | `nccl-test` | 리소스 이름 | +| `replicaCount` | `2` | 노드(Pod) 수. NNODES에 자동 반영 | +| `image.repository` | `nvcr.io/nvidia/pytorch` | 컨테이너 이미지 | +| `image.tag` | `24.10-py3` | 이미지 태그 | +| `image.pullPolicy` | `IfNotPresent` | 이미지 풀 정책 | +| `training.nprocPerNode` | `8` | 노드당 GPU 프로세스 수 | +| `training.masterPort` | `29500` | 마스터 rendezvous 포트 | +| `training.batchSize` | `1200` | 글로벌 배치 사이즈 | +| `training.epochs` | `90` | 학습 에포크 수 | +| `training.lr` | `0.01` | 학습률 | +| `resources.gpu` | `8` | nvidia.com/gpu 요청 수 | +| `resources.hostdev` | `2` | nvidia.com/hostdev 요청 수 | +| `shmSize` | `128Gi` | /dev/shm 크기 | +| `network.nadName` | `hostdevice-net` | Multus NAD 이름 | +| `nccl.debug` | `INFO` | NCCL_DEBUG 레벨 | +| `nccl.debugSubsys` | `INIT,NET,IB` | NCCL_DEBUG_SUBSYS | +| `nccl.ibDisable` | `0` | NCCL_IB_DISABLE (0=IB 사용) | +| `nccl.socketIfname` | `net` | NCCL_SOCKET_IFNAME | +| `nccl.ibHCA` | `""` | NCCL_IB_HCA (비어있으면 미설정) | +| `nccl.debugFile` | `""` | NCCL_DEBUG_FILE (비어있으면 미설정) | +| `volumes.infiniband.hostPath` | `/dev/infiniband` | IB 장치 경로 | +| `volumes.dataset.hostPath` | `/home/ubuntu/cifar-10-batches-py` | CIFAR-10 데이터셋 경로 | + +## 예시: 3노드 4GPU 설정 + +```yaml +# my-values.yaml +replicaCount: 3 + +training: + nprocPerNode: 4 + batchSize: 600 + +resources: + gpu: 4 + hostdev: 1 + +nccl: + ibHCA: "mlx5_0,mlx5_1" +``` + +```bash +helm install nccl-3node ./nccl-perftest -f my-values.yaml +``` diff --git a/hostdev-node1-pod1.yaml b/hostdev-node1-pod1.yaml new file mode 100644 index 0000000..0a3f7ab --- /dev/null +++ b/hostdev-node1-pod1.yaml @@ -0,0 +1,376 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nccl-test-scripts +data: + run.sh: | + #!/usr/bin/env bash + set -ex + + # ----- StatefulSet Pod 이름에서 node_rank 추출 ----- + # Pod 이름: nccl-test-0, nccl-test-1, nccl-test-2 ... + NODE_RANK=${HOSTNAME##*-} + echo "[INFO] HOSTNAME=$HOSTNAME, NODE_RANK=$NODE_RANK" + + # ----- NCCL 환경변수 ----- + export NCCL_DEBUG=INFO + export NCCL_DEBUG_SUBSYS=INIT,NET,IB + export NCCL_IB_DISABLE=0 + export NCCL_SOCKET_IFNAME="net" + # export NCCL_IB_HCA="mlx5_0,mlx5_1,mlx5_2,mlx5_3" + # export NCCL_DEBUG_FILE="/tmp/nccl-%h-%p.log" # 주석처리: tee로 파일+stdout 동시 출력 + + # ----- 학습 실행 ----- + torchrun \ + --nnodes=${NNODES} \ + --nproc_per_node=${NPROC_PER_NODE} \ + --rdzv_backend=c10d \ + --node_rank=${NODE_RANK} \ + --rdzv_endpoint=${MASTER_ADDR}:${MASTER_PORT} \ + /workspace/train_nccl.py 2>&1 | tee /tmp/nccl-aggregate.log + + train_nccl.py: | + import os + import time + import torch + import argparse + import torch.distributed as dist + import torch.multiprocessing as mp + from torch.optim.lr_scheduler import StepLR + + # for dataset + from torchvision.datasets.cifar import CIFAR10 + import torchvision.transforms as tfs + from torch.utils.data import DataLoader + from torch.utils.data.distributed import DistributedSampler + + # for model + from torchvision.models import vgg11 + from torch.nn.parallel import DistributedDataParallel as DDP + + import numpy as np + import random + import datetime + + + def set_random_seeds(random_seed=0): + torch.manual_seed(random_seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + np.random.seed(random_seed) + random.seed(random_seed) + + + def get_args_parser(): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument('--lr', type=float, default=0.01) + parser.add_argument('--epoch', type=int, default=90) + parser.add_argument('--batch_size', type=int, default=1200) + parser.add_argument('--global_rank', type=int, default=0) + parser.add_argument('--vis_step', type=int, default=10) + parser.add_argument('--num_workers', type=int, default=24) + parser.add_argument("--local_rank", type=int, + help="Local rank. Necessary for using the torch.distributed.launch utility.") + parser.add_argument('--world_size', type=int, default=0) + parser.add_argument('--port', type=int, default=2022) + parser.add_argument('--root', type=str, default='data') + parser.add_argument('--start_epoch', type=int, default=0) + parser.add_argument('--save_path', type=str, default='./save') + parser.add_argument('--save_file_name', type=str, default='vgg_cifar') + return parser + + + def main(opts): + # 1. set random seeds + set_random_seeds(random_seed=0) + + # 2. initialization + init_for_distributed(opts) + + # 3. visdom + vis = None + + # 4. data set + transform_train = tfs.Compose([ + tfs.Resize(256), + tfs.RandomCrop(224), + tfs.RandomHorizontalFlip(), + tfs.ToTensor(), + tfs.Normalize(mean=(0.4914, 0.4822, 0.4465), + std=(0.2023, 0.1994, 0.2010)), + ]) + + transform_test = tfs.Compose([ + tfs.Resize(256), + tfs.CenterCrop(224), + tfs.ToTensor(), + tfs.Normalize(mean=(0.4914, 0.4822, 0.4465), + std=(0.2023, 0.1994, 0.2010)), + ]) + + train_set = CIFAR10(root=opts.root, + train=True, + transform=transform_train, + download=True) + + test_set = CIFAR10(root=opts.root, + train=False, + transform=transform_test, + download=True) + + train_sampler = DistributedSampler(dataset=train_set, shuffle=True) + test_sampler = DistributedSampler(dataset=test_set, shuffle=False) + + train_loader = DataLoader(dataset=train_set, + batch_size=int(opts.batch_size / opts.world_size), + shuffle=False, + num_workers=int(opts.num_workers / opts.world_size), + sampler=train_sampler, + pin_memory=True) + + test_loader = DataLoader(dataset=test_set, + batch_size=int(opts.batch_size / opts.world_size), + shuffle=False, + num_workers=int(opts.num_workers / opts.world_size), + sampler=test_sampler, + pin_memory=True) + + # 5. model + model = vgg11(pretrained=False) + model = model.cuda(opts.local_rank) + model = DDP(module=model, + device_ids=[opts.local_rank]) + + # 6. criterion + criterion = torch.nn.CrossEntropyLoss().to(opts.local_rank) + + # 7. optimizer + optimizer = torch.optim.SGD(params=model.parameters(), + lr=0.01, + weight_decay=0.0005, + momentum=0.9) + + # 8. scheduler + scheduler = StepLR(optimizer=optimizer, + step_size=30, + gamma=0.1) + + if opts.start_epoch != 0: + + checkpoint = torch.load(os.path.join(opts.save_path, opts.save_file_name) + '.{}.pth.tar' + .format(opts.start_epoch - 1), + map_location=torch.device('cuda:{}'.format(opts.local_rank))) + model.load_state_dict(checkpoint['model_state_dict']) # load model state dict + optimizer.load_state_dict(checkpoint['optimizer_state_dict']) # load optim state dict + scheduler.load_state_dict(checkpoint['scheduler_state_dict']) # load sched state dict + if opts.global_rank == 0: + print('\nLoaded checkpoint from epoch %d.\n' % (int(opts.start_epoch) - 1)) + + for epoch in range(opts.start_epoch, opts.epoch): + + # 9. train + tic = time.time() + model.train() + train_sampler.set_epoch(epoch) + + for i, (images, labels) in enumerate(train_loader): + images = images.to(opts.local_rank) + labels = labels.to(opts.local_rank) + outputs = model(images) + + # ----------- update ----------- + optimizer.zero_grad() + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + + # get lr + for param_group in optimizer.param_groups: + lr = param_group['lr'] + + # time + toc = time.time() + + # visualization + if (i % opts.vis_step == 0 or i == len(train_loader) - 1): + print('GPU[{0}] Epoch [{1}/{2}], Iter [{3}/{4}], Loss: {5:.4f}, LR: {6:.5f}, Time: {7:.2f}'.format(opts.global_rank, + epoch, + opts.epoch, + i, + len(train_loader), + loss.item(), + lr, + toc - tic)) + + # save pth file + if opts.local_rank == 0: + if not os.path.exists(opts.save_path): + os.mkdir(opts.save_path) + + checkpoint = {'epoch': epoch, + 'model_state_dict': model.state_dict(), + 'optimizer_state_dict': optimizer.state_dict(), + 'scheduler_state_dict': scheduler.state_dict()} + + torch.save(checkpoint, os.path.join(opts.save_path, opts.save_file_name + '.{}.pth.tar'.format(epoch))) + print("save pth.tar {} epoch!".format(epoch)) + + # 10. test + model.eval() + + val_avg_loss = 0 + correct_top1 = 0 + correct_top5 = 0 + total = 0 + + with torch.no_grad(): + for i, (images, labels) in enumerate(test_loader): + images = images.to(opts.local_rank) # [100, 3, 224, 224] + labels = labels.to(opts.local_rank) # [100] + outputs = model(images) + loss = criterion(outputs, labels) + val_avg_loss += loss.item() + # ------------------------------------------------------------------------------ + # rank 1 + _, pred = torch.max(outputs, 1) + total += labels.size(0) + correct_top1 += (pred == labels).sum().item() + + # ------------------------------------------------------------------------------ + # rank 5 + _, rank5 = outputs.topk(5, 1, True, True) + rank5 = rank5.t() + correct5 = rank5.eq(labels.view(1, -1).expand_as(rank5)) + + # ------------------------------------------------------------------------------ + for k in range(5): # 0, 1, 2, 3, 4, 5 + correct_k = correct5[:k+1].reshape(-1).float().sum(0, keepdim=True) + correct_top5 += correct_k.item() + + accuracy_top1 = correct_top1 / total + accuracy_top5 = correct_top5 / total + + val_avg_loss = val_avg_loss / len(test_loader) # make mean loss + + print("top-1 percentage : {0:0.3f}%".format(correct_top1 / total * 100)) + print("top-5 percentage : {0:0.3f}%".format(correct_top5 / total * 100)) + scheduler.step() + + return 0 + + + def init_for_distributed(opts): + + # 1. setting for distributed training + opts.global_rank = int(os.environ['RANK']) + opts.local_rank = int(os.environ['LOCAL_RANK']) + opts.world_size = int(os.environ['WORLD_SIZE']) + torch.cuda.set_device(opts.local_rank) + if opts.global_rank is not None and opts.local_rank is not None: + print("Use GPU: [{}/{}] for training".format(opts.global_rank, opts.local_rank)) + + # 2. init_process_group + dist.init_process_group( + backend="nccl", + rank=opts.global_rank, + world_size=opts.world_size, + device_id=torch.device(f"cuda:{opts.local_rank}"), + timeout=datetime.timedelta(seconds=60) + ) + + return + + + if __name__ == '__main__': + + parser = argparse.ArgumentParser('vgg11 cifar training', parents=[get_args_parser()]) + opts = parser.parse_args() + main(opts) +--- +apiVersion: v1 +kind: Service +metadata: + name: nccl-test + labels: { app: nccl-test } +spec: + clusterIP: None # Headless Service + selector: { app: nccl-test } + ports: + - name: rdzv + port: 29500 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nccl-test +spec: + serviceName: nccl-test + replicas: 2 # 3노드 (ib-1, ib-2, ib-3) + selector: + matchLabels: { app: nccl-test } + template: + metadata: + labels: { app: nccl-test } + annotations: + k8s.v1.cni.cncf.io/networks: hostdevice-net + spec: + containers: + - name: worker + image: nvcr.io/nvidia/pytorch:24.10-py3 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + capabilities: + add: + - IPC_LOCK + command: + - bash + - -lc + - | + cp /config_scripts/* /workspace/ && \ + chmod +x /workspace/run.sh && \ + cd /workspace && ./run.sh; \ + sleep infinity + env: + # ----- 분산 학습 설정 ----- + - name: NNODES + value: "2" + - name: NPROC_PER_NODE + value: "8" + - name: MASTER_ADDR + value: "nccl-test-0.nccl-test" + - name: MASTER_PORT + value: "29500" + resources: + limits: + nvidia.com/gpu: "8" + nvidia.com/hostdev: 2 + requests: + nvidia.com/gpu: "8" + nvidia.com/hostdev: 2 + volumeMounts: + - name: scripts + mountPath: /config_scripts + - name: shared-memory + mountPath: /dev/shm + - name: infiniband + mountPath: /dev/infiniband + - name: cifar-data + mountPath: /workspace/data/cifar-10-batches-py + volumes: + - name: scripts + configMap: + name: nccl-test-scripts + defaultMode: 0755 + - name: shared-memory + emptyDir: + medium: Memory + sizeLimit: 128Gi # GPU 8장 x 32GB = 256GB, 권장: 50% = 128Gi + - name: infiniband + hostPath: + path: /dev/infiniband + type: Directory + - name: cifar-data + hostPath: + path: /home/ubuntu/cifar-10-batches-py + type: Directory diff --git a/nccl-perftest/Chart.yaml b/nccl-perftest/Chart.yaml new file mode 100644 index 0000000..369f598 --- /dev/null +++ b/nccl-perftest/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: nccl-perftest +description: Multi-node NCCL distributed training performance test +type: application +version: 0.1.0 +appVersion: "1.0.0" diff --git a/nccl-perftest/templates/_helpers.tpl b/nccl-perftest/templates/_helpers.tpl new file mode 100644 index 0000000..89b6b2a --- /dev/null +++ b/nccl-perftest/templates/_helpers.tpl @@ -0,0 +1,33 @@ +{{/* +Chart name, overridden by nameOverride. +*/}} +{{- define "nccl-perftest.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Fully qualified app name (release-aware). +*/}} +{{- define "nccl-perftest.fullname" -}} +{{- if .Values.nameOverride }} +{{- .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name .Chart.Name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} + +{{/* +Common labels. +*/}} +{{- define "nccl-perftest.labels" -}} +app: {{ include "nccl-perftest.name" . }} +helm.sh/chart: {{ printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels. +*/}} +{{- define "nccl-perftest.selectorLabels" -}} +app: {{ include "nccl-perftest.name" . }} +{{- end }} diff --git a/nccl-perftest/templates/configmap.yaml b/nccl-perftest/templates/configmap.yaml new file mode 100644 index 0000000..507c31b --- /dev/null +++ b/nccl-perftest/templates/configmap.yaml @@ -0,0 +1,294 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "nccl-perftest.name" . }}-scripts + labels: + {{- include "nccl-perftest.labels" . | nindent 4 }} +data: + run.sh: | + #!/usr/bin/env bash + set -ex + + # ----- StatefulSet Pod 이름에서 node_rank 추출 ----- + # Pod 이름: nccl-test-0, nccl-test-1, nccl-test-2 ... + NODE_RANK=${HOSTNAME##*-} + echo "[INFO] HOSTNAME=$HOSTNAME, NODE_RANK=$NODE_RANK" + + # ----- NCCL 환경변수 ----- + export NCCL_DEBUG={{ .Values.nccl.debug }} + export NCCL_DEBUG_SUBSYS={{ .Values.nccl.debugSubsys }} + export NCCL_IB_DISABLE={{ .Values.nccl.ibDisable }} + export NCCL_SOCKET_IFNAME="{{ .Values.nccl.socketIfname }}" + {{- if .Values.nccl.ibHCA }} + export NCCL_IB_HCA="{{ .Values.nccl.ibHCA }}" + {{- end }} + {{- if .Values.nccl.debugFile }} + export NCCL_DEBUG_FILE="{{ .Values.nccl.debugFile }}" + {{- end }} + + # ----- 학습 실행 ----- + torchrun \ + --nnodes=${NNODES} \ + --nproc_per_node=${NPROC_PER_NODE} \ + --rdzv_backend=c10d \ + --node_rank=${NODE_RANK} \ + --rdzv_endpoint=${MASTER_ADDR}:${MASTER_PORT} \ + /workspace/train_nccl.py 2>&1 | tee /tmp/nccl-aggregate.log + + train_nccl.py: | + import os + import time + import torch + import argparse + import torch.distributed as dist + import torch.multiprocessing as mp + from torch.optim.lr_scheduler import StepLR + + # for dataset + from torchvision.datasets.cifar import CIFAR10 + import torchvision.transforms as tfs + from torch.utils.data import DataLoader + from torch.utils.data.distributed import DistributedSampler + + # for model + from torchvision.models import vgg11 + from torch.nn.parallel import DistributedDataParallel as DDP + + import numpy as np + import random + import datetime + + + def set_random_seeds(random_seed=0): + torch.manual_seed(random_seed) + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + np.random.seed(random_seed) + random.seed(random_seed) + + + def get_args_parser(): + parser = argparse.ArgumentParser(add_help=False) + parser.add_argument('--lr', type=float, default={{ .Values.training.lr }}) + parser.add_argument('--epoch', type=int, default={{ .Values.training.epochs }}) + parser.add_argument('--batch_size', type=int, default={{ .Values.training.batchSize }}) + parser.add_argument('--global_rank', type=int, default=0) + parser.add_argument('--vis_step', type=int, default=10) + parser.add_argument('--num_workers', type=int, default=24) + parser.add_argument("--local_rank", type=int, + help="Local rank. Necessary for using the torch.distributed.launch utility.") + parser.add_argument('--world_size', type=int, default=0) + parser.add_argument('--port', type=int, default=2022) + parser.add_argument('--root', type=str, default='data') + parser.add_argument('--start_epoch', type=int, default=0) + parser.add_argument('--save_path', type=str, default='./save') + parser.add_argument('--save_file_name', type=str, default='vgg_cifar') + return parser + + + def main(opts): + # 1. set random seeds + set_random_seeds(random_seed=0) + + # 2. initialization + init_for_distributed(opts) + + # 3. visdom + vis = None + + # 4. data set + transform_train = tfs.Compose([ + tfs.Resize(256), + tfs.RandomCrop(224), + tfs.RandomHorizontalFlip(), + tfs.ToTensor(), + tfs.Normalize(mean=(0.4914, 0.4822, 0.4465), + std=(0.2023, 0.1994, 0.2010)), + ]) + + transform_test = tfs.Compose([ + tfs.Resize(256), + tfs.CenterCrop(224), + tfs.ToTensor(), + tfs.Normalize(mean=(0.4914, 0.4822, 0.4465), + std=(0.2023, 0.1994, 0.2010)), + ]) + + train_set = CIFAR10(root=opts.root, + train=True, + transform=transform_train, + download=True) + + test_set = CIFAR10(root=opts.root, + train=False, + transform=transform_test, + download=True) + + train_sampler = DistributedSampler(dataset=train_set, shuffle=True) + test_sampler = DistributedSampler(dataset=test_set, shuffle=False) + + train_loader = DataLoader(dataset=train_set, + batch_size=int(opts.batch_size / opts.world_size), + shuffle=False, + num_workers=int(opts.num_workers / opts.world_size), + sampler=train_sampler, + pin_memory=True) + + test_loader = DataLoader(dataset=test_set, + batch_size=int(opts.batch_size / opts.world_size), + shuffle=False, + num_workers=int(opts.num_workers / opts.world_size), + sampler=test_sampler, + pin_memory=True) + + # 5. model + model = vgg11(pretrained=False) + model = model.cuda(opts.local_rank) + model = DDP(module=model, + device_ids=[opts.local_rank]) + + # 6. criterion + criterion = torch.nn.CrossEntropyLoss().to(opts.local_rank) + + # 7. optimizer + optimizer = torch.optim.SGD(params=model.parameters(), + lr=0.01, + weight_decay=0.0005, + momentum=0.9) + + # 8. scheduler + scheduler = StepLR(optimizer=optimizer, + step_size=30, + gamma=0.1) + + if opts.start_epoch != 0: + + checkpoint = torch.load(os.path.join(opts.save_path, opts.save_file_name) + '.{}.pth.tar' + .format(opts.start_epoch - 1), + map_location=torch.device('cuda:{}'.format(opts.local_rank))) + model.load_state_dict(checkpoint['model_state_dict']) # load model state dict + optimizer.load_state_dict(checkpoint['optimizer_state_dict']) # load optim state dict + scheduler.load_state_dict(checkpoint['scheduler_state_dict']) # load sched state dict + if opts.global_rank == 0: + print('\nLoaded checkpoint from epoch %d.\n' % (int(opts.start_epoch) - 1)) + + for epoch in range(opts.start_epoch, opts.epoch): + + # 9. train + tic = time.time() + model.train() + train_sampler.set_epoch(epoch) + + for i, (images, labels) in enumerate(train_loader): + images = images.to(opts.local_rank) + labels = labels.to(opts.local_rank) + outputs = model(images) + + # ----------- update ----------- + optimizer.zero_grad() + loss = criterion(outputs, labels) + loss.backward() + optimizer.step() + + # get lr + for param_group in optimizer.param_groups: + lr = param_group['lr'] + + # time + toc = time.time() + + # visualization + if (i % opts.vis_step == 0 or i == len(train_loader) - 1): + print('GPU[{0}] Epoch [{1}/{2}], Iter [{3}/{4}], Loss: {5:.4f}, LR: {6:.5f}, Time: {7:.2f}'.format(opts.global_rank, + epoch, + opts.epoch, + i, + len(train_loader), + loss.item(), + lr, + toc - tic)) + + # save pth file + if opts.local_rank == 0: + if not os.path.exists(opts.save_path): + os.mkdir(opts.save_path) + + checkpoint = {'epoch': epoch, + 'model_state_dict': model.state_dict(), + 'optimizer_state_dict': optimizer.state_dict(), + 'scheduler_state_dict': scheduler.state_dict()} + + torch.save(checkpoint, os.path.join(opts.save_path, opts.save_file_name + '.{}.pth.tar'.format(epoch))) + print("save pth.tar {} epoch!".format(epoch)) + + # 10. test + model.eval() + + val_avg_loss = 0 + correct_top1 = 0 + correct_top5 = 0 + total = 0 + + with torch.no_grad(): + for i, (images, labels) in enumerate(test_loader): + images = images.to(opts.local_rank) # [100, 3, 224, 224] + labels = labels.to(opts.local_rank) # [100] + outputs = model(images) + loss = criterion(outputs, labels) + val_avg_loss += loss.item() + # ------------------------------------------------------------------------------ + # rank 1 + _, pred = torch.max(outputs, 1) + total += labels.size(0) + correct_top1 += (pred == labels).sum().item() + + # ------------------------------------------------------------------------------ + # rank 5 + _, rank5 = outputs.topk(5, 1, True, True) + rank5 = rank5.t() + correct5 = rank5.eq(labels.view(1, -1).expand_as(rank5)) + + # ------------------------------------------------------------------------------ + for k in range(5): # 0, 1, 2, 3, 4, 5 + correct_k = correct5[:k+1].reshape(-1).float().sum(0, keepdim=True) + correct_top5 += correct_k.item() + + accuracy_top1 = correct_top1 / total + accuracy_top5 = correct_top5 / total + + val_avg_loss = val_avg_loss / len(test_loader) # make mean loss + + print("top-1 percentage : {0:0.3f}%".format(correct_top1 / total * 100)) + print("top-5 percentage : {0:0.3f}%".format(correct_top5 / total * 100)) + scheduler.step() + + return 0 + + + def init_for_distributed(opts): + + # 1. setting for distributed training + opts.global_rank = int(os.environ['RANK']) + opts.local_rank = int(os.environ['LOCAL_RANK']) + opts.world_size = int(os.environ['WORLD_SIZE']) + torch.cuda.set_device(opts.local_rank) + if opts.global_rank is not None and opts.local_rank is not None: + print("Use GPU: [{}/{}] for training".format(opts.global_rank, opts.local_rank)) + + # 2. init_process_group + dist.init_process_group( + backend="nccl", + rank=opts.global_rank, + world_size=opts.world_size, + device_id=torch.device(f"cuda:{opts.local_rank}"), + timeout=datetime.timedelta(seconds=60) + ) + + return + + + if __name__ == '__main__': + + parser = argparse.ArgumentParser('vgg11 cifar training', parents=[get_args_parser()]) + opts = parser.parse_args() + main(opts) diff --git a/nccl-perftest/templates/service.yaml b/nccl-perftest/templates/service.yaml new file mode 100644 index 0000000..5c056c8 --- /dev/null +++ b/nccl-perftest/templates/service.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "nccl-perftest.name" . }} + labels: + {{- include "nccl-perftest.labels" . | nindent 4 }} +spec: + clusterIP: None + selector: + {{- include "nccl-perftest.selectorLabels" . | nindent 4 }} + ports: + - name: rdzv + port: {{ .Values.training.masterPort }} diff --git a/nccl-perftest/templates/statefulset.yaml b/nccl-perftest/templates/statefulset.yaml new file mode 100644 index 0000000..95e2c16 --- /dev/null +++ b/nccl-perftest/templates/statefulset.yaml @@ -0,0 +1,78 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "nccl-perftest.name" . }} + labels: + {{- include "nccl-perftest.labels" . | nindent 4 }} +spec: + serviceName: {{ include "nccl-perftest.name" . }} + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "nccl-perftest.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + {{- include "nccl-perftest.selectorLabels" . | nindent 8 }} + annotations: + k8s.v1.cni.cncf.io/networks: {{ .Values.network.nadName }} + spec: + containers: + - name: worker + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + securityContext: + privileged: true + capabilities: + add: + - IPC_LOCK + command: + - bash + - -lc + - | + cp /config_scripts/* /workspace/ && \ + chmod +x /workspace/run.sh && \ + cd /workspace && ./run.sh; \ + sleep infinity + env: + - name: NNODES + value: {{ .Values.replicaCount | quote }} + - name: NPROC_PER_NODE + value: {{ .Values.training.nprocPerNode | quote }} + - name: MASTER_ADDR + value: "{{ include "nccl-perftest.name" . }}-0.{{ include "nccl-perftest.name" . }}" + - name: MASTER_PORT + value: {{ .Values.training.masterPort | quote }} + resources: + limits: + nvidia.com/gpu: {{ .Values.resources.gpu | quote }} + nvidia.com/hostdev: {{ .Values.resources.hostdev }} + requests: + nvidia.com/gpu: {{ .Values.resources.gpu | quote }} + nvidia.com/hostdev: {{ .Values.resources.hostdev }} + volumeMounts: + - name: scripts + mountPath: /config_scripts + - name: shared-memory + mountPath: /dev/shm + - name: infiniband + mountPath: /dev/infiniband + - name: cifar-data + mountPath: /workspace/data/cifar-10-batches-py + volumes: + - name: scripts + configMap: + name: {{ include "nccl-perftest.name" . }}-scripts + defaultMode: 0755 + - name: shared-memory + emptyDir: + medium: Memory + sizeLimit: {{ .Values.shmSize }} + - name: infiniband + hostPath: + path: {{ .Values.volumes.infiniband.hostPath }} + type: Directory + - name: cifar-data + hostPath: + path: {{ .Values.volumes.dataset.hostPath }} + type: Directory diff --git a/nccl-perftest/values.yaml b/nccl-perftest/values.yaml new file mode 100644 index 0000000..c8a2249 --- /dev/null +++ b/nccl-perftest/values.yaml @@ -0,0 +1,37 @@ +nameOverride: "nccl-test" +replicaCount: 2 + +image: + repository: nvcr.io/nvidia/pytorch + tag: "24.10-py3" + pullPolicy: IfNotPresent + +training: + nprocPerNode: 8 + masterPort: 29500 + batchSize: 1200 + epochs: 90 + lr: 0.01 + +resources: + gpu: 8 + hostdev: 2 + +shmSize: "128Gi" + +network: + nadName: "hostdevice-net" + +nccl: + debug: "INFO" + debugSubsys: "INIT,NET,IB" + ibDisable: 0 + socketIfname: "net" + ibHCA: "" # 비어있으면 설정 안 함 + debugFile: "" # 비어있으면 설정 안 함 + +volumes: + infiniband: + hostPath: "/dev/infiniband" + dataset: + hostPath: "/home/ubuntu/cifar-10-batches-py"