From 59fbb40e28676edcb7df240b07444954a1744525 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 2 Dec 2025 17:20:54 +0900 Subject: [PATCH] net_operator, nccl, nerdctl init --- nccl_test/demo1.yaml | 137 ++ nccl_test/demo2.yaml | 65 + nccl_test/sample.yaml | 169 ++ nccl_test/simple-test/run.sh | 55 + nccl_test/simple-test/template.yaml | 152 ++ nccl_test/simple-test/train_nccl.py | 18 + nccl_test/trainer/trainer.yaml | 187 +++ nerdctl_test/helper.yaml | 32 + nerdctl_test/single.yaml | 19 + network-operator/.helmignore | 23 + network-operator/Chart.lock | 15 + network-operator/Chart.yaml | 30 + network-operator/README.md | 243 +++ .../maintenance-operator-chart/.helmignore | 23 + .../maintenance-operator-chart/Chart.yaml | 6 + .../maintenance-operator-chart/README.md | 34 + ...nvidia.com_maintenanceoperatorconfigs.yaml | 93 ++ ...intenance.nvidia.com_nodemaintenances.yaml | 275 ++++ .../templates/_helpers.tpl | 62 + .../templates/certificates.yaml | 43 + .../templates/deployment.yaml | 88 + .../templates/metrics-service.yaml | 18 + .../templates/operatorconfig.yaml | 21 + .../templates/role.yaml | 154 ++ .../templates/role_binding.yaml | 36 + .../templates/serviceaccount.yaml | 12 + .../templates/webhook.yaml | 48 + .../maintenance-operator-chart/values.yaml | 109 ++ .../.helmignore | 23 + .../Chart.yaml | 6 + .../README.md | 27 + ....nvidia.com_nicconfigurationtemplates.yaml | 174 ++ ...nfiguration.net.nvidia.com_nicdevices.yaml | 264 +++ .../templates/_helpers.tpl | 58 + .../templates/config-daemon.yaml | 68 + .../templates/operator.yaml | 61 + .../templates/role.yaml | 105 ++ .../templates/role_binding.yaml | 16 + .../templates/serviceaccount.yaml | 12 + .../supported-nic-firmware-configmap.yaml | 32 + .../values.yaml | 67 + .../charts/node-feature-discovery/.helmignore | 23 + .../charts/node-feature-discovery/Chart.yaml | 14 + .../charts/node-feature-discovery/README.md | 10 + .../crds/nfd-api-crds.yaml | 711 +++++++++ .../templates/_helpers.tpl | 107 ++ .../templates/clusterrole.yaml | 140 ++ .../templates/clusterrolebinding.yaml | 52 + .../templates/master.yaml | 170 ++ .../templates/nfd-gc.yaml | 88 + .../templates/nfd-master-conf.yaml | 12 + .../templates/nfd-topologyupdater-conf.yaml | 12 + .../templates/nfd-worker-conf.yaml | 12 + .../templates/post-delete-job.yaml | 94 ++ .../templates/prometheus.yaml | 26 + .../templates/role.yaml | 24 + .../templates/rolebinding.yaml | 18 + .../templates/serviceaccount.yaml | 58 + .../templates/topologyupdater-crds.yaml | 278 ++++ .../templates/topologyupdater.yaml | 188 +++ .../templates/worker.yaml | 195 +++ .../charts/node-feature-discovery/values.yaml | 599 +++++++ .../charts/sriov-network-operator/.helmignore | 23 + .../charts/sriov-network-operator/Chart.yaml | 13 + .../charts/sriov-network-operator/README.md | 167 ++ ...f.io_networkattachmentdefinitions_crd.yaml | 57 + ...sriovnetwork.openshift.io_ovsnetworks.yaml | 105 ++ ...vnetwork.openshift.io_sriovibnetworks.yaml | 78 + ...openshift.io_sriovnetworknodepolicies.yaml | 213 +++ ...k.openshift.io_sriovnetworknodestates.yaml | 369 +++++ ....openshift.io_sriovnetworkpoolconfigs.yaml | 129 ++ ...iovnetwork.openshift.io_sriovnetworks.yaml | 136 ++ ...ork.openshift.io_sriovoperatorconfigs.yaml | 114 ++ .../templates/NOTES.txt | 6 + .../templates/_helpers.tpl | 62 + .../templates/certificate.yaml | 71 + .../templates/clusterrole.yaml | 54 + .../templates/clusterrolebinding.yaml | 29 + .../templates/configmap.yaml | 47 + .../templates/extra-list.yaml | 8 + .../templates/operator.yaml | 137 ++ .../templates/pre-delete-webooks.yaml | 33 + .../templates/role.yaml | 138 ++ .../templates/rolebinding.yaml | 31 + .../templates/serviceaccount.yaml | 15 + .../templates/sriovoperatorconfig.yaml | 21 + .../charts/sriov-network-operator/values.yaml | 125 ++ ...f.io_networkattachmentdefinitions_crd.yaml | 57 + .../crds/mellanox.com_hostdevicenetworks.yaml | 111 ++ .../crds/mellanox.com_ipoibnetworks.yaml | 83 + .../crds/mellanox.com_macvlannetworks.yaml | 97 ++ .../crds/mellanox.com_nicclusterpolicies.yaml | 1414 +++++++++++++++++ network-operator/hostdev/demo1.yaml | 24 + network-operator/hostdev/demo2.yaml | 24 + network-operator/hostdev/direct-rdma-1.yaml | 24 + network-operator/hostdev/direct-rdma-2.yaml | 24 + network-operator/hostdev/host-device-net.yaml | 39 + .../hostdev/nicclusterpolicy-test.yaml | 47 + .../hostdev/nicclusterpolicy.yaml | 56 + network-operator/hostdev/sample-pod.yaml | 28 + network-operator/ipiob/nicclusterpolicy.yaml | 51 + network-operator/sriov/demo1.yaml | 24 + network-operator/sriov/demo2.yaml | 24 + network-operator/sriov/nicclusterpolicy.yaml | 23 + network-operator/sriov/nodepolicy_gcore1.yaml | 27 + network-operator/sriov/nodepolicy_gcore2.yaml | 27 + network-operator/sriov/sriov_net.yaml | 40 + network-operator/sriov/test.log | 401 +++++ network-operator/sriov/test2.log | 586 +++++++ network-operator/templates/NOTES.txt | 3 + network-operator/templates/_helpers.tpl | 86 + .../templates/admission_controller.yaml | 129 ++ network-operator/templates/keep-ncp-hook.yaml | 84 + .../templates/nodefeaturerules.yaml | 16 + network-operator/templates/operator.yaml | 120 ++ network-operator/templates/role.yaml | 390 +++++ network-operator/templates/role_binding.yaml | 27 + .../templates/service_account.yaml | 20 + .../templates/upgrade-crd-hook.yaml | 100 ++ network-operator/values.yaml | 309 ++++ .../dev/gpu-operator/custom/dcgm_config.yaml | 8 + 121 files changed, 12615 insertions(+) create mode 100644 nccl_test/demo1.yaml create mode 100644 nccl_test/demo2.yaml create mode 100644 nccl_test/sample.yaml create mode 100644 nccl_test/simple-test/run.sh create mode 100644 nccl_test/simple-test/template.yaml create mode 100644 nccl_test/simple-test/train_nccl.py create mode 100644 nccl_test/trainer/trainer.yaml create mode 100644 nerdctl_test/helper.yaml create mode 100644 nerdctl_test/single.yaml create mode 100644 network-operator/.helmignore create mode 100644 network-operator/Chart.lock create mode 100644 network-operator/Chart.yaml create mode 100644 network-operator/README.md create mode 100644 network-operator/charts/maintenance-operator-chart/.helmignore create mode 100644 network-operator/charts/maintenance-operator-chart/Chart.yaml create mode 100644 network-operator/charts/maintenance-operator-chart/README.md create mode 100644 network-operator/charts/maintenance-operator-chart/crds/maintenance.nvidia.com_maintenanceoperatorconfigs.yaml create mode 100644 network-operator/charts/maintenance-operator-chart/crds/maintenance.nvidia.com_nodemaintenances.yaml create mode 100644 network-operator/charts/maintenance-operator-chart/templates/_helpers.tpl create mode 100644 network-operator/charts/maintenance-operator-chart/templates/certificates.yaml create mode 100644 network-operator/charts/maintenance-operator-chart/templates/deployment.yaml create mode 100644 network-operator/charts/maintenance-operator-chart/templates/metrics-service.yaml create mode 100644 network-operator/charts/maintenance-operator-chart/templates/operatorconfig.yaml create mode 100644 network-operator/charts/maintenance-operator-chart/templates/role.yaml create mode 100644 network-operator/charts/maintenance-operator-chart/templates/role_binding.yaml create mode 100644 network-operator/charts/maintenance-operator-chart/templates/serviceaccount.yaml create mode 100644 network-operator/charts/maintenance-operator-chart/templates/webhook.yaml create mode 100644 network-operator/charts/maintenance-operator-chart/values.yaml create mode 100644 network-operator/charts/nic-configuration-operator-chart/.helmignore create mode 100644 network-operator/charts/nic-configuration-operator-chart/Chart.yaml create mode 100644 network-operator/charts/nic-configuration-operator-chart/README.md create mode 100644 network-operator/charts/nic-configuration-operator-chart/crds/configuration.net.nvidia.com_nicconfigurationtemplates.yaml create mode 100644 network-operator/charts/nic-configuration-operator-chart/crds/configuration.net.nvidia.com_nicdevices.yaml create mode 100644 network-operator/charts/nic-configuration-operator-chart/templates/_helpers.tpl create mode 100644 network-operator/charts/nic-configuration-operator-chart/templates/config-daemon.yaml create mode 100644 network-operator/charts/nic-configuration-operator-chart/templates/operator.yaml create mode 100644 network-operator/charts/nic-configuration-operator-chart/templates/role.yaml create mode 100644 network-operator/charts/nic-configuration-operator-chart/templates/role_binding.yaml create mode 100644 network-operator/charts/nic-configuration-operator-chart/templates/serviceaccount.yaml create mode 100644 network-operator/charts/nic-configuration-operator-chart/templates/supported-nic-firmware-configmap.yaml create mode 100644 network-operator/charts/nic-configuration-operator-chart/values.yaml create mode 100644 network-operator/charts/node-feature-discovery/.helmignore create mode 100644 network-operator/charts/node-feature-discovery/Chart.yaml create mode 100644 network-operator/charts/node-feature-discovery/README.md create mode 100644 network-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml create mode 100644 network-operator/charts/node-feature-discovery/templates/_helpers.tpl create mode 100644 network-operator/charts/node-feature-discovery/templates/clusterrole.yaml create mode 100644 network-operator/charts/node-feature-discovery/templates/clusterrolebinding.yaml create mode 100644 network-operator/charts/node-feature-discovery/templates/master.yaml create mode 100644 network-operator/charts/node-feature-discovery/templates/nfd-gc.yaml create mode 100644 network-operator/charts/node-feature-discovery/templates/nfd-master-conf.yaml create mode 100644 network-operator/charts/node-feature-discovery/templates/nfd-topologyupdater-conf.yaml create mode 100644 network-operator/charts/node-feature-discovery/templates/nfd-worker-conf.yaml create mode 100644 network-operator/charts/node-feature-discovery/templates/post-delete-job.yaml create mode 100644 network-operator/charts/node-feature-discovery/templates/prometheus.yaml create mode 100644 network-operator/charts/node-feature-discovery/templates/role.yaml create mode 100644 network-operator/charts/node-feature-discovery/templates/rolebinding.yaml create mode 100644 network-operator/charts/node-feature-discovery/templates/serviceaccount.yaml create mode 100644 network-operator/charts/node-feature-discovery/templates/topologyupdater-crds.yaml create mode 100644 network-operator/charts/node-feature-discovery/templates/topologyupdater.yaml create mode 100644 network-operator/charts/node-feature-discovery/templates/worker.yaml create mode 100644 network-operator/charts/node-feature-discovery/values.yaml create mode 100644 network-operator/charts/sriov-network-operator/.helmignore create mode 100644 network-operator/charts/sriov-network-operator/Chart.yaml create mode 100644 network-operator/charts/sriov-network-operator/README.md create mode 100644 network-operator/charts/sriov-network-operator/crds/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml create mode 100644 network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_ovsnetworks.yaml create mode 100644 network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovibnetworks.yaml create mode 100644 network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml create mode 100644 network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml create mode 100644 network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml create mode 100644 network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworks.yaml create mode 100644 network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml create mode 100644 network-operator/charts/sriov-network-operator/templates/NOTES.txt create mode 100644 network-operator/charts/sriov-network-operator/templates/_helpers.tpl create mode 100644 network-operator/charts/sriov-network-operator/templates/certificate.yaml create mode 100644 network-operator/charts/sriov-network-operator/templates/clusterrole.yaml create mode 100644 network-operator/charts/sriov-network-operator/templates/clusterrolebinding.yaml create mode 100644 network-operator/charts/sriov-network-operator/templates/configmap.yaml create mode 100644 network-operator/charts/sriov-network-operator/templates/extra-list.yaml create mode 100644 network-operator/charts/sriov-network-operator/templates/operator.yaml create mode 100644 network-operator/charts/sriov-network-operator/templates/pre-delete-webooks.yaml create mode 100644 network-operator/charts/sriov-network-operator/templates/role.yaml create mode 100644 network-operator/charts/sriov-network-operator/templates/rolebinding.yaml create mode 100644 network-operator/charts/sriov-network-operator/templates/serviceaccount.yaml create mode 100644 network-operator/charts/sriov-network-operator/templates/sriovoperatorconfig.yaml create mode 100644 network-operator/charts/sriov-network-operator/values.yaml create mode 100644 network-operator/crds/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml create mode 100644 network-operator/crds/mellanox.com_hostdevicenetworks.yaml create mode 100644 network-operator/crds/mellanox.com_ipoibnetworks.yaml create mode 100644 network-operator/crds/mellanox.com_macvlannetworks.yaml create mode 100644 network-operator/crds/mellanox.com_nicclusterpolicies.yaml create mode 100644 network-operator/hostdev/demo1.yaml create mode 100644 network-operator/hostdev/demo2.yaml create mode 100644 network-operator/hostdev/direct-rdma-1.yaml create mode 100644 network-operator/hostdev/direct-rdma-2.yaml create mode 100644 network-operator/hostdev/host-device-net.yaml create mode 100644 network-operator/hostdev/nicclusterpolicy-test.yaml create mode 100644 network-operator/hostdev/nicclusterpolicy.yaml create mode 100644 network-operator/hostdev/sample-pod.yaml create mode 100644 network-operator/ipiob/nicclusterpolicy.yaml create mode 100644 network-operator/sriov/demo1.yaml create mode 100644 network-operator/sriov/demo2.yaml create mode 100644 network-operator/sriov/nicclusterpolicy.yaml create mode 100644 network-operator/sriov/nodepolicy_gcore1.yaml create mode 100644 network-operator/sriov/nodepolicy_gcore2.yaml create mode 100644 network-operator/sriov/sriov_net.yaml create mode 100644 network-operator/sriov/test.log create mode 100644 network-operator/sriov/test2.log create mode 100644 network-operator/templates/NOTES.txt create mode 100644 network-operator/templates/_helpers.tpl create mode 100644 network-operator/templates/admission_controller.yaml create mode 100644 network-operator/templates/keep-ncp-hook.yaml create mode 100644 network-operator/templates/nodefeaturerules.yaml create mode 100644 network-operator/templates/operator.yaml create mode 100644 network-operator/templates/role.yaml create mode 100644 network-operator/templates/role_binding.yaml create mode 100644 network-operator/templates/service_account.yaml create mode 100644 network-operator/templates/upgrade-crd-hook.yaml create mode 100644 network-operator/values.yaml diff --git a/nccl_test/demo1.yaml b/nccl_test/demo1.yaml new file mode 100644 index 0000000..8f61e3d --- /dev/null +++ b/nccl_test/demo1.yaml @@ -0,0 +1,137 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + # 이 이름은 파드 YAML의 volumeMounts에서 참조해야 합니다. + name: ddp-test-script + namespace: default +data: + ddp_rdma_test.py: | + # 파일명: ddp_rdma_test.py + import os + import torch + import torch.distributed as dist + from torch.nn.parallel import DistributedDataParallel as DDP + + # torchrun에 의해 자동 주입되는 환경 변수들을 가져옵니다. + def setup(): + # 1. 환경 변수를 읽어 분산 통신 초기화 + # torchrun은 MASTER_ADDR, MASTER_PORT, RANK, WORLD_SIZE를 설정해줍니다. + + # 1.1. NCCL 백엔드 사용 + dist.init_process_group("nccl") + + # 1.2. torchrun이 설정한 로컬 랭크(해당 파드 내 GPU ID)를 가져옵니다. + # LOCAL_RANK 환경 변수는 torchrun에 의해 자동 설정됩니다. + local_rank = int(os.environ["LOCAL_RANK"]) + + # 1.3. 해당 프로세스에 GPU를 할당합니다. + torch.cuda.set_device(local_rank) + + return local_rank + + def run_ddp(): + local_rank = setup() + + # torchrun이 설정한 전체 정보를 가져옵니다. + global_rank = dist.get_rank() + world_size = dist.get_world_size() + # NODE_RANK는 파드 환경 변수로 설정되어야 합니다. + node_rank = int(os.environ.get("NODE_RANK", 0)) + + print(f"--- Process Initialized ---") + print(f"Global Rank: {global_rank} | Local Rank (GPU ID): {local_rank} | Node Rank: {node_rank}") + + # 간단한 모델 정의 및 GPU 로드 + model = torch.nn.Linear(10, 1).cuda() + ddp_model = DDP(model, device_ids=[local_rank]) + + # ---- RDMA 통신 검증을 위한 All-Reduce 연산 수행 ---- + tensor = torch.ones(10, dtype=torch.float32).cuda() + + # All-Reduce 실행: NCCL의 GPU Direct RDMA 기능이 활용됩니다. + dist.all_reduce(tensor, op=dist.ReduceOp.SUM) + + # ---------------------------------------------------- + + expected_sum = world_size * 10 + + # 결과 검증 및 출력 + if global_rank == 0: + print(f"\n--- Validation Check ---") + print(f"Total World Size: {world_size}") + + print(f"Global Rank {global_rank} / GPU {local_rank}: All-Reduce 결과 합계: {tensor.sum().item()}") + + # 모든 GPU에서 결과가 일치하는지 확인 + assert torch.allclose(tensor.sum().item(), torch.tensor(expected_sum, dtype=torch.float32).item()), "All-Reduce 결과 오류! 분산 통신 실패 또는 잘못된 계산." + + dist.destroy_process_group() + + if __name__ == '__main__': + # torchrun은 이 파일을 실행하며, 내부에서 run_ddp 로직을 반복 실행합니다. + run_ddp() +--- +apiVersion: v1 +kind: Pod +metadata: + name: nccl-gdrdma-pod-gcore-1 + annotations: + # 1. Multus CNI를 통해 'hostdevice-net' 네트워크에 연결합니다. + k8s.v1.cni.cncf.io/networks: hostdevice-net + # 2. 파드를 Node 0에 강제 배포합니다. (실제 클러스터에서는 스케줄러가 처리) + scheduler.alpha.kubernetes.io/node-selector: kubernetes.io/hostname=nhn-aideveloper-d40ce770 +spec: + # serviceAccountName: <적절한 SA 이름> # (필요시) + containers: + - name: gdrdma-client + # NVIDIA 공식 PyTorch 이미지 사용 + image: nvcr.io/nvidia/pytorch:24.03-py3 + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-c"] + args: + - | + # 1. RDMA 디바이스 정보 확인 및 테스트 도구 설치 (검증을 위해) + apt update && apt install -y rdma-core perftest + + # 2. 분산 학습 환경 변수 설정 + export NCCL_DEBUG=INFO # NCCL 디버그 레벨 (RDMA 사용 검증) + export NCCL_DEBUG_SUBSYS=NET # 네트워크 통신 계층 디버깅 활성화 + export MASTER_ADDR=192.168.240.1 # Node 0 파드의 hostdevice-net IP + export MASTER_PORT=29500 + export WORLD_SIZE=16 # 총 GPU 개수 (8 * 2) + export NODE_RANK=0 # 현재 파드의 노드 순위 (Node 0) + export LOCAL_WORLD_SIZE=8 # 이 파드에 할당된 GPU 개수 + + # 3. ddp_rdma_test.py 스크립트 실행 (미리 파드에 복사되어 있어야 함) + # torch.distributed.launch 또는 torchrun을 사용하여 8개 프로세스 실행 + torchrun \ + --nnodes=$((WORLD_SIZE / LOCAL_WORLD_SIZE)) \ + --nproc_per_node=$LOCAL_WORLD_SIZE \ + --rdzv_id=100 \ + --rdzv_backend=c10d \ + --rdzv_endpoint=$MASTER_ADDR:$MASTER_PORT \ + ddp_rdma_test.py + + # 3. 중요: torchrun 실행 대신, 무한 대기합니다. + echo "Setup complete. Pod is now in a waiting state. Use 'kubectl exec' to run torchrun." + sleep infinity # <-- 이 명령어가 컨테이너가 종료되지 않도록 유지합니다. + + securityContext: + capabilities: + add: ["IPC_LOCK"] # GPU Direct RDMA를 위한 필수 설정 + resources: + limits: + # GPU 8개와 Host Device NIC 1개를 요청합니다. + # 이 Host Device는 SR-IOV Device Plugin에 의해 RDMA 기능이 활성화됩니다. + nvidia.com/gpu: "8" + nvidia.com/hostdev: "8" + requests: + nvidia.com/gpu: "8" + nvidia.com/hostdev: "8" + volumeMounts: + - name: ddp-script-volume + mountPath: /app + volumes: + - name: ddp-script-volume + configMap: + name: ddp-test-script diff --git a/nccl_test/demo2.yaml b/nccl_test/demo2.yaml new file mode 100644 index 0000000..007ea14 --- /dev/null +++ b/nccl_test/demo2.yaml @@ -0,0 +1,65 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nccl-gdrdma-pod-gcore-2 + annotations: + # 1. Multus CNI를 통해 'hostdevice-net' 네트워크에 연결합니다. + k8s.v1.cni.cncf.io/networks: hostdevice-net + # 2. 파드를 Node 0에 강제 배포합니다. (실제 클러스터에서는 스케줄러가 처리) + scheduler.alpha.kubernetes.io/node-selector: kubernetes.io/hostname=nhn-aideveloper-f0fde370 +spec: + # serviceAccountName: <적절한 SA 이름> # (필요시) + containers: + - name: gdrdma-client + # NVIDIA 공식 PyTorch 이미지 사용 + image: nvcr.io/nvidia/pytorch:24.03-py3 + imagePullPolicy: IfNotPresent + command: ["/bin/bash", "-c"] + args: + - | + # 1. RDMA 디바이스 정보 확인 및 테스트 도구 설치 (검증을 위해) + apt update && apt install -y rdma-core perftest + + # 2. 분산 학습 환경 변수 설정 + export NCCL_DEBUG=INFO # NCCL 디버그 레벨 (RDMA 사용 검증) + export NCCL_DEBUG_SUBSYS=NET # 네트워크 통신 계층 디버깅 활성화 + export MASTER_ADDR=192.168.240.1 # Node 0 파드의 hostdevice-net IP + export MASTER_PORT=29500 + export WORLD_SIZE=16 # 총 GPU 개수 (8 * 2) + export NODE_RANK=1 # 현재 파드의 노드 순위 (Node 0) + export LOCAL_WORLD_SIZE=8 # 이 파드에 할당된 GPU 개수 + + # 3. ddp_rdma_test.py 스크립트 실행 (미리 파드에 복사되어 있어야 함) + # torch.distributed.launch 또는 torchrun을 사용하여 8개 프로세스 실행 + torchrun \ + --nnodes=$((WORLD_SIZE / LOCAL_WORLD_SIZE)) \ + --nproc_per_node=$LOCAL_WORLD_SIZE \ + --rdzv_id=100 \ + --rdzv_backend=c10d \ + --rdzv_endpoint=$MASTER_ADDR:$MASTER_PORT \ + ddp_rdma_test.py + + # 3. 중요: torchrun 실행 대신, 무한 대기합니다. + echo "Setup complete. Pod is now in a waiting state. Use 'kubectl exec' to run torchrun." + sleep infinity # <-- 이 명령어가 컨테이너가 종료되지 않도록 유지합니다. + + securityContext: + capabilities: + add: ["IPC_LOCK"] # GPU Direct RDMA를 위한 필수 설정 + resources: + limits: + # GPU 8개와 Host Device NIC 1개를 요청합니다. + # 이 Host Device는 SR-IOV Device Plugin에 의해 RDMA 기능이 활성화됩니다. + nvidia.com/gpu: "8" + nvidia.com/hostdev: "8" + requests: + nvidia.com/gpu: "8" + nvidia.com/hostdev: "8" + volumeMounts: + - name: ddp-script-volume + mountPath: /app + volumes: + - name: ddp-script-volume + configMap: + name: ddp-test-script + diff --git a/nccl_test/sample.yaml b/nccl_test/sample.yaml new file mode 100644 index 0000000..9bda473 --- /dev/null +++ b/nccl_test/sample.yaml @@ -0,0 +1,169 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nccl-scripts +data: + run.sh: | + #!/usr/bin/env bash + set -euo pipefail + + # ----- 기본 분산 설정 ----- + export MASTER_ADDR="${MASTER_ADDR:-nccl-0.nccl}" # 첫 파드 + export MASTER_PORT="${MASTER_PORT:-23456}" + export WORLD_SIZE="${WORLD_SIZE:-2}" # 파드 수 + export NUM_GPUS="${NUM_GPUS:-1}" # 파드당 프로세스(=GPU) 수 + + # 파드 이름이 nccl-0, nccl-1 ... 이므로, 끝의 ordinal을 RANK로 사용 + ordinal="$(hostname | awk -F- '{print $NF}')" + export RANK="${RANK:-$ordinal}" + + # ----- (선택) 특정 GPU만 사용하고 싶다면: 2번 GPU만 보이도록 고정 ----- + # 예: kubectl patch 로 아래 값을 "2"로 넣으면 그 노드의 물리 GPU #2만 노출됨 + if [[ -n "${CUDA_VISIBLE_DEVICES_OVERRIDE:-}" ]]; then + export CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES_OVERRIDE}" + fi + + # ----- RDMA/IB 강제 ----- + export NCCL_DEBUG=INFO + export NCCL_DEBUG_SUBSYS=INIT,NET + export NCCL_NET=IB + export NCCL_IB_DISABLE=0 + export NCCL_SOCKET_IFNAME="${NCCL_SOCKET_IFNAME:-net1}" # HostDeviceNetwork로 붙인 NIC 이름 + export NCCL_IB_HCA="${NCCL_IB_HCA:-mlx5*}" + # (RoCE 환경에 따라 필요 시) export NCCL_IB_GID_INDEX=3 + + # 프로세스별 로그 파일 경로 (노드/프로세스마다 별도 파일) + export NCCL_DEBUG_FILE="/tmp/nccl-%h-%p.log" + + # ----- 학습 실행 (각 파드가 1개 프로세스씩 구동) ----- + echo "[RUN] RANK=$RANK WORLD_SIZE=$WORLD_SIZE on $(hostname)" + torchrun \ + --nnodes="${WORLD_SIZE}" \ + --nproc_per_node="${NUM_GPUS}" \ + --rdzv_backend=c10d \ + --rdzv_endpoint="${MASTER_ADDR}:${MASTER_PORT}" \ + /workspace/train_nccl.py 2>&1 | tee /tmp/nccl-aggregate.log + + # ----- RDMA 사용 여부 집계 ----- + total=0; ok=0 + for f in /tmp/nccl-*.log; do + [[ -f "$f" ]] || continue + total=$((total+1)) + if grep -q "NET/IB" "$f"; then + ok=$((ok+1)) + fi + done + echo "[RDMA-CHECK] used IB on $ok / $total local ranks (processes) in $(hostname)" + + # 모든 랭크가 IB를 썼는지(파드 단위) 판단: 로컬 프로세스 수와 비교 + if [[ "$ok" -eq "$NUM_GPUS" ]]; then + echo "[RDMA-CHECK] ✅ RDMA path OK for all local GPUs" + exit 0 + else + echo "[RDMA-CHECK] ❌ RDMA path NOT used by all local GPUs" + echo "---- NCCL NET/IB related lines ----" + grep -h "NET/" /tmp/nccl-*.log || true + exit 1 + fi + train_nccl.py: | + import os, time, torch + import torch.distributed as dist + + def main(): + # 분산 초기화 + rank = int(os.getenv("RANK", "0")) + world_size = int(os.getenv("WORLD_SIZE", "1")) + local_rank = int(os.getenv("LOCAL_RANK", "0")) # torchrun이 설정 + + # (선택) 특정 GPU 인덱스로 고정하고 싶을 때: LOCAL_RANK 대신 고정값 사용 + device_index = int(os.getenv("GPU_INDEX", str(local_rank))) + torch.cuda.set_device(device_index) + + dist.init_process_group( + backend="nccl", + init_method="env://", + world_size=world_size, + rank=rank, + timeout=torch.distributed.constants.default_pg_timeout + ) + + # 더미 학습 스텝: all-reduce로 통신 발생 + x = torch.ones(8 * 1024 * 1024, device=device_index, dtype=torch.float32) # ~32MB + for i in range(5): + dist.all_reduce(x, op=dist.ReduceOp.SUM) + torch.cuda.synchronize() + if rank == 0: + print(f"[STEP {i}] all_reduce done; tensor sum={x[0].item():.1f}") + + # 약간 대기해서 로그 flush + time.sleep(2.0) + dist.destroy_process_group() + + if __name__ == "__main__": + main() +--- +apiVersion: v1 +kind: Service +metadata: + name: nccl + labels: { app: nccl } +spec: + clusterIP: None # Headless + selector: { app: nccl } + ports: + - name: rdv + port: 23456 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nccl +spec: + serviceName: nccl + replicas: 2 # >=2 노드에서 스케줄되도록 노드 리소스 준비 필요 + selector: + matchLabels: { app: nccl } + template: + metadata: + labels: { app: nccl } + annotations: + k8s.v1.cni.cncf.io/networks: hostdevice-net # HostDeviceNetwork (RDMA) + spec: + # RDMA 장치나 /dev/infiniband 노출이 안 보이면 네트워크 오퍼레이터의 RDMA 플러그인 구성을 확인하세요. + # (환경에 따라 RDMA Shared Device Plugin이 필요할 수 있음) + containers: + - name: worker + image: nvcr.io/nvidia/pytorch:24.10-py3 + imagePullPolicy: IfNotPresent + securityContext: + capabilities: { add: ["IPC_LOCK"] } + command: [ "bash", "-lc", "cp /config_scripts/* /workspace/ && chmod +x /workspace/run.sh && /workspace/run.sh || sleep infinity" ] + env: + # --- (선택) 특정 GPU만 사용하려면 아래 값을 물리 인덱스로 설정 (예: "2") --- + # - name: CUDA_VISIBLE_DEVICES_OVERRIDE + # value: "2" + - name: WORLD_SIZE + value: "2" # StatefulSet replicas와 동일 + - name: NUM_GPUS + value: "8" # 파드당 사용할 GPU 개수 + - name: MASTER_ADDR + value: "nccl-0.nccl" # 첫 파드의 DNS 이름 + - name: MASTER_PORT + value: "23456" + # RDMA 관련 기본값은 run.sh에서 설정 + resources: + limits: + nvidia.com/gpu: "8" + nvidia.com/hostdev: "8" # SR-IOV Device Plugin이 광고한 RDMA NIC 리소스 + requests: + nvidia.com/gpu: "8" + nvidia.com/hostdev: "8" + volumeMounts: + - name: scripts + mountPath: /config_scripts + volumes: + - name: scripts + configMap: + name: nccl-scripts + defaultMode: 0755 + diff --git a/nccl_test/simple-test/run.sh b/nccl_test/simple-test/run.sh new file mode 100644 index 0000000..8cabea6 --- /dev/null +++ b/nccl_test/simple-test/run.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -ex + +# 현재 실행 시작 시 타임스탬프 저장 +START_TIME=$(date +%s) + +# ----- RDMA/IB 강제 ----- +export NCCL_DEBUG=INFO +export NCCL_DEBUG_SUBSYS=INIT,NET,IB +#export NCCL_NET=IB +#export NCCL_IB_DISABLE=0 +export NCCL_SOCKET_IFNAME="${NCCL_SOCKET_IFNAME:-net1}" # HostDeviceNetwork로 붙인 NIC 이름 +export NCCL_IB_HCA="mlx5_0,mlx5_1,mlx5_2,mlx5_8,mlx5_9,mlx5_5,mlx5_6,mlx5_7" +export NCCL_NET_GDR_LEVEL=2 +# (RoCE 환경에 따라 필요 시) export NCCL_IB_GID_INDEX=3 + +# 프로세스별 로그 파일 경로 (노드/프로세스마다 별도 파일) +export NCCL_DEBUG_FILE="/tmp/nccl-%h-%p.log" + +# ----- 학습 실행 (각 파드가 1개 프로세스씩 구동) ----- +echo "[RUN] RANK=$RANK on $(hostname)" +torchrun \ + --nnodes=2 \ + --nproc_per_node=8 \ + --rdzv_backend=c10d \ + --node_rank=0 \ + --rdzv_endpoint=nccl-0.nccl:23456 \ + /workspace/train_nccl.py 2>&1 | tee /tmp/nccl-aggregate.log + +# ----- RDMA 사용 여부 집계 ----- + +# 새로 생성된 로그 목록 가져오기: START_TIME 이후 생성된 파일만 +new_logs=$(find /tmp -name "nccl-nccl-*.log" -type f -newermt "@${START_TIME}") + +total=0; ok=0 +for f in $new_logs; do + [[ -f "$f" ]] || continue + total=$((total+1)) + if rdma_line=$(grep -E "Channel .*GDRDMA" "$f"); then + echo "[OK] RDMA: $rdma_line" + ok=$((ok+1)) + fi +done +echo "[RDMA-CHECK] used IB on $ok / $total local ranks (processes) in $(hostname)" + +# 모든 랭크가 IB를 썼는지(파드 단위) 판단: 로컬 프로세스 수와 비교 +if [[ "$ok" -eq 8 ]]; then + echo "[RDMA-CHECK] ✅ RDMA path OK for all local GPUs" + exit 0 +else + echo "[RDMA-CHECK] ❌ RDMA path NOT used by all local GPUs" + echo "---- NCCL NET/IB related lines ----" + grep -h "NET/" /tmp/nccl-*.log || true + exit 1 +fi diff --git a/nccl_test/simple-test/template.yaml b/nccl_test/simple-test/template.yaml new file mode 100644 index 0000000..a1417bf --- /dev/null +++ b/nccl_test/simple-test/template.yaml @@ -0,0 +1,152 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nccl-scripts +data: + run.sh: | + + #!/usr/bin/env bash + set -ex + + # 현재 실행 시작 시 타임스탬프 저장 + START_TIME=$(date +%s) + + # ----- RDMA/IB 강제 ----- + export NCCL_DEBUG=INFO + export NCCL_DEBUG_SUBSYS=INIT,NET,IB + #export NCCL_NET=IB + #export NCCL_IB_DISABLE=0 + export NCCL_SOCKET_IFNAME="${NCCL_SOCKET_IFNAME:-net1}" # HostDeviceNetwork로 붙인 NIC 이름 + export NCCL_IB_HCA="mlx5_0,mlx5_1,mlx5_2,mlx5_8,mlx5_9,mlx5_5,mlx5_6,mlx5_7" + export NCCL_NET_GDR_LEVEL=2 + # (RoCE 환경에 따라 필요 시) export NCCL_IB_GID_INDEX=3 + + # 프로세스별 로그 파일 경로 (노드/프로세스마다 별도 파일) + export NCCL_DEBUG_FILE="/tmp/nccl-%h-%p.log" + + # ----- 학습 실행 (각 파드가 1개 프로세스씩 구동) ----- + echo "[RUN] RANK=$RANK on $(hostname)" + torchrun \ + --nnodes=2 \ + --nproc_per_node=8 \ + --rdzv_backend=c10d \ + --node_rank=0 \ + --rdzv_endpoint=nccl-0.nccl:23456 \ + /workspace/train_nccl.py 2>&1 | tee /tmp/nccl-aggregate.log + + # ----- RDMA 사용 여부 집계 ----- + + # 새로 생성된 로그 목록 가져오기: START_TIME 이후 생성된 파일만 + new_logs=$(find /tmp -name "nccl-nccl-*.log" -type f -newermt "@${START_TIME}") + + total=0; ok=0 + for f in $new_logs; do + [[ -f "$f" ]] || continue + total=$((total+1)) + if rdma_line=$(grep -E "Channel .*GDRDMA" "$f"); then + echo "[OK] RDMA: $rdma_line" + ok=$((ok+1)) + fi + done + echo "[RDMA-CHECK] used IB on $ok / $total local ranks (processes) in $(hostname)" + + # 모든 랭크가 IB를 썼는지(파드 단위) 판단: 로컬 프로세스 수와 비교 + if [[ "$ok" -eq 8 ]]; then + echo "[RDMA-CHECK] ✅ RDMA path OK for all local GPUs" + exit 0 + else + echo "[RDMA-CHECK] ❌ RDMA path NOT used by all local GPUs" + echo "---- NCCL NET/IB related lines ----" + grep -h "NET/" /tmp/nccl-*.log || true + exit 1 + fi + train_nccl.py: | + import os + import torch + import torch.distributed as dist + + def main(): + dist.init_process_group("nccl") + local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local_rank) + + print(f"Rank {dist.get_rank()} initialized on GPU {local_rank}") + + x = torch.ones(10).cuda() + dist.all_reduce(x) + print(f"Rank {dist.get_rank()} result: {x[0].item()}") + + if __name__ == "__main__": + main() +--- +apiVersion: v1 +kind: Service +metadata: + name: nccl + labels: { app: nccl } +spec: + clusterIP: None # Headless + selector: { app: nccl } + ports: + - name: rdv + port: 23456 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: nccl +spec: + serviceName: nccl + replicas: 2 # >=2 노드에서 스케줄되도록 노드 리소스 준비 필요 + selector: + matchLabels: { app: nccl } + template: + metadata: + labels: { app: nccl } + annotations: + k8s.v1.cni.cncf.io/networks: hostdevice-net # HostDeviceNetwork (RDMA) + spec: +# hostNetwork: true +# dnsPolicy: ClusterFirstWithHostNet + # RDMA 장치나 /dev/infiniband 노출이 안 보이면 네트워크 오퍼레이터의 RDMA 플러그인 구성을 확인하세요. + # (환경에 따라 RDMA Shared Device Plugin이 필요할 수 있음) + containers: + - name: worker + image: nvcr.io/nvidia/pytorch:24.10-py3 + imagePullPolicy: IfNotPresent + securityContext: + privileged: true + capabilities: + add: + - IPC_LOCK + - NET_ADMIN +# capabilities: { add: ["IPC_LOCK"] } + command: [ "bash", "-lc", "cp /config_scripts/* /workspace/ && chmod +x /workspace/run.sh && sleep infinity" ] + env: + # --- (선택) 특정 GPU만 사용하려면 아래 값을 물리 인덱스로 설정 (예: "2") --- + # - name: CUDA_VISIBLE_DEVICES_OVERRIDE + # value: "2" + # RDMA 관련 기본값은 run.sh에서 설정 + resources: + limits: + nvidia.com/gpu: "8" + nvidia.com/hostdev: "8" # SR-IOV Device Plugin이 광고한 RDMA NIC 리소스 + requests: + nvidia.com/gpu: "8" + nvidia.com/hostdev: "8" + volumeMounts: + - name: scripts + mountPath: /config_scripts + - name: shared-memory + mountPath: /dev/shm + volumes: + - name: scripts + configMap: + name: nccl-scripts + defaultMode: 0755 + - name: shared-memory + emptyDir: + medium: Memory + sizeLimit: 118541097369600m + + diff --git a/nccl_test/simple-test/train_nccl.py b/nccl_test/simple-test/train_nccl.py new file mode 100644 index 0000000..0e6a7c8 --- /dev/null +++ b/nccl_test/simple-test/train_nccl.py @@ -0,0 +1,18 @@ +import os +import torch +import torch.distributed as dist + +def main(): + dist.init_process_group("nccl") + local_rank = int(os.environ["LOCAL_RANK"]) + torch.cuda.set_device(local_rank) + + print(f"Rank {dist.get_rank()} initialized on GPU {local_rank}") + + x = torch.ones(10).cuda() + dist.all_reduce(x) + print(f"Rank {dist.get_rank()} result: {x[0].item()}") + +if __name__ == "__main__": + main() + diff --git a/nccl_test/trainer/trainer.yaml b/nccl_test/trainer/trainer.yaml new file mode 100644 index 0000000..49a4010 --- /dev/null +++ b/nccl_test/trainer/trainer.yaml @@ -0,0 +1,187 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: pytorch-train-script +data: + train.py: | + import os + import time + import torch + import torch.distributed as dist + + def main(): + rank = int(os.environ["RANK"]) + world_size = int(os.environ["WORLD_SIZE"]) + + print(f"[Rank {rank}] Starting process_group init...", flush=True) + + dist.init_process_group( + backend="nccl", + init_method="env://", + rank=rank, + world_size=world_size + ) + + print(f"[Rank {rank}] process_group initialized!", flush=True) + + # GPU 텐서 생성 + torch.cuda.set_device(rank % torch.cuda.device_count()) + tensor = torch.ones(1).cuda() + + # NCCL broadcast test + print(f"[Rank {rank}] tensor before: {tensor}", flush=True) + + dist.broadcast(tensor, src=0) + + print(f"[Rank {rank}] tensor after broadcast: {tensor}", flush=True) + + # 종료되지 않도록 대기 + print(f"[Rank {rank}] Sleeping for 30 seconds so logs can be seen...", flush=True) + time.sleep(600) + + if __name__ == "__main__": + main() + +--- +apiVersion: kubeflow.org/v1 +kind: PyTorchJob +metadata: + name: pytorch-multinode-rdma +spec: + runPolicy: + cleanPodPolicy: None + + pytorchReplicaSpecs: + Master: + replicas: 1 # 노드 수 = 2 (원하면 증가 가능) + restartPolicy: OnFailure + template: + metadata: + annotations: + k8s.v1.cni.cncf.io/networks: hostdevice-net + spec: + nodeSelector: + nodegroup: infini + initContainers: + - name: copy-train-script + image: busybox + command: + - sh + - -c + - | + echo "Copying train.py to workspace..." + cp /config/train.py /workspace/train.py + chmod +x /workspace/train.py + volumeMounts: + - name: train-config + mountPath: /config + - name: workspace + mountPath: /workspace + containers: + - name: pytorch + image: nvcr.io/nvidia/pytorch:24.01-py3 # RDMA 지원되는 NVIDIA PT 이미지 권장 + imagePullPolicy: IfNotPresent + command: + - "python" + - "/workspace/train.py" + env: + # --- PyTorch DDP --- + - name: MASTER_ADDR + value: "pytorch-multinode-rdma-worker-0" + - name: MASTER_PORT + value: "23456" + + # --- NCCL RDMA 설정 --- + - name: NCCL_DEBUG + value: "INFO" + - name: NCCL_IB_HCA + value: "mlx5_*" + - name: NCCL_IB_DISABLE + value: "0" + - name: NCCL_NET_GDR_LEVEL + value: "2" + - name: NCCL_SOCKET_IFNAME + value: "net1" # Multus의 RDMA 인터페이스를 여기에 넣어도 됨 (ipoib: ib0) + - name: NCCL_IB_PCI_RELAXED_ORDERING + value: "1" + + resources: + limits: + nvidia.com/gpu: 8 + nvidia.com/hostdev: "8" # 또는 rdma/hca: 8 (리소스 명칭에 따름) + volumeMounts: + - name: workspace + mountPath: /workspace + volumes: + - name: train-config # ConfigMap (read-only) + configMap: + name: pytorch-train-script + - name: workspace # EmptyDir (read-write) + emptyDir: {} + Worker: + replicas: 1 # 노드 수 = 2 (원하면 증가 가능) + restartPolicy: OnFailure + template: + metadata: + annotations: + k8s.v1.cni.cncf.io/networks: hostdevice-net + spec: + nodeSelector: + nodegroup: infini + initContainers: + - name: copy-train-script + image: busybox + command: + - sh + - -c + - | + echo "Copying train.py to workspace..." + cp /config/train.py /workspace/train.py + chmod +x /workspace/train.py + volumeMounts: + - name: train-config + mountPath: /config + - name: workspace + mountPath: /workspace + containers: + - name: pytorch + image: nvcr.io/nvidia/pytorch:24.01-py3 # RDMA 지원되는 NVIDIA PT 이미지 권장 + imagePullPolicy: IfNotPresent + command: + - "python" + - "/workspace/train.py" + env: + # --- PyTorch DDP --- + - name: MASTER_ADDR + value: "pytorch-multinode-rdma-worker-0" + - name: MASTER_PORT + value: "23456" + + # --- NCCL RDMA 설정 --- + - name: NCCL_DEBUG + value: "INFO" + - name: NCCL_IB_HCA + value: "mlx5_*" + - name: NCCL_IB_DISABLE + value: "0" + - name: NCCL_NET_GDR_LEVEL + value: "2" + - name: NCCL_SOCKET_IFNAME + value: "eth0" # Multus의 RDMA 인터페이스를 여기에 넣어도 됨 (ipoib: ib0) + - name: NCCL_IB_PCI_RELAXED_ORDERING + value: "1" + + resources: + limits: + nvidia.com/gpu: 8 + nvidia.com/hostdev: "8" # 또는 rdma/hca: 8 (리소스 명칭에 따름) + volumeMounts: + - name: workspace + mountPath: /workspace + volumes: + - name: train-config # ConfigMap (read-only) + configMap: + name: pytorch-train-script + - name: workspace # EmptyDir (read-write) + emptyDir: {} + diff --git a/nerdctl_test/helper.yaml b/nerdctl_test/helper.yaml new file mode 100644 index 0000000..9544d4a --- /dev/null +++ b/nerdctl_test/helper.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: Pod +metadata: + name: nerdctl-helper + namespace: default +spec: + nodeSelector: + kubernetes.io/hostname: v1002 + + hostPID: true + hostNetwork: true + containers: + - name: nerdctl + image: alpine:3.20 + securityContext: + privileged: true + command: + - /bin/sh + - -c + - | + apk add --no-cache bash curl nerdctl + echo "nerdctl ready. Use 'kubectl exec -it nerdctl-helper -- bash'" + sleep infinity + volumeMounts: + - name: containerd-sock + mountPath: /run/containerd/containerd.sock + restartPolicy: Never + volumes: + - name: containerd-sock + hostPath: + path: /run/containerd/containerd.sock + type: Socket diff --git a/nerdctl_test/single.yaml b/nerdctl_test/single.yaml new file mode 100644 index 0000000..e801da4 --- /dev/null +++ b/nerdctl_test/single.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + name: ssh-client + namespace: default +spec: + nodeSelector: + nodegroup: svc + containers: + - name: ssh-client + image: alpine:3.20 + command: + - /bin/sh + - -c + - | + apk add --no-cache openssh-client bash + echo "Pod ready. Use 'kubectl exec -it ssh-client -- bash' to enter." + sleep infinity + restartPolicy: Never diff --git a/network-operator/.helmignore b/network-operator/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/network-operator/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/network-operator/Chart.lock b/network-operator/Chart.lock new file mode 100644 index 0000000..1c42dd1 --- /dev/null +++ b/network-operator/Chart.lock @@ -0,0 +1,15 @@ +dependencies: +- name: node-feature-discovery + repository: http://kubernetes-sigs.github.io/node-feature-discovery/charts + version: 0.17.0 +- name: sriov-network-operator + repository: "" + version: 0.1.0 +- name: nic-configuration-operator-chart + repository: "" + version: 0.0.1 +- name: maintenance-operator-chart + repository: "" + version: 0.0.1 +digest: sha256:f1908d13fa2d7e56ffc4441f1bc34f1111d50b88c6e914f1d4ca5d479b8b43c6 +generated: "2025-02-26T19:04:38.740888136+02:00" diff --git a/network-operator/Chart.yaml b/network-operator/Chart.yaml new file mode 100644 index 0000000..705860c --- /dev/null +++ b/network-operator/Chart.yaml @@ -0,0 +1,30 @@ +apiVersion: v2 +appVersion: v25.1.0 +dependencies: +- condition: nfd.enabled + name: node-feature-discovery + repository: http://kubernetes-sigs.github.io/node-feature-discovery/charts + version: 0.17.0 +- condition: sriovNetworkOperator.enabled + name: sriov-network-operator + repository: "" + version: 0.1.0 +- condition: nicConfigurationOperator.enabled + name: nic-configuration-operator-chart + repository: "" + version: 0.0.1 +- condition: maintenanceOperator.enabled + name: maintenance-operator-chart + repository: "" + version: 0.0.1 +description: Nvidia network operator +home: https://mellanox.github.io/network-operator +keywords: +- gpu-direct +- rdma +kubeVersion: '>= 1.21.0' +name: network-operator +sources: +- https://github.com/Mellanox/network-operator +type: application +version: 25.1.0 diff --git a/network-operator/README.md b/network-operator/README.md new file mode 100644 index 0000000..5504351 --- /dev/null +++ b/network-operator/README.md @@ -0,0 +1,243 @@ +# Nvidia Network Operator Helm Chart + +Nvidia Network Operator Helm Chart provides an easy way to install and manage the lifecycle of Nvidia network operator. + +## Nvidia Network Operator + +Nvidia Network Operator +leverages [Kubernetes CRDs](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) +and [Operator SDK](https://github.com/operator-framework/operator-sdk) to manage Networking related Components in order +to enable Fast networking, RDMA and GPUDirect for workloads in a Kubernetes cluster. Network Operator works in +conjunction with [GPU-Operator](https://github.com/NVIDIA/gpu-operator) to enable GPU-Direct RDMA on compatible systems. + +The Goal of Network Operator is to manage _all_ networking related components to enable execution of RDMA and GPUDirect +RDMA workloads in a kubernetes cluster including: + +* Mellanox Networking drivers to enable advanced features +* Kubernetes device plugins to provide hardware resources for fast network +* Kubernetes secondary network for Network intensive workloads + +### Documentation + +For more information please visit the official [documentation](https://docs.nvidia.com/networking/software/cloud-orchestration/index.html). + +## Additional components + +### Node Feature Discovery + +Nvidia Network Operator relies on the existance of specific node labels to operate properly. e.g label a node as having +Nvidia networking hardware available. This can be achieved by either manually labeling Kubernetes nodes or using +[Node Feature Discovery](https://github.com/kubernetes-sigs/node-feature-discovery) to perform the labeling. + +To allow zero touch deployment of the Operator we provide a helm chart to be used to optionally deploy Node Feature +Discovery in the cluster. This is enabled via `nfd.enabled` chart parameter. + +### SR-IOV Network Operator + +Nvidia Network Operator can operate in unison with SR-IOV Network Operator to enable SR-IOV workloads in a Kubernetes +cluster. We provide a helm chart to be used to optionally +deploy [SR-IOV Network Operator](https://github.com/k8snetworkplumbingwg/sriov-network-operator) in the cluster. This is +enabled via `sriovNetworkOperator.enabled` chart parameter. + +SR-IOV Network Operator can work in conjuction with [IB Kubernetes](#ib-kubernetes) to use InfiniBand PKEY Membership +Types + +For more information on how to configure SR-IOV in your Kubernetes cluster using SR-IOV Network Operator refer to the +project's github. + +## QuickStart + +### System Requirements + +* RDMA capable hardware: Mellanox ConnectX-5 NIC or newer. +* NVIDIA GPU and driver supporting GPUDirect e.g Quadro RTX 6000/8000 or Tesla T4 or Tesla V100 or Tesla V100. + (GPU-Direct only) +* Operating Systems: Ubuntu 20.04 LTS. + +> __NOTE__: ConnectX-6 Lx is not supported. + +### Tested Network Adapters + +The following Network Adapters have been tested with network-operator: + +* ConnectX-5 +* ConnectX-6 Dx + +### Prerequisites + +- Kubernetes v1.17+ +- Helm v3.5.3+ +- Ubuntu 20.04 LTS + +### Install Helm + +Helm provides an install script to copy helm binary to your system: + +``` +$ curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 +$ chmod 500 get_helm.sh +$ ./get_helm.sh +``` + +For additional information and methods for installing Helm, refer to the official [helm website](https://helm.sh/) + +### Deploy Network Operator + +``` +# Add Repo +$ helm repo add nvidia https://helm.ngc.nvidia.com/nvidia +$ helm repo update + +# Install Operator +$ helm install -n network-operator --create-namespace --wait network-operator nvidia/network-operator + +# View deployed resources +$ kubectl -n network-operator get pods +``` + +#### Deploy Network Operator without Node Feature Discovery + +By default the network operator +deploys [Node Feature Discovery (NFD)](https://github.com/kubernetes-sigs/node-feature-discovery) +in order to perform node labeling in the cluster to allow proper scheduling of Network Operator resources. If the nodes +where already labeled by other means (either deployed from upstream or deployed within another deployment), it is possible to disable the deployment of NFD by setting +`nfd.enabled=false` chart parameter and make sure that the installed version is `v0.13.2` or newer and has NodeFeatureApi enabled. + +##### Deploy NFD from upstream with NodeFeatureApi enabled +``` +$ export NFD_NS=node-feature-discovery +$ helm repo add nfd https://kubernetes-sigs.github.io/node-feature-discovery/charts +$ helm repo update +$ helm install nfd/node-feature-discovery --namespace $NFD_NS --create-namespace --generate-name --set enableNodeFeatureApi='true' +``` +For additional information , refer to the official [NVD deployment with Helm](https://kubernetes-sigs.github.io/node-feature-discovery/v0.13/deployment/helm.html) + +##### Deploy Network Operator without Node Feature Discovery +``` +$ helm install --set nfd.enabled=false -n network-operator --create-namespace --wait network-operator nvidia/network-operator +``` + +##### Currently the following NFD labels are used: + +| Label | Where | +|-----------------------------------------------|---------------------------------------------------| +| `feature.node.kubernetes.io/pci-15b3.present` | Nodes bearing Nvidia Mellanox Networking hardware | +| `nvidia.com/gpu.present` | Nodes bearing Nvidia GPU hardware | + +> __Note:__ The labels which Network Operator depends on may change between releases. + +> __Note:__ By default the operator is deployed without an instance of `NicClusterPolicy` and `MacvlanNetwork` +> custom resources. The user is required to create it later with configuration matching the cluster or use chart parameters to deploy it together with the operator. + +#### Deploy development version of Network Operator + +To install development version of Network Operator you need to clone repository first and install helm chart from the +local directory: + +``` +# Clone Network Operator Repository +$ git clone https://github.com/Mellanox/network-operator.git + +# Update chart dependencies +$ cd network-operator/deployment/network-operator && helm dependency update + +# Install Operator +$ helm install -n network-operator --create-namespace --wait network-operator ./ + +# View deployed resources +$ kubectl -n network-operator get pods +``` + +#### Deploy Network Operator with Admission Controller + +The Admission Controller can be optionally included as part of the Network Operator installation process. +It has the capability to validate supported Custom Resource Definitions (CRDs), which currently include NicClusterPolicy and HostDeviceNetwork. +By default, the deployment of the admission controller is disabled. To enable it, you must set `operator.admissionController.enabled` to `true`. + +Enabling the admission controller provides you with two options for managing certificates. +You can either utilize [cert-manager](https://cert-manager.io/docs/installation/) for generating a self-signed certificate automatically, or you can provide your own self-signed certificate. + +To use `cert-manager`, ensure that `operator.admissionController.useCertManager` is set to `true`. Additionally, make sure that you deploy cert-manager before initiating the Network Operator deployment. + +If you prefer not to use `cert-manager`, set `operator.admissionController.useCertManager` to `false`, and then provide your custom certificate and key using `operator.admissionController.certificate.tlsCrt` and `operator.admissionController.certificate.tlsKey`. + +> __NOTE__: When using your own certificate, the certificate must be valid for -webhook-service.< +> Release_Namespace>.svc, e.g. network-operator-webhook-service.network-operator.svc + +> __NOTE__: When deploying network operator with admission controller using helm, you need to append `--wait` to helm install and helm upgrade commands +> + +##### Generating self-signed certificate using OpenSSL + +To generate a self-signed SSL certificate valid for a specific hostname, you can use the `openssl` command-line tool. +First, navigate to the directory where you want to store your certificate and key files. Then, run the following +command: + +```bash +SVCNAME="network-operator-webhook-service.network-operator.svc" +openssl req -x509 -nodes -batch -newkey rsa:2048 -keyout server.key -out server.crt -days 365 -addext "subjectAltName=DNS:$SVCNAME" +``` + +Replace `SVCNAME` with the SVC name follows this convention -webhook-service..svc. +This command will generate a new RSA key pair with 2048 bits and create a self-signed certificate (`server.crt`) and +private key (`server.key`) that are valid for 365 days. + + +## Upgrade + +> __NOTE__: Upgrade capabilities are limited now. Additional manual actions required when containerized OFED driver is used + +Before starting the upgrade to a specific release version, please, check release notes for this version to ensure that +no additional actions are required. + + +### Check available releases + +``` +helm search repo nvidia/network-operator -l +``` + +> __NOTE__: add `--devel` option if you want to list beta releases as well + +### Upgrade CRDs to compatible version + +The network-operator helm chart contains a hook(pre-install, pre-upgrade) that will automatically upgrade required CRDs in the cluster. +The hook is enabled by default. If you don't want to upgrade CRDs with helm automatically, +you can disable auto upgrade by setting `upgradeCRDs: false` in the helm chart values. +Then you can follow the guide below to download and apply CRDs for the concrete version of the network-operator. + +It is possible to retrieve updated CRDs from the Helm chart or from the release branch on GitHub. Example bellow show +how to download and unpack Helm chart for specified release and then apply CRDs update from it. + +``` +helm pull nvidia/network-operator --version --untar --untardir network-operator-chart +``` + +> __NOTE__: `--devel` option required if you want to use the beta release + +``` +kubectl apply -f network-operator-chart/network-operator/crds \ + -f network-operator-chart/network-operator/charts/sriov-network-operator/crds +``` + +### Prepare Helm values for the new release + +Download Helm values for the specific release + +``` +helm show values nvidia/network-operator --version= > values-.yaml +``` + +Edit `values-.yaml` file as required for your cluster. + +### Apply Helm chart update + +``` +helm upgrade -n network-operator network-operator nvidia/network-operator --version= -f values-.yaml --force +``` + +> __NOTE__: `--devel` option required if you want to use the beta release + +## Chart parameters + +In order to tailor the deployment of the network operator to your cluster needs, Chart parameters are available. See official [documentation](https://docs.nvidia.com/networking/software/cloud-orchestration/index.html). diff --git a/network-operator/charts/maintenance-operator-chart/.helmignore b/network-operator/charts/maintenance-operator-chart/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/network-operator/charts/maintenance-operator-chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/network-operator/charts/maintenance-operator-chart/Chart.yaml b/network-operator/charts/maintenance-operator-chart/Chart.yaml new file mode 100644 index 0000000..61dd2ce --- /dev/null +++ b/network-operator/charts/maintenance-operator-chart/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +appVersion: latest +description: Maintenance Operator Helm Chart +name: maintenance-operator-chart +type: application +version: 0.0.1 diff --git a/network-operator/charts/maintenance-operator-chart/README.md b/network-operator/charts/maintenance-operator-chart/README.md new file mode 100644 index 0000000..8c5d686 --- /dev/null +++ b/network-operator/charts/maintenance-operator-chart/README.md @@ -0,0 +1,34 @@ +# maintenance-operator-chart + +![Version: 0.0.1](https://img.shields.io/badge/Version-0.0.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square) + +Maintenance Operator Helm Chart + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| imagePullSecrets | list | `[]` | image pull secrets for the operator | +| metricsService | object | `{"ports":[{"name":"https","port":8443,"protocol":"TCP","targetPort":"https"}],"type":"ClusterIP"}` | metrics service configurations | +| operator.admissionController.certificates.certManager.enable | bool | `true` | use cert-manager for certificates | +| operator.admissionController.certificates.certManager.generateSelfSigned | bool | `true` | generate self-signed certificiates with cert-manager | +| operator.admissionController.certificates.custom.enable | bool | `false` | enable custom certificates using secrets | +| operator.admissionController.certificates.secretNames.operator | string | `"operator-webhook-cert"` | secret name containing certificates for the operator admission controller | +| operator.admissionController.enable | bool | `true` | enable admission controller of the operator | +| operator.affinity | object | `{"nodeAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/master","operator":"Exists"}]},"weight":1},{"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/control-plane","operator":"Exists"}]},"weight":1}]}}` | node affinity for the operator | +| operator.image.imagePullPolicy | string | `nil` | image pull policy for the operator image | +| operator.image.repository | string | `"ghcr.io/mellanox"` | repository to use for the operator image | +| operator.image.name | string | `"maintenance-operator"` | image name to use for the operator image | +| operator.image.tag | string | `nil` | image tag to use for the operator image | +| operator.nodeSelector | object | `{}` | node selector for the operator | +| operator.replicas | int | `1` | operator deployment number of repplicas | +| operator.resources | object | `{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"10m","memory":"64Mi"}}` | specify resource requests and limits for the operator | +| operator.serviceAccount.annotations | object | `{}` | set annotations for the operator service account | +| operator.tolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master","operator":"Exists"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane","operator":"Exists"}]` | toleration for the operator | +| operatorConfig | object | `{"logLevel":"info","maxNodeMaintenanceTimeSeconds":null,"maxParallelOperations":null,"maxUnavailable":null}` | operator configuration values. fields here correspond to fields in MaintenanceOperatorConfig CR | +| operatorConfig.logLevel | string | `"info"` | log level configuration | +| operatorConfig.maxNodeMaintenanceTimeSeconds | string | `nil` | max time for node maintenance | +| operatorConfig.maxParallelOperations | string | `nil` | max number of parallel operations | +| operatorConfig.maxUnavailable | string | `nil` | max number of unavailable nodes | +| webhookService | object | `{"ports":[{"port":443,"protocol":"TCP","targetPort":9443}],"type":"ClusterIP"}` | webhook service configurations | + diff --git a/network-operator/charts/maintenance-operator-chart/crds/maintenance.nvidia.com_maintenanceoperatorconfigs.yaml b/network-operator/charts/maintenance-operator-chart/crds/maintenance.nvidia.com_maintenanceoperatorconfigs.yaml new file mode 100644 index 0000000..450d351 --- /dev/null +++ b/network-operator/charts/maintenance-operator-chart/crds/maintenance.nvidia.com_maintenanceoperatorconfigs.yaml @@ -0,0 +1,93 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: maintenanceoperatorconfigs.maintenance.nvidia.com +spec: + group: maintenance.nvidia.com + names: + kind: MaintenanceOperatorConfig + listKind: MaintenanceOperatorConfigList + plural: maintenanceoperatorconfigs + singular: maintenanceoperatorconfig + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: MaintenanceOperatorConfig is the Schema for the maintenanceoperatorconfigs + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: MaintenanceOperatorConfigSpec defines the desired state of + MaintenanceOperatorConfig + properties: + logLevel: + default: info + description: LogLevel is the operator logging level + enum: + - debug + - info + - error + type: string + maxNodeMaintenanceTimeSeconds: + default: 1600 + description: |- + MaxNodeMaintenanceTimeSeconds is the time from when a NodeMaintenance is marked as ready (phase: Ready) + until the NodeMaintenance is considered stale and removed by the operator. + should be less than idle time for any autoscaler that is running. + default to 30m (1600 seconds) + format: int32 + minimum: 0 + type: integer + maxParallelOperations: + anyOf: + - type: integer + - type: string + default: 1 + description: |- + MaxParallelOperations indicates the maximal number nodes that can undergo maintenance + at a given time. 0 means no limit + value can be an absolute number (ex: 5) or a percentage of total nodes in the cluster (ex: 10%). + absolute number is calculated from percentage by rounding up. + defaults to 1. The actual number of nodes that can undergo maintenance may be lower depending + on the value of MaintenanceOperatorConfigSpec.MaxUnavailable. + x-kubernetes-int-or-string: true + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + MaxUnavailable is the maximum number of nodes that can become unavailable in the cluster. + value can be an absolute number (ex: 5) or a percentage of total nodes in the cluster (ex: 10%). + absolute number is calculated from percentage by rounding up. + by default, unset. + new nodes will not be processed if the number of unavailable node will exceed this value + x-kubernetes-int-or-string: true + type: object + status: + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/network-operator/charts/maintenance-operator-chart/crds/maintenance.nvidia.com_nodemaintenances.yaml b/network-operator/charts/maintenance-operator-chart/crds/maintenance.nvidia.com_nodemaintenances.yaml new file mode 100644 index 0000000..b78e584 --- /dev/null +++ b/network-operator/charts/maintenance-operator-chart/crds/maintenance.nvidia.com_nodemaintenances.yaml @@ -0,0 +1,275 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.5 + name: nodemaintenances.maintenance.nvidia.com +spec: + group: maintenance.nvidia.com + names: + kind: NodeMaintenance + listKind: NodeMaintenanceList + plural: nodemaintenances + singular: nodemaintenance + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.nodeName + name: Node + type: string + - jsonPath: .spec.requestorID + name: Requestor + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].status + name: Ready + type: string + - jsonPath: .status.conditions[?(@.type=='Ready')].reason + name: Phase + type: string + - jsonPath: .status.conditions[?(@.type=='Failed')].reason + name: Failed + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: NodeMaintenance is the Schema for the nodemaintenances API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NodeMaintenanceSpec defines the desired state of NodeMaintenance + properties: + additionalRequestors: + description: |- + AdditionalRequestors is a set of additional requestor IDs which are using the same NodeMaintenance + request. addition or removal of requiestor IDs to this list MUST be made with update operation (and retry on failure) + which will replace the entire list. + items: + type: string + type: array + x-kubernetes-list-type: set + cordon: + default: true + description: Cordon if set, marks node as unschedulable during maintenance + operation + type: boolean + drainSpec: + description: DrainSpec specifies how a node will be drained. if not + provided, no draining will be performed. + properties: + deleteEmptyDir: + default: false + description: |- + DeleteEmptyDir indicates if should continue even if there are pods using emptyDir + (local data that will be deleted when the node is drained) + type: boolean + force: + default: false + description: Force draining even if there are pods that do not + declare a controller + type: boolean + podEvictionFilters: + description: |- + PodEvictionFilters specifies filters for pods that need to undergo eviction during drain. + if specified. only pods that match PodEvictionFilters will be evicted during drain operation. + if unspecified. all non-daemonset pods will be evicted. + logical OR is performed between filter entires. logical AND is performed within different filters + in a filter entry. + items: + description: PodEvictionFiterEntry defines filters for Pod evictions + during drain operation + properties: + byResourceNameRegex: + description: ByResourceNameRegex filters pods by the name + of the resources they consume using regex. + type: string + type: object + type: array + podSelector: + description: |- + PodSelector specifies a label selector to filter pods on the node that need to be drained + For more details on label selectors, see: + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + type: string + timeoutSeconds: + default: 300 + description: TimeoutSecond specifies the length of time in seconds + to wait before giving up drain, zero means infinite + format: int32 + minimum: 0 + type: integer + type: object + nodeName: + description: |- + NodeName is The name of the node that maintenance operation will be performed on + creation fails if node obj does not exist (webhook) + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + requestorID: + description: |- + RequestorID MUST follow domain name notation format (https://tools.ietf.org/html/rfc1035#section-2.3.1) + It MUST be 63 characters or less, beginning and ending with an alphanumeric + character ([a-z0-9A-Z]) with dashes (-), dots (.), and alphanumerics between. + caller SHOULD NOT create multiple objects with same requestorID and nodeName. + This field identifies the requestor of the operation. + maxLength: 63 + minLength: 2 + pattern: ^([a-z0-9A-Z]([-a-z0-9A-Z]*[a-z0-9A-Z])?(\.[a-z0-9A-Z]([-a-z0-9A-Z]*[a-z0-9A-Z])?)*)$ + type: string + x-kubernetes-validations: + - message: Value is immutable + rule: self == oldSelf + waitForPodCompletion: + description: |- + WaitForPodCompletion specifies pods via selector to wait for completion before performing drain operation + if not provided, will not wait for pods to complete + properties: + podSelector: + description: |- + PodSelector specifies a label selector for the pods to wait for completion + For more details on label selectors, see: + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + example: app=my-workloads + type: string + timeoutSeconds: + default: 0 + description: |- + TimeoutSecond specifies the length of time in seconds + to wait before giving up on pod termination, zero means infinite + format: int32 + minimum: 0 + type: integer + type: object + required: + - nodeName + - requestorID + type: object + status: + description: NodeMaintenanceStatus defines the observed state of NodeMaintenance + properties: + conditions: + description: Conditions represents observations of NodeMaintenance + current state + items: + description: Condition contains details for one aspect of the current + state of this API Resource. + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: type of condition in CamelCase or in foo.example.com/CamelCase. + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + x-kubernetes-list-map-keys: + - type + x-kubernetes-list-type: map + drain: + description: Drain represents the drain status of the node + properties: + drainProgress: + description: DrainProgress represents the draining progress as + percentage + format: int32 + minimum: 0 + type: integer + evictionPods: + description: EvictionPods is the total number of pods that need + to be evicted at the time NodeMaintenance started draining + format: int32 + minimum: 0 + type: integer + totalPods: + description: TotalPods is the number of pods on the node at the + time NodeMaintenance started draining + format: int32 + minimum: 0 + type: integer + waitForEviction: + description: WaitForEviction is the list of namespaced named pods + that need to be evicted + items: + type: string + type: array + required: + - drainProgress + - evictionPods + - totalPods + type: object + waitForCompletion: + description: WaitForCompletion is the list of namespaced named pods + that we wait to complete + items: + type: string + type: array + type: object + type: object + selectableFields: + - jsonPath: .spec.nodeName + - jsonPath: .spec.requestorID + served: true + storage: true + subresources: + status: {} diff --git a/network-operator/charts/maintenance-operator-chart/templates/_helpers.tpl b/network-operator/charts/maintenance-operator-chart/templates/_helpers.tpl new file mode 100644 index 0000000..ae44554 --- /dev/null +++ b/network-operator/charts/maintenance-operator-chart/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "maintenance-operator.name" -}} +{{- default "maintenance-operator" .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "maintenance-operator.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default "maintenance-operator" .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "maintenance-operator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "maintenance-operator.labels" -}} +helm.sh/chart: {{ include "maintenance-operator.chart" . }} +{{ include "maintenance-operator.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "maintenance-operator.selectorLabels" -}} +app.kubernetes.io/name: {{ include "maintenance-operator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "maintenance-operator.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "maintenance-operator.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/network-operator/charts/maintenance-operator-chart/templates/certificates.yaml b/network-operator/charts/maintenance-operator-chart/templates/certificates.yaml new file mode 100644 index 0000000..68383b5 --- /dev/null +++ b/network-operator/charts/maintenance-operator-chart/templates/certificates.yaml @@ -0,0 +1,43 @@ +{{- if and .Values.operator.admissionController.enable }} +{{- if and .Values.operator.admissionController.certificates.certManager.enable + .Values.operator.admissionController.certificates.certManager.generateSelfSigned }} +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ include "maintenance-operator.fullname" . }}-selfsigned-issuer + namespace: {{ .Release.Namespace }} + labels: + {{- include "maintenance-operator.labels" . | nindent 4 }} +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ include "maintenance-operator.fullname" . }}-serving-cert + namespace: {{ .Release.Namespace }} + labels: + {{- include "maintenance-operator.labels" . | nindent 4 }} +spec: + dnsNames: + - '{{ include "maintenance-operator.fullname" . }}-webhook-service.{{ .Release.Namespace + }}.svc' + - '{{ include "maintenance-operator.fullname" . }}-webhook-service.{{ .Release.Namespace + }}.svc.{{ .Values.kubernetesClusterDomain }}' + issuerRef: + kind: Issuer + name: '{{ include "maintenance-operator.fullname" . }}-selfsigned-issuer' + secretName: {{ .Values.operator.admissionController.certificates.secretNames.operator }} +{{- else if and (not .Values.operator.admissionController.certManager.enable) .Values.operator.admissionController.custom.enable }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.operator.admissionController.certificates.secretNames.operator }} + namespace: {{ .Release.Namespace }} +type: Opaque +data: + ca.crt: {{ .Values.operator.admissionController.certificates.custom.operator.caCrt | b64enc | b64enc | quote }} + tls.crt: {{ .Values.operator.admissionController.certificates.custom.operator.tlsCrt | b64enc | quote }} + tls.key: {{ .Values.operator.admissionController.certificates.custom.operator.tlsKey | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/network-operator/charts/maintenance-operator-chart/templates/deployment.yaml b/network-operator/charts/maintenance-operator-chart/templates/deployment.yaml new file mode 100644 index 0000000..033b8fc --- /dev/null +++ b/network-operator/charts/maintenance-operator-chart/templates/deployment.yaml @@ -0,0 +1,88 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "maintenance-operator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: maintenance-operator-controller-manager + app.kubernetes.io/created-by: maintenance-operator + app.kubernetes.io/part-of: maintenance-operator + control-plane: {{ .Release.Name }}-controller-manager + {{- include "maintenance-operator.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.operator.replicas }} + selector: + matchLabels: + control-plane: {{ .Release.Name }}-controller-manager + {{- include "maintenance-operator.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + control-plane: {{ .Release.Name }}-controller-manager + app.kubernetes.io/component: maintenance-operator-controller-manager + {{- include "maintenance-operator.selectorLabels" . | nindent 8 }} + annotations: + kubectl.kubernetes.io/default-container: manager + spec: + tolerations: {{- toYaml .Values.operator.tolerations | nindent 8 }} + nodeSelector: {{- toYaml .Values.operator.nodeSelector | nindent 8 }} + affinity: {{- toYaml .Values.operator.affinity | nindent 8 }} + imagePullSecrets: {{ .Values.imagePullSecrets | default list | toJson }} + securityContext: + runAsNonRoot: true + serviceAccountName: {{ include "maintenance-operator.fullname" . }}-controller-manager + terminationGracePeriodSeconds: 10 + containers: + - name: manager + command: + - /manager + args: + - --leader-elect + env: + - name: OPERATOR_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: ENABLE_WEBHOOKS + value: {{ quote .Values.operator.admissionController.enable }} + image: {{ .Values.operator.image.repository }}/{{ .Values.operator.image.name }}:{{ .Values.operator.image.tag | default .Chart.AppVersion }} + {{- if .Values.operator.image.imagePullPolicy }} + imagePullPolicy: {{ .Values.operator.image.imagePullPolicy }} + {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + {{- if .Values.operator.admissionController.enable }} + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + {{- end }} + resources: {{- toYaml .Values.operator.resources | nindent 10 }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + volumeMounts: + {{- if .Values.operator.admissionController.enable }} + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + {{- end }} + volumes: + {{- if .Values.operator.admissionController.enable }} + - name: cert + secret: + defaultMode: 420 + secretName: {{ .Values.operator.admissionController.certificates.secretNames.operator }} + {{- end }} diff --git a/network-operator/charts/maintenance-operator-chart/templates/metrics-service.yaml b/network-operator/charts/maintenance-operator-chart/templates/metrics-service.yaml new file mode 100644 index 0000000..780fa80 --- /dev/null +++ b/network-operator/charts/maintenance-operator-chart/templates/metrics-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "maintenance-operator.name" . }}-metrics-service + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: kube-rbac-proxy + app.kubernetes.io/created-by: maintenance-operator + app.kubernetes.io/part-of: maintenance-operator + control-plane: {{ .Release.Name }}-controller-manager + {{- include "maintenance-operator.labels" . | nindent 4 }} +spec: + type: {{ .Values.metricsService.type }} + selector: + control-plane: {{ .Release.Name }}-controller-manager + {{- include "maintenance-operator.selectorLabels" . | nindent 4 }} + ports: + {{- .Values.metricsService.ports | toYaml | nindent 2 }} \ No newline at end of file diff --git a/network-operator/charts/maintenance-operator-chart/templates/operatorconfig.yaml b/network-operator/charts/maintenance-operator-chart/templates/operatorconfig.yaml new file mode 100644 index 0000000..60f4b49 --- /dev/null +++ b/network-operator/charts/maintenance-operator-chart/templates/operatorconfig.yaml @@ -0,0 +1,21 @@ +apiVersion: maintenance.nvidia.com/v1alpha1 +kind: MaintenanceOperatorConfig +metadata: + name: default + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: config + app.kubernetes.io/created-by: maintenance-operator + app.kubernetes.io/part-of: maintenance-operator + {{- include "maintenance-operator.labels" . | nindent 4 }} +spec: + logLevel: {{ .Values.operatorConfig.logLevel }} +{{- if .Values.operatorConfig.maxParallelOperations }} + maxParallelOperations: {{ .Values.operatorConfig.maxParallelOperations }} +{{- end }} +{{- if .Values.operatorConfig.maxUnavailable }} + maxUnavailable: {{ .Values.operatorConfig.maxUnavailable }} +{{- end }} +{{- if .Values.operatorConfig.maxNodeMaintenanceTimeSeconds }} + maxNodeMaintenanceTimeSeconds: {{ .Values.operatorConfig.maxNodeMaintenanceTimeSeconds }} +{{- end }} diff --git a/network-operator/charts/maintenance-operator-chart/templates/role.yaml b/network-operator/charts/maintenance-operator-chart/templates/role.yaml new file mode 100644 index 0000000..464e1a1 --- /dev/null +++ b/network-operator/charts/maintenance-operator-chart/templates/role.yaml @@ -0,0 +1,154 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "maintenance-operator.fullname" . }}-manager-role + labels: + {{- include "maintenance-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create + - delete + - get + - list + - patch + - update +- apiGroups: + - apps + resources: + - daemonsets + verbs: + - get + - list + - watch +- apiGroups: + - config.openshift.io + resources: + - infrastructures + verbs: + - get + - list + - watch +- apiGroups: + - machineconfiguration.openshift.io + resources: + - machineconfigpools + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - machineconfiguration.openshift.io + resources: + - machineconfigs + verbs: + - get + - list + - watch +- apiGroups: + - maintenance.nvidia.com + resources: + - maintenanceoperatorconfigs + - nodemaintenances + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - maintenance.nvidia.com + resources: + - maintenanceoperatorconfigs/finalizers + - nodemaintenances/finalizers + verbs: + - update +- apiGroups: + - maintenance.nvidia.com + resources: + - maintenanceoperatorconfigs/status + - nodemaintenances/status + verbs: + - get + - patch + - update +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "maintenance-operator.fullname" . }}-manager-role + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: maintenance-operator + app.kubernetes.io/part-of: maintenance-operator + {{- include "maintenance-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - get + - list + - watch + - create + - update + - patch + - delete +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch diff --git a/network-operator/charts/maintenance-operator-chart/templates/role_binding.yaml b/network-operator/charts/maintenance-operator-chart/templates/role_binding.yaml new file mode 100644 index 0000000..6f3491d --- /dev/null +++ b/network-operator/charts/maintenance-operator-chart/templates/role_binding.yaml @@ -0,0 +1,36 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "maintenance-operator.fullname" . }}-manager-rolebinding + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: maintenance-operator + app.kubernetes.io/part-of: maintenance-operator + {{- include "maintenance-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: '{{ include "maintenance-operator.fullname" . }}-manager-role' +subjects: +- kind: ServiceAccount + name: '{{ include "maintenance-operator.fullname" . }}-controller-manager' + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "maintenance-operator.fullname" . }}-manager-rolebinding + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: maintenance-operator + app.kubernetes.io/part-of: maintenance-operator + {{- include "maintenance-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: '{{ include "maintenance-operator.fullname" . }}-manager-role' +subjects: +- kind: ServiceAccount + name: '{{ include "maintenance-operator.fullname" . }}-controller-manager' + namespace: {{ .Release.Namespace }} diff --git a/network-operator/charts/maintenance-operator-chart/templates/serviceaccount.yaml b/network-operator/charts/maintenance-operator-chart/templates/serviceaccount.yaml new file mode 100644 index 0000000..c948f0c --- /dev/null +++ b/network-operator/charts/maintenance-operator-chart/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "maintenance-operator.fullname" . }}-controller-manager + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: maintenance-operator + app.kubernetes.io/part-of: maintenance-operator + {{- include "maintenance-operator.labels" . | nindent 4 }} + annotations: + {{- toYaml .Values.operator.serviceAccount.annotations | nindent 4 }} diff --git a/network-operator/charts/maintenance-operator-chart/templates/webhook.yaml b/network-operator/charts/maintenance-operator-chart/templates/webhook.yaml new file mode 100644 index 0000000..5e904c6 --- /dev/null +++ b/network-operator/charts/maintenance-operator-chart/templates/webhook.yaml @@ -0,0 +1,48 @@ +{{- if .Values.operator.admissionController.enable }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "maintenance-operator.fullname" . }}-webhook-service + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: maintenance-operator + app.kubernetes.io/part-of: maintenance-operator + {{- include "maintenance-operator.labels" . | nindent 4 }} +spec: + type: {{ .Values.webhookService.type }} + selector: + control-plane: {{ .Release.Name }}-controller-manager + {{- include "maintenance-operator.selectorLabels" . | nindent 4 }} + ports: + {{- .Values.webhookService.ports | toYaml | nindent 2 }} +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + name: {{ include "maintenance-operator.fullname" . }}-validating-webhook-configuration + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ include "maintenance-operator.fullname" . }}-serving-cert + labels: + {{- include "maintenance-operator.labels" . | nindent 4 }} +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: '{{ include "maintenance-operator.fullname" . }}-webhook-service' + namespace: {{ .Release.Namespace }} + path: /validate-maintenance-nvidia-com-v1alpha1-nodemaintenance + failurePolicy: Fail + name: vnodemaintenance.kb.io + rules: + - apiGroups: + - maintenance.nvidia.com + apiVersions: + - v1alpha1 + operations: + - CREATE + resources: + - nodemaintenances + sideEffects: None +{{- end }} diff --git a/network-operator/charts/maintenance-operator-chart/values.yaml b/network-operator/charts/maintenance-operator-chart/values.yaml new file mode 100644 index 0000000..8a0511b --- /dev/null +++ b/network-operator/charts/maintenance-operator-chart/values.yaml @@ -0,0 +1,109 @@ +operator: + image: + # -- repository to use for the operator image + repository: ghcr.io/mellanox + # -- image name to use for the operator image + name: maintenance-operator + # -- image tag to use for the operator image + tag: null + # -- image pull policy for the operator image + imagePullPolicy: null + # -- toleration for the operator + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + # -- node selector for the operator + nodeSelector: {} + # -- node affinity for the operator + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: Exists + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: Exists + # -- specify resource requests and limits for the operator + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + # -- operator deployment number of repplicas + replicas: 1 + serviceAccount: + # -- set annotations for the operator service account + annotations: {} + admissionController: + # -- enable admission controller of the operator + enable: true + certificates: + secretNames: + # -- secret name containing certificates for the operator admission controller + operator: "operator-webhook-cert" + certManager: + # -- use cert-manager for certificates + enable: true + # -- generate self-signed certificiates with cert-manager + generateSelfSigned: true + custom: + # -- enable custom certificates using secrets + enable: false + # operator: + # caCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsKey: | + # -----BEGIN EC PRIVATE KEY----- + # MHcl4wOuDwKQa+upc8GftXE2C//4mKANBC6It01gUaTIpo= + # ... + # -----END EC PRIVATE KEY----- + +# -- operator configuration values. fields here correspond to fields in MaintenanceOperatorConfig CR +operatorConfig: + # -- log level configuration + logLevel: info + # operatorConfig.maxParallelOperations -- max number of parallel operations + maxParallelOperations: null + # -- max number of unavailable nodes + maxUnavailable: null + # -- max time for node maintenance + maxNodeMaintenanceTimeSeconds: null + +# -- image pull secrets for the operator +imagePullSecrets: [] + +# -- metrics service configurations +metricsService: + ports: + - name: https + port: 8443 + protocol: TCP + targetPort: https + type: ClusterIP + +# -- webhook service configurations +webhookService: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + type: ClusterIP diff --git a/network-operator/charts/nic-configuration-operator-chart/.helmignore b/network-operator/charts/nic-configuration-operator-chart/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/network-operator/charts/nic-configuration-operator-chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/network-operator/charts/nic-configuration-operator-chart/Chart.yaml b/network-operator/charts/nic-configuration-operator-chart/Chart.yaml new file mode 100644 index 0000000..754af6b --- /dev/null +++ b/network-operator/charts/nic-configuration-operator-chart/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +appVersion: latest +description: A Helm chart for NIC Configuration Operator +name: nic-configuration-operator-chart +type: application +version: 0.0.1 diff --git a/network-operator/charts/nic-configuration-operator-chart/README.md b/network-operator/charts/nic-configuration-operator-chart/README.md new file mode 100644 index 0000000..67b2826 --- /dev/null +++ b/network-operator/charts/nic-configuration-operator-chart/README.md @@ -0,0 +1,27 @@ +# nic-configuration-operator-chart + +![Version: 0.0.1](https://img.shields.io/badge/Version-0.0.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: latest](https://img.shields.io/badge/AppVersion-latest-informational?style=flat-square) + +A Helm chart for NIC Configuration Operator + +## Values + +| Key | Type | Default | Description | +|-----|------|---------|-------------| +| configDaemon.image.name | string | `"nic-configuration-operator-daemon"` | | +| configDaemon.image.repository | string | `"ghcr.io/mellanox"` | repository to use for the config daemon image | +| configDaemon.image.tag | string | `"latest"` | image tag to use for the config daemon image | +| configDaemon.nodeSelector | object | `{}` | node selector for the config daemon | +| configDaemon.resources | object | `{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"10m","memory":"64Mi"}}` | resources and limits for the config daemon | +| imagePullSecrets | list | `[]` | image pull secrets for both the operator and the config daemon | +| logLevel | string | `"info"` | log level configuration (debug|info) | +| operator.affinity | object | `{"nodeAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/master","operator":"Exists"}]},"weight":1},{"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/control-plane","operator":"Exists"}]},"weight":1}]}}` | node affinity for the operator | +| operator.image.name | string | `"nic-configuration-operator"` | | +| operator.image.repository | string | `"ghcr.io/mellanox"` | repository to use for the operator image | +| operator.image.tag | string | `"latest"` | image tag to use for the operator image | +| operator.nodeSelector | object | `{}` | node selector for the operator | +| operator.replicas | int | `1` | operator deployment number of replicas | +| operator.resources | object | `{"limits":{"cpu":"500m","memory":"128Mi"},"requests":{"cpu":"10m","memory":"64Mi"}}` | specify resource requests and limits for the operator | +| operator.serviceAccount.annotations | object | `{}` | set annotations for the operator service account | +| operator.tolerations | list | `[{"effect":"NoSchedule","key":"node-role.kubernetes.io/master","operator":"Exists"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane","operator":"Exists"}]` | tolerations for the operator | + diff --git a/network-operator/charts/nic-configuration-operator-chart/crds/configuration.net.nvidia.com_nicconfigurationtemplates.yaml b/network-operator/charts/nic-configuration-operator-chart/crds/configuration.net.nvidia.com_nicconfigurationtemplates.yaml new file mode 100644 index 0000000..fdceca4 --- /dev/null +++ b/network-operator/charts/nic-configuration-operator-chart/crds/configuration.net.nvidia.com_nicconfigurationtemplates.yaml @@ -0,0 +1,174 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: nicconfigurationtemplates.configuration.net.nvidia.com +spec: + group: configuration.net.nvidia.com + names: + kind: NicConfigurationTemplate + listKind: NicConfigurationTemplateList + plural: nicconfigurationtemplates + singular: nicconfigurationtemplate + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NicConfigurationTemplate is the Schema for the nicconfigurationtemplates + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Defines the desired state of NICs + properties: + nicSelector: + description: NIC selector configuration + properties: + nicType: + description: Type of the NIC to be selected, e.g. 101d,1015,a2d6 + etc. + type: string + pciAddresses: + description: Array of PCI addresses to be selected, e.g. "0000:03:00.0" + items: + type: string + type: array + serialNumbers: + description: Serial numbers of the NICs to be selected, e.g. MT2116X09299 + items: + type: string + type: array + required: + - nicType + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector contains labels required on the node + type: object + resetToDefault: + default: false + description: |- + ResetToDefault specifies whether node agent needs to perform a reset flow + The following operations will be performed: + * Nvconfig reset of all non-volatile configurations + - Mstconfig -d reset for each PF + - Mstconfig -d set ADVANCED_PCI_SETTINGS=1 + * Node reboot + - Applies new NIC NV config + - Will undo any runtime configuration previously performed for the device/driver + type: boolean + template: + description: Configuration template to be applied to matching devices + properties: + gpuDirectOptimized: + description: GPU Direct optimization settings + properties: + enabled: + description: Optimize GPU Direct + type: boolean + env: + description: GPU direct environment, e.g. Baremetal + type: string + required: + - enabled + - env + type: object + linkType: + description: LinkType to be configured, Ethernet|Infiniband + enum: + - Ethernet + - Infiniband + type: string + numVfs: + description: Number of VFs to be configured + type: integer + pciPerformanceOptimized: + description: PCI performance optimization settings + properties: + enabled: + description: Specifies whether to enable PCI performance optimization + type: boolean + maxAccOutRead: + description: Specifies the PCIe Max Accumulative Outstanding + read bytes + type: integer + maxReadRequest: + description: Specifies the size of a single PCI read request + in bytes + enum: + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + type: integer + required: + - enabled + type: object + roceOptimized: + description: RoCE optimization settings + properties: + enabled: + description: Optimize RoCE + type: boolean + qos: + description: Quality of Service settings + properties: + pfc: + description: Priority-based Flow Control configuration, + e.g. "0,0,0,1,0,0,0,0" + pattern: ^([01],){7}[01]$ + type: string + trust: + description: Trust mode for QoS settings, e.g. trust-dscp + type: string + required: + - pfc + - trust + type: object + required: + - enabled + type: object + required: + - linkType + - numVfs + type: object + required: + - nicSelector + - template + type: object + status: + description: Defines the observed state of NicConfigurationTemplate + properties: + nicDevices: + description: NicDevice CRs matching this configuration template + items: + type: string + type: array + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/network-operator/charts/nic-configuration-operator-chart/crds/configuration.net.nvidia.com_nicdevices.yaml b/network-operator/charts/nic-configuration-operator-chart/crds/configuration.net.nvidia.com_nicdevices.yaml new file mode 100644 index 0000000..93c5227 --- /dev/null +++ b/network-operator/charts/nic-configuration-operator-chart/crds/configuration.net.nvidia.com_nicdevices.yaml @@ -0,0 +1,264 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: nicdevices.configuration.net.nvidia.com +spec: + group: configuration.net.nvidia.com + names: + kind: NicDevice + listKind: NicDeviceList + plural: nicdevices + singular: nicdevice + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NicDevice is the Schema for the nicdevices API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: NicDeviceSpec defines the desired state of NicDevice + properties: + configuration: + description: Configuration specifies the configuration requested by + NicConfigurationTemplate + properties: + resetToDefault: + description: |- + ResetToDefault specifies whether node agent needs to perform a reset flow. + In NIC Configuration Operator template v0.1.14 BF2/BF3 DPUs (not SuperNics) FW reset flow isn't supported. + The following operations will be performed: + * Nvconfig reset of all non-volatile configurations + - Mstconfig -d reset for each PF + - Mstconfig -d set ADVANCED_PCI_SETTINGS=1 + * Node reboot + - Applies new NIC NV config + - Will undo any runtime configuration previously performed for the device/driver + type: boolean + template: + description: Configuration template applied from the NicConfigurationTemplate + CR + properties: + gpuDirectOptimized: + description: GPU Direct optimization settings + properties: + enabled: + description: Optimize GPU Direct + type: boolean + env: + description: GPU direct environment, e.g. Baremetal + type: string + required: + - enabled + - env + type: object + linkType: + description: LinkType to be configured, Ethernet|Infiniband + enum: + - Ethernet + - Infiniband + type: string + numVfs: + description: Number of VFs to be configured + type: integer + pciPerformanceOptimized: + description: PCI performance optimization settings + properties: + enabled: + description: Specifies whether to enable PCI performance + optimization + type: boolean + maxAccOutRead: + description: Specifies the PCIe Max Accumulative Outstanding + read bytes + type: integer + maxReadRequest: + description: Specifies the size of a single PCI read request + in bytes + enum: + - 128 + - 256 + - 512 + - 1024 + - 2048 + - 4096 + type: integer + required: + - enabled + type: object + roceOptimized: + description: RoCE optimization settings + properties: + enabled: + description: Optimize RoCE + type: boolean + qos: + description: Quality of Service settings + properties: + pfc: + description: Priority-based Flow Control configuration, + e.g. "0,0,0,1,0,0,0,0" + pattern: ^([01],){7}[01]$ + type: string + trust: + description: Trust mode for QoS settings, e.g. trust-dscp + type: string + required: + - pfc + - trust + type: object + required: + - enabled + type: object + required: + - linkType + - numVfs + type: object + type: object + type: object + status: + description: NicDeviceStatus defines the observed state of NicDevice + properties: + conditions: + description: List of conditions observed for the device + items: + description: "Condition contains details for one aspect of the current + state of this API Resource.\n---\nThis struct is intended for + direct use as an array at the field path .status.conditions. For + example,\n\n\n\ttype FooStatus struct{\n\t // Represents the + observations of a foo's current state.\n\t // Known .status.conditions.type + are: \"Available\", \"Progressing\", and \"Degraded\"\n\t // + +patchMergeKey=type\n\t // +patchStrategy=merge\n\t // +listType=map\n\t + \ // +listMapKey=type\n\t Conditions []metav1.Condition `json:\"conditions,omitempty\" + patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`\n\n\n\t + \ // other fields\n\t}" + properties: + lastTransitionTime: + description: |- + lastTransitionTime is the last time the condition transitioned from one status to another. + This should be when the underlying condition changed. If that is not known, then using the time when the API field changed is acceptable. + format: date-time + type: string + message: + description: |- + message is a human readable message indicating details about the transition. + This may be an empty string. + maxLength: 32768 + type: string + observedGeneration: + description: |- + observedGeneration represents the .metadata.generation that the condition was set based upon. + For instance, if .metadata.generation is currently 12, but the .status.conditions[x].observedGeneration is 9, the condition is out of date + with respect to the current state of the instance. + format: int64 + minimum: 0 + type: integer + reason: + description: |- + reason contains a programmatic identifier indicating the reason for the condition's last transition. + Producers of specific condition types may define expected values and meanings for this field, + and whether the values are considered a guaranteed API. + The value should be a CamelCase string. + This field may not be empty. + maxLength: 1024 + minLength: 1 + pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ + type: string + status: + description: status of the condition, one of True, False, Unknown. + enum: + - "True" + - "False" + - Unknown + type: string + type: + description: |- + type of condition in CamelCase or in foo.example.com/CamelCase. + --- + Many .condition.type values are consistent across resources like Available, but because arbitrary conditions can be + useful (see .node.status.conditions), the ability to deconflict is important. + The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) + maxLength: 316 + pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ + type: string + required: + - lastTransitionTime + - message + - reason + - status + - type + type: object + type: array + firmwareVersion: + description: Firmware version currently installed on the device, e.g. + 22.31.1014 + type: string + node: + description: Node where the device is located + type: string + partNumber: + description: Part number of the device, e.g. MCX713106AEHEA_QP1 + type: string + ports: + description: List of ports for the device + items: + description: NicDevicePortSpec describes the ports of the NIC + properties: + networkInterface: + description: NetworkInterface is the name of the network interface + for this port, e.g. eth1 + type: string + pci: + description: PCI is a PCI address of the port, e.g. 0000:3b:00.0 + type: string + rdmaInterface: + description: RdmaInterface is the name of the rdma interface + for this port, e.g. mlx5_1 + type: string + required: + - pci + type: object + type: array + psid: + description: Product Serial ID of the device, e.g. MT_0000000221 + type: string + serialNumber: + description: Serial number of the device, e.g. MT2116X09299 + type: string + type: + description: Type of device, e.g. ConnectX7 + type: string + required: + - firmwareVersion + - node + - partNumber + - ports + - psid + - serialNumber + - type + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/network-operator/charts/nic-configuration-operator-chart/templates/_helpers.tpl b/network-operator/charts/nic-configuration-operator-chart/templates/_helpers.tpl new file mode 100644 index 0000000..9c367f0 --- /dev/null +++ b/network-operator/charts/nic-configuration-operator-chart/templates/_helpers.tpl @@ -0,0 +1,58 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "nic-configuration-operator.name" -}} +{{- default "nic-configuration-operator" .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "nic-configuration-operator.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default "nic-configuration-operator" .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "nic-configuration-operator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "nic-configuration-operator.labels" -}} +helm.sh/chart: {{ include "nic-configuration-operator.chart" . }} +{{ include "nic-configuration-operator.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "nic-configuration-operator.selectorLabels" -}} +app.kubernetes.io/name: {{ include "nic-configuration-operator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "nic-configuration-operator.serviceAccountName" -}} +{{- include "nic-configuration-operator.fullname" . }} +{{- end }} diff --git a/network-operator/charts/nic-configuration-operator-chart/templates/config-daemon.yaml b/network-operator/charts/nic-configuration-operator-chart/templates/config-daemon.yaml new file mode 100644 index 0000000..9715cd0 --- /dev/null +++ b/network-operator/charts/nic-configuration-operator-chart/templates/config-daemon.yaml @@ -0,0 +1,68 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: nic-configuration-daemon + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: nic-configuration-daemon + app.kubernetes.io/created-by: nic-configuration-operator + app.kubernetes.io/part-of: nic-configuration-operator + {{- include "nic-configuration-operator.labels" . | nindent 4 }} +spec: + selector: + matchLabels: + control-plane: nic-configuration-daemon + {{- include "nic-configuration-operator.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: nic-configuration-daemon + labels: + control-plane: nic-configuration-daemon + {{- include "nic-configuration-operator.selectorLabels" . | nindent 8 }} + spec: + nodeSelector: {{- toYaml .Values.operator.nodeSelector | nindent 8 }} + serviceAccountName: {{ include "nic-configuration-operator.serviceAccountName" . }} + terminationGracePeriodSeconds: 10 + hostNetwork: true + hostPID: true + priorityClassName: system-node-critical + containers: + - image: "{{ .Values.configDaemon.image.repository }}/{{ .Values.configDaemon.image.name }}:{{ .Values.configDaemon.image.tag | default .Chart.AppVersion }}" + name: nic-configuration-daemon + securityContext: + privileged: true + resources: {{- toYaml .Values.configDaemon.resources | nindent 12 }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if .Values.logLevel}} + - name: LOG_LEVEL + value: {{ .Values.logLevel }} + {{- end}} + volumeMounts: + - name: sys + mountPath: /sys + readOnly: false + - name: proc + mountPath: /proc + readOnly: false + - name: host + mountPath: /host + readOnly: true + volumes: + - name: sys + hostPath: + path: /sys + - name: proc + hostPath: + path: /proc + - name: host + hostPath: + path: / diff --git a/network-operator/charts/nic-configuration-operator-chart/templates/operator.yaml b/network-operator/charts/nic-configuration-operator-chart/templates/operator.yaml new file mode 100644 index 0000000..0358a6d --- /dev/null +++ b/network-operator/charts/nic-configuration-operator-chart/templates/operator.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "nic-configuration-operator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: manager + app.kubernetes.io/created-by: nic-configuration-operator + app.kubernetes.io/part-of: nic-configuration-operator + {{- include "nic-configuration-operator.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + control-plane: {{ .Release.Name }}-controller-manager + {{- include "nic-configuration-operator.selectorLabels" . | nindent 6 }} + template: + metadata: + labels: + control-plane: {{ .Release.Name }}-controller-manager + {{- include "nic-configuration-operator.selectorLabels" . | nindent 8 }} + annotations: + kubectl.kubernetes.io/default-container: manager + spec: + tolerations: {{- toYaml .Values.operator.tolerations | nindent 8 }} + nodeSelector: {{- toYaml .Values.operator.nodeSelector | nindent 8 }} + affinity: {{- toYaml .Values.operator.affinity | nindent 8 }} + imagePullSecrets: {{ .Values.imagePullSecrets | default list | toJson }} + securityContext: + runAsNonRoot: true + serviceAccountName: {{ include "nic-configuration-operator.serviceAccountName" . }} + terminationGracePeriodSeconds: 10 + containers: + - name: manager + command: + - /manager + image: "{{ .Values.operator.image.repository }}/{{ .Values.operator.image.name }}:{{ .Values.operator.image.tag | default .Chart.AppVersion }}" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + {{- if .Values.logLevel}} + env: + - name: LOG_LEVEL + value: {{ .Values.logLevel }} + {{- end}} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + resources: + {{- toYaml .Values.resources | nindent 12 }} diff --git a/network-operator/charts/nic-configuration-operator-chart/templates/role.yaml b/network-operator/charts/nic-configuration-operator-chart/templates/role.yaml new file mode 100644 index 0000000..fbffac7 --- /dev/null +++ b/network-operator/charts/nic-configuration-operator-chart/templates/role.yaml @@ -0,0 +1,105 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "nic-configuration-operator.fullname" . }}-role + labels: + {{- include "nic-configuration-operator.labels" . | nindent 4}} +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - list +- apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - configuration.net.nvidia.com + resources: + - nicconfigurationtemplates + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - configuration.net.nvidia.com + resources: + - nicconfigurationtemplates/finalizers + verbs: + - update +- apiGroups: + - configuration.net.nvidia.com + resources: + - nicconfigurationtemplates/status + verbs: + - get + - patch + - update +- apiGroups: + - configuration.net.nvidia.com + resources: + - nicdevices + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - configuration.net.nvidia.com + resources: + - nicdevices/finalizers + verbs: + - update +- apiGroups: + - configuration.net.nvidia.com + resources: + - nicdevices/status + verbs: + - get + - patch + - update +- apiGroups: + - maintenance.nvidia.com + resources: + - nodemaintenances + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/network-operator/charts/nic-configuration-operator-chart/templates/role_binding.yaml b/network-operator/charts/nic-configuration-operator-chart/templates/role_binding.yaml new file mode 100644 index 0000000..0d10563 --- /dev/null +++ b/network-operator/charts/nic-configuration-operator-chart/templates/role_binding.yaml @@ -0,0 +1,16 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: nic-configuration-operator + app.kubernetes.io/part-of: nic-configuration-operator + name: {{ include "nic-configuration-operator.fullname" . }}-rolebinding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "nic-configuration-operator.fullname" . }}-role +subjects: +- kind: ServiceAccount + name: {{ include "nic-configuration-operator.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/network-operator/charts/nic-configuration-operator-chart/templates/serviceaccount.yaml b/network-operator/charts/nic-configuration-operator-chart/templates/serviceaccount.yaml new file mode 100644 index 0000000..4f8e5ca --- /dev/null +++ b/network-operator/charts/nic-configuration-operator-chart/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "nic-configuration-operator.serviceAccountName" . }} + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: nic-configuration-operator + app.kubernetes.io/part-of: nic-configuration-operator + {{- include "nic-configuration-operator.labels" . | nindent 4 }} + annotations: + {{- toYaml .Values.operator.serviceAccount.annotations | nindent 4 }} diff --git a/network-operator/charts/nic-configuration-operator-chart/templates/supported-nic-firmware-configmap.yaml b/network-operator/charts/nic-configuration-operator-chart/templates/supported-nic-firmware-configmap.yaml new file mode 100644 index 0000000..5f8e9a9 --- /dev/null +++ b/network-operator/charts/nic-configuration-operator-chart/templates/supported-nic-firmware-configmap.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: nic-configuration-operator-supported-nic-firmware +data: + Nvidia_mlx5_ConnectX-4-24.07: "1013 24.07-0.6.1 12.28.2006" + Nvidia_mlx5_ConnectX-5-24.07: "1017 24.07-0.6.1 16.35.4030" + Nvidia_mlx5_ConnectX-5_Ex-24.07: "1019 24.07-0.6.1 16.35.4030" + Nvidia_mlx5_ConnectX-6-24.07: "101b 24.07-0.6.1 20.42.1000" + Nvidia_mlx5_ConnectX-6_Dx-24.07: "101d 24.07-0.6.1 22.42.1000" + Nvidia_mlx5_ConnectX-6_Lx-24.07: "101f 24.07-0.6.1 26.42.1000" + Nvidia_mlx5_ConnectX-7-24.07: "1021 24.07-0.6.1 28.42.1000" + Nvidia_mlx5_MT42822_BlueField-2_integrated_ConnectX-6_Dx-24.07: "a2d6 24.07-0.6.1 24.42.1000" + Nvidia_mlx5_ConnectX-4-24.10: "1013 24.10-0.7.0 12.28.2006" + Nvidia_mlx5_ConnectX-4_Lx-24.10: "1013 24.10-0.7.0 14.32.1010" + Nvidia_mlx5_ConnectX-5-24.10: "1017 24.10-0.7.0 16.35.4030" + Nvidia_mlx5_ConnectX-5_Ex-24.10: "1019 24.10-0.7.0 16.35.4030" + Nvidia_mlx5_ConnectX-6-24.10: "101b 24.10-0.7.0 20.43.1014" + Nvidia_mlx5_ConnectX-6_Dx-24.10: "101d 24.10-0.7.0 22.43.1014" + Nvidia_mlx5_ConnectX-6_Lx-24.10: "101f 24.10-0.7.0 26.43.1014" + Nvidia_mlx5_ConnectX-7-24.10: "1021 24.10-0.7.0 28.43.1014" + Nvidia_mlx5_MT42822_BlueField-2_integrated_ConnectX-6_Dx-24.10: "a2d6 24.10-0.7.0 24.43.2026" + Nvidia_mlx5_MT43244_BlueField-3_integrated_ConnectX-7_Dx-24.10: "a2dc 24.10-0.7.0 32.43.2026" + Nvidia_mlx5_ConnectX-4_Lx-25.01: "1013 25.01-0.6.0 14.32.1900" + Nvidia_mlx5_ConnectX-5-25.01: "1017 25.01-0.6.0 16.35.4030" + Nvidia_mlx5_ConnectX-5_Ex-25.01: "1019 25.01-0.6.0 16.35.4030" + Nvidia_mlx5_ConnectX-6-25.01: "101b 25.01-0.6.0 20.43.2026" + Nvidia_mlx5_ConnectX-6_Dx-25.01: "101d 25.01-0.6.0 22.44.1036" + Nvidia_mlx5_ConnectX-6_Lx-25.01: "101f 25.01-0.6.0 26.44.1036" + Nvidia_mlx5_ConnectX-7-25.01: "1021 25.01-0.6.0 28.44.1036" + Nvidia_mlx5_MT42822_BlueField-2_integrated_ConnectX-6_Dx-25.01: "a2d6 25.01-0.6.0 24.44.1036" + Nvidia_mlx5_MT43244_BlueField-3_integrated_ConnectX-7_Dx-25.01: "a2dc 25.01-0.6.0 32.44.1036" diff --git a/network-operator/charts/nic-configuration-operator-chart/values.yaml b/network-operator/charts/nic-configuration-operator-chart/values.yaml new file mode 100644 index 0000000..c945c79 --- /dev/null +++ b/network-operator/charts/nic-configuration-operator-chart/values.yaml @@ -0,0 +1,67 @@ +operator: + image: + # -- repository to use for the operator image + repository: ghcr.io/mellanox + name: nic-configuration-operator + # -- image tag to use for the operator image + tag: latest + # -- tolerations for the operator + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + # -- node selector for the operator + nodeSelector: {} + # -- node affinity for the operator + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: Exists + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: Exists + # -- specify resource requests and limits for the operator + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + # -- operator deployment number of replicas + replicas: 1 + serviceAccount: + # -- set annotations for the operator service account + annotations: {} + +configDaemon: + image: + # -- repository to use for the config daemon image + repository: ghcr.io/mellanox + name: nic-configuration-operator-daemon + # -- image tag to use for the config daemon image + tag: latest + # -- node selector for the config daemon + nodeSelector: {} + # -- resources and limits for the config daemon + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 10m + memory: 64Mi + +# -- log level configuration (debug|info) +logLevel: info +# -- image pull secrets for both the operator and the config daemon +imagePullSecrets: [] diff --git a/network-operator/charts/node-feature-discovery/.helmignore b/network-operator/charts/node-feature-discovery/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/network-operator/charts/node-feature-discovery/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/network-operator/charts/node-feature-discovery/Chart.yaml b/network-operator/charts/node-feature-discovery/Chart.yaml new file mode 100644 index 0000000..1f72fa8 --- /dev/null +++ b/network-operator/charts/node-feature-discovery/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: v2 +appVersion: v0.17.0 +description: 'Detects hardware features available on each node in a Kubernetes cluster, + and advertises those features using node labels. ' +home: https://github.com/kubernetes-sigs/node-feature-discovery +keywords: +- feature-discovery +- feature-detection +- node-labels +name: node-feature-discovery +sources: +- https://github.com/kubernetes-sigs/node-feature-discovery +type: application +version: 0.17.0 diff --git a/network-operator/charts/node-feature-discovery/README.md b/network-operator/charts/node-feature-discovery/README.md new file mode 100644 index 0000000..02f7b17 --- /dev/null +++ b/network-operator/charts/node-feature-discovery/README.md @@ -0,0 +1,10 @@ +# Node Feature Discovery + +Node Feature Discovery (NFD) is a Kubernetes add-on for detecting hardware +features and system configuration. Detected features are advertised as node +labels. NFD provides flexible configuration and extension points for a wide +range of vendor and application specific node labeling needs. + +See +[NFD documentation](https://kubernetes-sigs.github.io/node-feature-discovery/v0.17/deployment/helm.html) +for deployment instructions. diff --git a/network-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml b/network-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml new file mode 100644 index 0000000..9f62da6 --- /dev/null +++ b/network-operator/charts/node-feature-discovery/crds/nfd-api-crds.yaml @@ -0,0 +1,711 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: nodefeatures.nfd.k8s-sigs.io +spec: + group: nfd.k8s-sigs.io + names: + kind: NodeFeature + listKind: NodeFeatureList + plural: nodefeatures + singular: nodefeature + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + NodeFeature resource holds the features discovered for one node in the + cluster. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Specification of the NodeFeature, containing features discovered + for a node. + properties: + features: + description: Features is the full "raw" features data that has been + discovered. + properties: + attributes: + additionalProperties: + description: AttributeFeatureSet is a set of features having + string value. + properties: + elements: + additionalProperties: + type: string + description: Individual features of the feature set. + type: object + required: + - elements + type: object + description: Attributes contains all the attribute-type features + of the node. + type: object + flags: + additionalProperties: + description: FlagFeatureSet is a set of simple features only + containing names without values. + properties: + elements: + additionalProperties: + description: |- + Nil is a dummy empty struct for protobuf compatibility. + NOTE: protobuf definitions have been removed but this is kept for API compatibility. + type: object + description: Individual features of the feature set. + type: object + required: + - elements + type: object + description: Flags contains all the flag-type features of the + node. + type: object + instances: + additionalProperties: + description: InstanceFeatureSet is a set of features each of + which is an instance having multiple attributes. + properties: + elements: + description: Individual features of the feature set. + items: + description: InstanceFeature represents one instance of + a complex features, e.g. a device. + properties: + attributes: + additionalProperties: + type: string + description: Attributes of the instance feature. + type: object + required: + - attributes + type: object + type: array + required: + - elements + type: object + description: Instances contains all the instance-type features + of the node. + type: object + type: object + labels: + additionalProperties: + type: string + description: Labels is the set of node labels that are requested to + be created. + type: object + type: object + required: + - spec + type: object + served: true + storage: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: nodefeaturegroups.nfd.k8s-sigs.io +spec: + group: nfd.k8s-sigs.io + names: + kind: NodeFeatureGroup + listKind: NodeFeatureGroupList + plural: nodefeaturegroups + shortNames: + - nfg + singular: nodefeaturegroup + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NodeFeatureGroup resource holds Node pools by featureGroup + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec defines the rules to be evaluated. + properties: + featureGroupRules: + description: List of rules to evaluate to determine nodes that belong + in this group. + items: + description: GroupRule defines a rule for nodegroup filtering. + properties: + matchAny: + description: MatchAny specifies a list of matchers one of which + must match. + items: + description: MatchAnyElem specifies one sub-matcher of MatchAny. + properties: + matchFeatures: + description: MatchFeatures specifies a set of matcher + terms all of which must match. + items: + description: |- + FeatureMatcherTerm defines requirements against one feature set. All + requirements (specified as MatchExpressions) are evaluated against each + element in the feature set. + properties: + feature: + description: Feature is the name of the feature + set to match against. + type: string + matchExpressions: + additionalProperties: + description: |- + MatchExpression specifies an expression to evaluate against a set of input + values. It contains an operator that is applied when matching the input and + an array of values that the operator evaluates the input against. + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Lt + - GtLt + - IsTrue + - IsFalse + type: string + value: + description: |- + Value is the list of values that the operand evaluates the input + against. Value should be empty if the operator is Exists, DoesNotExist, + IsTrue or IsFalse. Value should contain exactly one element if the + operator is Gt or Lt and exactly two elements if the operator is GtLt. + In other cases Value should contain at least one element. + items: + type: string + type: array + required: + - op + type: object + description: |- + MatchExpressions is the set of per-element expressions evaluated. These + match against the value of the specified elements. + type: object + matchName: + description: |- + MatchName in an expression that is matched against the name of each + element in the feature set. + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Lt + - GtLt + - IsTrue + - IsFalse + type: string + value: + description: |- + Value is the list of values that the operand evaluates the input + against. Value should be empty if the operator is Exists, DoesNotExist, + IsTrue or IsFalse. Value should contain exactly one element if the + operator is Gt or Lt and exactly two elements if the operator is GtLt. + In other cases Value should contain at least one element. + items: + type: string + type: array + required: + - op + type: object + required: + - feature + type: object + type: array + required: + - matchFeatures + type: object + type: array + matchFeatures: + description: MatchFeatures specifies a set of matcher terms + all of which must match. + items: + description: |- + FeatureMatcherTerm defines requirements against one feature set. All + requirements (specified as MatchExpressions) are evaluated against each + element in the feature set. + properties: + feature: + description: Feature is the name of the feature set to + match against. + type: string + matchExpressions: + additionalProperties: + description: |- + MatchExpression specifies an expression to evaluate against a set of input + values. It contains an operator that is applied when matching the input and + an array of values that the operator evaluates the input against. + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Lt + - GtLt + - IsTrue + - IsFalse + type: string + value: + description: |- + Value is the list of values that the operand evaluates the input + against. Value should be empty if the operator is Exists, DoesNotExist, + IsTrue or IsFalse. Value should contain exactly one element if the + operator is Gt or Lt and exactly two elements if the operator is GtLt. + In other cases Value should contain at least one element. + items: + type: string + type: array + required: + - op + type: object + description: |- + MatchExpressions is the set of per-element expressions evaluated. These + match against the value of the specified elements. + type: object + matchName: + description: |- + MatchName in an expression that is matched against the name of each + element in the feature set. + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Lt + - GtLt + - IsTrue + - IsFalse + type: string + value: + description: |- + Value is the list of values that the operand evaluates the input + against. Value should be empty if the operator is Exists, DoesNotExist, + IsTrue or IsFalse. Value should contain exactly one element if the + operator is Gt or Lt and exactly two elements if the operator is GtLt. + In other cases Value should contain at least one element. + items: + type: string + type: array + required: + - op + type: object + required: + - feature + type: object + type: array + name: + description: Name of the rule. + type: string + required: + - name + type: object + type: array + required: + - featureGroupRules + type: object + status: + description: |- + Status of the NodeFeatureGroup after the most recent evaluation of the + specification. + properties: + nodes: + description: Nodes is a list of FeatureGroupNode in the cluster that + match the featureGroupRules + items: + properties: + name: + description: Name of the node. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.3 + name: nodefeaturerules.nfd.k8s-sigs.io +spec: + group: nfd.k8s-sigs.io + names: + kind: NodeFeatureRule + listKind: NodeFeatureRuleList + plural: nodefeaturerules + shortNames: + - nfr + singular: nodefeaturerule + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: |- + NodeFeatureRule resource specifies a configuration for feature-based + customization of node objects, such as node labeling. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Spec defines the rules to be evaluated. + properties: + rules: + description: Rules is a list of node customization rules. + items: + description: Rule defines a rule for node customization such as + labeling. + properties: + annotations: + additionalProperties: + type: string + description: Annotations to create if the rule matches. + type: object + extendedResources: + additionalProperties: + type: string + description: ExtendedResources to create if the rule matches. + type: object + labels: + additionalProperties: + type: string + description: Labels to create if the rule matches. + type: object + labelsTemplate: + description: |- + LabelsTemplate specifies a template to expand for dynamically generating + multiple labels. Data (after template expansion) must be keys with an + optional value ([=]) separated by newlines. + type: string + matchAny: + description: MatchAny specifies a list of matchers one of which + must match. + items: + description: MatchAnyElem specifies one sub-matcher of MatchAny. + properties: + matchFeatures: + description: MatchFeatures specifies a set of matcher + terms all of which must match. + items: + description: |- + FeatureMatcherTerm defines requirements against one feature set. All + requirements (specified as MatchExpressions) are evaluated against each + element in the feature set. + properties: + feature: + description: Feature is the name of the feature + set to match against. + type: string + matchExpressions: + additionalProperties: + description: |- + MatchExpression specifies an expression to evaluate against a set of input + values. It contains an operator that is applied when matching the input and + an array of values that the operator evaluates the input against. + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Lt + - GtLt + - IsTrue + - IsFalse + type: string + value: + description: |- + Value is the list of values that the operand evaluates the input + against. Value should be empty if the operator is Exists, DoesNotExist, + IsTrue or IsFalse. Value should contain exactly one element if the + operator is Gt or Lt and exactly two elements if the operator is GtLt. + In other cases Value should contain at least one element. + items: + type: string + type: array + required: + - op + type: object + description: |- + MatchExpressions is the set of per-element expressions evaluated. These + match against the value of the specified elements. + type: object + matchName: + description: |- + MatchName in an expression that is matched against the name of each + element in the feature set. + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Lt + - GtLt + - IsTrue + - IsFalse + type: string + value: + description: |- + Value is the list of values that the operand evaluates the input + against. Value should be empty if the operator is Exists, DoesNotExist, + IsTrue or IsFalse. Value should contain exactly one element if the + operator is Gt or Lt and exactly two elements if the operator is GtLt. + In other cases Value should contain at least one element. + items: + type: string + type: array + required: + - op + type: object + required: + - feature + type: object + type: array + required: + - matchFeatures + type: object + type: array + matchFeatures: + description: MatchFeatures specifies a set of matcher terms + all of which must match. + items: + description: |- + FeatureMatcherTerm defines requirements against one feature set. All + requirements (specified as MatchExpressions) are evaluated against each + element in the feature set. + properties: + feature: + description: Feature is the name of the feature set to + match against. + type: string + matchExpressions: + additionalProperties: + description: |- + MatchExpression specifies an expression to evaluate against a set of input + values. It contains an operator that is applied when matching the input and + an array of values that the operator evaluates the input against. + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Lt + - GtLt + - IsTrue + - IsFalse + type: string + value: + description: |- + Value is the list of values that the operand evaluates the input + against. Value should be empty if the operator is Exists, DoesNotExist, + IsTrue or IsFalse. Value should contain exactly one element if the + operator is Gt or Lt and exactly two elements if the operator is GtLt. + In other cases Value should contain at least one element. + items: + type: string + type: array + required: + - op + type: object + description: |- + MatchExpressions is the set of per-element expressions evaluated. These + match against the value of the specified elements. + type: object + matchName: + description: |- + MatchName in an expression that is matched against the name of each + element in the feature set. + properties: + op: + description: Op is the operator to be applied. + enum: + - In + - NotIn + - InRegexp + - Exists + - DoesNotExist + - Gt + - Lt + - GtLt + - IsTrue + - IsFalse + type: string + value: + description: |- + Value is the list of values that the operand evaluates the input + against. Value should be empty if the operator is Exists, DoesNotExist, + IsTrue or IsFalse. Value should contain exactly one element if the + operator is Gt or Lt and exactly two elements if the operator is GtLt. + In other cases Value should contain at least one element. + items: + type: string + type: array + required: + - op + type: object + required: + - feature + type: object + type: array + name: + description: Name of the rule. + type: string + taints: + description: Taints to create if the rule matches. + items: + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. + properties: + effect: + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to + a node. + type: string + timeAdded: + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint + key. + type: string + required: + - effect + - key + type: object + type: array + vars: + additionalProperties: + type: string + description: |- + Vars is the variables to store if the rule matches. Variables do not + directly inflict any changes in the node object. However, they can be + referenced from other rules enabling more complex rule hierarchies, + without exposing intermediary output values as labels. + type: object + varsTemplate: + description: |- + VarsTemplate specifies a template to expand for dynamically generating + multiple variables. Data (after template expansion) must be keys with an + optional value ([=]) separated by newlines. + type: string + required: + - name + type: object + type: array + required: + - rules + type: object + required: + - spec + type: object + served: true + storage: true diff --git a/network-operator/charts/node-feature-discovery/templates/_helpers.tpl b/network-operator/charts/node-feature-discovery/templates/_helpers.tpl new file mode 100644 index 0000000..928ece7 --- /dev/null +++ b/network-operator/charts/node-feature-discovery/templates/_helpers.tpl @@ -0,0 +1,107 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "node-feature-discovery.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "node-feature-discovery.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "node-feature-discovery.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "node-feature-discovery.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "node-feature-discovery.labels" -}} +helm.sh/chart: {{ include "node-feature-discovery.chart" . }} +{{ include "node-feature-discovery.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} + +{{/* +Selector labels +*/}} +{{- define "node-feature-discovery.selectorLabels" -}} +app.kubernetes.io/name: {{ include "node-feature-discovery.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end -}} + +{{/* +Create the name of the service account which the nfd master will use +*/}} +{{- define "node-feature-discovery.master.serviceAccountName" -}} +{{- if .Values.master.serviceAccount.create -}} + {{ default (include "node-feature-discovery.fullname" .) .Values.master.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.master.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account which the nfd worker will use +*/}} +{{- define "node-feature-discovery.worker.serviceAccountName" -}} +{{- if .Values.worker.serviceAccount.create -}} + {{ default (printf "%s-worker" (include "node-feature-discovery.fullname" .)) .Values.worker.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.worker.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account which topologyUpdater will use +*/}} +{{- define "node-feature-discovery.topologyUpdater.serviceAccountName" -}} +{{- if .Values.topologyUpdater.serviceAccount.create -}} + {{ default (printf "%s-topology-updater" (include "node-feature-discovery.fullname" .)) .Values.topologyUpdater.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.topologyUpdater.serviceAccount.name }} +{{- end -}} +{{- end -}} + +{{/* +Create the name of the service account which nfd-gc will use +*/}} +{{- define "node-feature-discovery.gc.serviceAccountName" -}} +{{- if .Values.gc.serviceAccount.create -}} + {{ default (printf "%s-gc" (include "node-feature-discovery.fullname" .)) .Values.gc.serviceAccount.name }} +{{- else -}} + {{ default "default" .Values.gc.serviceAccount.name }} +{{- end -}} +{{- end -}} diff --git a/network-operator/charts/node-feature-discovery/templates/clusterrole.yaml b/network-operator/charts/node-feature-discovery/templates/clusterrole.yaml new file mode 100644 index 0000000..ea6e3e3 --- /dev/null +++ b/network-operator/charts/node-feature-discovery/templates/clusterrole.yaml @@ -0,0 +1,140 @@ +{{- if and .Values.master.enable .Values.master.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "node-feature-discovery.fullname" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - namespaces + verbs: + - watch + - list +- apiGroups: + - "" + resources: + - nodes + - nodes/status + verbs: + - get + - patch + - update + - list +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeatures + - nodefeaturerules + - nodefeaturegroups + verbs: + - get + - list + - watch +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeaturegroups/status + verbs: + - patch + - update +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - coordination.k8s.io + resources: + - leases + resourceNames: + - "nfd-master.nfd.kubernetes.io" + verbs: + - get + - update +{{- end }} + +{{- if and .Values.topologyUpdater.enable .Values.topologyUpdater.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-topology-updater + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list +- apiGroups: + - "" + resources: + - namespaces + verbs: + - get +- apiGroups: + - "" + resources: + - nodes/proxy + verbs: + - get +- apiGroups: + - "" + resources: + - pods + verbs: + - get +- apiGroups: + - topology.node.k8s.io + resources: + - noderesourcetopologies + verbs: + - create + - get + - update +{{- end }} + +{{- if and .Values.gc.enable .Values.gc.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-gc + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes/proxy + verbs: + - get +- apiGroups: + - topology.node.k8s.io + resources: + - noderesourcetopologies + verbs: + - delete + - list +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeatures + verbs: + - delete + - list +{{- end }} diff --git a/network-operator/charts/node-feature-discovery/templates/clusterrolebinding.yaml b/network-operator/charts/node-feature-discovery/templates/clusterrolebinding.yaml new file mode 100644 index 0000000..8331019 --- /dev/null +++ b/network-operator/charts/node-feature-discovery/templates/clusterrolebinding.yaml @@ -0,0 +1,52 @@ +{{- if and .Values.master.enable .Values.master.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "node-feature-discovery.fullname" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "node-feature-discovery.fullname" . }} +subjects: +- kind: ServiceAccount + name: {{ include "node-feature-discovery.master.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} +{{- end }} + +{{- if and .Values.topologyUpdater.enable .Values.topologyUpdater.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-topology-updater + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "node-feature-discovery.fullname" . }}-topology-updater +subjects: +- kind: ServiceAccount + name: {{ include "node-feature-discovery.topologyUpdater.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} +{{- end }} + +{{- if and .Values.gc.enable .Values.gc.rbac.create }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-gc + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "node-feature-discovery.fullname" . }}-gc +subjects: +- kind: ServiceAccount + name: {{ include "node-feature-discovery.gc.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} +{{- end }} diff --git a/network-operator/charts/node-feature-discovery/templates/master.yaml b/network-operator/charts/node-feature-discovery/templates/master.yaml new file mode 100644 index 0000000..da3ca24 --- /dev/null +++ b/network-operator/charts/node-feature-discovery/templates/master.yaml @@ -0,0 +1,170 @@ +{{- if .Values.master.enable }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-master + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + role: master + {{- with .Values.master.deploymentAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.master.replicaCount }} + revisionHistoryLimit: {{ .Values.master.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} + role: master + template: + metadata: + labels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 8 }} + role: master + annotations: + checksum/config: {{ include (print $.Template.BasePath "/nfd-master-conf.yaml") . | sha256sum }} + {{- with .Values.master.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "node-feature-discovery.master.serviceAccountName" . }} + enableServiceLinks: false + securityContext: + {{- toYaml .Values.master.podSecurityContext | nindent 8 }} + hostNetwork: {{ .Values.master.hostNetwork }} + containers: + - name: master + securityContext: + {{- toYaml .Values.master.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + startupProbe: + grpc: + port: {{ .Values.master.healthPort | default "8082" }} + {{- with .Values.master.startupProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.master.startupProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.master.startupProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.master.startupProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + livenessProbe: + grpc: + port: {{ .Values.master.healthPort | default "8082" }} + {{- with .Values.master.livenessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.master.livenessProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.master.livenessProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.master.livenessProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + readinessProbe: + grpc: + port: {{ .Values.master.healthPort | default "8082" }} + {{- with .Values.master.readinessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.master.readinessProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.master.readinessProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.master.readinessProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + {{- with .Values.master.readinessProbe.successThreshold }} + successThreshold: {{ . }} + {{- end }} + ports: + - containerPort: {{ .Values.master.metricsPort | default "8081" }} + name: metrics + - containerPort: {{ .Values.master.healthPort | default "8082" }} + name: health + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- with .Values.master.extraEnvs }} + {{- toYaml . | nindent 8 }} + {{- end}} + command: + - "nfd-master" + resources: + {{- toYaml .Values.master.resources | nindent 12 }} + args: + {{- if .Values.master.instance | empty | not }} + - "-instance={{ .Values.master.instance }}" + {{- end }} + - "-enable-leader-election" + {{- if .Values.master.extraLabelNs | empty | not }} + - "-extra-label-ns={{- join "," .Values.master.extraLabelNs }}" + {{- end }} + {{- if .Values.master.denyLabelNs | empty | not }} + - "-deny-label-ns={{- join "," .Values.master.denyLabelNs }}" + {{- end }} + {{- if .Values.master.enableTaints }} + - "-enable-taints" + {{- end }} + {{- if .Values.master.featureRulesController | kindIs "invalid" | not }} + - "-featurerules-controller={{ .Values.master.featureRulesController }}" + {{- end }} + {{- if .Values.master.resyncPeriod }} + - "-resync-period={{ .Values.master.resyncPeriod }}" + {{- end }} + {{- if .Values.master.nfdApiParallelism | empty | not }} + - "-nfd-api-parallelism={{ .Values.master.nfdApiParallelism }}" + {{- end }} + # Go over featureGates and add the feature-gate flag + {{- range $key, $value := .Values.featureGates }} + - "-feature-gates={{ $key }}={{ $value }}" + {{- end }} + - "-metrics={{ .Values.master.metricsPort | default "8081" }}" + - "-grpc-health={{ .Values.master.healthPort | default "8082" }}" + {{- with .Values.master.extraArgs }} + {{- toYaml . | nindent 12 }} + {{- end }} + volumeMounts: + - name: nfd-master-conf + mountPath: "/etc/kubernetes/node-feature-discovery" + readOnly: true + volumes: + - name: nfd-master-conf + configMap: + name: {{ include "node-feature-discovery.fullname" . }}-master-conf + items: + - key: nfd-master.conf + path: nfd-master.conf + {{- with .Values.master.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.master.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.master.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/network-operator/charts/node-feature-discovery/templates/nfd-gc.yaml b/network-operator/charts/node-feature-discovery/templates/nfd-gc.yaml new file mode 100644 index 0000000..3642aa6 --- /dev/null +++ b/network-operator/charts/node-feature-discovery/templates/nfd-gc.yaml @@ -0,0 +1,88 @@ +{{- if and .Values.gc.enable -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-gc + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + role: gc + {{- with .Values.gc.deploymentAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + replicas: {{ .Values.gc.replicaCount | default 1 }} + revisionHistoryLimit: {{ .Values.gc.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} + role: gc + template: + metadata: + labels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 8 }} + role: gc + {{- with .Values.gc.annotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "node-feature-discovery.gc.serviceAccountName" . }} + dnsPolicy: ClusterFirstWithHostNet + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.gc.podSecurityContext | nindent 8 }} + hostNetwork: {{ .Values.gc.hostNetwork }} + containers: + - name: gc + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- with .Values.gc.extraEnvs }} + {{- toYaml . | nindent 8 }} + {{- end}} + command: + - "nfd-gc" + args: + {{- if .Values.gc.interval | empty | not }} + - "-gc-interval={{ .Values.gc.interval }}" + {{- end }} + {{- with .Values.gc.extraArgs }} + {{- toYaml . | nindent 10 }} + {{- end }} + resources: + {{- toYaml .Values.gc.resources | nindent 12 }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ "ALL" ] + readOnlyRootFilesystem: true + runAsNonRoot: true + ports: + - name: metrics + containerPort: {{ .Values.gc.metricsPort | default "8081"}} + + {{- with .Values.gc.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gc.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.gc.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/network-operator/charts/node-feature-discovery/templates/nfd-master-conf.yaml b/network-operator/charts/node-feature-discovery/templates/nfd-master-conf.yaml new file mode 100644 index 0000000..9c6e01c --- /dev/null +++ b/network-operator/charts/node-feature-discovery/templates/nfd-master-conf.yaml @@ -0,0 +1,12 @@ +{{- if .Values.master.enable }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-master-conf + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +data: + nfd-master.conf: |- + {{- .Values.master.config | toYaml | nindent 4 }} +{{- end }} diff --git a/network-operator/charts/node-feature-discovery/templates/nfd-topologyupdater-conf.yaml b/network-operator/charts/node-feature-discovery/templates/nfd-topologyupdater-conf.yaml new file mode 100644 index 0000000..8d03aa2 --- /dev/null +++ b/network-operator/charts/node-feature-discovery/templates/nfd-topologyupdater-conf.yaml @@ -0,0 +1,12 @@ +{{- if .Values.topologyUpdater.enable -}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-topology-updater-conf + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +data: + nfd-topology-updater.conf: |- + {{- .Values.topologyUpdater.config | toYaml | nindent 4 }} +{{- end }} diff --git a/network-operator/charts/node-feature-discovery/templates/nfd-worker-conf.yaml b/network-operator/charts/node-feature-discovery/templates/nfd-worker-conf.yaml new file mode 100644 index 0000000..a2299de --- /dev/null +++ b/network-operator/charts/node-feature-discovery/templates/nfd-worker-conf.yaml @@ -0,0 +1,12 @@ +{{- if .Values.worker.enable }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-worker-conf + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +data: + nfd-worker.conf: |- + {{- .Values.worker.config | toYaml | nindent 4 }} +{{- end }} diff --git a/network-operator/charts/node-feature-discovery/templates/post-delete-job.yaml b/network-operator/charts/node-feature-discovery/templates/post-delete-job.yaml new file mode 100644 index 0000000..4364f1a --- /dev/null +++ b/network-operator/charts/node-feature-discovery/templates/post-delete-job.yaml @@ -0,0 +1,94 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-prune + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-prune + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +rules: +- apiGroups: + - "" + resources: + - nodes + - nodes/status + verbs: + - get + - patch + - update + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-prune + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "node-feature-discovery.fullname" . }}-prune +subjects: +- kind: ServiceAccount + name: {{ include "node-feature-discovery.fullname" . }}-prune + namespace: {{ include "node-feature-discovery.namespace" . }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-prune + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": post-delete + "helm.sh/hook-delete-policy": before-hook-creation,hook-succeeded +spec: + template: + metadata: + labels: + {{- include "node-feature-discovery.labels" . | nindent 8 }} + role: prune + spec: + serviceAccountName: {{ include "node-feature-discovery.fullname" . }}-prune + containers: + - name: nfd-master + securityContext: + {{- toYaml .Values.master.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - "nfd-master" + args: + - "-prune" + {{- if .Values.master.instance | empty | not }} + - "-instance={{ .Values.master.instance }}" + {{- end }} + restartPolicy: Never + {{- with .Values.master.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.master.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.master.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/network-operator/charts/node-feature-discovery/templates/prometheus.yaml b/network-operator/charts/node-feature-discovery/templates/prometheus.yaml new file mode 100644 index 0000000..3d680e2 --- /dev/null +++ b/network-operator/charts/node-feature-discovery/templates/prometheus.yaml @@ -0,0 +1,26 @@ +{{- if .Values.prometheus.enable }} +# Prometheus Monitor Service (Metrics) +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ include "node-feature-discovery.fullname" . }} + labels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 4 }} + {{- with .Values.prometheus.labels }} + {{ toYaml . | nindent 4 }} + {{- end }} +spec: + podMetricsEndpoints: + - honorLabels: true + interval: {{ .Values.prometheus.scrapeInterval }} + path: /metrics + port: metrics + scheme: http + namespaceSelector: + matchNames: + - {{ include "node-feature-discovery.namespace" . }} + selector: + matchExpressions: + - {key: app.kubernetes.io/instance, operator: In, values: ["{{ .Release.Name }}"]} + - {key: app.kubernetes.io/name, operator: In, values: ["{{ include "node-feature-discovery.name" . }}"]} +{{- end }} diff --git a/network-operator/charts/node-feature-discovery/templates/role.yaml b/network-operator/charts/node-feature-discovery/templates/role.yaml new file mode 100644 index 0000000..52c69eb --- /dev/null +++ b/network-operator/charts/node-feature-discovery/templates/role.yaml @@ -0,0 +1,24 @@ +{{- if and .Values.worker.enable .Values.worker.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-worker + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +rules: +- apiGroups: + - nfd.k8s-sigs.io + resources: + - nodefeatures + verbs: + - create + - get + - update +- apiGroups: + - "" + resources: + - pods + verbs: + - get +{{- end }} diff --git a/network-operator/charts/node-feature-discovery/templates/rolebinding.yaml b/network-operator/charts/node-feature-discovery/templates/rolebinding.yaml new file mode 100644 index 0000000..a640d5f --- /dev/null +++ b/network-operator/charts/node-feature-discovery/templates/rolebinding.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.worker.enable .Values.worker.rbac.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-worker + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ include "node-feature-discovery.fullname" . }}-worker +subjects: +- kind: ServiceAccount + name: {{ include "node-feature-discovery.worker.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} +{{- end }} + diff --git a/network-operator/charts/node-feature-discovery/templates/serviceaccount.yaml b/network-operator/charts/node-feature-discovery/templates/serviceaccount.yaml new file mode 100644 index 0000000..47c75a7 --- /dev/null +++ b/network-operator/charts/node-feature-discovery/templates/serviceaccount.yaml @@ -0,0 +1,58 @@ +{{- if and .Values.master.enable .Values.master.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "node-feature-discovery.master.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + {{- with .Values.master.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} + +{{- if and .Values.topologyUpdater.enable .Values.topologyUpdater.serviceAccount.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "node-feature-discovery.topologyUpdater.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + {{- with .Values.topologyUpdater.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} + +{{- if and .Values.gc.enable .Values.gc.serviceAccount.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "node-feature-discovery.gc.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + {{- with .Values.gc.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} + +{{- if and .Values.worker.enable .Values.worker.serviceAccount.create }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "node-feature-discovery.worker.serviceAccountName" . }} + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + {{- with .Values.worker.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/network-operator/charts/node-feature-discovery/templates/topologyupdater-crds.yaml b/network-operator/charts/node-feature-discovery/templates/topologyupdater-crds.yaml new file mode 100644 index 0000000..b6b9196 --- /dev/null +++ b/network-operator/charts/node-feature-discovery/templates/topologyupdater-crds.yaml @@ -0,0 +1,278 @@ +{{- if and .Values.topologyUpdater.enable .Values.topologyUpdater.createCRDs -}} +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + api-approved.kubernetes.io: https://github.com/kubernetes/enhancements/pull/1870 + controller-gen.kubebuilder.io/version: v0.11.2 + creationTimestamp: null + name: noderesourcetopologies.topology.node.k8s.io +spec: + group: topology.node.k8s.io + names: + kind: NodeResourceTopology + listKind: NodeResourceTopologyList + plural: noderesourcetopologies + shortNames: + - node-res-topo + singular: noderesourcetopology + scope: Cluster + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: NodeResourceTopology describes node resources and their topology. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + topologyPolicies: + items: + type: string + type: array + zones: + description: ZoneList contains an array of Zone objects. + items: + description: Zone represents a resource topology zone, e.g. socket, + node, die or core. + properties: + attributes: + description: AttributeList contains an array of AttributeInfo objects. + items: + description: AttributeInfo contains one attribute of a Zone. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + costs: + description: CostList contains an array of CostInfo objects. + items: + description: CostInfo describes the cost (or distance) between + two Zones. + properties: + name: + type: string + value: + format: int64 + type: integer + required: + - name + - value + type: object + type: array + name: + type: string + parent: + type: string + resources: + description: ResourceInfoList contains an array of ResourceInfo + objects. + items: + description: ResourceInfo contains information about one resource + type. + properties: + allocatable: + anyOf: + - type: integer + - type: string + description: Allocatable quantity of the resource, corresponding + to allocatable in node status, i.e. total amount of this + resource available to be used by pods. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + available: + anyOf: + - type: integer + - type: string + description: Available is the amount of this resource currently + available for new (to be scheduled) pods, i.e. Allocatable + minus the resources reserved by currently running pods. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + capacity: + anyOf: + - type: integer + - type: string + description: Capacity of the resource, corresponding to capacity + in node status, i.e. total amount of this resource that + the node has. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: Name of the resource. + type: string + required: + - allocatable + - available + - capacity + - name + type: object + type: array + type: + type: string + required: + - name + - type + type: object + type: array + required: + - topologyPolicies + - zones + type: object + served: true + storage: false + - name: v1alpha2 + schema: + openAPIV3Schema: + description: NodeResourceTopology describes node resources and their topology. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + attributes: + description: AttributeList contains an array of AttributeInfo objects. + items: + description: AttributeInfo contains one attribute of a Zone. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + topologyPolicies: + description: 'DEPRECATED (to be removed in v1beta1): use top level attributes + if needed' + items: + type: string + type: array + zones: + description: ZoneList contains an array of Zone objects. + items: + description: Zone represents a resource topology zone, e.g. socket, + node, die or core. + properties: + attributes: + description: AttributeList contains an array of AttributeInfo objects. + items: + description: AttributeInfo contains one attribute of a Zone. + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + costs: + description: CostList contains an array of CostInfo objects. + items: + description: CostInfo describes the cost (or distance) between + two Zones. + properties: + name: + type: string + value: + format: int64 + type: integer + required: + - name + - value + type: object + type: array + name: + type: string + parent: + type: string + resources: + description: ResourceInfoList contains an array of ResourceInfo + objects. + items: + description: ResourceInfo contains information about one resource + type. + properties: + allocatable: + anyOf: + - type: integer + - type: string + description: Allocatable quantity of the resource, corresponding + to allocatable in node status, i.e. total amount of this + resource available to be used by pods. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + available: + anyOf: + - type: integer + - type: string + description: Available is the amount of this resource currently + available for new (to be scheduled) pods, i.e. Allocatable + minus the resources reserved by currently running pods. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + capacity: + anyOf: + - type: integer + - type: string + description: Capacity of the resource, corresponding to capacity + in node status, i.e. total amount of this resource that + the node has. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + name: + description: Name of the resource. + type: string + required: + - allocatable + - available + - capacity + - name + type: object + type: array + type: + type: string + required: + - name + - type + type: object + type: array + required: + - zones + type: object + served: true + storage: true +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +{{- end }} diff --git a/network-operator/charts/node-feature-discovery/templates/topologyupdater.yaml b/network-operator/charts/node-feature-discovery/templates/topologyupdater.yaml new file mode 100644 index 0000000..9a466f8 --- /dev/null +++ b/network-operator/charts/node-feature-discovery/templates/topologyupdater.yaml @@ -0,0 +1,188 @@ +{{- if .Values.topologyUpdater.enable -}} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-topology-updater + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + role: topology-updater + {{- with .Values.topologyUpdater.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + revisionHistoryLimit: {{ .Values.topologyUpdater.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} + role: topology-updater + template: + metadata: + labels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 8 }} + role: topology-updater + annotations: + checksum/config: {{ include (print $.Template.BasePath "/nfd-topologyupdater-conf.yaml") . | sha256sum }} + {{- with .Values.topologyUpdater.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + serviceAccountName: {{ include "node-feature-discovery.topologyUpdater.serviceAccountName" . }} + dnsPolicy: ClusterFirstWithHostNet + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + {{- toYaml .Values.topologyUpdater.podSecurityContext | nindent 8 }} + hostNetwork: {{ .Values.topologyUpdater.hostNetwork }} + containers: + - name: topology-updater + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + livenessProbe: + grpc: + port: {{ .Values.topologyUpdater.healthPort | default "8082" }} + {{- with .Values.topologyUpdater.livenessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.topologyUpdater.livenessProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.topologyUpdater.livenessProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.topologyUpdater.livenessProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + readinessProbe: + grpc: + port: {{ .Values.topologyUpdater.healthPort | default "8082" }} + {{- with .Values.topologyUpdater.readinessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.topologyUpdater.readinessProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.topologyUpdater.readinessProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.topologyUpdater.readinessProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + {{- with .Values.topologyUpdater.readinessProbe.successThreshold }} + successThreshold: {{ . }} + {{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NODE_ADDRESS + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- with .Values.topologyUpdater.extraEnvs }} + {{- toYaml . | nindent 8 }} + {{- end}} + command: + - "nfd-topology-updater" + args: + - "-podresources-socket=/host-var/lib/kubelet-podresources/kubelet.sock" + {{- if .Values.topologyUpdater.updateInterval | empty | not }} + - "-sleep-interval={{ .Values.topologyUpdater.updateInterval }}" + {{- else }} + - "-sleep-interval=3s" + {{- end }} + {{- if .Values.topologyUpdater.watchNamespace | empty | not }} + - "-watch-namespace={{ .Values.topologyUpdater.watchNamespace }}" + {{- else }} + - "-watch-namespace=*" + {{- end }} + {{- if not .Values.topologyUpdater.podSetFingerprint }} + - "-pods-fingerprint=false" + {{- end }} + {{- if .Values.topologyUpdater.kubeletConfigPath | empty | not }} + - "-kubelet-config-uri=file:///host-var/kubelet-config" + {{- end }} + {{- if .Values.topologyUpdater.kubeletStateDir | empty }} + # Disable kubelet state tracking by giving an empty path + - "-kubelet-state-dir=" + {{- end }} + - "-metrics={{ .Values.topologyUpdater.metricsPort | default "8081"}}" + - "-grpc-health={{ .Values.topologyUpdater.healthPort | default "8082" }}" + {{- with .Values.topologyUpdater.extraArgs }} + {{- toYaml . | nindent 10 }} + {{- end }} + ports: + - containerPort: {{ .Values.topologyUpdater.metricsPort | default "8081"}} + name: metrics + - containerPort: {{ .Values.topologyUpdater.healthPort | default "8082" }} + name: health + volumeMounts: + {{- if .Values.topologyUpdater.kubeletConfigPath | empty | not }} + - name: kubelet-config + mountPath: /host-var/kubelet-config + {{- end }} + - name: kubelet-podresources-sock + mountPath: /host-var/lib/kubelet-podresources/kubelet.sock + - name: host-sys + mountPath: /host-sys + {{- if .Values.topologyUpdater.kubeletStateDir | empty | not }} + - name: kubelet-state-files + mountPath: /host-var/lib/kubelet + readOnly: true + {{- end }} + - name: nfd-topology-updater-conf + mountPath: "/etc/kubernetes/node-feature-discovery" + readOnly: true + + resources: + {{- toYaml .Values.topologyUpdater.resources | nindent 12 }} + securityContext: + {{- toYaml .Values.topologyUpdater.securityContext | nindent 12 }} + volumes: + - name: host-sys + hostPath: + path: "/sys" + {{- if .Values.topologyUpdater.kubeletConfigPath | empty | not }} + - name: kubelet-config + hostPath: + path: {{ .Values.topologyUpdater.kubeletConfigPath }} + {{- end }} + - name: kubelet-podresources-sock + hostPath: + {{- if .Values.topologyUpdater.kubeletPodResourcesSockPath | empty | not }} + path: {{ .Values.topologyUpdater.kubeletPodResourcesSockPath }} + {{- else }} + path: /var/lib/kubelet/pod-resources/kubelet.sock + {{- end }} + {{- if .Values.topologyUpdater.kubeletStateDir | empty | not }} + - name: kubelet-state-files + hostPath: + path: {{ .Values.topologyUpdater.kubeletStateDir }} + {{- end }} + - name: nfd-topology-updater-conf + configMap: + name: {{ include "node-feature-discovery.fullname" . }}-topology-updater-conf + items: + - key: nfd-topology-updater.conf + path: nfd-topology-updater.conf + + {{- with .Values.topologyUpdater.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.topologyUpdater.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.topologyUpdater.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/network-operator/charts/node-feature-discovery/templates/worker.yaml b/network-operator/charts/node-feature-discovery/templates/worker.yaml new file mode 100644 index 0000000..5fd1ab7 --- /dev/null +++ b/network-operator/charts/node-feature-discovery/templates/worker.yaml @@ -0,0 +1,195 @@ +{{- if .Values.worker.enable }} +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ include "node-feature-discovery.fullname" . }}-worker + namespace: {{ include "node-feature-discovery.namespace" . }} + labels: + {{- include "node-feature-discovery.labels" . | nindent 4 }} + role: worker + {{- with .Values.worker.daemonsetAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + revisionHistoryLimit: {{ .Values.worker.revisionHistoryLimit }} + selector: + matchLabels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 6 }} + role: worker + template: + metadata: + labels: + {{- include "node-feature-discovery.selectorLabels" . | nindent 8 }} + role: worker + annotations: + checksum/config: {{ include (print $.Template.BasePath "/nfd-worker-conf.yaml") . | sha256sum }} + {{- with .Values.worker.annotations }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + dnsPolicy: ClusterFirstWithHostNet + {{- with .Values.priorityClassName }} + priorityClassName: {{ . }} + {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "node-feature-discovery.worker.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.worker.podSecurityContext | nindent 8 }} + hostNetwork: {{ .Values.worker.hostNetwork }} + containers: + - name: worker + securityContext: + {{- toYaml .Values.worker.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + livenessProbe: + grpc: + port: {{ .Values.worker.healthPort | default "8082" }} + {{- with .Values.worker.livenessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.worker.livenessProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.worker.livenessProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.worker.livenessProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + readinessProbe: + grpc: + port: {{ .Values.worker.healthPort | default "8082" }} + {{- with .Values.worker.readinessProbe.initialDelaySeconds }} + initialDelaySeconds: {{ . }} + {{- end }} + {{- with .Values.worker.readinessProbe.failureThreshold }} + failureThreshold: {{ . }} + {{- end }} + {{- with .Values.worker.readinessProbe.periodSeconds }} + periodSeconds: {{ . }} + {{- end }} + {{- with .Values.worker.readinessProbe.timeoutSeconds }} + timeoutSeconds: {{ . }} + {{- end }} + {{- with .Values.worker.readinessProbe.successThreshold }} + successThreshold: {{ . }} + {{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + {{- with .Values.worker.extraEnvs }} + {{- toYaml . | nindent 8 }} + {{- end}} + resources: + {{- toYaml .Values.worker.resources | nindent 12 }} + command: + - "nfd-worker" + args: + # Go over featureGate and add the feature-gate flag + {{- range $key, $value := .Values.featureGates }} + - "-feature-gates={{ $key }}={{ $value }}" + {{- end }} + - "-metrics={{ .Values.worker.metricsPort | default "8081"}}" + - "-grpc-health={{ .Values.worker.healthPort | default "8082" }}" + {{- with .Values.gc.extraArgs }} + {{- toYaml . | nindent 8 }} + {{- end }} + ports: + - containerPort: {{ .Values.worker.metricsPort | default "8081"}} + name: metrics + - containerPort: {{ .Values.worker.healthPort | default "8082" }} + name: health + volumeMounts: + - name: host-boot + mountPath: "/host-boot" + readOnly: true + - name: host-os-release + mountPath: "/host-etc/os-release" + readOnly: true + - name: host-sys + mountPath: "/host-sys" + readOnly: true + - name: host-usr-lib + mountPath: "/host-usr/lib" + readOnly: true + - name: host-lib + mountPath: "/host-lib" + readOnly: true + - name: host-proc-swaps + mountPath: "/host-proc/swaps" + readOnly: true + {{- if .Values.worker.mountUsrSrc }} + - name: host-usr-src + mountPath: "/host-usr/src" + readOnly: true + {{- end }} + - name: features-d + mountPath: "/etc/kubernetes/node-feature-discovery/features.d/" + readOnly: true + - name: nfd-worker-conf + mountPath: "/etc/kubernetes/node-feature-discovery" + readOnly: true + volumes: + - name: host-boot + hostPath: + path: "/boot" + - name: host-os-release + hostPath: + path: "/etc/os-release" + - name: host-sys + hostPath: + path: "/sys" + - name: host-usr-lib + hostPath: + path: "/usr/lib" + - name: host-lib + hostPath: + path: "/lib" + - name: host-proc-swaps + hostPath: + path: "/proc/swaps" + {{- if .Values.worker.mountUsrSrc }} + - name: host-usr-src + hostPath: + path: "/usr/src" + {{- end }} + - name: features-d + hostPath: + path: "/etc/kubernetes/node-feature-discovery/features.d/" + - name: nfd-worker-conf + configMap: + name: {{ include "node-feature-discovery.fullname" . }}-worker-conf + items: + - key: nfd-worker.conf + path: nfd-worker.conf + {{- with .Values.worker.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.worker.priorityClassName }} + priorityClassName: {{ . | quote }} + {{- end }} +{{- end }} diff --git a/network-operator/charts/node-feature-discovery/values.yaml b/network-operator/charts/node-feature-discovery/values.yaml new file mode 100644 index 0000000..18aa7bc --- /dev/null +++ b/network-operator/charts/node-feature-discovery/values.yaml @@ -0,0 +1,599 @@ +image: + repository: registry.k8s.io/nfd/node-feature-discovery + # This should be set to 'IfNotPresent' for released version + pullPolicy: IfNotPresent + # tag, if defined will use the given image tag, else Chart.AppVersion will be used + # tag +imagePullSecrets: [] + +nameOverride: "" +fullnameOverride: "" +namespaceOverride: "" + +featureGates: + NodeFeatureGroupAPI: false + +priorityClassName: "" + +master: + enable: true + extraArgs: [] + extraEnvs: [] + hostNetwork: false + config: ### + # noPublish: false + # autoDefaultNs: true + # extraLabelNs: ["added.ns.io","added.kubernets.io"] + # denyLabelNs: ["denied.ns.io","denied.kubernetes.io"] + # enableTaints: false + # labelWhiteList: "foo" + # resyncPeriod: "2h" + # restrictions: + # disableLabels: true + # disableTaints: true + # disableExtendedResources: true + # disableAnnotations: true + # allowOverwrite: false + # denyNodeFeatureLabels: true + # nodeFeatureNamespaceSelector: + # matchLabels: + # kubernetes.io/metadata.name: "node-feature-discovery" + # matchExpressions: + # - key: "kubernetes.io/metadata.name" + # operator: "In" + # values: + # - "node-feature-discovery" + # klog: + # addDirHeader: false + # alsologtostderr: false + # logBacktraceAt: + # logtostderr: true + # skipHeaders: false + # stderrthreshold: 2 + # v: 0 + # vmodule: + ## NOTE: the following options are not dynamically run-time configurable + ## and require a nfd-master restart to take effect after being changed + # logDir: + # logFile: + # logFileMaxSize: 1800 + # skipLogHeaders: false + # leaderElection: + # leaseDuration: 15s + # # this value has to be lower than leaseDuration and greater than retryPeriod*1.2 + # renewDeadline: 10s + # # this value has to be greater than 0 + # retryPeriod: 2s + # nfdApiParallelism: 10 + ### + metricsPort: 8081 + healthPort: 8082 + instance: + featureApi: + resyncPeriod: + denyLabelNs: [] + extraLabelNs: [] + enableTaints: false + featureRulesController: null + nfdApiParallelism: null + deploymentAnnotations: {} + replicaCount: 1 + + podSecurityContext: {} + # fsGroup: 2000 + + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ "ALL" ] + readOnlyRootFilesystem: true + runAsNonRoot: true + # runAsUser: 1000 + + serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + + # specify how many old ReplicaSets for the Deployment to retain. + revisionHistoryLimit: + + rbac: + create: true + + resources: + limits: + memory: 4Gi + requests: + cpu: 100m + # You may want to use the same value for `requests.memory` and `limits.memory`. The “requests” value affects scheduling to accommodate pods on nodes. + # If there is a large difference between “requests” and “limits” and nodes experience memory pressure, the kernel may invoke + # the OOM Killer, even if the memory does not exceed the “limits” threshold. This can cause unexpected pod evictions. Memory + # cannot be compressed and once allocated to a pod, it can only be reclaimed by killing the pod. + # Natan Yellin 22/09/2022 https://home.robusta.dev/blog/kubernetes-memory-limit + memory: 128Mi + + nodeSelector: {} + + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Equal" + value: "" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Equal" + value: "" + effect: "NoSchedule" + + annotations: {} + + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: In + values: [""] + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: In + values: [""] + + startupProbe: + grpc: + port: 8082 + failureThreshold: 30 + # periodSeconds: 10 + livenessProbe: + grpc: + port: 8082 + # failureThreshold: 3 + # initialDelaySeconds: 0 + # periodSeconds: 10 + # timeoutSeconds: 1 + readinessProbe: + grpc: + port: 8082 + failureThreshold: 10 + # initialDelaySeconds: 0 + # periodSeconds: 10 + # timeoutSeconds: 1 + # successThreshold: 1 + +worker: + enable: true + extraArgs: [] + extraEnvs: [] + hostNetwork: false + config: ### + #core: + # labelWhiteList: + # noPublish: false + # noOwnerRefs: false + # sleepInterval: 60s + # featureSources: [all] + # labelSources: [all] + # klog: + # addDirHeader: false + # alsologtostderr: false + # logBacktraceAt: + # logtostderr: true + # skipHeaders: false + # stderrthreshold: 2 + # v: 0 + # vmodule: + ## NOTE: the following options are not dynamically run-time configurable + ## and require a nfd-worker restart to take effect after being changed + # logDir: + # logFile: + # logFileMaxSize: 1800 + # skipLogHeaders: false + #sources: + # cpu: + # cpuid: + ## NOTE: whitelist has priority over blacklist + # attributeBlacklist: + # - "AVX10" + # - "BMI1" + # - "BMI2" + # - "CLMUL" + # - "CMOV" + # - "CX16" + # - "ERMS" + # - "F16C" + # - "HTT" + # - "LZCNT" + # - "MMX" + # - "MMXEXT" + # - "NX" + # - "POPCNT" + # - "RDRAND" + # - "RDSEED" + # - "RDTSCP" + # - "SGX" + # - "SSE" + # - "SSE2" + # - "SSE3" + # - "SSE4" + # - "SSE42" + # - "SSSE3" + # - "TDX_GUEST" + # attributeWhitelist: + # kernel: + # kconfigFile: "/path/to/kconfig" + # configOpts: + # - "NO_HZ" + # - "X86" + # - "DMI" + # pci: + # deviceClassWhitelist: + # - "0200" + # - "03" + # - "12" + # deviceLabelFields: + # - "class" + # - "vendor" + # - "device" + # - "subsystem_vendor" + # - "subsystem_device" + # usb: + # deviceClassWhitelist: + # - "0e" + # - "ef" + # - "fe" + # - "ff" + # deviceLabelFields: + # - "class" + # - "vendor" + # - "device" + # custom: + # # The following feature demonstrates the capabilities of the matchFeatures + # - name: "my custom rule" + # labels: + # "vendor.io/my-ng-feature": "true" + # # matchFeatures implements a logical AND over all matcher terms in the + # # list (i.e. all of the terms, or per-feature matchers, must match) + # matchFeatures: + # - feature: cpu.cpuid + # matchExpressions: + # AVX512F: {op: Exists} + # - feature: cpu.cstate + # matchExpressions: + # enabled: {op: IsTrue} + # - feature: cpu.pstate + # matchExpressions: + # no_turbo: {op: IsFalse} + # scaling_governor: {op: In, value: ["performance"]} + # - feature: cpu.rdt + # matchExpressions: + # RDTL3CA: {op: Exists} + # - feature: cpu.sst + # matchExpressions: + # bf.enabled: {op: IsTrue} + # - feature: cpu.topology + # matchExpressions: + # hardware_multithreading: {op: IsFalse} + # + # - feature: kernel.config + # matchExpressions: + # X86: {op: Exists} + # LSM: {op: InRegexp, value: ["apparmor"]} + # - feature: kernel.loadedmodule + # matchExpressions: + # e1000e: {op: Exists} + # - feature: kernel.selinux + # matchExpressions: + # enabled: {op: IsFalse} + # - feature: kernel.version + # matchExpressions: + # major: {op: In, value: ["5"]} + # minor: {op: Gt, value: ["10"]} + # + # - feature: storage.block + # matchExpressions: + # rotational: {op: In, value: ["0"]} + # dax: {op: In, value: ["0"]} + # + # - feature: network.device + # matchExpressions: + # operstate: {op: In, value: ["up"]} + # speed: {op: Gt, value: ["100"]} + # + # - feature: memory.numa + # matchExpressions: + # node_count: {op: Gt, value: ["2"]} + # - feature: memory.nv + # matchExpressions: + # devtype: {op: In, value: ["nd_dax"]} + # mode: {op: In, value: ["memory"]} + # + # - feature: system.osrelease + # matchExpressions: + # ID: {op: In, value: ["fedora", "centos"]} + # - feature: system.name + # matchExpressions: + # nodename: {op: InRegexp, value: ["^worker-X"]} + # + # - feature: local.label + # matchExpressions: + # custom-feature-knob: {op: Gt, value: ["100"]} + # + # # The following feature demonstrates the capabilities of the matchAny + # - name: "my matchAny rule" + # labels: + # "vendor.io/my-ng-feature-2": "my-value" + # # matchAny implements a logical IF over all elements (sub-matchers) in + # # the list (i.e. at least one feature matcher must match) + # matchAny: + # - matchFeatures: + # - feature: kernel.loadedmodule + # matchExpressions: + # driver-module-X: {op: Exists} + # - feature: pci.device + # matchExpressions: + # vendor: {op: In, value: ["8086"]} + # class: {op: In, value: ["0200"]} + # - matchFeatures: + # - feature: kernel.loadedmodule + # matchExpressions: + # driver-module-Y: {op: Exists} + # - feature: usb.device + # matchExpressions: + # vendor: {op: In, value: ["8086"]} + # class: {op: In, value: ["02"]} + # + # - name: "avx wildcard rule" + # labels: + # "my-avx-feature": "true" + # matchFeatures: + # - feature: cpu.cpuid + # matchName: {op: InRegexp, value: ["^AVX512"]} + # + # # The following features demonstreate label templating capabilities + # - name: "my template rule" + # labelsTemplate: | + # {{ range .system.osrelease }}vendor.io/my-system-feature.{{ .Name }}={{ .Value }} + # {{ end }} + # matchFeatures: + # - feature: system.osrelease + # matchExpressions: + # ID: {op: InRegexp, value: ["^open.*"]} + # VERSION_ID.major: {op: In, value: ["13", "15"]} + # + # - name: "my template rule 2" + # labelsTemplate: | + # {{ range .pci.device }}vendor.io/my-pci-device.{{ .class }}-{{ .device }}=with-cpuid + # {{ end }} + # matchFeatures: + # - feature: pci.device + # matchExpressions: + # class: {op: InRegexp, value: ["^06"]} + # vendor: ["8086"] + # - feature: cpu.cpuid + # matchExpressions: + # AVX: {op: Exists} + # + # # The following examples demonstrate vars field and back-referencing + # # previous labels and vars + # - name: "my dummy kernel rule" + # labels: + # "vendor.io/my.kernel.feature": "true" + # matchFeatures: + # - feature: kernel.version + # matchExpressions: + # major: {op: Gt, value: ["2"]} + # + # - name: "my dummy rule with no labels" + # vars: + # "my.dummy.var": "1" + # matchFeatures: + # - feature: cpu.cpuid + # matchExpressions: {} + # + # - name: "my rule using backrefs" + # labels: + # "vendor.io/my.backref.feature": "true" + # matchFeatures: + # - feature: rule.matched + # matchExpressions: + # vendor.io/my.kernel.feature: {op: IsTrue} + # my.dummy.var: {op: Gt, value: ["0"]} + # + # - name: "kconfig template rule" + # labelsTemplate: | + # {{ range .kernel.config }}kconfig-{{ .Name }}={{ .Value }} + # {{ end }} + # matchFeatures: + # - feature: kernel.config + # matchName: {op: In, value: ["SWAP", "X86", "ARM"]} +### + + metricsPort: 8081 + healthPort: 8082 + daemonsetAnnotations: {} + podSecurityContext: {} + # fsGroup: 2000 + + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ "ALL" ] + readOnlyRootFilesystem: true + runAsNonRoot: true + # runAsUser: 1000 + + livenessProbe: + grpc: + port: 8082 + initialDelaySeconds: 10 + # failureThreshold: 3 + # periodSeconds: 10 + # timeoutSeconds: 1 + readinessProbe: + grpc: + port: 8082 + initialDelaySeconds: 5 + failureThreshold: 10 + # periodSeconds: 10 + # timeoutSeconds: 1 + # successThreshold: 1 + + serviceAccount: + # Specifies whether a service account should be created. + # We create this by default to make it easier for downstream users to apply PodSecurityPolicies. + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: + + # specify how many old ControllerRevisions for the DaemonSet to retain. + revisionHistoryLimit: + + rbac: + create: true + + # Allow users to mount the hostPath /usr/src, useful for RHCOS on s390x + # Does not work on systems without /usr/src AND a read-only /usr, such as Talos + mountUsrSrc: false + + resources: + limits: + memory: 512Mi + requests: + cpu: 5m + memory: 64Mi + + nodeSelector: {} + + tolerations: [] + + annotations: {} + + affinity: {} + + priorityClassName: "" + +topologyUpdater: + config: ### + ## key = node name, value = list of resources to be excluded. + ## use * to exclude from all nodes. + ## an example for how the exclude list should looks like + #excludeList: + # node1: [cpu] + # node2: [memory, example/deviceA] + # *: [hugepages-2Mi] +### + + enable: false + createCRDs: false + extraArgs: [] + extraEnvs: [] + hostNetwork: false + + serviceAccount: + create: true + annotations: {} + name: + + # specify how many old ControllerRevisions for the DaemonSet to retain. + revisionHistoryLimit: + + rbac: + create: true + + metricsPort: 8081 + healthPort: 8082 + kubeletConfigPath: + kubeletPodResourcesSockPath: + updateInterval: 60s + watchNamespace: "*" + kubeletStateDir: /var/lib/kubelet + + podSecurityContext: {} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ "ALL" ] + readOnlyRootFilesystem: true + runAsUser: 0 + + livenessProbe: + grpc: + port: 8082 + initialDelaySeconds: 10 + # failureThreshold: 3 + # periodSeconds: 10 + # timeoutSeconds: 1 + readinessProbe: + grpc: + port: 8082 + initialDelaySeconds: 5 + failureThreshold: 10 + # periodSeconds: 10 + # timeoutSeconds: 1 + # successThreshold: 1 + + resources: + limits: + memory: 60Mi + requests: + cpu: 50m + memory: 40Mi + + nodeSelector: {} + tolerations: [] + annotations: {} + daemonsetAnnotations: {} + affinity: {} + podSetFingerprint: true + +gc: + enable: true + extraArgs: [] + extraEnvs: [] + hostNetwork: false + replicaCount: 1 + + serviceAccount: + create: true + annotations: {} + name: + rbac: + create: true + + interval: 1h + + podSecurityContext: {} + + resources: + limits: + memory: 1Gi + requests: + cpu: 10m + memory: 128Mi + + metricsPort: 8081 + + nodeSelector: {} + tolerations: [] + annotations: {} + deploymentAnnotations: {} + affinity: {} + + # specify how many old ReplicaSets for the Deployment to retain. + revisionHistoryLimit: + +prometheus: + enable: false + scrapeInterval: 10s + labels: {} diff --git a/network-operator/charts/sriov-network-operator/.helmignore b/network-operator/charts/sriov-network-operator/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/network-operator/charts/sriov-network-operator/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/network-operator/charts/sriov-network-operator/Chart.yaml b/network-operator/charts/sriov-network-operator/Chart.yaml new file mode 100644 index 0000000..5d7b7b1 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/Chart.yaml @@ -0,0 +1,13 @@ +apiVersion: v2 +appVersion: 1.2.0 +description: SR-IOV network operator configures and manages SR-IOV networks in the + kubernetes cluster +home: https://github.com/k8snetworkplumbingwg/sriov-network-operator +keywords: +- sriov +kubeVersion: '>= 1.16.0-0' +name: sriov-network-operator +sources: +- https://github.com/k8snetworkplumbingwg/sriov-network-operator +type: application +version: 0.1.0 diff --git a/network-operator/charts/sriov-network-operator/README.md b/network-operator/charts/sriov-network-operator/README.md new file mode 100644 index 0000000..d5d529d --- /dev/null +++ b/network-operator/charts/sriov-network-operator/README.md @@ -0,0 +1,167 @@ +# SR-IOV Network Operator Helm Chart + +SR-IOV Network Operator Helm Chart provides an easy way to install, configure and manage +the lifecycle of SR-IOV network operator. + +## SR-IOV Network Operator +SR-IOV Network Operator leverages [Kubernetes CRDs](https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/) +and [Operator SDK](https://github.com/operator-framework/operator-sdk) to configure and manage SR-IOV networks in a Kubernetes cluster. + +SR-IOV Network Operator features: +- Initialize the supported SR-IOV NIC types on selected nodes. +- Provision/upgrade SR-IOV device plugin executable on selected node. +- Provision/upgrade SR-IOV CNI plugin executable on selected nodes. +- Manage configuration of SR-IOV device plugin on host. +- Generate net-att-def CRs for SR-IOV CNI plugin +- Supports operation in a virtualized Kubernetes deployment + - Discovers VFs attached to the Virtual Machine (VM) + - Does not require attached of associated PFs + - VFs can be associated to SriovNetworks by selecting the appropriate PciAddress as the RootDevice in the SriovNetworkNodePolicy + +## QuickStart + +### Prerequisites + +- Kubernetes v1.17+ +- Helm v3 + +### Install Helm + +Helm provides an install script to copy helm binary to your system: +``` +$ curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 +$ chmod 500 get_helm.sh +$ ./get_helm.sh +``` + +For additional information and methods for installing Helm, refer to the official [helm website](https://helm.sh/) + +### Deploy SR-IOV Network Operator + +#### Deploy from OCI repo + +``` +$ helm install -n sriov-network-operator --create-namespace --version 1.3.0 --set sriovOperatorConfig.deploy=true sriov-network-operator oci://ghcr.io/k8snetworkplumbingwg/sriov-network-operator-chart +``` + +#### Deploy from project sources + +``` +# Clone project +$ git clone https://github.com/k8snetworkplumbingwg/sriov-network-operator.git ; cd sriov-network-operator + +# Install Operator +$ helm install -n sriov-network-operator --create-namespace --wait --set sriovOperatorConfig.deploy=true sriov-network-operator ./deployment/sriov-network-operator-chart + +# View deployed resources +$ kubectl -n sriov-network-operator get pods +``` + +In the case that [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) is enabled, the sriov network operator namespace will require a security level of 'privileged' +``` +$ kubectl label ns sriov-network-operator pod-security.kubernetes.io/enforce=privileged +``` + +## Chart parameters + +In order to tailor the deployment of the network operator to your cluster needs +We have introduced the following Chart parameters. + +| Name | Type | Default | description | +| ---- |------|---------|-------------| +| `imagePullSecrets` | list | `[]` | An optional list of references to secrets to use for pulling any of the SR-IOV Network Operator image | +| `supportedExtraNICs` | list | `[]` | An optional list of whitelisted NICs | + +### Operator parameters + +| Name | Type | Default | description | +| ---- | ---- | ------- | ----------- | +| `operator.tolerations` | list | `[{"key":"node-role.kubernetes.io/master","operator":"Exists","effect":"NoSchedule"},{"key":"node-role.kubernetes.io/control-plane","operator":"Exists","effect":"NoSchedule"}]` | Operator's tolerations | +| `operator.nodeSelector` | object | {} | Operator's node selector | +| `operator.affinity` | object | `{"nodeAffinity":{"preferredDuringSchedulingIgnoredDuringExecution":[{"weight":1,"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/master","operator":"In","values":[""]}]}},{"weight":1,"preference":{"matchExpressions":[{"key":"node-role.kubernetes.io/control-plane","operator":"In","values":[""]}]}}]}}` | Operator's afffinity configuration | +| `operator.nameOverride` | string | `` | Operator's resource name override | +| `operator.fullnameOverride` | string | `` | Operator's resource full name override | +| `operator.resourcePrefix` | string | `openshift.io` | Device plugin resource prefix | +| `operator.cniBinPath` | string | `/opt/cni/bin` | Path for CNI binary | +| `operator.clustertype` | string | `kubernetes` | Cluster environment type | +| `operator.metricsExporter.port` | string | `9110` | Port where the Network Metrics Exporter listen | +| `operator.metricsExporter.certificates.secretName` | string | `metrics-exporter-cert` | Secret name to serve metrics via TLS. The secret must have the same fields as `operator.admissionControllers.certificates.secretNames` | +| `operator.metricsExporter.prometheusOperator.enabled` | bool | false | Wheter the operator shoud configure Prometheus resources or not (e.g. `ServiceMonitors`). | +| `operator.metricsExporter.prometheusOperator.serviceAccount` | string | `prometheus-k8s` | The service account used by the Prometheus Operator. This is used to give Prometheus the permission to list resource in the SR-IOV operator namespace | +| `operator.metricsExporter.prometheusOperator.namespace` | string | `monitoring` | The namespace where the Prometheus Operator is installed. Setting this variable makes the operator deploy `monitoring.coreos.com` resources. | +| `operator.metricsExporter.prometheusOperator.deployRules` | bool | false | Whether the operator should deploy `PrometheusRules` to scrape namespace version of metrics. | + +#### Admission Controllers parameters + +The admission controllers can be enabled by switching on a single parameter `operator.admissionControllers.enabled`. By +default, the user needs to pre-create Kubernetes Secrets that match the names provided in +`operator.admissionControllers.certificates.secretNames`. The secrets should have 3 fields populated with the relevant +content: +* `ca.crt` (value needs to be base64 encoded twice) +* `tls.crt` +* `tls.key` + +Aside from the aforementioned mode, the chart supports 3 more modes for certificate consumption by the admission +controllers, which can be found in the table below. In a nutshell, the modes that are supported are: +* Consume pre-created Certificates managed by cert-manager +* Generate self signed Certificates managed by cert-manager +* Specify the content of the certificates as Helm values + +| Name | Type | Default | description | +| ---- | ---- | ------- | ----------- | +| `operator.admissionControllers.enabled` | bool | false | Flag that switches on the admission controllers | +| `operator.admissionControllers.certificates.secretNames.operator` | string | `operator-webhook-cert` | Secret that stores the certificate for the Operator's admission controller | +| `operator.admissionControllers.certificates.secretNames.injector` | string | `network-resources-injector-cert` | Secret that stores the certificate for the Network Resources Injector's admission controller | +| `operator.admissionControllers.certificates.certManager.enabled` | bool | false | Flag that switches on consumption of certificates managed by cert-manager | +| `operator.admissionControllers.certificates.certManager.generateSelfSigned` | bool | false | Flag that switches on generation of self signed certificates managed by cert-manager. The secrets in which the certificates are stored will have the names provided in `operator.admissionControllers.certificates.secretNames` | +| `operator.admissionControllers.certificates.custom.enabled` | bool | false | Flag that switches on consumption of user provided certificates that are part of `operator.admissionControllers.certificates.custom.operator` and `operator.admissionControllers.certificates.custom.injector` objects | +| `operator.admissionControllers.certificates.custom.operator.caCrt` | string | `` | The CA certificate to be used by the Operator's admission controller | +| `operator.admissionControllers.certificates.custom.operator.tlsCrt` | string | `` | The public part of the certificate to be used by the Operator's admission controller | +| `operator.admissionControllers.certificates.custom.operator.tlsKey` | string | `` | The private part of the certificate to be used by the Operator's admission controller | +| `operator.admissionControllers.certificates.custom.injector.caCrt` | string | `` | The CA certificate to be used by the Network Resources Injector's admission controller | +| `operator.admissionControllers.certificates.custom.injector.tlsCrt` | string | `` | The public part of the certificate to be used by the Network Resources Injector's admission controller | +| `operator.admissionControllers.certificates.custom.injector.tlsKey` | string | `` | The private part of the certificate to be used by the Network Resources Injector's admission controller | + +### SR-IOV Operator Configuration Parameters + +This section contains general parameters that apply to both the operator and daemon componets of SR-IOV Network Operator. + +| Name | Type | Default | description | +| ---- | ---- | ------- | ----------- | +| `sriovOperatorConfig.deploy` | bool | `false` | deploy SriovOperatorConfig custom resource | +| `sriovOperatorConfig.configDaemonNodeSelector` | map[string]string | `{}` | node selectors for sriov-network-config-daemon | +| `sriovOperatorConfig.logLevel` | int | `2` | log level for both operator and sriov-network-config-daemon | +| `sriovOperatorConfig.disableDrain` | bool | `false` | disable node draining when configuring SR-IOV, set to true in case of a single node cluster or any other justifiable reason | +| `sriovOperatorConfig.configurationMode` | string | `daemon` | sriov-network-config-daemon configuration mode. either `daemon` or `systemd` | +| `sriovOperatorConfig.featureGates` | map[string]bool | `{}` | feature gates to enable/disable | + +**Note** + +When `sriovOperatorConfig.configurationMode` is configured as `systemd`, configurations files and `systemd` service files are created on the node. +Upon chart deletion, those files are not cleaned up. For cases where this is not acceptable, users should rather configured the `daemon` mode. + +### Images parameters + +| Name | description | +| ---- | ----------- | +| `images.operator` | Operator controller image | +| `images.sriovConfigDaemon` | Daemon node agent image | +| `images.sriovCni` | SR-IOV CNI image | +| `images.ibSriovCni` | InfiniBand SR-IOV CNI image | +| `images.ovsCni` | OVS CNI image | +| `images.rdmaCni` | RDMA CNI image | +| `images.sriovDevicePlugin` | SR-IOV device plugin image | +| `images.resourcesInjector` | Resources Injector image | +| `images.webhook` | Operator Webhook image | +| `images.metricsExporter` | Network Metrics Exporter image | +| `images.metricsExporterKubeRbacProxy` | Kube RBAC Proxy image used for metrics exporter | + +### Extra objects parameters + +**Disclaimer**: + +Please note that any resources deployed using the `extraDeploy` in this Helm chart are the sole responsibility of the user. It is important to review and understand the implications of these deployed resources. The maintainers of this Helm chart take no responsibility for any issues or damages caused by the deployment or operation of these resources. + +| Name | description | +| ---- | ------------| +|`extraDeploy`| Array of extra objects to deploy with the release | diff --git a/network-operator/charts/sriov-network-operator/crds/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml b/network-operator/charts/sriov-network-operator/crds/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml new file mode 100644 index 0000000..5916055 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/crds/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml @@ -0,0 +1,57 @@ +# Copyright 2020 NVIDIA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: network-attachment-definitions.k8s.cni.cncf.io +spec: + group: k8s.cni.cncf.io + scope: Namespaced + names: + plural: network-attachment-definitions + singular: network-attachment-definition + kind: NetworkAttachmentDefinition + shortNames: + - net-attach-def + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing + Working Group to express the intent for attaching pods to one or more logical or physical + networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec' + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this represen + tation of an object. Servers should convert recognized schemas to the + latest internal value, and may reject unrecognized values. More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment' + type: object + properties: + config: + description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration' + type: string diff --git a/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_ovsnetworks.yaml b/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_ovsnetworks.yaml new file mode 100644 index 0000000..8c4f8f9 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_ovsnetworks.yaml @@ -0,0 +1,105 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: ovsnetworks.sriovnetwork.openshift.io +spec: + group: sriovnetwork.openshift.io + names: + kind: OVSNetwork + listKind: OVSNetworkList + plural: ovsnetworks + singular: ovsnetwork + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: OVSNetwork is the Schema for the ovsnetworks API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: OVSNetworkSpec defines the desired state of OVSNetwork + properties: + bridge: + description: |- + name of the OVS bridge, if not set OVS will automatically select bridge + based on VF PCI address + type: string + capabilities: + description: |- + Capabilities to be configured for this network. + Capabilities supported: (mac|ips), e.g. '{"mac": true}' + type: string + interfaceType: + description: The type of interface on ovs. + type: string + ipam: + description: IPAM configuration to be used for this network. + type: string + metaPlugins: + description: MetaPluginsConfig configuration to be used in order to + chain metaplugins + type: string + mtu: + description: Mtu for the OVS port + type: integer + networkNamespace: + description: Namespace of the NetworkAttachmentDefinition custom resource + type: string + resourceName: + description: OVS Network device plugin endpoint resource name + type: string + trunk: + description: Trunk configuration for the OVS port + items: + description: TrunkConfig contains configuration for bridge trunk + properties: + id: + maximum: 4095 + minimum: 0 + type: integer + maxID: + maximum: 4095 + minimum: 0 + type: integer + minID: + maximum: 4095 + minimum: 0 + type: integer + type: object + type: array + vlan: + description: Vlan to assign for the OVS port + maximum: 4095 + minimum: 0 + type: integer + required: + - resourceName + type: object + status: + description: OVSNetworkStatus defines the observed state of OVSNetwork + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovibnetworks.yaml b/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovibnetworks.yaml new file mode 100644 index 0000000..4b4b44d --- /dev/null +++ b/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovibnetworks.yaml @@ -0,0 +1,78 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: sriovibnetworks.sriovnetwork.openshift.io +spec: + group: sriovnetwork.openshift.io + names: + kind: SriovIBNetwork + listKind: SriovIBNetworkList + plural: sriovibnetworks + singular: sriovibnetwork + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: SriovIBNetwork is the Schema for the sriovibnetworks API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SriovIBNetworkSpec defines the desired state of SriovIBNetwork + properties: + capabilities: + description: |- + Capabilities to be configured for this network. + Capabilities supported: (infinibandGUID), e.g. '{"infinibandGUID": true}' + type: string + ipam: + description: IPAM configuration to be used for this network. + type: string + linkState: + description: VF link state (enable|disable|auto) + enum: + - auto + - enable + - disable + type: string + metaPlugins: + description: |- + MetaPluginsConfig configuration to be used in order to chain metaplugins to the sriov interface returned + by the operator. + type: string + networkNamespace: + description: Namespace of the NetworkAttachmentDefinition custom resource + type: string + resourceName: + description: SRIOV Network device plugin endpoint resource name + type: string + required: + - resourceName + type: object + status: + description: SriovIBNetworkStatus defines the observed state of SriovIBNetwork + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml b/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml new file mode 100644 index 0000000..524c512 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodepolicies.yaml @@ -0,0 +1,213 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: sriovnetworknodepolicies.sriovnetwork.openshift.io +spec: + group: sriovnetwork.openshift.io + names: + kind: SriovNetworkNodePolicy + listKind: SriovNetworkNodePolicyList + plural: sriovnetworknodepolicies + singular: sriovnetworknodepolicy + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: SriovNetworkNodePolicy is the Schema for the sriovnetworknodepolicies + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SriovNetworkNodePolicySpec defines the desired state of SriovNetworkNodePolicy + properties: + bridge: + description: |- + contains bridge configuration for matching PFs, + valid only for eSwitchMode==switchdev + properties: + ovs: + description: contains configuration for the OVS bridge, + properties: + bridge: + description: contains bridge level settings + properties: + datapathType: + description: configure datapath_type field in the Bridge + table in OVSDB + type: string + externalIDs: + additionalProperties: + type: string + description: IDs to inject to external_ids field in the + Bridge table in OVSDB + type: object + otherConfig: + additionalProperties: + type: string + description: additional options to inject to other_config + field in the bridge table in OVSDB + type: object + type: object + uplink: + description: contains settings for uplink (PF) + properties: + interface: + description: contains settings for PF interface in the + OVS bridge + properties: + externalIDs: + additionalProperties: + type: string + description: external_ids field in the Interface table + in OVSDB + type: object + mtuRequest: + description: mtu_request field in the Interface table + in OVSDB + type: integer + options: + additionalProperties: + type: string + description: options field in the Interface table + in OVSDB + type: object + otherConfig: + additionalProperties: + type: string + description: other_config field in the Interface table + in OVSDB + type: object + type: + description: type field in the Interface table in + OVSDB + type: string + type: object + type: object + type: object + type: object + deviceType: + default: netdevice + description: The driver type for configured VFs. Allowed value "netdevice", + "vfio-pci". Defaults to netdevice. + enum: + - netdevice + - vfio-pci + type: string + eSwitchMode: + description: NIC Device Mode. Allowed value "legacy","switchdev". + enum: + - legacy + - switchdev + type: string + excludeTopology: + description: Exclude device's NUMA node when advertising this resource + by SRIOV network device plugin. Default to false. + type: boolean + externallyManaged: + description: don't create the virtual function only allocated them + to the device plugin. Defaults to false. + type: boolean + isRdma: + description: RDMA mode. Defaults to false. + type: boolean + linkType: + description: NIC Link Type. Allowed value "eth", "ETH", "ib", and + "IB". + enum: + - eth + - ETH + - ib + - IB + type: string + mtu: + description: MTU of VF + minimum: 1 + type: integer + needVhostNet: + description: mount vhost-net device. Defaults to false. + type: boolean + nicSelector: + description: NicSelector selects the NICs to be configured + properties: + deviceID: + description: The device hex code of SR-IoV device. Allowed value + "0d58", "1572", "158b", "1013", "1015", "1017", "101b". + type: string + netFilter: + description: Infrastructure Networking selection filter. Allowed + value "openstack/NetworkID:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" + type: string + pfNames: + description: Name of SR-IoV PF. + items: + type: string + type: array + rootDevices: + description: PCI address of SR-IoV PF. + items: + type: string + type: array + vendor: + description: The vendor hex code of SR-IoV device. Allowed value + "8086", "15b3". + type: string + type: object + nodeSelector: + additionalProperties: + type: string + description: NodeSelector selects the nodes to be configured + type: object + numVfs: + description: Number of VFs for each PF + minimum: 0 + type: integer + priority: + description: Priority of the policy, higher priority policies can + override lower ones. + maximum: 99 + minimum: 0 + type: integer + resourceName: + description: SRIOV Network device plugin endpoint resource name + type: string + vdpaType: + description: VDPA device type. Allowed value "virtio", "vhost" + enum: + - virtio + - vhost + type: string + required: + - nicSelector + - nodeSelector + - numVfs + - resourceName + type: object + status: + description: SriovNetworkNodePolicyStatus defines the observed state of + SriovNetworkNodePolicy + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml b/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml new file mode 100644 index 0000000..7535346 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworknodestates.yaml @@ -0,0 +1,369 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: sriovnetworknodestates.sriovnetwork.openshift.io +spec: + group: sriovnetwork.openshift.io + names: + kind: SriovNetworkNodeState + listKind: SriovNetworkNodeStateList + plural: sriovnetworknodestates + singular: sriovnetworknodestate + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.syncStatus + name: Sync Status + type: string + - jsonPath: .metadata.annotations.sriovnetwork\.openshift\.io/desired-state + name: Desired Sync State + type: string + - jsonPath: .metadata.annotations.sriovnetwork\.openshift\.io/current-state + name: Current Sync State + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: SriovNetworkNodeState is the Schema for the sriovnetworknodestates + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SriovNetworkNodeStateSpec defines the desired state of SriovNetworkNodeState + properties: + bridges: + description: Bridges contains list of bridges + properties: + ovs: + items: + description: OVSConfigExt contains configuration for the concrete + OVS bridge + properties: + bridge: + description: bridge-level configuration for the bridge + properties: + datapathType: + description: configure datapath_type field in the Bridge + table in OVSDB + type: string + externalIDs: + additionalProperties: + type: string + description: IDs to inject to external_ids field in + the Bridge table in OVSDB + type: object + otherConfig: + additionalProperties: + type: string + description: additional options to inject to other_config + field in the bridge table in OVSDB + type: object + type: object + name: + description: name of the bridge + type: string + uplinks: + description: |- + uplink-level bridge configuration for each uplink(PF). + currently must contain only one element + items: + description: OVSUplinkConfigExt contains configuration + for the concrete OVS uplink(PF) + properties: + interface: + description: configuration from the Interface OVS + table for the PF + properties: + externalIDs: + additionalProperties: + type: string + description: external_ids field in the Interface + table in OVSDB + type: object + mtuRequest: + description: mtu_request field in the Interface + table in OVSDB + type: integer + options: + additionalProperties: + type: string + description: options field in the Interface table + in OVSDB + type: object + otherConfig: + additionalProperties: + type: string + description: other_config field in the Interface + table in OVSDB + type: object + type: + description: type field in the Interface table + in OVSDB + type: string + type: object + name: + description: name of the PF interface + type: string + pciAddress: + description: pci address of the PF + type: string + required: + - pciAddress + type: object + type: array + required: + - name + type: object + type: array + type: object + interfaces: + items: + properties: + eSwitchMode: + type: string + externallyManaged: + type: boolean + linkType: + type: string + mtu: + type: integer + name: + type: string + numVfs: + type: integer + pciAddress: + type: string + vfGroups: + items: + properties: + deviceType: + type: string + isRdma: + type: boolean + mtu: + type: integer + policyName: + type: string + resourceName: + type: string + vdpaType: + type: string + vfRange: + type: string + type: object + type: array + required: + - pciAddress + type: object + type: array + system: + properties: + rdmaMode: + description: RDMA subsystem. Allowed value "shared", "exclusive". + enum: + - shared + - exclusive + type: string + type: object + type: object + status: + description: SriovNetworkNodeStateStatus defines the observed state of + SriovNetworkNodeState + properties: + bridges: + description: Bridges contains list of bridges + properties: + ovs: + items: + description: OVSConfigExt contains configuration for the concrete + OVS bridge + properties: + bridge: + description: bridge-level configuration for the bridge + properties: + datapathType: + description: configure datapath_type field in the Bridge + table in OVSDB + type: string + externalIDs: + additionalProperties: + type: string + description: IDs to inject to external_ids field in + the Bridge table in OVSDB + type: object + otherConfig: + additionalProperties: + type: string + description: additional options to inject to other_config + field in the bridge table in OVSDB + type: object + type: object + name: + description: name of the bridge + type: string + uplinks: + description: |- + uplink-level bridge configuration for each uplink(PF). + currently must contain only one element + items: + description: OVSUplinkConfigExt contains configuration + for the concrete OVS uplink(PF) + properties: + interface: + description: configuration from the Interface OVS + table for the PF + properties: + externalIDs: + additionalProperties: + type: string + description: external_ids field in the Interface + table in OVSDB + type: object + mtuRequest: + description: mtu_request field in the Interface + table in OVSDB + type: integer + options: + additionalProperties: + type: string + description: options field in the Interface table + in OVSDB + type: object + otherConfig: + additionalProperties: + type: string + description: other_config field in the Interface + table in OVSDB + type: object + type: + description: type field in the Interface table + in OVSDB + type: string + type: object + name: + description: name of the PF interface + type: string + pciAddress: + description: pci address of the PF + type: string + required: + - pciAddress + type: object + type: array + required: + - name + type: object + type: array + type: object + interfaces: + items: + properties: + Vfs: + items: + properties: + Vlan: + type: integer + assigned: + type: string + deviceID: + type: string + driver: + type: string + guid: + type: string + mac: + type: string + mtu: + type: integer + name: + type: string + pciAddress: + type: string + representorName: + type: string + vdpaType: + type: string + vendor: + type: string + vfID: + type: integer + required: + - pciAddress + - vfID + type: object + type: array + deviceID: + type: string + driver: + type: string + eSwitchMode: + type: string + externallyManaged: + type: boolean + linkAdminState: + type: string + linkSpeed: + type: string + linkType: + type: string + mac: + type: string + mtu: + type: integer + name: + type: string + netFilter: + type: string + numVfs: + type: integer + pciAddress: + type: string + totalvfs: + type: integer + vendor: + type: string + required: + - pciAddress + type: object + type: array + lastSyncError: + type: string + syncStatus: + type: string + system: + properties: + rdmaMode: + description: RDMA subsystem. Allowed value "shared", "exclusive". + enum: + - shared + - exclusive + type: string + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml b/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml new file mode 100644 index 0000000..3d8a6a1 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworkpoolconfigs.yaml @@ -0,0 +1,129 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: sriovnetworkpoolconfigs.sriovnetwork.openshift.io +spec: + group: sriovnetwork.openshift.io + names: + kind: SriovNetworkPoolConfig + listKind: SriovNetworkPoolConfigList + plural: sriovnetworkpoolconfigs + singular: sriovnetworkpoolconfig + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: SriovNetworkPoolConfig is the Schema for the sriovnetworkpoolconfigs + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SriovNetworkPoolConfigSpec defines the desired state of SriovNetworkPoolConfig + properties: + maxUnavailable: + anyOf: + - type: integer + - type: string + description: |- + maxUnavailable defines either an integer number or percentage + of nodes in the pool that can go Unavailable during an update. + + + A value larger than 1 will mean multiple nodes going unavailable during + the update, which may affect your workload stress on the remaining nodes. + Drain will respect Pod Disruption Budgets (PDBs) such as etcd quorum guards, + even if maxUnavailable is greater than one. + x-kubernetes-int-or-string: true + nodeSelector: + description: nodeSelector specifies a label selector for Nodes + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + ovsHardwareOffloadConfig: + description: OvsHardwareOffloadConfig describes the OVS HWOL configuration + for selected Nodes + properties: + name: + description: |- + Name is mandatory and must be unique. + On Kubernetes: + Name is the name of OvsHardwareOffloadConfig + On OpenShift: + Name is the name of MachineConfigPool to be enabled with OVS hardware offload + type: string + type: object + rdmaMode: + description: RDMA subsystem. Allowed value "shared", "exclusive". + enum: + - shared + - exclusive + type: string + type: object + status: + description: SriovNetworkPoolConfigStatus defines the observed state of + SriovNetworkPoolConfig + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworks.yaml b/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworks.yaml new file mode 100644 index 0000000..e33b9a3 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovnetworks.yaml @@ -0,0 +1,136 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: sriovnetworks.sriovnetwork.openshift.io +spec: + group: sriovnetwork.openshift.io + names: + kind: SriovNetwork + listKind: SriovNetworkList + plural: sriovnetworks + singular: sriovnetwork + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: SriovNetwork is the Schema for the sriovnetworks API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SriovNetworkSpec defines the desired state of SriovNetwork + properties: + capabilities: + description: |- + Capabilities to be configured for this network. + Capabilities supported: (mac|ips), e.g. '{"mac": true}' + type: string + ipam: + description: IPAM configuration to be used for this network. + type: string + linkState: + description: VF link state (enable|disable|auto) + enum: + - auto + - enable + - disable + type: string + logFile: + description: |- + LogFile sets the log file of the SRIOV CNI plugin logs. If unset (default), this will log to stderr and thus + to multus and container runtime logs. + type: string + logLevel: + default: info + description: |- + LogLevel sets the log level of the SRIOV CNI plugin - either of panic, error, warning, info, debug. Defaults + to info if left blank. + enum: + - panic + - error + - warning + - info + - debug + - "" + type: string + maxTxRate: + description: Maximum tx rate, in Mbps, for the VF. Defaults to 0 (no + rate limiting) + minimum: 0 + type: integer + metaPlugins: + description: |- + MetaPluginsConfig configuration to be used in order to chain metaplugins to the sriov interface returned + by the operator. + type: string + minTxRate: + description: Minimum tx rate, in Mbps, for the VF. Defaults to 0 (no + rate limiting). min_tx_rate should be <= max_tx_rate. + minimum: 0 + type: integer + networkNamespace: + description: Namespace of the NetworkAttachmentDefinition custom resource + type: string + resourceName: + description: SRIOV Network device plugin endpoint resource name + type: string + spoofChk: + description: VF spoof check, (on|off) + enum: + - "on" + - "off" + type: string + trust: + description: VF trust mode (on|off) + enum: + - "on" + - "off" + type: string + vlan: + description: VLAN ID to assign for the VF. Defaults to 0. + maximum: 4096 + minimum: 0 + type: integer + vlanProto: + description: VLAN proto to assign for the VF. Defaults to 802.1q. + enum: + - 802.1q + - 802.1Q + - 802.1ad + - 802.1AD + type: string + vlanQoS: + description: VLAN QoS ID to assign for the VF. Defaults to 0. + maximum: 7 + minimum: 0 + type: integer + required: + - resourceName + type: object + status: + description: SriovNetworkStatus defines the observed state of SriovNetwork + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml b/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml new file mode 100644 index 0000000..5d94491 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/crds/sriovnetwork.openshift.io_sriovoperatorconfigs.yaml @@ -0,0 +1,114 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.14.0 + name: sriovoperatorconfigs.sriovnetwork.openshift.io +spec: + group: sriovnetwork.openshift.io + names: + kind: SriovOperatorConfig + listKind: SriovOperatorConfigList + plural: sriovoperatorconfigs + singular: sriovoperatorconfig + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: SriovOperatorConfig is the Schema for the sriovoperatorconfigs + API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: SriovOperatorConfigSpec defines the desired state of SriovOperatorConfig + properties: + configDaemonNodeSelector: + additionalProperties: + type: string + description: NodeSelector selects the nodes to be configured + type: object + configurationMode: + description: |- + Flag to enable the sriov-network-config-daemon to use a systemd service to configure SR-IOV devices on boot + Default mode: daemon + enum: + - daemon + - systemd + type: string + disableDrain: + description: Flag to disable nodes drain during debugging + type: boolean + disablePlugins: + description: DisablePlugins is a list of sriov-network-config-daemon + plugins to disable + items: + description: PluginNameValue defines the plugin name + enum: + - mellanox + type: string + type: array + enableInjector: + description: Flag to control whether the network resource injector + webhook shall be deployed + type: boolean + enableOperatorWebhook: + description: Flag to control whether the operator admission controller + webhook shall be deployed + type: boolean + enableOvsOffload: + description: Flag to enable OVS hardware offload. Set to 'true' to + provision switchdev-configuration.service and enable OpenvSwitch + hw-offload on nodes. + type: boolean + featureGates: + additionalProperties: + type: boolean + description: FeatureGates to enable experimental features + type: object + logLevel: + description: Flag to control the log verbose level of the operator. + Set to '0' to show only the basic logs. And set to '2' to show all + the available logs. + maximum: 2 + minimum: 0 + type: integer + useCDI: + description: Flag to enable Container Device Interface mode for SR-IOV + Network Device Plugin + type: boolean + type: object + status: + description: SriovOperatorConfigStatus defines the observed state of SriovOperatorConfig + properties: + injector: + description: Show the runtime status of the network resource injector + webhook + type: string + operatorWebhook: + description: Show the runtime status of the operator admission controller + webhook + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/network-operator/charts/sriov-network-operator/templates/NOTES.txt b/network-operator/charts/sriov-network-operator/templates/NOTES.txt new file mode 100644 index 0000000..94dcf31 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/templates/NOTES.txt @@ -0,0 +1,6 @@ +Get Network Operator deployed resources by running the following commands: + +$ kubectl -n {{ .Release.Namespace }} get pods + +For additional instructions on how to use SR-IOV network operator, +refer to: https://github.com/k8snetworkplumbingwg/sriov-network-operator \ No newline at end of file diff --git a/network-operator/charts/sriov-network-operator/templates/_helpers.tpl b/network-operator/charts/sriov-network-operator/templates/_helpers.tpl new file mode 100644 index 0000000..2cde681 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "sriov-network-operator.name" -}} +{{- default "sriov-network-operator" .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "sriov-network-operator.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default "sriov-network-operator" .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "sriov-network-operator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "sriov-network-operator.labels" -}} +helm.sh/chart: {{ include "sriov-network-operator.chart" . }} +{{ include "sriov-network-operator.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "sriov-network-operator.selectorLabels" -}} +app.kubernetes.io/name: {{ include "sriov-network-operator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "sriov-network-operator.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "sriov-network-operator.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/network-operator/charts/sriov-network-operator/templates/certificate.yaml b/network-operator/charts/sriov-network-operator/templates/certificate.yaml new file mode 100644 index 0000000..add29a9 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/templates/certificate.yaml @@ -0,0 +1,71 @@ +{{- if .Values.operator.admissionControllers.enabled }} +{{- if and (.Values.operator.admissionControllers.certificates.certManager.enabled) (.Values.operator.admissionControllers.certificates.certManager.generateSelfSigned) }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ .Values.operator.admissionControllers.certificates.secretNames.operator }} + namespace: {{ .Release.Namespace }} +spec: + dnsNames: + - operator-webhook-service.{{ .Release.Namespace }}.svc + - operator-webhook-service.{{ .Release.Namespace }}.svc.cluster.local + issuerRef: + kind: Issuer + name: operator-webhook-selfsigned-issuer + secretName: {{ .Values.operator.admissionControllers.certificates.secretNames.operator }} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: operator-webhook-selfsigned-issuer + namespace: {{ .Release.Namespace }} +spec: + selfSigned: {} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ .Values.operator.admissionControllers.certificates.secretNames.injector }} + namespace: {{ .Release.Namespace }} +spec: + dnsNames: + - network-resources-injector-service.{{ .Release.Namespace }}.svc + - network-resources-injector-service.{{ .Release.Namespace }}.svc.cluster.local + issuerRef: + kind: Issuer + name: network-resources-injector-selfsigned-issuer + secretName: {{ .Values.operator.admissionControllers.certificates.secretNames.injector }} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: network-resources-injector-selfsigned-issuer + namespace: {{ .Release.Namespace }} +spec: + selfSigned: {} +{{- else if and (not .Values.operator.admissionControllers.certificates.certManager.enabled) (.Values.operator.admissionControllers.certificates.custom.enabled) }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.operator.admissionControllers.certificates.secretNames.operator }} + namespace: {{ .Release.Namespace }} +type: Opaque +data: + ca.crt: {{ .Values.operator.admissionControllers.certificates.custom.operator.caCrt | b64enc | b64enc | quote }} + tls.crt: {{ .Values.operator.admissionControllers.certificates.custom.operator.tlsCrt | b64enc | quote }} + tls.key: {{ .Values.operator.admissionControllers.certificates.custom.operator.tlsKey | b64enc | quote }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.operator.admissionControllers.certificates.secretNames.injector }} + namespace: {{ .Release.Namespace }} +type: Opaque +data: + ca.crt: {{ .Values.operator.admissionControllers.certificates.custom.injector.caCrt | b64enc | b64enc | quote }} + tls.crt: {{ .Values.operator.admissionControllers.certificates.custom.injector.tlsCrt | b64enc | quote }} + tls.key: {{ .Values.operator.admissionControllers.certificates.custom.injector.tlsKey | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/network-operator/charts/sriov-network-operator/templates/clusterrole.yaml b/network-operator/charts/sriov-network-operator/templates/clusterrole.yaml new file mode 100644 index 0000000..519d2c0 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/templates/clusterrole.yaml @@ -0,0 +1,54 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "sriov-network-operator.fullname" . }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "patch", "update"] + - apiGroups: [""] + resources: ["pods"] + verbs: ["*"] + - apiGroups: [""] + resources: ["pods/eviction"] + verbs: ["create"] + - apiGroups: ["apps"] + resources: ["daemonsets"] + verbs: ["get"] + - apiGroups: [""] + resources: ["namespaces", "serviceaccounts"] + verbs: ["*"] + - apiGroups: ["k8s.cni.cncf.io"] + resources: ["network-attachment-definitions"] + verbs: ["*"] + - apiGroups: ["rbac.authorization.k8s.io"] + resources: [clusterroles, clusterrolebindings] + verbs: ["*"] + - apiGroups: ["admissionregistration.k8s.io"] + resources: ["mutatingwebhookconfigurations", "validatingwebhookconfigurations"] + verbs: ["*"] + - apiGroups: ["sriovnetwork.openshift.io"] + resources: ["*"] + verbs: ["*"] + - apiGroups: ["machineconfiguration.openshift.io"] + resources: ["*"] + verbs: ["*"] + - apiGroups: ["config.openshift.io"] + resources: ["infrastructures"] + verbs: ["get", "list", "watch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: sriov-network-config-daemon + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch", "patch", "update"] + - apiGroups: [ "config.openshift.io" ] + resources: [ "infrastructures" ] + verbs: [ "get", "list", "watch" ] diff --git a/network-operator/charts/sriov-network-operator/templates/clusterrolebinding.yaml b/network-operator/charts/sriov-network-operator/templates/clusterrolebinding.yaml new file mode 100644 index 0000000..c10aa9b --- /dev/null +++ b/network-operator/charts/sriov-network-operator/templates/clusterrolebinding.yaml @@ -0,0 +1,29 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "sriov-network-operator.fullname" . }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +roleRef: + kind: ClusterRole + name: {{ include "sriov-network-operator.fullname" . }} + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + namespace: {{ .Release.Namespace }} + name: {{ include "sriov-network-operator.fullname" . }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: sriov-network-config-daemon + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +roleRef: + kind: ClusterRole + name: sriov-network-config-daemon + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + namespace: {{ .Release.Namespace }} + name: sriov-network-config-daemon diff --git a/network-operator/charts/sriov-network-operator/templates/configmap.yaml b/network-operator/charts/sriov-network-operator/templates/configmap.yaml new file mode 100644 index 0000000..95901c0 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/templates/configmap.yaml @@ -0,0 +1,47 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: supported-nic-ids +data: + Intel_i40e_XXV710: "8086 158a 154c" + Intel_i40e_25G_SFP28: "8086 158b 154c" + Intel_i40e_10G_X710_SFP: "8086 1572 154c" + Intel_ixgbe_10G_X550: "8086 1563 1565" + Intel_ixgbe_82576: "8086 10c9 10ca" + Intel_i40e_X710_X557_AT_10G: "8086 1589 154c" + Intel_i40e_10G_X710_BACKPLANE: "8086 1581 154c" + Intel_i40e_10G_X710_BASE_T: "8086 15ff 154c" + Intel_i40e_XXV710_N3000: "8086 0d58 154c" + Intel_i40e_40G_XL710_QSFP: "8086 1583 154c" + Intel_ice_Columbiaville_E810-CQDA2_2CQDA2: "8086 1592 1889" + Intel_ice_Columbiaville_E810-XXVDA4: "8086 1593 1889" + Intel_ice_Columbiaville_E810-XXVDA2: "8086 159b 1889" + Intel_ice_Columbiaville_E810-XXV_BACKPLANE: "8086 1599 1889" + Intel_ice_Columbiaville_E810: "8086 1591 1889" + Intel_ice_Columbiapark_E823C: "8086 188a 1889" + Intel_ice_Columbiapark_E823L_SFP: "8086 124d 1889" + Intel_ice_Columbiapark_E823L_BACKPLANE: "8086 124c 1889" + Nvidia_mlx5_ConnectX-4: "15b3 1013 1014" + Nvidia_mlx5_ConnectX-4LX: "15b3 1015 1016" + Nvidia_mlx5_ConnectX-5: "15b3 1017 1018" + Nvidia_mlx5_ConnectX-5_Ex: "15b3 1019 101a" + Nvidia_mlx5_ConnectX-6: "15b3 101b 101c" + Nvidia_mlx5_ConnectX-6_Dx: "15b3 101d 101e" + Nvidia_mlx5_ConnectX-6_Lx: "15b3 101f 101e" + Nvidia_mlx5_ConnectX-7: "15b3 1021 101e" + Nvidia_mlx5_ConnectX-8: "15b3 1023 101e" + Nvidia_mlx5_MT42822_BlueField-2_integrated_ConnectX-6_Dx: "15b3 a2d6 101e" + Nvidia_mlx5_MT43244_BlueField-3_integrated_ConnectX-7_Dx: "15b3 a2dc 101e" + Broadcom_bnxt_BCM57414_2x25G: "14e4 16d7 16dc" + Broadcom_bnxt_BCM75508_2x100G: "14e4 1750 1806" + Qlogic_qede_QL45000_50G: "1077 1654 1664" + Red_Hat_Virtio_network_device: "1af4 1000 1000" + Red_Hat_Virtio_1_0_network_device: "1af4 1041 1041" + Marvell_OCTEON_TX2_CN96XX: "177d b200 b203" + Marvell_OCTEON_TX2_CN98XX: "177d b100 b103" + Marvell_OCTEON_Fusion_CNF95XX: "177d b600 b603" + Marvell_OCTEON10_CN10XXX: "177d b900 b903" + Marvell_OCTEON_Fusion_CNF105XX: "177d ba00 ba03" + {{- range .Values.supportedExtraNICs }} + {{ . }} + {{- end }} diff --git a/network-operator/charts/sriov-network-operator/templates/extra-list.yaml b/network-operator/charts/sriov-network-operator/templates/extra-list.yaml new file mode 100644 index 0000000..6968e7d --- /dev/null +++ b/network-operator/charts/sriov-network-operator/templates/extra-list.yaml @@ -0,0 +1,8 @@ +{{- range .Values.extraDeploy }} +--- +{{- if typeIs "string" . }} + {{- tpl . $ }} +{{- else }} + {{- tpl (. | toYaml) $ }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/network-operator/charts/sriov-network-operator/templates/operator.yaml b/network-operator/charts/sriov-network-operator/templates/operator.yaml new file mode 100644 index 0000000..c2a813f --- /dev/null +++ b/network-operator/charts/sriov-network-operator/templates/operator.yaml @@ -0,0 +1,137 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "sriov-network-operator.fullname" . }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +spec: + replicas: 1 + selector: + matchLabels: + name: sriov-network-operator + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 33% + template: + metadata: + annotations: + openshift.io/required-scc: restricted-v2 + labels: + name: sriov-network-operator + spec: + {{- with .Values.operator.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.affinity }} + affinity: + {{- toYaml . | nindent 8}} + {{- end }} + {{- with .Values.operator.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "sriov-network-operator.fullname" . }} + priorityClassName: "system-node-critical" + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + containers: + - name: {{ include "sriov-network-operator.fullname" . }} + image: {{ .Values.images.operator }} + command: + - sriov-network-operator + resources: + requests: + cpu: 100m + memory: 100Mi + env: + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: SRIOV_CNI_IMAGE + value: {{ .Values.images.sriovCni }} + - name: SRIOV_INFINIBAND_CNI_IMAGE + value: {{ .Values.images.ibSriovCni }} + - name: OVS_CNI_IMAGE + value: {{ .Values.images.ovsCni }} + - name: RDMA_CNI_IMAGE + value: {{ .Values.images.rdmaCni }} + - name: SRIOV_DEVICE_PLUGIN_IMAGE + value: {{ .Values.images.sriovDevicePlugin }} + - name: NETWORK_RESOURCES_INJECTOR_IMAGE + value: {{ .Values.images.resourcesInjector }} + - name: OPERATOR_NAME + value: sriov-network-operator + - name: SRIOV_NETWORK_CONFIG_DAEMON_IMAGE + value: {{ .Values.images.sriovConfigDaemon }} + - name: SRIOV_NETWORK_WEBHOOK_IMAGE + value: {{ .Values.images.webhook }} + - name: METRICS_EXPORTER_IMAGE + value: {{ .Values.images.metricsExporter }} + - name: METRICS_EXPORTER_PORT + value: "{{ .Values.operator.metricsExporter.port }}" + - name: METRICS_EXPORTER_SECRET_NAME + value: {{ .Values.operator.metricsExporter.certificates.secretName }} + - name: METRICS_EXPORTER_KUBE_RBAC_PROXY_IMAGE + value: {{ .Values.images.metricsExporterKubeRbacProxy }} + {{- if .Values.operator.metricsExporter.prometheusOperator.enabled }} + - name: METRICS_EXPORTER_PROMETHEUS_OPERATOR_ENABLED + value: {{ .Values.operator.metricsExporter.prometheusOperator.enabled | quote}} + - name: METRICS_EXPORTER_PROMETHEUS_DEPLOY_RULES + value: {{ .Values.operator.metricsExporter.prometheusOperator.deployRules | quote}} + - name: METRICS_EXPORTER_PROMETHEUS_OPERATOR_SERVICE_ACCOUNT + value: {{ .Values.operator.metricsExporter.prometheusOperator.serviceAccount }} + - name: METRICS_EXPORTER_PROMETHEUS_OPERATOR_NAMESPACE + value: {{ .Values.operator.metricsExporter.prometheusOperator.namespace }} + {{- end }} + - name: RESOURCE_PREFIX + value: {{ .Values.operator.resourcePrefix }} + - name: IMAGE_PULL_SECRETS + value: {{ join "," .Values.imagePullSecrets }} + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: RELEASE_VERSION + value: {{ .Release.AppVersion }} + - name: SRIOV_CNI_BIN_PATH + value: {{ .Values.operator.cniBinPath }} + - name: CLUSTER_TYPE + value: {{ .Values.operator.clusterType }} + - name: STALE_NODE_STATE_CLEANUP_DELAY_MINUTES + value: "{{ .Values.operator.staleNodeStateCleanupDelayMinutes }}" + {{- if .Values.operator.admissionControllers.enabled }} + - name: ADMISSION_CONTROLLERS_CERTIFICATES_OPERATOR_SECRET_NAME + value: {{ .Values.operator.admissionControllers.certificates.secretNames.operator }} + - name: ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_SECRET_NAME + value: {{ .Values.operator.admissionControllers.certificates.secretNames.injector }} + {{- if .Values.operator.admissionControllers.certificates.certManager.enabled }} + - name: ADMISSION_CONTROLLERS_CERTIFICATES_CERT_MANAGER_ENABLED + value: {{ .Values.operator.admissionControllers.certificates.certManager.enabled | quote }} + {{- else }} + - name: ADMISSION_CONTROLLERS_CERTIFICATES_OPERATOR_CA_CRT + valueFrom: + secretKeyRef: + name: {{ .Values.operator.admissionControllers.certificates.secretNames.operator }} + key: ca.crt + - name: ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_CA_CRT + valueFrom: + secretKeyRef: + name: {{ .Values.operator.admissionControllers.certificates.secretNames.injector }} + key: ca.crt + {{- end }} + {{- end }} diff --git a/network-operator/charts/sriov-network-operator/templates/pre-delete-webooks.yaml b/network-operator/charts/sriov-network-operator/templates/pre-delete-webooks.yaml new file mode 100644 index 0000000..2022756 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/templates/pre-delete-webooks.yaml @@ -0,0 +1,33 @@ +# The following job will be used as Helm pre-delete hook. It executes a small go-client binary +# which intent to delete 'default' SriovOperatorConfig, that triggers operator removal of generated cluster objects +# e.g. mutating/validating webhooks, within operator's recoinciling loop and +# preventing operator cluster object remainings while using helm uninstall +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ include "sriov-network-operator.fullname" . }}-pre-delete-hook + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": hook-succeeded,hook-failed +spec: + template: + spec: + serviceAccountName: {{ include "sriov-network-operator.fullname" . }} + {{- if .Values.imagePullSecrets }} + imagePullSecrets: + {{- range .Values.imagePullSecrets }} + - name: {{ . }} + {{- end }} + {{- end }} + containers: + - name: cleanup + image: {{ .Values.images.operator }} + command: + - sriov-network-operator-config-cleanup + args: + - --namespace + - {{ .Release.Namespace }} + restartPolicy: Never + backoffLimit: 2 + diff --git a/network-operator/charts/sriov-network-operator/templates/role.yaml b/network-operator/charts/sriov-network-operator/templates/role.yaml new file mode 100644 index 0000000..56e5a54 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/templates/role.yaml @@ -0,0 +1,138 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + creationTimestamp: null + name: {{ include "sriov-network-operator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - pods + - services + - endpoints + - persistentvolumeclaims + - events + - configmaps + - secrets + verbs: + - '*' + - apiGroups: + - apps + resources: + - deployments + - daemonsets + - replicasets + - statefulsets + verbs: + - '*' + - apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + - prometheusrules + verbs: + - get + - create + - update + - delete + - apiGroups: + - apps + resourceNames: + - sriov-network-operator + resources: + - deployments/finalizers + verbs: + - update + - apiGroups: + - rbac.authorization.k8s.io + resources: + - serviceaccounts + - roles + - rolebindings + verbs: + - '*' + - apiGroups: + - config.openshift.io + resources: + - infrastructures + verbs: + - get + - list + - watch + - apiGroups: + - 'coordination.k8s.io' + resources: + - 'leases' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: sriov-network-config-daemon + namespace: {{ .Release.Namespace }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - "get" + - "list" + - "watch" + - "delete" + - apiGroups: + - sriovnetwork.openshift.io + resources: + - '*' + - sriovnetworknodestates + verbs: + - '*' + - apiGroups: + - security.openshift.io + resourceNames: + - privileged + resources: + - securitycontextconstraints + verbs: + - use + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - update + - apiGroups: + - 'coordination.k8s.io' + resources: + - 'leases' + verbs: + - '*' + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: operator-webhook-sa + namespace: {{ .Release.Namespace }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get diff --git a/network-operator/charts/sriov-network-operator/templates/rolebinding.yaml b/network-operator/charts/sriov-network-operator/templates/rolebinding.yaml new file mode 100644 index 0000000..2450cc0 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/templates/rolebinding.yaml @@ -0,0 +1,31 @@ +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "sriov-network-operator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: {{ include "sriov-network-operator.fullname" . }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: {{ include "sriov-network-operator.fullname" . }} + apiGroup: rbac.authorization.k8s.io +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: sriov-network-config-daemon + namespace: {{ .Release.Namespace }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +subjects: + - kind: ServiceAccount + name: sriov-network-config-daemon + namespace: {{ .Release.Namespace }} +roleRef: + kind: Role + name: sriov-network-config-daemon + apiGroup: rbac.authorization.k8s.io diff --git a/network-operator/charts/sriov-network-operator/templates/serviceaccount.yaml b/network-operator/charts/sriov-network-operator/templates/serviceaccount.yaml new file mode 100644 index 0000000..fc0bb57 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/templates/serviceaccount.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "sriov-network-operator.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: sriov-network-config-daemon + namespace: {{ .Release.Namespace }} + labels: + {{- include "sriov-network-operator.labels" . | nindent 4 }} diff --git a/network-operator/charts/sriov-network-operator/templates/sriovoperatorconfig.yaml b/network-operator/charts/sriov-network-operator/templates/sriovoperatorconfig.yaml new file mode 100644 index 0000000..3bec8d3 --- /dev/null +++ b/network-operator/charts/sriov-network-operator/templates/sriovoperatorconfig.yaml @@ -0,0 +1,21 @@ +{{ if .Values.sriovOperatorConfig.deploy }} +apiVersion: sriovnetwork.openshift.io/v1 +kind: SriovOperatorConfig +metadata: + name: default + namespace: {{ .Release.Namespace }} +spec: + enableInjector: {{ .Values.operator.admissionControllers.enabled }} + enableOperatorWebhook: {{ .Values.operator.admissionControllers.enabled }} + {{- with .Values.sriovOperatorConfig.configDaemonNodeSelector }} + configDaemonNodeSelector: + {{- range $k, $v := .}}{{printf "%s: \"%s\"" $k $v | nindent 4 }}{{ end }} + {{- end }} + logLevel: {{ .Values.sriovOperatorConfig.logLevel }} + disableDrain: {{ .Values.sriovOperatorConfig.disableDrain }} + configurationMode: {{ .Values.sriovOperatorConfig.configurationMode }} + {{- with .Values.sriovOperatorConfig.featureGates }} + featureGates: + {{- range $k, $v := .}}{{printf "%s: %t" $k $v | nindent 4 }}{{ end }} + {{- end }} +{{ end }} \ No newline at end of file diff --git a/network-operator/charts/sriov-network-operator/values.yaml b/network-operator/charts/sriov-network-operator/values.yaml new file mode 100644 index 0000000..ec9323b --- /dev/null +++ b/network-operator/charts/sriov-network-operator/values.yaml @@ -0,0 +1,125 @@ +operator: + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + nodeSelector: {} + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: In + values: [""] + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: In + values: [""] + nameOverride: "" + fullnameOverride: "" + resourcePrefix: "openshift.io" + cniBinPath: "/opt/cni/bin" + clusterType: "kubernetes" + # minimal amount of time (in minutes) the operator will wait before removing + # stale SriovNetworkNodeState objects (objects that doesn't match node with the daemon) + # "0" means no extra delay, in this case the CR will be removed by the next reconcilation cycle (may take up to 5 minutes) + staleNodeStateCleanupDelayMinutes: "30" + metricsExporter: + port: "9110" + certificates: + secretName: "metrics-exporter-cert" + prometheusOperator: + enabled: false + serviceAccount: "prometheus-k8s" + namespace: "monitoring" + deployRules: false + admissionControllers: + enabled: false + certificates: + secretNames: + operator: "operator-webhook-cert" + injector: "network-resources-injector-cert" + certManager: + # When enabled, makes use of certificates managed by cert-manager. + enabled: false + # When enabled, certificates are generated via cert-manager and then name will match the name of the secrets + # defined above + generateSelfSigned: false + # If not specified, no secret is created and secrets with the names defined above are expected to exist in the + # cluster. In that case, the ca.crt must be base64 encoded twice since it ends up being an env variable. + custom: + enabled: false + # operator: + # caCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsKey: | + # -----BEGIN EC PRIVATE KEY----- + # MHcl4wOuDwKQa+upc8GftXE2C//4mKANBC6It01gUaTIpo= + # ... + # -----END EC PRIVATE KEY----- + # injector: + # caCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsKey: | + # -----BEGIN EC PRIVATE KEY----- + # MHcl4wOuDwKQa+upc8GftXE2C//4mKANBC6It01gUaTIpo= + # ... + # -----END EC PRIVATE KEY----- + +sriovOperatorConfig: + # deploy sriovOperatorConfig CR with the below values + deploy: false + # node selectors for sriov-network-config-daemon + configDaemonNodeSelector: {} + # log level for both operator and sriov-network-config-daemon + logLevel: 2 + # disable node draining when configuring SR-IOV, set to true in case of a single node + # cluster or any other justifiable reason + disableDrain: false + # sriov-network-config-daemon configuration mode. either "daemon" or "systemd" + configurationMode: daemon + # feature gates to enable/disable + featureGates: {} + +# Example for supportedExtraNICs values ['MyNIC: "8086 1521 1520"'] +supportedExtraNICs: [] + +# Image URIs for sriov-network-operator components +images: + operator: ghcr.io/k8snetworkplumbingwg/sriov-network-operator + sriovConfigDaemon: ghcr.io/k8snetworkplumbingwg/sriov-network-operator-config-daemon + sriovCni: ghcr.io/k8snetworkplumbingwg/sriov-cni + ibSriovCni: ghcr.io/k8snetworkplumbingwg/ib-sriov-cni + ovsCni: ghcr.io/k8snetworkplumbingwg/ovs-cni-plugin + rdmaCni: ghcr.io/k8snetworkplumbingwg/rdma-cni + sriovDevicePlugin: ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin + resourcesInjector: ghcr.io/k8snetworkplumbingwg/network-resources-injector + webhook: ghcr.io/k8snetworkplumbingwg/sriov-network-operator-webhook + metricsExporter: ghcr.io/k8snetworkplumbingwg/sriov-network-metrics-exporter + metricsExporterKubeRbacProxy: gcr.io/kubebuilder/kube-rbac-proxy:v0.15.0 + +imagePullSecrets: [] +extraDeploy: [] diff --git a/network-operator/crds/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml b/network-operator/crds/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml new file mode 100644 index 0000000..5916055 --- /dev/null +++ b/network-operator/crds/k8s.cni.cncf.io_networkattachmentdefinitions_crd.yaml @@ -0,0 +1,57 @@ +# Copyright 2020 NVIDIA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: network-attachment-definitions.k8s.cni.cncf.io +spec: + group: k8s.cni.cncf.io + scope: Namespaced + names: + plural: network-attachment-definitions + singular: network-attachment-definition + kind: NetworkAttachmentDefinition + shortNames: + - net-attach-def + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + description: 'NetworkAttachmentDefinition is a CRD schema specified by the Network Plumbing + Working Group to express the intent for attaching pods to one or more logical or physical + networks. More information available at: https://github.com/k8snetworkplumbingwg/multi-net-spec' + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this represen + tation of an object. Servers should convert recognized schemas to the + latest internal value, and may reject unrecognized values. More info: + https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: 'NetworkAttachmentDefinition spec defines the desired state of a network attachment' + type: object + properties: + config: + description: 'NetworkAttachmentDefinition config is a JSON-formatted CNI configuration' + type: string diff --git a/network-operator/crds/mellanox.com_hostdevicenetworks.yaml b/network-operator/crds/mellanox.com_hostdevicenetworks.yaml new file mode 100644 index 0000000..859c055 --- /dev/null +++ b/network-operator/crds/mellanox.com_hostdevicenetworks.yaml @@ -0,0 +1,111 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: hostdevicenetworks.mellanox.com +spec: + group: mellanox.com + names: + kind: HostDeviceNetwork + listKind: HostDeviceNetworkList + plural: hostdevicenetworks + singular: hostdevicenetwork + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: HostDeviceNetwork is the Schema for the hostdevicenetworks API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Defines the desired state of HostDeviceNetwork + properties: + ipam: + description: IPAM configuration to be used for this network + type: string + networkNamespace: + description: Namespace of the NetworkAttachmentDefinition custom resource + type: string + resourceName: + description: Host device resource pool name + type: string + type: object + status: + description: Defines the observed state of HostDeviceNetwork + properties: + appliedStates: + description: AppliedStates provide a finer view of the observed state + items: + description: AppliedState defines a finer-grained view of the observed + state of NicClusterPolicy + properties: + message: + description: |- + Message is a human readable message indicating details about why + the state is in this condition + type: string + name: + description: Name of the deployed component this state refers + to + type: string + state: + description: The state of the deployed component. ("ready", + "notReady", "ignore", "error") + enum: + - ready + - notReady + - ignore + - error + type: string + required: + - name + - state + type: object + type: array + hostDeviceNetworkAttachmentDef: + description: Network attachment definition generated from HostDeviceNetworkSpec + type: string + reason: + description: Informative string in case the observed state is error + type: string + state: + description: Reflects the state of the HostDeviceNetwork + enum: + - notReady + - ready + - error + type: string + required: + - state + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/network-operator/crds/mellanox.com_ipoibnetworks.yaml b/network-operator/crds/mellanox.com_ipoibnetworks.yaml new file mode 100644 index 0000000..91e2895 --- /dev/null +++ b/network-operator/crds/mellanox.com_ipoibnetworks.yaml @@ -0,0 +1,83 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: ipoibnetworks.mellanox.com +spec: + group: mellanox.com + names: + kind: IPoIBNetwork + listKind: IPoIBNetworkList + plural: ipoibnetworks + singular: ipoibnetwork + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: IPoIBNetwork is the Schema for the ipoibnetworks API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Defines the desired state of IPoIBNetwork + properties: + ipam: + description: IPAM configuration to be used for this network. + type: string + master: + description: Name of the host interface to enslave. Defaults to default + route interface + type: string + networkNamespace: + description: Namespace of the NetworkAttachmentDefinition custom resource + type: string + type: object + status: + description: Defines the observed state of IPoIBNetwork + properties: + ipoibNetworkAttachmentDef: + description: Network attachment definition generated from IPoIBNetworkSpec + type: string + reason: + description: Informative string in case the observed state is error + type: string + state: + description: Reflects the state of the IPoIBNetwork + enum: + - notReady + - ready + - error + type: string + required: + - state + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/network-operator/crds/mellanox.com_macvlannetworks.yaml b/network-operator/crds/mellanox.com_macvlannetworks.yaml new file mode 100644 index 0000000..5bc9d40 --- /dev/null +++ b/network-operator/crds/mellanox.com_macvlannetworks.yaml @@ -0,0 +1,97 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: macvlannetworks.mellanox.com +spec: + group: mellanox.com + names: + kind: MacvlanNetwork + listKind: MacvlanNetworkList + plural: macvlannetworks + singular: macvlannetwork + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: MacvlanNetwork is the Schema for the macvlannetworks API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Defines the desired state of MacvlanNetworkSpec + properties: + ipam: + description: IPAM configuration to be used for this network. + type: string + master: + description: Name of the host interface to enslave. Defaults to default + route interface + type: string + mode: + description: Mode of interface one of "bridge", "private", "vepa", + "passthru" + enum: + - bridge + - private + - vepa + - passthru + type: string + mtu: + description: MTU of interface to the specified value. 0 for master's + MTU + minimum: 0 + type: integer + networkNamespace: + description: Namespace of the NetworkAttachmentDefinition custom resource + type: string + type: object + status: + description: Defines the observed state of MacvlanNetwork + properties: + macvlanNetworkAttachmentDef: + description: Network attachment definition generated from MacvlanNetworkSpec + type: string + reason: + description: Informative string in case the observed state is error + type: string + state: + description: Reflects the state of the MacvlanNetwork + enum: + - notReady + - ready + - error + type: string + required: + - state + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/network-operator/crds/mellanox.com_nicclusterpolicies.yaml b/network-operator/crds/mellanox.com_nicclusterpolicies.yaml new file mode 100644 index 0000000..690153b --- /dev/null +++ b/network-operator/crds/mellanox.com_nicclusterpolicies.yaml @@ -0,0 +1,1414 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.16.4 + name: nicclusterpolicies.mellanox.com +spec: + group: mellanox.com + names: + kind: NicClusterPolicy + listKind: NicClusterPolicyList + plural: nicclusterpolicies + shortNames: + - ncp + singular: nicclusterpolicy + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .status.state + name: Status + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: NicClusterPolicy is the Schema for the nicclusterpolicies API + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: Defines the desired state of NicClusterPolicy + properties: + docaTelemetryService: + description: |- + DOCATelemetryService exposes telemetry from NVIDIA networking components to prometheus. + See: https://docs.nvidia.com/doca/sdk/nvidia+doca+telemetry+service+guide/index.html + properties: + config: + description: |- + Config contains custom config for the DOCATelemetryService. + If set no default config will be deployed. + properties: + fromConfigMap: + description: |- + FromConfigMap sets the configMap the DOCATelemetryService gets its configuration from. The ConfigMap must be in + the same namespace as the NICClusterPolicy. + type: string + type: object + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + description: Version of the image to use + type: string + required: + - image + - repository + - version + type: object + ibKubernetes: + description: |- + IBKubernetes provides a daemon that works in conjunction with the SR-IOV Network Device Plugin. + It acts on Kubernetes pod object changes and reads the pod's network annotation. + From there it fetches the corresponding network CRD and reads the PKey. + This is done in order to add the newly generated GUID or the predefined GUID in the GUID field of the CRD. + This is then passed in cni-args to that PKey for pods with mellanox.infiniband.app annotation. + See: https://github.com/Mellanox/ib-kubernetes + properties: + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + pKeyGUIDPoolRangeEnd: + description: The last guid in the pool + type: string + pKeyGUIDPoolRangeStart: + description: The first guid in the pool + type: string + periodicUpdateSeconds: + default: 5 + description: Interval of updates in seconds + minimum: 0 + type: integer + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + ufmSecret: + description: Secret containing credentials to UFM service + type: string + version: + description: Version of the image to use + type: string + required: + - image + - pKeyGUIDPoolRangeEnd + - pKeyGUIDPoolRangeStart + - repository + - ufmSecret + - version + type: object + nicFeatureDiscovery: + description: |- + NicFeatureDiscovery works with NodeFeatureDiscovery to expose information about NVIDIA NICs. + https://github.com/Mellanox/nic-feature-discovery + properties: + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + description: Version of the image to use + type: string + required: + - image + - repository + - version + type: object + nodeAffinity: + description: NodeAffinity rules to inject to the DaemonSets objects + that are managed by the operator + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: |- + The scheduler will prefer to schedule pods to nodes that satisfy + the affinity expressions specified by this field, but it may choose + a node that violates one or more of the expressions. The node that is + most preferred is the one with the greatest sum of weights, i.e. + for each node that meets all of the scheduling requirements (resource + request, requiredDuringScheduling affinity expressions, etc.), + compute a sum by iterating through the elements of this field and adding + "weight" to the sum if the node matches the corresponding matchExpressions; the + node(s) with the highest sum are the most preferred. + items: + description: |- + An empty preferred scheduling term matches all objects with implicit weight 0 + (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding + weight. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding + nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + x-kubernetes-list-type: atomic + requiredDuringSchedulingIgnoredDuringExecution: + description: |- + If the affinity requirements specified by this field are not met at + scheduling time, the pod will not be scheduled onto the node. + If the affinity requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The + terms are ORed. + items: + description: |- + A null or empty node selector term matches no objects. The requirements of + them are ANDed. + The TopologySelectorTerm type implements a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements by + node's labels. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchFields: + description: A list of node selector requirements by + node's fields. + items: + description: |- + A node selector requirement is a selector that contains values, a key, and an operator + that relates the key and values. + properties: + key: + description: The label key that the selector applies + to. + type: string + operator: + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + type: object + x-kubernetes-map-type: atomic + type: array + x-kubernetes-list-type: atomic + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + nvIpam: + description: |- + NvIpam is an IPAM provider that dynamically assigns IP addresses with speed and performance in mind. + Note: NvIPam requires certificate management e.g. cert-manager or OpenShift cert management. + See https://github.com/Mellanox/nvidia-k8s-ipam + properties: + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + enableWebhook: + description: Enable deployment of the validation webhook + type: boolean + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + description: Version of the image to use + type: string + required: + - image + - repository + - version + type: object + ofedDriver: + description: |- + OFEDDriver is a specialized driver for NVIDIA NICs which can replace the inbox driver that comes with an OS. + See https://network.nvidia.com/support/mlnx-ofed-matrix/ + properties: + certConfig: + description: 'Optional: Custom TLS certificates configuration + for DOCA driver container' + properties: + name: + description: Name of the ConfigMap + type: string + type: object + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + env: + description: List of environment variables to set in the DOCA + driver container. + items: + description: EnvVar represents an environment variable present + in a Container. + properties: + name: + description: Name of the environment variable. Must be a + C_IDENTIFIER. + type: string + value: + description: |- + Variable references $(VAR_NAME) are expanded + using the previously defined environment variables in the container and + any service environment variables. If a variable cannot be resolved, + the reference in the input string will be unchanged. Double $$ are reduced + to a single $, which allows for escaping the $(VAR_NAME) syntax: i.e. + "$$(VAR_NAME)" will produce the string literal "$(VAR_NAME)". + Escaped references will never be expanded, regardless of whether the variable + exists or not. + Defaults to "". + type: string + valueFrom: + description: Source for the environment variable's value. + Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the ConfigMap or its + key must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + description: |- + Selects a field of the pod: supports metadata.name, metadata.namespace, `metadata.labels['']`, `metadata.annotations['']`, + spec.nodeName, spec.serviceAccountName, status.hostIP, status.podIP, status.podIPs. + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select in the + specified API version. + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + description: |- + Selects a resource of the container: only resources limits and requests + (limits.cpu, limits.memory, limits.ephemeral-storage, requests.cpu, requests.memory and requests.ephemeral-storage) are currently supported. + properties: + containerName: + description: 'Container name: required for volumes, + optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format of the + exposed resources, defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + description: Selects a key of a secret in the pod's + namespace + properties: + key: + description: The key of the secret to select from. Must + be a valid secret key. + type: string + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: Specify whether the Secret or its key + must be defined + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + forcePrecompiled: + default: false + description: |- + ForcePrecompiled specifies if only DOCA driver precompiled images are allowed + If set to false and precompiled image does not exists, DOCA driver will be compiled on Nodes + If set to true and precompiled image does not exists, OFED state will be Error. + type: boolean + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + livenessProbe: + description: Pod liveness probe settings + properties: + initialDelaySeconds: + description: Number of seconds after the container has started + before the probe is initiated + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe + type: integer + required: + - initialDelaySeconds + - periodSeconds + type: object + readinessProbe: + description: Pod readiness probe settings + properties: + initialDelaySeconds: + description: Number of seconds after the container has started + before the probe is initiated + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe + type: integer + required: + - initialDelaySeconds + - periodSeconds + type: object + repoConfig: + description: 'Optional: Custom package repository configuration + for DOCA driver container' + properties: + name: + description: Name of the ConfigMap + type: string + type: object + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + startupProbe: + description: Pod startup probe settings + properties: + initialDelaySeconds: + description: Number of seconds after the container has started + before the probe is initiated + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe + type: integer + required: + - initialDelaySeconds + - periodSeconds + type: object + terminationGracePeriodSeconds: + default: 300 + description: |- + TerminationGracePeriodSeconds specifies the length of time in seconds + to wait before killing the DOCA driver container pod on termination + format: int64 + minimum: 0 + type: integer + upgradePolicy: + description: DOCA driver auto-upgrade settings + properties: + autoUpgrade: + default: false + description: |- + AutoUpgrade is a global switch for automatic upgrade feature + if set to false all other options are ignored + type: boolean + drain: + description: The configuration for node drain during automatic + upgrade + properties: + deleteEmptyDir: + default: false + description: |- + DeleteEmptyDir indicates if should continue even if there are pods using emptyDir + (local data that will be deleted when the node is drained) + type: boolean + enable: + default: true + description: Enable indicates if node draining is allowed + during upgrade + type: boolean + force: + default: false + description: Force indicates if force draining is allowed + type: boolean + podSelector: + description: |- + PodSelector specifies a label selector to filter pods on the node that need to be drained + For more details on label selectors, see: + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + type: string + timeoutSeconds: + default: 300 + description: TimeoutSecond specifies the length of time + in seconds to wait before giving up drain, zero means + infinite + minimum: 0 + type: integer + type: object + maxParallelUpgrades: + default: 1 + description: |- + MaxParallelUpgrades indicates how many nodes can be upgraded in parallel + 0 means no limit, all nodes will be upgraded in parallel + minimum: 0 + type: integer + safeLoad: + default: false + description: SafeLoad turn on safe driver loading (cordon + and drain the node before loading the driver) + type: boolean + waitForCompletion: + description: The configuration for waiting on pods completions + properties: + podSelector: + description: |- + PodSelector specifies a label selector for the pods to wait for completion + For more details on label selectors, see: + https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors + type: string + timeoutSeconds: + default: 0 + description: |- + TimeoutSecond specifies the length of time in seconds + to wait before giving up on pod termination, zero means infinite + minimum: 0 + type: integer + type: object + type: object + version: + description: Version of the image to use + type: string + required: + - image + - repository + - version + type: object + rdmaSharedDevicePlugin: + description: |- + RdmaSharedDevicePlugin manages support IB and RoCE HCAs through the Kubernetes device plugin framework. + The config field is a json representation of the RDMA shared device plugin configuration. + See https://github.com/Mellanox/k8s-rdma-shared-dev-plugin + properties: + config: + description: Configuration for the component as a string + type: string + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + useCdi: + description: |- + Enables use of container device interface (CDI) + NOTE: NVIDIA Network Operator does not configure container runtime to enable CDI. + type: boolean + version: + description: Version of the image to use + type: string + required: + - image + - repository + - version + type: object + secondaryNetwork: + description: |- + SecondaryNetwork Specifies components to deploy in order to facilitate a secondary network in Kubernetes. + It consists of the following optionally deployed components: + - Multus-CNI: Delegate CNI plugin to support secondary networks in Kubernetes + - CNI plugins: Currently only containernetworking-plugins is supported + - IPAM CNI: Currently only Whereabout IPAM CNI is supported as a part of the secondaryNetwork section. + - IPoIB CNI: Allows the user to create IPoIB child link and move it to the pod + properties: + cniPlugins: + description: Image information for CNI plugins + properties: + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute + resource requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements + are set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + description: Version of the image to use + type: string + required: + - image + - repository + - version + type: object + ipamPlugin: + description: Image information for IPAM plugin + properties: + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute + resource requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements + are set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + description: Version of the image to use + type: string + required: + - image + - repository + - version + type: object + ipoib: + description: Image information for IPoIB CNI + properties: + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute + resource requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements + are set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + description: Version of the image to use + type: string + required: + - image + - repository + - version + type: object + multus: + description: Image and configuration information for multus + properties: + config: + description: Configuration for the component as a string + type: string + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute + resource requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements + are set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + version: + description: Version of the image to use + type: string + required: + - image + - repository + - version + type: object + type: object + sriovDevicePlugin: + description: |- + SriovDevicePlugin manages SRIOV through the Kubernetes device plugin framework. + The config field is a json representation of the RDMA shared device plugin configuration. + See https://github.com/k8snetworkplumbingwg/sriov-network-device-plugin + properties: + config: + description: Configuration for the component as a string + type: string + containerResources: + description: ResourceRequirements describes the compute resource + requirements + items: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Limits describes the maximum amount of compute resources allowed. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + name: + description: Name of the container the requirements are + set for + type: string + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: |- + Requests describes the minimum amount of compute resources required. + If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, + otherwise to an implementation-defined value. Requests cannot exceed Limits. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ + type: object + required: + - name + type: object + type: array + image: + description: Name of the image + pattern: '[a-zA-Z0-9\-]+' + type: string + imagePullSecrets: + default: [] + description: |- + ImagePullSecrets is an optional list of references to secrets in the same + namespace to use for pulling the image + items: + type: string + type: array + repository: + description: Address of the registry that stores the image + pattern: '[a-zA-Z0-9\.\-\/]+' + type: string + useCdi: + description: |- + Enables use of container device interface (CDI) + NOTE: NVIDIA Network Operator does not configure container runtime to enable CDI. + type: boolean + version: + description: Version of the image to use + type: string + required: + - image + - repository + - version + type: object + tolerations: + description: Tolerations to inject to the DaemonSets objects that + are managed by the operator + items: + description: |- + The pod this Toleration is attached to tolerates any taint that matches + the triple using the matching operator . + properties: + effect: + description: |- + Effect indicates the taint effect to match. Empty means match all taint effects. + When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: |- + Key is the taint key that the toleration applies to. Empty means match all taint keys. + If the key is empty, operator must be Exists; this combination means to match all values and all keys. + type: string + operator: + description: |- + Operator represents a key's relationship to the value. + Valid operators are Exists and Equal. Defaults to Equal. + Exists is equivalent to wildcard for value, so that a pod can + tolerate all taints of a particular category. + type: string + tolerationSeconds: + description: |- + TolerationSeconds represents the period of time the toleration (which must be + of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default, + it is not set, which means tolerate the taint forever (do not evict). Zero and + negative values will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: |- + Value is the taint value the toleration matches to. + If the operator is Exists, the value should be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + status: + description: Defines the observed state of NicClusterPolicy + properties: + appliedStates: + description: AppliedStates provide a finer view of the observed state + items: + description: AppliedState defines a finer-grained view of the observed + state of NicClusterPolicy + properties: + message: + description: |- + Message is a human readable message indicating details about why + the state is in this condition + type: string + name: + description: Name of the deployed component this state refers + to + type: string + state: + description: The state of the deployed component. ("ready", + "notReady", "ignore", "error") + enum: + - ready + - notReady + - ignore + - error + type: string + required: + - name + - state + type: object + type: array + reason: + description: Informative string in case the observed state is error + type: string + state: + description: Reflects the current state of the cluster policy + enum: + - ignore + - notReady + - ready + - error + type: string + required: + - state + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/network-operator/hostdev/demo1.yaml b/network-operator/hostdev/demo1.yaml new file mode 100644 index 0000000..734a827 --- /dev/null +++ b/network-operator/hostdev/demo1.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Pod +metadata: + name: demo-pod-monitoring-1 + annotations: + k8s.v1.cni.cncf.io/networks: hostdevice-net +spec: + nodeSelector: + # Note: Replace hostname or remove selector altogether + kubernetes.io/hostname: nhn-aideveloper-d40ce770 + restartPolicy: OnFailure + containers: + - image: mellanox/cuda-perftest + name: rdma-gpu-test-ctr + securityContext: + capabilities: + add: [ "IPC_LOCK" ] + resources: + limits: + nvidia.com/gpu: 8 + nvidia.com/hostdev: 8 + requests: + nvidia.com/gpu: 8 + nvidia.com/hostdev: 8 diff --git a/network-operator/hostdev/demo2.yaml b/network-operator/hostdev/demo2.yaml new file mode 100644 index 0000000..43900e4 --- /dev/null +++ b/network-operator/hostdev/demo2.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Pod +metadata: + name: demo-pod-monitoring-2 + annotations: + k8s.v1.cni.cncf.io/networks: hostdevice-net +spec: + nodeSelector: + # Note: Replace hostname or remove selector altogether + kubernetes.io/hostname: nhn-aideveloper-f0fde370 + restartPolicy: OnFailure + containers: + - image: mellanox/cuda-perftest + name: rdma-gpu-test-ctr + securityContext: + capabilities: + add: [ "IPC_LOCK" ] + resources: + limits: + nvidia.com/gpu: 8 + nvidia.com/hostdev: 8 + requests: + nvidia.com/gpu: 8 + nvidia.com/hostdev: 8 diff --git a/network-operator/hostdev/direct-rdma-1.yaml b/network-operator/hostdev/direct-rdma-1.yaml new file mode 100644 index 0000000..755dae1 --- /dev/null +++ b/network-operator/hostdev/direct-rdma-1.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Pod +metadata: + name: demo-pod-1 + annotations: + k8s.v1.cni.cncf.io/networks: "hostdevice-net, hostdevice-net, hostdevice-net, hostdevice-net, hostdevice-net, hostdevice-net, hostdevice-net, hostdevice-net" +spec: + nodeSelector: + # Note: Replace hostname or remove selector altogether + kubernetes.io/hostname: nhn-aideveloper-d40ce770 + restartPolicy: OnFailure + containers: + - image: mellanox/cuda-perftest + name: rdma-gpu-test-ctr + securityContext: + capabilities: + add: [ "IPC_LOCK" ] + resources: + limits: + nvidia.com/gpu: 8 + nvidia.com/hostdev: 8 + requests: + nvidia.com/gpu: 8 + nvidia.com/hostdev: 8 diff --git a/network-operator/hostdev/direct-rdma-2.yaml b/network-operator/hostdev/direct-rdma-2.yaml new file mode 100644 index 0000000..d4e5613 --- /dev/null +++ b/network-operator/hostdev/direct-rdma-2.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Pod +metadata: + name: demo-pod-2 + annotations: + k8s.v1.cni.cncf.io/networks: "hostdevice-net, hostdevice-net, hostdevice-net, hostdevice-net, hostdevice-net, hostdevice-net, hostdevice-net, hostdevice-net" +spec: + nodeSelector: + # Note: Replace hostname or remove selector altogether + kubernetes.io/hostname: nhn-aideveloper-f0fde370 + restartPolicy: OnFailure + containers: + - image: mellanox/cuda-perftest + name: rdma-gpu-test-ctr + securityContext: + capabilities: + add: [ "IPC_LOCK" ] + resources: + limits: + nvidia.com/gpu: 8 + nvidia.com/hostdev: 8 + requests: + nvidia.com/gpu: 8 + nvidia.com/hostdev: 8 diff --git a/network-operator/hostdev/host-device-net.yaml b/network-operator/hostdev/host-device-net.yaml new file mode 100644 index 0000000..dfbb6b5 --- /dev/null +++ b/network-operator/hostdev/host-device-net.yaml @@ -0,0 +1,39 @@ +apiVersion: mellanox.com/v1alpha1 +kind: HostDeviceNetwork +metadata: + name: hostdevice-net + namespace: nvidia-network-operator +spec: + ipam: | + { + "type": "whereabouts", + "datastore": "kubernetes", + "kubernetes": { + "kubeconfig": "/etc/cni/net.d/whereabouts.d/whereabouts.kubeconfig" + }, + "range": "192.168.240.0/20", + "exclude": [ + "192.168.240.0/32", + "192.168.255.255/32", + "192.168.240.70/32", + "192.168.240.151/32", + "192.168.240.217/32", + "192.168.240.227/32", + "192.168.241.16/32", + "192.168.241.112/32", + "192.168.241.170/32", + "192.168.241.186/32", + "192.168.241.193/32", + "192.168.241.204/32", + "192.168.243.29/32", + "192.168.243.46/32", + "192.168.243.51/32", + "192.168.243.138/32", + "192.168.243.172/32", + "192.168.243.194/32" + ], + "log_file": "/var/log/whereabouts.log", + "log_level": "info" + } + resourceName: hostdev + networkNamespace: default diff --git a/network-operator/hostdev/nicclusterpolicy-test.yaml b/network-operator/hostdev/nicclusterpolicy-test.yaml new file mode 100644 index 0000000..f384a64 --- /dev/null +++ b/network-operator/hostdev/nicclusterpolicy-test.yaml @@ -0,0 +1,47 @@ +apiVersion: mellanox.com/v1alpha1 +kind: NicClusterPolicy +metadata: + name: nic-cluster-policy +spec: + sriovDevicePlugin: + image: sriov-network-device-plugin + repository: ghcr.io/k8snetworkplumbingwg + version: v3.9.0 + imagePullSecrets: [] + config: | + { + "resourceList": [ + { + "resourcePrefix": "nvidia.com", + "resourceName": "hostdev", + "selectors": { + "vendors": ["15b3"], + "devices": [], + "drivers": [], + "pfNames": [], + "pciAddresses": [], + "rootDevices": [], + "linkTypes": [], + "isRdma": true + } + } + ] + } + + secondaryNetwork: + cniPlugins: + image: plugins + repository: ghcr.io/k8snetworkplumbingwg + version: v1.5.0 + imagePullSecrets: [] + multus: + image: multus-cni + repository: ghcr.io/k8snetworkplumbingwg + version: v4.1.0 + imagePullSecrets: [] + ipamPlugin: + image: whereabouts + repository: ghcr.io/k8snetworkplumbingwg + version: v0.7.0 + imagePullSecrets: [] + diff --git a/network-operator/hostdev/nicclusterpolicy.yaml b/network-operator/hostdev/nicclusterpolicy.yaml new file mode 100644 index 0000000..9672216 --- /dev/null +++ b/network-operator/hostdev/nicclusterpolicy.yaml @@ -0,0 +1,56 @@ +apiVersion: mellanox.com/v1alpha1 +kind: NicClusterPolicy +metadata: + name: nic-cluster-policy +spec: + sriovDevicePlugin: + image: sriov-network-device-plugin + repository: ghcr.io/k8snetworkplumbingwg + version: v3.9.0 + imagePullSecrets: [] + config: | + { + "resourceList": [ + { + "resourcePrefix": "nvidia.com", + "resourceName": "hostdev", + "selectors": { + "vendors": ["15b3"], + "devices": [], + "drivers": [], + "pfNames": [], + "pciAddresses": [ + "0000:19:00.0", + "0000:29:00.0", + "0000:3b:00.0", + "0000:5c:00.0", + "0000:85:00.0", + "0000:8a:00.0", + "0000:92:00.0", + "0000:e3:00.0" + ], + "rootDevices": [], + "linkTypes": [], + "isRdma": true + } + } + ] + } + + secondaryNetwork: + cniPlugins: + image: plugins + repository: ghcr.io/k8snetworkplumbingwg + version: v1.5.0 + imagePullSecrets: [] + multus: + image: multus-cni + repository: ghcr.io/k8snetworkplumbingwg + version: v4.1.0 + imagePullSecrets: [] + ipamPlugin: + image: whereabouts + repository: ghcr.io/k8snetworkplumbingwg + version: v0.7.0 + imagePullSecrets: [] + diff --git a/network-operator/hostdev/sample-pod.yaml b/network-operator/hostdev/sample-pod.yaml new file mode 100644 index 0000000..e705a58 --- /dev/null +++ b/network-operator/hostdev/sample-pod.yaml @@ -0,0 +1,28 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-hostdevice-pod + annotations: + k8s.v1.cni.cncf.io/networks: hostdevice-net +spec: + nodeSelector: + nodegroup: infini + containers: + - name: test-hostdevice-pod + image: ubuntu:22.04 + imagePullPolicy: IfNotPresent + securityContext: + capabilities: + add: ["IPC_LOCK"] + command: + - sh + - -c + - sleep inf + resources: + requests: + nvidia.com/hostdev: '1' + nvidia.com/gpu: '1' + limits: + nvidia.com/hostdev: '1' + nvidia.com/gpu: '1' + diff --git a/network-operator/ipiob/nicclusterpolicy.yaml b/network-operator/ipiob/nicclusterpolicy.yaml new file mode 100644 index 0000000..3a680c1 --- /dev/null +++ b/network-operator/ipiob/nicclusterpolicy.yaml @@ -0,0 +1,51 @@ +apiVersion: mellanox.com/v1alpha1 +kind: NicClusterPolicy +metadata: + name: nic-cluster-policy +spec: + rdmaSharedDevicePlugin: + # [map[ifNames:[ibs1f0] name:rdma_shared_device_a]] + image: k8s-rdma-shared-device-plugin + repository: ghcr.io/mellanox + version: v1.5.2 + imagePullSecrets: [] + # The config below directly propagates to k8s-rdma-shared-device-plugin configuration. + # Replace 'devices' with your (RDMA capable) netdevice name. + config: | + { + "configList": [ + { + "resourceName": "rdma_shared_device", + "rdmaHcaMax": 63, + "selectors": { + "vendors": [], + "deviceIDs": [], + "drivers": [], + "ifNames": ["ib0", "ib1", "ib2", "ib3", "ib4", "ib5", "ib6", "ib7"], + "linkTypes": [] + } + } + ] + } + + secondaryNetwork: + cniPlugins: + image: plugins + repository: ghcr.io/k8snetworkplumbingwg + version: v1.5.0 + imagePullSecrets: [] + multus: + image: multus-cni + repository: ghcr.io/k8snetworkplumbingwg + version: v4.1.0 + imagePullSecrets: [] + ipoib: + image: ipoib-cni + repository: ghcr.io/mellanox + version: 428715a57c0b633e48ec7620f6e3af6863149ccf + ipamPlugin: + image: whereabouts + repository: ghcr.io/k8snetworkplumbingwg + version: v0.7.0 + imagePullSecrets: [] + diff --git a/network-operator/sriov/demo1.yaml b/network-operator/sriov/demo1.yaml new file mode 100644 index 0000000..8d63577 --- /dev/null +++ b/network-operator/sriov/demo1.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Pod +metadata: + name: demo-pod-monitoring-1 + annotations: + k8s.v1.cni.cncf.io/networks: sriov-ib-network +spec: + nodeSelector: + # Note: Replace hostname or remove selector altogether + kubernetes.io/hostname: nhn-aideveloper-d40ce770 + restartPolicy: OnFailure + containers: + - image: mellanox/cuda-perftest + name: rdma-gpu-test-ctr + securityContext: + capabilities: + add: [ "IPC_LOCK" ] + resources: + limits: + nvidia.com/gpu: 8 + nvidia.com/mlnxnics: 8 + requests: + nvidia.com/gpu: 8 + nvidia.com/mlnxnics: 8 diff --git a/network-operator/sriov/demo2.yaml b/network-operator/sriov/demo2.yaml new file mode 100644 index 0000000..13f8ad8 --- /dev/null +++ b/network-operator/sriov/demo2.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Pod +metadata: + name: demo-pod-monitoring-2 + annotations: + k8s.v1.cni.cncf.io/networks: sriov-ib-network +spec: + nodeSelector: + # Note: Replace hostname or remove selector altogether + kubernetes.io/hostname: nhn-aideveloper-f0fde370 + restartPolicy: OnFailure + containers: + - image: mellanox/cuda-perftest + name: rdma-gpu-test-ctr + securityContext: + capabilities: + add: [ "IPC_LOCK" ] + resources: + limits: + nvidia.com/gpu: 8 + nvidia.com/mlnxnics: 8 + requests: + nvidia.com/gpu: 8 + nvidia.com/mlnxnics: 8 diff --git a/network-operator/sriov/nicclusterpolicy.yaml b/network-operator/sriov/nicclusterpolicy.yaml new file mode 100644 index 0000000..3493256 --- /dev/null +++ b/network-operator/sriov/nicclusterpolicy.yaml @@ -0,0 +1,23 @@ +apiVersion: mellanox.com/v1alpha1 +kind: NicClusterPolicy +metadata: + name: nic-cluster-policy +spec: + secondaryNetwork: + cniPlugins: + image: plugins + repository: ghcr.io/k8snetworkplumbingwg + version: v1.5.0 + imagePullSecrets: [] + multus: + image: multus-cni + repository: ghcr.io/k8snetworkplumbingwg + version: v4.1.0 + imagePullSecrets: [] + ipamPlugin: + image: whereabouts + repository: ghcr.io/k8snetworkplumbingwg + version: v0.7.0 + imagePullSecrets: [] + + diff --git a/network-operator/sriov/nodepolicy_gcore1.yaml b/network-operator/sriov/nodepolicy_gcore1.yaml new file mode 100644 index 0000000..6eec17a --- /dev/null +++ b/network-operator/sriov/nodepolicy_gcore1.yaml @@ -0,0 +1,27 @@ +apiVersion: sriovnetwork.openshift.io/v1 +kind: SriovNetworkNodePolicy +metadata: + name: infiniband-sriov-a + namespace: nvidia-network-operator +spec: + deviceType: netdevice + mtu: 1500 + nodeSelector: + feature.node.kubernetes.io/pci-15b3.present: "true" + sriov.network.nvidia/total-vfs: "vf1" + nicSelector: + vendor: "15b3" + pfNames: + - "ib0" + - "ib1" + - "ib2" + - "ib3" + - "ib4" + - "ib5" + - "ib6" + - "ib7" + linkType: IB + isRdma: true + numVfs: 1 + priority: 90 + resourceName: mlnxnics diff --git a/network-operator/sriov/nodepolicy_gcore2.yaml b/network-operator/sriov/nodepolicy_gcore2.yaml new file mode 100644 index 0000000..df29d20 --- /dev/null +++ b/network-operator/sriov/nodepolicy_gcore2.yaml @@ -0,0 +1,27 @@ +apiVersion: sriovnetwork.openshift.io/v1 +kind: SriovNetworkNodePolicy +metadata: + name: infiniband-sriov + namespace: nvidia-network-operator +spec: + deviceType: netdevice + mtu: 1500 + nodeSelector: + feature.node.kubernetes.io/pci-15b3.present: "true" + sriov.network.nvidia/total-vfs: "vf16" + nicSelector: + vendor: "15b3" + pfNames: + - "ib0" + - "ib1" + - "ib2" + - "ib3" + - "ib4" + - "ib5" + - "ib6" + - "ib7" + linkType: IB + isRdma: true + numVfs: 16 + priority: 90 + resourceName: mlnxnics diff --git a/network-operator/sriov/sriov_net.yaml b/network-operator/sriov/sriov_net.yaml new file mode 100644 index 0000000..6ee1cda --- /dev/null +++ b/network-operator/sriov/sriov_net.yaml @@ -0,0 +1,40 @@ +apiVersion: sriovnetwork.openshift.io/v1 +kind: SriovIBNetwork +metadata: + name: sriov-ib-network + namespace: nvidia-network-operator +spec: + ipam: | + { + "type": "whereabouts", + "datastore": "kubernetes", + "kubernetes": { + "kubeconfig": "/etc/cni/net.d/whereabouts.d/whereabouts.kubeconfig" + }, + "range": "192.168.240.0/20", + "exclude": [ + "192.168.240.0/32", + "192.168.255.255/32", + "192.168.240.70/32", + "192.168.240.151/32", + "192.168.240.217/32", + "192.168.240.227/32", + "192.168.241.16/32", + "192.168.241.112/32", + "192.168.241.170/32", + "192.168.241.186/32", + "192.168.241.193/32", + "192.168.241.204/32", + "192.168.243.29/32", + "192.168.243.46/32", + "192.168.243.51/32", + "192.168.243.138/32", + "192.168.243.172/32", + "192.168.243.194/32" + ], + "log_file": "/var/log/whereabouts.log", + "log_level": "info" + } + resourceName: mlnxnics + linkState: enable + networkNamespace: default diff --git a/network-operator/sriov/test.log b/network-operator/sriov/test.log new file mode 100644 index 0000000..e0db5bb --- /dev/null +++ b/network-operator/sriov/test.log @@ -0,0 +1,401 @@ +2025-11-17T00:59:45.92409528Z INFO host/manager.go:61 infiniband.New(): ib guid config doesn't exist, continuing without it {"config path": "/etc/sriov-operator/infiniband/guids"} +2025-11-17T00:59:45.925745127Z INFO host/manager.go:61 infiniband.New(): ib guid config doesn't exist, continuing without it {"config path": "/etc/sriov-operator/infiniband/guids"} +2025-11-17T00:59:45.925936236Z INFO sriov-network-config-daemon cobra@v1.8.0/command.go:983 starting node writer +2025-11-17T00:59:45.959766078Z INFO sriov-network-config-daemon cobra@v1.8.0/command.go:983 Running on {"platform": "Baremetal"} +2025-11-17T00:59:45.998051223Z INFO sriov-network-config-daemon/start.go:277 RunOnce() +2025-11-17T00:59:45.998082634Z INFO sriov-network-config-daemon/start.go:277 RunOnce(): first poll for nic status +2025-11-17T00:59:46.225663023Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib0/speed", "error": "read /sys/class/net/ib0/speed: invalid argument"} +2025-11-17T00:59:46.226234114Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib1/speed", "error": "read /sys/class/net/ib1/speed: invalid argument"} +2025-11-17T00:59:46.231101563Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth0/speed", "error": "read /sys/class/net/eth0/speed: invalid argument"} +2025-11-17T00:59:46.23137743Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth1/speed", "error": "read /sys/class/net/eth1/speed: invalid argument"} +2025-11-17T00:59:46.232906162Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib6/speed", "error": "read /sys/class/net/ib6/speed: invalid argument"} +2025-11-17T00:59:46.233161697Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib7/speed", "error": "read /sys/class/net/ib7/speed: invalid argument"} +2025-11-17T00:59:46.243903326Z INFO daemon/writer.go:166 setNodeStateStatus(): status {"sync-status": "", "last-sync-error": ""} +2025-11-17T00:59:46.269166624Z INFO daemon/writer.go:83 writeCheckpointFile(): try to decode the checkpoint file +2025-11-17T00:59:46.269193495Z ERROR daemon/writer.go:83 writeCheckpointFile(): fail to decode, writing new file instead {"error": "EOF"} +2025-11-17T00:59:46.269210954Z INFO daemon/writer.go:83 writeCheckpointFile(): write checkpoint file +2025-11-17T00:59:46.269534063Z INFO runtime/asm_amd64.s:1695 Run(): start writer +2025-11-17T00:59:46.304248767Z INFO cobra@v1.8.0/command.go:983 Enabled featureGates {"featureGates": ""} +2025-11-17T00:59:46.304289678Z INFO sriov-network-config-daemon cobra@v1.8.0/command.go:983 Starting SriovNetworkConfigDaemon +2025-11-17T00:59:46.30434292Z INFO sriov-network-config-daemon/start.go:310 Run() {"node": "nhn-aideveloper-f0fde370"} +2025-11-17T00:59:46.304354684Z INFO sriov-network-config-daemon/start.go:310 Run(): start daemon. +2025-11-17T00:59:46.304360178Z INFO sriov-network-config-daemon/start.go:310 Run(): daemon running in daemon mode +2025-11-17T00:59:46.304370815Z INFO kernel/kernel.go:266 RunCommand() {"command": "/bin/sh", "args": ["-c", "chroot /host lsmod | grep --quiet 'mlx5_core'"]} +2025-11-17T00:59:46.327694971Z INFO kernel/kernel.go:284 RunCommand() {"command": "/bin/sh", "args": ["-c", "chroot /host lsmod | grep --quiet '\\(^ib\\|^rdma\\)'"]} +2025-11-17T00:59:46.336000978Z INFO kernel/kernel.go:75 LoadKernelModule(): try to load kernel module {"name": "tun", "args": []} +2025-11-17T00:59:46.336034332Z INFO kernel/kernel.go:33 IsKernelModuleLoaded(): check if kernel module is loaded {"name": "tun"} +2025-11-17T00:59:46.336044318Z INFO kernel/kernel.go:54 RunCommand() {"command": "/bin/sh", "args": ["-c", "chroot /host lsmod | grep \"^tun\""]} +2025-11-17T00:59:46.343881827Z INFO kernel/kernel.go:42 RunCommand() {"command": "/bin/sh", "args": ["-c", "chroot /host modprobe tun "]} +2025-11-17T00:59:46.347562828Z INFO kernel/kernel.go:81 LoadKernelModule(): try to load kernel module {"name": "vhost_net", "args": []} +2025-11-17T00:59:46.347592796Z INFO kernel/kernel.go:33 IsKernelModuleLoaded(): check if kernel module is loaded {"name": "vhost_net"} +2025-11-17T00:59:46.34760228Z INFO kernel/kernel.go:54 RunCommand() {"command": "/bin/sh", "args": ["-c", "chroot /host lsmod | grep \"^vhost_net\""]} +2025-11-17T00:59:46.359435844Z INFO kernel/kernel.go:42 RunCommand() {"command": "/bin/sh", "args": ["-c", "chroot /host modprobe vhost_net "]} +2025-11-17T00:59:46.374595365Z INFO udev/udev.go:41 RunCommand() {"command": "/bin/bash", "args": ["/bindata/scripts/udev-find-sriov-pf.sh"]} +2025-11-17T00:59:46.406541243Z INFO daemon/daemon.go:301 Set log verbose level {"new-level": 2, "current-level": 0} +2025-11-17T00:59:51.482409576Z INFO sriov-network-config-daemon/start.go:310 Starting workers +2025-11-17T00:59:51.482457526Z INFO sriov-network-config-daemon/start.go:310 Started workers +2025-11-17T00:59:51.482488729Z LEVEL(-2) daemon/daemon.go:225 processNextWorkItem {"worker-queue-size": 1} +2025-11-17T00:59:51.48253801Z LEVEL(-2) daemon/daemon.go:225 get item from queue {"item": 2} +2025-11-17T00:59:51.492736165Z INFO daemon/daemon.go:264 nodeStateSyncHandler(): new generation {"generation": 2} +2025-11-17T00:59:51.492760542Z INFO daemon/daemon.go:331 loadPlugins(): loading plugins +2025-11-17T00:59:51.493624528Z INFO daemon/daemon.go:331 loadPlugins(): loaded plugins {"plugins": ["mellanox", "intel", "k8s", "generic"]} +2025-11-17T00:59:51.493644864Z INFO runtime/asm_amd64.s:1695 Run(): refresh trigger +2025-11-17T00:59:51.493659399Z LEVEL(-2) daemon/writer.go:99 pollNicStatus() +2025-11-17T00:59:51.493683302Z LEVEL(-2) daemon/writer.go:130 DiscoverSriovDevices +2025-11-17T00:59:51.651457865Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib0"} +2025-11-17T00:59:51.65177688Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib0"} +2025-11-17T00:59:51.651809345Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib0/speed", "error": "read /sys/class/net/ib0/speed: invalid argument"} +2025-11-17T00:59:51.651820627Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib0"} +2025-11-17T00:59:51.651919153Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:19:00.0"} +2025-11-17T00:59:51.652024583Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib1"} +2025-11-17T00:59:51.652060955Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib1"} +2025-11-17T00:59:51.652075796Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib1/speed", "error": "read /sys/class/net/ib1/speed: invalid argument"} +2025-11-17T00:59:51.652081039Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib1"} +2025-11-17T00:59:51.652135525Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:29:00.0"} +2025-11-17T00:59:51.65223407Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib2"} +2025-11-17T00:59:51.652283638Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib2"} +2025-11-17T00:59:51.652591598Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib2"} +2025-11-17T00:59:51.652649113Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:3b:00.0"} +2025-11-17T00:59:51.652801692Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth2"} +2025-11-17T00:59:51.653432667Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth2"} +2025-11-17T00:59:51.653490268Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.0"} +2025-11-17T00:59:51.653617099Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth3"} +2025-11-17T00:59:51.654232365Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth3"} +2025-11-17T00:59:51.65429165Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.1"} +2025-11-17T00:59:51.654373917Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth0"} +2025-11-17T00:59:51.654418439Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth0"} +2025-11-17T00:59:51.65443549Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth0/speed", "error": "read /sys/class/net/eth0/speed: invalid argument"} +2025-11-17T00:59:51.654440713Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth0"} +2025-11-17T00:59:51.654493364Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.0"} +2025-11-17T00:59:51.654557678Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth1"} +2025-11-17T00:59:51.654598441Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth1"} +2025-11-17T00:59:51.654612631Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth1/speed", "error": "read /sys/class/net/eth1/speed: invalid argument"} +2025-11-17T00:59:51.654617764Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth1"} +2025-11-17T00:59:51.654670563Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.1"} +2025-11-17T00:59:51.654741889Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib3"} +2025-11-17T00:59:51.654785384Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib3"} +2025-11-17T00:59:51.654970092Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib3"} +2025-11-17T00:59:51.655028431Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:5c:00.0"} +2025-11-17T00:59:51.655107115Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib4"} +2025-11-17T00:59:51.655161874Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib4"} +2025-11-17T00:59:51.655346665Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib4"} +2025-11-17T00:59:51.655409423Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:85:00.0"} +2025-11-17T00:59:51.65549153Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib5"} +2025-11-17T00:59:51.655534542Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib5"} +2025-11-17T00:59:51.655724384Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib5"} +2025-11-17T00:59:51.655784227Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:8a:00.0"} +2025-11-17T00:59:51.655863831Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib6"} +2025-11-17T00:59:51.655914356Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib6"} +2025-11-17T00:59:51.655932055Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib6/speed", "error": "read /sys/class/net/ib6/speed: invalid argument"} +2025-11-17T00:59:51.655937476Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib6"} +2025-11-17T00:59:51.655991302Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:92:00.0"} +2025-11-17T00:59:51.656067198Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib7"} +2025-11-17T00:59:51.656115868Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib7"} +2025-11-17T00:59:51.656129586Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib7/speed", "error": "read /sys/class/net/ib7/speed: invalid argument"} +2025-11-17T00:59:51.656134479Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib7"} +2025-11-17T00:59:51.656186935Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:e3:00.0"} +2025-11-17T00:59:51.66626594Z INFO daemon/writer.go:166 setNodeStateStatus(): status {"sync-status": "InProgress", "last-sync-error": ""} +2025-11-17T00:59:51.700102408Z INFO daemon/daemon.go:264 nodeStateSyncHandler(): calling OnNodeStateChange for a new node state +2025-11-17T00:59:51.700143833Z INFO daemon/daemon.go:421 mellanox plugin OnNodeStateChange() +2025-11-17T00:59:51.700166088Z INFO kernel/kernel.go:304 RunCommand() {"command": "cat", "args": ["/host/sys/kernel/security/lockdown"]} +2025-11-17T00:59:51.70173717Z LEVEL(-2) kernel/kernel.go:304 RunCommand() {"output": "[none] integrity confidentiality\n", "error": null} +2025-11-17T00:59:51.701768702Z LEVEL(-2) mellanox/mellanox_plugin.go:86 IsKernelLockdownMode() {"output": "[none] integrity confidentiality\n", "error": null} +2025-11-17T00:59:51.701783549Z INFO mellanox/mellanox_plugin.go:102 mellanox-plugin getMlnxNicFwData() {"device": "0000:85:00.0"} +2025-11-17T00:59:51.701791757Z INFO mellanox/mellanox.go:208 MstConfigReadData() {"device": "0000:85:00.0"} +2025-11-17T00:59:51.701797927Z INFO mellanox/mellanox.go:81 RunCommand() {"command": "mstconfig", "args": ["-e", "-d", "0000:85:00.0", "q"]} +2025-11-17T01:00:16.615640949Z LEVEL(-2) mellanox/mellanox.go:81 RunCommand() {"output": "\nDevice #1:\n----------\n\nDevice type: ConnectX7 \nName: MCX75310AAS-NEA_Ax \nDescription: NVIDIA ConnectX-7 HHHL Adapter card; 400GbE / NDR IB (default mode); Single-port OSFP; PCIe 5.0 x16; Crypto Disabled; Secure Boot Enabled;\nDevice: 0000:85:00.0 \n\nConfigurations: Default Current Next Boot\n MODULE_SPLIT_M0 Array[0..15] Array[0..15] Array[0..15] \n MEMIC_BAR_SIZE 0 0 0 \n MEMIC_SIZE_LIMIT _256KB(1) _256KB(1) _256KB(1) \n MEMIC_ATOMIC MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2)\n HOST_CHAINING_MODE DISABLED(0) DISABLED(0) DISABLED(0) \n HOST_CHAINING_CACHE_DISABLE False(0) False(0) False(0) \n HOST_CHAINING_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n HOST_CHAINING_TOTAL_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n FLEX_PARSER_PROFILE_ENABLE 0 0 0 \n PROG_PARSE_GRAPH False(0) False(0) False(0) \n FLEX_IPV4_OVER_VXLAN_PORT 0 0 0 \n ROCE_NEXT_PROTOCOL 254 254 254 \n ESWITCH_HAIRPIN_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n ESWITCH_HAIRPIN_TOT_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n DPA_AUTHENTICATION True(1) True(1) True(1) \n PF_BAR2_SIZE 0 0 0 \n PF_NUM_OF_VF_VALID False(0) False(0) False(0) \n NON_PREFETCHABLE_PF_BAR False(0) False(0) False(0) \n VF_VPD_ENABLE False(0) False(0) False(0) \n PF_NUM_PF_MSIX_VALID False(0) False(0) False(0) \n PER_PF_NUM_SF False(0) False(0) False(0) \n STRICT_VF_MSIX_NUM False(0) False(0) False(0) \n VF_NODNIC_ENABLE False(0) False(0) False(0) \n NUM_PF_MSIX_VALID True(1) True(1) True(1) \n* NUM_OF_VFS 16 16 1 \n NUM_OF_PF 1 1 1 \n PF_BAR2_ENABLE False(0) False(0) False(0) \n SRIOV_EN True(1) True(1) True(1) \n PF_LOG_BAR_SIZE 5 5 5 \n VF_LOG_BAR_SIZE 1 1 1 \n NUM_PF_MSIX 63 63 63 \n NUM_VF_MSIX 11 11 11 \n INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0) AUTOMATIC(0) AUTOMATIC(0) \n PCIE_CREDIT_TOKEN_TIMEOUT 0 0 0 \n RT_PPS_ENABLED_ON_POWERUP False(0) False(0) False(0) \n LAG_RESOURCE_ALLOCATION DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ACCURATE_TX_SCHEDULER False(0) False(0) False(0) \n PARTIAL_RESET_EN False(0) False(0) False(0) \n RESET_WITH_HOST_ON_ERRORS False(0) False(0) False(0) \n PCI_SWITCH_EMULATION_NUM_PORT 16 16 16 \n PCI_SWITCH_EMULATION_ENABLE False(0) False(0) False(0) \n PCI_DOWNSTREAM_PORT_OWNER Array[0..15] Array[0..15] Array[0..15] \n CQE_COMPRESSION BALANCED(0) BALANCED(0) BALANCED(0) \n IP_OVER_VXLAN_EN False(0) False(0) False(0) \n MKEY_BY_NAME False(0) False(0) False(0) \n PRIO_TAG_REQUIRED_EN False(0) False(0) False(0) \n UCTX_EN True(1) True(1) True(1) \n REAL_TIME_CLOCK_ENABLE False(0) False(0) False(0) \n RDMA_SELECTIVE_REPEAT_EN False(0) False(0) False(0) \n PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)\n TUNNEL_ECN_COPY_DISABLE False(0) False(0) False(0) \n LRO_LOG_TIMEOUT0 6 6 6 \n LRO_LOG_TIMEOUT1 7 7 7 \n LRO_LOG_TIMEOUT2 8 8 8 \n LRO_LOG_TIMEOUT3 13 13 13 \n LOG_TX_PSN_WINDOW 7 7 7 \n VF_MIGRATION_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n LOG_MAX_OUTSTANDING_WQE 7 7 7 \n ROCE_ADAPTIVE_ROUTING_EN False(0) False(0) False(0) \n TUNNEL_IP_PROTO_ENTROPY_DISABLE False(0) False(0) False(0) \n PCC_HANDLE_CORE_UTIL DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PCC_INT_NP_RTT_DSCP 26 26 26 \n PCC_INT_NP_RTT_DSCP_EN False(0) False(0) False(0) \n PCC_INT_NP_RTT_DATA_MODE RTT_V0(64) RTT_V0(64) RTT_V0(64) \n PCC_INT_EN False(0) False(0) False(0) \n PCC_INT_SYSTEM_RTT 0 0 0 \n STEERING_CACHE_REFRESH 0 0 0 \n ICM_CACHE_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n HAIRPIN_DATA_BUFFER_LOCK False(0) False(0) False(0) \n TX_SCHEDULER_BURST 0 0 0 \n ZERO_TOUCH_TUNING_ENABLE False(0) False(0) False(0) \n LOG_MAX_QUEUE 17 17 17 \n LARGE_MTU_TWEAK_64 False(0) False(0) False(0) \n AES_XTS_TWEAK_INC_64 False(0) False(0) False(0) \n* CRYPTO_POLICY UNRESTRICTED(1) DEVICE_DEFAULT(0) UNRESTRICTED(1) \n RDE_DISABLE False(0) False(0) False(0) \n PLDM_FW_UPDATE_DISABLE False(0) False(0) False(0) \n RBT_DISABLE False(0) False(0) False(0) \n PCIE_SMBUS_DISABLE False(0) False(0) False(0) \n PCIE_IN_BAND_VDM_DISABLE False(0) False(0) False(0) \n ADVANCED_TESTABILITY False(0) False(0) False(0) \n LOG_DCR_HASH_TABLE_SIZE 11 11 11 \n MAX_PACKET_LIFETIME 0 0 0 \n DCR_LIFO_SIZE 16384 16384 16384 \n LINK_TYPE_P1 IB(1) IB(1) IB(1) \n ROCE_CC_PRIO_MASK_P1 255 255 255 \n ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ROCE_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n IB_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1) True(1) True(1) \n CLAMP_TGT_RATE_P1 False(0) False(0) False(0) \n RPG_TIME_RESET_P1 300 300 300 \n RPG_BYTE_RESET_P1 32767 32767 32767 \n RPG_THRESHOLD_P1 1 1 1 \n RPG_MAX_RATE_P1 0 0 0 \n RPG_AI_RATE_P1 5 5 5 \n RPG_HAI_RATE_P1 50 50 50 \n RPG_GD_P1 11 11 11 \n RPG_MIN_DEC_FAC_P1 50 50 50 \n RPG_MIN_RATE_P1 1 1 1 \n RATE_TO_SET_ON_FIRST_CNP_P1 0 0 0 \n DCE_TCP_G_P1 1019 1019 1019 \n DCE_TCP_RTT_P1 1 1 1 \n RATE_REDUCE_MONITOR_PERIOD_P1 4 4 4 \n INITIAL_ALPHA_VALUE_P1 1023 1023 1023 \n MIN_TIME_BETWEEN_CNPS_P1 4 4 4 \n CNP_802P_PRIO_P1 6 6 6 \n CNP_DSCP_P1 48 48 48 \n LLDP_NB_DCBX_P1 False(0) False(0) False(0) \n LLDP_NB_RX_MODE_P1 OFF(0) OFF(0) OFF(0) \n LLDP_NB_TX_MODE_P1 OFF(0) OFF(0) OFF(0) \n ROCE_RTT_RESP_DSCP_P1 0 0 0 \n ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n DCBX_IEEE_P1 True(1) True(1) True(1) \n DCBX_CEE_P1 True(1) True(1) True(1) \n DCBX_WILLING_P1 True(1) True(1) True(1) \n KEEP_ETH_LINK_UP_P1 True(1) True(1) True(1) \n KEEP_IB_LINK_UP_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_BOOT_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_STANDBY_P1 False(0) False(0) False(0) \n DO_NOT_CLEAR_PORT_STATS_P1 False(0) False(0) False(0) \n AUTO_POWER_SAVE_LINK_DOWN_P1 False(0) False(0) False(0) \n NUM_OF_VL_P1 _4_VLs(3) _4_VLs(3) _4_VLs(3) \n NUM_OF_TC_P1 _8_TCs(0) _8_TCs(0) _8_TCs(0) \n NUM_OF_PFC_P1 8 8 8 \n VL15_BUFFER_SIZE_P1 0 0 0 \n QOS_TRUST_STATE_P1 TRUST_PCP(1) TRUST_PCP(1) TRUST_PCP(1) \n ETS_SCHED_MODE_P1 device_default(0) device_default(0) device_default(0) \n DUP_MAC_ACTION_P1 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) \n MPFS_MC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n MPFS_UC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n SRIOV_IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n PHY_AUTO_NEG_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PHY_RATE_MASK_OVERRIDE_P1 False(0) False(0) False(0) \n PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PF_TOTAL_SF 0 0 0 \n PF_SD_GROUP 0 0 0 \n PF_SF_BAR_SIZE 0 0 0 \n PF_NUM_PF_MSIX 63 63 63 \n SILENT_MODE False(0) False(0) False(0) \n MKEY_BY_NAME_RANGE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n* ROCE_CONTROL ROCE_ENABLE(2) DEVICE_DEFAULT(0) ROCE_ENABLE(2) \n PCI_WR_ORDERING per_mkey(0) per_mkey(0) per_mkey(0) \n MULTI_PORT_VHCA_EN False(0) False(0) False(0) \n PORT_OWNER True(1) True(1) True(1) \n ALLOW_RD_COUNTERS True(1) True(1) True(1) \n RENEG_ON_CHANGE True(1) True(1) True(1) \n TRACER_ENABLE True(1) True(1) True(1) \n IP_VER IPv4(0) IPv4(0) IPv4(0) \n BOOT_UNDI_NETWORK_WAIT 0 0 0 \n UEFI_HII_EN True(1) True(1) True(1) \n BOOT_DBG_LOG False(0) False(0) False(0) \n UEFI_LOGS DISABLED(0) DISABLED(0) DISABLED(0) \n BOOT_VLAN 1 1 1 \n LEGACY_BOOT_PROTOCOL PXE(1) PXE(1) PXE(1) \n BOOT_INTERRUPT_DIS False(0) False(0) False(0) \n BOOT_LACP_DIS True(1) True(1) True(1) \n BOOT_VLAN_EN False(0) False(0) False(0) \n BOOT_PKEY 0 0 0 \n P2P_ORDERING_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ATS_ENABLED False(0) False(0) False(0) \n DYNAMIC_VF_MSIX_TABLE False(0) False(0) False(0) \n EXP_ROM_UEFI_ARM_ENABLE True(1) True(1) True(1) \n EXP_ROM_UEFI_x86_ENABLE True(1) True(1) True(1) \n EXP_ROM_PXE_ENABLE True(1) True(1) True(1) \n ADVANCED_PCI_SETTINGS False(0) False(0) False(0) \n SAFE_MODE_THRESHOLD 10 10 10 \n SAFE_MODE_ENABLE True(1) True(1) True(1) \nThe '*' shows parameters with next value different from default/current value.\n", "error": null} +2025-11-17T01:00:16.615807877Z INFO mellanox/mellanox.go:213 ParseMstconfigOutput() {"attributes": ["NUM_OF_VFS", "SRIOV_EN", "LINK_TYPE_P1", "LINK_TYPE_P2"]} +2025-11-17T01:00:16.615862667Z INFO mellanox/mellanox.go:214 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"16","SRIOV_EN":"True(1)"}} +2025-11-17T01:00:16.615877772Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:00:16.615883646Z INFO mellanox/mellanox.go:219 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"1","SRIOV_EN":"True(1)"}} +2025-11-17T01:00:16.615889419Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:00:16.615893194Z INFO mellanox/mellanox_plugin.go:107 mellanox-plugin IsDualPort() {"address": "0000:85:00.0"} +2025-11-17T01:00:16.615901924Z LEVEL(-2) mellanox/mellanox_plugin.go:112 Changing TotalVfs, needs reboot {"current": 16, "requested": 1} +2025-11-17T01:00:16.615909974Z INFO mellanox/mellanox.go:343 mellanox-plugin isLinkTypeRequireChange() {"device": "0000:85:00.0"} +2025-11-17T01:00:16.615918724Z INFO mellanox/mellanox_plugin.go:102 mellanox-plugin getMlnxNicFwData() {"device": "0000:8a:00.0"} +2025-11-17T01:00:16.615923647Z INFO mellanox/mellanox.go:208 MstConfigReadData() {"device": "0000:8a:00.0"} +2025-11-17T01:00:16.615928652Z INFO mellanox/mellanox.go:81 RunCommand() {"command": "mstconfig", "args": ["-e", "-d", "0000:8a:00.0", "q"]} +2025-11-17T01:00:21.690785152Z LEVEL(-2) runtime/asm_amd64.s:1695 Run(): period refresh +2025-11-17T01:00:21.69082225Z LEVEL(-2) daemon/writer.go:109 pollNicStatus() +2025-11-17T01:00:21.69082916Z LEVEL(-2) daemon/writer.go:130 DiscoverSriovDevices +2025-11-17T01:00:21.789480064Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib0"} +2025-11-17T01:00:21.789690068Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib0"} +2025-11-17T01:00:21.789710025Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib0/speed", "error": "read /sys/class/net/ib0/speed: invalid argument"} +2025-11-17T01:00:21.789718479Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib0"} +2025-11-17T01:00:21.78978272Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:19:00.0"} +2025-11-17T01:00:21.789891362Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib1"} +2025-11-17T01:00:21.789933121Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib1"} +2025-11-17T01:00:21.789947271Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib1/speed", "error": "read /sys/class/net/ib1/speed: invalid argument"} +2025-11-17T01:00:21.789951592Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib1"} +2025-11-17T01:00:21.79000417Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:29:00.0"} +2025-11-17T01:00:21.790095769Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib2"} +2025-11-17T01:00:21.790140484Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib2"} +2025-11-17T01:00:21.790451259Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib2"} +2025-11-17T01:00:21.790594943Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:3b:00.0"} +2025-11-17T01:00:21.790765833Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth2"} +2025-11-17T01:00:21.791408754Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth2"} +2025-11-17T01:00:21.791472547Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.0"} +2025-11-17T01:00:21.791630074Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth3"} +2025-11-17T01:00:21.792284606Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth3"} +2025-11-17T01:00:21.792351694Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.1"} +2025-11-17T01:00:21.792440208Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth0"} +2025-11-17T01:00:21.792488762Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth0"} +2025-11-17T01:00:21.792503123Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth0/speed", "error": "read /sys/class/net/eth0/speed: invalid argument"} +2025-11-17T01:00:21.792509393Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth0"} +2025-11-17T01:00:21.792567857Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.0"} +2025-11-17T01:00:21.792638161Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth1"} +2025-11-17T01:00:21.792684174Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth1"} +2025-11-17T01:00:21.792699424Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth1/speed", "error": "read /sys/class/net/eth1/speed: invalid argument"} +2025-11-17T01:00:21.792703512Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth1"} +2025-11-17T01:00:21.792761207Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.1"} +2025-11-17T01:00:21.792841345Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib3"} +2025-11-17T01:00:21.792900909Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib3"} +2025-11-17T01:00:21.793135959Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib3"} +2025-11-17T01:00:21.793203815Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:5c:00.0"} +2025-11-17T01:00:21.793297952Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib4"} +2025-11-17T01:00:21.793353047Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib4"} +2025-11-17T01:00:21.793593041Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib4"} +2025-11-17T01:00:21.793652785Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:85:00.0"} +2025-11-17T01:00:21.793735298Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib5"} +2025-11-17T01:00:21.793776859Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib5"} +2025-11-17T01:00:21.793868015Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib5"} +2025-11-17T01:00:21.793929225Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:8a:00.0"} +2025-11-17T01:00:21.794013481Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib6"} +2025-11-17T01:00:21.794055379Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib6"} +2025-11-17T01:00:21.794070615Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib6/speed", "error": "read /sys/class/net/ib6/speed: invalid argument"} +2025-11-17T01:00:21.794076417Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib6"} +2025-11-17T01:00:21.794137253Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:92:00.0"} +2025-11-17T01:00:21.794222303Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib7"} +2025-11-17T01:00:21.794265077Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib7"} +2025-11-17T01:00:21.794279847Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib7/speed", "error": "read /sys/class/net/ib7/speed: invalid argument"} +2025-11-17T01:00:21.794284601Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib7"} +2025-11-17T01:00:21.794340042Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:e3:00.0"} +2025-11-17T01:00:21.80554676Z INFO daemon/writer.go:166 setNodeStateStatus(): status {"sync-status": "InProgress", "last-sync-error": ""} +2025-11-17T01:00:39.907287069Z LEVEL(-2) mellanox/mellanox.go:81 RunCommand() {"output": "\nDevice #1:\n----------\n\nDevice type: ConnectX7 \nName: MCX75310AAS-NEA_Ax \nDescription: NVIDIA ConnectX-7 HHHL Adapter card; 400GbE / NDR IB (default mode); Single-port OSFP; PCIe 5.0 x16; Crypto Disabled; Secure Boot Enabled;\nDevice: 0000:8a:00.0 \n\nConfigurations: Default Current Next Boot\n MODULE_SPLIT_M0 Array[0..15] Array[0..15] Array[0..15] \n MEMIC_BAR_SIZE 0 0 0 \n MEMIC_SIZE_LIMIT _256KB(1) _256KB(1) _256KB(1) \n MEMIC_ATOMIC MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2)\n HOST_CHAINING_MODE DISABLED(0) DISABLED(0) DISABLED(0) \n HOST_CHAINING_CACHE_DISABLE False(0) False(0) False(0) \n HOST_CHAINING_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n HOST_CHAINING_TOTAL_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n FLEX_PARSER_PROFILE_ENABLE 0 0 0 \n PROG_PARSE_GRAPH False(0) False(0) False(0) \n FLEX_IPV4_OVER_VXLAN_PORT 0 0 0 \n ROCE_NEXT_PROTOCOL 254 254 254 \n ESWITCH_HAIRPIN_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n ESWITCH_HAIRPIN_TOT_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n DPA_AUTHENTICATION True(1) True(1) True(1) \n PF_BAR2_SIZE 0 0 0 \n PF_NUM_OF_VF_VALID False(0) False(0) False(0) \n NON_PREFETCHABLE_PF_BAR False(0) False(0) False(0) \n VF_VPD_ENABLE False(0) False(0) False(0) \n PF_NUM_PF_MSIX_VALID False(0) False(0) False(0) \n PER_PF_NUM_SF False(0) False(0) False(0) \n STRICT_VF_MSIX_NUM False(0) False(0) False(0) \n VF_NODNIC_ENABLE False(0) False(0) False(0) \n NUM_PF_MSIX_VALID True(1) True(1) True(1) \n* NUM_OF_VFS 16 16 1 \n NUM_OF_PF 1 1 1 \n PF_BAR2_ENABLE False(0) False(0) False(0) \n SRIOV_EN True(1) True(1) True(1) \n PF_LOG_BAR_SIZE 5 5 5 \n VF_LOG_BAR_SIZE 1 1 1 \n NUM_PF_MSIX 63 63 63 \n NUM_VF_MSIX 11 11 11 \n INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0) AUTOMATIC(0) AUTOMATIC(0) \n PCIE_CREDIT_TOKEN_TIMEOUT 0 0 0 \n RT_PPS_ENABLED_ON_POWERUP False(0) False(0) False(0) \n LAG_RESOURCE_ALLOCATION DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ACCURATE_TX_SCHEDULER False(0) False(0) False(0) \n PARTIAL_RESET_EN False(0) False(0) False(0) \n RESET_WITH_HOST_ON_ERRORS False(0) False(0) False(0) \n PCI_SWITCH_EMULATION_NUM_PORT 16 16 16 \n PCI_SWITCH_EMULATION_ENABLE False(0) False(0) False(0) \n PCI_DOWNSTREAM_PORT_OWNER Array[0..15] Array[0..15] Array[0..15] \n CQE_COMPRESSION BALANCED(0) BALANCED(0) BALANCED(0) \n IP_OVER_VXLAN_EN False(0) False(0) False(0) \n MKEY_BY_NAME False(0) False(0) False(0) \n PRIO_TAG_REQUIRED_EN False(0) False(0) False(0) \n UCTX_EN True(1) True(1) True(1) \n REAL_TIME_CLOCK_ENABLE False(0) False(0) False(0) \n RDMA_SELECTIVE_REPEAT_EN False(0) False(0) False(0) \n PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)\n TUNNEL_ECN_COPY_DISABLE False(0) False(0) False(0) \n LRO_LOG_TIMEOUT0 6 6 6 \n LRO_LOG_TIMEOUT1 7 7 7 \n LRO_LOG_TIMEOUT2 8 8 8 \n LRO_LOG_TIMEOUT3 13 13 13 \n LOG_TX_PSN_WINDOW 7 7 7 \n VF_MIGRATION_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n LOG_MAX_OUTSTANDING_WQE 7 7 7 \n ROCE_ADAPTIVE_ROUTING_EN False(0) False(0) False(0) \n TUNNEL_IP_PROTO_ENTROPY_DISABLE False(0) False(0) False(0) \n PCC_HANDLE_CORE_UTIL DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PCC_INT_NP_RTT_DSCP 26 26 26 \n PCC_INT_NP_RTT_DSCP_EN False(0) False(0) False(0) \n PCC_INT_NP_RTT_DATA_MODE RTT_V0(64) RTT_V0(64) RTT_V0(64) \n PCC_INT_EN False(0) False(0) False(0) \n PCC_INT_SYSTEM_RTT 0 0 0 \n STEERING_CACHE_REFRESH 0 0 0 \n ICM_CACHE_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n HAIRPIN_DATA_BUFFER_LOCK False(0) False(0) False(0) \n TX_SCHEDULER_BURST 0 0 0 \n ZERO_TOUCH_TUNING_ENABLE False(0) False(0) False(0) \n LOG_MAX_QUEUE 17 17 17 \n LARGE_MTU_TWEAK_64 False(0) False(0) False(0) \n AES_XTS_TWEAK_INC_64 False(0) False(0) False(0) \n* CRYPTO_POLICY UNRESTRICTED(1) DEVICE_DEFAULT(0) UNRESTRICTED(1) \n RDE_DISABLE False(0) False(0) False(0) \n PLDM_FW_UPDATE_DISABLE False(0) False(0) False(0) \n RBT_DISABLE False(0) False(0) False(0) \n PCIE_SMBUS_DISABLE False(0) False(0) False(0) \n PCIE_IN_BAND_VDM_DISABLE False(0) False(0) False(0) \n ADVANCED_TESTABILITY False(0) False(0) False(0) \n LOG_DCR_HASH_TABLE_SIZE 11 11 11 \n MAX_PACKET_LIFETIME 0 0 0 \n DCR_LIFO_SIZE 16384 16384 16384 \n LINK_TYPE_P1 IB(1) IB(1) IB(1) \n ROCE_CC_PRIO_MASK_P1 255 255 255 \n ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ROCE_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n IB_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1) True(1) True(1) \n CLAMP_TGT_RATE_P1 False(0) False(0) False(0) \n RPG_TIME_RESET_P1 300 300 300 \n RPG_BYTE_RESET_P1 32767 32767 32767 \n RPG_THRESHOLD_P1 1 1 1 \n RPG_MAX_RATE_P1 0 0 0 \n RPG_AI_RATE_P1 5 5 5 \n RPG_HAI_RATE_P1 50 50 50 \n RPG_GD_P1 11 11 11 \n RPG_MIN_DEC_FAC_P1 50 50 50 \n RPG_MIN_RATE_P1 1 1 1 \n RATE_TO_SET_ON_FIRST_CNP_P1 0 0 0 \n DCE_TCP_G_P1 1019 1019 1019 \n DCE_TCP_RTT_P1 1 1 1 \n RATE_REDUCE_MONITOR_PERIOD_P1 4 4 4 \n INITIAL_ALPHA_VALUE_P1 1023 1023 1023 \n MIN_TIME_BETWEEN_CNPS_P1 4 4 4 \n CNP_802P_PRIO_P1 6 6 6 \n CNP_DSCP_P1 48 48 48 \n LLDP_NB_DCBX_P1 False(0) False(0) False(0) \n LLDP_NB_RX_MODE_P1 OFF(0) OFF(0) OFF(0) \n LLDP_NB_TX_MODE_P1 OFF(0) OFF(0) OFF(0) \n ROCE_RTT_RESP_DSCP_P1 0 0 0 \n ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n DCBX_IEEE_P1 True(1) True(1) True(1) \n DCBX_CEE_P1 True(1) True(1) True(1) \n DCBX_WILLING_P1 True(1) True(1) True(1) \n KEEP_ETH_LINK_UP_P1 True(1) True(1) True(1) \n KEEP_IB_LINK_UP_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_BOOT_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_STANDBY_P1 False(0) False(0) False(0) \n DO_NOT_CLEAR_PORT_STATS_P1 False(0) False(0) False(0) \n AUTO_POWER_SAVE_LINK_DOWN_P1 False(0) False(0) False(0) \n NUM_OF_VL_P1 _4_VLs(3) _4_VLs(3) _4_VLs(3) \n NUM_OF_TC_P1 _8_TCs(0) _8_TCs(0) _8_TCs(0) \n NUM_OF_PFC_P1 8 8 8 \n VL15_BUFFER_SIZE_P1 0 0 0 \n QOS_TRUST_STATE_P1 TRUST_PCP(1) TRUST_PCP(1) TRUST_PCP(1) \n ETS_SCHED_MODE_P1 device_default(0) device_default(0) device_default(0) \n DUP_MAC_ACTION_P1 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) \n MPFS_MC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n MPFS_UC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n SRIOV_IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n PHY_AUTO_NEG_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PHY_RATE_MASK_OVERRIDE_P1 False(0) False(0) False(0) \n PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PF_TOTAL_SF 0 0 0 \n PF_SD_GROUP 0 0 0 \n PF_SF_BAR_SIZE 0 0 0 \n PF_NUM_PF_MSIX 63 63 63 \n SILENT_MODE False(0) False(0) False(0) \n MKEY_BY_NAME_RANGE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n* ROCE_CONTROL ROCE_ENABLE(2) DEVICE_DEFAULT(0) ROCE_ENABLE(2) \n PCI_WR_ORDERING per_mkey(0) per_mkey(0) per_mkey(0) \n MULTI_PORT_VHCA_EN False(0) False(0) False(0) \n PORT_OWNER True(1) True(1) True(1) \n ALLOW_RD_COUNTERS True(1) True(1) True(1) \n RENEG_ON_CHANGE True(1) True(1) True(1) \n TRACER_ENABLE True(1) True(1) True(1) \n IP_VER IPv4(0) IPv4(0) IPv4(0) \n BOOT_UNDI_NETWORK_WAIT 0 0 0 \n UEFI_HII_EN True(1) True(1) True(1) \n BOOT_DBG_LOG False(0) False(0) False(0) \n UEFI_LOGS DISABLED(0) DISABLED(0) DISABLED(0) \n BOOT_VLAN 1 1 1 \n LEGACY_BOOT_PROTOCOL PXE(1) PXE(1) PXE(1) \n BOOT_INTERRUPT_DIS False(0) False(0) False(0) \n BOOT_LACP_DIS True(1) True(1) True(1) \n BOOT_VLAN_EN False(0) False(0) False(0) \n BOOT_PKEY 0 0 0 \n P2P_ORDERING_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ATS_ENABLED False(0) False(0) False(0) \n DYNAMIC_VF_MSIX_TABLE False(0) False(0) False(0) \n EXP_ROM_UEFI_ARM_ENABLE True(1) True(1) True(1) \n EXP_ROM_UEFI_x86_ENABLE True(1) True(1) True(1) \n EXP_ROM_PXE_ENABLE True(1) True(1) True(1) \n ADVANCED_PCI_SETTINGS False(0) False(0) False(0) \n SAFE_MODE_THRESHOLD 10 10 10 \n SAFE_MODE_ENABLE True(1) True(1) True(1) \nThe '*' shows parameters with next value different from default/current value.\n", "error": null} +2025-11-17T01:00:39.907437047Z INFO mellanox/mellanox.go:213 ParseMstconfigOutput() {"attributes": ["NUM_OF_VFS", "SRIOV_EN", "LINK_TYPE_P1", "LINK_TYPE_P2"]} +2025-11-17T01:00:39.907487454Z INFO mellanox/mellanox.go:214 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"16","SRIOV_EN":"True(1)"}} +2025-11-17T01:00:39.907506598Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:00:39.907510883Z INFO mellanox/mellanox.go:219 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"1","SRIOV_EN":"True(1)"}} +2025-11-17T01:00:39.90751561Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:00:39.907520961Z INFO mellanox/mellanox_plugin.go:107 mellanox-plugin IsDualPort() {"address": "0000:8a:00.0"} +2025-11-17T01:00:39.907527198Z LEVEL(-2) mellanox/mellanox_plugin.go:112 Changing TotalVfs, needs reboot {"current": 16, "requested": 1} +2025-11-17T01:00:39.907534304Z INFO mellanox/mellanox.go:343 mellanox-plugin isLinkTypeRequireChange() {"device": "0000:8a:00.0"} +2025-11-17T01:00:39.90754034Z INFO mellanox/mellanox_plugin.go:102 mellanox-plugin getMlnxNicFwData() {"device": "0000:92:00.0"} +2025-11-17T01:00:39.907544912Z INFO mellanox/mellanox.go:208 MstConfigReadData() {"device": "0000:92:00.0"} +2025-11-17T01:00:39.907554847Z INFO mellanox/mellanox.go:81 RunCommand() {"command": "mstconfig", "args": ["-e", "-d", "0000:92:00.0", "q"]} +2025-11-17T01:00:51.820519109Z LEVEL(-2) runtime/asm_amd64.s:1695 Run(): period refresh +2025-11-17T01:00:51.820578916Z LEVEL(-2) daemon/writer.go:109 pollNicStatus() +2025-11-17T01:00:51.82058592Z LEVEL(-2) daemon/writer.go:130 DiscoverSriovDevices +2025-11-17T01:00:51.926311479Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib0"} +2025-11-17T01:00:51.926509244Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib0"} +2025-11-17T01:00:51.926530085Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib0/speed", "error": "read /sys/class/net/ib0/speed: invalid argument"} +2025-11-17T01:00:51.926539371Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib0"} +2025-11-17T01:00:51.926606148Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:19:00.0"} +2025-11-17T01:00:51.92671356Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib1"} +2025-11-17T01:00:51.926804146Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib1"} +2025-11-17T01:00:51.926819025Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib1/speed", "error": "read /sys/class/net/ib1/speed: invalid argument"} +2025-11-17T01:00:51.926824667Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib1"} +2025-11-17T01:00:51.926881886Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:29:00.0"} +2025-11-17T01:00:51.926975615Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib2"} +2025-11-17T01:00:51.92706782Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib2"} +2025-11-17T01:00:51.92716798Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib2"} +2025-11-17T01:00:51.927226876Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:3b:00.0"} +2025-11-17T01:00:51.927439535Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth2"} +2025-11-17T01:00:51.92801106Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth2"} +2025-11-17T01:00:51.92807487Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.0"} +2025-11-17T01:00:51.928286193Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth3"} +2025-11-17T01:00:51.928879252Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth3"} +2025-11-17T01:00:51.929117362Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.1"} +2025-11-17T01:00:51.929248512Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth0"} +2025-11-17T01:00:51.929332537Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth0"} +2025-11-17T01:00:51.929350006Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth0/speed", "error": "read /sys/class/net/eth0/speed: invalid argument"} +2025-11-17T01:00:51.92935947Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth0"} +2025-11-17T01:00:51.929429491Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.0"} +2025-11-17T01:00:51.929504489Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth1"} +2025-11-17T01:00:51.929577957Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth1"} +2025-11-17T01:00:51.929592632Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth1/speed", "error": "read /sys/class/net/eth1/speed: invalid argument"} +2025-11-17T01:00:51.929597665Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth1"} +2025-11-17T01:00:51.929660264Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.1"} +2025-11-17T01:00:51.929736819Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib3"} +2025-11-17T01:00:51.929836549Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib3"} +2025-11-17T01:00:51.929938912Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib3"} +2025-11-17T01:00:51.930006228Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:5c:00.0"} +2025-11-17T01:00:51.93010338Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib4"} +2025-11-17T01:00:51.930187225Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib4"} +2025-11-17T01:00:51.93027948Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib4"} +2025-11-17T01:00:51.930343439Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:85:00.0"} +2025-11-17T01:00:51.930428192Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib5"} +2025-11-17T01:00:51.930495226Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib5"} +2025-11-17T01:00:51.930589779Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib5"} +2025-11-17T01:00:51.930650965Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:8a:00.0"} +2025-11-17T01:00:51.930735354Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib6"} +2025-11-17T01:00:51.930809886Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib6"} +2025-11-17T01:00:51.930825175Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib6/speed", "error": "read /sys/class/net/ib6/speed: invalid argument"} +2025-11-17T01:00:51.930831913Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib6"} +2025-11-17T01:00:51.930890876Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:92:00.0"} +2025-11-17T01:00:51.930974592Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib7"} +2025-11-17T01:00:51.931047431Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib7"} +2025-11-17T01:00:51.931062064Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib7/speed", "error": "read /sys/class/net/ib7/speed: invalid argument"} +2025-11-17T01:00:51.931067641Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib7"} +2025-11-17T01:00:51.931126459Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:e3:00.0"} +2025-11-17T01:00:51.941884132Z INFO daemon/writer.go:166 setNodeStateStatus(): status {"sync-status": "InProgress", "last-sync-error": ""} +2025-11-17T01:01:03.498419319Z LEVEL(-2) mellanox/mellanox.go:81 RunCommand() {"output": "\nDevice #1:\n----------\n\nDevice type: ConnectX7 \nName: MCX75310AAS-NEA_Ax \nDescription: NVIDIA ConnectX-7 HHHL Adapter card; 400GbE / NDR IB (default mode); Single-port OSFP; PCIe 5.0 x16; Crypto Disabled; Secure Boot Enabled;\nDevice: 0000:92:00.0 \n\nConfigurations: Default Current Next Boot\n MODULE_SPLIT_M0 Array[0..15] Array[0..15] Array[0..15] \n MEMIC_BAR_SIZE 0 0 0 \n MEMIC_SIZE_LIMIT _256KB(1) _256KB(1) _256KB(1) \n MEMIC_ATOMIC MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2)\n HOST_CHAINING_MODE DISABLED(0) DISABLED(0) DISABLED(0) \n HOST_CHAINING_CACHE_DISABLE False(0) False(0) False(0) \n HOST_CHAINING_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n HOST_CHAINING_TOTAL_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n FLEX_PARSER_PROFILE_ENABLE 0 0 0 \n PROG_PARSE_GRAPH False(0) False(0) False(0) \n FLEX_IPV4_OVER_VXLAN_PORT 0 0 0 \n ROCE_NEXT_PROTOCOL 254 254 254 \n ESWITCH_HAIRPIN_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n ESWITCH_HAIRPIN_TOT_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n DPA_AUTHENTICATION True(1) True(1) True(1) \n PF_BAR2_SIZE 0 0 0 \n PF_NUM_OF_VF_VALID False(0) False(0) False(0) \n NON_PREFETCHABLE_PF_BAR False(0) False(0) False(0) \n VF_VPD_ENABLE False(0) False(0) False(0) \n PF_NUM_PF_MSIX_VALID False(0) False(0) False(0) \n PER_PF_NUM_SF False(0) False(0) False(0) \n STRICT_VF_MSIX_NUM False(0) False(0) False(0) \n VF_NODNIC_ENABLE False(0) False(0) False(0) \n NUM_PF_MSIX_VALID True(1) True(1) True(1) \n* NUM_OF_VFS 16 16 1 \n NUM_OF_PF 1 1 1 \n PF_BAR2_ENABLE False(0) False(0) False(0) \n SRIOV_EN True(1) True(1) True(1) \n PF_LOG_BAR_SIZE 5 5 5 \n VF_LOG_BAR_SIZE 1 1 1 \n NUM_PF_MSIX 63 63 63 \n NUM_VF_MSIX 11 11 11 \n INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0) AUTOMATIC(0) AUTOMATIC(0) \n PCIE_CREDIT_TOKEN_TIMEOUT 0 0 0 \n RT_PPS_ENABLED_ON_POWERUP False(0) False(0) False(0) \n LAG_RESOURCE_ALLOCATION DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ACCURATE_TX_SCHEDULER False(0) False(0) False(0) \n PARTIAL_RESET_EN False(0) False(0) False(0) \n RESET_WITH_HOST_ON_ERRORS False(0) False(0) False(0) \n PCI_SWITCH_EMULATION_NUM_PORT 16 16 16 \n PCI_SWITCH_EMULATION_ENABLE False(0) False(0) False(0) \n PCI_DOWNSTREAM_PORT_OWNER Array[0..15] Array[0..15] Array[0..15] \n CQE_COMPRESSION BALANCED(0) BALANCED(0) BALANCED(0) \n IP_OVER_VXLAN_EN False(0) False(0) False(0) \n MKEY_BY_NAME False(0) False(0) False(0) \n PRIO_TAG_REQUIRED_EN False(0) False(0) False(0) \n UCTX_EN True(1) True(1) True(1) \n REAL_TIME_CLOCK_ENABLE False(0) False(0) False(0) \n RDMA_SELECTIVE_REPEAT_EN False(0) False(0) False(0) \n PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)\n TUNNEL_ECN_COPY_DISABLE False(0) False(0) False(0) \n LRO_LOG_TIMEOUT0 6 6 6 \n LRO_LOG_TIMEOUT1 7 7 7 \n LRO_LOG_TIMEOUT2 8 8 8 \n LRO_LOG_TIMEOUT3 13 13 13 \n LOG_TX_PSN_WINDOW 7 7 7 \n VF_MIGRATION_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n LOG_MAX_OUTSTANDING_WQE 7 7 7 \n ROCE_ADAPTIVE_ROUTING_EN False(0) False(0) False(0) \n TUNNEL_IP_PROTO_ENTROPY_DISABLE False(0) False(0) False(0) \n PCC_HANDLE_CORE_UTIL DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PCC_INT_NP_RTT_DSCP 26 26 26 \n PCC_INT_NP_RTT_DSCP_EN False(0) False(0) False(0) \n PCC_INT_NP_RTT_DATA_MODE RTT_V0(64) RTT_V0(64) RTT_V0(64) \n PCC_INT_EN False(0) False(0) False(0) \n PCC_INT_SYSTEM_RTT 0 0 0 \n STEERING_CACHE_REFRESH 0 0 0 \n ICM_CACHE_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n HAIRPIN_DATA_BUFFER_LOCK False(0) False(0) False(0) \n TX_SCHEDULER_BURST 0 0 0 \n ZERO_TOUCH_TUNING_ENABLE False(0) False(0) False(0) \n LOG_MAX_QUEUE 17 17 17 \n LARGE_MTU_TWEAK_64 False(0) False(0) False(0) \n AES_XTS_TWEAK_INC_64 False(0) False(0) False(0) \n* CRYPTO_POLICY UNRESTRICTED(1) DEVICE_DEFAULT(0) UNRESTRICTED(1) \n RDE_DISABLE False(0) False(0) False(0) \n PLDM_FW_UPDATE_DISABLE False(0) False(0) False(0) \n RBT_DISABLE False(0) False(0) False(0) \n PCIE_SMBUS_DISABLE False(0) False(0) False(0) \n PCIE_IN_BAND_VDM_DISABLE False(0) False(0) False(0) \n ADVANCED_TESTABILITY False(0) False(0) False(0) \n LOG_DCR_HASH_TABLE_SIZE 11 11 11 \n MAX_PACKET_LIFETIME 0 0 0 \n DCR_LIFO_SIZE 16384 16384 16384 \n LINK_TYPE_P1 IB(1) IB(1) IB(1) \n ROCE_CC_PRIO_MASK_P1 255 255 255 \n ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ROCE_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n IB_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1) True(1) True(1) \n CLAMP_TGT_RATE_P1 False(0) False(0) False(0) \n RPG_TIME_RESET_P1 300 300 300 \n RPG_BYTE_RESET_P1 32767 32767 32767 \n RPG_THRESHOLD_P1 1 1 1 \n RPG_MAX_RATE_P1 0 0 0 \n RPG_AI_RATE_P1 5 5 5 \n RPG_HAI_RATE_P1 50 50 50 \n RPG_GD_P1 11 11 11 \n RPG_MIN_DEC_FAC_P1 50 50 50 \n RPG_MIN_RATE_P1 1 1 1 \n RATE_TO_SET_ON_FIRST_CNP_P1 0 0 0 \n DCE_TCP_G_P1 1019 1019 1019 \n DCE_TCP_RTT_P1 1 1 1 \n RATE_REDUCE_MONITOR_PERIOD_P1 4 4 4 \n INITIAL_ALPHA_VALUE_P1 1023 1023 1023 \n MIN_TIME_BETWEEN_CNPS_P1 4 4 4 \n CNP_802P_PRIO_P1 6 6 6 \n CNP_DSCP_P1 48 48 48 \n LLDP_NB_DCBX_P1 False(0) False(0) False(0) \n LLDP_NB_RX_MODE_P1 OFF(0) OFF(0) OFF(0) \n LLDP_NB_TX_MODE_P1 OFF(0) OFF(0) OFF(0) \n ROCE_RTT_RESP_DSCP_P1 0 0 0 \n ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n DCBX_IEEE_P1 True(1) True(1) True(1) \n DCBX_CEE_P1 True(1) True(1) True(1) \n DCBX_WILLING_P1 True(1) True(1) True(1) \n KEEP_ETH_LINK_UP_P1 True(1) True(1) True(1) \n KEEP_IB_LINK_UP_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_BOOT_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_STANDBY_P1 False(0) False(0) False(0) \n DO_NOT_CLEAR_PORT_STATS_P1 False(0) False(0) False(0) \n AUTO_POWER_SAVE_LINK_DOWN_P1 False(0) False(0) False(0) \n NUM_OF_VL_P1 _4_VLs(3) _4_VLs(3) _4_VLs(3) \n NUM_OF_TC_P1 _8_TCs(0) _8_TCs(0) _8_TCs(0) \n NUM_OF_PFC_P1 8 8 8 \n VL15_BUFFER_SIZE_P1 0 0 0 \n QOS_TRUST_STATE_P1 TRUST_PCP(1) TRUST_PCP(1) TRUST_PCP(1) \n ETS_SCHED_MODE_P1 device_default(0) device_default(0) device_default(0) \n DUP_MAC_ACTION_P1 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) \n MPFS_MC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n MPFS_UC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n SRIOV_IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n PHY_AUTO_NEG_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PHY_RATE_MASK_OVERRIDE_P1 False(0) False(0) False(0) \n PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PF_TOTAL_SF 0 0 0 \n PF_SD_GROUP 0 0 0 \n PF_SF_BAR_SIZE 0 0 0 \n PF_NUM_PF_MSIX 63 63 63 \n SILENT_MODE False(0) False(0) False(0) \n MKEY_BY_NAME_RANGE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n* ROCE_CONTROL ROCE_ENABLE(2) DEVICE_DEFAULT(0) ROCE_ENABLE(2) \n PCI_WR_ORDERING per_mkey(0) per_mkey(0) per_mkey(0) \n MULTI_PORT_VHCA_EN False(0) False(0) False(0) \n PORT_OWNER True(1) True(1) True(1) \n ALLOW_RD_COUNTERS True(1) True(1) True(1) \n RENEG_ON_CHANGE True(1) True(1) True(1) \n TRACER_ENABLE True(1) True(1) True(1) \n IP_VER IPv4(0) IPv4(0) IPv4(0) \n BOOT_UNDI_NETWORK_WAIT 0 0 0 \n UEFI_HII_EN True(1) True(1) True(1) \n BOOT_DBG_LOG False(0) False(0) False(0) \n UEFI_LOGS DISABLED(0) DISABLED(0) DISABLED(0) \n BOOT_VLAN 1 1 1 \n LEGACY_BOOT_PROTOCOL PXE(1) PXE(1) PXE(1) \n BOOT_INTERRUPT_DIS False(0) False(0) False(0) \n BOOT_LACP_DIS True(1) True(1) True(1) \n BOOT_VLAN_EN False(0) False(0) False(0) \n BOOT_PKEY 0 0 0 \n P2P_ORDERING_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ATS_ENABLED False(0) False(0) False(0) \n DYNAMIC_VF_MSIX_TABLE False(0) False(0) False(0) \n EXP_ROM_UEFI_ARM_ENABLE True(1) True(1) True(1) \n EXP_ROM_UEFI_x86_ENABLE True(1) True(1) True(1) \n EXP_ROM_PXE_ENABLE True(1) True(1) True(1) \n ADVANCED_PCI_SETTINGS False(0) False(0) False(0) \n SAFE_MODE_THRESHOLD 10 10 10 \n SAFE_MODE_ENABLE True(1) True(1) True(1) \nThe '*' shows parameters with next value different from default/current value.\n", "error": null} +2025-11-17T01:01:03.498537882Z INFO mellanox/mellanox.go:213 ParseMstconfigOutput() {"attributes": ["NUM_OF_VFS", "SRIOV_EN", "LINK_TYPE_P1", "LINK_TYPE_P2"]} +2025-11-17T01:01:03.498590848Z INFO mellanox/mellanox.go:214 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"16","SRIOV_EN":"True(1)"}} +2025-11-17T01:01:03.49861004Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:01:03.498613855Z INFO mellanox/mellanox.go:219 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"1","SRIOV_EN":"True(1)"}} +2025-11-17T01:01:03.498619122Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:01:03.498622714Z INFO mellanox/mellanox_plugin.go:107 mellanox-plugin IsDualPort() {"address": "0000:92:00.0"} +2025-11-17T01:01:03.498630457Z LEVEL(-2) mellanox/mellanox_plugin.go:112 Changing TotalVfs, needs reboot {"current": 16, "requested": 1} +2025-11-17T01:01:03.498637594Z INFO mellanox/mellanox.go:343 mellanox-plugin isLinkTypeRequireChange() {"device": "0000:92:00.0"} +2025-11-17T01:01:03.498644176Z INFO mellanox/mellanox_plugin.go:102 mellanox-plugin getMlnxNicFwData() {"device": "0000:e3:00.0"} +2025-11-17T01:01:03.498648601Z INFO mellanox/mellanox.go:208 MstConfigReadData() {"device": "0000:e3:00.0"} +2025-11-17T01:01:03.49865965Z INFO mellanox/mellanox.go:81 RunCommand() {"command": "mstconfig", "args": ["-e", "-d", "0000:e3:00.0", "q"]} +2025-11-17T01:01:21.956182426Z LEVEL(-2) runtime/asm_amd64.s:1695 Run(): period refresh +2025-11-17T01:01:21.9562125Z LEVEL(-2) daemon/writer.go:109 pollNicStatus() +2025-11-17T01:01:21.956219243Z LEVEL(-2) daemon/writer.go:130 DiscoverSriovDevices +2025-11-17T01:01:22.064793713Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib0"} +2025-11-17T01:01:22.065031112Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib0"} +2025-11-17T01:01:22.065053741Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib0/speed", "error": "read /sys/class/net/ib0/speed: invalid argument"} +2025-11-17T01:01:22.065064108Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib0"} +2025-11-17T01:01:22.065134801Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:19:00.0"} +2025-11-17T01:01:22.065243888Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib1"} +2025-11-17T01:01:22.065278995Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib1"} +2025-11-17T01:01:22.065296022Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib1/speed", "error": "read /sys/class/net/ib1/speed: invalid argument"} +2025-11-17T01:01:22.065301382Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib1"} +2025-11-17T01:01:22.065366772Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:29:00.0"} +2025-11-17T01:01:22.065475557Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib2"} +2025-11-17T01:01:22.06552379Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib2"} +2025-11-17T01:01:22.065905693Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib2"} +2025-11-17T01:01:22.066053136Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:3b:00.0"} +2025-11-17T01:01:22.066262514Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth2"} +2025-11-17T01:01:22.066943603Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth2"} +2025-11-17T01:01:22.067023029Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.0"} +2025-11-17T01:01:22.067209202Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth3"} +2025-11-17T01:01:22.067836113Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth3"} +2025-11-17T01:01:22.067905794Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.1"} +2025-11-17T01:01:22.06799878Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth0"} +2025-11-17T01:01:22.068046443Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth0"} +2025-11-17T01:01:22.068062179Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth0/speed", "error": "read /sys/class/net/eth0/speed: invalid argument"} +2025-11-17T01:01:22.068069082Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth0"} +2025-11-17T01:01:22.068143783Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.0"} +2025-11-17T01:01:22.068233734Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth1"} +2025-11-17T01:01:22.068277321Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth1"} +2025-11-17T01:01:22.068293345Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth1/speed", "error": "read /sys/class/net/eth1/speed: invalid argument"} +2025-11-17T01:01:22.068303782Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth1"} +2025-11-17T01:01:22.068362733Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.1"} +2025-11-17T01:01:22.068450878Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib3"} +2025-11-17T01:01:22.068505129Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib3"} +2025-11-17T01:01:22.068595012Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib3"} +2025-11-17T01:01:22.068656234Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:5c:00.0"} +2025-11-17T01:01:22.068767457Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib4"} +2025-11-17T01:01:22.068828029Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib4"} +2025-11-17T01:01:22.068919404Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib4"} +2025-11-17T01:01:22.068979983Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:85:00.0"} +2025-11-17T01:01:22.069063372Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib5"} +2025-11-17T01:01:22.069117752Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib5"} +2025-11-17T01:01:22.069217352Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib5"} +2025-11-17T01:01:22.069279113Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:8a:00.0"} +2025-11-17T01:01:22.06936444Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib6"} +2025-11-17T01:01:22.069422457Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib6"} +2025-11-17T01:01:22.069438407Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib6/speed", "error": "read /sys/class/net/ib6/speed: invalid argument"} +2025-11-17T01:01:22.069445249Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib6"} +2025-11-17T01:01:22.06950334Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:92:00.0"} +2025-11-17T01:01:22.069616855Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib7"} +2025-11-17T01:01:22.06966502Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib7"} +2025-11-17T01:01:22.069680856Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib7/speed", "error": "read /sys/class/net/ib7/speed: invalid argument"} +2025-11-17T01:01:22.069686574Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib7"} +2025-11-17T01:01:22.069750872Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:e3:00.0"} +2025-11-17T01:01:22.080242404Z INFO daemon/writer.go:166 setNodeStateStatus(): status {"sync-status": "InProgress", "last-sync-error": ""} +2025-11-17T01:01:27.034612027Z LEVEL(-2) mellanox/mellanox.go:81 RunCommand() {"output": "\nDevice #1:\n----------\n\nDevice type: ConnectX7 \nName: MCX75310AAS-NEA_Ax \nDescription: NVIDIA ConnectX-7 HHHL Adapter card; 400GbE / NDR IB (default mode); Single-port OSFP; PCIe 5.0 x16; Crypto Disabled; Secure Boot Enabled;\nDevice: 0000:e3:00.0 \n\nConfigurations: Default Current Next Boot\n MODULE_SPLIT_M0 Array[0..15] Array[0..15] Array[0..15] \n MEMIC_BAR_SIZE 0 0 0 \n MEMIC_SIZE_LIMIT _256KB(1) _256KB(1) _256KB(1) \n MEMIC_ATOMIC MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2)\n HOST_CHAINING_MODE DISABLED(0) DISABLED(0) DISABLED(0) \n HOST_CHAINING_CACHE_DISABLE False(0) False(0) False(0) \n HOST_CHAINING_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n HOST_CHAINING_TOTAL_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n FLEX_PARSER_PROFILE_ENABLE 0 0 0 \n PROG_PARSE_GRAPH False(0) False(0) False(0) \n FLEX_IPV4_OVER_VXLAN_PORT 0 0 0 \n ROCE_NEXT_PROTOCOL 254 254 254 \n ESWITCH_HAIRPIN_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n ESWITCH_HAIRPIN_TOT_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n DPA_AUTHENTICATION True(1) True(1) True(1) \n PF_BAR2_SIZE 0 0 0 \n PF_NUM_OF_VF_VALID False(0) False(0) False(0) \n NON_PREFETCHABLE_PF_BAR False(0) False(0) False(0) \n VF_VPD_ENABLE False(0) False(0) False(0) \n PF_NUM_PF_MSIX_VALID False(0) False(0) False(0) \n PER_PF_NUM_SF False(0) False(0) False(0) \n STRICT_VF_MSIX_NUM False(0) False(0) False(0) \n VF_NODNIC_ENABLE False(0) False(0) False(0) \n NUM_PF_MSIX_VALID True(1) True(1) True(1) \n* NUM_OF_VFS 16 16 1 \n NUM_OF_PF 1 1 1 \n PF_BAR2_ENABLE False(0) False(0) False(0) \n SRIOV_EN True(1) True(1) True(1) \n PF_LOG_BAR_SIZE 5 5 5 \n VF_LOG_BAR_SIZE 1 1 1 \n NUM_PF_MSIX 63 63 63 \n NUM_VF_MSIX 11 11 11 \n INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0) AUTOMATIC(0) AUTOMATIC(0) \n PCIE_CREDIT_TOKEN_TIMEOUT 0 0 0 \n RT_PPS_ENABLED_ON_POWERUP False(0) False(0) False(0) \n LAG_RESOURCE_ALLOCATION DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ACCURATE_TX_SCHEDULER False(0) False(0) False(0) \n PARTIAL_RESET_EN False(0) False(0) False(0) \n RESET_WITH_HOST_ON_ERRORS False(0) False(0) False(0) \n PCI_SWITCH_EMULATION_NUM_PORT 16 16 16 \n PCI_SWITCH_EMULATION_ENABLE False(0) False(0) False(0) \n PCI_DOWNSTREAM_PORT_OWNER Array[0..15] Array[0..15] Array[0..15] \n CQE_COMPRESSION BALANCED(0) BALANCED(0) BALANCED(0) \n IP_OVER_VXLAN_EN False(0) False(0) False(0) \n MKEY_BY_NAME False(0) False(0) False(0) \n PRIO_TAG_REQUIRED_EN False(0) False(0) False(0) \n UCTX_EN True(1) True(1) True(1) \n REAL_TIME_CLOCK_ENABLE False(0) False(0) False(0) \n RDMA_SELECTIVE_REPEAT_EN False(0) False(0) False(0) \n PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)\n TUNNEL_ECN_COPY_DISABLE False(0) False(0) False(0) \n LRO_LOG_TIMEOUT0 6 6 6 \n LRO_LOG_TIMEOUT1 7 7 7 \n LRO_LOG_TIMEOUT2 8 8 8 \n LRO_LOG_TIMEOUT3 13 13 13 \n LOG_TX_PSN_WINDOW 7 7 7 \n VF_MIGRATION_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n LOG_MAX_OUTSTANDING_WQE 7 7 7 \n ROCE_ADAPTIVE_ROUTING_EN False(0) False(0) False(0) \n TUNNEL_IP_PROTO_ENTROPY_DISABLE False(0) False(0) False(0) \n PCC_HANDLE_CORE_UTIL DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PCC_INT_NP_RTT_DSCP 26 26 26 \n PCC_INT_NP_RTT_DSCP_EN False(0) False(0) False(0) \n PCC_INT_NP_RTT_DATA_MODE RTT_V0(64) RTT_V0(64) RTT_V0(64) \n PCC_INT_EN False(0) False(0) False(0) \n PCC_INT_SYSTEM_RTT 0 0 0 \n STEERING_CACHE_REFRESH 0 0 0 \n ICM_CACHE_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n HAIRPIN_DATA_BUFFER_LOCK False(0) False(0) False(0) \n TX_SCHEDULER_BURST 0 0 0 \n ZERO_TOUCH_TUNING_ENABLE False(0) False(0) False(0) \n LOG_MAX_QUEUE 17 17 17 \n LARGE_MTU_TWEAK_64 False(0) False(0) False(0) \n AES_XTS_TWEAK_INC_64 False(0) False(0) False(0) \n* CRYPTO_POLICY UNRESTRICTED(1) DEVICE_DEFAULT(0) UNRESTRICTED(1) \n RDE_DISABLE False(0) False(0) False(0) \n PLDM_FW_UPDATE_DISABLE False(0) False(0) False(0) \n RBT_DISABLE False(0) False(0) False(0) \n PCIE_SMBUS_DISABLE False(0) False(0) False(0) \n PCIE_IN_BAND_VDM_DISABLE False(0) False(0) False(0) \n ADVANCED_TESTABILITY False(0) False(0) False(0) \n LOG_DCR_HASH_TABLE_SIZE 11 11 11 \n MAX_PACKET_LIFETIME 0 0 0 \n DCR_LIFO_SIZE 16384 16384 16384 \n LINK_TYPE_P1 IB(1) IB(1) IB(1) \n ROCE_CC_PRIO_MASK_P1 255 255 255 \n ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ROCE_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n IB_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1) True(1) True(1) \n CLAMP_TGT_RATE_P1 False(0) False(0) False(0) \n RPG_TIME_RESET_P1 300 300 300 \n RPG_BYTE_RESET_P1 32767 32767 32767 \n RPG_THRESHOLD_P1 1 1 1 \n RPG_MAX_RATE_P1 0 0 0 \n RPG_AI_RATE_P1 5 5 5 \n RPG_HAI_RATE_P1 50 50 50 \n RPG_GD_P1 11 11 11 \n RPG_MIN_DEC_FAC_P1 50 50 50 \n RPG_MIN_RATE_P1 1 1 1 \n RATE_TO_SET_ON_FIRST_CNP_P1 0 0 0 \n DCE_TCP_G_P1 1019 1019 1019 \n DCE_TCP_RTT_P1 1 1 1 \n RATE_REDUCE_MONITOR_PERIOD_P1 4 4 4 \n INITIAL_ALPHA_VALUE_P1 1023 1023 1023 \n MIN_TIME_BETWEEN_CNPS_P1 4 4 4 \n CNP_802P_PRIO_P1 6 6 6 \n CNP_DSCP_P1 48 48 48 \n LLDP_NB_DCBX_P1 False(0) False(0) False(0) \n LLDP_NB_RX_MODE_P1 OFF(0) OFF(0) OFF(0) \n LLDP_NB_TX_MODE_P1 OFF(0) OFF(0) OFF(0) \n ROCE_RTT_RESP_DSCP_P1 0 0 0 \n ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n DCBX_IEEE_P1 True(1) True(1) True(1) \n DCBX_CEE_P1 True(1) True(1) True(1) \n DCBX_WILLING_P1 True(1) True(1) True(1) \n KEEP_ETH_LINK_UP_P1 True(1) True(1) True(1) \n KEEP_IB_LINK_UP_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_BOOT_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_STANDBY_P1 False(0) False(0) False(0) \n DO_NOT_CLEAR_PORT_STATS_P1 False(0) False(0) False(0) \n AUTO_POWER_SAVE_LINK_DOWN_P1 False(0) False(0) False(0) \n NUM_OF_VL_P1 _4_VLs(3) _4_VLs(3) _4_VLs(3) \n NUM_OF_TC_P1 _8_TCs(0) _8_TCs(0) _8_TCs(0) \n NUM_OF_PFC_P1 8 8 8 \n VL15_BUFFER_SIZE_P1 0 0 0 \n QOS_TRUST_STATE_P1 TRUST_PCP(1) TRUST_PCP(1) TRUST_PCP(1) \n ETS_SCHED_MODE_P1 device_default(0) device_default(0) device_default(0) \n DUP_MAC_ACTION_P1 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) \n MPFS_MC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n MPFS_UC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n SRIOV_IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n PHY_AUTO_NEG_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PHY_RATE_MASK_OVERRIDE_P1 False(0) False(0) False(0) \n PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PF_TOTAL_SF 0 0 0 \n PF_SD_GROUP 0 0 0 \n PF_SF_BAR_SIZE 0 0 0 \n PF_NUM_PF_MSIX 63 63 63 \n SILENT_MODE False(0) False(0) False(0) \n MKEY_BY_NAME_RANGE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n* ROCE_CONTROL ROCE_ENABLE(2) DEVICE_DEFAULT(0) ROCE_ENABLE(2) \n PCI_WR_ORDERING per_mkey(0) per_mkey(0) per_mkey(0) \n MULTI_PORT_VHCA_EN False(0) False(0) False(0) \n PORT_OWNER True(1) True(1) True(1) \n ALLOW_RD_COUNTERS True(1) True(1) True(1) \n RENEG_ON_CHANGE True(1) True(1) True(1) \n TRACER_ENABLE True(1) True(1) True(1) \n IP_VER IPv4(0) IPv4(0) IPv4(0) \n BOOT_UNDI_NETWORK_WAIT 0 0 0 \n UEFI_HII_EN True(1) True(1) True(1) \n BOOT_DBG_LOG False(0) False(0) False(0) \n UEFI_LOGS DISABLED(0) DISABLED(0) DISABLED(0) \n BOOT_VLAN 1 1 1 \n LEGACY_BOOT_PROTOCOL PXE(1) PXE(1) PXE(1) \n BOOT_INTERRUPT_DIS False(0) False(0) False(0) \n BOOT_LACP_DIS True(1) True(1) True(1) \n BOOT_VLAN_EN False(0) False(0) False(0) \n BOOT_PKEY 0 0 0 \n P2P_ORDERING_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ATS_ENABLED False(0) False(0) False(0) \n DYNAMIC_VF_MSIX_TABLE False(0) False(0) False(0) \n EXP_ROM_UEFI_ARM_ENABLE True(1) True(1) True(1) \n EXP_ROM_UEFI_x86_ENABLE True(1) True(1) True(1) \n EXP_ROM_PXE_ENABLE True(1) True(1) True(1) \n ADVANCED_PCI_SETTINGS False(0) False(0) False(0) \n SAFE_MODE_THRESHOLD 10 10 10 \n SAFE_MODE_ENABLE True(1) True(1) True(1) \nThe '*' shows parameters with next value different from default/current value.\n", "error": null} +2025-11-17T01:01:27.034732068Z INFO mellanox/mellanox.go:213 ParseMstconfigOutput() {"attributes": ["NUM_OF_VFS", "SRIOV_EN", "LINK_TYPE_P1", "LINK_TYPE_P2"]} +2025-11-17T01:01:27.034775472Z INFO mellanox/mellanox.go:214 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"16","SRIOV_EN":"True(1)"}} +2025-11-17T01:01:27.034792914Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:01:27.03479703Z INFO mellanox/mellanox.go:219 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"1","SRIOV_EN":"True(1)"}} +2025-11-17T01:01:27.034802359Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:01:27.034805586Z INFO mellanox/mellanox_plugin.go:107 mellanox-plugin IsDualPort() {"address": "0000:e3:00.0"} +2025-11-17T01:01:27.034811725Z LEVEL(-2) mellanox/mellanox_plugin.go:112 Changing TotalVfs, needs reboot {"current": 16, "requested": 1} +2025-11-17T01:01:27.034818896Z INFO mellanox/mellanox.go:343 mellanox-plugin isLinkTypeRequireChange() {"device": "0000:e3:00.0"} +2025-11-17T01:01:27.034824277Z INFO mellanox/mellanox_plugin.go:102 mellanox-plugin getMlnxNicFwData() {"device": "0000:19:00.0"} +2025-11-17T01:01:27.034827983Z INFO mellanox/mellanox.go:208 MstConfigReadData() {"device": "0000:19:00.0"} +2025-11-17T01:01:27.034831823Z INFO mellanox/mellanox.go:81 RunCommand() {"command": "mstconfig", "args": ["-e", "-d", "0000:19:00.0", "q"]} +2025-11-17T01:01:51.283986837Z LEVEL(-2) mellanox/mellanox.go:81 RunCommand() {"output": "\nDevice #1:\n----------\n\nDevice type: ConnectX7 \nName: MCX75310AAS-NEA_Ax \nDescription: NVIDIA ConnectX-7 HHHL Adapter card; 400GbE / NDR IB (default mode); Single-port OSFP; PCIe 5.0 x16; Crypto Disabled; Secure Boot Enabled;\nDevice: 0000:19:00.0 \n\nConfigurations: Default Current Next Boot\n MODULE_SPLIT_M0 Array[0..15] Array[0..15] Array[0..15] \n MEMIC_BAR_SIZE 0 0 0 \n MEMIC_SIZE_LIMIT _256KB(1) _256KB(1) _256KB(1) \n MEMIC_ATOMIC MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2)\n HOST_CHAINING_MODE DISABLED(0) DISABLED(0) DISABLED(0) \n HOST_CHAINING_CACHE_DISABLE False(0) False(0) False(0) \n HOST_CHAINING_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n HOST_CHAINING_TOTAL_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n FLEX_PARSER_PROFILE_ENABLE 0 0 0 \n PROG_PARSE_GRAPH False(0) False(0) False(0) \n FLEX_IPV4_OVER_VXLAN_PORT 0 0 0 \n ROCE_NEXT_PROTOCOL 254 254 254 \n ESWITCH_HAIRPIN_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n ESWITCH_HAIRPIN_TOT_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n DPA_AUTHENTICATION True(1) True(1) True(1) \n PF_BAR2_SIZE 0 0 0 \n PF_NUM_OF_VF_VALID False(0) False(0) False(0) \n NON_PREFETCHABLE_PF_BAR False(0) False(0) False(0) \n VF_VPD_ENABLE False(0) False(0) False(0) \n PF_NUM_PF_MSIX_VALID False(0) False(0) False(0) \n PER_PF_NUM_SF False(0) False(0) False(0) \n STRICT_VF_MSIX_NUM False(0) False(0) False(0) \n VF_NODNIC_ENABLE False(0) False(0) False(0) \n NUM_PF_MSIX_VALID True(1) True(1) True(1) \n* NUM_OF_VFS 16 16 1 \n NUM_OF_PF 1 1 1 \n PF_BAR2_ENABLE False(0) False(0) False(0) \n SRIOV_EN True(1) True(1) True(1) \n PF_LOG_BAR_SIZE 5 5 5 \n VF_LOG_BAR_SIZE 1 1 1 \n NUM_PF_MSIX 63 63 63 \n NUM_VF_MSIX 11 11 11 \n INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0) AUTOMATIC(0) AUTOMATIC(0) \n PCIE_CREDIT_TOKEN_TIMEOUT 0 0 0 \n RT_PPS_ENABLED_ON_POWERUP False(0) False(0) False(0) \n LAG_RESOURCE_ALLOCATION DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ACCURATE_TX_SCHEDULER False(0) False(0) False(0) \n PARTIAL_RESET_EN False(0) False(0) False(0) \n RESET_WITH_HOST_ON_ERRORS False(0) False(0) False(0) \n PCI_SWITCH_EMULATION_NUM_PORT 16 16 16 \n PCI_SWITCH_EMULATION_ENABLE False(0) False(0) False(0) \n PCI_DOWNSTREAM_PORT_OWNER Array[0..15] Array[0..15] Array[0..15] \n CQE_COMPRESSION BALANCED(0) BALANCED(0) BALANCED(0) \n IP_OVER_VXLAN_EN False(0) False(0) False(0) \n MKEY_BY_NAME False(0) False(0) False(0) \n PRIO_TAG_REQUIRED_EN False(0) False(0) False(0) \n UCTX_EN True(1) True(1) True(1) \n REAL_TIME_CLOCK_ENABLE False(0) False(0) False(0) \n RDMA_SELECTIVE_REPEAT_EN False(0) False(0) False(0) \n PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)\n TUNNEL_ECN_COPY_DISABLE False(0) False(0) False(0) \n LRO_LOG_TIMEOUT0 6 6 6 \n LRO_LOG_TIMEOUT1 7 7 7 \n LRO_LOG_TIMEOUT2 8 8 8 \n LRO_LOG_TIMEOUT3 13 13 13 \n LOG_TX_PSN_WINDOW 7 7 7 \n VF_MIGRATION_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n LOG_MAX_OUTSTANDING_WQE 7 7 7 \n ROCE_ADAPTIVE_ROUTING_EN False(0) False(0) False(0) \n TUNNEL_IP_PROTO_ENTROPY_DISABLE False(0) False(0) False(0) \n PCC_HANDLE_CORE_UTIL DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PCC_INT_NP_RTT_DSCP 26 26 26 \n PCC_INT_NP_RTT_DSCP_EN False(0) False(0) False(0) \n PCC_INT_NP_RTT_DATA_MODE RTT_V0(64) RTT_V0(64) RTT_V0(64) \n PCC_INT_EN False(0) False(0) False(0) \n PCC_INT_SYSTEM_RTT 0 0 0 \n STEERING_CACHE_REFRESH 0 0 0 \n ICM_CACHE_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n HAIRPIN_DATA_BUFFER_LOCK False(0) False(0) False(0) \n TX_SCHEDULER_BURST 0 0 0 \n ZERO_TOUCH_TUNING_ENABLE False(0) False(0) False(0) \n LOG_MAX_QUEUE 17 17 17 \n LARGE_MTU_TWEAK_64 False(0) False(0) False(0) \n AES_XTS_TWEAK_INC_64 False(0) False(0) False(0) \n* CRYPTO_POLICY UNRESTRICTED(1) DEVICE_DEFAULT(0) UNRESTRICTED(1) \n RDE_DISABLE False(0) False(0) False(0) \n PLDM_FW_UPDATE_DISABLE False(0) False(0) False(0) \n RBT_DISABLE False(0) False(0) False(0) \n PCIE_SMBUS_DISABLE False(0) False(0) False(0) \n PCIE_IN_BAND_VDM_DISABLE False(0) False(0) False(0) \n ADVANCED_TESTABILITY False(0) False(0) False(0) \n LOG_DCR_HASH_TABLE_SIZE 11 11 11 \n MAX_PACKET_LIFETIME 0 0 0 \n DCR_LIFO_SIZE 16384 16384 16384 \n LINK_TYPE_P1 IB(1) IB(1) IB(1) \n ROCE_CC_PRIO_MASK_P1 255 255 255 \n ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ROCE_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n IB_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1) True(1) True(1) \n CLAMP_TGT_RATE_P1 False(0) False(0) False(0) \n RPG_TIME_RESET_P1 300 300 300 \n RPG_BYTE_RESET_P1 32767 32767 32767 \n RPG_THRESHOLD_P1 1 1 1 \n RPG_MAX_RATE_P1 0 0 0 \n RPG_AI_RATE_P1 5 5 5 \n RPG_HAI_RATE_P1 50 50 50 \n RPG_GD_P1 11 11 11 \n RPG_MIN_DEC_FAC_P1 50 50 50 \n RPG_MIN_RATE_P1 1 1 1 \n RATE_TO_SET_ON_FIRST_CNP_P1 0 0 0 \n DCE_TCP_G_P1 1019 1019 1019 \n DCE_TCP_RTT_P1 1 1 1 \n RATE_REDUCE_MONITOR_PERIOD_P1 4 4 4 \n INITIAL_ALPHA_VALUE_P1 1023 1023 1023 \n MIN_TIME_BETWEEN_CNPS_P1 4 4 4 \n CNP_802P_PRIO_P1 6 6 6 \n CNP_DSCP_P1 48 48 48 \n LLDP_NB_DCBX_P1 False(0) False(0) False(0) \n LLDP_NB_RX_MODE_P1 OFF(0) OFF(0) OFF(0) \n LLDP_NB_TX_MODE_P1 OFF(0) OFF(0) OFF(0) \n ROCE_RTT_RESP_DSCP_P1 0 0 0 \n ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n DCBX_IEEE_P1 True(1) True(1) True(1) \n DCBX_CEE_P1 True(1) True(1) True(1) \n DCBX_WILLING_P1 True(1) True(1) True(1) \n KEEP_ETH_LINK_UP_P1 True(1) True(1) True(1) \n KEEP_IB_LINK_UP_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_BOOT_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_STANDBY_P1 False(0) False(0) False(0) \n DO_NOT_CLEAR_PORT_STATS_P1 False(0) False(0) False(0) \n AUTO_POWER_SAVE_LINK_DOWN_P1 False(0) False(0) False(0) \n NUM_OF_VL_P1 _4_VLs(3) _4_VLs(3) _4_VLs(3) \n NUM_OF_TC_P1 _8_TCs(0) _8_TCs(0) _8_TCs(0) \n NUM_OF_PFC_P1 8 8 8 \n VL15_BUFFER_SIZE_P1 0 0 0 \n QOS_TRUST_STATE_P1 TRUST_PCP(1) TRUST_PCP(1) TRUST_PCP(1) \n ETS_SCHED_MODE_P1 device_default(0) device_default(0) device_default(0) \n DUP_MAC_ACTION_P1 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) \n MPFS_MC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n MPFS_UC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n SRIOV_IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n PHY_AUTO_NEG_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PHY_RATE_MASK_OVERRIDE_P1 False(0) False(0) False(0) \n PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PF_TOTAL_SF 0 0 0 \n PF_SD_GROUP 0 0 0 \n PF_SF_BAR_SIZE 0 0 0 \n PF_NUM_PF_MSIX 63 63 63 \n SILENT_MODE False(0) False(0) False(0) \n MKEY_BY_NAME_RANGE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n* ROCE_CONTROL ROCE_ENABLE(2) DEVICE_DEFAULT(0) ROCE_ENABLE(2) \n PCI_WR_ORDERING per_mkey(0) per_mkey(0) per_mkey(0) \n MULTI_PORT_VHCA_EN False(0) False(0) False(0) \n PORT_OWNER True(1) True(1) True(1) \n ALLOW_RD_COUNTERS True(1) True(1) True(1) \n RENEG_ON_CHANGE True(1) True(1) True(1) \n TRACER_ENABLE True(1) True(1) True(1) \n IP_VER IPv4(0) IPv4(0) IPv4(0) \n BOOT_UNDI_NETWORK_WAIT 0 0 0 \n UEFI_HII_EN True(1) True(1) True(1) \n BOOT_DBG_LOG False(0) False(0) False(0) \n UEFI_LOGS DISABLED(0) DISABLED(0) DISABLED(0) \n BOOT_VLAN 1 1 1 \n LEGACY_BOOT_PROTOCOL PXE(1) PXE(1) PXE(1) \n BOOT_INTERRUPT_DIS False(0) False(0) False(0) \n BOOT_LACP_DIS True(1) True(1) True(1) \n BOOT_VLAN_EN False(0) False(0) False(0) \n BOOT_PKEY 0 0 0 \n P2P_ORDERING_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ATS_ENABLED False(0) False(0) False(0) \n DYNAMIC_VF_MSIX_TABLE False(0) False(0) False(0) \n EXP_ROM_UEFI_ARM_ENABLE True(1) True(1) True(1) \n EXP_ROM_UEFI_x86_ENABLE True(1) True(1) True(1) \n EXP_ROM_PXE_ENABLE True(1) True(1) True(1) \n ADVANCED_PCI_SETTINGS False(0) False(0) False(0) \n SAFE_MODE_THRESHOLD 10 10 10 \n SAFE_MODE_ENABLE True(1) True(1) True(1) \nThe '*' shows parameters with next value different from default/current value.\n", "error": null} +2025-11-17T01:01:51.284090177Z INFO mellanox/mellanox.go:213 ParseMstconfigOutput() {"attributes": ["NUM_OF_VFS", "SRIOV_EN", "LINK_TYPE_P1", "LINK_TYPE_P2"]} +2025-11-17T01:01:51.284141806Z INFO mellanox/mellanox.go:214 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"16","SRIOV_EN":"True(1)"}} +2025-11-17T01:01:51.284155512Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:01:51.284159743Z INFO mellanox/mellanox.go:219 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"1","SRIOV_EN":"True(1)"}} +2025-11-17T01:01:51.284169177Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:01:51.284173885Z INFO mellanox/mellanox_plugin.go:107 mellanox-plugin IsDualPort() {"address": "0000:19:00.0"} +2025-11-17T01:01:51.284181628Z LEVEL(-2) mellanox/mellanox_plugin.go:112 Changing TotalVfs, needs reboot {"current": 16, "requested": 1} +2025-11-17T01:01:51.284189795Z INFO mellanox/mellanox.go:343 mellanox-plugin isLinkTypeRequireChange() {"device": "0000:19:00.0"} +2025-11-17T01:01:51.28419698Z INFO mellanox/mellanox_plugin.go:102 mellanox-plugin getMlnxNicFwData() {"device": "0000:29:00.0"} +2025-11-17T01:01:51.284201307Z INFO mellanox/mellanox.go:208 MstConfigReadData() {"device": "0000:29:00.0"} +2025-11-17T01:01:51.284205437Z INFO mellanox/mellanox.go:81 RunCommand() {"command": "mstconfig", "args": ["-e", "-d", "0000:29:00.0", "q"]} +2025-11-17T01:01:52.094994285Z LEVEL(-2) runtime/asm_amd64.s:1695 Run(): period refresh +2025-11-17T01:01:52.095015436Z LEVEL(-2) daemon/writer.go:109 pollNicStatus() +2025-11-17T01:01:52.095087463Z LEVEL(-2) daemon/writer.go:130 DiscoverSriovDevices +2025-11-17T01:01:52.195459744Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib0"} +2025-11-17T01:01:52.195694217Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib0"} +2025-11-17T01:01:52.195713667Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib0/speed", "error": "read /sys/class/net/ib0/speed: invalid argument"} +2025-11-17T01:01:52.195720067Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib0"} +2025-11-17T01:01:52.195788106Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:19:00.0"} +2025-11-17T01:01:52.195902852Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib1"} +2025-11-17T01:01:52.19594694Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib1"} +2025-11-17T01:01:52.195959802Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib1/speed", "error": "read /sys/class/net/ib1/speed: invalid argument"} +2025-11-17T01:01:52.195963889Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib1"} +2025-11-17T01:01:52.196015904Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:29:00.0"} +2025-11-17T01:01:52.1961013Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib2"} +2025-11-17T01:01:52.196146743Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib2"} +2025-11-17T01:01:52.196443911Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib2"} +2025-11-17T01:01:52.196500097Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:3b:00.0"} +2025-11-17T01:01:52.196651411Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth2"} +2025-11-17T01:01:52.197310219Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth2"} +2025-11-17T01:01:52.197369379Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.0"} +2025-11-17T01:01:52.197515971Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth3"} +2025-11-17T01:01:52.198534537Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth3"} +2025-11-17T01:01:52.1986868Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.1"} +2025-11-17T01:01:52.198790646Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth0"} +2025-11-17T01:01:52.198841794Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth0"} +2025-11-17T01:01:52.198857238Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth0/speed", "error": "read /sys/class/net/eth0/speed: invalid argument"} +2025-11-17T01:01:52.198863433Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth0"} +2025-11-17T01:01:52.198931129Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.0"} +2025-11-17T01:01:52.19900123Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth1"} +2025-11-17T01:01:52.199050195Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth1"} +2025-11-17T01:01:52.19906583Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth1/speed", "error": "read /sys/class/net/eth1/speed: invalid argument"} +2025-11-17T01:01:52.199069665Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth1"} +2025-11-17T01:01:52.199132912Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.1"} +2025-11-17T01:01:52.199211326Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib3"} +2025-11-17T01:01:52.199264724Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib3"} +2025-11-17T01:01:52.199388244Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib3"} +2025-11-17T01:01:52.199578106Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:5c:00.0"} +2025-11-17T01:01:52.199704636Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib4"} +2025-11-17T01:01:52.199761474Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib4"} +2025-11-17T01:01:52.199850844Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib4"} +2025-11-17T01:01:52.199919422Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:85:00.0"} +2025-11-17T01:01:52.200008818Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib5"} +2025-11-17T01:01:52.200057236Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib5"} +2025-11-17T01:01:52.200289781Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib5"} +2025-11-17T01:01:52.200358184Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:8a:00.0"} +2025-11-17T01:01:52.200437723Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib6"} +2025-11-17T01:01:52.200482326Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib6"} +2025-11-17T01:01:52.200497506Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib6/speed", "error": "read /sys/class/net/ib6/speed: invalid argument"} +2025-11-17T01:01:52.200505458Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib6"} +2025-11-17T01:01:52.200569189Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:92:00.0"} +2025-11-17T01:01:52.200652739Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib7"} +2025-11-17T01:01:52.200694957Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib7"} +2025-11-17T01:01:52.200709294Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib7/speed", "error": "read /sys/class/net/ib7/speed: invalid argument"} +2025-11-17T01:01:52.2007135Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib7"} +2025-11-17T01:01:52.20077191Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:e3:00.0"} +2025-11-17T01:01:52.211192385Z INFO daemon/writer.go:166 setNodeStateStatus(): status {"sync-status": "InProgress", "last-sync-error": ""} +2025-11-17T01:02:15.020220837Z LEVEL(-2) mellanox/mellanox.go:81 RunCommand() {"output": "\nDevice #1:\n----------\n\nDevice type: ConnectX7 \nName: MCX75310AAS-NEA_Ax \nDescription: NVIDIA ConnectX-7 HHHL Adapter card; 400GbE / NDR IB (default mode); Single-port OSFP; PCIe 5.0 x16; Crypto Disabled; Secure Boot Enabled;\nDevice: 0000:29:00.0 \n\nConfigurations: Default Current Next Boot\n MODULE_SPLIT_M0 Array[0..15] Array[0..15] Array[0..15] \n MEMIC_BAR_SIZE 0 0 0 \n MEMIC_SIZE_LIMIT _256KB(1) _256KB(1) _256KB(1) \n MEMIC_ATOMIC MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2)\n HOST_CHAINING_MODE DISABLED(0) DISABLED(0) DISABLED(0) \n HOST_CHAINING_CACHE_DISABLE False(0) False(0) False(0) \n HOST_CHAINING_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n HOST_CHAINING_TOTAL_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n FLEX_PARSER_PROFILE_ENABLE 0 0 0 \n PROG_PARSE_GRAPH False(0) False(0) False(0) \n FLEX_IPV4_OVER_VXLAN_PORT 0 0 0 \n ROCE_NEXT_PROTOCOL 254 254 254 \n ESWITCH_HAIRPIN_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n ESWITCH_HAIRPIN_TOT_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n DPA_AUTHENTICATION True(1) True(1) True(1) \n PF_BAR2_SIZE 0 0 0 \n PF_NUM_OF_VF_VALID False(0) False(0) False(0) \n NON_PREFETCHABLE_PF_BAR False(0) False(0) False(0) \n VF_VPD_ENABLE False(0) False(0) False(0) \n PF_NUM_PF_MSIX_VALID False(0) False(0) False(0) \n PER_PF_NUM_SF False(0) False(0) False(0) \n STRICT_VF_MSIX_NUM False(0) False(0) False(0) \n VF_NODNIC_ENABLE False(0) False(0) False(0) \n NUM_PF_MSIX_VALID True(1) True(1) True(1) \n* NUM_OF_VFS 16 16 1 \n NUM_OF_PF 1 1 1 \n PF_BAR2_ENABLE False(0) False(0) False(0) \n SRIOV_EN True(1) True(1) True(1) \n PF_LOG_BAR_SIZE 5 5 5 \n VF_LOG_BAR_SIZE 1 1 1 \n NUM_PF_MSIX 63 63 63 \n NUM_VF_MSIX 11 11 11 \n INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0) AUTOMATIC(0) AUTOMATIC(0) \n PCIE_CREDIT_TOKEN_TIMEOUT 0 0 0 \n RT_PPS_ENABLED_ON_POWERUP False(0) False(0) False(0) \n LAG_RESOURCE_ALLOCATION DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ACCURATE_TX_SCHEDULER False(0) False(0) False(0) \n PARTIAL_RESET_EN False(0) False(0) False(0) \n RESET_WITH_HOST_ON_ERRORS False(0) False(0) False(0) \n PCI_SWITCH_EMULATION_NUM_PORT 16 16 16 \n PCI_SWITCH_EMULATION_ENABLE False(0) False(0) False(0) \n PCI_DOWNSTREAM_PORT_OWNER Array[0..15] Array[0..15] Array[0..15] \n CQE_COMPRESSION BALANCED(0) BALANCED(0) BALANCED(0) \n IP_OVER_VXLAN_EN False(0) False(0) False(0) \n MKEY_BY_NAME False(0) False(0) False(0) \n PRIO_TAG_REQUIRED_EN False(0) False(0) False(0) \n UCTX_EN True(1) True(1) True(1) \n REAL_TIME_CLOCK_ENABLE False(0) False(0) False(0) \n RDMA_SELECTIVE_REPEAT_EN False(0) False(0) False(0) \n PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)\n TUNNEL_ECN_COPY_DISABLE False(0) False(0) False(0) \n LRO_LOG_TIMEOUT0 6 6 6 \n LRO_LOG_TIMEOUT1 7 7 7 \n LRO_LOG_TIMEOUT2 8 8 8 \n LRO_LOG_TIMEOUT3 13 13 13 \n LOG_TX_PSN_WINDOW 7 7 7 \n VF_MIGRATION_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n LOG_MAX_OUTSTANDING_WQE 7 7 7 \n ROCE_ADAPTIVE_ROUTING_EN False(0) False(0) False(0) \n TUNNEL_IP_PROTO_ENTROPY_DISABLE False(0) False(0) False(0) \n PCC_HANDLE_CORE_UTIL DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PCC_INT_NP_RTT_DSCP 26 26 26 \n PCC_INT_NP_RTT_DSCP_EN False(0) False(0) False(0) \n PCC_INT_NP_RTT_DATA_MODE RTT_V0(64) RTT_V0(64) RTT_V0(64) \n PCC_INT_EN False(0) False(0) False(0) \n PCC_INT_SYSTEM_RTT 0 0 0 \n STEERING_CACHE_REFRESH 0 0 0 \n ICM_CACHE_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n HAIRPIN_DATA_BUFFER_LOCK False(0) False(0) False(0) \n TX_SCHEDULER_BURST 0 0 0 \n ZERO_TOUCH_TUNING_ENABLE False(0) False(0) False(0) \n LOG_MAX_QUEUE 17 17 17 \n LARGE_MTU_TWEAK_64 False(0) False(0) False(0) \n AES_XTS_TWEAK_INC_64 False(0) False(0) False(0) \n* CRYPTO_POLICY UNRESTRICTED(1) DEVICE_DEFAULT(0) UNRESTRICTED(1) \n RDE_DISABLE False(0) False(0) False(0) \n PLDM_FW_UPDATE_DISABLE False(0) False(0) False(0) \n RBT_DISABLE False(0) False(0) False(0) \n PCIE_SMBUS_DISABLE False(0) False(0) False(0) \n PCIE_IN_BAND_VDM_DISABLE False(0) False(0) False(0) \n ADVANCED_TESTABILITY False(0) False(0) False(0) \n LOG_DCR_HASH_TABLE_SIZE 11 11 11 \n MAX_PACKET_LIFETIME 0 0 0 \n DCR_LIFO_SIZE 16384 16384 16384 \n LINK_TYPE_P1 IB(1) IB(1) IB(1) \n ROCE_CC_PRIO_MASK_P1 255 255 255 \n ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ROCE_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n IB_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1) True(1) True(1) \n CLAMP_TGT_RATE_P1 False(0) False(0) False(0) \n RPG_TIME_RESET_P1 300 300 300 \n RPG_BYTE_RESET_P1 32767 32767 32767 \n RPG_THRESHOLD_P1 1 1 1 \n RPG_MAX_RATE_P1 0 0 0 \n RPG_AI_RATE_P1 5 5 5 \n RPG_HAI_RATE_P1 50 50 50 \n RPG_GD_P1 11 11 11 \n RPG_MIN_DEC_FAC_P1 50 50 50 \n RPG_MIN_RATE_P1 1 1 1 \n RATE_TO_SET_ON_FIRST_CNP_P1 0 0 0 \n DCE_TCP_G_P1 1019 1019 1019 \n DCE_TCP_RTT_P1 1 1 1 \n RATE_REDUCE_MONITOR_PERIOD_P1 4 4 4 \n INITIAL_ALPHA_VALUE_P1 1023 1023 1023 \n MIN_TIME_BETWEEN_CNPS_P1 4 4 4 \n CNP_802P_PRIO_P1 6 6 6 \n CNP_DSCP_P1 48 48 48 \n LLDP_NB_DCBX_P1 False(0) False(0) False(0) \n LLDP_NB_RX_MODE_P1 OFF(0) OFF(0) OFF(0) \n LLDP_NB_TX_MODE_P1 OFF(0) OFF(0) OFF(0) \n ROCE_RTT_RESP_DSCP_P1 0 0 0 \n ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n DCBX_IEEE_P1 True(1) True(1) True(1) \n DCBX_CEE_P1 True(1) True(1) True(1) \n DCBX_WILLING_P1 True(1) True(1) True(1) \n KEEP_ETH_LINK_UP_P1 True(1) True(1) True(1) \n KEEP_IB_LINK_UP_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_BOOT_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_STANDBY_P1 False(0) False(0) False(0) \n DO_NOT_CLEAR_PORT_STATS_P1 False(0) False(0) False(0) \n AUTO_POWER_SAVE_LINK_DOWN_P1 False(0) False(0) False(0) \n NUM_OF_VL_P1 _4_VLs(3) _4_VLs(3) _4_VLs(3) \n NUM_OF_TC_P1 _8_TCs(0) _8_TCs(0) _8_TCs(0) \n NUM_OF_PFC_P1 8 8 8 \n VL15_BUFFER_SIZE_P1 0 0 0 \n QOS_TRUST_STATE_P1 TRUST_PCP(1) TRUST_PCP(1) TRUST_PCP(1) \n ETS_SCHED_MODE_P1 device_default(0) device_default(0) device_default(0) \n DUP_MAC_ACTION_P1 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) \n MPFS_MC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n MPFS_UC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n SRIOV_IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n PHY_AUTO_NEG_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PHY_RATE_MASK_OVERRIDE_P1 False(0) False(0) False(0) \n PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PF_TOTAL_SF 0 0 0 \n PF_SD_GROUP 0 0 0 \n PF_SF_BAR_SIZE 0 0 0 \n PF_NUM_PF_MSIX 63 63 63 \n SILENT_MODE False(0) False(0) False(0) \n MKEY_BY_NAME_RANGE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n* ROCE_CONTROL ROCE_ENABLE(2) DEVICE_DEFAULT(0) ROCE_ENABLE(2) \n PCI_WR_ORDERING per_mkey(0) per_mkey(0) per_mkey(0) \n MULTI_PORT_VHCA_EN False(0) False(0) False(0) \n PORT_OWNER True(1) True(1) True(1) \n ALLOW_RD_COUNTERS True(1) True(1) True(1) \n RENEG_ON_CHANGE True(1) True(1) True(1) \n TRACER_ENABLE True(1) True(1) True(1) \n IP_VER IPv4(0) IPv4(0) IPv4(0) \n BOOT_UNDI_NETWORK_WAIT 0 0 0 \n UEFI_HII_EN True(1) True(1) True(1) \n BOOT_DBG_LOG False(0) False(0) False(0) \n UEFI_LOGS DISABLED(0) DISABLED(0) DISABLED(0) \n BOOT_VLAN 1 1 1 \n LEGACY_BOOT_PROTOCOL PXE(1) PXE(1) PXE(1) \n BOOT_INTERRUPT_DIS False(0) False(0) False(0) \n BOOT_LACP_DIS True(1) True(1) True(1) \n BOOT_VLAN_EN False(0) False(0) False(0) \n BOOT_PKEY 0 0 0 \n P2P_ORDERING_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ATS_ENABLED False(0) False(0) False(0) \n DYNAMIC_VF_MSIX_TABLE False(0) False(0) False(0) \n EXP_ROM_UEFI_ARM_ENABLE True(1) True(1) True(1) \n EXP_ROM_UEFI_x86_ENABLE True(1) True(1) True(1) \n EXP_ROM_PXE_ENABLE True(1) True(1) True(1) \n ADVANCED_PCI_SETTINGS False(0) False(0) False(0) \n SAFE_MODE_THRESHOLD 10 10 10 \n SAFE_MODE_ENABLE True(1) True(1) True(1) \nThe '*' shows parameters with next value different from default/current value.\n", "error": null} +2025-11-17T01:02:15.020377294Z INFO mellanox/mellanox.go:213 ParseMstconfigOutput() {"attributes": ["NUM_OF_VFS", "SRIOV_EN", "LINK_TYPE_P1", "LINK_TYPE_P2"]} +2025-11-17T01:02:15.020443499Z INFO mellanox/mellanox.go:214 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"16","SRIOV_EN":"True(1)"}} +2025-11-17T01:02:15.020461727Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:02:15.02046593Z INFO mellanox/mellanox.go:219 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"1","SRIOV_EN":"True(1)"}} +2025-11-17T01:02:15.020469741Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:02:15.020473643Z INFO mellanox/mellanox_plugin.go:107 mellanox-plugin IsDualPort() {"address": "0000:29:00.0"} +2025-11-17T01:02:15.020483056Z LEVEL(-2) mellanox/mellanox_plugin.go:112 Changing TotalVfs, needs reboot {"current": 16, "requested": 1} +2025-11-17T01:02:15.020490603Z INFO mellanox/mellanox.go:343 mellanox-plugin isLinkTypeRequireChange() {"device": "0000:29:00.0"} +2025-11-17T01:02:15.020497173Z INFO mellanox/mellanox_plugin.go:102 mellanox-plugin getMlnxNicFwData() {"device": "0000:3b:00.0"} +2025-11-17T01:02:15.020502591Z INFO mellanox/mellanox.go:208 MstConfigReadData() {"device": "0000:3b:00.0"} +2025-11-17T01:02:15.020507819Z INFO mellanox/mellanox.go:81 RunCommand() {"command": "mstconfig", "args": ["-e", "-d", "0000:3b:00.0", "q"]} diff --git a/network-operator/sriov/test2.log b/network-operator/sriov/test2.log new file mode 100644 index 0000000..512219a --- /dev/null +++ b/network-operator/sriov/test2.log @@ -0,0 +1,586 @@ +2025-11-17T00:59:45.92409528Z INFO host/manager.go:61 infiniband.New(): ib guid config doesn't exist, continuing without it {"config path": "/etc/sriov-operator/infiniband/guids"} +2025-11-17T00:59:45.925745127Z INFO host/manager.go:61 infiniband.New(): ib guid config doesn't exist, continuing without it {"config path": "/etc/sriov-operator/infiniband/guids"} +2025-11-17T00:59:45.925936236Z INFO sriov-network-config-daemon cobra@v1.8.0/command.go:983 starting node writer +2025-11-17T00:59:45.959766078Z INFO sriov-network-config-daemon cobra@v1.8.0/command.go:983 Running on {"platform": "Baremetal"} +2025-11-17T00:59:45.998051223Z INFO sriov-network-config-daemon/start.go:277 RunOnce() +2025-11-17T00:59:45.998082634Z INFO sriov-network-config-daemon/start.go:277 RunOnce(): first poll for nic status +2025-11-17T00:59:46.225663023Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib0/speed", "error": "read /sys/class/net/ib0/speed: invalid argument"} +2025-11-17T00:59:46.226234114Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib1/speed", "error": "read /sys/class/net/ib1/speed: invalid argument"} +2025-11-17T00:59:46.231101563Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth0/speed", "error": "read /sys/class/net/eth0/speed: invalid argument"} +2025-11-17T00:59:46.23137743Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth1/speed", "error": "read /sys/class/net/eth1/speed: invalid argument"} +2025-11-17T00:59:46.232906162Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib6/speed", "error": "read /sys/class/net/ib6/speed: invalid argument"} +2025-11-17T00:59:46.233161697Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib7/speed", "error": "read /sys/class/net/ib7/speed: invalid argument"} +2025-11-17T00:59:46.243903326Z INFO daemon/writer.go:166 setNodeStateStatus(): status {"sync-status": "", "last-sync-error": ""} +2025-11-17T00:59:46.269166624Z INFO daemon/writer.go:83 writeCheckpointFile(): try to decode the checkpoint file +2025-11-17T00:59:46.269193495Z ERROR daemon/writer.go:83 writeCheckpointFile(): fail to decode, writing new file instead {"error": "EOF"} +2025-11-17T00:59:46.269210954Z INFO daemon/writer.go:83 writeCheckpointFile(): write checkpoint file +2025-11-17T00:59:46.269534063Z INFO runtime/asm_amd64.s:1695 Run(): start writer +2025-11-17T00:59:46.304248767Z INFO cobra@v1.8.0/command.go:983 Enabled featureGates {"featureGates": ""} +2025-11-17T00:59:46.304289678Z INFO sriov-network-config-daemon cobra@v1.8.0/command.go:983 Starting SriovNetworkConfigDaemon +2025-11-17T00:59:46.30434292Z INFO sriov-network-config-daemon/start.go:310 Run() {"node": "nhn-aideveloper-f0fde370"} +2025-11-17T00:59:46.304354684Z INFO sriov-network-config-daemon/start.go:310 Run(): start daemon. +2025-11-17T00:59:46.304360178Z INFO sriov-network-config-daemon/start.go:310 Run(): daemon running in daemon mode +2025-11-17T00:59:46.304370815Z INFO kernel/kernel.go:266 RunCommand() {"command": "/bin/sh", "args": ["-c", "chroot /host lsmod | grep --quiet 'mlx5_core'"]} +2025-11-17T00:59:46.327694971Z INFO kernel/kernel.go:284 RunCommand() {"command": "/bin/sh", "args": ["-c", "chroot /host lsmod | grep --quiet '\\(^ib\\|^rdma\\)'"]} +2025-11-17T00:59:46.336000978Z INFO kernel/kernel.go:75 LoadKernelModule(): try to load kernel module {"name": "tun", "args": []} +2025-11-17T00:59:46.336034332Z INFO kernel/kernel.go:33 IsKernelModuleLoaded(): check if kernel module is loaded {"name": "tun"} +2025-11-17T00:59:46.336044318Z INFO kernel/kernel.go:54 RunCommand() {"command": "/bin/sh", "args": ["-c", "chroot /host lsmod | grep \"^tun\""]} +2025-11-17T00:59:46.343881827Z INFO kernel/kernel.go:42 RunCommand() {"command": "/bin/sh", "args": ["-c", "chroot /host modprobe tun "]} +2025-11-17T00:59:46.347562828Z INFO kernel/kernel.go:81 LoadKernelModule(): try to load kernel module {"name": "vhost_net", "args": []} +2025-11-17T00:59:46.347592796Z INFO kernel/kernel.go:33 IsKernelModuleLoaded(): check if kernel module is loaded {"name": "vhost_net"} +2025-11-17T00:59:46.34760228Z INFO kernel/kernel.go:54 RunCommand() {"command": "/bin/sh", "args": ["-c", "chroot /host lsmod | grep \"^vhost_net\""]} +2025-11-17T00:59:46.359435844Z INFO kernel/kernel.go:42 RunCommand() {"command": "/bin/sh", "args": ["-c", "chroot /host modprobe vhost_net "]} +2025-11-17T00:59:46.374595365Z INFO udev/udev.go:41 RunCommand() {"command": "/bin/bash", "args": ["/bindata/scripts/udev-find-sriov-pf.sh"]} +2025-11-17T00:59:46.406541243Z INFO daemon/daemon.go:301 Set log verbose level {"new-level": 2, "current-level": 0} +2025-11-17T00:59:51.482409576Z INFO sriov-network-config-daemon/start.go:310 Starting workers +2025-11-17T00:59:51.482457526Z INFO sriov-network-config-daemon/start.go:310 Started workers +2025-11-17T00:59:51.482488729Z LEVEL(-2) daemon/daemon.go:225 processNextWorkItem {"worker-queue-size": 1} +2025-11-17T00:59:51.48253801Z LEVEL(-2) daemon/daemon.go:225 get item from queue {"item": 2} +2025-11-17T00:59:51.492736165Z INFO daemon/daemon.go:264 nodeStateSyncHandler(): new generation {"generation": 2} +2025-11-17T00:59:51.492760542Z INFO daemon/daemon.go:331 loadPlugins(): loading plugins +2025-11-17T00:59:51.493624528Z INFO daemon/daemon.go:331 loadPlugins(): loaded plugins {"plugins": ["mellanox", "intel", "k8s", "generic"]} +2025-11-17T00:59:51.493644864Z INFO runtime/asm_amd64.s:1695 Run(): refresh trigger +2025-11-17T00:59:51.493659399Z LEVEL(-2) daemon/writer.go:99 pollNicStatus() +2025-11-17T00:59:51.493683302Z LEVEL(-2) daemon/writer.go:130 DiscoverSriovDevices +2025-11-17T00:59:51.651457865Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib0"} +2025-11-17T00:59:51.65177688Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib0"} +2025-11-17T00:59:51.651809345Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib0/speed", "error": "read /sys/class/net/ib0/speed: invalid argument"} +2025-11-17T00:59:51.651820627Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib0"} +2025-11-17T00:59:51.651919153Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:19:00.0"} +2025-11-17T00:59:51.652024583Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib1"} +2025-11-17T00:59:51.652060955Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib1"} +2025-11-17T00:59:51.652075796Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib1/speed", "error": "read /sys/class/net/ib1/speed: invalid argument"} +2025-11-17T00:59:51.652081039Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib1"} +2025-11-17T00:59:51.652135525Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:29:00.0"} +2025-11-17T00:59:51.65223407Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib2"} +2025-11-17T00:59:51.652283638Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib2"} +2025-11-17T00:59:51.652591598Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib2"} +2025-11-17T00:59:51.652649113Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:3b:00.0"} +2025-11-17T00:59:51.652801692Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth2"} +2025-11-17T00:59:51.653432667Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth2"} +2025-11-17T00:59:51.653490268Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.0"} +2025-11-17T00:59:51.653617099Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth3"} +2025-11-17T00:59:51.654232365Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth3"} +2025-11-17T00:59:51.65429165Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.1"} +2025-11-17T00:59:51.654373917Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth0"} +2025-11-17T00:59:51.654418439Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth0"} +2025-11-17T00:59:51.65443549Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth0/speed", "error": "read /sys/class/net/eth0/speed: invalid argument"} +2025-11-17T00:59:51.654440713Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth0"} +2025-11-17T00:59:51.654493364Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.0"} +2025-11-17T00:59:51.654557678Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth1"} +2025-11-17T00:59:51.654598441Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth1"} +2025-11-17T00:59:51.654612631Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth1/speed", "error": "read /sys/class/net/eth1/speed: invalid argument"} +2025-11-17T00:59:51.654617764Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth1"} +2025-11-17T00:59:51.654670563Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.1"} +2025-11-17T00:59:51.654741889Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib3"} +2025-11-17T00:59:51.654785384Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib3"} +2025-11-17T00:59:51.654970092Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib3"} +2025-11-17T00:59:51.655028431Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:5c:00.0"} +2025-11-17T00:59:51.655107115Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib4"} +2025-11-17T00:59:51.655161874Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib4"} +2025-11-17T00:59:51.655346665Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib4"} +2025-11-17T00:59:51.655409423Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:85:00.0"} +2025-11-17T00:59:51.65549153Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib5"} +2025-11-17T00:59:51.655534542Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib5"} +2025-11-17T00:59:51.655724384Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib5"} +2025-11-17T00:59:51.655784227Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:8a:00.0"} +2025-11-17T00:59:51.655863831Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib6"} +2025-11-17T00:59:51.655914356Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib6"} +2025-11-17T00:59:51.655932055Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib6/speed", "error": "read /sys/class/net/ib6/speed: invalid argument"} +2025-11-17T00:59:51.655937476Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib6"} +2025-11-17T00:59:51.655991302Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:92:00.0"} +2025-11-17T00:59:51.656067198Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib7"} +2025-11-17T00:59:51.656115868Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib7"} +2025-11-17T00:59:51.656129586Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib7/speed", "error": "read /sys/class/net/ib7/speed: invalid argument"} +2025-11-17T00:59:51.656134479Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib7"} +2025-11-17T00:59:51.656186935Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:e3:00.0"} +2025-11-17T00:59:51.66626594Z INFO daemon/writer.go:166 setNodeStateStatus(): status {"sync-status": "InProgress", "last-sync-error": ""} +2025-11-17T00:59:51.700102408Z INFO daemon/daemon.go:264 nodeStateSyncHandler(): calling OnNodeStateChange for a new node state +2025-11-17T00:59:51.700143833Z INFO daemon/daemon.go:421 mellanox plugin OnNodeStateChange() +2025-11-17T00:59:51.700166088Z INFO kernel/kernel.go:304 RunCommand() {"command": "cat", "args": ["/host/sys/kernel/security/lockdown"]} +2025-11-17T00:59:51.70173717Z LEVEL(-2) kernel/kernel.go:304 RunCommand() {"output": "[none] integrity confidentiality\n", "error": null} +2025-11-17T00:59:51.701768702Z LEVEL(-2) mellanox/mellanox_plugin.go:86 IsKernelLockdownMode() {"output": "[none] integrity confidentiality\n", "error": null} +2025-11-17T00:59:51.701783549Z INFO mellanox/mellanox_plugin.go:102 mellanox-plugin getMlnxNicFwData() {"device": "0000:85:00.0"} +2025-11-17T00:59:51.701791757Z INFO mellanox/mellanox.go:208 MstConfigReadData() {"device": "0000:85:00.0"} +2025-11-17T00:59:51.701797927Z INFO mellanox/mellanox.go:81 RunCommand() {"command": "mstconfig", "args": ["-e", "-d", "0000:85:00.0", "q"]} +2025-11-17T01:00:16.615640949Z LEVEL(-2) mellanox/mellanox.go:81 RunCommand() {"output": "\nDevice #1:\n----------\n\nDevice type: ConnectX7 \nName: MCX75310AAS-NEA_Ax \nDescription: NVIDIA ConnectX-7 HHHL Adapter card; 400GbE / NDR IB (default mode); Single-port OSFP; PCIe 5.0 x16; Crypto Disabled; Secure Boot Enabled;\nDevice: 0000:85:00.0 \n\nConfigurations: Default Current Next Boot\n MODULE_SPLIT_M0 Array[0..15] Array[0..15] Array[0..15] \n MEMIC_BAR_SIZE 0 0 0 \n MEMIC_SIZE_LIMIT _256KB(1) _256KB(1) _256KB(1) \n MEMIC_ATOMIC MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2)\n HOST_CHAINING_MODE DISABLED(0) DISABLED(0) DISABLED(0) \n HOST_CHAINING_CACHE_DISABLE False(0) False(0) False(0) \n HOST_CHAINING_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n HOST_CHAINING_TOTAL_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n FLEX_PARSER_PROFILE_ENABLE 0 0 0 \n PROG_PARSE_GRAPH False(0) False(0) False(0) \n FLEX_IPV4_OVER_VXLAN_PORT 0 0 0 \n ROCE_NEXT_PROTOCOL 254 254 254 \n ESWITCH_HAIRPIN_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n ESWITCH_HAIRPIN_TOT_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n DPA_AUTHENTICATION True(1) True(1) True(1) \n PF_BAR2_SIZE 0 0 0 \n PF_NUM_OF_VF_VALID False(0) False(0) False(0) \n NON_PREFETCHABLE_PF_BAR False(0) False(0) False(0) \n VF_VPD_ENABLE False(0) False(0) False(0) \n PF_NUM_PF_MSIX_VALID False(0) False(0) False(0) \n PER_PF_NUM_SF False(0) False(0) False(0) \n STRICT_VF_MSIX_NUM False(0) False(0) False(0) \n VF_NODNIC_ENABLE False(0) False(0) False(0) \n NUM_PF_MSIX_VALID True(1) True(1) True(1) \n* NUM_OF_VFS 16 16 1 \n NUM_OF_PF 1 1 1 \n PF_BAR2_ENABLE False(0) False(0) False(0) \n SRIOV_EN True(1) True(1) True(1) \n PF_LOG_BAR_SIZE 5 5 5 \n VF_LOG_BAR_SIZE 1 1 1 \n NUM_PF_MSIX 63 63 63 \n NUM_VF_MSIX 11 11 11 \n INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0) AUTOMATIC(0) AUTOMATIC(0) \n PCIE_CREDIT_TOKEN_TIMEOUT 0 0 0 \n RT_PPS_ENABLED_ON_POWERUP False(0) False(0) False(0) \n LAG_RESOURCE_ALLOCATION DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ACCURATE_TX_SCHEDULER False(0) False(0) False(0) \n PARTIAL_RESET_EN False(0) False(0) False(0) \n RESET_WITH_HOST_ON_ERRORS False(0) False(0) False(0) \n PCI_SWITCH_EMULATION_NUM_PORT 16 16 16 \n PCI_SWITCH_EMULATION_ENABLE False(0) False(0) False(0) \n PCI_DOWNSTREAM_PORT_OWNER Array[0..15] Array[0..15] Array[0..15] \n CQE_COMPRESSION BALANCED(0) BALANCED(0) BALANCED(0) \n IP_OVER_VXLAN_EN False(0) False(0) False(0) \n MKEY_BY_NAME False(0) False(0) False(0) \n PRIO_TAG_REQUIRED_EN False(0) False(0) False(0) \n UCTX_EN True(1) True(1) True(1) \n REAL_TIME_CLOCK_ENABLE False(0) False(0) False(0) \n RDMA_SELECTIVE_REPEAT_EN False(0) False(0) False(0) \n PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)\n TUNNEL_ECN_COPY_DISABLE False(0) False(0) False(0) \n LRO_LOG_TIMEOUT0 6 6 6 \n LRO_LOG_TIMEOUT1 7 7 7 \n LRO_LOG_TIMEOUT2 8 8 8 \n LRO_LOG_TIMEOUT3 13 13 13 \n LOG_TX_PSN_WINDOW 7 7 7 \n VF_MIGRATION_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n LOG_MAX_OUTSTANDING_WQE 7 7 7 \n ROCE_ADAPTIVE_ROUTING_EN False(0) False(0) False(0) \n TUNNEL_IP_PROTO_ENTROPY_DISABLE False(0) False(0) False(0) \n PCC_HANDLE_CORE_UTIL DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PCC_INT_NP_RTT_DSCP 26 26 26 \n PCC_INT_NP_RTT_DSCP_EN False(0) False(0) False(0) \n PCC_INT_NP_RTT_DATA_MODE RTT_V0(64) RTT_V0(64) RTT_V0(64) \n PCC_INT_EN False(0) False(0) False(0) \n PCC_INT_SYSTEM_RTT 0 0 0 \n STEERING_CACHE_REFRESH 0 0 0 \n ICM_CACHE_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n HAIRPIN_DATA_BUFFER_LOCK False(0) False(0) False(0) \n TX_SCHEDULER_BURST 0 0 0 \n ZERO_TOUCH_TUNING_ENABLE False(0) False(0) False(0) \n LOG_MAX_QUEUE 17 17 17 \n LARGE_MTU_TWEAK_64 False(0) False(0) False(0) \n AES_XTS_TWEAK_INC_64 False(0) False(0) False(0) \n* CRYPTO_POLICY UNRESTRICTED(1) DEVICE_DEFAULT(0) UNRESTRICTED(1) \n RDE_DISABLE False(0) False(0) False(0) \n PLDM_FW_UPDATE_DISABLE False(0) False(0) False(0) \n RBT_DISABLE False(0) False(0) False(0) \n PCIE_SMBUS_DISABLE False(0) False(0) False(0) \n PCIE_IN_BAND_VDM_DISABLE False(0) False(0) False(0) \n ADVANCED_TESTABILITY False(0) False(0) False(0) \n LOG_DCR_HASH_TABLE_SIZE 11 11 11 \n MAX_PACKET_LIFETIME 0 0 0 \n DCR_LIFO_SIZE 16384 16384 16384 \n LINK_TYPE_P1 IB(1) IB(1) IB(1) \n ROCE_CC_PRIO_MASK_P1 255 255 255 \n ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ROCE_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n IB_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1) True(1) True(1) \n CLAMP_TGT_RATE_P1 False(0) False(0) False(0) \n RPG_TIME_RESET_P1 300 300 300 \n RPG_BYTE_RESET_P1 32767 32767 32767 \n RPG_THRESHOLD_P1 1 1 1 \n RPG_MAX_RATE_P1 0 0 0 \n RPG_AI_RATE_P1 5 5 5 \n RPG_HAI_RATE_P1 50 50 50 \n RPG_GD_P1 11 11 11 \n RPG_MIN_DEC_FAC_P1 50 50 50 \n RPG_MIN_RATE_P1 1 1 1 \n RATE_TO_SET_ON_FIRST_CNP_P1 0 0 0 \n DCE_TCP_G_P1 1019 1019 1019 \n DCE_TCP_RTT_P1 1 1 1 \n RATE_REDUCE_MONITOR_PERIOD_P1 4 4 4 \n INITIAL_ALPHA_VALUE_P1 1023 1023 1023 \n MIN_TIME_BETWEEN_CNPS_P1 4 4 4 \n CNP_802P_PRIO_P1 6 6 6 \n CNP_DSCP_P1 48 48 48 \n LLDP_NB_DCBX_P1 False(0) False(0) False(0) \n LLDP_NB_RX_MODE_P1 OFF(0) OFF(0) OFF(0) \n LLDP_NB_TX_MODE_P1 OFF(0) OFF(0) OFF(0) \n ROCE_RTT_RESP_DSCP_P1 0 0 0 \n ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n DCBX_IEEE_P1 True(1) True(1) True(1) \n DCBX_CEE_P1 True(1) True(1) True(1) \n DCBX_WILLING_P1 True(1) True(1) True(1) \n KEEP_ETH_LINK_UP_P1 True(1) True(1) True(1) \n KEEP_IB_LINK_UP_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_BOOT_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_STANDBY_P1 False(0) False(0) False(0) \n DO_NOT_CLEAR_PORT_STATS_P1 False(0) False(0) False(0) \n AUTO_POWER_SAVE_LINK_DOWN_P1 False(0) False(0) False(0) \n NUM_OF_VL_P1 _4_VLs(3) _4_VLs(3) _4_VLs(3) \n NUM_OF_TC_P1 _8_TCs(0) _8_TCs(0) _8_TCs(0) \n NUM_OF_PFC_P1 8 8 8 \n VL15_BUFFER_SIZE_P1 0 0 0 \n QOS_TRUST_STATE_P1 TRUST_PCP(1) TRUST_PCP(1) TRUST_PCP(1) \n ETS_SCHED_MODE_P1 device_default(0) device_default(0) device_default(0) \n DUP_MAC_ACTION_P1 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) \n MPFS_MC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n MPFS_UC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n SRIOV_IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n PHY_AUTO_NEG_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PHY_RATE_MASK_OVERRIDE_P1 False(0) False(0) False(0) \n PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PF_TOTAL_SF 0 0 0 \n PF_SD_GROUP 0 0 0 \n PF_SF_BAR_SIZE 0 0 0 \n PF_NUM_PF_MSIX 63 63 63 \n SILENT_MODE False(0) False(0) False(0) \n MKEY_BY_NAME_RANGE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n* ROCE_CONTROL ROCE_ENABLE(2) DEVICE_DEFAULT(0) ROCE_ENABLE(2) \n PCI_WR_ORDERING per_mkey(0) per_mkey(0) per_mkey(0) \n MULTI_PORT_VHCA_EN False(0) False(0) False(0) \n PORT_OWNER True(1) True(1) True(1) \n ALLOW_RD_COUNTERS True(1) True(1) True(1) \n RENEG_ON_CHANGE True(1) True(1) True(1) \n TRACER_ENABLE True(1) True(1) True(1) \n IP_VER IPv4(0) IPv4(0) IPv4(0) \n BOOT_UNDI_NETWORK_WAIT 0 0 0 \n UEFI_HII_EN True(1) True(1) True(1) \n BOOT_DBG_LOG False(0) False(0) False(0) \n UEFI_LOGS DISABLED(0) DISABLED(0) DISABLED(0) \n BOOT_VLAN 1 1 1 \n LEGACY_BOOT_PROTOCOL PXE(1) PXE(1) PXE(1) \n BOOT_INTERRUPT_DIS False(0) False(0) False(0) \n BOOT_LACP_DIS True(1) True(1) True(1) \n BOOT_VLAN_EN False(0) False(0) False(0) \n BOOT_PKEY 0 0 0 \n P2P_ORDERING_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ATS_ENABLED False(0) False(0) False(0) \n DYNAMIC_VF_MSIX_TABLE False(0) False(0) False(0) \n EXP_ROM_UEFI_ARM_ENABLE True(1) True(1) True(1) \n EXP_ROM_UEFI_x86_ENABLE True(1) True(1) True(1) \n EXP_ROM_PXE_ENABLE True(1) True(1) True(1) \n ADVANCED_PCI_SETTINGS False(0) False(0) False(0) \n SAFE_MODE_THRESHOLD 10 10 10 \n SAFE_MODE_ENABLE True(1) True(1) True(1) \nThe '*' shows parameters with next value different from default/current value.\n", "error": null} +2025-11-17T01:00:16.615807877Z INFO mellanox/mellanox.go:213 ParseMstconfigOutput() {"attributes": ["NUM_OF_VFS", "SRIOV_EN", "LINK_TYPE_P1", "LINK_TYPE_P2"]} +2025-11-17T01:00:16.615862667Z INFO mellanox/mellanox.go:214 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"16","SRIOV_EN":"True(1)"}} +2025-11-17T01:00:16.615877772Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:00:16.615883646Z INFO mellanox/mellanox.go:219 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"1","SRIOV_EN":"True(1)"}} +2025-11-17T01:00:16.615889419Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:00:16.615893194Z INFO mellanox/mellanox_plugin.go:107 mellanox-plugin IsDualPort() {"address": "0000:85:00.0"} +2025-11-17T01:00:16.615901924Z LEVEL(-2) mellanox/mellanox_plugin.go:112 Changing TotalVfs, needs reboot {"current": 16, "requested": 1} +2025-11-17T01:00:16.615909974Z INFO mellanox/mellanox.go:343 mellanox-plugin isLinkTypeRequireChange() {"device": "0000:85:00.0"} +2025-11-17T01:00:16.615918724Z INFO mellanox/mellanox_plugin.go:102 mellanox-plugin getMlnxNicFwData() {"device": "0000:8a:00.0"} +2025-11-17T01:00:16.615923647Z INFO mellanox/mellanox.go:208 MstConfigReadData() {"device": "0000:8a:00.0"} +2025-11-17T01:00:16.615928652Z INFO mellanox/mellanox.go:81 RunCommand() {"command": "mstconfig", "args": ["-e", "-d", "0000:8a:00.0", "q"]} +2025-11-17T01:00:21.690785152Z LEVEL(-2) runtime/asm_amd64.s:1695 Run(): period refresh +2025-11-17T01:00:21.69082225Z LEVEL(-2) daemon/writer.go:109 pollNicStatus() +2025-11-17T01:00:21.69082916Z LEVEL(-2) daemon/writer.go:130 DiscoverSriovDevices +2025-11-17T01:00:21.789480064Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib0"} +2025-11-17T01:00:21.789690068Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib0"} +2025-11-17T01:00:21.789710025Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib0/speed", "error": "read /sys/class/net/ib0/speed: invalid argument"} +2025-11-17T01:00:21.789718479Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib0"} +2025-11-17T01:00:21.78978272Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:19:00.0"} +2025-11-17T01:00:21.789891362Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib1"} +2025-11-17T01:00:21.789933121Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib1"} +2025-11-17T01:00:21.789947271Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib1/speed", "error": "read /sys/class/net/ib1/speed: invalid argument"} +2025-11-17T01:00:21.789951592Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib1"} +2025-11-17T01:00:21.79000417Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:29:00.0"} +2025-11-17T01:00:21.790095769Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib2"} +2025-11-17T01:00:21.790140484Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib2"} +2025-11-17T01:00:21.790451259Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib2"} +2025-11-17T01:00:21.790594943Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:3b:00.0"} +2025-11-17T01:00:21.790765833Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth2"} +2025-11-17T01:00:21.791408754Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth2"} +2025-11-17T01:00:21.791472547Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.0"} +2025-11-17T01:00:21.791630074Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth3"} +2025-11-17T01:00:21.792284606Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth3"} +2025-11-17T01:00:21.792351694Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.1"} +2025-11-17T01:00:21.792440208Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth0"} +2025-11-17T01:00:21.792488762Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth0"} +2025-11-17T01:00:21.792503123Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth0/speed", "error": "read /sys/class/net/eth0/speed: invalid argument"} +2025-11-17T01:00:21.792509393Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth0"} +2025-11-17T01:00:21.792567857Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.0"} +2025-11-17T01:00:21.792638161Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth1"} +2025-11-17T01:00:21.792684174Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth1"} +2025-11-17T01:00:21.792699424Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth1/speed", "error": "read /sys/class/net/eth1/speed: invalid argument"} +2025-11-17T01:00:21.792703512Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth1"} +2025-11-17T01:00:21.792761207Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.1"} +2025-11-17T01:00:21.792841345Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib3"} +2025-11-17T01:00:21.792900909Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib3"} +2025-11-17T01:00:21.793135959Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib3"} +2025-11-17T01:00:21.793203815Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:5c:00.0"} +2025-11-17T01:00:21.793297952Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib4"} +2025-11-17T01:00:21.793353047Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib4"} +2025-11-17T01:00:21.793593041Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib4"} +2025-11-17T01:00:21.793652785Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:85:00.0"} +2025-11-17T01:00:21.793735298Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib5"} +2025-11-17T01:00:21.793776859Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib5"} +2025-11-17T01:00:21.793868015Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib5"} +2025-11-17T01:00:21.793929225Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:8a:00.0"} +2025-11-17T01:00:21.794013481Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib6"} +2025-11-17T01:00:21.794055379Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib6"} +2025-11-17T01:00:21.794070615Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib6/speed", "error": "read /sys/class/net/ib6/speed: invalid argument"} +2025-11-17T01:00:21.794076417Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib6"} +2025-11-17T01:00:21.794137253Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:92:00.0"} +2025-11-17T01:00:21.794222303Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib7"} +2025-11-17T01:00:21.794265077Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib7"} +2025-11-17T01:00:21.794279847Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib7/speed", "error": "read /sys/class/net/ib7/speed: invalid argument"} +2025-11-17T01:00:21.794284601Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib7"} +2025-11-17T01:00:21.794340042Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:e3:00.0"} +2025-11-17T01:00:21.80554676Z INFO daemon/writer.go:166 setNodeStateStatus(): status {"sync-status": "InProgress", "last-sync-error": ""} +2025-11-17T01:00:39.907287069Z LEVEL(-2) mellanox/mellanox.go:81 RunCommand() {"output": "\nDevice #1:\n----------\n\nDevice type: ConnectX7 \nName: MCX75310AAS-NEA_Ax \nDescription: NVIDIA ConnectX-7 HHHL Adapter card; 400GbE / NDR IB (default mode); Single-port OSFP; PCIe 5.0 x16; Crypto Disabled; Secure Boot Enabled;\nDevice: 0000:8a:00.0 \n\nConfigurations: Default Current Next Boot\n MODULE_SPLIT_M0 Array[0..15] Array[0..15] Array[0..15] \n MEMIC_BAR_SIZE 0 0 0 \n MEMIC_SIZE_LIMIT _256KB(1) _256KB(1) _256KB(1) \n MEMIC_ATOMIC MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2)\n HOST_CHAINING_MODE DISABLED(0) DISABLED(0) DISABLED(0) \n HOST_CHAINING_CACHE_DISABLE False(0) False(0) False(0) \n HOST_CHAINING_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n HOST_CHAINING_TOTAL_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n FLEX_PARSER_PROFILE_ENABLE 0 0 0 \n PROG_PARSE_GRAPH False(0) False(0) False(0) \n FLEX_IPV4_OVER_VXLAN_PORT 0 0 0 \n ROCE_NEXT_PROTOCOL 254 254 254 \n ESWITCH_HAIRPIN_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n ESWITCH_HAIRPIN_TOT_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n DPA_AUTHENTICATION True(1) True(1) True(1) \n PF_BAR2_SIZE 0 0 0 \n PF_NUM_OF_VF_VALID False(0) False(0) False(0) \n NON_PREFETCHABLE_PF_BAR False(0) False(0) False(0) \n VF_VPD_ENABLE False(0) False(0) False(0) \n PF_NUM_PF_MSIX_VALID False(0) False(0) False(0) \n PER_PF_NUM_SF False(0) False(0) False(0) \n STRICT_VF_MSIX_NUM False(0) False(0) False(0) \n VF_NODNIC_ENABLE False(0) False(0) False(0) \n NUM_PF_MSIX_VALID True(1) True(1) True(1) \n* NUM_OF_VFS 16 16 1 \n NUM_OF_PF 1 1 1 \n PF_BAR2_ENABLE False(0) False(0) False(0) \n SRIOV_EN True(1) True(1) True(1) \n PF_LOG_BAR_SIZE 5 5 5 \n VF_LOG_BAR_SIZE 1 1 1 \n NUM_PF_MSIX 63 63 63 \n NUM_VF_MSIX 11 11 11 \n INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0) AUTOMATIC(0) AUTOMATIC(0) \n PCIE_CREDIT_TOKEN_TIMEOUT 0 0 0 \n RT_PPS_ENABLED_ON_POWERUP False(0) False(0) False(0) \n LAG_RESOURCE_ALLOCATION DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ACCURATE_TX_SCHEDULER False(0) False(0) False(0) \n PARTIAL_RESET_EN False(0) False(0) False(0) \n RESET_WITH_HOST_ON_ERRORS False(0) False(0) False(0) \n PCI_SWITCH_EMULATION_NUM_PORT 16 16 16 \n PCI_SWITCH_EMULATION_ENABLE False(0) False(0) False(0) \n PCI_DOWNSTREAM_PORT_OWNER Array[0..15] Array[0..15] Array[0..15] \n CQE_COMPRESSION BALANCED(0) BALANCED(0) BALANCED(0) \n IP_OVER_VXLAN_EN False(0) False(0) False(0) \n MKEY_BY_NAME False(0) False(0) False(0) \n PRIO_TAG_REQUIRED_EN False(0) False(0) False(0) \n UCTX_EN True(1) True(1) True(1) \n REAL_TIME_CLOCK_ENABLE False(0) False(0) False(0) \n RDMA_SELECTIVE_REPEAT_EN False(0) False(0) False(0) \n PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)\n TUNNEL_ECN_COPY_DISABLE False(0) False(0) False(0) \n LRO_LOG_TIMEOUT0 6 6 6 \n LRO_LOG_TIMEOUT1 7 7 7 \n LRO_LOG_TIMEOUT2 8 8 8 \n LRO_LOG_TIMEOUT3 13 13 13 \n LOG_TX_PSN_WINDOW 7 7 7 \n VF_MIGRATION_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n LOG_MAX_OUTSTANDING_WQE 7 7 7 \n ROCE_ADAPTIVE_ROUTING_EN False(0) False(0) False(0) \n TUNNEL_IP_PROTO_ENTROPY_DISABLE False(0) False(0) False(0) \n PCC_HANDLE_CORE_UTIL DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PCC_INT_NP_RTT_DSCP 26 26 26 \n PCC_INT_NP_RTT_DSCP_EN False(0) False(0) False(0) \n PCC_INT_NP_RTT_DATA_MODE RTT_V0(64) RTT_V0(64) RTT_V0(64) \n PCC_INT_EN False(0) False(0) False(0) \n PCC_INT_SYSTEM_RTT 0 0 0 \n STEERING_CACHE_REFRESH 0 0 0 \n ICM_CACHE_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n HAIRPIN_DATA_BUFFER_LOCK False(0) False(0) False(0) \n TX_SCHEDULER_BURST 0 0 0 \n ZERO_TOUCH_TUNING_ENABLE False(0) False(0) False(0) \n LOG_MAX_QUEUE 17 17 17 \n LARGE_MTU_TWEAK_64 False(0) False(0) False(0) \n AES_XTS_TWEAK_INC_64 False(0) False(0) False(0) \n* CRYPTO_POLICY UNRESTRICTED(1) DEVICE_DEFAULT(0) UNRESTRICTED(1) \n RDE_DISABLE False(0) False(0) False(0) \n PLDM_FW_UPDATE_DISABLE False(0) False(0) False(0) \n RBT_DISABLE False(0) False(0) False(0) \n PCIE_SMBUS_DISABLE False(0) False(0) False(0) \n PCIE_IN_BAND_VDM_DISABLE False(0) False(0) False(0) \n ADVANCED_TESTABILITY False(0) False(0) False(0) \n LOG_DCR_HASH_TABLE_SIZE 11 11 11 \n MAX_PACKET_LIFETIME 0 0 0 \n DCR_LIFO_SIZE 16384 16384 16384 \n LINK_TYPE_P1 IB(1) IB(1) IB(1) \n ROCE_CC_PRIO_MASK_P1 255 255 255 \n ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ROCE_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n IB_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1) True(1) True(1) \n CLAMP_TGT_RATE_P1 False(0) False(0) False(0) \n RPG_TIME_RESET_P1 300 300 300 \n RPG_BYTE_RESET_P1 32767 32767 32767 \n RPG_THRESHOLD_P1 1 1 1 \n RPG_MAX_RATE_P1 0 0 0 \n RPG_AI_RATE_P1 5 5 5 \n RPG_HAI_RATE_P1 50 50 50 \n RPG_GD_P1 11 11 11 \n RPG_MIN_DEC_FAC_P1 50 50 50 \n RPG_MIN_RATE_P1 1 1 1 \n RATE_TO_SET_ON_FIRST_CNP_P1 0 0 0 \n DCE_TCP_G_P1 1019 1019 1019 \n DCE_TCP_RTT_P1 1 1 1 \n RATE_REDUCE_MONITOR_PERIOD_P1 4 4 4 \n INITIAL_ALPHA_VALUE_P1 1023 1023 1023 \n MIN_TIME_BETWEEN_CNPS_P1 4 4 4 \n CNP_802P_PRIO_P1 6 6 6 \n CNP_DSCP_P1 48 48 48 \n LLDP_NB_DCBX_P1 False(0) False(0) False(0) \n LLDP_NB_RX_MODE_P1 OFF(0) OFF(0) OFF(0) \n LLDP_NB_TX_MODE_P1 OFF(0) OFF(0) OFF(0) \n ROCE_RTT_RESP_DSCP_P1 0 0 0 \n ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n DCBX_IEEE_P1 True(1) True(1) True(1) \n DCBX_CEE_P1 True(1) True(1) True(1) \n DCBX_WILLING_P1 True(1) True(1) True(1) \n KEEP_ETH_LINK_UP_P1 True(1) True(1) True(1) \n KEEP_IB_LINK_UP_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_BOOT_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_STANDBY_P1 False(0) False(0) False(0) \n DO_NOT_CLEAR_PORT_STATS_P1 False(0) False(0) False(0) \n AUTO_POWER_SAVE_LINK_DOWN_P1 False(0) False(0) False(0) \n NUM_OF_VL_P1 _4_VLs(3) _4_VLs(3) _4_VLs(3) \n NUM_OF_TC_P1 _8_TCs(0) _8_TCs(0) _8_TCs(0) \n NUM_OF_PFC_P1 8 8 8 \n VL15_BUFFER_SIZE_P1 0 0 0 \n QOS_TRUST_STATE_P1 TRUST_PCP(1) TRUST_PCP(1) TRUST_PCP(1) \n ETS_SCHED_MODE_P1 device_default(0) device_default(0) device_default(0) \n DUP_MAC_ACTION_P1 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) \n MPFS_MC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n MPFS_UC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n SRIOV_IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n PHY_AUTO_NEG_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PHY_RATE_MASK_OVERRIDE_P1 False(0) False(0) False(0) \n PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PF_TOTAL_SF 0 0 0 \n PF_SD_GROUP 0 0 0 \n PF_SF_BAR_SIZE 0 0 0 \n PF_NUM_PF_MSIX 63 63 63 \n SILENT_MODE False(0) False(0) False(0) \n MKEY_BY_NAME_RANGE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n* ROCE_CONTROL ROCE_ENABLE(2) DEVICE_DEFAULT(0) ROCE_ENABLE(2) \n PCI_WR_ORDERING per_mkey(0) per_mkey(0) per_mkey(0) \n MULTI_PORT_VHCA_EN False(0) False(0) False(0) \n PORT_OWNER True(1) True(1) True(1) \n ALLOW_RD_COUNTERS True(1) True(1) True(1) \n RENEG_ON_CHANGE True(1) True(1) True(1) \n TRACER_ENABLE True(1) True(1) True(1) \n IP_VER IPv4(0) IPv4(0) IPv4(0) \n BOOT_UNDI_NETWORK_WAIT 0 0 0 \n UEFI_HII_EN True(1) True(1) True(1) \n BOOT_DBG_LOG False(0) False(0) False(0) \n UEFI_LOGS DISABLED(0) DISABLED(0) DISABLED(0) \n BOOT_VLAN 1 1 1 \n LEGACY_BOOT_PROTOCOL PXE(1) PXE(1) PXE(1) \n BOOT_INTERRUPT_DIS False(0) False(0) False(0) \n BOOT_LACP_DIS True(1) True(1) True(1) \n BOOT_VLAN_EN False(0) False(0) False(0) \n BOOT_PKEY 0 0 0 \n P2P_ORDERING_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ATS_ENABLED False(0) False(0) False(0) \n DYNAMIC_VF_MSIX_TABLE False(0) False(0) False(0) \n EXP_ROM_UEFI_ARM_ENABLE True(1) True(1) True(1) \n EXP_ROM_UEFI_x86_ENABLE True(1) True(1) True(1) \n EXP_ROM_PXE_ENABLE True(1) True(1) True(1) \n ADVANCED_PCI_SETTINGS False(0) False(0) False(0) \n SAFE_MODE_THRESHOLD 10 10 10 \n SAFE_MODE_ENABLE True(1) True(1) True(1) \nThe '*' shows parameters with next value different from default/current value.\n", "error": null} +2025-11-17T01:00:39.907437047Z INFO mellanox/mellanox.go:213 ParseMstconfigOutput() {"attributes": ["NUM_OF_VFS", "SRIOV_EN", "LINK_TYPE_P1", "LINK_TYPE_P2"]} +2025-11-17T01:00:39.907487454Z INFO mellanox/mellanox.go:214 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"16","SRIOV_EN":"True(1)"}} +2025-11-17T01:00:39.907506598Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:00:39.907510883Z INFO mellanox/mellanox.go:219 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"1","SRIOV_EN":"True(1)"}} +2025-11-17T01:00:39.90751561Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:00:39.907520961Z INFO mellanox/mellanox_plugin.go:107 mellanox-plugin IsDualPort() {"address": "0000:8a:00.0"} +2025-11-17T01:00:39.907527198Z LEVEL(-2) mellanox/mellanox_plugin.go:112 Changing TotalVfs, needs reboot {"current": 16, "requested": 1} +2025-11-17T01:00:39.907534304Z INFO mellanox/mellanox.go:343 mellanox-plugin isLinkTypeRequireChange() {"device": "0000:8a:00.0"} +2025-11-17T01:00:39.90754034Z INFO mellanox/mellanox_plugin.go:102 mellanox-plugin getMlnxNicFwData() {"device": "0000:92:00.0"} +2025-11-17T01:00:39.907544912Z INFO mellanox/mellanox.go:208 MstConfigReadData() {"device": "0000:92:00.0"} +2025-11-17T01:00:39.907554847Z INFO mellanox/mellanox.go:81 RunCommand() {"command": "mstconfig", "args": ["-e", "-d", "0000:92:00.0", "q"]} +2025-11-17T01:00:51.820519109Z LEVEL(-2) runtime/asm_amd64.s:1695 Run(): period refresh +2025-11-17T01:00:51.820578916Z LEVEL(-2) daemon/writer.go:109 pollNicStatus() +2025-11-17T01:00:51.82058592Z LEVEL(-2) daemon/writer.go:130 DiscoverSriovDevices +2025-11-17T01:00:51.926311479Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib0"} +2025-11-17T01:00:51.926509244Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib0"} +2025-11-17T01:00:51.926530085Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib0/speed", "error": "read /sys/class/net/ib0/speed: invalid argument"} +2025-11-17T01:00:51.926539371Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib0"} +2025-11-17T01:00:51.926606148Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:19:00.0"} +2025-11-17T01:00:51.92671356Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib1"} +2025-11-17T01:00:51.926804146Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib1"} +2025-11-17T01:00:51.926819025Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib1/speed", "error": "read /sys/class/net/ib1/speed: invalid argument"} +2025-11-17T01:00:51.926824667Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib1"} +2025-11-17T01:00:51.926881886Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:29:00.0"} +2025-11-17T01:00:51.926975615Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib2"} +2025-11-17T01:00:51.92706782Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib2"} +2025-11-17T01:00:51.92716798Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib2"} +2025-11-17T01:00:51.927226876Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:3b:00.0"} +2025-11-17T01:00:51.927439535Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth2"} +2025-11-17T01:00:51.92801106Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth2"} +2025-11-17T01:00:51.92807487Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.0"} +2025-11-17T01:00:51.928286193Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth3"} +2025-11-17T01:00:51.928879252Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth3"} +2025-11-17T01:00:51.929117362Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.1"} +2025-11-17T01:00:51.929248512Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth0"} +2025-11-17T01:00:51.929332537Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth0"} +2025-11-17T01:00:51.929350006Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth0/speed", "error": "read /sys/class/net/eth0/speed: invalid argument"} +2025-11-17T01:00:51.92935947Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth0"} +2025-11-17T01:00:51.929429491Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.0"} +2025-11-17T01:00:51.929504489Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth1"} +2025-11-17T01:00:51.929577957Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth1"} +2025-11-17T01:00:51.929592632Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth1/speed", "error": "read /sys/class/net/eth1/speed: invalid argument"} +2025-11-17T01:00:51.929597665Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth1"} +2025-11-17T01:00:51.929660264Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.1"} +2025-11-17T01:00:51.929736819Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib3"} +2025-11-17T01:00:51.929836549Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib3"} +2025-11-17T01:00:51.929938912Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib3"} +2025-11-17T01:00:51.930006228Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:5c:00.0"} +2025-11-17T01:00:51.93010338Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib4"} +2025-11-17T01:00:51.930187225Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib4"} +2025-11-17T01:00:51.93027948Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib4"} +2025-11-17T01:00:51.930343439Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:85:00.0"} +2025-11-17T01:00:51.930428192Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib5"} +2025-11-17T01:00:51.930495226Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib5"} +2025-11-17T01:00:51.930589779Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib5"} +2025-11-17T01:00:51.930650965Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:8a:00.0"} +2025-11-17T01:00:51.930735354Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib6"} +2025-11-17T01:00:51.930809886Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib6"} +2025-11-17T01:00:51.930825175Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib6/speed", "error": "read /sys/class/net/ib6/speed: invalid argument"} +2025-11-17T01:00:51.930831913Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib6"} +2025-11-17T01:00:51.930890876Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:92:00.0"} +2025-11-17T01:00:51.930974592Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib7"} +2025-11-17T01:00:51.931047431Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib7"} +2025-11-17T01:00:51.931062064Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib7/speed", "error": "read /sys/class/net/ib7/speed: invalid argument"} +2025-11-17T01:00:51.931067641Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib7"} +2025-11-17T01:00:51.931126459Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:e3:00.0"} +2025-11-17T01:00:51.941884132Z INFO daemon/writer.go:166 setNodeStateStatus(): status {"sync-status": "InProgress", "last-sync-error": ""} +2025-11-17T01:01:03.498419319Z LEVEL(-2) mellanox/mellanox.go:81 RunCommand() {"output": "\nDevice #1:\n----------\n\nDevice type: ConnectX7 \nName: MCX75310AAS-NEA_Ax \nDescription: NVIDIA ConnectX-7 HHHL Adapter card; 400GbE / NDR IB (default mode); Single-port OSFP; PCIe 5.0 x16; Crypto Disabled; Secure Boot Enabled;\nDevice: 0000:92:00.0 \n\nConfigurations: Default Current Next Boot\n MODULE_SPLIT_M0 Array[0..15] Array[0..15] Array[0..15] \n MEMIC_BAR_SIZE 0 0 0 \n MEMIC_SIZE_LIMIT _256KB(1) _256KB(1) _256KB(1) \n MEMIC_ATOMIC MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2)\n HOST_CHAINING_MODE DISABLED(0) DISABLED(0) DISABLED(0) \n HOST_CHAINING_CACHE_DISABLE False(0) False(0) False(0) \n HOST_CHAINING_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n HOST_CHAINING_TOTAL_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n FLEX_PARSER_PROFILE_ENABLE 0 0 0 \n PROG_PARSE_GRAPH False(0) False(0) False(0) \n FLEX_IPV4_OVER_VXLAN_PORT 0 0 0 \n ROCE_NEXT_PROTOCOL 254 254 254 \n ESWITCH_HAIRPIN_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n ESWITCH_HAIRPIN_TOT_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n DPA_AUTHENTICATION True(1) True(1) True(1) \n PF_BAR2_SIZE 0 0 0 \n PF_NUM_OF_VF_VALID False(0) False(0) False(0) \n NON_PREFETCHABLE_PF_BAR False(0) False(0) False(0) \n VF_VPD_ENABLE False(0) False(0) False(0) \n PF_NUM_PF_MSIX_VALID False(0) False(0) False(0) \n PER_PF_NUM_SF False(0) False(0) False(0) \n STRICT_VF_MSIX_NUM False(0) False(0) False(0) \n VF_NODNIC_ENABLE False(0) False(0) False(0) \n NUM_PF_MSIX_VALID True(1) True(1) True(1) \n* NUM_OF_VFS 16 16 1 \n NUM_OF_PF 1 1 1 \n PF_BAR2_ENABLE False(0) False(0) False(0) \n SRIOV_EN True(1) True(1) True(1) \n PF_LOG_BAR_SIZE 5 5 5 \n VF_LOG_BAR_SIZE 1 1 1 \n NUM_PF_MSIX 63 63 63 \n NUM_VF_MSIX 11 11 11 \n INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0) AUTOMATIC(0) AUTOMATIC(0) \n PCIE_CREDIT_TOKEN_TIMEOUT 0 0 0 \n RT_PPS_ENABLED_ON_POWERUP False(0) False(0) False(0) \n LAG_RESOURCE_ALLOCATION DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ACCURATE_TX_SCHEDULER False(0) False(0) False(0) \n PARTIAL_RESET_EN False(0) False(0) False(0) \n RESET_WITH_HOST_ON_ERRORS False(0) False(0) False(0) \n PCI_SWITCH_EMULATION_NUM_PORT 16 16 16 \n PCI_SWITCH_EMULATION_ENABLE False(0) False(0) False(0) \n PCI_DOWNSTREAM_PORT_OWNER Array[0..15] Array[0..15] Array[0..15] \n CQE_COMPRESSION BALANCED(0) BALANCED(0) BALANCED(0) \n IP_OVER_VXLAN_EN False(0) False(0) False(0) \n MKEY_BY_NAME False(0) False(0) False(0) \n PRIO_TAG_REQUIRED_EN False(0) False(0) False(0) \n UCTX_EN True(1) True(1) True(1) \n REAL_TIME_CLOCK_ENABLE False(0) False(0) False(0) \n RDMA_SELECTIVE_REPEAT_EN False(0) False(0) False(0) \n PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)\n TUNNEL_ECN_COPY_DISABLE False(0) False(0) False(0) \n LRO_LOG_TIMEOUT0 6 6 6 \n LRO_LOG_TIMEOUT1 7 7 7 \n LRO_LOG_TIMEOUT2 8 8 8 \n LRO_LOG_TIMEOUT3 13 13 13 \n LOG_TX_PSN_WINDOW 7 7 7 \n VF_MIGRATION_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n LOG_MAX_OUTSTANDING_WQE 7 7 7 \n ROCE_ADAPTIVE_ROUTING_EN False(0) False(0) False(0) \n TUNNEL_IP_PROTO_ENTROPY_DISABLE False(0) False(0) False(0) \n PCC_HANDLE_CORE_UTIL DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PCC_INT_NP_RTT_DSCP 26 26 26 \n PCC_INT_NP_RTT_DSCP_EN False(0) False(0) False(0) \n PCC_INT_NP_RTT_DATA_MODE RTT_V0(64) RTT_V0(64) RTT_V0(64) \n PCC_INT_EN False(0) False(0) False(0) \n PCC_INT_SYSTEM_RTT 0 0 0 \n STEERING_CACHE_REFRESH 0 0 0 \n ICM_CACHE_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n HAIRPIN_DATA_BUFFER_LOCK False(0) False(0) False(0) \n TX_SCHEDULER_BURST 0 0 0 \n ZERO_TOUCH_TUNING_ENABLE False(0) False(0) False(0) \n LOG_MAX_QUEUE 17 17 17 \n LARGE_MTU_TWEAK_64 False(0) False(0) False(0) \n AES_XTS_TWEAK_INC_64 False(0) False(0) False(0) \n* CRYPTO_POLICY UNRESTRICTED(1) DEVICE_DEFAULT(0) UNRESTRICTED(1) \n RDE_DISABLE False(0) False(0) False(0) \n PLDM_FW_UPDATE_DISABLE False(0) False(0) False(0) \n RBT_DISABLE False(0) False(0) False(0) \n PCIE_SMBUS_DISABLE False(0) False(0) False(0) \n PCIE_IN_BAND_VDM_DISABLE False(0) False(0) False(0) \n ADVANCED_TESTABILITY False(0) False(0) False(0) \n LOG_DCR_HASH_TABLE_SIZE 11 11 11 \n MAX_PACKET_LIFETIME 0 0 0 \n DCR_LIFO_SIZE 16384 16384 16384 \n LINK_TYPE_P1 IB(1) IB(1) IB(1) \n ROCE_CC_PRIO_MASK_P1 255 255 255 \n ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ROCE_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n IB_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1) True(1) True(1) \n CLAMP_TGT_RATE_P1 False(0) False(0) False(0) \n RPG_TIME_RESET_P1 300 300 300 \n RPG_BYTE_RESET_P1 32767 32767 32767 \n RPG_THRESHOLD_P1 1 1 1 \n RPG_MAX_RATE_P1 0 0 0 \n RPG_AI_RATE_P1 5 5 5 \n RPG_HAI_RATE_P1 50 50 50 \n RPG_GD_P1 11 11 11 \n RPG_MIN_DEC_FAC_P1 50 50 50 \n RPG_MIN_RATE_P1 1 1 1 \n RATE_TO_SET_ON_FIRST_CNP_P1 0 0 0 \n DCE_TCP_G_P1 1019 1019 1019 \n DCE_TCP_RTT_P1 1 1 1 \n RATE_REDUCE_MONITOR_PERIOD_P1 4 4 4 \n INITIAL_ALPHA_VALUE_P1 1023 1023 1023 \n MIN_TIME_BETWEEN_CNPS_P1 4 4 4 \n CNP_802P_PRIO_P1 6 6 6 \n CNP_DSCP_P1 48 48 48 \n LLDP_NB_DCBX_P1 False(0) False(0) False(0) \n LLDP_NB_RX_MODE_P1 OFF(0) OFF(0) OFF(0) \n LLDP_NB_TX_MODE_P1 OFF(0) OFF(0) OFF(0) \n ROCE_RTT_RESP_DSCP_P1 0 0 0 \n ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n DCBX_IEEE_P1 True(1) True(1) True(1) \n DCBX_CEE_P1 True(1) True(1) True(1) \n DCBX_WILLING_P1 True(1) True(1) True(1) \n KEEP_ETH_LINK_UP_P1 True(1) True(1) True(1) \n KEEP_IB_LINK_UP_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_BOOT_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_STANDBY_P1 False(0) False(0) False(0) \n DO_NOT_CLEAR_PORT_STATS_P1 False(0) False(0) False(0) \n AUTO_POWER_SAVE_LINK_DOWN_P1 False(0) False(0) False(0) \n NUM_OF_VL_P1 _4_VLs(3) _4_VLs(3) _4_VLs(3) \n NUM_OF_TC_P1 _8_TCs(0) _8_TCs(0) _8_TCs(0) \n NUM_OF_PFC_P1 8 8 8 \n VL15_BUFFER_SIZE_P1 0 0 0 \n QOS_TRUST_STATE_P1 TRUST_PCP(1) TRUST_PCP(1) TRUST_PCP(1) \n ETS_SCHED_MODE_P1 device_default(0) device_default(0) device_default(0) \n DUP_MAC_ACTION_P1 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) \n MPFS_MC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n MPFS_UC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n SRIOV_IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n PHY_AUTO_NEG_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PHY_RATE_MASK_OVERRIDE_P1 False(0) False(0) False(0) \n PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PF_TOTAL_SF 0 0 0 \n PF_SD_GROUP 0 0 0 \n PF_SF_BAR_SIZE 0 0 0 \n PF_NUM_PF_MSIX 63 63 63 \n SILENT_MODE False(0) False(0) False(0) \n MKEY_BY_NAME_RANGE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n* ROCE_CONTROL ROCE_ENABLE(2) DEVICE_DEFAULT(0) ROCE_ENABLE(2) \n PCI_WR_ORDERING per_mkey(0) per_mkey(0) per_mkey(0) \n MULTI_PORT_VHCA_EN False(0) False(0) False(0) \n PORT_OWNER True(1) True(1) True(1) \n ALLOW_RD_COUNTERS True(1) True(1) True(1) \n RENEG_ON_CHANGE True(1) True(1) True(1) \n TRACER_ENABLE True(1) True(1) True(1) \n IP_VER IPv4(0) IPv4(0) IPv4(0) \n BOOT_UNDI_NETWORK_WAIT 0 0 0 \n UEFI_HII_EN True(1) True(1) True(1) \n BOOT_DBG_LOG False(0) False(0) False(0) \n UEFI_LOGS DISABLED(0) DISABLED(0) DISABLED(0) \n BOOT_VLAN 1 1 1 \n LEGACY_BOOT_PROTOCOL PXE(1) PXE(1) PXE(1) \n BOOT_INTERRUPT_DIS False(0) False(0) False(0) \n BOOT_LACP_DIS True(1) True(1) True(1) \n BOOT_VLAN_EN False(0) False(0) False(0) \n BOOT_PKEY 0 0 0 \n P2P_ORDERING_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ATS_ENABLED False(0) False(0) False(0) \n DYNAMIC_VF_MSIX_TABLE False(0) False(0) False(0) \n EXP_ROM_UEFI_ARM_ENABLE True(1) True(1) True(1) \n EXP_ROM_UEFI_x86_ENABLE True(1) True(1) True(1) \n EXP_ROM_PXE_ENABLE True(1) True(1) True(1) \n ADVANCED_PCI_SETTINGS False(0) False(0) False(0) \n SAFE_MODE_THRESHOLD 10 10 10 \n SAFE_MODE_ENABLE True(1) True(1) True(1) \nThe '*' shows parameters with next value different from default/current value.\n", "error": null} +2025-11-17T01:01:03.498537882Z INFO mellanox/mellanox.go:213 ParseMstconfigOutput() {"attributes": ["NUM_OF_VFS", "SRIOV_EN", "LINK_TYPE_P1", "LINK_TYPE_P2"]} +2025-11-17T01:01:03.498590848Z INFO mellanox/mellanox.go:214 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"16","SRIOV_EN":"True(1)"}} +2025-11-17T01:01:03.49861004Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:01:03.498613855Z INFO mellanox/mellanox.go:219 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"1","SRIOV_EN":"True(1)"}} +2025-11-17T01:01:03.498619122Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:01:03.498622714Z INFO mellanox/mellanox_plugin.go:107 mellanox-plugin IsDualPort() {"address": "0000:92:00.0"} +2025-11-17T01:01:03.498630457Z LEVEL(-2) mellanox/mellanox_plugin.go:112 Changing TotalVfs, needs reboot {"current": 16, "requested": 1} +2025-11-17T01:01:03.498637594Z INFO mellanox/mellanox.go:343 mellanox-plugin isLinkTypeRequireChange() {"device": "0000:92:00.0"} +2025-11-17T01:01:03.498644176Z INFO mellanox/mellanox_plugin.go:102 mellanox-plugin getMlnxNicFwData() {"device": "0000:e3:00.0"} +2025-11-17T01:01:03.498648601Z INFO mellanox/mellanox.go:208 MstConfigReadData() {"device": "0000:e3:00.0"} +2025-11-17T01:01:03.49865965Z INFO mellanox/mellanox.go:81 RunCommand() {"command": "mstconfig", "args": ["-e", "-d", "0000:e3:00.0", "q"]} +2025-11-17T01:01:21.956182426Z LEVEL(-2) runtime/asm_amd64.s:1695 Run(): period refresh +2025-11-17T01:01:21.9562125Z LEVEL(-2) daemon/writer.go:109 pollNicStatus() +2025-11-17T01:01:21.956219243Z LEVEL(-2) daemon/writer.go:130 DiscoverSriovDevices +2025-11-17T01:01:22.064793713Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib0"} +2025-11-17T01:01:22.065031112Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib0"} +2025-11-17T01:01:22.065053741Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib0/speed", "error": "read /sys/class/net/ib0/speed: invalid argument"} +2025-11-17T01:01:22.065064108Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib0"} +2025-11-17T01:01:22.065134801Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:19:00.0"} +2025-11-17T01:01:22.065243888Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib1"} +2025-11-17T01:01:22.065278995Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib1"} +2025-11-17T01:01:22.065296022Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib1/speed", "error": "read /sys/class/net/ib1/speed: invalid argument"} +2025-11-17T01:01:22.065301382Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib1"} +2025-11-17T01:01:22.065366772Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:29:00.0"} +2025-11-17T01:01:22.065475557Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib2"} +2025-11-17T01:01:22.06552379Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib2"} +2025-11-17T01:01:22.065905693Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib2"} +2025-11-17T01:01:22.066053136Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:3b:00.0"} +2025-11-17T01:01:22.066262514Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth2"} +2025-11-17T01:01:22.066943603Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth2"} +2025-11-17T01:01:22.067023029Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.0"} +2025-11-17T01:01:22.067209202Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth3"} +2025-11-17T01:01:22.067836113Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth3"} +2025-11-17T01:01:22.067905794Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.1"} +2025-11-17T01:01:22.06799878Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth0"} +2025-11-17T01:01:22.068046443Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth0"} +2025-11-17T01:01:22.068062179Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth0/speed", "error": "read /sys/class/net/eth0/speed: invalid argument"} +2025-11-17T01:01:22.068069082Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth0"} +2025-11-17T01:01:22.068143783Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.0"} +2025-11-17T01:01:22.068233734Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth1"} +2025-11-17T01:01:22.068277321Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth1"} +2025-11-17T01:01:22.068293345Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth1/speed", "error": "read /sys/class/net/eth1/speed: invalid argument"} +2025-11-17T01:01:22.068303782Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth1"} +2025-11-17T01:01:22.068362733Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.1"} +2025-11-17T01:01:22.068450878Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib3"} +2025-11-17T01:01:22.068505129Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib3"} +2025-11-17T01:01:22.068595012Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib3"} +2025-11-17T01:01:22.068656234Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:5c:00.0"} +2025-11-17T01:01:22.068767457Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib4"} +2025-11-17T01:01:22.068828029Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib4"} +2025-11-17T01:01:22.068919404Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib4"} +2025-11-17T01:01:22.068979983Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:85:00.0"} +2025-11-17T01:01:22.069063372Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib5"} +2025-11-17T01:01:22.069117752Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib5"} +2025-11-17T01:01:22.069217352Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib5"} +2025-11-17T01:01:22.069279113Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:8a:00.0"} +2025-11-17T01:01:22.06936444Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib6"} +2025-11-17T01:01:22.069422457Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib6"} +2025-11-17T01:01:22.069438407Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib6/speed", "error": "read /sys/class/net/ib6/speed: invalid argument"} +2025-11-17T01:01:22.069445249Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib6"} +2025-11-17T01:01:22.06950334Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:92:00.0"} +2025-11-17T01:01:22.069616855Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib7"} +2025-11-17T01:01:22.06966502Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib7"} +2025-11-17T01:01:22.069680856Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib7/speed", "error": "read /sys/class/net/ib7/speed: invalid argument"} +2025-11-17T01:01:22.069686574Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib7"} +2025-11-17T01:01:22.069750872Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:e3:00.0"} +2025-11-17T01:01:22.080242404Z INFO daemon/writer.go:166 setNodeStateStatus(): status {"sync-status": "InProgress", "last-sync-error": ""} +2025-11-17T01:01:27.034612027Z LEVEL(-2) mellanox/mellanox.go:81 RunCommand() {"output": "\nDevice #1:\n----------\n\nDevice type: ConnectX7 \nName: MCX75310AAS-NEA_Ax \nDescription: NVIDIA ConnectX-7 HHHL Adapter card; 400GbE / NDR IB (default mode); Single-port OSFP; PCIe 5.0 x16; Crypto Disabled; Secure Boot Enabled;\nDevice: 0000:e3:00.0 \n\nConfigurations: Default Current Next Boot\n MODULE_SPLIT_M0 Array[0..15] Array[0..15] Array[0..15] \n MEMIC_BAR_SIZE 0 0 0 \n MEMIC_SIZE_LIMIT _256KB(1) _256KB(1) _256KB(1) \n MEMIC_ATOMIC MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2)\n HOST_CHAINING_MODE DISABLED(0) DISABLED(0) DISABLED(0) \n HOST_CHAINING_CACHE_DISABLE False(0) False(0) False(0) \n HOST_CHAINING_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n HOST_CHAINING_TOTAL_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n FLEX_PARSER_PROFILE_ENABLE 0 0 0 \n PROG_PARSE_GRAPH False(0) False(0) False(0) \n FLEX_IPV4_OVER_VXLAN_PORT 0 0 0 \n ROCE_NEXT_PROTOCOL 254 254 254 \n ESWITCH_HAIRPIN_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n ESWITCH_HAIRPIN_TOT_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n DPA_AUTHENTICATION True(1) True(1) True(1) \n PF_BAR2_SIZE 0 0 0 \n PF_NUM_OF_VF_VALID False(0) False(0) False(0) \n NON_PREFETCHABLE_PF_BAR False(0) False(0) False(0) \n VF_VPD_ENABLE False(0) False(0) False(0) \n PF_NUM_PF_MSIX_VALID False(0) False(0) False(0) \n PER_PF_NUM_SF False(0) False(0) False(0) \n STRICT_VF_MSIX_NUM False(0) False(0) False(0) \n VF_NODNIC_ENABLE False(0) False(0) False(0) \n NUM_PF_MSIX_VALID True(1) True(1) True(1) \n* NUM_OF_VFS 16 16 1 \n NUM_OF_PF 1 1 1 \n PF_BAR2_ENABLE False(0) False(0) False(0) \n SRIOV_EN True(1) True(1) True(1) \n PF_LOG_BAR_SIZE 5 5 5 \n VF_LOG_BAR_SIZE 1 1 1 \n NUM_PF_MSIX 63 63 63 \n NUM_VF_MSIX 11 11 11 \n INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0) AUTOMATIC(0) AUTOMATIC(0) \n PCIE_CREDIT_TOKEN_TIMEOUT 0 0 0 \n RT_PPS_ENABLED_ON_POWERUP False(0) False(0) False(0) \n LAG_RESOURCE_ALLOCATION DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ACCURATE_TX_SCHEDULER False(0) False(0) False(0) \n PARTIAL_RESET_EN False(0) False(0) False(0) \n RESET_WITH_HOST_ON_ERRORS False(0) False(0) False(0) \n PCI_SWITCH_EMULATION_NUM_PORT 16 16 16 \n PCI_SWITCH_EMULATION_ENABLE False(0) False(0) False(0) \n PCI_DOWNSTREAM_PORT_OWNER Array[0..15] Array[0..15] Array[0..15] \n CQE_COMPRESSION BALANCED(0) BALANCED(0) BALANCED(0) \n IP_OVER_VXLAN_EN False(0) False(0) False(0) \n MKEY_BY_NAME False(0) False(0) False(0) \n PRIO_TAG_REQUIRED_EN False(0) False(0) False(0) \n UCTX_EN True(1) True(1) True(1) \n REAL_TIME_CLOCK_ENABLE False(0) False(0) False(0) \n RDMA_SELECTIVE_REPEAT_EN False(0) False(0) False(0) \n PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)\n TUNNEL_ECN_COPY_DISABLE False(0) False(0) False(0) \n LRO_LOG_TIMEOUT0 6 6 6 \n LRO_LOG_TIMEOUT1 7 7 7 \n LRO_LOG_TIMEOUT2 8 8 8 \n LRO_LOG_TIMEOUT3 13 13 13 \n LOG_TX_PSN_WINDOW 7 7 7 \n VF_MIGRATION_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n LOG_MAX_OUTSTANDING_WQE 7 7 7 \n ROCE_ADAPTIVE_ROUTING_EN False(0) False(0) False(0) \n TUNNEL_IP_PROTO_ENTROPY_DISABLE False(0) False(0) False(0) \n PCC_HANDLE_CORE_UTIL DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PCC_INT_NP_RTT_DSCP 26 26 26 \n PCC_INT_NP_RTT_DSCP_EN False(0) False(0) False(0) \n PCC_INT_NP_RTT_DATA_MODE RTT_V0(64) RTT_V0(64) RTT_V0(64) \n PCC_INT_EN False(0) False(0) False(0) \n PCC_INT_SYSTEM_RTT 0 0 0 \n STEERING_CACHE_REFRESH 0 0 0 \n ICM_CACHE_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n HAIRPIN_DATA_BUFFER_LOCK False(0) False(0) False(0) \n TX_SCHEDULER_BURST 0 0 0 \n ZERO_TOUCH_TUNING_ENABLE False(0) False(0) False(0) \n LOG_MAX_QUEUE 17 17 17 \n LARGE_MTU_TWEAK_64 False(0) False(0) False(0) \n AES_XTS_TWEAK_INC_64 False(0) False(0) False(0) \n* CRYPTO_POLICY UNRESTRICTED(1) DEVICE_DEFAULT(0) UNRESTRICTED(1) \n RDE_DISABLE False(0) False(0) False(0) \n PLDM_FW_UPDATE_DISABLE False(0) False(0) False(0) \n RBT_DISABLE False(0) False(0) False(0) \n PCIE_SMBUS_DISABLE False(0) False(0) False(0) \n PCIE_IN_BAND_VDM_DISABLE False(0) False(0) False(0) \n ADVANCED_TESTABILITY False(0) False(0) False(0) \n LOG_DCR_HASH_TABLE_SIZE 11 11 11 \n MAX_PACKET_LIFETIME 0 0 0 \n DCR_LIFO_SIZE 16384 16384 16384 \n LINK_TYPE_P1 IB(1) IB(1) IB(1) \n ROCE_CC_PRIO_MASK_P1 255 255 255 \n ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ROCE_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n IB_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1) True(1) True(1) \n CLAMP_TGT_RATE_P1 False(0) False(0) False(0) \n RPG_TIME_RESET_P1 300 300 300 \n RPG_BYTE_RESET_P1 32767 32767 32767 \n RPG_THRESHOLD_P1 1 1 1 \n RPG_MAX_RATE_P1 0 0 0 \n RPG_AI_RATE_P1 5 5 5 \n RPG_HAI_RATE_P1 50 50 50 \n RPG_GD_P1 11 11 11 \n RPG_MIN_DEC_FAC_P1 50 50 50 \n RPG_MIN_RATE_P1 1 1 1 \n RATE_TO_SET_ON_FIRST_CNP_P1 0 0 0 \n DCE_TCP_G_P1 1019 1019 1019 \n DCE_TCP_RTT_P1 1 1 1 \n RATE_REDUCE_MONITOR_PERIOD_P1 4 4 4 \n INITIAL_ALPHA_VALUE_P1 1023 1023 1023 \n MIN_TIME_BETWEEN_CNPS_P1 4 4 4 \n CNP_802P_PRIO_P1 6 6 6 \n CNP_DSCP_P1 48 48 48 \n LLDP_NB_DCBX_P1 False(0) False(0) False(0) \n LLDP_NB_RX_MODE_P1 OFF(0) OFF(0) OFF(0) \n LLDP_NB_TX_MODE_P1 OFF(0) OFF(0) OFF(0) \n ROCE_RTT_RESP_DSCP_P1 0 0 0 \n ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n DCBX_IEEE_P1 True(1) True(1) True(1) \n DCBX_CEE_P1 True(1) True(1) True(1) \n DCBX_WILLING_P1 True(1) True(1) True(1) \n KEEP_ETH_LINK_UP_P1 True(1) True(1) True(1) \n KEEP_IB_LINK_UP_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_BOOT_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_STANDBY_P1 False(0) False(0) False(0) \n DO_NOT_CLEAR_PORT_STATS_P1 False(0) False(0) False(0) \n AUTO_POWER_SAVE_LINK_DOWN_P1 False(0) False(0) False(0) \n NUM_OF_VL_P1 _4_VLs(3) _4_VLs(3) _4_VLs(3) \n NUM_OF_TC_P1 _8_TCs(0) _8_TCs(0) _8_TCs(0) \n NUM_OF_PFC_P1 8 8 8 \n VL15_BUFFER_SIZE_P1 0 0 0 \n QOS_TRUST_STATE_P1 TRUST_PCP(1) TRUST_PCP(1) TRUST_PCP(1) \n ETS_SCHED_MODE_P1 device_default(0) device_default(0) device_default(0) \n DUP_MAC_ACTION_P1 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) \n MPFS_MC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n MPFS_UC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n SRIOV_IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n PHY_AUTO_NEG_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PHY_RATE_MASK_OVERRIDE_P1 False(0) False(0) False(0) \n PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PF_TOTAL_SF 0 0 0 \n PF_SD_GROUP 0 0 0 \n PF_SF_BAR_SIZE 0 0 0 \n PF_NUM_PF_MSIX 63 63 63 \n SILENT_MODE False(0) False(0) False(0) \n MKEY_BY_NAME_RANGE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n* ROCE_CONTROL ROCE_ENABLE(2) DEVICE_DEFAULT(0) ROCE_ENABLE(2) \n PCI_WR_ORDERING per_mkey(0) per_mkey(0) per_mkey(0) \n MULTI_PORT_VHCA_EN False(0) False(0) False(0) \n PORT_OWNER True(1) True(1) True(1) \n ALLOW_RD_COUNTERS True(1) True(1) True(1) \n RENEG_ON_CHANGE True(1) True(1) True(1) \n TRACER_ENABLE True(1) True(1) True(1) \n IP_VER IPv4(0) IPv4(0) IPv4(0) \n BOOT_UNDI_NETWORK_WAIT 0 0 0 \n UEFI_HII_EN True(1) True(1) True(1) \n BOOT_DBG_LOG False(0) False(0) False(0) \n UEFI_LOGS DISABLED(0) DISABLED(0) DISABLED(0) \n BOOT_VLAN 1 1 1 \n LEGACY_BOOT_PROTOCOL PXE(1) PXE(1) PXE(1) \n BOOT_INTERRUPT_DIS False(0) False(0) False(0) \n BOOT_LACP_DIS True(1) True(1) True(1) \n BOOT_VLAN_EN False(0) False(0) False(0) \n BOOT_PKEY 0 0 0 \n P2P_ORDERING_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ATS_ENABLED False(0) False(0) False(0) \n DYNAMIC_VF_MSIX_TABLE False(0) False(0) False(0) \n EXP_ROM_UEFI_ARM_ENABLE True(1) True(1) True(1) \n EXP_ROM_UEFI_x86_ENABLE True(1) True(1) True(1) \n EXP_ROM_PXE_ENABLE True(1) True(1) True(1) \n ADVANCED_PCI_SETTINGS False(0) False(0) False(0) \n SAFE_MODE_THRESHOLD 10 10 10 \n SAFE_MODE_ENABLE True(1) True(1) True(1) \nThe '*' shows parameters with next value different from default/current value.\n", "error": null} +2025-11-17T01:01:27.034732068Z INFO mellanox/mellanox.go:213 ParseMstconfigOutput() {"attributes": ["NUM_OF_VFS", "SRIOV_EN", "LINK_TYPE_P1", "LINK_TYPE_P2"]} +2025-11-17T01:01:27.034775472Z INFO mellanox/mellanox.go:214 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"16","SRIOV_EN":"True(1)"}} +2025-11-17T01:01:27.034792914Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:01:27.03479703Z INFO mellanox/mellanox.go:219 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"1","SRIOV_EN":"True(1)"}} +2025-11-17T01:01:27.034802359Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:01:27.034805586Z INFO mellanox/mellanox_plugin.go:107 mellanox-plugin IsDualPort() {"address": "0000:e3:00.0"} +2025-11-17T01:01:27.034811725Z LEVEL(-2) mellanox/mellanox_plugin.go:112 Changing TotalVfs, needs reboot {"current": 16, "requested": 1} +2025-11-17T01:01:27.034818896Z INFO mellanox/mellanox.go:343 mellanox-plugin isLinkTypeRequireChange() {"device": "0000:e3:00.0"} +2025-11-17T01:01:27.034824277Z INFO mellanox/mellanox_plugin.go:102 mellanox-plugin getMlnxNicFwData() {"device": "0000:19:00.0"} +2025-11-17T01:01:27.034827983Z INFO mellanox/mellanox.go:208 MstConfigReadData() {"device": "0000:19:00.0"} +2025-11-17T01:01:27.034831823Z INFO mellanox/mellanox.go:81 RunCommand() {"command": "mstconfig", "args": ["-e", "-d", "0000:19:00.0", "q"]} +2025-11-17T01:01:51.283986837Z LEVEL(-2) mellanox/mellanox.go:81 RunCommand() {"output": "\nDevice #1:\n----------\n\nDevice type: ConnectX7 \nName: MCX75310AAS-NEA_Ax \nDescription: NVIDIA ConnectX-7 HHHL Adapter card; 400GbE / NDR IB (default mode); Single-port OSFP; PCIe 5.0 x16; Crypto Disabled; Secure Boot Enabled;\nDevice: 0000:19:00.0 \n\nConfigurations: Default Current Next Boot\n MODULE_SPLIT_M0 Array[0..15] Array[0..15] Array[0..15] \n MEMIC_BAR_SIZE 0 0 0 \n MEMIC_SIZE_LIMIT _256KB(1) _256KB(1) _256KB(1) \n MEMIC_ATOMIC MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2)\n HOST_CHAINING_MODE DISABLED(0) DISABLED(0) DISABLED(0) \n HOST_CHAINING_CACHE_DISABLE False(0) False(0) False(0) \n HOST_CHAINING_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n HOST_CHAINING_TOTAL_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n FLEX_PARSER_PROFILE_ENABLE 0 0 0 \n PROG_PARSE_GRAPH False(0) False(0) False(0) \n FLEX_IPV4_OVER_VXLAN_PORT 0 0 0 \n ROCE_NEXT_PROTOCOL 254 254 254 \n ESWITCH_HAIRPIN_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n ESWITCH_HAIRPIN_TOT_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n DPA_AUTHENTICATION True(1) True(1) True(1) \n PF_BAR2_SIZE 0 0 0 \n PF_NUM_OF_VF_VALID False(0) False(0) False(0) \n NON_PREFETCHABLE_PF_BAR False(0) False(0) False(0) \n VF_VPD_ENABLE False(0) False(0) False(0) \n PF_NUM_PF_MSIX_VALID False(0) False(0) False(0) \n PER_PF_NUM_SF False(0) False(0) False(0) \n STRICT_VF_MSIX_NUM False(0) False(0) False(0) \n VF_NODNIC_ENABLE False(0) False(0) False(0) \n NUM_PF_MSIX_VALID True(1) True(1) True(1) \n* NUM_OF_VFS 16 16 1 \n NUM_OF_PF 1 1 1 \n PF_BAR2_ENABLE False(0) False(0) False(0) \n SRIOV_EN True(1) True(1) True(1) \n PF_LOG_BAR_SIZE 5 5 5 \n VF_LOG_BAR_SIZE 1 1 1 \n NUM_PF_MSIX 63 63 63 \n NUM_VF_MSIX 11 11 11 \n INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0) AUTOMATIC(0) AUTOMATIC(0) \n PCIE_CREDIT_TOKEN_TIMEOUT 0 0 0 \n RT_PPS_ENABLED_ON_POWERUP False(0) False(0) False(0) \n LAG_RESOURCE_ALLOCATION DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ACCURATE_TX_SCHEDULER False(0) False(0) False(0) \n PARTIAL_RESET_EN False(0) False(0) False(0) \n RESET_WITH_HOST_ON_ERRORS False(0) False(0) False(0) \n PCI_SWITCH_EMULATION_NUM_PORT 16 16 16 \n PCI_SWITCH_EMULATION_ENABLE False(0) False(0) False(0) \n PCI_DOWNSTREAM_PORT_OWNER Array[0..15] Array[0..15] Array[0..15] \n CQE_COMPRESSION BALANCED(0) BALANCED(0) BALANCED(0) \n IP_OVER_VXLAN_EN False(0) False(0) False(0) \n MKEY_BY_NAME False(0) False(0) False(0) \n PRIO_TAG_REQUIRED_EN False(0) False(0) False(0) \n UCTX_EN True(1) True(1) True(1) \n REAL_TIME_CLOCK_ENABLE False(0) False(0) False(0) \n RDMA_SELECTIVE_REPEAT_EN False(0) False(0) False(0) \n PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)\n TUNNEL_ECN_COPY_DISABLE False(0) False(0) False(0) \n LRO_LOG_TIMEOUT0 6 6 6 \n LRO_LOG_TIMEOUT1 7 7 7 \n LRO_LOG_TIMEOUT2 8 8 8 \n LRO_LOG_TIMEOUT3 13 13 13 \n LOG_TX_PSN_WINDOW 7 7 7 \n VF_MIGRATION_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n LOG_MAX_OUTSTANDING_WQE 7 7 7 \n ROCE_ADAPTIVE_ROUTING_EN False(0) False(0) False(0) \n TUNNEL_IP_PROTO_ENTROPY_DISABLE False(0) False(0) False(0) \n PCC_HANDLE_CORE_UTIL DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PCC_INT_NP_RTT_DSCP 26 26 26 \n PCC_INT_NP_RTT_DSCP_EN False(0) False(0) False(0) \n PCC_INT_NP_RTT_DATA_MODE RTT_V0(64) RTT_V0(64) RTT_V0(64) \n PCC_INT_EN False(0) False(0) False(0) \n PCC_INT_SYSTEM_RTT 0 0 0 \n STEERING_CACHE_REFRESH 0 0 0 \n ICM_CACHE_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n HAIRPIN_DATA_BUFFER_LOCK False(0) False(0) False(0) \n TX_SCHEDULER_BURST 0 0 0 \n ZERO_TOUCH_TUNING_ENABLE False(0) False(0) False(0) \n LOG_MAX_QUEUE 17 17 17 \n LARGE_MTU_TWEAK_64 False(0) False(0) False(0) \n AES_XTS_TWEAK_INC_64 False(0) False(0) False(0) \n* CRYPTO_POLICY UNRESTRICTED(1) DEVICE_DEFAULT(0) UNRESTRICTED(1) \n RDE_DISABLE False(0) False(0) False(0) \n PLDM_FW_UPDATE_DISABLE False(0) False(0) False(0) \n RBT_DISABLE False(0) False(0) False(0) \n PCIE_SMBUS_DISABLE False(0) False(0) False(0) \n PCIE_IN_BAND_VDM_DISABLE False(0) False(0) False(0) \n ADVANCED_TESTABILITY False(0) False(0) False(0) \n LOG_DCR_HASH_TABLE_SIZE 11 11 11 \n MAX_PACKET_LIFETIME 0 0 0 \n DCR_LIFO_SIZE 16384 16384 16384 \n LINK_TYPE_P1 IB(1) IB(1) IB(1) \n ROCE_CC_PRIO_MASK_P1 255 255 255 \n ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ROCE_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n IB_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1) True(1) True(1) \n CLAMP_TGT_RATE_P1 False(0) False(0) False(0) \n RPG_TIME_RESET_P1 300 300 300 \n RPG_BYTE_RESET_P1 32767 32767 32767 \n RPG_THRESHOLD_P1 1 1 1 \n RPG_MAX_RATE_P1 0 0 0 \n RPG_AI_RATE_P1 5 5 5 \n RPG_HAI_RATE_P1 50 50 50 \n RPG_GD_P1 11 11 11 \n RPG_MIN_DEC_FAC_P1 50 50 50 \n RPG_MIN_RATE_P1 1 1 1 \n RATE_TO_SET_ON_FIRST_CNP_P1 0 0 0 \n DCE_TCP_G_P1 1019 1019 1019 \n DCE_TCP_RTT_P1 1 1 1 \n RATE_REDUCE_MONITOR_PERIOD_P1 4 4 4 \n INITIAL_ALPHA_VALUE_P1 1023 1023 1023 \n MIN_TIME_BETWEEN_CNPS_P1 4 4 4 \n CNP_802P_PRIO_P1 6 6 6 \n CNP_DSCP_P1 48 48 48 \n LLDP_NB_DCBX_P1 False(0) False(0) False(0) \n LLDP_NB_RX_MODE_P1 OFF(0) OFF(0) OFF(0) \n LLDP_NB_TX_MODE_P1 OFF(0) OFF(0) OFF(0) \n ROCE_RTT_RESP_DSCP_P1 0 0 0 \n ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n DCBX_IEEE_P1 True(1) True(1) True(1) \n DCBX_CEE_P1 True(1) True(1) True(1) \n DCBX_WILLING_P1 True(1) True(1) True(1) \n KEEP_ETH_LINK_UP_P1 True(1) True(1) True(1) \n KEEP_IB_LINK_UP_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_BOOT_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_STANDBY_P1 False(0) False(0) False(0) \n DO_NOT_CLEAR_PORT_STATS_P1 False(0) False(0) False(0) \n AUTO_POWER_SAVE_LINK_DOWN_P1 False(0) False(0) False(0) \n NUM_OF_VL_P1 _4_VLs(3) _4_VLs(3) _4_VLs(3) \n NUM_OF_TC_P1 _8_TCs(0) _8_TCs(0) _8_TCs(0) \n NUM_OF_PFC_P1 8 8 8 \n VL15_BUFFER_SIZE_P1 0 0 0 \n QOS_TRUST_STATE_P1 TRUST_PCP(1) TRUST_PCP(1) TRUST_PCP(1) \n ETS_SCHED_MODE_P1 device_default(0) device_default(0) device_default(0) \n DUP_MAC_ACTION_P1 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) \n MPFS_MC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n MPFS_UC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n SRIOV_IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n PHY_AUTO_NEG_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PHY_RATE_MASK_OVERRIDE_P1 False(0) False(0) False(0) \n PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PF_TOTAL_SF 0 0 0 \n PF_SD_GROUP 0 0 0 \n PF_SF_BAR_SIZE 0 0 0 \n PF_NUM_PF_MSIX 63 63 63 \n SILENT_MODE False(0) False(0) False(0) \n MKEY_BY_NAME_RANGE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n* ROCE_CONTROL ROCE_ENABLE(2) DEVICE_DEFAULT(0) ROCE_ENABLE(2) \n PCI_WR_ORDERING per_mkey(0) per_mkey(0) per_mkey(0) \n MULTI_PORT_VHCA_EN False(0) False(0) False(0) \n PORT_OWNER True(1) True(1) True(1) \n ALLOW_RD_COUNTERS True(1) True(1) True(1) \n RENEG_ON_CHANGE True(1) True(1) True(1) \n TRACER_ENABLE True(1) True(1) True(1) \n IP_VER IPv4(0) IPv4(0) IPv4(0) \n BOOT_UNDI_NETWORK_WAIT 0 0 0 \n UEFI_HII_EN True(1) True(1) True(1) \n BOOT_DBG_LOG False(0) False(0) False(0) \n UEFI_LOGS DISABLED(0) DISABLED(0) DISABLED(0) \n BOOT_VLAN 1 1 1 \n LEGACY_BOOT_PROTOCOL PXE(1) PXE(1) PXE(1) \n BOOT_INTERRUPT_DIS False(0) False(0) False(0) \n BOOT_LACP_DIS True(1) True(1) True(1) \n BOOT_VLAN_EN False(0) False(0) False(0) \n BOOT_PKEY 0 0 0 \n P2P_ORDERING_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ATS_ENABLED False(0) False(0) False(0) \n DYNAMIC_VF_MSIX_TABLE False(0) False(0) False(0) \n EXP_ROM_UEFI_ARM_ENABLE True(1) True(1) True(1) \n EXP_ROM_UEFI_x86_ENABLE True(1) True(1) True(1) \n EXP_ROM_PXE_ENABLE True(1) True(1) True(1) \n ADVANCED_PCI_SETTINGS False(0) False(0) False(0) \n SAFE_MODE_THRESHOLD 10 10 10 \n SAFE_MODE_ENABLE True(1) True(1) True(1) \nThe '*' shows parameters with next value different from default/current value.\n", "error": null} +2025-11-17T01:01:51.284090177Z INFO mellanox/mellanox.go:213 ParseMstconfigOutput() {"attributes": ["NUM_OF_VFS", "SRIOV_EN", "LINK_TYPE_P1", "LINK_TYPE_P2"]} +2025-11-17T01:01:51.284141806Z INFO mellanox/mellanox.go:214 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"16","SRIOV_EN":"True(1)"}} +2025-11-17T01:01:51.284155512Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:01:51.284159743Z INFO mellanox/mellanox.go:219 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"1","SRIOV_EN":"True(1)"}} +2025-11-17T01:01:51.284169177Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:01:51.284173885Z INFO mellanox/mellanox_plugin.go:107 mellanox-plugin IsDualPort() {"address": "0000:19:00.0"} +2025-11-17T01:01:51.284181628Z LEVEL(-2) mellanox/mellanox_plugin.go:112 Changing TotalVfs, needs reboot {"current": 16, "requested": 1} +2025-11-17T01:01:51.284189795Z INFO mellanox/mellanox.go:343 mellanox-plugin isLinkTypeRequireChange() {"device": "0000:19:00.0"} +2025-11-17T01:01:51.28419698Z INFO mellanox/mellanox_plugin.go:102 mellanox-plugin getMlnxNicFwData() {"device": "0000:29:00.0"} +2025-11-17T01:01:51.284201307Z INFO mellanox/mellanox.go:208 MstConfigReadData() {"device": "0000:29:00.0"} +2025-11-17T01:01:51.284205437Z INFO mellanox/mellanox.go:81 RunCommand() {"command": "mstconfig", "args": ["-e", "-d", "0000:29:00.0", "q"]} +2025-11-17T01:01:52.094994285Z LEVEL(-2) runtime/asm_amd64.s:1695 Run(): period refresh +2025-11-17T01:01:52.095015436Z LEVEL(-2) daemon/writer.go:109 pollNicStatus() +2025-11-17T01:01:52.095087463Z LEVEL(-2) daemon/writer.go:130 DiscoverSriovDevices +2025-11-17T01:01:52.195459744Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib0"} +2025-11-17T01:01:52.195694217Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib0"} +2025-11-17T01:01:52.195713667Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib0/speed", "error": "read /sys/class/net/ib0/speed: invalid argument"} +2025-11-17T01:01:52.195720067Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib0"} +2025-11-17T01:01:52.195788106Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:19:00.0"} +2025-11-17T01:01:52.195902852Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib1"} +2025-11-17T01:01:52.19594694Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib1"} +2025-11-17T01:01:52.195959802Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib1/speed", "error": "read /sys/class/net/ib1/speed: invalid argument"} +2025-11-17T01:01:52.195963889Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib1"} +2025-11-17T01:01:52.196015904Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:29:00.0"} +2025-11-17T01:01:52.1961013Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib2"} +2025-11-17T01:01:52.196146743Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib2"} +2025-11-17T01:01:52.196443911Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib2"} +2025-11-17T01:01:52.196500097Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:3b:00.0"} +2025-11-17T01:01:52.196651411Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth2"} +2025-11-17T01:01:52.197310219Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth2"} +2025-11-17T01:01:52.197369379Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.0"} +2025-11-17T01:01:52.197515971Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth3"} +2025-11-17T01:01:52.198534537Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth3"} +2025-11-17T01:01:52.1986868Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.1"} +2025-11-17T01:01:52.198790646Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth0"} +2025-11-17T01:01:52.198841794Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth0"} +2025-11-17T01:01:52.198857238Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth0/speed", "error": "read /sys/class/net/eth0/speed: invalid argument"} +2025-11-17T01:01:52.198863433Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth0"} +2025-11-17T01:01:52.198931129Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.0"} +2025-11-17T01:01:52.19900123Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth1"} +2025-11-17T01:01:52.199050195Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth1"} +2025-11-17T01:01:52.19906583Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth1/speed", "error": "read /sys/class/net/eth1/speed: invalid argument"} +2025-11-17T01:01:52.199069665Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth1"} +2025-11-17T01:01:52.199132912Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.1"} +2025-11-17T01:01:52.199211326Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib3"} +2025-11-17T01:01:52.199264724Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib3"} +2025-11-17T01:01:52.199388244Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib3"} +2025-11-17T01:01:52.199578106Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:5c:00.0"} +2025-11-17T01:01:52.199704636Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib4"} +2025-11-17T01:01:52.199761474Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib4"} +2025-11-17T01:01:52.199850844Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib4"} +2025-11-17T01:01:52.199919422Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:85:00.0"} +2025-11-17T01:01:52.200008818Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib5"} +2025-11-17T01:01:52.200057236Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib5"} +2025-11-17T01:01:52.200289781Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib5"} +2025-11-17T01:01:52.200358184Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:8a:00.0"} +2025-11-17T01:01:52.200437723Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib6"} +2025-11-17T01:01:52.200482326Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib6"} +2025-11-17T01:01:52.200497506Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib6/speed", "error": "read /sys/class/net/ib6/speed: invalid argument"} +2025-11-17T01:01:52.200505458Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib6"} +2025-11-17T01:01:52.200569189Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:92:00.0"} +2025-11-17T01:01:52.200652739Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib7"} +2025-11-17T01:01:52.200694957Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib7"} +2025-11-17T01:01:52.200709294Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib7/speed", "error": "read /sys/class/net/ib7/speed: invalid argument"} +2025-11-17T01:01:52.2007135Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib7"} +2025-11-17T01:01:52.20077191Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:e3:00.0"} +2025-11-17T01:01:52.211192385Z INFO daemon/writer.go:166 setNodeStateStatus(): status {"sync-status": "InProgress", "last-sync-error": ""} +2025-11-17T01:02:15.020220837Z LEVEL(-2) mellanox/mellanox.go:81 RunCommand() {"output": "\nDevice #1:\n----------\n\nDevice type: ConnectX7 \nName: MCX75310AAS-NEA_Ax \nDescription: NVIDIA ConnectX-7 HHHL Adapter card; 400GbE / NDR IB (default mode); Single-port OSFP; PCIe 5.0 x16; Crypto Disabled; Secure Boot Enabled;\nDevice: 0000:29:00.0 \n\nConfigurations: Default Current Next Boot\n MODULE_SPLIT_M0 Array[0..15] Array[0..15] Array[0..15] \n MEMIC_BAR_SIZE 0 0 0 \n MEMIC_SIZE_LIMIT _256KB(1) _256KB(1) _256KB(1) \n MEMIC_ATOMIC MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2)\n HOST_CHAINING_MODE DISABLED(0) DISABLED(0) DISABLED(0) \n HOST_CHAINING_CACHE_DISABLE False(0) False(0) False(0) \n HOST_CHAINING_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n HOST_CHAINING_TOTAL_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n FLEX_PARSER_PROFILE_ENABLE 0 0 0 \n PROG_PARSE_GRAPH False(0) False(0) False(0) \n FLEX_IPV4_OVER_VXLAN_PORT 0 0 0 \n ROCE_NEXT_PROTOCOL 254 254 254 \n ESWITCH_HAIRPIN_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n ESWITCH_HAIRPIN_TOT_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n DPA_AUTHENTICATION True(1) True(1) True(1) \n PF_BAR2_SIZE 0 0 0 \n PF_NUM_OF_VF_VALID False(0) False(0) False(0) \n NON_PREFETCHABLE_PF_BAR False(0) False(0) False(0) \n VF_VPD_ENABLE False(0) False(0) False(0) \n PF_NUM_PF_MSIX_VALID False(0) False(0) False(0) \n PER_PF_NUM_SF False(0) False(0) False(0) \n STRICT_VF_MSIX_NUM False(0) False(0) False(0) \n VF_NODNIC_ENABLE False(0) False(0) False(0) \n NUM_PF_MSIX_VALID True(1) True(1) True(1) \n* NUM_OF_VFS 16 16 1 \n NUM_OF_PF 1 1 1 \n PF_BAR2_ENABLE False(0) False(0) False(0) \n SRIOV_EN True(1) True(1) True(1) \n PF_LOG_BAR_SIZE 5 5 5 \n VF_LOG_BAR_SIZE 1 1 1 \n NUM_PF_MSIX 63 63 63 \n NUM_VF_MSIX 11 11 11 \n INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0) AUTOMATIC(0) AUTOMATIC(0) \n PCIE_CREDIT_TOKEN_TIMEOUT 0 0 0 \n RT_PPS_ENABLED_ON_POWERUP False(0) False(0) False(0) \n LAG_RESOURCE_ALLOCATION DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ACCURATE_TX_SCHEDULER False(0) False(0) False(0) \n PARTIAL_RESET_EN False(0) False(0) False(0) \n RESET_WITH_HOST_ON_ERRORS False(0) False(0) False(0) \n PCI_SWITCH_EMULATION_NUM_PORT 16 16 16 \n PCI_SWITCH_EMULATION_ENABLE False(0) False(0) False(0) \n PCI_DOWNSTREAM_PORT_OWNER Array[0..15] Array[0..15] Array[0..15] \n CQE_COMPRESSION BALANCED(0) BALANCED(0) BALANCED(0) \n IP_OVER_VXLAN_EN False(0) False(0) False(0) \n MKEY_BY_NAME False(0) False(0) False(0) \n PRIO_TAG_REQUIRED_EN False(0) False(0) False(0) \n UCTX_EN True(1) True(1) True(1) \n REAL_TIME_CLOCK_ENABLE False(0) False(0) False(0) \n RDMA_SELECTIVE_REPEAT_EN False(0) False(0) False(0) \n PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)\n TUNNEL_ECN_COPY_DISABLE False(0) False(0) False(0) \n LRO_LOG_TIMEOUT0 6 6 6 \n LRO_LOG_TIMEOUT1 7 7 7 \n LRO_LOG_TIMEOUT2 8 8 8 \n LRO_LOG_TIMEOUT3 13 13 13 \n LOG_TX_PSN_WINDOW 7 7 7 \n VF_MIGRATION_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n LOG_MAX_OUTSTANDING_WQE 7 7 7 \n ROCE_ADAPTIVE_ROUTING_EN False(0) False(0) False(0) \n TUNNEL_IP_PROTO_ENTROPY_DISABLE False(0) False(0) False(0) \n PCC_HANDLE_CORE_UTIL DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PCC_INT_NP_RTT_DSCP 26 26 26 \n PCC_INT_NP_RTT_DSCP_EN False(0) False(0) False(0) \n PCC_INT_NP_RTT_DATA_MODE RTT_V0(64) RTT_V0(64) RTT_V0(64) \n PCC_INT_EN False(0) False(0) False(0) \n PCC_INT_SYSTEM_RTT 0 0 0 \n STEERING_CACHE_REFRESH 0 0 0 \n ICM_CACHE_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n HAIRPIN_DATA_BUFFER_LOCK False(0) False(0) False(0) \n TX_SCHEDULER_BURST 0 0 0 \n ZERO_TOUCH_TUNING_ENABLE False(0) False(0) False(0) \n LOG_MAX_QUEUE 17 17 17 \n LARGE_MTU_TWEAK_64 False(0) False(0) False(0) \n AES_XTS_TWEAK_INC_64 False(0) False(0) False(0) \n* CRYPTO_POLICY UNRESTRICTED(1) DEVICE_DEFAULT(0) UNRESTRICTED(1) \n RDE_DISABLE False(0) False(0) False(0) \n PLDM_FW_UPDATE_DISABLE False(0) False(0) False(0) \n RBT_DISABLE False(0) False(0) False(0) \n PCIE_SMBUS_DISABLE False(0) False(0) False(0) \n PCIE_IN_BAND_VDM_DISABLE False(0) False(0) False(0) \n ADVANCED_TESTABILITY False(0) False(0) False(0) \n LOG_DCR_HASH_TABLE_SIZE 11 11 11 \n MAX_PACKET_LIFETIME 0 0 0 \n DCR_LIFO_SIZE 16384 16384 16384 \n LINK_TYPE_P1 IB(1) IB(1) IB(1) \n ROCE_CC_PRIO_MASK_P1 255 255 255 \n ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ROCE_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n IB_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1) True(1) True(1) \n CLAMP_TGT_RATE_P1 False(0) False(0) False(0) \n RPG_TIME_RESET_P1 300 300 300 \n RPG_BYTE_RESET_P1 32767 32767 32767 \n RPG_THRESHOLD_P1 1 1 1 \n RPG_MAX_RATE_P1 0 0 0 \n RPG_AI_RATE_P1 5 5 5 \n RPG_HAI_RATE_P1 50 50 50 \n RPG_GD_P1 11 11 11 \n RPG_MIN_DEC_FAC_P1 50 50 50 \n RPG_MIN_RATE_P1 1 1 1 \n RATE_TO_SET_ON_FIRST_CNP_P1 0 0 0 \n DCE_TCP_G_P1 1019 1019 1019 \n DCE_TCP_RTT_P1 1 1 1 \n RATE_REDUCE_MONITOR_PERIOD_P1 4 4 4 \n INITIAL_ALPHA_VALUE_P1 1023 1023 1023 \n MIN_TIME_BETWEEN_CNPS_P1 4 4 4 \n CNP_802P_PRIO_P1 6 6 6 \n CNP_DSCP_P1 48 48 48 \n LLDP_NB_DCBX_P1 False(0) False(0) False(0) \n LLDP_NB_RX_MODE_P1 OFF(0) OFF(0) OFF(0) \n LLDP_NB_TX_MODE_P1 OFF(0) OFF(0) OFF(0) \n ROCE_RTT_RESP_DSCP_P1 0 0 0 \n ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n DCBX_IEEE_P1 True(1) True(1) True(1) \n DCBX_CEE_P1 True(1) True(1) True(1) \n DCBX_WILLING_P1 True(1) True(1) True(1) \n KEEP_ETH_LINK_UP_P1 True(1) True(1) True(1) \n KEEP_IB_LINK_UP_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_BOOT_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_STANDBY_P1 False(0) False(0) False(0) \n DO_NOT_CLEAR_PORT_STATS_P1 False(0) False(0) False(0) \n AUTO_POWER_SAVE_LINK_DOWN_P1 False(0) False(0) False(0) \n NUM_OF_VL_P1 _4_VLs(3) _4_VLs(3) _4_VLs(3) \n NUM_OF_TC_P1 _8_TCs(0) _8_TCs(0) _8_TCs(0) \n NUM_OF_PFC_P1 8 8 8 \n VL15_BUFFER_SIZE_P1 0 0 0 \n QOS_TRUST_STATE_P1 TRUST_PCP(1) TRUST_PCP(1) TRUST_PCP(1) \n ETS_SCHED_MODE_P1 device_default(0) device_default(0) device_default(0) \n DUP_MAC_ACTION_P1 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) \n MPFS_MC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n MPFS_UC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n SRIOV_IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n PHY_AUTO_NEG_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PHY_RATE_MASK_OVERRIDE_P1 False(0) False(0) False(0) \n PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PF_TOTAL_SF 0 0 0 \n PF_SD_GROUP 0 0 0 \n PF_SF_BAR_SIZE 0 0 0 \n PF_NUM_PF_MSIX 63 63 63 \n SILENT_MODE False(0) False(0) False(0) \n MKEY_BY_NAME_RANGE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n* ROCE_CONTROL ROCE_ENABLE(2) DEVICE_DEFAULT(0) ROCE_ENABLE(2) \n PCI_WR_ORDERING per_mkey(0) per_mkey(0) per_mkey(0) \n MULTI_PORT_VHCA_EN False(0) False(0) False(0) \n PORT_OWNER True(1) True(1) True(1) \n ALLOW_RD_COUNTERS True(1) True(1) True(1) \n RENEG_ON_CHANGE True(1) True(1) True(1) \n TRACER_ENABLE True(1) True(1) True(1) \n IP_VER IPv4(0) IPv4(0) IPv4(0) \n BOOT_UNDI_NETWORK_WAIT 0 0 0 \n UEFI_HII_EN True(1) True(1) True(1) \n BOOT_DBG_LOG False(0) False(0) False(0) \n UEFI_LOGS DISABLED(0) DISABLED(0) DISABLED(0) \n BOOT_VLAN 1 1 1 \n LEGACY_BOOT_PROTOCOL PXE(1) PXE(1) PXE(1) \n BOOT_INTERRUPT_DIS False(0) False(0) False(0) \n BOOT_LACP_DIS True(1) True(1) True(1) \n BOOT_VLAN_EN False(0) False(0) False(0) \n BOOT_PKEY 0 0 0 \n P2P_ORDERING_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ATS_ENABLED False(0) False(0) False(0) \n DYNAMIC_VF_MSIX_TABLE False(0) False(0) False(0) \n EXP_ROM_UEFI_ARM_ENABLE True(1) True(1) True(1) \n EXP_ROM_UEFI_x86_ENABLE True(1) True(1) True(1) \n EXP_ROM_PXE_ENABLE True(1) True(1) True(1) \n ADVANCED_PCI_SETTINGS False(0) False(0) False(0) \n SAFE_MODE_THRESHOLD 10 10 10 \n SAFE_MODE_ENABLE True(1) True(1) True(1) \nThe '*' shows parameters with next value different from default/current value.\n", "error": null} +2025-11-17T01:02:15.020377294Z INFO mellanox/mellanox.go:213 ParseMstconfigOutput() {"attributes": ["NUM_OF_VFS", "SRIOV_EN", "LINK_TYPE_P1", "LINK_TYPE_P2"]} +2025-11-17T01:02:15.020443499Z INFO mellanox/mellanox.go:214 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"16","SRIOV_EN":"True(1)"}} +2025-11-17T01:02:15.020461727Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:02:15.02046593Z INFO mellanox/mellanox.go:219 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"1","SRIOV_EN":"True(1)"}} +2025-11-17T01:02:15.020469741Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:02:15.020473643Z INFO mellanox/mellanox_plugin.go:107 mellanox-plugin IsDualPort() {"address": "0000:29:00.0"} +2025-11-17T01:02:15.020483056Z LEVEL(-2) mellanox/mellanox_plugin.go:112 Changing TotalVfs, needs reboot {"current": 16, "requested": 1} +2025-11-17T01:02:15.020490603Z INFO mellanox/mellanox.go:343 mellanox-plugin isLinkTypeRequireChange() {"device": "0000:29:00.0"} +2025-11-17T01:02:15.020497173Z INFO mellanox/mellanox_plugin.go:102 mellanox-plugin getMlnxNicFwData() {"device": "0000:3b:00.0"} +2025-11-17T01:02:15.020502591Z INFO mellanox/mellanox.go:208 MstConfigReadData() {"device": "0000:3b:00.0"} +2025-11-17T01:02:15.020507819Z INFO mellanox/mellanox.go:81 RunCommand() {"command": "mstconfig", "args": ["-e", "-d", "0000:3b:00.0", "q"]} +2025-11-17T01:02:22.225460594Z LEVEL(-2) runtime/asm_amd64.s:1695 Run(): period refresh +2025-11-17T01:02:22.225514219Z LEVEL(-2) daemon/writer.go:109 pollNicStatus() +2025-11-17T01:02:22.225521097Z LEVEL(-2) daemon/writer.go:130 DiscoverSriovDevices +2025-11-17T01:02:22.325959224Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib0"} +2025-11-17T01:02:22.326187592Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib0"} +2025-11-17T01:02:22.326207692Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib0/speed", "error": "read /sys/class/net/ib0/speed: invalid argument"} +2025-11-17T01:02:22.326216033Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib0"} +2025-11-17T01:02:22.326278363Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:19:00.0"} +2025-11-17T01:02:22.326386081Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib1"} +2025-11-17T01:02:22.326427673Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib1"} +2025-11-17T01:02:22.32644162Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib1/speed", "error": "read /sys/class/net/ib1/speed: invalid argument"} +2025-11-17T01:02:22.326446021Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib1"} +2025-11-17T01:02:22.326503222Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:29:00.0"} +2025-11-17T01:02:22.326591559Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib2"} +2025-11-17T01:02:22.326640291Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib2"} +2025-11-17T01:02:22.326993406Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib2"} +2025-11-17T01:02:22.327158729Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:3b:00.0"} +2025-11-17T01:02:22.32734452Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth2"} +2025-11-17T01:02:22.327993345Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth2"} +2025-11-17T01:02:22.328061357Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.0"} +2025-11-17T01:02:22.328211332Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth3"} +2025-11-17T01:02:22.328877844Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth3"} +2025-11-17T01:02:22.328944264Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.1"} +2025-11-17T01:02:22.329034688Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth0"} +2025-11-17T01:02:22.329080441Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth0"} +2025-11-17T01:02:22.329096364Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth0/speed", "error": "read /sys/class/net/eth0/speed: invalid argument"} +2025-11-17T01:02:22.329102767Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth0"} +2025-11-17T01:02:22.329165752Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.0"} +2025-11-17T01:02:22.329237016Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth1"} +2025-11-17T01:02:22.329278454Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth1"} +2025-11-17T01:02:22.329294443Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth1/speed", "error": "read /sys/class/net/eth1/speed: invalid argument"} +2025-11-17T01:02:22.329299323Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth1"} +2025-11-17T01:02:22.329360067Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.1"} +2025-11-17T01:02:22.329442761Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib3"} +2025-11-17T01:02:22.329492391Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib3"} +2025-11-17T01:02:22.329612299Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib3"} +2025-11-17T01:02:22.329803549Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:5c:00.0"} +2025-11-17T01:02:22.329927855Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib4"} +2025-11-17T01:02:22.329981189Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib4"} +2025-11-17T01:02:22.3301776Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib4"} +2025-11-17T01:02:22.330245217Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:85:00.0"} +2025-11-17T01:02:22.330328873Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib5"} +2025-11-17T01:02:22.330369095Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib5"} +2025-11-17T01:02:22.330556066Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib5"} +2025-11-17T01:02:22.330618565Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:8a:00.0"} +2025-11-17T01:02:22.330710508Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib6"} +2025-11-17T01:02:22.33075328Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib6"} +2025-11-17T01:02:22.330768319Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib6/speed", "error": "read /sys/class/net/ib6/speed: invalid argument"} +2025-11-17T01:02:22.330776323Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib6"} +2025-11-17T01:02:22.330834269Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:92:00.0"} +2025-11-17T01:02:22.330915846Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib7"} +2025-11-17T01:02:22.330958612Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib7"} +2025-11-17T01:02:22.330973755Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib7/speed", "error": "read /sys/class/net/ib7/speed: invalid argument"} +2025-11-17T01:02:22.33097887Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib7"} +2025-11-17T01:02:22.331037725Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:e3:00.0"} +2025-11-17T01:02:22.341534112Z INFO daemon/writer.go:166 setNodeStateStatus(): status {"sync-status": "InProgress", "last-sync-error": ""} +2025-11-17T01:02:38.675663748Z LEVEL(-2) mellanox/mellanox.go:81 RunCommand() {"output": "\nDevice #1:\n----------\n\nDevice type: ConnectX7 \nName: MCX75310AAS-NEA_Ax \nDescription: NVIDIA ConnectX-7 HHHL Adapter card; 400GbE / NDR IB (default mode); Single-port OSFP; PCIe 5.0 x16; Crypto Disabled; Secure Boot Enabled;\nDevice: 0000:3b:00.0 \n\nConfigurations: Default Current Next Boot\n MODULE_SPLIT_M0 Array[0..15] Array[0..15] Array[0..15] \n MEMIC_BAR_SIZE 0 0 0 \n MEMIC_SIZE_LIMIT _256KB(1) _256KB(1) _256KB(1) \n MEMIC_ATOMIC MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2)\n HOST_CHAINING_MODE DISABLED(0) DISABLED(0) DISABLED(0) \n HOST_CHAINING_CACHE_DISABLE False(0) False(0) False(0) \n HOST_CHAINING_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n HOST_CHAINING_TOTAL_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n FLEX_PARSER_PROFILE_ENABLE 0 0 0 \n PROG_PARSE_GRAPH False(0) False(0) False(0) \n FLEX_IPV4_OVER_VXLAN_PORT 0 0 0 \n ROCE_NEXT_PROTOCOL 254 254 254 \n ESWITCH_HAIRPIN_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n ESWITCH_HAIRPIN_TOT_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n DPA_AUTHENTICATION True(1) True(1) True(1) \n PF_BAR2_SIZE 0 0 0 \n PF_NUM_OF_VF_VALID False(0) False(0) False(0) \n NON_PREFETCHABLE_PF_BAR False(0) False(0) False(0) \n VF_VPD_ENABLE False(0) False(0) False(0) \n PF_NUM_PF_MSIX_VALID False(0) False(0) False(0) \n PER_PF_NUM_SF False(0) False(0) False(0) \n STRICT_VF_MSIX_NUM False(0) False(0) False(0) \n VF_NODNIC_ENABLE False(0) False(0) False(0) \n NUM_PF_MSIX_VALID True(1) True(1) True(1) \n* NUM_OF_VFS 16 16 1 \n NUM_OF_PF 1 1 1 \n PF_BAR2_ENABLE False(0) False(0) False(0) \n SRIOV_EN True(1) True(1) True(1) \n PF_LOG_BAR_SIZE 5 5 5 \n VF_LOG_BAR_SIZE 1 1 1 \n NUM_PF_MSIX 63 63 63 \n NUM_VF_MSIX 11 11 11 \n INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0) AUTOMATIC(0) AUTOMATIC(0) \n PCIE_CREDIT_TOKEN_TIMEOUT 0 0 0 \n RT_PPS_ENABLED_ON_POWERUP False(0) False(0) False(0) \n LAG_RESOURCE_ALLOCATION DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ACCURATE_TX_SCHEDULER False(0) False(0) False(0) \n PARTIAL_RESET_EN False(0) False(0) False(0) \n RESET_WITH_HOST_ON_ERRORS False(0) False(0) False(0) \n PCI_SWITCH_EMULATION_NUM_PORT 16 16 16 \n PCI_SWITCH_EMULATION_ENABLE False(0) False(0) False(0) \n PCI_DOWNSTREAM_PORT_OWNER Array[0..15] Array[0..15] Array[0..15] \n CQE_COMPRESSION BALANCED(0) BALANCED(0) BALANCED(0) \n IP_OVER_VXLAN_EN False(0) False(0) False(0) \n MKEY_BY_NAME False(0) False(0) False(0) \n PRIO_TAG_REQUIRED_EN False(0) False(0) False(0) \n UCTX_EN True(1) True(1) True(1) \n REAL_TIME_CLOCK_ENABLE False(0) False(0) False(0) \n RDMA_SELECTIVE_REPEAT_EN False(0) False(0) False(0) \n PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)\n TUNNEL_ECN_COPY_DISABLE False(0) False(0) False(0) \n LRO_LOG_TIMEOUT0 6 6 6 \n LRO_LOG_TIMEOUT1 7 7 7 \n LRO_LOG_TIMEOUT2 8 8 8 \n LRO_LOG_TIMEOUT3 13 13 13 \n LOG_TX_PSN_WINDOW 7 7 7 \n VF_MIGRATION_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n LOG_MAX_OUTSTANDING_WQE 7 7 7 \n ROCE_ADAPTIVE_ROUTING_EN False(0) False(0) False(0) \n TUNNEL_IP_PROTO_ENTROPY_DISABLE False(0) False(0) False(0) \n PCC_HANDLE_CORE_UTIL DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PCC_INT_NP_RTT_DSCP 26 26 26 \n PCC_INT_NP_RTT_DSCP_EN False(0) False(0) False(0) \n PCC_INT_NP_RTT_DATA_MODE RTT_V0(64) RTT_V0(64) RTT_V0(64) \n PCC_INT_EN False(0) False(0) False(0) \n PCC_INT_SYSTEM_RTT 0 0 0 \n STEERING_CACHE_REFRESH 0 0 0 \n ICM_CACHE_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n HAIRPIN_DATA_BUFFER_LOCK False(0) False(0) False(0) \n TX_SCHEDULER_BURST 0 0 0 \n ZERO_TOUCH_TUNING_ENABLE False(0) False(0) False(0) \n LOG_MAX_QUEUE 17 17 17 \n LARGE_MTU_TWEAK_64 False(0) False(0) False(0) \n AES_XTS_TWEAK_INC_64 False(0) False(0) False(0) \n* CRYPTO_POLICY UNRESTRICTED(1) DEVICE_DEFAULT(0) UNRESTRICTED(1) \n RDE_DISABLE False(0) False(0) False(0) \n PLDM_FW_UPDATE_DISABLE False(0) False(0) False(0) \n RBT_DISABLE False(0) False(0) False(0) \n PCIE_SMBUS_DISABLE False(0) False(0) False(0) \n PCIE_IN_BAND_VDM_DISABLE False(0) False(0) False(0) \n ADVANCED_TESTABILITY False(0) False(0) False(0) \n LOG_DCR_HASH_TABLE_SIZE 11 11 11 \n MAX_PACKET_LIFETIME 0 0 0 \n DCR_LIFO_SIZE 16384 16384 16384 \n LINK_TYPE_P1 IB(1) IB(1) IB(1) \n ROCE_CC_PRIO_MASK_P1 255 255 255 \n ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ROCE_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n IB_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1) True(1) True(1) \n CLAMP_TGT_RATE_P1 False(0) False(0) False(0) \n RPG_TIME_RESET_P1 300 300 300 \n RPG_BYTE_RESET_P1 32767 32767 32767 \n RPG_THRESHOLD_P1 1 1 1 \n RPG_MAX_RATE_P1 0 0 0 \n RPG_AI_RATE_P1 5 5 5 \n RPG_HAI_RATE_P1 50 50 50 \n RPG_GD_P1 11 11 11 \n RPG_MIN_DEC_FAC_P1 50 50 50 \n RPG_MIN_RATE_P1 1 1 1 \n RATE_TO_SET_ON_FIRST_CNP_P1 0 0 0 \n DCE_TCP_G_P1 1019 1019 1019 \n DCE_TCP_RTT_P1 1 1 1 \n RATE_REDUCE_MONITOR_PERIOD_P1 4 4 4 \n INITIAL_ALPHA_VALUE_P1 1023 1023 1023 \n MIN_TIME_BETWEEN_CNPS_P1 4 4 4 \n CNP_802P_PRIO_P1 6 6 6 \n CNP_DSCP_P1 48 48 48 \n LLDP_NB_DCBX_P1 False(0) False(0) False(0) \n LLDP_NB_RX_MODE_P1 OFF(0) OFF(0) OFF(0) \n LLDP_NB_TX_MODE_P1 OFF(0) OFF(0) OFF(0) \n ROCE_RTT_RESP_DSCP_P1 0 0 0 \n ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n DCBX_IEEE_P1 True(1) True(1) True(1) \n DCBX_CEE_P1 True(1) True(1) True(1) \n DCBX_WILLING_P1 True(1) True(1) True(1) \n KEEP_ETH_LINK_UP_P1 True(1) True(1) True(1) \n KEEP_IB_LINK_UP_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_BOOT_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_STANDBY_P1 False(0) False(0) False(0) \n DO_NOT_CLEAR_PORT_STATS_P1 False(0) False(0) False(0) \n AUTO_POWER_SAVE_LINK_DOWN_P1 False(0) False(0) False(0) \n NUM_OF_VL_P1 _4_VLs(3) _4_VLs(3) _4_VLs(3) \n NUM_OF_TC_P1 _8_TCs(0) _8_TCs(0) _8_TCs(0) \n NUM_OF_PFC_P1 8 8 8 \n VL15_BUFFER_SIZE_P1 0 0 0 \n QOS_TRUST_STATE_P1 TRUST_PCP(1) TRUST_PCP(1) TRUST_PCP(1) \n ETS_SCHED_MODE_P1 device_default(0) device_default(0) device_default(0) \n DUP_MAC_ACTION_P1 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) \n MPFS_MC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n MPFS_UC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n SRIOV_IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n PHY_AUTO_NEG_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PHY_RATE_MASK_OVERRIDE_P1 False(0) False(0) False(0) \n PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PF_TOTAL_SF 0 0 0 \n PF_SD_GROUP 0 0 0 \n PF_SF_BAR_SIZE 0 0 0 \n PF_NUM_PF_MSIX 63 63 63 \n SILENT_MODE False(0) False(0) False(0) \n MKEY_BY_NAME_RANGE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n* ROCE_CONTROL ROCE_ENABLE(2) DEVICE_DEFAULT(0) ROCE_ENABLE(2) \n PCI_WR_ORDERING per_mkey(0) per_mkey(0) per_mkey(0) \n MULTI_PORT_VHCA_EN False(0) False(0) False(0) \n PORT_OWNER True(1) True(1) True(1) \n ALLOW_RD_COUNTERS True(1) True(1) True(1) \n RENEG_ON_CHANGE True(1) True(1) True(1) \n TRACER_ENABLE True(1) True(1) True(1) \n IP_VER IPv4(0) IPv4(0) IPv4(0) \n BOOT_UNDI_NETWORK_WAIT 0 0 0 \n UEFI_HII_EN True(1) True(1) True(1) \n BOOT_DBG_LOG False(0) False(0) False(0) \n UEFI_LOGS DISABLED(0) DISABLED(0) DISABLED(0) \n BOOT_VLAN 1 1 1 \n LEGACY_BOOT_PROTOCOL PXE(1) PXE(1) PXE(1) \n BOOT_INTERRUPT_DIS False(0) False(0) False(0) \n BOOT_LACP_DIS True(1) True(1) True(1) \n BOOT_VLAN_EN False(0) False(0) False(0) \n BOOT_PKEY 0 0 0 \n P2P_ORDERING_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ATS_ENABLED False(0) False(0) False(0) \n DYNAMIC_VF_MSIX_TABLE False(0) False(0) False(0) \n EXP_ROM_UEFI_ARM_ENABLE True(1) True(1) True(1) \n EXP_ROM_UEFI_x86_ENABLE True(1) True(1) True(1) \n EXP_ROM_PXE_ENABLE True(1) True(1) True(1) \n ADVANCED_PCI_SETTINGS False(0) False(0) False(0) \n SAFE_MODE_THRESHOLD 10 10 10 \n SAFE_MODE_ENABLE True(1) True(1) True(1) \nThe '*' shows parameters with next value different from default/current value.\n", "error": null} +2025-11-17T01:02:38.675816845Z INFO mellanox/mellanox.go:213 ParseMstconfigOutput() {"attributes": ["NUM_OF_VFS", "SRIOV_EN", "LINK_TYPE_P1", "LINK_TYPE_P2"]} +2025-11-17T01:02:38.675864297Z INFO mellanox/mellanox.go:214 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"16","SRIOV_EN":"True(1)"}} +2025-11-17T01:02:38.675881642Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:02:38.675886047Z INFO mellanox/mellanox.go:219 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"1","SRIOV_EN":"True(1)"}} +2025-11-17T01:02:38.675890402Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:02:38.675893775Z INFO mellanox/mellanox_plugin.go:107 mellanox-plugin IsDualPort() {"address": "0000:3b:00.0"} +2025-11-17T01:02:38.67590068Z LEVEL(-2) mellanox/mellanox_plugin.go:112 Changing TotalVfs, needs reboot {"current": 16, "requested": 1} +2025-11-17T01:02:38.675906981Z INFO mellanox/mellanox.go:343 mellanox-plugin isLinkTypeRequireChange() {"device": "0000:3b:00.0"} +2025-11-17T01:02:38.675912389Z INFO mellanox/mellanox_plugin.go:102 mellanox-plugin getMlnxNicFwData() {"device": "0000:5c:00.0"} +2025-11-17T01:02:38.675916781Z INFO mellanox/mellanox.go:208 MstConfigReadData() {"device": "0000:5c:00.0"} +2025-11-17T01:02:38.675921227Z INFO mellanox/mellanox.go:81 RunCommand() {"command": "mstconfig", "args": ["-e", "-d", "0000:5c:00.0", "q"]} +2025-11-17T01:02:52.356139128Z LEVEL(-2) runtime/asm_amd64.s:1695 Run(): period refresh +2025-11-17T01:02:52.356203921Z LEVEL(-2) daemon/writer.go:109 pollNicStatus() +2025-11-17T01:02:52.356211177Z LEVEL(-2) daemon/writer.go:130 DiscoverSriovDevices +2025-11-17T01:02:52.461723367Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib0"} +2025-11-17T01:02:52.46186985Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib0"} +2025-11-17T01:02:52.461889413Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib0/speed", "error": "read /sys/class/net/ib0/speed: invalid argument"} +2025-11-17T01:02:52.461898342Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib0"} +2025-11-17T01:02:52.461965148Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:19:00.0"} +2025-11-17T01:02:52.462076028Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib1"} +2025-11-17T01:02:52.462119566Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib1"} +2025-11-17T01:02:52.462132755Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib1/speed", "error": "read /sys/class/net/ib1/speed: invalid argument"} +2025-11-17T01:02:52.462138098Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib1"} +2025-11-17T01:02:52.462192939Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:29:00.0"} +2025-11-17T01:02:52.462274837Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib2"} +2025-11-17T01:02:52.46232221Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib2"} +2025-11-17T01:02:52.462555422Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib2"} +2025-11-17T01:02:52.462615667Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:3b:00.0"} +2025-11-17T01:02:52.462766787Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth2"} +2025-11-17T01:02:52.463541012Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth2"} +2025-11-17T01:02:52.463601735Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.0"} +2025-11-17T01:02:52.463761509Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth3"} +2025-11-17T01:02:52.464392108Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth3"} +2025-11-17T01:02:52.464454781Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:53:00.1"} +2025-11-17T01:02:52.46454693Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth0"} +2025-11-17T01:02:52.464590409Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth0"} +2025-11-17T01:02:52.464605114Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth0/speed", "error": "read /sys/class/net/eth0/speed: invalid argument"} +2025-11-17T01:02:52.464610255Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth0"} +2025-11-17T01:02:52.464660447Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.0"} +2025-11-17T01:02:52.464728651Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "eth1"} +2025-11-17T01:02:52.464775062Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "eth1"} +2025-11-17T01:02:52.464789505Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/eth1/speed", "error": "read /sys/class/net/eth1/speed: invalid argument"} +2025-11-17T01:02:52.464794691Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "eth1"} +2025-11-17T01:02:52.464848368Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:56:00.1"} +2025-11-17T01:02:52.464918876Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib3"} +2025-11-17T01:02:52.464963608Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib3"} +2025-11-17T01:02:52.465169761Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib3"} +2025-11-17T01:02:52.465227071Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:5c:00.0"} +2025-11-17T01:02:52.465310876Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib4"} +2025-11-17T01:02:52.465359445Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib4"} +2025-11-17T01:02:52.465454338Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib4"} +2025-11-17T01:02:52.465534825Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:85:00.0"} +2025-11-17T01:02:52.465611083Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib5"} +2025-11-17T01:02:52.465656288Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib5"} +2025-11-17T01:02:52.465841321Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib5"} +2025-11-17T01:02:52.465903943Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:8a:00.0"} +2025-11-17T01:02:52.465980753Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib6"} +2025-11-17T01:02:52.466023129Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib6"} +2025-11-17T01:02:52.466037503Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib6/speed", "error": "read /sys/class/net/ib6/speed: invalid argument"} +2025-11-17T01:02:52.466042347Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib6"} +2025-11-17T01:02:52.46609594Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:92:00.0"} +2025-11-17T01:02:52.466171812Z LEVEL(-2) sriov/sriov.go:256 TryGetInterfaceName() {"name": "ib7"} +2025-11-17T01:02:52.466212806Z LEVEL(-2) sriov/sriov.go:278 GetNetDevLinkSpeed(): get LinkSpeed {"device": "ib7"} +2025-11-17T01:02:52.466226951Z ERROR sriov/sriov.go:278 GetNetDevLinkSpeed(): fail to read Link Speed file {"path": "/sys/class/net/ib7/speed", "error": "read /sys/class/net/ib7/speed: invalid argument"} +2025-11-17T01:02:52.466231779Z LEVEL(-2) sriov/sriov.go:279 GetNetDevLinkAdminState(): get LinkAdminState {"device": "ib7"} +2025-11-17T01:02:52.466285062Z LEVEL(-2) sriov/sriov.go:294 GetNicSriovMode() {"device": "0000:e3:00.0"} +2025-11-17T01:02:52.477174067Z INFO daemon/writer.go:166 setNodeStateStatus(): status {"sync-status": "InProgress", "last-sync-error": ""} +2025-11-17T01:03:03.447705748Z LEVEL(-2) mellanox/mellanox.go:81 RunCommand() {"output": "\nDevice #1:\n----------\n\nDevice type: ConnectX7 \nName: MCX75310AAS-NEA_Ax \nDescription: NVIDIA ConnectX-7 HHHL Adapter card; 400GbE / NDR IB (default mode); Single-port OSFP; PCIe 5.0 x16; Crypto Disabled; Secure Boot Enabled;\nDevice: 0000:5c:00.0 \n\nConfigurations: Default Current Next Boot\n MODULE_SPLIT_M0 Array[0..15] Array[0..15] Array[0..15] \n MEMIC_BAR_SIZE 0 0 0 \n MEMIC_SIZE_LIMIT _256KB(1) _256KB(1) _256KB(1) \n MEMIC_ATOMIC MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2) MEMIC_ATOMIC_ENABLE(2)\n HOST_CHAINING_MODE DISABLED(0) DISABLED(0) DISABLED(0) \n HOST_CHAINING_CACHE_DISABLE False(0) False(0) False(0) \n HOST_CHAINING_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n HOST_CHAINING_TOTAL_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n FLEX_PARSER_PROFILE_ENABLE 0 0 0 \n PROG_PARSE_GRAPH False(0) False(0) False(0) \n FLEX_IPV4_OVER_VXLAN_PORT 0 0 0 \n ROCE_NEXT_PROTOCOL 254 254 254 \n ESWITCH_HAIRPIN_DESCRIPTORS Array[0..7] Array[0..7] Array[0..7] \n ESWITCH_HAIRPIN_TOT_BUFFER_SIZE Array[0..7] Array[0..7] Array[0..7] \n DPA_AUTHENTICATION True(1) True(1) True(1) \n PF_BAR2_SIZE 0 0 0 \n PF_NUM_OF_VF_VALID False(0) False(0) False(0) \n NON_PREFETCHABLE_PF_BAR False(0) False(0) False(0) \n VF_VPD_ENABLE False(0) False(0) False(0) \n PF_NUM_PF_MSIX_VALID False(0) False(0) False(0) \n PER_PF_NUM_SF False(0) False(0) False(0) \n STRICT_VF_MSIX_NUM False(0) False(0) False(0) \n VF_NODNIC_ENABLE False(0) False(0) False(0) \n NUM_PF_MSIX_VALID True(1) True(1) True(1) \n* NUM_OF_VFS 16 16 1 \n NUM_OF_PF 1 1 1 \n PF_BAR2_ENABLE False(0) False(0) False(0) \n SRIOV_EN True(1) True(1) True(1) \n PF_LOG_BAR_SIZE 5 5 5 \n VF_LOG_BAR_SIZE 1 1 1 \n NUM_PF_MSIX 63 63 63 \n NUM_VF_MSIX 11 11 11 \n INT_LOG_MAX_PAYLOAD_SIZE AUTOMATIC(0) AUTOMATIC(0) AUTOMATIC(0) \n PCIE_CREDIT_TOKEN_TIMEOUT 0 0 0 \n RT_PPS_ENABLED_ON_POWERUP False(0) False(0) False(0) \n LAG_RESOURCE_ALLOCATION DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ACCURATE_TX_SCHEDULER False(0) False(0) False(0) \n PARTIAL_RESET_EN False(0) False(0) False(0) \n RESET_WITH_HOST_ON_ERRORS False(0) False(0) False(0) \n PCI_SWITCH_EMULATION_NUM_PORT 16 16 16 \n PCI_SWITCH_EMULATION_ENABLE False(0) False(0) False(0) \n PCI_DOWNSTREAM_PORT_OWNER Array[0..15] Array[0..15] Array[0..15] \n CQE_COMPRESSION BALANCED(0) BALANCED(0) BALANCED(0) \n IP_OVER_VXLAN_EN False(0) False(0) False(0) \n MKEY_BY_NAME False(0) False(0) False(0) \n PRIO_TAG_REQUIRED_EN False(0) False(0) False(0) \n UCTX_EN True(1) True(1) True(1) \n REAL_TIME_CLOCK_ENABLE False(0) False(0) False(0) \n RDMA_SELECTIVE_REPEAT_EN False(0) False(0) False(0) \n PCI_ATOMIC_MODE PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0) PCI_ATOMIC_DISABLED_EXT_ATOMIC_ENABLED(0)\n TUNNEL_ECN_COPY_DISABLE False(0) False(0) False(0) \n LRO_LOG_TIMEOUT0 6 6 6 \n LRO_LOG_TIMEOUT1 7 7 7 \n LRO_LOG_TIMEOUT2 8 8 8 \n LRO_LOG_TIMEOUT3 13 13 13 \n LOG_TX_PSN_WINDOW 7 7 7 \n VF_MIGRATION_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n LOG_MAX_OUTSTANDING_WQE 7 7 7 \n ROCE_ADAPTIVE_ROUTING_EN False(0) False(0) False(0) \n TUNNEL_IP_PROTO_ENTROPY_DISABLE False(0) False(0) False(0) \n PCC_HANDLE_CORE_UTIL DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PCC_INT_NP_RTT_DSCP 26 26 26 \n PCC_INT_NP_RTT_DSCP_EN False(0) False(0) False(0) \n PCC_INT_NP_RTT_DATA_MODE RTT_V0(64) RTT_V0(64) RTT_V0(64) \n PCC_INT_EN False(0) False(0) False(0) \n PCC_INT_SYSTEM_RTT 0 0 0 \n STEERING_CACHE_REFRESH 0 0 0 \n ICM_CACHE_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n HAIRPIN_DATA_BUFFER_LOCK False(0) False(0) False(0) \n TX_SCHEDULER_BURST 0 0 0 \n ZERO_TOUCH_TUNING_ENABLE False(0) False(0) False(0) \n LOG_MAX_QUEUE 17 17 17 \n LARGE_MTU_TWEAK_64 False(0) False(0) False(0) \n AES_XTS_TWEAK_INC_64 False(0) False(0) False(0) \n* CRYPTO_POLICY UNRESTRICTED(1) DEVICE_DEFAULT(0) UNRESTRICTED(1) \n RDE_DISABLE False(0) False(0) False(0) \n PLDM_FW_UPDATE_DISABLE False(0) False(0) False(0) \n RBT_DISABLE False(0) False(0) False(0) \n PCIE_SMBUS_DISABLE False(0) False(0) False(0) \n PCIE_IN_BAND_VDM_DISABLE False(0) False(0) False(0) \n ADVANCED_TESTABILITY False(0) False(0) False(0) \n LOG_DCR_HASH_TABLE_SIZE 11 11 11 \n MAX_PACKET_LIFETIME 0 0 0 \n DCR_LIFO_SIZE 16384 16384 16384 \n LINK_TYPE_P1 IB(1) IB(1) IB(1) \n ROCE_CC_PRIO_MASK_P1 255 255 255 \n ROCE_CC_CNP_MODERATION_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ROCE_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n IB_CC_SHAPER_COALESCE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n CLAMP_TGT_RATE_AFTER_TIME_INC_P1 True(1) True(1) True(1) \n CLAMP_TGT_RATE_P1 False(0) False(0) False(0) \n RPG_TIME_RESET_P1 300 300 300 \n RPG_BYTE_RESET_P1 32767 32767 32767 \n RPG_THRESHOLD_P1 1 1 1 \n RPG_MAX_RATE_P1 0 0 0 \n RPG_AI_RATE_P1 5 5 5 \n RPG_HAI_RATE_P1 50 50 50 \n RPG_GD_P1 11 11 11 \n RPG_MIN_DEC_FAC_P1 50 50 50 \n RPG_MIN_RATE_P1 1 1 1 \n RATE_TO_SET_ON_FIRST_CNP_P1 0 0 0 \n DCE_TCP_G_P1 1019 1019 1019 \n DCE_TCP_RTT_P1 1 1 1 \n RATE_REDUCE_MONITOR_PERIOD_P1 4 4 4 \n INITIAL_ALPHA_VALUE_P1 1023 1023 1023 \n MIN_TIME_BETWEEN_CNPS_P1 4 4 4 \n CNP_802P_PRIO_P1 6 6 6 \n CNP_DSCP_P1 48 48 48 \n LLDP_NB_DCBX_P1 False(0) False(0) False(0) \n LLDP_NB_RX_MODE_P1 OFF(0) OFF(0) OFF(0) \n LLDP_NB_TX_MODE_P1 OFF(0) OFF(0) OFF(0) \n ROCE_RTT_RESP_DSCP_P1 0 0 0 \n ROCE_RTT_RESP_DSCP_MODE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n DCBX_IEEE_P1 True(1) True(1) True(1) \n DCBX_CEE_P1 True(1) True(1) True(1) \n DCBX_WILLING_P1 True(1) True(1) True(1) \n KEEP_ETH_LINK_UP_P1 True(1) True(1) True(1) \n KEEP_IB_LINK_UP_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_BOOT_P1 False(0) False(0) False(0) \n KEEP_LINK_UP_ON_STANDBY_P1 False(0) False(0) False(0) \n DO_NOT_CLEAR_PORT_STATS_P1 False(0) False(0) False(0) \n AUTO_POWER_SAVE_LINK_DOWN_P1 False(0) False(0) False(0) \n NUM_OF_VL_P1 _4_VLs(3) _4_VLs(3) _4_VLs(3) \n NUM_OF_TC_P1 _8_TCs(0) _8_TCs(0) _8_TCs(0) \n NUM_OF_PFC_P1 8 8 8 \n VL15_BUFFER_SIZE_P1 0 0 0 \n QOS_TRUST_STATE_P1 TRUST_PCP(1) TRUST_PCP(1) TRUST_PCP(1) \n ETS_SCHED_MODE_P1 device_default(0) device_default(0) device_default(0) \n DUP_MAC_ACTION_P1 LAST_CFG(0) LAST_CFG(0) LAST_CFG(0) \n MPFS_MC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n MPFS_UC_LOOPBACK_DISABLE_P1 False(0) False(0) False(0) \n SRIOV_IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n IB_ROUTING_MODE_P1 LID(1) LID(1) LID(1) \n PHY_AUTO_NEG_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PHY_RATE_MASK_OVERRIDE_P1 False(0) False(0) False(0) \n PHY_FEC_OVERRIDE_P1 DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n PF_TOTAL_SF 0 0 0 \n PF_SD_GROUP 0 0 0 \n PF_SF_BAR_SIZE 0 0 0 \n PF_NUM_PF_MSIX 63 63 63 \n SILENT_MODE False(0) False(0) False(0) \n MKEY_BY_NAME_RANGE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n* ROCE_CONTROL ROCE_ENABLE(2) DEVICE_DEFAULT(0) ROCE_ENABLE(2) \n PCI_WR_ORDERING per_mkey(0) per_mkey(0) per_mkey(0) \n MULTI_PORT_VHCA_EN False(0) False(0) False(0) \n PORT_OWNER True(1) True(1) True(1) \n ALLOW_RD_COUNTERS True(1) True(1) True(1) \n RENEG_ON_CHANGE True(1) True(1) True(1) \n TRACER_ENABLE True(1) True(1) True(1) \n IP_VER IPv4(0) IPv4(0) IPv4(0) \n BOOT_UNDI_NETWORK_WAIT 0 0 0 \n UEFI_HII_EN True(1) True(1) True(1) \n BOOT_DBG_LOG False(0) False(0) False(0) \n UEFI_LOGS DISABLED(0) DISABLED(0) DISABLED(0) \n BOOT_VLAN 1 1 1 \n LEGACY_BOOT_PROTOCOL PXE(1) PXE(1) PXE(1) \n BOOT_INTERRUPT_DIS False(0) False(0) False(0) \n BOOT_LACP_DIS True(1) True(1) True(1) \n BOOT_VLAN_EN False(0) False(0) False(0) \n BOOT_PKEY 0 0 0 \n P2P_ORDERING_MODE DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) DEVICE_DEFAULT(0) \n ATS_ENABLED False(0) False(0) False(0) \n DYNAMIC_VF_MSIX_TABLE False(0) False(0) False(0) \n EXP_ROM_UEFI_ARM_ENABLE True(1) True(1) True(1) \n EXP_ROM_UEFI_x86_ENABLE True(1) True(1) True(1) \n EXP_ROM_PXE_ENABLE True(1) True(1) True(1) \n ADVANCED_PCI_SETTINGS False(0) False(0) False(0) \n SAFE_MODE_THRESHOLD 10 10 10 \n SAFE_MODE_ENABLE True(1) True(1) True(1) \nThe '*' shows parameters with next value different from default/current value.\n", "error": null} +2025-11-17T01:03:03.447861825Z INFO mellanox/mellanox.go:213 ParseMstconfigOutput() {"attributes": ["NUM_OF_VFS", "SRIOV_EN", "LINK_TYPE_P1", "LINK_TYPE_P2"]} +2025-11-17T01:03:03.447923652Z INFO mellanox/mellanox.go:214 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"16","SRIOV_EN":"True(1)"}} +2025-11-17T01:03:03.447942645Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:03:03.447947322Z INFO mellanox/mellanox.go:219 mellanox-plugin mlnxNicFromMap() {"data": {"LINK_TYPE_P1":"IB(1)","NUM_OF_VFS":"1","SRIOV_EN":"True(1)"}} +2025-11-17T01:03:03.447952892Z INFO mellanox/mellanox.go:387 mellanox-plugin getLinkType() {"link-type": "IB(1)"} +2025-11-17T01:03:03.447957009Z INFO mellanox/mellanox_plugin.go:107 mellanox-plugin IsDualPort() {"address": "0000:5c:00.0"} +2025-11-17T01:03:03.447963685Z LEVEL(-2) mellanox/mellanox_plugin.go:112 Changing TotalVfs, needs reboot {"current": 16, "requested": 1} +2025-11-17T01:03:03.447972246Z INFO mellanox/mellanox.go:343 mellanox-plugin isLinkTypeRequireChange() {"device": "0000:5c:00.0"} +2025-11-17T01:03:03.447997403Z LEVEL(-2) daemon/daemon.go:421 None of the ports are configured by the operator skipping firmware reset {"portMap": {"0000:53:00.0":{"name":"eth2","mac":"58:a2:e1:12:20:ec","driver":"mlx5_core","pciAddress":"0000:53:00.0","vendor":"15b3","deviceID":"101d","mtu":1500,"linkSpeed":"100000 Mb/s","linkType":"ETH","linkAdminState":"up","eSwitchMode":"legacy","totalvfs":8},"0000:53:00.1":{"name":"eth3","mac":"58:a2:e1:12:20:ec","driver":"mlx5_core","pciAddress":"0000:53:00.1","vendor":"15b3","deviceID":"101d","mtu":1500,"linkSpeed":"100000 Mb/s","linkType":"ETH","linkAdminState":"up","eSwitchMode":"legacy","totalvfs":8}}} +2025-11-17T01:03:03.448030724Z LEVEL(-2) daemon/daemon.go:421 mellanox plugin {"need-drain": true, "need-reboot": true} +2025-11-17T01:03:03.448037673Z INFO daemon/daemon.go:264 nodeStateSyncHandler(): OnNodeStateChange result {"plugin": "mellanox", "drain-required": true, "reboot-required": true} +2025-11-17T01:03:03.448044344Z INFO daemon/daemon.go:264 nodeStateSyncHandler(): calling OnNodeStateChange for a new node state +2025-11-17T01:03:03.448048132Z INFO daemon/daemon.go:421 intel-plugin OnNodeStateChange() +2025-11-17T01:03:03.448052124Z INFO daemon/daemon.go:264 nodeStateSyncHandler(): OnNodeStateChange result {"plugin": "intel", "drain-required": false, "reboot-required": false} +2025-11-17T01:03:03.448056193Z INFO daemon/daemon.go:264 nodeStateSyncHandler(): calling OnNodeStateChange for a new node state +2025-11-17T01:03:03.448059616Z INFO daemon/daemon.go:421 k8s plugin OnNodeStateChange() +2025-11-17T01:03:03.448064467Z INFO daemon/daemon.go:264 nodeStateSyncHandler(): OnNodeStateChange result {"plugin": "k8s", "drain-required": false, "reboot-required": false} +2025-11-17T01:03:03.448068145Z INFO daemon/daemon.go:264 nodeStateSyncHandler(): calling OnNodeStateChange for a new node state +2025-11-17T01:03:03.4480721Z INFO daemon/daemon.go:421 generic plugin OnNodeStateChange() +2025-11-17T01:03:03.448077648Z LEVEL(-2) generic/generic_plugin.go:157 generic plugin needDrainNode() {"current": {"interfaces":[{"name":"ib0","mac":"00:00:10:4a:fe:80:00:00:00:00:00:00:a0:88:c2:03:00:d5:8e:fa","driver":"mlx5_core","pciAddress":"0000:19:00.0","vendor":"15b3","deviceID":"1021","mtu":4092,"linkType":"IB","linkAdminState":"down","eSwitchMode":"legacy","totalvfs":16},{"name":"ib1","mac":"00:00:10:4a:fe:80:00:00:00:00:00:00:a0:88:c2:03:00:d6:34:fa","driver":"mlx5_core","pciAddress":"0000:29:00.0","vendor":"15b3","deviceID":"1021","mtu":4092,"linkType":"IB","linkAdminState":"down","eSwitchMode":"legacy","totalvfs":16},{"name":"ib2","mac":"00:00:10:4a:fe:80:00:00:00:00:00:00:a0:88:c2:03:00:d6:34:72","driver":"mlx5_core","pciAddress":"0000:3b:00.0","vendor":"15b3","deviceID":"1021","mtu":2044,"linkSpeed":"400000 Mb/s","linkType":"IB","linkAdminState":"up","eSwitchMode":"legacy","totalvfs":16},{"name":"eth2","mac":"58:a2:e1:12:20:ec","driver":"mlx5_core","pciAddress":"0000:53:00.0","vendor":"15b3","deviceID":"101d","mtu":1500,"linkSpeed":"100000 Mb/s","linkType":"ETH","linkAdminState":"up","eSwitchMode":"legacy","totalvfs":8},{"name":"eth3","mac":"58:a2:e1:12:20:ec","driver":"mlx5_core","pciAddress":"0000:53:00.1","vendor":"15b3","deviceID":"101d","mtu":1500,"linkSpeed":"100000 Mb/s","linkType":"ETH","linkAdminState":"up","eSwitchMode":"legacy","totalvfs":8},{"name":"eth0","mac":"7c:c2:55:7b:32:ce","driver":"ixgbe","pciAddress":"0000:56:00.0","vendor":"8086","deviceID":"1563","mtu":1500,"linkType":"ETH","linkAdminState":"down","eSwitchMode":"legacy","totalvfs":63},{"name":"eth1","mac":"7c:c2:55:7b:32:cf","driver":"ixgbe","pciAddress":"0000:56:00.1","vendor":"8086","deviceID":"1563","mtu":1500,"linkType":"ETH","linkAdminState":"down","eSwitchMode":"legacy","totalvfs":63},{"name":"ib3","mac":"00:00:10:4a:fe:80:00:00:00:00:00:00:a0:88:c2:03:00:d6:36:72","driver":"mlx5_core","pciAddress":"0000:5c:00.0","vendor":"15b3","deviceID":"1021","mtu":2044,"linkSpeed":"400000 Mb/s","linkType":"IB","linkAdminState":"up","eSwitchMode":"legacy","totalvfs":16},{"name":"ib4","mac":"00:00:10:4a:fe:80:00:00:00:00:00:00:a0:88:c2:03:00:d6:44:02","driver":"mlx5_core","pciAddress":"0000:85:00.0","vendor":"15b3","deviceID":"1021","mtu":2044,"linkSpeed":"400000 Mb/s","linkType":"IB","linkAdminState":"up","eSwitchMode":"legacy","totalvfs":16},{"name":"ib5","mac":"00:00:10:4a:fe:80:00:00:00:00:00:00:a0:88:c2:03:00:d6:3d:ba","driver":"mlx5_core","pciAddress":"0000:8a:00.0","vendor":"15b3","deviceID":"1021","mtu":2044,"linkSpeed":"400000 Mb/s","linkType":"IB","linkAdminState":"up","eSwitchMode":"legacy","totalvfs":16},{"name":"ib6","mac":"00:00:10:49:fe:80:00:00:00:00:00:00:a0:88:c2:03:00:d6:44:0a","driver":"mlx5_core","pciAddress":"0000:92:00.0","vendor":"15b3","deviceID":"1021","mtu":4092,"linkType":"IB","linkAdminState":"down","eSwitchMode":"legacy","totalvfs":16},{"name":"ib7","mac":"00:00:10:49:fe:80:00:00:00:00:00:00:a0:88:c2:03:00:d6:44:4a","driver":"mlx5_core","pciAddress":"0000:e3:00.0","vendor":"15b3","deviceID":"1021","mtu":4092,"linkType":"IB","linkAdminState":"down","eSwitchMode":"legacy","totalvfs":16}],"bridges":{},"system":{"rdmaMode":"shared"},"syncStatus":"InProgress"}, "desired": {"interfaces":[{"pciAddress":"0000:19:00.0","numVfs":1,"mtu":1500,"name":"ib0","linkType":"IB","vfGroups":[{"resourceName":"mlnxnics","deviceType":"netdevice","vfRange":"0-0","policyName":"infiniband-sriov","mtu":1500,"isRdma":true}]},{"pciAddress":"0000:29:00.0","numVfs":1,"mtu":1500,"name":"ib1","linkType":"IB","vfGroups":[{"resourceName":"mlnxnics","deviceType":"netdevice","vfRange":"0-0","policyName":"infiniband-sriov","mtu":1500,"isRdma":true}]},{"pciAddress":"0000:3b:00.0","numVfs":1,"mtu":1500,"name":"ib2","linkType":"IB","vfGroups":[{"resourceName":"mlnxnics","deviceType":"netdevice","vfRange":"0-0","policyName":"infiniband-sriov","mtu":1500,"isRdma":true}]},{"pciAddress":"0000:5c:00.0","numVfs":1,"mtu":1500,"name":"ib3","linkType":"IB","vfGroups":[{"resourceName":"mlnxnics","deviceType":"netdevice","vfRange":"0-0","policyName":"infiniband-sriov","mtu":1500,"isRdma":true}]},{"pciAddress":"0000:85:00.0","numVfs":1,"mtu":1500,"name":"ib4","linkType":"IB","vfGroups":[{"resourceName":"mlnxnics","deviceType":"netdevice","vfRange":"0-0","policyName":"infiniband-sriov","mtu":1500,"isRdma":true}]},{"pciAddress":"0000:8a:00.0","numVfs":1,"mtu":1500,"name":"ib5","linkType":"IB","vfGroups":[{"resourceName":"mlnxnics","deviceType":"netdevice","vfRange":"0-0","policyName":"infiniband-sriov","mtu":1500,"isRdma":true}]},{"pciAddress":"0000:92:00.0","numVfs":1,"mtu":1500,"name":"ib6","linkType":"IB","vfGroups":[{"resourceName":"mlnxnics","deviceType":"netdevice","vfRange":"0-0","policyName":"infiniband-sriov","mtu":1500,"isRdma":true}]},{"pciAddress":"0000:e3:00.0","numVfs":1,"mtu":1500,"name":"ib7","linkType":"IB","vfGroups":[{"resourceName":"mlnxnics","deviceType":"netdevice","vfRange":"0-0","policyName":"infiniband-sriov","mtu":1500,"isRdma":true}]}],"bridges":{},"system":{}}} +2025-11-17T01:03:03.448109992Z LEVEL(-2) generic/generic_plugin.go:367 generic plugin needToUpdateVFs(): no need drain, for PCI address, current NumVfs is 0 {"address": "0000:19:00.0"} +2025-11-17T01:03:03.448114159Z LEVEL(-2) generic/generic_plugin.go:367 generic plugin needToUpdateVFs(): no need drain, for PCI address, current NumVfs is 0 {"address": "0000:29:00.0"} +2025-11-17T01:03:03.448118366Z LEVEL(-2) generic/generic_plugin.go:367 generic plugin needToUpdateVFs(): no need drain, for PCI address, current NumVfs is 0 {"address": "0000:3b:00.0"} +2025-11-17T01:03:03.448122266Z LEVEL(-2) generic/generic_plugin.go:367 generic plugin needToUpdateVFs(): no need drain, for PCI address, current NumVfs is 0 {"address": "0000:5c:00.0"} +2025-11-17T01:03:03.448126328Z LEVEL(-2) generic/generic_plugin.go:367 generic plugin needToUpdateVFs(): no need drain, for PCI address, current NumVfs is 0 {"address": "0000:85:00.0"} +2025-11-17T01:03:03.448129397Z LEVEL(-2) generic/generic_plugin.go:367 generic plugin needToUpdateVFs(): no need drain, for PCI address, current NumVfs is 0 {"address": "0000:8a:00.0"} +2025-11-17T01:03:03.448132001Z LEVEL(-2) generic/generic_plugin.go:367 generic plugin needToUpdateVFs(): no need drain, for PCI address, current NumVfs is 0 {"address": "0000:92:00.0"} +2025-11-17T01:03:03.448135186Z LEVEL(-2) generic/generic_plugin.go:367 generic plugin needToUpdateVFs(): no need drain, for PCI address, current NumVfs is 0 {"address": "0000:e3:00.0"} +2025-11-17T01:03:03.448138789Z INFO generic/generic_plugin.go:467 generic plugin disableDesiredKernelArgs(): disable kernel arg {"karg": "ib_core.netns_mode=0"} +2025-11-17T01:03:03.448143632Z INFO generic/generic_plugin.go:468 generic plugin disableDesiredKernelArgs(): disable kernel arg {"karg": "ib_core.netns_mode=1"} +2025-11-17T01:03:03.448148135Z INFO generic/generic_plugin.go:481 SetRDMASubsystem(): Updating RDMA subsystem mode {"mode": ""} +2025-11-17T01:03:03.448190634Z INFO generic/generic_plugin.go:350 generic plugin editKernelArg() {"mode": "remove", "karg": "intel_iommu=on"} +2025-11-17T01:03:03.448202959Z INFO generic/generic_plugin.go:285 RunCommand() {"command": "/bin/bash", "args": ["/bindata/scripts/kargs.sh", "remove", "intel_iommu=on"]} +2025-11-17T01:03:03.871911435Z LEVEL(-2) generic/generic_plugin.go:285 RunCommand() {"output": "0\n", "error": null} +2025-11-17T01:03:03.871946843Z INFO generic/generic_plugin.go:350 generic plugin editKernelArg() {"mode": "remove", "karg": "iommu=pt"} +2025-11-17T01:03:03.871954055Z INFO generic/generic_plugin.go:285 RunCommand() {"command": "/bin/bash", "args": ["/bindata/scripts/kargs.sh", "remove", "iommu=pt"]} +2025-11-17T01:03:04.269322735Z LEVEL(-2) generic/generic_plugin.go:285 RunCommand() {"output": "0\n", "error": null} +2025-11-17T01:03:04.269358343Z INFO generic/generic_plugin.go:350 generic plugin editKernelArg() {"mode": "remove", "karg": "iommu.passthrough=1"} +2025-11-17T01:03:04.269365077Z INFO generic/generic_plugin.go:285 RunCommand() {"command": "/bin/bash", "args": ["/bindata/scripts/kargs.sh", "remove", "iommu.passthrough=1"]} +2025-11-17T01:03:04.673632149Z LEVEL(-2) generic/generic_plugin.go:285 RunCommand() {"output": "0\n", "error": null} +2025-11-17T01:03:04.67367179Z INFO generic/generic_plugin.go:350 generic plugin editKernelArg() {"mode": "remove", "karg": "ib_core.netns_mode=1"} +2025-11-17T01:03:04.673678917Z INFO generic/generic_plugin.go:285 RunCommand() {"command": "/bin/bash", "args": ["/bindata/scripts/kargs.sh", "remove", "ib_core.netns_mode=1"]} +2025-11-17T01:03:05.094428914Z LEVEL(-2) generic/generic_plugin.go:285 RunCommand() {"output": "0\n", "error": null} +2025-11-17T01:03:05.094466786Z INFO generic/generic_plugin.go:350 generic plugin editKernelArg() {"mode": "remove", "karg": "ib_core.netns_mode=0"} +2025-11-17T01:03:05.094475127Z INFO generic/generic_plugin.go:285 RunCommand() {"command": "/bin/bash", "args": ["/bindata/scripts/kargs.sh", "remove", "ib_core.netns_mode=0"]} +2025-11-17T01:03:05.498012314Z LEVEL(-2) generic/generic_plugin.go:285 RunCommand() {"output": "0\n", "error": null} +2025-11-17T01:03:05.498071319Z INFO generic/generic_plugin.go:350 generic plugin editKernelArg() {"mode": "remove", "karg": "pci=realloc"} +2025-11-17T01:03:05.498078857Z INFO generic/generic_plugin.go:285 RunCommand() {"command": "/bin/bash", "args": ["/bindata/scripts/kargs.sh", "remove", "pci=realloc"]} +2025-11-17T01:03:05.894527205Z LEVEL(-2) generic/generic_plugin.go:285 RunCommand() {"output": "0\n", "error": null} +2025-11-17T01:03:05.89456556Z INFO daemon/daemon.go:264 nodeStateSyncHandler(): OnNodeStateChange result {"plugin": "generic", "drain-required": false, "reboot-required": false} +2025-11-17T01:03:05.894578909Z INFO daemon/daemon.go:264 nodeStateSyncHandler(): aggregated daemon {"drain-required": true, "reboot-required": true, "disable-drain": false} +2025-11-17T01:03:05.894585486Z INFO daemon/daemon.go:470 handleDrain(): the node complete the draining +2025-11-17T01:03:05.894597988Z INFO kernel/kernel.go:304 RunCommand() {"command": "cat", "args": ["/host/sys/kernel/security/lockdown"]} +2025-11-17T01:03:05.895575799Z LEVEL(-2) kernel/kernel.go:304 RunCommand() {"output": "[none] integrity confidentiality\n", "error": null} +2025-11-17T01:03:05.89558445Z LEVEL(-2) mellanox/mellanox_plugin.go:207 IsKernelLockdownMode() {"output": "[none] integrity confidentiality\n", "error": null} +2025-11-17T01:03:05.895592404Z INFO daemon/daemon.go:484 mellanox plugin Apply() +2025-11-17T01:03:05.89559818Z INFO mellanox/mellanox_plugin.go:212 mellanox-plugin configFW() +2025-11-17T01:03:05.895604278Z LEVEL(-2) mellanox/mellanox.go:166 MellanoxBlueFieldMode(): checking mode for device {"device": "0000:8a:00.0"} +2025-11-17T01:03:05.895609468Z INFO mellanox/mellanox.go:87 MstConfigReadData() {"device": "0000:8a:00.0"} +2025-11-17T01:03:05.895614624Z INFO mellanox/mellanox.go:81 RunCommand() {"command": "mstconfig", "args": ["-e", "-d", "0000:8a:00.0", "q"]} diff --git a/network-operator/templates/NOTES.txt b/network-operator/templates/NOTES.txt new file mode 100644 index 0000000..6f03935 --- /dev/null +++ b/network-operator/templates/NOTES.txt @@ -0,0 +1,3 @@ +Get Network Operator deployed resources by running the following commands: + +$ kubectl -n {{ .Release.Namespace }} get pods diff --git a/network-operator/templates/_helpers.tpl b/network-operator/templates/_helpers.tpl new file mode 100644 index 0000000..198e47a --- /dev/null +++ b/network-operator/templates/_helpers.tpl @@ -0,0 +1,86 @@ +{{/* + Copyright 2020 NVIDIA + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "network-operator.name" -}} +{{- default .Chart.Name .Values.operator.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "network-operator.fullname" -}} +{{- if .Values.operator.fullnameOverride }} +{{- .Values.operator.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.operator.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "network-operator.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "network-operator.labels" -}} +helm.sh/chart: {{ include "network-operator.chart" . }} +{{ include "network-operator.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "network-operator.selectorLabels" -}} +app.kubernetes.io/name: {{ include "network-operator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +imagePullSecrets helpers +*/}} +{{- define "network-operator.operator.imagePullSecrets" }} +{{- $imagePullSecrets := list }} +{{- if .Values.operator.imagePullSecrets }} +{{- range .Values.operator.imagePullSecrets }} +{{- $imagePullSecrets = append $imagePullSecrets (dict "name" . ) }} +{{- end }} +{{- else }} +{{- if .Values.imagePullSecrets }} +{{- range .Values.imagePullSecrets }} +{{- $imagePullSecrets = append $imagePullSecrets (dict "name" . ) }} +{{- end }} +{{- end }} +{{- end }} +{{- $imagePullSecrets | toJson }} +{{- end }} \ No newline at end of file diff --git a/network-operator/templates/admission_controller.yaml b/network-operator/templates/admission_controller.yaml new file mode 100644 index 0000000..a619c02 --- /dev/null +++ b/network-operator/templates/admission_controller.yaml @@ -0,0 +1,129 @@ +--- +{{- if .Values.operator.admissionController.enabled }} +apiVersion: v1 +kind: Service +metadata: + name: {{ .Release.Name }}-webhook-service + namespace: {{ .Release.Namespace }} +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 9443 + selector: + control-plane: {{ .Release.Name }}-controller +{{- end }} +--- +{{- if and .Values.operator.admissionController.enabled .Values.operator.admissionController.useCertManager }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + labels: + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: {{ .Release.Name }} + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/name: certificate + app.kubernetes.io/part-of: {{ .Release.Name }} + name: {{ .Release.Name }}-serving-cert + namespace: {{ .Release.Namespace }} +spec: + dnsNames: + - {{ .Release.Name }}-webhook-service.{{ .Release.Namespace }}.svc + - {{ .Release.Name }}-webhook-service.{{ .Release.Namespace }}.svc.cluster.local + issuerRef: + kind: Issuer + name: {{ .Release.Name }}-selfsigned-issuer + secretName: webhook-server-cert +{{- end }} +--- +{{- if and .Values.operator.admissionController.enabled .Values.operator.admissionController.useCertManager }} +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + labels: + app.kubernetes.io/component: certificate + app.kubernetes.io/created-by: {{ .Release.Name }} + app.kubernetes.io/instance: serving-cert + app.kubernetes.io/name: certificate + app.kubernetes.io/part-of: {{ .Release.Name }} + name: {{ .Release.Name }}-selfsigned-issuer + namespace: {{ .Release.Namespace }} +spec: + selfSigned: {} +{{- end }} +--- +{{- if .Values.operator.admissionController.enabled }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + {{- if .Values.operator.admissionController.useCertManager }} + annotations: + cert-manager.io/inject-ca-from: {{ .Release.Namespace }}/{{ .Release.Name }}-serving-cert + {{- end }} + labels: + app.kubernetes.io/component: webhook + app.kubernetes.io/created-by: {{ .Release.Name }} + app.kubernetes.io/instance: validating-webhook-configuration + app.kubernetes.io/name: validatingwebhookconfiguration + app.kubernetes.io/part-of: {{ .Release.Name }} + name: {{ .Release.Name }}-validating-webhook-configuration +webhooks: +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: {{ .Release.Name }}-webhook-service + namespace: {{ .Release.Namespace }} + path: /validate-mellanox-com-v1alpha1-hostdevicenetwork + {{- if not .Values.operator.admissionController.useCertManager }} + caBundle: {{ .Values.operator.admissionController.certificate.caCrt | b64enc | quote }} + {{- end }} + failurePolicy: Fail + name: vhostdevicenetwork.kb.io + rules: + - apiGroups: + - mellanox.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - hostdevicenetworks + sideEffects: None +- admissionReviewVersions: + - v1 + clientConfig: + service: + name: {{ .Release.Name }}-webhook-service + namespace: {{ .Release.Namespace }} + path: /validate-mellanox-com-v1alpha1-nicclusterpolicy + {{- if not .Values.operator.admissionController.useCertManager }} + caBundle: {{ .Values.operator.admissionController.certificate.caCrt | b64enc | quote }} + {{- end }} + failurePolicy: Fail + name: vnicclusterpolicy.kb.io + rules: + - apiGroups: + - mellanox.com + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - nicclusterpolicies + sideEffects: None +{{- end }} +--- +{{- if and .Values.operator.admissionController.enabled (not .Values.operator.admissionController.useCertManager) }} +apiVersion: v1 +kind: Secret +metadata: + name: webhook-server-cert + namespace: {{ .Release.Namespace }} +type: Opaque +data: + tls.crt: {{ .Values.operator.admissionController.certificate.tlsCrt | b64enc | quote }} + tls.key: {{ .Values.operator.admissionController.certificate.tlsKey | b64enc | quote }} +{{- end }} diff --git a/network-operator/templates/keep-ncp-hook.yaml b/network-operator/templates/keep-ncp-hook.yaml new file mode 100644 index 0000000..c14479e --- /dev/null +++ b/network-operator/templates/keep-ncp-hook.yaml @@ -0,0 +1,84 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "network-operator.fullname" . }}-hooks-keep-ncp-sa + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "network-operator.fullname" . }}-hooks-keep-ncp-role + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +rules: + - apiGroups: + - mellanox.com + resources: + - nicclusterpolicies + verbs: + - "*" +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "network-operator.fullname" . }}-hooks-scale-binding + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +subjects: + - kind: ServiceAccount + name: {{ include "network-operator.fullname" . }}-hooks-keep-ncp-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "network-operator.fullname" . }}-hooks-keep-ncp-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: network-operator-keep-ncp + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation + labels: + {{- include "network-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: "network-operator" +spec: + template: + metadata: + name: network-operator-keep-ncp + labels: + {{- include "network-operator.labels" . | nindent 8 }} + app.kubernetes.io/component: "network-operator" + spec: + {{- with .Values.operator.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.affinity}} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "network-operator.fullname" . }}-hooks-keep-ncp-sa + imagePullSecrets: {{ include "network-operator.operator.imagePullSecrets" . }} + containers: + - name: keep-ncp + image: "{{ .Values.operator.repository }}/{{ .Values.operator.image }}:{{ .Values.operator.tag | default .Chart.AppVersion }}" + imagePullPolicy: IfNotPresent + command: + - /keep-ncp + restartPolicy: OnFailure diff --git a/network-operator/templates/nodefeaturerules.yaml b/network-operator/templates/nodefeaturerules.yaml new file mode 100644 index 0000000..0dc04f8 --- /dev/null +++ b/network-operator/templates/nodefeaturerules.yaml @@ -0,0 +1,16 @@ +{{- if .Values.nfd.deployNodeFeatureRules }} +apiVersion: nfd.k8s-sigs.io/v1alpha1 +kind: NodeFeatureRule +metadata: + name: nvidia-nics-rules +spec: + rules: + - name: "Nvidia NICs PCI" + labels: + "pci-15b3.present": "true" + matchFeatures: + - feature: pci.device + matchExpressions: + vendor: {op: In, value: ["15b3"]} + class: {op: In, value: ["0200", "0207"]} +{{- end }} diff --git a/network-operator/templates/operator.yaml b/network-operator/templates/operator.yaml new file mode 100644 index 0000000..23997b6 --- /dev/null +++ b/network-operator/templates/operator.yaml @@ -0,0 +1,120 @@ +{{/* + Copyright 2020 NVIDIA + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "network-operator.fullname" . }} + labels: + {{- include "network-operator.labels" . | nindent 4 }} + control-plane: {{ .Release.Name }}-controller + namespace: {{ .Release.Namespace }} +spec: + replicas: 1 + selector: + matchLabels: + {{- include "network-operator.selectorLabels" . | nindent 6 }} + template: + metadata: + annotations: + kubectl.kubernetes.io/default-container: {{ .Chart.Name }} + labels: + nvidia.com/ofed-driver-upgrade-drain.skip: "true" + control-plane: {{ .Release.Name }}-controller + {{- include "network-operator.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.operator.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.affinity}} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "network-operator.fullname" . }} + imagePullSecrets: {{ include "network-operator.operator.imagePullSecrets" . }} + containers: + - name: {{ .Chart.Name }} + # Replace this with the built image name + image: "{{ .Values.operator.repository }}/{{ .Values.operator.image }}:{{ .Values.operator.tag | default .Chart.AppVersion }}" + {{- if .Values.operator.admissionController.enabled }} + ports: + - containerPort: 9443 + name: webhook-server + protocol: TCP + volumeMounts: + - mountPath: /tmp/k8s-webhook-server/serving-certs + name: cert + readOnly: true + {{- end }} + command: + - /manager + args: + - --leader-elect + env: + - name: STATE_MANIFEST_BASE_DIR + value: "/manifests" + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPERATOR_NAME + value: "network-operator" + - name: ENABLE_WEBHOOKS + value: "{{ .Values.operator.admissionController.enabled }}" + - name: USE_DTK + value: "{{ .Values.operator.useDTK }}" + {{- if .Values.operator.cniBinDirectory }} + - name: CNI_BIN_DIR + value: "{{ .Values.operator.cniBinDirectory }}" + {{- end }} + {{- if and .Values.operator.ofedDriver.initContainer .Values.operator.ofedDriver.initContainer.enable }} + - name: OFED_INIT_CONTAINER_IMAGE + {{- with .Values.operator.ofedDriver.initContainer }} + value: "{{ .repository }}/{{ .image }}:{{ .version }}" + {{- end }} + {{- end }} + securityContext: + allowPrivilegeEscalation: false + livenessProbe: + httpGet: + path: /healthz + port: 8081 + initialDelaySeconds: 15 + periodSeconds: 20 + readinessProbe: + httpGet: + path: /readyz + port: 8081 + initialDelaySeconds: 5 + periodSeconds: 10 + {{- with .Values.operator.resources }} + resources: + {{- toYaml . | nindent 12 }} + {{- end }} + securityContext: + runAsUser: 65532 + terminationGracePeriodSeconds: 10 + {{- if .Values.operator.admissionController.enabled }} + volumes: + - name: cert + secret: + defaultMode: 420 + secretName: webhook-server-cert + {{- end }} diff --git a/network-operator/templates/role.yaml b/network-operator/templates/role.yaml new file mode 100644 index 0000000..41890db --- /dev/null +++ b/network-operator/templates/role.yaml @@ -0,0 +1,390 @@ +{{/* + Copyright 2020 NVIDIA + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: {{ include "network-operator.fullname" . }} +rules: +- apiGroups: + - "" + resources: + - configmaps + - events + - persistentvolumeclaims + - secrets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - endpoints + - namespaces + - pods + - pods/status + - serviceaccounts + - services + - services/finalizers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list + - patch + - update + - watch +- apiGroups: + - "" + resources: + - pods + verbs: + - list +- apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - admissionregistration.k8s.io + resources: + - validatingwebhookconfigurations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - controllerrevisions + - daemonsets + - deployments + - replicasets + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - daemonsets + - deployments + - replicasets + - statefulsets + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - apps + resources: + - deployments/finalizers + verbs: + - update +- apiGroups: + - batch + resources: + - cronjobs + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - cert-manager.io + resources: + - certificates + - issuers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - config.openshift.io + resources: + - clusterversions + - proxies + verbs: + - get + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - events.k8s.io + resources: + - events + verbs: + - create + - get + - patch + - update +- apiGroups: + - image.openshift.io + resources: + - imagestreams + verbs: + - get + - list + - watch +- apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - mellanox.com + resources: + - hostdevicenetworks + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - mellanox.com + resources: + - hostdevicenetworks/finalizers + verbs: + - update +- apiGroups: + - mellanox.com + resources: + - hostdevicenetworks/status + verbs: + - get + - patch + - update +- apiGroups: + - mellanox.com + resources: + - ipoibnetworks + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - mellanox.com + resources: + - ipoibnetworks/finalizers + verbs: + - update +- apiGroups: + - mellanox.com + resources: + - ipoibnetworks/status + verbs: + - get + - patch + - update +- apiGroups: + - mellanox.com + resources: + - macvlannetworks + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - mellanox.com + resources: + - macvlannetworks/finalizers + verbs: + - update +- apiGroups: + - mellanox.com + resources: + - macvlannetworks/status + verbs: + - get + - patch + - update +- apiGroups: + - mellanox.com + resources: + - nicclusterpolicies + - nicclusterpolicies/status + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - mellanox.com + resources: + - nicclusterpolicies/finalizers + verbs: + - update +- apiGroups: + - monitoring.coreos.com + resources: + - servicemonitors + verbs: + - create + - get + - list + - watch +- apiGroups: + - nv-ipam.nvidia.com + resources: + - ippools + - cidrpools + verbs: + - create + - get + - list + - watch +- apiGroups: + - nv-ipam.nvidia.com + resources: + - ippools/status + - cidrpools/status + verbs: + - get + - patch + - update +- apiGroups: + - rbac.authorization.k8s.io + resources: + - clusterrolebindings + - clusterroles + - rolebindings + - roles + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - security.openshift.io + resourceNames: + - privileged + resources: + - securitycontextconstraints + verbs: + - use +- apiGroups: + - whereabouts.cni.cncf.io + resources: + - ippools + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - whereabouts.cni.cncf.io + resources: + - overlappingrangeipreservations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch diff --git a/network-operator/templates/role_binding.yaml b/network-operator/templates/role_binding.yaml new file mode 100644 index 0000000..9f4d698 --- /dev/null +++ b/network-operator/templates/role_binding.yaml @@ -0,0 +1,27 @@ +{{/* + Copyright 2020 NVIDIA + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "network-operator.fullname" . }} +subjects: + - kind: Group + apiGroup: rbac.authorization.k8s.io + name: system:serviceaccounts:{{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "network-operator.fullname" . }} + apiGroup: rbac.authorization.k8s.io diff --git a/network-operator/templates/service_account.yaml b/network-operator/templates/service_account.yaml new file mode 100644 index 0000000..b98dc38 --- /dev/null +++ b/network-operator/templates/service_account.yaml @@ -0,0 +1,20 @@ +{{/* + Copyright 2020 NVIDIA + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "network-operator.fullname" . }} + namespace: {{ .Release.Namespace }} diff --git a/network-operator/templates/upgrade-crd-hook.yaml b/network-operator/templates/upgrade-crd-hook.yaml new file mode 100644 index 0000000..e0c80e4 --- /dev/null +++ b/network-operator/templates/upgrade-crd-hook.yaml @@ -0,0 +1,100 @@ +{{- if .Values.upgradeCRDs }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "network-operator.fullname" . }}-hooks-sa + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "network-operator.fullname" . }}-hooks-role + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +rules: + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - "*" +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ include "network-operator.fullname" . }}-hooks-binding + annotations: + helm.sh/hook: pre-install,pre-upgrade + helm.sh/hook-delete-policy: hook-succeeded,before-hook-creation + helm.sh/hook-weight: "0" +subjects: + - kind: ServiceAccount + name: {{ include "network-operator.fullname" . }}-hooks-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "network-operator.fullname" . }}-hooks-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: network-operator-upgrade-crd + namespace: {{ .Release.Namespace }} + annotations: + "helm.sh/hook": pre-install,pre-upgrade + "helm.sh/hook-weight": "1" + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation + labels: + {{- include "network-operator.labels" . | nindent 4 }} + app.kubernetes.io/component: "network-operator" +spec: + template: + metadata: + name: network-operator-upgrade-crd + labels: + {{- include "network-operator.labels" . | nindent 8 }} + app.kubernetes.io/component: "network-operator" + spec: + {{- with .Values.operator.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.affinity}} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.operator.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "network-operator.fullname" . }}-hooks-sa + imagePullSecrets: {{ include "network-operator.operator.imagePullSecrets" . }} + containers: + - name: upgrade-crd + image: "{{ .Values.operator.repository }}/{{ .Values.operator.image }}:{{ .Values.operator.tag | default .Chart.AppVersion }}" + imagePullPolicy: IfNotPresent + command: + - /apply-crds + args: + - --crds-dir=/crds/network-operator + {{- if .Values.sriovNetworkOperator.enabled }} + - --crds-dir=/crds/sriov-network-operator + {{- end }} + {{- if .Values.nfd.enabled }} + - --crds-dir=/crds/node-feature-discovery + {{- end }} + {{- if .Values.nicConfigurationOperator.enabled }} + - --crds-dir=/crds/nic-configuration-operator + {{- end }} + {{- if .Values.maintenanceOperator.enabled }} + - --crds-dir=/crds/maintenance-operator + {{- end }} + restartPolicy: OnFailure +{{- end }} diff --git a/network-operator/values.yaml b/network-operator/values.yaml new file mode 100644 index 0000000..c7f1a20 --- /dev/null +++ b/network-operator/values.yaml @@ -0,0 +1,309 @@ +# Copyright 2020 NVIDIA +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for network-operator. +# This is a YAML-formatted file. + +nfd: + # -- Deploy Node Feature Discovery operator. + enabled: true + # -- Deploy Node Feature Rules to label the nodes with the discovered features. + deployNodeFeatureRules: true + +# -- Enable CRDs upgrade with helm pre-install and pre-upgrade hooks. +upgradeCRDs: true + +sriovNetworkOperator: + # -- Deploy SR-IOV Network Operator. + enabled: false + +nicConfigurationOperator: + # -- Deploy NIC Configuration Operator. + enabled: false + +maintenanceOperator: + # -- Deploy Maintenance Operator. + enabled: false + +# Set both enableNodeFeatureApi and NodeFeatureAPI feature gate to false to disable. +node-feature-discovery: + # -- The Node Feature API enable communication between nfd master and worker + # through NodeFeature CRs. Otherwise communication is through gRPC. + enableNodeFeatureApi: true + featureGates: + NodeFeatureAPI: true + + # -- NFD master deployment configuration. + # @notationType -- yaml + master: + serviceAccount: + name: node-feature-discovery + create: true + config: + extraLabelNs: ["nvidia.com"] + gc: + # -- Specifies whether the NFD Garbage Collector should be created + enable: true + # -- Specifies the number of replicas for the NFD Garbage Collector + replicaCount: 1 + serviceAccount: + # -- The name of the service account for garbage collector to use. + # If not set and create is true, a name is generated using the fullname + # template and -gc suffix. + name: node-feature-discovery + # -- disable creation to avoid duplicate serviceaccount creation by master + # spec above. + create: false + # -- NFD worker daemonset configuration. + # @notationType -- yaml + worker: + serviceAccount: + # disable creation to avoid duplicate serviceaccount creation by master spec + # above + name: node-feature-discovery + create: false + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + config: + sources: + pci: + deviceClassWhitelist: + - "0300" + - "0302" + deviceLabelFields: + - vendor + +# SR-IOV Network Operator chart related values. +sriov-network-operator: + operator: + # -- Prefix to be used for resources names. + resourcePrefix: "nvidia.com" + # -- Enable admission controller. + # @notationType -- yaml + admissionControllers: + enabled: false + certificates: + secretNames: + operator: "operator-webhook-cert" + injector: "network-resources-injector-cert" + certManager: + # -- When enabled, makes use of certificates managed by cert-manager. + enabled: true + # -- When enabled, certificates are generated via cert-manager and then + # name will match the name of the secrets defined above. + generateSelfSigned: true + # -- If not specified, no secret is created and secrets with the names + # defined above are expected to exist in the cluster. In that case, + # the ca.crt must be base64 encoded twice since it ends up being an env variable. + custom: + enabled: false + # operator: + # caCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsKey: | + # -----BEGIN EC PRIVATE KEY----- + # MHcl4wOuDwKQa+upc8GftXE2C//4mKANBC6It01gUaTIpo= + # ... + # -----END EC PRIVATE KEY----- + # injector: + # caCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsKey: | + # -----BEGIN EC PRIVATE KEY----- + # MHcl4wOuDwKQa+upc8GftXE2C//4mKANBC6It01gUaTIpo= + # ... + # -----END EC PRIVATE KEY----- + + # Image URIs for sriov-network-operator components. + images: + operator: nvcr.io/nvidia/mellanox/sriov-network-operator:network-operator-25.1.0 + sriovConfigDaemon: nvcr.io/nvidia/mellanox/sriov-network-operator-config-daemon:network-operator-25.1.0 + sriovCni: ghcr.io/k8snetworkplumbingwg/sriov-cni:v2.8.1 + ibSriovCni: ghcr.io/k8snetworkplumbingwg/ib-sriov-cni:v1.1.1 + ovsCni: ghcr.io/k8snetworkplumbingwg/ovs-cni-plugin:v0.37.0 + # rdmaCni: ghcr.io/k8snetworkplumbingwg/rdma-cni:v1.2.0 + sriovDevicePlugin: ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin:v3.9.0 + resourcesInjector: ghcr.io/k8snetworkplumbingwg/network-resources-injector:v1.7.0 + webhook: nvcr.io/nvidia/mellanox/sriov-network-operator-webhook:network-operator-25.1.0 + # imagePullSecrest for SR-IOV Network Operator related images + # imagePullSecrets: [] + sriovOperatorConfig: + # -- Deploy ``SriovOperatorConfig`` custom resource + deploy: true + # -- Selects the nodes to be configured + # @notationType -- yaml + configDaemonNodeSelector: + beta.kubernetes.io/os: "linux" + network.nvidia.com/operator.mofed.wait: "false" + +# Nic Configuration Operator chart related values. +nic-configuration-operator-chart: + operator: + image: + repository: ghcr.io/mellanox + name: nic-configuration-operator + tag: v0.1.17 + configDaemon: + image: + repository: ghcr.io/mellanox + name: nic-configuration-operator-daemon + tag: v0.1.17 + +# Maintenance Operator chart related values. +maintenance-operator-chart: + operator: + image: + repository: ghcr.io/mellanox + name: maintenance-operator + tag: v0.2.0 + admissionController: + # -- enable admission controller of the operator + enable: false + certificates: + secretNames: + # -- secret name containing certificates for the operator admission controller + operator: "maintenance-webhook-cert" + certManager: + # -- use cert-manager for certificates + enable: false + # -- generate self-signed certificates with cert-manager + generateSelfSigned: false + custom: + # -- enable custom certificates using secrets + enable: false + +# General Operator related values +# The operator element allows to deploy network operator from an alternate location +operator: + # -- Optional `resource requests and limits `_ + # for the operator. + # @notationType -- yaml + resources: + limits: + cpu: 500m + memory: 128Mi + requests: + cpu: 5m + memory: 64Mi + # -- Set additional tolerations for various Daemonsets deployed by the operator. + # @notationType -- yaml + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Equal" + value: "" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Equal" + value: "" + effect: "NoSchedule" + # -- Configure node selector settings for the operator. + nodeSelector: {} + affinity: + # -- Configure node affinity settings for the operator. + # @notationType -- yaml + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/master" + operator: In + values: [""] + - weight: 1 + preference: + matchExpressions: + - key: "node-role.kubernetes.io/control-plane" + operator: In + values: [ "" ] + # -- Network Operator image repository. + repository: nvcr.io/nvidia/cloud-native + # -- Network Operator image name + image: network-operator + # imagePullSecrets: [] + # -- Name to be used as part of objects name generation. + nameOverride: "" + # -- Name to be used to replace generated names. + fullnameOverride: "" + # tag, if defined will use the given image tag, else Chart. AppVersion will be used. + # tag + # -- Directory, where CNI binaries will be deployed on the nodes. Setting for + # the sriov-network-operator is set with ``sriov-network-operator.cniBinPath`` + # parameter. Note that the CNI bin directory should be aligned with the CNI bin + # directory in the container runtime. + cniBinDirectory: /opt/cni/bin + # -- Enable the use of Driver ToolKit to compile DOCA Drivers (OpenShift only). + useDTK: true + admissionController: + # -- Deploy with admission controller. + enabled: false + # -- Use cert-manager for generating self-signed certificate. + useCertManager: true + # certificate: + # caCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsCrt: | + # -----BEGIN CERTIFICATE----- + # MIIMIICLDCCAdKgAwIBAgIBADAKBggqhkjOPQQDAjB9MQswCQYDVQQGEwJCRTEPMA0G + # ... + # -----END CERTIFICATE----- + # tlsKey: | + # -----BEGIN EC PRIVATE KEY----- + # MHcl4wOuDwKQa+upc8GftXE2C//4mKANBC6It01gUaTIpo= + # ... + # -----END EC PRIVATE KEY----- + ofedDriver: + initContainer: + # -- Deploy init container. + enable: true + # -- Init container image repository. + repository: ghcr.io/mellanox + # -- Init container image name. + image: network-operator-init-container + # -- Init container image version. + version: v0.0.3 + +# -- An optional list of references to secrets to use for pulling any of the +# Network Operator images. +imagePullSecrets: [] + +# @ignore +test: + pf: ens2f0 diff --git a/template/dev/gpu-operator/custom/dcgm_config.yaml b/template/dev/gpu-operator/custom/dcgm_config.yaml index 1b958ea..d79cd85 100644 --- a/template/dev/gpu-operator/custom/dcgm_config.yaml +++ b/template/dev/gpu-operator/custom/dcgm_config.yaml @@ -95,3 +95,11 @@ data: DCGM_FI_PROF_PCIE_TX_BYTES, gauge, The rate of data transmitted over the PCIe bus - including both protocol headers and data payloads - in bytes per second. DCGM_FI_PROF_PCIE_RX_BYTES, gauge, The rate of data received over the PCIe bus - including both protocol headers and data payloads - in bytes per second. + # NVLINK more metrics + DCGM_FI_DEV_NVLINK_COUNT_LINK_RECOVERY_FAILED_EVENTS, counter, Number of times link went from Up to recovery - failed and link was declared down. + DCGM_FI_DEV_NVLINK_COUNT_LOCAL_LINK_INTEGRITY_ERRORS, counter, Total number of times that the count of local errors exceeded a threshold. + DCGM_FI_PROF_NVLINK_RX_BYTES, gauge, The total number of bytes of active NvLink rx (read) data including both header and payload. + DCGM_FI_PROF_NVLINK_TX_BYTES, gauge, The total number of bytes of active NvLink tx (transmit) data including both header and payload. + DCGM_FI_DEV_NVLINK_COUNT_RX_ERRORS, counter, Total number of packets with errors Rx on a link. + DCGM_FI_DEV_NVLINK_COUNT_TX_DISCARDS, counter, Total number of tx error packets that were discarded. +