443 lines
13 KiB
YAML
443 lines
13 KiB
YAML
# Exascaler CSI Driver (2.2.4)
|
|
#
|
|
# This driver version works with Kubernetes version >=1.14
|
|
#
|
|
# In production, each CSI driver deployment has to be customized to avoid conflicts,
|
|
# use non-default namespace and different names for non-namespaced entities like the ClusterRole
|
|
#
|
|
# Install to Kubernetes:
|
|
# $ kubectl apply -f deploy/kubernetes/exascaler-csi-file-driver.yaml
|
|
#
|
|
|
|
|
|
# ----------------------
|
|
# Exascaler CSI Driver
|
|
# ----------------------
|
|
|
|
apiVersion: storage.k8s.io/v1
|
|
kind: CSIDriver
|
|
metadata:
|
|
name: exa.csi.ddn.com
|
|
spec:
|
|
attachRequired: false
|
|
podInfoOnMount: false
|
|
fsGroupPolicy: ReadWriteOnceWithFSType
|
|
---
|
|
|
|
|
|
# ----
|
|
# RBAC
|
|
# ----
|
|
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: exascaler-csi-controller-service-account
|
|
namespace: default # replace with non-default namespace name if needed
|
|
---
|
|
|
|
kind: ClusterRole
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: exascaler-csi-controller-cluster-role
|
|
rules:
|
|
- apiGroups: ['']
|
|
resources: ['secrets']
|
|
verbs: ['get', 'list', "watch"]
|
|
- apiGroups: ['']
|
|
resources: ['persistentvolumes']
|
|
verbs: ['get', 'list', 'watch', 'create', 'update', 'delete'] # "update" for attacher
|
|
- apiGroups: ['']
|
|
resources: ['persistentvolumeclaims']
|
|
verbs: ['get', 'list', 'watch', 'update']
|
|
- apiGroups: ['storage.k8s.io']
|
|
resources: ['storageclasses']
|
|
verbs: ['get', 'list', 'watch']
|
|
- apiGroups: ['']
|
|
resources: ['events']
|
|
verbs: ['list', 'watch', 'create', 'update', 'patch']
|
|
# attacher specific
|
|
- apiGroups: ['']
|
|
resources: ['nodes', 'pods']
|
|
verbs: ['get', 'list', 'watch']
|
|
- apiGroups: ['csi.storage.k8s.io']
|
|
resources: ['csinodeinfos']
|
|
verbs: ['get', 'list', 'watch']
|
|
- apiGroups: ['storage.k8s.io']
|
|
resources: ['volumeattachments']
|
|
verbs: ['get', 'list', 'watch', 'update']
|
|
- apiGroups: ['storage.k8s.io']
|
|
resources: ['volumeattachments/status']
|
|
verbs: ['get', 'list', 'watch', 'update', 'patch']
|
|
# snapshotter specific
|
|
- apiGroups: ['snapshot.storage.k8s.io']
|
|
resources: ['volumesnapshotclasses']
|
|
verbs: ['get', 'list', 'watch']
|
|
- apiGroups: ['snapshot.storage.k8s.io']
|
|
resources: ['volumesnapshotcontents']
|
|
verbs: ['create', 'get', 'list', 'watch', 'update', 'delete', 'patch']
|
|
- apiGroups: ['snapshot.storage.k8s.io']
|
|
resources: ['volumesnapshots']
|
|
verbs: ['get', 'list', 'watch', 'update']
|
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
|
resources: ["volumesnapshots/status"]
|
|
verbs: ["update"]
|
|
- apiGroups: ["snapshot.storage.k8s.io"]
|
|
resources: ["volumesnapshotcontents/status"]
|
|
verbs: ["update"]
|
|
- apiGroups: ['apiextensions.k8s.io']
|
|
resources: ['customresourcedefinitions']
|
|
verbs: ['create', 'list', 'watch', 'delete']
|
|
- apiGroups: [""]
|
|
resources: ["persistentvolumeclaims/status"]
|
|
verbs: ["update", "patch"]
|
|
# CSINode specific
|
|
- apiGroups: ["storage.k8s.io"]
|
|
resources: ["csinodes"]
|
|
verbs: ["watch", "list", "get"]
|
|
---
|
|
|
|
kind: ClusterRoleBinding
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: exascaler-csi-controller-cluster-role-binding
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: exascaler-csi-controller-service-account
|
|
namespace: default # replace with non-default namespace name if needed
|
|
roleRef:
|
|
kind: ClusterRole
|
|
name: exascaler-csi-controller-cluster-role
|
|
apiGroup: rbac.authorization.k8s.io
|
|
---
|
|
|
|
# External Resizer
|
|
kind: ClusterRole
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: csi-resizer-role
|
|
rules:
|
|
# The following rule should be uncommented for plugins that require secrets
|
|
# for provisioning.
|
|
- apiGroups: [""]
|
|
resources: ["secrets"]
|
|
verbs: ["get", "list", "watch"]
|
|
- apiGroups: [""]
|
|
resources: ["persistentvolumes"]
|
|
verbs: ["get", "list", "watch", "update", "patch"]
|
|
- apiGroups: [""]
|
|
resources: ["persistentvolumeclaims"]
|
|
verbs: ["get", "list", "watch"]
|
|
- apiGroups: [""]
|
|
resources: ["persistentvolumeclaims/status"]
|
|
verbs: ["update", "patch"]
|
|
- apiGroups: ["storage.k8s.io"]
|
|
resources: ["storageclasses"]
|
|
verbs: ["get", "list", "watch"]
|
|
- apiGroups: [""]
|
|
resources: ["events"]
|
|
verbs: ["list", "watch", "create", "update", "patch"]
|
|
|
|
---
|
|
kind: ClusterRoleBinding
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: csi-resizer-binding
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: exascaler-csi-controller-service-account
|
|
namespace: default
|
|
roleRef:
|
|
kind: ClusterRole
|
|
name: csi-resizer-role
|
|
apiGroup: rbac.authorization.k8s.io
|
|
|
|
---
|
|
kind: Role
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
namespace: default
|
|
name: external-resizer-cfg
|
|
rules:
|
|
- apiGroups: ["coordination.k8s.io"]
|
|
resources: ["leases"]
|
|
verbs: ["get", "watch", "list", "delete", "update", "create"]
|
|
|
|
---
|
|
kind: RoleBinding
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: csi-resizer-role-cfg
|
|
namespace: default
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: exascaler-csi-controller-service-account
|
|
namespace: default
|
|
roleRef:
|
|
kind: Role
|
|
name: external-resizer-cfg
|
|
apiGroup: rbac.authorization.k8s.io
|
|
---
|
|
|
|
kind: Service
|
|
apiVersion: v1
|
|
metadata:
|
|
name: exascaler-csi-controller-service
|
|
labels:
|
|
app: exascaler-csi-controller
|
|
spec:
|
|
selector:
|
|
app: exascaler-csi-controller
|
|
ports:
|
|
- name: dummy
|
|
port: 12345
|
|
---
|
|
|
|
# ---------------------------------
|
|
# Exascaler CSI Controller Server
|
|
# ---------------------------------
|
|
#
|
|
# Runs single driver controller server (driver + provisioner + attacher + snapshotter) on one of the nodes.
|
|
# Controller driver deployment does not support running multiple replicas.
|
|
#
|
|
kind: Deployment
|
|
apiVersion: apps/v1
|
|
metadata:
|
|
name: exascaler-csi-controller
|
|
spec:
|
|
# serviceName: exascaler-csi-controller-service
|
|
selector:
|
|
matchLabels:
|
|
app: exascaler-csi-controller # has to match .spec.template.metadata.labels
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: exascaler-csi-controller
|
|
spec:
|
|
serviceAccount: exascaler-csi-controller-service-account
|
|
containers:
|
|
# csi-provisioner: sidecar container that watches Kubernetes PersistentVolumeClaim objects
|
|
# and triggers CreateVolume/DeleteVolume against a CSI endpoint
|
|
- name: csi-provisioner
|
|
image: k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- --csi-address=/var/lib/csi/sockets/pluginproxy/csi.sock
|
|
- --volume-name-prefix=pvc-exa
|
|
- --strict-topology
|
|
- --immediate-topology=false
|
|
- --feature-gates=Topology=true
|
|
- --timeout=120m
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /var/lib/csi/sockets/pluginproxy
|
|
- name: csi-attacher
|
|
image: k8s.gcr.io/sig-storage/csi-attacher:v3.5.0
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- --csi-address=$(ADDRESS)
|
|
- --v=2
|
|
- --leader-election=true
|
|
env:
|
|
- name: ADDRESS
|
|
value: /var/lib/csi/sockets/pluginproxy/csi.sock
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /var/lib/csi/sockets/pluginproxy/
|
|
- name: csi-snapshotter
|
|
image: registry.k8s.io/sig-storage/csi-snapshotter:v5.0.1
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- -v=3
|
|
- --csi-address=/var/lib/csi/sockets/pluginproxy/csi.sock
|
|
- --timeout=120m
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /var/lib/csi/sockets/pluginproxy
|
|
- name: csi-resizer
|
|
image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0
|
|
args:
|
|
- "--csi-address=$(ADDRESS)"
|
|
env:
|
|
- name: ADDRESS
|
|
value: /var/lib/csi/sockets/pluginproxy/csi.sock
|
|
imagePullPolicy: "IfNotPresent"
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /var/lib/csi/sockets/pluginproxy/
|
|
- name: driver
|
|
securityContext:
|
|
privileged: true
|
|
capabilities:
|
|
add: ['SYS_ADMIN']
|
|
allowPrivilegeEscalation: true
|
|
image: quay.io/ddn/exascaler-csi-file-driver:v2.3.5
|
|
imagePullPolicy: Always
|
|
args:
|
|
- --nodeid=$(KUBE_NODE_NAME)
|
|
- --endpoint=unix://csi/csi.sock
|
|
- --role=controller
|
|
env:
|
|
- name: KUBE_NODE_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: spec.nodeName
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /csi
|
|
- name: secret
|
|
mountPath: /config
|
|
readOnly: true
|
|
- name: host
|
|
mountPath: /host
|
|
mountPropagation: Bidirectional
|
|
volumes:
|
|
- name: socket-dir
|
|
emptyDir:
|
|
- name: secret
|
|
secret:
|
|
secretName: exascaler-csi-file-driver-config
|
|
- name: host
|
|
hostPath:
|
|
path: /
|
|
type: Directory
|
|
---
|
|
|
|
|
|
# ---------------------------
|
|
# exascaler CSI Node Server
|
|
# ---------------------------
|
|
#
|
|
# Runs driver node server (driver + registrar) on each node
|
|
#
|
|
|
|
apiVersion: v1
|
|
kind: ServiceAccount
|
|
metadata:
|
|
name: exascaler-csi-node-service-account
|
|
namespace: default # replace with non-default namespace name if needed
|
|
---
|
|
|
|
kind: ClusterRole
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: exascaler-csi-node-cluster-role
|
|
rules:
|
|
- apiGroups: ['']
|
|
resources: ['events']
|
|
verbs: ['get', 'list', 'watch', 'create', 'update', 'patch']
|
|
- apiGroups: ['']
|
|
resources: ['nodes']
|
|
verbs: ['get']
|
|
---
|
|
|
|
kind: ClusterRoleBinding
|
|
apiVersion: rbac.authorization.k8s.io/v1
|
|
metadata:
|
|
name: exascaler-csi-node-cluster-role-binding
|
|
subjects:
|
|
- kind: ServiceAccount
|
|
name: exascaler-csi-node-service-account
|
|
namespace: default # replace with non-default namespace name if needed
|
|
roleRef:
|
|
kind: ClusterRole
|
|
name: exascaler-csi-node-cluster-role
|
|
apiGroup: rbac.authorization.k8s.io
|
|
---
|
|
|
|
# exascaler Node Server as a daemon
|
|
|
|
kind: DaemonSet
|
|
apiVersion: apps/v1
|
|
metadata:
|
|
name: exascaler-csi-node
|
|
spec:
|
|
selector:
|
|
matchLabels:
|
|
app: exascaler-csi-node
|
|
template:
|
|
metadata:
|
|
labels:
|
|
app: exascaler-csi-node
|
|
spec:
|
|
serviceAccount: exascaler-csi-node-service-account
|
|
hostNetwork: true
|
|
containers:
|
|
# driver-registrar: sidecar container that:
|
|
# 1) registers the CSI driver with kubelet
|
|
# 2) adds the drivers custom NodeId to a label on the Kubernetes Node API Object
|
|
- name: driver-registrar
|
|
image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0
|
|
imagePullPolicy: IfNotPresent
|
|
args:
|
|
- --v=5
|
|
- --csi-address=/csi/csi.sock
|
|
- --kubelet-registration-path=/mnt/vdb/kubelet/plugins/exa.csi.ddn.com/csi.sock
|
|
livenessProbe:
|
|
exec:
|
|
command:
|
|
- /csi-node-driver-registrar
|
|
- --kubelet-registration-path=/mnt/vdb/kubelet/plugins/exa.csi.ddn.com/csi.sock
|
|
- --mode=kubelet-registration-probe
|
|
initialDelaySeconds: 30
|
|
timeoutSeconds: 15
|
|
env:
|
|
- name: KUBE_NODE_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: spec.nodeName
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /csi
|
|
- name: registration-dir
|
|
mountPath: /registration
|
|
- name: driver
|
|
securityContext:
|
|
privileged: true
|
|
capabilities:
|
|
add: ['SYS_ADMIN']
|
|
allowPrivilegeEscalation: true
|
|
image: quay.io/ddn/exascaler-csi-file-driver:v2.3.5
|
|
imagePullPolicy: Always
|
|
args:
|
|
- --nodeid=$(KUBE_NODE_NAME)
|
|
- --endpoint=unix://csi/csi.sock
|
|
- --role=node
|
|
env:
|
|
- name: KUBE_NODE_NAME
|
|
valueFrom:
|
|
fieldRef:
|
|
fieldPath: spec.nodeName
|
|
volumeMounts:
|
|
- name: socket-dir
|
|
mountPath: /csi
|
|
- name: secret
|
|
mountPath: /config
|
|
- name: host
|
|
mountPath: /host
|
|
mountPropagation: Bidirectional
|
|
- name: pods-mount-dir
|
|
mountPath: /mnt/vdb/kubelet/pods
|
|
mountPropagation: Bidirectional
|
|
volumes:
|
|
- name: socket-dir
|
|
hostPath:
|
|
path: /mnt/vdb/kubelet/plugins/exa.csi.ddn.com
|
|
type: DirectoryOrCreate
|
|
- name: registration-dir
|
|
hostPath:
|
|
path: /mnt/vdb/kubelet/plugins_registry/
|
|
type: Directory
|
|
- name: pods-mount-dir
|
|
hostPath:
|
|
path: /mnt/vdb/kubelet/pods
|
|
type: Directory
|
|
- name: host
|
|
hostPath:
|
|
path: /
|
|
type: Directory
|
|
- name: secret
|
|
secret:
|
|
secretName: exascaler-csi-file-driver-config
|
|
---
|