# Exascaler CSI Driver (2.2.4)
#
# This driver version works with Kubernetes version >=1.14
#
# In production, each CSI driver deployment has to be customized to avoid conflicts,
# use non-default namespace and different names for non-namespaced entities like the ClusterRole
#
# Install to Kubernetes:
# $ kubectl apply -f deploy/kubernetes/exascaler-csi-file-driver.yaml
#


# ----------------------
# Exascaler CSI Driver
# ----------------------

apiVersion: storage.k8s.io/v1
kind: CSIDriver
metadata:
  name: exa.csi.ddn.com
spec:
  attachRequired: false
  podInfoOnMount: false
  fsGroupPolicy: File
---


# ----
# RBAC
# ----

apiVersion: v1
kind: ServiceAccount
metadata:
  name: exascaler-csi-controller-service-account
  namespace: default # replace with non-default namespace name if needed
---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: exascaler-csi-controller-cluster-role
rules:
  - apiGroups: ['']
    resources: ['secrets']
    verbs: ['get', 'list', "watch"]
  - apiGroups: ['']
    resources: ['persistentvolumes']
    verbs: ['get', 'list', 'watch', 'create', 'update', 'delete'] # "update" for attacher
  - apiGroups: ['']
    resources: ['persistentvolumeclaims']
    verbs: ['get', 'list', 'watch', 'update']
  - apiGroups: ['storage.k8s.io']
    resources: ['storageclasses']
    verbs: ['get', 'list', 'watch']
  - apiGroups: ['']
    resources: ['events']
    verbs: ['list', 'watch', 'create', 'update', 'patch']
  # attacher specific
  - apiGroups: ['']
    resources: ['nodes', 'pods']
    verbs: ['get', 'list', 'watch']
  - apiGroups: ['csi.storage.k8s.io']
    resources: ['csinodeinfos']
    verbs: ['get', 'list', 'watch']
  - apiGroups: ['storage.k8s.io']
    resources: ['volumeattachments']
    verbs: ['get', 'list', 'watch', 'update']
  - apiGroups: ['storage.k8s.io']
    resources: ['volumeattachments/status']
    verbs: ['get', 'list', 'watch', 'update', 'patch']
  # snapshotter specific
  - apiGroups: ['snapshot.storage.k8s.io']
    resources: ['volumesnapshotclasses']
    verbs: ['get', 'list', 'watch']
  - apiGroups: ['snapshot.storage.k8s.io']
    resources: ['volumesnapshotcontents']
    verbs: ['create', 'get', 'list', 'watch', 'update', 'delete', 'patch']
  - apiGroups: ['snapshot.storage.k8s.io']
    resources: ['volumesnapshots']
    verbs: ['get', 'list', 'watch', 'update']
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshots/status"]
    verbs: ["update"]
  - apiGroups: ["snapshot.storage.k8s.io"]
    resources: ["volumesnapshotcontents/status"]
    verbs: ["update"]
  - apiGroups: ['apiextensions.k8s.io']
    resources: ['customresourcedefinitions']
    verbs: ['create', 'list', 'watch', 'delete']
  - apiGroups: [""]
    resources: ["persistentvolumeclaims/status"]
    verbs: ["update", "patch"]
  # CSINode specific
  - apiGroups: ["storage.k8s.io"]
    resources: ["csinodes"]
    verbs: ["watch", "list", "get"]
---

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: exascaler-csi-controller-cluster-role-binding
subjects:
  - kind: ServiceAccount
    name: exascaler-csi-controller-service-account
    namespace: default # replace with non-default namespace name if needed
roleRef:
  kind: ClusterRole
  name: exascaler-csi-controller-cluster-role
  apiGroup: rbac.authorization.k8s.io
---

# External Resizer
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: csi-resizer-role
rules:
  # The following rule should be uncommented for plugins that require secrets
  # for provisioning.
  - apiGroups: [""]
    resources: ["secrets"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "update", "patch"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims/status"]
    verbs: ["update", "patch"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["list", "watch", "create", "update", "patch"]

---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: csi-resizer-binding
subjects:
  - kind: ServiceAccount
    name: exascaler-csi-controller-service-account
    namespace: default
roleRef:
  kind: ClusterRole
  name: csi-resizer-role
  apiGroup: rbac.authorization.k8s.io

---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  namespace: default
  name: external-resizer-cfg
rules:
- apiGroups: ["coordination.k8s.io"]
  resources: ["leases"]
  verbs: ["get", "watch", "list", "delete", "update", "create"]

---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: csi-resizer-role-cfg
  namespace: default
subjects:
  - kind: ServiceAccount
    name: exascaler-csi-controller-service-account
    namespace: default
roleRef:
  kind: Role
  name: external-resizer-cfg
  apiGroup: rbac.authorization.k8s.io
---

kind: Service
apiVersion: v1
metadata:
  name: exascaler-csi-controller-service
  labels:
    app: exascaler-csi-controller
spec:
  selector:
    app: exascaler-csi-controller
  ports:
    - name: dummy
      port: 12345
---

# ---------------------------------
# Exascaler CSI Controller Server
# ---------------------------------
#
# Runs single driver controller server (driver + provisioner + attacher + snapshotter) on one of the nodes.
# Controller driver deployment does not support running multiple replicas.
#
kind: Deployment
apiVersion: apps/v1
metadata:
  name: exascaler-csi-controller
spec:
  # serviceName: exascaler-csi-controller-service
  selector:
    matchLabels:
      app: exascaler-csi-controller # has to match .spec.template.metadata.labels
  template:
    metadata:
      labels:
        app: exascaler-csi-controller
    spec:
      serviceAccount: exascaler-csi-controller-service-account
      containers:
        # csi-provisioner: sidecar container that watches Kubernetes PersistentVolumeClaim objects
        # and triggers CreateVolume/DeleteVolume against a CSI endpoint
        - name: csi-provisioner
          image: k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0
          imagePullPolicy: IfNotPresent
          args:
            - --csi-address=/var/lib/csi/sockets/pluginproxy/csi.sock
            - --volume-name-prefix=pvc-exa
            - --strict-topology
            - --immediate-topology=false
            - --feature-gates=Topology=true
            - --timeout=120m
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy
        - name: csi-attacher
          image: k8s.gcr.io/sig-storage/csi-attacher:v3.5.0
          imagePullPolicy: IfNotPresent
          args:
            - --csi-address=$(ADDRESS)
            - --v=2
            - --leader-election=true
          env:
            - name: ADDRESS
              value: /var/lib/csi/sockets/pluginproxy/csi.sock
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
        - name: csi-snapshotter
          image: registry.k8s.io/sig-storage/csi-snapshotter:v5.0.1
          imagePullPolicy: IfNotPresent
          args:
            - -v=3
            - --csi-address=/var/lib/csi/sockets/pluginproxy/csi.sock
            - --timeout=120m
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy
        - name: csi-resizer
          image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0
          args:
            - "--csi-address=$(ADDRESS)"
          env:
            - name: ADDRESS
              value: /var/lib/csi/sockets/pluginproxy/csi.sock
          imagePullPolicy: "IfNotPresent"
          volumeMounts:
            - name: socket-dir
              mountPath: /var/lib/csi/sockets/pluginproxy/
        - name: driver
          securityContext:
            privileged: true
            capabilities:
              add: ['SYS_ADMIN']
            allowPrivilegeEscalation: true
          image: quay.io/ddn/exascaler-csi-file-driver:v2.2.6
          imagePullPolicy: Always
          args:
            - --nodeid=$(KUBE_NODE_NAME)
            - --endpoint=unix://csi/csi.sock
            - --role=controller
          env:
            - name: KUBE_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          volumeMounts:
            - name: socket-dir
              mountPath: /csi
            - name: secret
              mountPath: /config
              readOnly: true
            - name: host
              mountPath: /host
              mountPropagation: Bidirectional
      volumes:
        - name: socket-dir
          emptyDir:
        - name: secret
          secret:
            secretName: exascaler-csi-file-driver-config
        - name: host
          hostPath:
            path: /
            type: Directory
---


# ---------------------------
# exascaler CSI Node Server
# ---------------------------
#
# Runs driver node server (driver + registrar) on each node
#

apiVersion: v1
kind: ServiceAccount
metadata:
  name: exascaler-csi-node-service-account
  namespace: default # replace with non-default namespace name if needed
---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: exascaler-csi-node-cluster-role
rules:
  - apiGroups: ['']
    resources: ['events']
    verbs: ['get', 'list', 'watch', 'create', 'update', 'patch']
  - apiGroups: ['']
    resources: ['nodes']
    verbs: ['get']
---

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: exascaler-csi-node-cluster-role-binding
subjects:
  - kind: ServiceAccount
    name: exascaler-csi-node-service-account
    namespace: default # replace with non-default namespace name if needed
roleRef:
  kind: ClusterRole
  name: exascaler-csi-node-cluster-role
  apiGroup: rbac.authorization.k8s.io
---

# exascaler Node Server as a daemon

kind: DaemonSet
apiVersion: apps/v1
metadata:
  name: exascaler-csi-node
spec:
  selector:
    matchLabels:
      app: exascaler-csi-node
  template:
    metadata:
      labels:
        app: exascaler-csi-node
    spec:
      serviceAccount: exascaler-csi-node-service-account
      hostNetwork: true
      containers:
        # driver-registrar: sidecar container that:
        # 1) registers the CSI driver with kubelet
        # 2) adds the drivers custom NodeId to a label on the Kubernetes Node API Object
        - name: driver-registrar
          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0
          imagePullPolicy: IfNotPresent
          args:
            - --v=5
            - --csi-address=/csi/csi.sock
            - --kubelet-registration-path=/var/lib/kubelet/plugins/exa.csi.ddn.com/csi.sock
          livenessProbe:
            exec:
              command:
              - /csi-node-driver-registrar
              - --kubelet-registration-path=/var/lib/kubelet/plugins/exa.csi.ddn.com/csi.sock
              - --mode=kubelet-registration-probe
            initialDelaySeconds: 30
            timeoutSeconds: 15
          env:
            - name: KUBE_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          volumeMounts:
            - name: socket-dir
              mountPath: /csi
            - name: registration-dir
              mountPath: /registration
        - name: driver
          securityContext:
            privileged: true
            capabilities:
              add: ['SYS_ADMIN']
            allowPrivilegeEscalation: true
          image: quay.io/ddn/exascaler-csi-file-driver:v2.2.6
          imagePullPolicy: Always
          args:
            - --nodeid=$(KUBE_NODE_NAME)
            - --endpoint=unix://csi/csi.sock
            - --role=node
          env:
            - name: KUBE_NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          volumeMounts:
            - name: socket-dir
              mountPath: /csi
            - name: secret
              mountPath: /config
            - name: host
              mountPath: /host
              mountPropagation: Bidirectional
            - name: pods-mount-dir
              mountPath: /var/lib/kubelet/pods
              mountPropagation: Bidirectional
      volumes:
        - name: socket-dir
          hostPath:
            path: /var/lib/kubelet/plugins/exa.csi.ddn.com
            type: DirectoryOrCreate
        - name: registration-dir
          hostPath:
            path: /var/lib/kubelet/plugins_registry/
            type: Directory
        - name: pods-mount-dir
          hostPath:
            path: /var/lib/kubelet/pods
            type: Directory
        - name: host
          hostPath:
            path: /
            type: Directory
        - name: secret
          secret:
            secretName: exascaler-csi-file-driver-config
---