start cloud

netshoot/netshoot-worker.yaml

@@ -2,10 +2,7 @@ apiVersion: v1
 kind: Pod
 metadata:
   name: tmp-shell2
-  namespace: org1
 spec:
-  nodeSelector:
-    nodegroup: a100
   containers:
     - name: shell
       image: nicolaka/netshoot

template/prd/argocd_v2.12.3/3_apply/ing.yaml

@@ -11,7 +11,7 @@ metadata:
 spec:
   ingressClassName: nginx
   rules:
-  - host: argocd.gpulive.nhncloud.com # 사전 생성된 인증서의 도메인적용.
+  - host: alpha-argocd.gpulive.apppaas.io # 사전 생성된 인증서의 도메인적용.
     http:
       paths:
       - path: /
@@ -23,6 +23,6 @@ spec:
               number: 443
   tls:
     - hosts:
-      - "argocd.gpulive.nhncloud.com"
+      - "alpha-argocd.gpulive.apppaas.io"
-      secretName: gpulive-new-ssl
+      secretName: gpulive-apppaas-io-ssl
 

template/prd/argocd_v2.12.3/crd.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# 원본 YAML 파일
+input_file="argocd_v2.12.3_add_ns.yaml"
+
+# CRD가 저장될 파일
+crd_file="crds.yaml"
+
+# CRD를 제외한 나머지 리소스를 저장할 파일
+output_file="deployment_without_crds.yaml"
+
+# CRD 분리
+awk '
+BEGIN { in_crd = 0 }
+/\r$/ { sub(/\r$/, ""); } # 줄 끝에 있는 \r 제거
+/^---/ {
+    if (in_crd) {
+        in_crd = 0
+    } else if ($0 ~ /^---/) {
+        if (!in_crd) print "---" > "'$output_file'"
+    }
+}
+/^apiVersion: apiextensions.k8s.io\/v1/ { in_crd = 1; print "---" > "'$crd_file'" }
+in_crd { print > "'$crd_file'" }
+!in_crd && $0 !~ /^---/ { print > "'$output_file'" }
+' "$input_file"
+
+echo "CRD 부분은 $crd_file에 저장되었고, 나머지 리소스는 $output_file에 저장되었습니다."

template/prd/argocd_v2.12.3/kustomization.yaml

@@ -0,0 +1,5 @@
+# kustomization.yaml
+namespace: argocd
+
+resources:
+  - argocd_v2.12.3.yaml

template/prd/calico_v3.28.2/custom-resources.yaml

@@ -14,8 +14,6 @@ spec:
       encapsulation: VXLAN         # openstack 에서는 VXLAN을 사용해야해안다 byljh
       natOutgoing: Enabled
       nodeSelector: all()
-    nodeAddressAutodetectionV4:
-      interface: "enp2s0|bond1"
 
 ---
 

template/prd/gitea/1_configure/5_secret.yaml

@@ -63,12 +63,12 @@ stringData:
   security: INSTALL_LOCK=true
   server: |-
     APP_DATA_PATH=/data
-    DOMAIN=gitea.gpulive.nhncloud.com
+    DOMAIN=alpha-gitea.gpulive.apppaas.io
     ENABLE_PPROF=false
     HTTP_PORT=3000
     PROTOCOL=http
-    ROOT_URL=https://gitea.gpulive.nhncloud.com
+    ROOT_URL=https://alpha-gitea.gpulive.apppaas.io
-    SSH_DOMAIN=gitea.gpulive.nhncloud.com
+    SSH_DOMAIN=alpha-gitea.gpulive.apppaas.io
     SSH_LISTEN_PORT=2222
     SSH_PORT=22
     START_SSH_SERVER=true

template/prd/gitea/1_configure/6_pvc.yaml

@@ -11,8 +11,4 @@ spec:
     #  storageClassName: sc-monitoring
   resources:
     requests:
-      storage: 500Gi # default: 10Gi
+      storage: 100Gi # default: 10Gi
-  selector:
-    matchLabels:
-      # to create 1-1 relationship for pod - persistent volume use unique labels
-      name: gitea-shared-storage-pv

template/prd/gitea/3_apply/ing.yaml

@@ -9,7 +9,7 @@ metadata:
 spec:
   ingressClassName: nginx
   rules:
-  - host: gitea.gpulive.nhncloud.com
+  - host: alpha-gitea.gpulive.apppaas.io
     http:
       paths:
       - path: /
@@ -21,6 +21,6 @@ spec:
               number: 3000
   tls:
     - hosts:
-      - "gitea.gpulive.nhncloud.com"
+      - "alpha-gitea.gpulive.apppaas.io"
-      secretName: gpulive-new-ssl
+      secretName: gpulive-apppaas-io-ssl
 

template/prd/harbor_2.9.1/1_configure/pvc.yaml

@@ -15,10 +15,6 @@ spec:
   resources:
     requests:
       storage: 10Gi
-  selector:
-    matchLabels:
-      # to create 1-1 relationship for pod - persistent volume use unique labels
-      name: harbor-jobservice-pv
 ---
 apiVersion: v1
 kind: PersistentVolumeClaim
@@ -36,8 +32,4 @@ spec:
   resources:
     requests:
       storage: 100Gi
-  selector:
-    matchLabels:
-      # to create 1-1 relationship for pod - persistent volume use unique labels
-      name: harbor-registry-pv
 

template/prd/harbor_2.9.1/2_configmap/configmap.yaml

@@ -14,7 +14,7 @@ data:
   CORE_LOCAL_URL: http://127.0.0.1:8080
   CORE_URL: http://harbor-core:80
   DATABASE_TYPE: postgresql
-  EXT_ENDPOINT: https://registry.gpulive.nhncloud.com
+  EXT_ENDPOINT: https://alpha-registry.gpulive.apppaas.io
   HTTP_PROXY: ""
   HTTPS_PROXY: ""
   JOBSERVICE_URL: http://harbor-jobservice

template/prd/harbor_2.9.1/3_apply/deploy.yaml

@@ -149,6 +149,8 @@ spec:
       automountServiceAccountToken: false
       containers:
       - env:
+        - name: TZ
+          value: Asia/Seoul
         - name: CORE_SECRET
           valueFrom:
             secretKeyRef:
@@ -262,6 +264,8 @@ spec:
       automountServiceAccountToken: false
       containers:
       - env:
+        - name: TZ
+          value: Asia/Seoul
         - name: CORE_SECRET
           valueFrom:
             secretKeyRef:
@@ -340,6 +344,9 @@ spec:
       containers:
       - image: goharbor/harbor-portal:v2.11.2
         imagePullPolicy: IfNotPresent
+        env:
+        - name: TZ
+          value: Asia/Seoul
         livenessProbe:
           httpGet:
             path: /
@@ -402,7 +409,9 @@ spec:
       - args:
         - serve
         - /etc/registry/config.yml
-        env: null
+        env:
+        - name: TZ
+          value: Asia/Seoul
         envFrom:
         - secretRef:
             name: harbor-registry
@@ -437,6 +446,8 @@ spec:
           name: registry-config
           subPath: config.yml
       - env:
+        - name: TZ
+          value: Asia/Seoul
         - name: CORE_SECRET
           valueFrom:
             secretKeyRef:
@@ -531,6 +542,8 @@ spec:
       automountServiceAccountToken: false
       containers:
       - env:
+        - name: TZ
+          value: Asia/Seoul
         - name: PGDATA
           value: /var/lib/postgresql/data/pgdata
         envFrom:
@@ -635,6 +648,9 @@ spec:
       automountServiceAccountToken: false
       containers:
       - image: goharbor/redis-photon:v2.11.2
+        env:
+        - name: TZ
+          value: Asia/Seoul
         imagePullPolicy: IfNotPresent
         livenessProbe:
           initialDelaySeconds: 300
@@ -650,6 +666,14 @@ spec:
         volumeMounts:
         - mountPath: /var/lib/redis
           name: data
+        - name: tz-config
+          mountPath: /etc/localtime
+          readOnly: true
+      volumes:
+      - name: tz-config
+        hostPath:
+          path: /usr/share/zoneinfo/Asia/Seoul
+          type: File
       securityContext:
         fsGroup: 999
         runAsUser: 999
@@ -697,6 +721,8 @@ spec:
       automountServiceAccountToken: false
       containers:
       - env:
+        - name: TZ
+          value: Asia/Seoul
         - name: HTTP_PROXY
           value: ""
         - name: HTTPS_PROXY
@@ -802,3 +828,4 @@ spec:
       resources:
         requests:
           storage: 50Gi
+

template/prd/harbor_2.9.1/3_apply/deploy.yaml.bak

@@ -0,0 +1,804 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: harbor
+    release: harbor
+  name: harbor-core
+  namespace: harbor
+spec:
+  ports:
+  - name: http-web
+    port: 80
+    targetPort: 8080
+  selector:
+    app: harbor
+    component: core
+    release: harbor
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: harbor
+    release: harbor
+  name: harbor-database
+  namespace: harbor
+spec:
+  ports:
+  - port: 5432
+  selector:
+    app: harbor
+    component: database
+    release: harbor
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: harbor
+    release: harbor
+  name: harbor-jobservice
+  namespace: harbor
+spec:
+  ports:
+  - name: http-jobservice
+    port: 80
+    targetPort: 8080
+  selector:
+    app: harbor
+    component: jobservice
+    release: harbor
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: harbor
+    release: harbor
+  name: harbor-portal
+  namespace: harbor
+spec:
+  ports:
+  - port: 80
+    targetPort: 8080
+  selector:
+    app: harbor
+    component: portal
+    release: harbor
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: harbor
+    release: harbor
+  name: harbor-redis
+  namespace: harbor
+spec:
+  ports:
+  - port: 6379
+  selector:
+    app: harbor
+    component: redis
+    release: harbor
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: harbor
+    release: harbor
+  name: harbor-registry
+  namespace: harbor
+spec:
+  ports:
+  - name: http-registry
+    port: 5000
+  - name: http-controller
+    port: 8080
+  selector:
+    app: harbor
+    component: registry
+    release: harbor
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app: harbor
+    release: harbor
+  name: harbor-trivy
+  namespace: harbor
+spec:
+  ports:
+  - name: http-trivy
+    port: 8080
+    protocol: TCP
+  selector:
+    app: harbor
+    component: trivy
+    release: harbor
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: harbor
+    component: core
+    release: harbor
+  name: harbor-core
+  namespace: harbor
+spec:
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app: harbor
+      component: core
+      release: harbor
+  template:
+    metadata:
+      labels:
+        app: harbor
+        component: core
+        release: harbor
+    spec:
+      nodeSelector:
+        nodegroup: nd
+      automountServiceAccountToken: false
+      containers:
+      - env:
+        - name: CORE_SECRET
+          valueFrom:
+            secretKeyRef:
+              key: secret
+              name: harbor-core
+        - name: JOBSERVICE_SECRET
+          valueFrom:
+            secretKeyRef:
+              key: JOBSERVICE_SECRET
+              name: harbor-jobservice
+        envFrom:
+        - configMapRef:
+            name: harbor-core
+        - secretRef:
+            name: harbor-core
+        image: goharbor/harbor-core:v2.11.2
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 2
+          httpGet:
+            path: /api/v2.0/ping
+            port: 8080
+            scheme: HTTP
+          periodSeconds: 10
+        name: core
+        ports:
+        - containerPort: 8080
+        readinessProbe:
+          failureThreshold: 2
+          httpGet:
+            path: /api/v2.0/ping
+            port: 8080
+            scheme: HTTP
+          periodSeconds: 10
+        startupProbe:
+          failureThreshold: 360
+          httpGet:
+            path: /api/v2.0/ping
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        volumeMounts:
+        - mountPath: /etc/core/app.conf
+          name: config
+          subPath: app.conf
+        - mountPath: /etc/core/key
+          name: secret-key
+          subPath: key
+        - mountPath: /etc/core/private_key.pem
+          name: token-service-private-key
+          subPath: tls.key
+        - mountPath: /etc/core/ca
+          name: ca-download
+        - mountPath: /etc/core/token
+          name: psc
+      securityContext:
+        fsGroup: 10000
+        runAsUser: 10000
+      terminationGracePeriodSeconds: 120
+      volumes:
+      - configMap:
+          items:
+          - key: app.conf
+            path: app.conf
+          name: harbor-core
+        name: config
+      - name: secret-key
+        secret:
+          items:
+          - key: secretKey
+            path: key
+          secretName: harbor-core
+      - name: token-service-private-key
+        secret:
+          secretName: harbor-core
+      - name: ca-download
+        secret:
+          secretName: harbor-cacerts
+      - emptyDir: {}
+        name: psc
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: harbor
+    component: jobservice
+    release: harbor
+  name: harbor-jobservice
+  namespace: harbor
+spec:
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app: harbor
+      component: jobservice
+      release: harbor
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: harbor
+        component: jobservice
+        release: harbor
+    spec:
+      nodeSelector:
+        nodegroup: nd
+      automountServiceAccountToken: false
+      containers:
+      - env:
+        - name: CORE_SECRET
+          valueFrom:
+            secretKeyRef:
+              key: secret
+              name: harbor-core
+        envFrom:
+        - configMapRef:
+            name: harbor-jobservice-env
+        - secretRef:
+            name: harbor-jobservice
+        image: goharbor/harbor-jobservice:v2.11.2
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /api/v1/stats
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 300
+          periodSeconds: 10
+        name: jobservice
+        ports:
+        - containerPort: 8080
+        readinessProbe:
+          httpGet:
+            path: /api/v1/stats
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 20
+          periodSeconds: 10
+        volumeMounts:
+        - mountPath: /etc/jobservice/config.yml
+          name: jobservice-config
+          subPath: config.yml
+        - mountPath: /var/log/jobs
+          name: job-logs
+          subPath: null
+      securityContext:
+        fsGroup: 10000
+        runAsUser: 10000
+      terminationGracePeriodSeconds: 120
+      volumes:
+      - configMap:
+          name: harbor-jobservice
+        name: jobservice-config
+      - name: job-logs
+        persistentVolumeClaim:
+          claimName: harbor-jobservice
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: harbor
+    component: portal
+    release: harbor
+  name: harbor-portal
+  namespace: harbor
+spec:
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app: harbor
+      component: portal
+      release: harbor
+  template:
+    metadata:
+      labels:
+        app: harbor
+        component: portal
+        release: harbor
+    spec:
+      nodeSelector:
+        nodegroup: nd
+      automountServiceAccountToken: false
+      containers:
+      - image: goharbor/harbor-portal:v2.11.2
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 300
+          periodSeconds: 10
+        name: portal
+        ports:
+        - containerPort: 8080
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 1
+          periodSeconds: 10
+        volumeMounts:
+        - mountPath: /etc/nginx/nginx.conf
+          name: portal-config
+          subPath: nginx.conf
+      securityContext:
+        fsGroup: 10000
+        runAsUser: 10000
+      volumes:
+      - configMap:
+          name: harbor-portal
+        name: portal-config
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    app: harbor
+    component: registry
+    release: harbor
+  name: harbor-registry
+  namespace: harbor
+spec:
+  replicas: 1
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app: harbor
+      component: registry
+      release: harbor
+  strategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: harbor
+        component: registry
+        release: harbor
+    spec:
+      nodeSelector:
+        nodegroup: nd
+      automountServiceAccountToken: false
+      containers:
+      - args:
+        - serve
+        - /etc/registry/config.yml
+        env: null
+        envFrom:
+        - secretRef:
+            name: harbor-registry
+        image: goharbor/registry-photon:v2.11.2
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 5000
+            scheme: HTTP
+          initialDelaySeconds: 300
+          periodSeconds: 10
+        name: registry
+        ports:
+        - containerPort: 5000
+        - containerPort: 5001
+        readinessProbe:
+          httpGet:
+            path: /
+            port: 5000
+            scheme: HTTP
+          initialDelaySeconds: 1
+          periodSeconds: 10
+        volumeMounts:
+        - mountPath: /storage
+          name: registry-data
+          subPath: null
+        - mountPath: /etc/registry/passwd
+          name: registry-htpasswd
+          subPath: passwd
+        - mountPath: /etc/registry/config.yml
+          name: registry-config
+          subPath: config.yml
+      - env:
+        - name: CORE_SECRET
+          valueFrom:
+            secretKeyRef:
+              key: secret
+              name: harbor-core
+        - name: JOBSERVICE_SECRET
+          valueFrom:
+            secretKeyRef:
+              key: JOBSERVICE_SECRET
+              name: harbor-jobservice
+        envFrom:
+        - configMapRef:
+            name: harbor-registryctl
+        - secretRef:
+            name: harbor-registry
+        - secretRef:
+            name: harbor-registryctl
+        image: goharbor/harbor-registryctl:v2.11.2
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          httpGet:
+            path: /api/health
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 300
+          periodSeconds: 10
+        name: registryctl
+        ports:
+        - containerPort: 8080
+        readinessProbe:
+          httpGet:
+            path: /api/health
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 1
+          periodSeconds: 10
+        volumeMounts:
+        - mountPath: /storage
+          name: registry-data
+          subPath: null
+        - mountPath: /etc/registry/config.yml
+          name: registry-config
+          subPath: config.yml
+        - mountPath: /etc/registryctl/config.yml
+          name: registry-config
+          subPath: ctl-config.yml
+      securityContext:
+        fsGroup: 10000
+        fsGroupChangePolicy: OnRootMismatch
+        runAsUser: 10000
+      terminationGracePeriodSeconds: 120
+      volumes:
+      - name: registry-htpasswd
+        secret:
+          items:
+          - key: REGISTRY_HTPASSWD
+            path: passwd
+          secretName: harbor-registry-htpasswd
+      - configMap:
+          name: harbor-registry
+        name: registry-config
+      - name: registry-data
+        persistentVolumeClaim:
+          claimName: harbor-registry
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    app: harbor
+    component: database
+    release: harbor
+  name: harbor-database
+  namespace: harbor
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: harbor
+      component: database
+      release: harbor
+  serviceName: harbor-database
+  template:
+    metadata:
+      labels:
+        app: harbor
+        component: database
+        release: harbor
+    spec:
+      nodeSelector:
+        nodegroup: nd
+      automountServiceAccountToken: false
+      containers:
+      - env:
+        - name: PGDATA
+          value: /var/lib/postgresql/data/pgdata
+        envFrom:
+        - secretRef:
+            name: harbor-database
+        image: goharbor/harbor-db:v2.11.2
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          exec:
+            command:
+            - /docker-healthcheck.sh
+          initialDelaySeconds: 300
+          periodSeconds: 10
+          timeoutSeconds: 1
+        name: database
+        readinessProbe:
+          exec:
+            command:
+            - /docker-healthcheck.sh
+          initialDelaySeconds: 1
+          periodSeconds: 10
+          timeoutSeconds: 1
+        volumeMounts:
+        - mountPath: /var/lib/postgresql/data
+          name: database-data
+        - mountPath: /dev/shm
+          name: shm-volume
+      initContainers:
+      - args:
+        - -c
+        - '[ -e /var/lib/postgresql/data/postgresql.conf ] && [ ! -d /var/lib/postgresql/data/pgdata
+          ] && mkdir -m 0700 /var/lib/postgresql/data/pgdata && mv /var/lib/postgresql/data/*
+          /var/lib/postgresql/data/pgdata/ || true'
+        command:
+        - /bin/sh
+        image: goharbor/harbor-db:v2.11.2
+        imagePullPolicy: IfNotPresent
+        name: data-migrator
+        volumeMounts:
+        - mountPath: /var/lib/postgresql/data
+          name: database-data
+      - args:
+        - -c
+        - chmod -R 700 /var/lib/postgresql/data/pgdata || true
+        command:
+        - /bin/sh
+        image: goharbor/harbor-db:v2.11.2
+        imagePullPolicy: IfNotPresent
+        name: data-permissions-ensurer
+        volumeMounts:
+        - mountPath: /var/lib/postgresql/data
+          name: database-data
+      securityContext:
+        fsGroup: 999
+        runAsUser: 999
+      terminationGracePeriodSeconds: 120
+      volumes:
+      - emptyDir:
+          medium: Memory
+          sizeLimit: 512Mi
+        name: shm-volume
+  volumeClaimTemplates:
+  - metadata:
+      annotations: null
+      labels:
+        app: harbor
+        release: harbor
+      name: database-data
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Gi
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    app: harbor
+    component: redis
+    release: harbor
+  name: harbor-redis
+  namespace: harbor
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: harbor
+      component: redis
+      release: harbor
+  serviceName: harbor-redis
+  template:
+    metadata:
+      labels:
+        app: harbor
+        component: redis
+        release: harbor
+    spec:
+      nodeSelector:
+        nodegroup: nd
+      automountServiceAccountToken: false
+      containers:
+      - image: goharbor/redis-photon:v2.11.2
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          initialDelaySeconds: 300
+          periodSeconds: 10
+          tcpSocket:
+            port: 6379
+        name: redis
+        readinessProbe:
+          initialDelaySeconds: 1
+          periodSeconds: 10
+          tcpSocket:
+            port: 6379
+        volumeMounts:
+        - mountPath: /var/lib/redis
+          name: data
+      securityContext:
+        fsGroup: 999
+        runAsUser: 999
+      terminationGracePeriodSeconds: 120
+  volumeClaimTemplates:
+  - metadata:
+      annotations: null
+      labels:
+        app: harbor
+        release: harbor
+      name: data
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 10Gi
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    app: harbor
+    component: trivy
+    release: harbor
+  name: harbor-trivy
+  namespace: harbor
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: harbor
+      component: trivy
+      release: harbor
+  serviceName: harbor-trivy
+  template:
+    metadata:
+      labels:
+        app: harbor
+        component: trivy
+        release: harbor
+    spec:
+      nodeSelector:
+        nodegroup: nd
+      automountServiceAccountToken: false
+      containers:
+      - env:
+        - name: HTTP_PROXY
+          value: ""
+        - name: HTTPS_PROXY
+          value: ""
+        - name: NO_PROXY
+          value: harbor-core,harbor-jobservice,harbor-database,harbor-registry,harbor-portal,harbor-trivy,harbor-exporter,127.0.0.1,localhost,.local,.internal
+        - name: SCANNER_LOG_LEVEL
+          value: info
+        - name: SCANNER_TRIVY_CACHE_DIR
+          value: /home/scanner/.cache/trivy
+        - name: SCANNER_TRIVY_REPORTS_DIR
+          value: /home/scanner/.cache/reports
+        - name: SCANNER_TRIVY_DEBUG_MODE
+          value: "false"
+        - name: SCANNER_TRIVY_VULN_TYPE
+          value: os,library
+        - name: SCANNER_TRIVY_TIMEOUT
+          value: 5m0s
+        - name: SCANNER_TRIVY_GITHUB_TOKEN
+          valueFrom:
+            secretKeyRef:
+              key: gitHubToken
+              name: harbor-trivy
+        - name: SCANNER_TRIVY_SEVERITY
+          value: UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL
+        - name: SCANNER_TRIVY_IGNORE_UNFIXED
+          value: "false"
+        - name: SCANNER_TRIVY_SKIP_UPDATE
+          value: "false"
+        - name: SCANNER_TRIVY_OFFLINE_SCAN
+          value: "false"
+        - name: SCANNER_TRIVY_SECURITY_CHECKS
+          value: vuln
+        - name: SCANNER_TRIVY_INSECURE
+          value: "false"
+        - name: SCANNER_API_SERVER_ADDR
+          value: :8080
+        - name: SCANNER_REDIS_URL
+          valueFrom:
+            secretKeyRef:
+              key: redisURL
+              name: harbor-trivy
+        - name: SCANNER_STORE_REDIS_URL
+          valueFrom:
+            secretKeyRef:
+              key: redisURL
+              name: harbor-trivy
+        - name: SCANNER_JOB_QUEUE_REDIS_URL
+          valueFrom:
+            secretKeyRef:
+              key: redisURL
+              name: harbor-trivy
+        image: goharbor/trivy-adapter-photon:v2.11.2
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 10
+          httpGet:
+            path: /probe/healthy
+            port: api-server
+            scheme: HTTP
+          initialDelaySeconds: 5
+          periodSeconds: 10
+          successThreshold: 1
+        name: trivy
+        ports:
+        - containerPort: 8080
+          name: api-server
+        readinessProbe:
+          failureThreshold: 3
+          httpGet:
+            path: /probe/ready
+            port: api-server
+            scheme: HTTP
+          initialDelaySeconds: 5
+          periodSeconds: 10
+          successThreshold: 1
+        resources:
+          limits:
+            cpu: 1
+            memory: 1Gi
+          requests:
+            cpu: 200m
+            memory: 512Mi
+        securityContext:
+          allowPrivilegeEscalation: false
+          privileged: false
+        volumeMounts:
+        - mountPath: /home/scanner/.cache
+          name: data
+          readOnly: false
+      securityContext:
+        fsGroup: 10000
+        runAsUser: 10000
+  volumeClaimTemplates:
+  - metadata:
+      labels:
+        app: harbor
+        release: harbor
+      name: data
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 50Gi

template/prd/harbor_2.9.1/3_apply/ing.yaml

@@ -0,0 +1,64 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  annotations:
+    ingress.kubernetes.io/proxy-body-size: "0"
+    nginx.ingress.kubernetes.io/proxy-body-size: "0"
+      #nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+  labels:
+    app: harbor
+    release: harbor
+  name: harbor-ingress
+  namespace: harbor
+spec:
+  ingressClassName: nginx
+  tls:
+  - hosts:
+    - "alpha-registry.gpulive.apppaas.io"
+    secretName: gpulive-apppaas-io-ssl
+  rules:
+  - host: alpha-registry.gpulive.apppaas.io
+    http:
+      paths:
+      - backend:
+          service:
+            name: harbor-core
+            port:
+              number: 80
+        path: /api/
+        pathType: Prefix
+      - backend:
+          service:
+            name: harbor-core
+            port:
+              number: 80
+        path: /service/
+        pathType: Prefix
+      - backend:
+          service:
+            name: harbor-core
+            port:
+              number: 80
+        path: /v2/
+        pathType: Prefix
+      - backend:
+          service:
+            name: harbor-core
+            port:
+              number: 80
+        path: /chartrepo/
+        pathType: Prefix
+      - backend:
+          service:
+            name: harbor-core
+            port:
+              number: 80
+        path: /c/
+        pathType: Prefix
+      - backend:
+          service:
+            name: harbor-portal
+            port:
+              number: 80
+        path: /
+        pathType: Prefix

template/prd/harbor_2.9.1/image-pull-test.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nginx-pod
+  labels:
+    app: nginx
+spec:
+  containers:
+    - name: nginx
+      image: harbor.nhngpuaas.com/test/nginx:test
+      ports:
+        - containerPort: 80
+

template/prd/harbor_2.9.1/pv-delete.py

@@ -0,0 +1,12 @@
+#!/usr/bin/python3
+import os
+import subprocess
+
+pv_str = os.popen("kubectl get -n default pv | grep harbor | awk '$1 > 0 {print $1}'").read()
+print(pv_str)
+pv_list = pv_str.split('\n')
+i = 0
+for pv in pv_list:
+    print(pv)
+    # os.system(f"kubectl delete -n {ns} pv {pv}")
+    os.system(f"kubectl delete pv {pv}")

template/prd/monitoring/base/grafana/cm.yaml

@@ -0,0 +1,27 @@
+# Source: grafana/templates/configmap.yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: grafana
+  namespace: monitoring
+  labels:
+    helm.sh/chart: grafana-8.10.0
+    app.kubernetes.io/name: grafana
+    app.kubernetes.io/instance: grafana
+    app.kubernetes.io/version: "11.5.1"
+data:
+  
+  grafana.ini: |
+    [analytics]
+    check_for_updates = true
+    [grafana_net]
+    url = https://grafana.net
+    [log]
+    mode = console
+    [paths]
+    data = /var/lib/grafana/
+    logs = /var/log/grafana
+    plugins = /var/lib/grafana/plugins
+    provisioning = /etc/grafana/provisioning
+    [server]
+    domain = ''

template/prd/monitoring/base/grafana/kustomization.yaml

@@ -0,0 +1,17 @@
+# Prometheus에 종속적으로 설치
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: monitoring
+
+resources:
+  - cm.yaml
+  - deploy.yaml
+  # - namespace.yaml
+  - role.yaml
+  - sa.yaml
+  - secret.yaml
+  - service.yaml
+  - ingress.yaml
+  - pvc.yaml
+  

template/prd/monitoring/base/grafana/pvc.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: grafana-pvc
+  namespace: monitoring
+  labels:
+    app.kubernetes.io/name: grafana
+spec:
+  accessModes:
+    - "ReadWriteOnce"
+  resources:
+    requests:
+      storage: "10Gi"

template/prd/monitoring/base/prometheus/kustomization.yaml

@@ -0,0 +1,19 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: monitoring
+
+resources:
+  - namespace.yaml
+  - sa.yaml
+  - cluster_role.yaml
+  - cm.yaml
+  - deploy.yaml
+  - daemonset.yaml
+  - pvc.yaml
+  - service.yaml
+  - state_full_set.yaml
+    #  - secret.yaml
+  - ingress_alertmanager.yaml
+  - ingress_prometheus.yaml
+  - cm_custom_alert_rules.yaml

template/prd/monitoring/base/prometheus/pvc.yaml

@@ -0,0 +1,19 @@
+# Source: prometheus/templates/pvc.yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  labels:
+    app.kubernetes.io/component: server
+    app.kubernetes.io/name: prometheus
+    app.kubernetes.io/instance: prometheus
+    app.kubernetes.io/version: v3.1.0
+    helm.sh/chart: prometheus-27.3.1
+    app.kubernetes.io/part-of: prometheus
+  name: prometheus-server
+  namespace: monitoring
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: "8Gi"

template/prd/monitoring/base/prometheus/state_full_set.yaml

@@ -0,0 +1,90 @@
+# Source: prometheus/charts/alertmanager/templates/statefulset.yaml
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  name: prometheus-alertmanager
+  labels:
+    helm.sh/chart: alertmanager-1.14.0
+    app.kubernetes.io/name: alertmanager
+    app.kubernetes.io/instance: prometheus
+    app.kubernetes.io/version: "v0.28.0"
+    app.kubernetes.io/managed-by: Helm
+  namespace: monitoring
+spec:
+  replicas: 1
+  minReadySeconds: 0
+  revisionHistoryLimit: 10
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: alertmanager
+      app.kubernetes.io/instance: prometheus
+  serviceName: prometheus-alertmanager-headless
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: alertmanager
+        app.kubernetes.io/instance: prometheus
+    spec:
+      nodeSelector:
+        nodegroup: nd
+      automountServiceAccountToken: true
+      serviceAccountName: prometheus-alertmanager
+      securityContext:
+        fsGroup: 65534
+        runAsGroup: 65534
+        runAsNonRoot: true
+        runAsUser: 65534
+      containers:
+        - name: alertmanager
+          securityContext:
+            runAsGroup: 65534
+            runAsNonRoot: true
+            runAsUser: 65534
+          image: "quay.io/prometheus/alertmanager:v0.28.0"
+          imagePullPolicy: Always
+          env:
+            - name: POD_IP
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: status.podIP
+          args:
+            - --storage.path=/alertmanager
+            - --config.file=/etc/alertmanager/alertmanager.yml
+          ports:
+            - name: http
+              containerPort: 9093
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /
+              port: http
+          readinessProbe:
+            httpGet:
+              path: /
+              port: http
+          resources:
+            requests:
+              cpu: 200m
+              memory: 512Mi
+            limits:
+              cpu: 200m
+              memory: 512Mi 
+          volumeMounts:
+            - name: config
+              mountPath: /etc/alertmanager
+            - name: storage
+              mountPath: /alertmanager
+      volumes:
+        - name: config
+          configMap:
+            name: prometheus-alertmanager
+  volumeClaimTemplates:
+    - metadata:
+        name: storage
+      spec:
+        accessModes:
+          - ReadWriteOnce
+        resources:
+          requests:
+            storage: 2Gi

template/prd/monitoring/overlays/default/prometheus/patch-ingress_alertmanager.yaml

@@ -0,0 +1,22 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: prometheus-alertmanager-ingress
+  namespace: monitoring
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: alpha-alert.gpulive.apppaas.io
+    http:
+      paths:
+      - backend:
+          service:
+            name: prometheus-alertmanager
+            port:
+              number: 9093
+        path: /
+        pathType: Prefix
+  tls:
+  - hosts:
+    - alpha-alert.gpulive.apppaas.io
+    secretName: gpulive-apppaas-io-ssl

template/prd/nfs-provisioner/nfs-provisioner.yaml

@@ -0,0 +1,41 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nfs-client-provisioner
+  labels:
+    app: nfs-client-provisioner
+  # replace with namespace where provisioner is deployed
+  namespace: default
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: nfs-client-provisioner
+  template:
+    metadata:
+      labels:
+        app: nfs-client-provisioner
+    spec:
+      nodeSelector:
+        nodegroup: nd
+      serviceAccountName: nfs-client-provisioner
+      containers:
+        - name: nfs-client-provisioner
+          image: registry.k8s.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2
+          volumeMounts:
+            - name: nfs-client-root
+              mountPath: /persistentvolumes
+          env:
+            - name: PROVISIONER_NAME
+              value: k8s-sigs.io/nfs-subdir-external-provisioner
+            - name: NFS_SERVER
+              value: 192.168.0.126
+            - name: NFS_PATH
+              value: /GJ_SHARE_FS2/afed9d8d-f9c8-4053-aa89-23cdc70938e3
+      volumes:
+        - name: nfs-client-root
+          nfs:
+            server: 192.168.0.126
+            path: /GJ_SHARE_FS2/afed9d8d-f9c8-4053-aa89-23cdc70938e3

template/prd/nfs-provisioner/rbac.yaml

@@ -0,0 +1,68 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: nfs-client-provisioner
+  # replace with namespace where provisioner is deployed
+  namespace: default
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: nfs-client-provisioner-runner
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "delete"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["create", "update", "patch"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: run-nfs-client-provisioner
+subjects:
+  - kind: ServiceAccount
+    name: nfs-client-provisioner
+    # replace with namespace where provisioner is deployed
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: nfs-client-provisioner-runner
+  apiGroup: rbac.authorization.k8s.io
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: leader-locking-nfs-client-provisioner
+  # replace with namespace where provisioner is deployed
+  namespace: default
+rules:
+  - apiGroups: [""]
+    resources: ["endpoints"]
+    verbs: ["get", "list", "watch", "create", "update", "patch"]
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: leader-locking-nfs-client-provisioner
+  # replace with namespace where provisioner is deployed
+  namespace: default
+subjects:
+  - kind: ServiceAccount
+    name: nfs-client-provisioner
+    # replace with namespace where provisioner is deployed
+    namespace: default
+roleRef:
+  kind: Role
+  name: leader-locking-nfs-client-provisioner
+  apiGroup: rbac.authorization.k8s.io

template/prd/nfs-provisioner/sc.yaml

@@ -0,0 +1,11 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: nfs-client
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: k8s-sigs.io/nfs-subdir-external-provisioner # or choose another name, must match deployment's env PROVISIONER_NAME'
+parameters:
+  archiveOnDelete: "false"
+  onDelete: Retain
+reclaimPolicy: Retain

template/prd/nfs-provisioner/test-claim.yaml

@@ -0,0 +1,11 @@
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: test-claim
+spec:
+  storageClassName: nfs-client
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Gi

template/prd/nfs-provisioner/test-pod.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: nhnent01-pod01
+  namespace: default
+  labels:
+    app: nginx
+spec:
+  containers:
+   - name: web-server
+     image: nginx
+     volumeMounts:
+       - name: mypvc
+         mountPath: /data
+     ports:
+     - containerPort: 80
+  volumes:
+   - name: mypvc
+     persistentVolumeClaim:
+       claimName: test-claim
+       readOnly: false
+

template/prd/nfs-provisioner/user-sc.yaml

@@ -0,0 +1,8 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: user-nas
+provisioner: k8s-sigs.io/nfs-subdir-external-provisioner # or choose another name, must match deployment's env PROVISIONER_NAME'
+parameters:
+  archiveOnDelete: "false"
+  onDelete: Retain

template/stg/argocd_v2.12.3/1_configure/init.yaml

@@ -0,0 +1,672 @@
+apiVersion: v1                  # ns 용 추가 by ljh
+kind: Namespace
+metadata:
+  name: argocd
+  labels:
+    name: argocd
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: application-controller
+    app.kubernetes.io/name: argocd-application-controller
+    app.kubernetes.io/part-of: argocd
+  name: argocd-application-controller
+  namespace: argocd
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: applicationset-controller
+    app.kubernetes.io/name: argocd-applicationset-controller
+    app.kubernetes.io/part-of: argocd
+  name: argocd-applicationset-controller
+  namespace: argocd
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: dex-server
+    app.kubernetes.io/name: argocd-dex-server
+    app.kubernetes.io/part-of: argocd
+  name: argocd-dex-server
+  namespace: argocd
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: notifications-controller
+    app.kubernetes.io/name: argocd-notifications-controller
+    app.kubernetes.io/part-of: argocd
+  name: argocd-notifications-controller
+  namespace: argocd
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: redis
+    app.kubernetes.io/name: argocd-redis
+    app.kubernetes.io/part-of: argocd
+  name: argocd-redis
+  namespace: argocd
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: repo-server
+    app.kubernetes.io/name: argocd-repo-server
+    app.kubernetes.io/part-of: argocd
+  name: argocd-repo-server
+  namespace: argocd
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    app.kubernetes.io/component: server
+    app.kubernetes.io/name: argocd-server
+    app.kubernetes.io/part-of: argocd
+  name: argocd-server
+  namespace: argocd
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/component: application-controller
+    app.kubernetes.io/name: argocd-application-controller
+    app.kubernetes.io/part-of: argocd
+  name: argocd-application-controller
+  namespace: argocd
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - argoproj.io
+  resources:
+  - applications
+  - appprojects
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - list
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/component: applicationset-controller
+    app.kubernetes.io/name: argocd-applicationset-controller
+    app.kubernetes.io/part-of: argocd
+  name: argocd-applicationset-controller
+  namespace: argocd
+rules:
+- apiGroups:
+  - argoproj.io
+  resources:
+  - applications
+  - applicationsets
+  - applicationsets/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - argoproj.io
+  resources:
+  - appprojects
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - argoproj.io
+  resources:
+  - applicationsets/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - deployments
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/component: dex-server
+    app.kubernetes.io/name: argocd-dex-server
+    app.kubernetes.io/part-of: argocd
+  name: argocd-dex-server
+  namespace: argocd
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  - configmaps
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/component: notifications-controller
+    app.kubernetes.io/name: argocd-notifications-controller
+    app.kubernetes.io/part-of: argocd
+  name: argocd-notifications-controller
+  namespace: argocd
+rules:
+- apiGroups:
+  - argoproj.io
+  resources:
+  - applications
+  - appprojects
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  - secrets
+  verbs:
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resourceNames:
+  - argocd-notifications-cm
+  resources:
+  - configmaps
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resourceNames:
+  - argocd-notifications-secret
+  resources:
+  - secrets
+  verbs:
+  - get
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/component: redis
+    app.kubernetes.io/name: argocd-redis
+    app.kubernetes.io/part-of: argocd
+  name: argocd-redis
+  namespace: argocd
+rules:
+- apiGroups:
+  - ""
+  resourceNames:
+  - argocd-redis
+  resources:
+  - secrets
+  verbs:
+  - get
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  labels:
+    app.kubernetes.io/component: server
+    app.kubernetes.io/name: argocd-server
+    app.kubernetes.io/part-of: argocd
+  name: argocd-server
+  namespace: argocd
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  - configmaps
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - argoproj.io
+  resources:
+  - applications
+  - appprojects
+  - applicationsets
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - delete
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/component: application-controller
+    app.kubernetes.io/name: argocd-application-controller
+    app.kubernetes.io/part-of: argocd
+  name: argocd-application-controller
+rules:
+- apiGroups:
+  - '*'
+  resources:
+  - '*'
+  verbs:
+  - '*'
+- nonResourceURLs:
+  - '*'
+  verbs:
+  - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/component: applicationset-controller
+    app.kubernetes.io/name: argocd-applicationset-controller
+    app.kubernetes.io/part-of: argocd
+  name: argocd-applicationset-controller
+rules:
+- apiGroups:
+  - argoproj.io
+  resources:
+  - applications
+  - applicationsets
+  - applicationsets/finalizers
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+- apiGroups:
+  - argoproj.io
+  resources:
+  - applicationsets/status
+  verbs:
+  - get
+  - patch
+  - update
+- apiGroups:
+  - argoproj.io
+  resources:
+  - appprojects
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - create
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - create
+  - update
+  - delete
+  - get
+  - list
+  - patch
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - apps
+  - extensions
+  resources:
+  - deployments
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  labels:
+    app.kubernetes.io/component: server
+    app.kubernetes.io/name: argocd-server
+    app.kubernetes.io/part-of: argocd
+  name: argocd-server
+rules:
+- apiGroups:
+  - '*'
+  resources:
+  - '*'
+  verbs:
+  - delete
+  - get
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - pods/log
+  verbs:
+  - get
+- apiGroups:
+  - argoproj.io
+  resources:
+  - applications
+  - applicationsets
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - batch
+  resources:
+  - jobs
+  verbs:
+  - create
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflows
+  verbs:
+  - create
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: application-controller
+    app.kubernetes.io/name: argocd-application-controller
+    app.kubernetes.io/part-of: argocd
+  name: argocd-application-controller
+  namespace: argocd
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: argocd-application-controller
+subjects:
+- kind: ServiceAccount
+  name: argocd-application-controller
+  namespace: argocd
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: applicationset-controller
+    app.kubernetes.io/name: argocd-applicationset-controller
+    app.kubernetes.io/part-of: argocd
+  name: argocd-applicationset-controller
+  namespace: argocd
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: argocd-applicationset-controller
+subjects:
+- kind: ServiceAccount
+  name: argocd-applicationset-controller
+  namespace: argocd
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: dex-server
+    app.kubernetes.io/name: argocd-dex-server
+    app.kubernetes.io/part-of: argocd
+  name: argocd-dex-server
+  namespace: argocd
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: argocd-dex-server
+subjects:
+- kind: ServiceAccount
+  name: argocd-dex-server
+  namespace: argocd
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: notifications-controller
+    app.kubernetes.io/name: argocd-notifications-controller
+    app.kubernetes.io/part-of: argocd
+  name: argocd-notifications-controller
+  namespace: argocd
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: argocd-notifications-controller
+subjects:
+- kind: ServiceAccount
+  name: argocd-notifications-controller
+  namespace: argocd
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: redis
+    app.kubernetes.io/name: argocd-redis
+    app.kubernetes.io/part-of: argocd
+  name: argocd-redis
+  namespace: argocd
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: argocd-redis
+subjects:
+- kind: ServiceAccount
+  name: argocd-redis
+  namespace: argocd
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: server
+    app.kubernetes.io/name: argocd-server
+    app.kubernetes.io/part-of: argocd
+  name: argocd-server
+  namespace: argocd
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: argocd-server
+subjects:
+- kind: ServiceAccount
+  name: argocd-server
+  namespace: argocd
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: application-controller
+    app.kubernetes.io/name: argocd-application-controller
+    app.kubernetes.io/part-of: argocd
+  name: argocd-application-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: argocd-application-controller
+subjects:
+- kind: ServiceAccount
+  name: argocd-application-controller
+  namespace: argocd
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: applicationset-controller
+    app.kubernetes.io/name: argocd-applicationset-controller
+    app.kubernetes.io/part-of: argocd
+  name: argocd-applicationset-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: argocd-applicationset-controller
+subjects:
+- kind: ServiceAccount
+  name: argocd-applicationset-controller
+  namespace: argocd
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  labels:
+    app.kubernetes.io/component: server
+    app.kubernetes.io/name: argocd-server
+    app.kubernetes.io/part-of: argocd
+  name: argocd-server
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: argocd-server
+subjects:
+- kind: ServiceAccount
+  name: argocd-server
+  namespace: argocd
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    app.kubernetes.io/component: notifications-controller
+    app.kubernetes.io/name: argocd-notifications-controller
+    app.kubernetes.io/part-of: argocd
+  name: argocd-notifications-secret
+  namespace: argocd
+type: Opaque
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    app.kubernetes.io/name: argocd-secret
+    app.kubernetes.io/part-of: argocd
+  name: argocd-secret
+  namespace: argocd
+type: Opaque

template/stg/argocd_v2.12.3/2_configmap/configmap.yaml

@@ -0,0 +1,81 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/name: argocd-cm
+    app.kubernetes.io/part-of: argocd
+  name: argocd-cm
+  namespace: argocd
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/name: argocd-cmd-params-cm
+    app.kubernetes.io/part-of: argocd
+  name: argocd-cmd-params-cm
+  namespace: argocd
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/name: argocd-gpg-keys-cm
+    app.kubernetes.io/part-of: argocd
+  name: argocd-gpg-keys-cm
+  namespace: argocd
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/component: notifications-controller
+    app.kubernetes.io/name: argocd-notifications-controller
+    app.kubernetes.io/part-of: argocd
+  name: argocd-notifications-cm
+  namespace: argocd
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/name: argocd-rbac-cm
+    app.kubernetes.io/part-of: argocd
+  name: argocd-rbac-cm
+  namespace: argocd
+---
+apiVersion: v1
+data:
+  ssh_known_hosts: |
+    # This file was automatically generated by hack/update-ssh-known-hosts.sh. DO NOT EDIT
+    [ssh.github.com]:443 ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
+    [ssh.github.com]:443 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
+    [ssh.github.com]:443 ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
+    bitbucket.org ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBPIQmuzMBuKdWeF4+a2sjSSpBK0iqitSQ+5BM9KhpexuGt20JpTVM7u5BDZngncgrqDMbWdxMWWOGtZ9UgbqgZE=
+    bitbucket.org ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIazEu89wgQZ4bqs3d63QSMzYVa0MuJ2e2gKTKqu+UUO
+    bitbucket.org ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDQeJzhupRu0u0cdegZIa8e86EG2qOCsIsD1Xw0xSeiPDlCr7kq97NLmMbpKTX6Esc30NuoqEEHCuc7yWtwp8dI76EEEB1VqY9QJq6vk+aySyboD5QF61I/1WeTwu+deCbgKMGbUijeXhtfbxSxm6JwGrXrhBdofTsbKRUsrN1WoNgUa8uqN1Vx6WAJw1JHPhglEGGHea6QICwJOAr/6mrui/oB7pkaWKHj3z7d1IC4KWLtY47elvjbaTlkN04Kc/5LFEirorGYVbt15kAUlqGM65pk6ZBxtaO3+30LVlORZkxOh+LKL/BvbZ/iRNhItLqNyieoQj/uh/7Iv4uyH/cV/0b4WDSd3DptigWq84lJubb9t/DnZlrJazxyDCulTmKdOR7vs9gMTo+uoIrPSb8ScTtvw65+odKAlBj59dhnVp9zd7QUojOpXlL62Aw56U4oO+FALuevvMjiWeavKhJqlR7i5n9srYcrNV7ttmDw7kf/97P5zauIhxcjX+xHv4M=
+    github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
+    github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
+    github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
+    gitlab.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFSMqzJeV9rUzU4kWitGjeR4PWSa29SPqJ1fVkhtj3Hw9xjLVXVYrU9QlYWrOLXBpQ6KWjbjTDTdDkoohFzgbEY=
+    gitlab.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAfuCHKVTjquxvt6CM6tdG4SLp1Btn/nOeHHE5UOzRdf
+    gitlab.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCsj2bNKTBSpIYDEGk9KxsGh3mySTRgMtXL583qmBpzeQ+jqCMRgBqB98u3z++J1sKlXHWfM9dyhSevkMwSbhoR8XIq/U0tCNyokEi/ueaBMCvbcTHhO7FcwzY92WK4Yt0aGROY5qX2UKSeOvuP4D6TPqKF1onrSzH9bx9XUf2lEdWT/ia1NEKjunUqu1xOB/StKDHMoX4/OKyIzuS0q/T1zOATthvasJFoPrAjkohTyaDUz2LN5JoH839hViyEG82yB+MjcFV5MU3N1l1QL3cVUCh93xSaua1N85qivl+siMkPGbO5xR/En4iEY6K2XPASUEMaieWVNTRCtJ4S8H+9
+    ssh.dev.azure.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7Hr1oTWqNqOlzGJOfGJ4NakVyIzf1rXYd4d7wo6jBlkLvCA4odBlL0mDUyZ0/QUfTTqeu+tm22gOsv+VrVTMk6vwRU75gY/y9ut5Mb3bR5BV58dKXyq9A9UeB5Cakehn5Zgm6x1mKoVyf+FFn26iYqXJRgzIZZcZ5V6hrE0Qg39kZm4az48o0AUbf6Sp4SLdvnuMa2sVNwHBboS7EJkm57XQPVU3/QpyNLHbWDdzwtrlS+ez30S3AdYhLKEOxAG8weOnyrtLJAUen9mTkol8oII1edf7mWWbWVf0nBmly21+nZcmCTISQBtdcyPaEno7fFQMDD26/s0lfKob4Kw8H
+    vs-ssh.visualstudio.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7Hr1oTWqNqOlzGJOfGJ4NakVyIzf1rXYd4d7wo6jBlkLvCA4odBlL0mDUyZ0/QUfTTqeu+tm22gOsv+VrVTMk6vwRU75gY/y9ut5Mb3bR5BV58dKXyq9A9UeB5Cakehn5Zgm6x1mKoVyf+FFn26iYqXJRgzIZZcZ5V6hrE0Qg39kZm4az48o0AUbf6Sp4SLdvnuMa2sVNwHBboS7EJkm57XQPVU3/QpyNLHbWDdzwtrlS+ez30S3AdYhLKEOxAG8weOnyrtLJAUen9mTkol8oII1edf7mWWbWVf0nBmly21+nZcmCTISQBtdcyPaEno7fFQMDD26/s0lfKob4Kw8H
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/name: argocd-ssh-known-hosts-cm
+    app.kubernetes.io/part-of: argocd
+  name: argocd-ssh-known-hosts-cm
+  namespace: argocd
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/name: argocd-tls-certs-cm
+    app.kubernetes.io/part-of: argocd
+  name: argocd-tls-certs-cm
+  namespace: argocd

template/stg/argocd_v2.12.3/3_apply/ing.yaml

@@ -0,0 +1,28 @@
+# Offloading setting.
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: ing-argocd
+  namespace: argocd
+  annotations:
+    #nginx.ingress.kubernetes.io/force-ssl-redirect: "true"
+    nginx.ingress.kubernetes.io/backend-protocol: "https"
+    nginx.ingress.kubernetes.io/ssl-passthrough: "true"
+spec:
+  ingressClassName: nginx
+  rules:
+  - host: argocd.gpulive.nhncloud.com # 사전 생성된 인증서의 도메인적용.
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: argocd-server
+            port:
+              number: 443
+  tls:
+    - hosts:
+      - "argocd.gpulive.nhncloud.com"
+      secretName: gpulive-new-ssl
+

template/stg/calico_v3.28.2/custom-resources.yaml

@@ -0,0 +1,29 @@
+# This section includes base Calico installation configuration.
+# For more information, see: https://docs.tigera.io/calico/latest/reference/installation/api#operator.tigera.io/v1.Installation
+apiVersion: operator.tigera.io/v1
+kind: Installation
+metadata:
+  name: default
+spec:
+  # Configures Calico networking.
+  calicoNetwork:
+    ipPools:
+    - name: default-ipv4-ippool
+      blockSize: 24
+      cidr: 10.1.0.0/16
+      encapsulation: VXLAN         # openstack 에서는 VXLAN을 사용해야해안다 byljh
+      natOutgoing: Enabled
+      nodeSelector: all()
+    nodeAddressAutodetectionV4:
+      interface: "enp2s0|bond1"
+
+---
+
+# This section configures the Calico API server.
+# For more information, see: https://docs.tigera.io/calico/latest/reference/installation/api#operator.tigera.io/v1.APIServer
+apiVersion: operator.tigera.io/v1
+kind: APIServer
+metadata:
+  name: default
+spec: {}
+

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/License.md

@@ -0,0 +1,22 @@
+DDN licenses the EXA CSI driver under its license at:  https://www.ddn.com/resources/legal/ddn-end-user-license-agreement/
+
+The DDN license is subject to 3rd party licenses applicable to modules of the exa-csi-driver as listed below. 
+
+ 
+
+Here is list of 3rd party modules<65> licenses:
+
+|Components |License Link|
+|--- |--- |
+|github.com/antonfisher/nested-logrus-formatter|https://github.com/antonfisher/nested-logrus-formatter/blob/master/LICENSE|
+|github.com/container-storage-interface/spec|https://github.com/container-storage-interface/spec/blob/master/LICENSE|
+|github.com/educlos/testrail|https://github.com/educlos/testrail/blob/master/LICENSE|
+|google.golang.org/protobuf|https://pkg.go.dev/google.golang.org/protobuf?tab=licenses|
+|github.com/kubernetes-csi/csi-lib-utils|https://github.com/kubernetes-csi/csi-lib-utils/blob/master/LICENSE|
+|github.com/sirupsen/logrus|https://github.com/sirupsen/logrus/blob/master/LICENSE|
+|golang.org/x/net|https://pkg.go.dev/golang.org/x/net?tab=licenses|
+|google.golang.org/grpc|https://pkg.go.dev/google.golang.org/grpc?tab=licenses|
+|gopkg.in/yaml.v2|https://github.com/go-yaml/yaml/blob/v3/LICENSE|
+|k8s.io/kubernetes|https://github.com/kubernetes/registry.k8s.io/blob/main/LICENSE|
+|k8s.io/utils|https://github.com/kubernetes/registry.k8s.io/blob/main/LICENSE|
+|base image - alpine linux|https://github.com/alpinelinux/alpine-wiki/blob/master/LICENSE|

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/README.md

@@ -0,0 +1,814 @@
+# Exascaler-csi-file-driver
+
+Releases can be found here - https://github.com/DDNStorage/exa-csi-driver/releases
+
+## Compatibility matrix
+|CSI driver version|EXAScaler client version|EXA Scaler server version|
+|--- |---|---|
+|>=v2.3.0|>=2.14.0-ddn182|>=6.3.2|
+
+## Feature List
+|Feature|Feature Status|CSI Driver Version|CSI Spec Version|Kubernetes Version|Openshift Version|
+|--- |--- |--- |--- |--- |--- |
+|Static Provisioning|GA|>= 1.0.0|>= 1.0.0|>=1.18|>=4.13|
+|Dynamic Provisioning|GA|>= 1.0.0|>= 1.0.0|>=1.18|>=4.13|
+|RW mode|GA|>= 1.0.0|>= 1.0.0|>=1.18|>=4.13|
+|RO mode|GA|>= 1.0.0|>= 1.0.0|>=1.18|>=4.13|
+|Expand volume|GA|>= 1.0.0|>= 1.1.0|>=1.18|>=4.13|
+|StorageClass Secrets|GA|>= 1.0.0|>=1.0.0|>=1.18|>=4.13|
+|Mount options|GA|>= 1.0.0|>= 1.0.0|>=1.18|>=4.13|
+|Topology|GA|>= 2.0.0|>= 1.0.0|>=1.17|>=4.13|
+|Snapshots|GA|>= 2.2.6|>= 1.0.0|>=1.17|>=4.13|
+|Exascaler Hot Nodes|GA|>= 2.3.0|>= 1.0.0|>=1.18| Not supported yet|
+
+## Access Modes support
+|Access mode| Supported in version|
+|--- |--- |
+|ReadWriteOnce| >=1.0.0 |
+|ReadOnlyMany| >=2.2.3 |
+|ReadWriteMany| >=1.0.0 |
+|ReadWriteOncePod| >=2.2.3 |
+
+## OpenShift Certification
+|OpenShift Version| CSI driver Version| EXA Version|
+|---|---|---|
+|v4.13|>=v2.2.3|>=v6.3.0|
+|v4.14|>=v2.2.4|>=v6.3.0|
+|v4.15|>=v2.2.4|>=v6.3.0|
+
+## OpenShift
+### Prerequisites
+Internal OpenShift image registry needs to be patched to allow building lustre modules with KMM.
+```bash
+oc patch configs.imageregistry.operator.openshift.io cluster --type merge --patch '{"spec":{"managementState":"Managed"}}'
+oc patch configs.imageregistry.operator.openshift.io cluster --type merge --patch '{"spec":{"storage":{"emptyDir":{}}}}'
+oc patch configs.imageregistry.operator.openshift.io/cluster --type merge -p '{"spec":{"defaultRoute":true}}'
+```
+
+### Building lustre rpms
+You will need a vm with the kernel version matching that of the OpenShift nodes. To check on nodes:
+```bash
+oc get nodes
+oc debug node/c1-pk6k4-worker-0-2j6w8
+uname -r
+```
+
+On the builder vm, install the matching kernel and log in to it, example for Rhel 9.2:
+```bash
+# login to subscription-manager
+subscription-manager register --username <username> --password <password> --auto-attach
+# list available kernels
+yum --showduplicates list available kernel
+yum install kernel-<version>-<revision>.<arch>  # e.g.: yum install kernel-5.14.0-284.25.1.el9_2.x86_64
+grubby --info=ALL | grep title
+title="Red Hat Enterprise Linux (5.14.0-284.11.1.el9_2.x86_64) 9.2 (Plow)"                <---- 0
+title="Red Hat Enterprise Linux (5.14.0-284.25.1.el9_2.x86_64) 9.2 (Plow)"                <---- 1
+
+grub2-set-default 1
+reboot
+```
+
+Copy EXAScaler client tar from the EXAScaler server:
+```bash
+scp root@<exa-server-ip>:/scratch/EXAScaler-<version>/exa-client-<version>.tar.gz .
+tar -xvf exa-client-<version>.tar.gz
+cd exa-client
+./exa_client_deploy.py -i
+```
+
+This will build the rpms and install the client.
+Upload the rpms to any repository available from the cluster and change deploy/openshift/lustre-module/lustre-dockerfile-configmap.yaml lines 12-13 accordingly.
+Make sure that `kmod-lustre-client-*.rpm`, `lustre-client-*.rpm` and `lustre-client-devel-*.rpm` packages are present.
+  ```
+  RUN git clone https://github.com/Qeas/rpms.git # change this to your repo with matching rpms
+  RUN yum -y install rpms/*.rpm
+  ```
+
+### Loading lustre modules in OpenShift
+
+Before loading the lustre modules, make sure to install OpenShift Kernel Module Management (KMM) via OpenShift console.
+
+```bash
+oc create -n openshift-kmm -f deploy/openshift/lustre-module/lustre-dockerfile-configmap.yaml
+oc apply -n openshift-kmm -f deploy/openshift/lustre-module/lnet-mod.yaml
+```
+Wait for the builder pod (e.g. `lnet-build-5f265-build`) to finish. After builder finishes,
+you should have `lnet-8b72w-6fjwh` running on each worker node.
+```bash
+# run ko2iblnd-mod if you are using Infiniband network
+oc apply -n openshift-kmm -f deploy/openshift/lustre-module/ko2iblnd-mod.yaml
+```
+
+Make changes to `deploy/openshift/lustre-module/lnet-configuration-ds.yaml` line 38 according to the cluster's network
+```
+lnetctl net add --net tcp --if br-ex # change interface according to your cluster
+```
+Configure lnet and install lustre
+```
+oc apply -n openshift-kmm -f deploy/openshift/lustre-module/lnet-configuration-ds.yaml
+oc apply -n openshift-kmm -f deploy/openshift/lustre-module/lustre-mod.yaml
+```
+
+### Installing the driver
+
+Make sure that `openshift: true` in `deploy/openshift/exascaler-csi-file-driver-config.yaml`.
+Create a secret from the config file and apply the driver yaml.
+
+```bash
+oc create -n openshift-kmm secret generic exascaler-csi-file-driver-config --from-file=deploy/openshift/exascaler-csi-file-driver-config.yaml
+oc apply -n openshift-kmm -f deploy/openshift/exascaler-csi-file-driver.yaml
+```
+
+### Uninstall
+
+```bash
+oc delete -n openshift-kmm secret exascaler-csi-file-driver-config
+oc delete -n openshift-kmm -f deploy/openshift/exascaler-csi-file-driver.yaml
+oc delete -n openshift-kmm -f deploy/openshift/lustre-module/lustre-mod.yaml
+oc delete -n openshift-kmm -f deploy/openshift/lustre-module/lnet-configuration-ds.yaml
+oc delete -n openshift-kmm -f deploy/openshift/lustre-module/ko2iblnd-mod.yaml
+oc delete -n openshift-kmm -f deploy/openshift/lustre-module/lnet-mod.yaml
+oc delete -n openshift-kmm -f deploy/openshift/lustre-module/lustre-dockerfile-configmap.yaml
+oc get images | grep lustre-client-moduleloader | awk '{print $1}' | xargs oc delete image
+```
+
+
+### Snapshots
+To use CSI snapshots, the snapshot CRDs along with the csi-snapshotter must be installed.
+```bash
+oc apply -f deploy/openshift/snapshots/
+```
+
+After that the snapshot class for EXA CSI must be created
+Snapshot parameters can be passed through snapshot class as can be seen is `examples/snapshot-class.yaml`
+List of available snapshot parameters:
+
+| Name           | Description                                                       | Example                              |
+|----------------|-------------------------------------------------------------------|--------------------------------------|
+| `snapshotFolder` | [Optional] Folder on ExaScaler filesystem where the snapshots will be created. | `csi-snapshots` |
+| `snapshotUtility` | [Optional] Either `tar` or `dtar`. Default is `tar`| `dtar` |
+| `snapshotMd5Verify` | [Optional] Defines whether the driver should do md5sum check on the snapshot. Ensures that the snapshot is not corrupt but reduces performance. Default is `false` | `true` |
+| `exaFS` | Same parameter as in storage class. If the original volume was created using storage class parameter for `exaFS` this MUST match the value of storage class. | `10.3.3.200@tcp:/csi-fs` |
+| `mountPoint` | Same parameter as in storage class. If the original volume was created using storage class parameter for `mountPoint` this MUST match the value of storage class. | `/exa-csi-mnt` |
+
+```bash
+oc apply -f examples/snapshot-class.yaml
+```
+
+Now a snapshot can be created, for example
+```bash
+oc apply -f examples/snapshot-from-dynamic.yaml
+```
+
+We can create a volume using the snapshot
+```bash
+oc apply -f examples/nginx-from-snapshot.yaml
+```
+
+Exascaler CSI driver supports 2 snapshot modes: `tar` or `dtar`.
+Default mode is `tar`. Dtar is much faster.
+To enable `dtar` set `snapshotUtility: dtar` in config.
+
+
+## Kubernetes
+### Requirements
+
+- Required API server and kubelet feature gates for k8s version < 1.16 (skip this step for k8s >= 1.16)
+  ([instructions](https://github.com/kubernetes-csi/docs/blob/735f1ef4adfcb157afce47c64d750b71012c8151/book/src/Setup.md#enabling-features)):
+  ```
+  --feature-gates=ExpandInUsePersistentVolumes=true,ExpandCSIVolumes=true,ExpandPersistentVolumes=true
+  ```
+- Mount propagation must be enabled, the Docker daemon for the cluster must allow shared mounts
+  ([instructions](https://kubernetes.io/docs/concepts/storage/volumes/#mount-propagation))
+
+- MpiFileUtils dtar must be installed on all kubernetes nodes in order to use `dtar` as snapshot utility. Not required if `tar` is used for snapshots (default).
+
+### Prerequisites
+EXAScaler client must be installed and configured on all kubernetes nodes. Please refer to EXAScaler Installation and Administration Guide.
+
+### Installation
+Clone or untar driver (depending on where you get the driver)
+```bash
+git clone -b <driver version> https://github.com/DDNStorage/exa-csi-driver.git /opt/exascaler-csi-file-driver
+```
+e.g:-
+```bash
+git clone -b 2.2.4 https://github.com/DDNStorage/exa-csi-driver.git /opt/exascaler-csi-file-driver
+```
+or
+```bash
+rpm -Uvh exa-csi-driver-1.0-1.el7.x86_64.rpm
+```
+
+### Using helm chart
+
+Pull latest helm chart configuration before installing or upgrading. 
+
+#### Install
+- Make changes to `deploy/helm-chart/values.yaml` according to your Kubernetes and EXAScaler clusters environment.
+
+- If metrics exporter is required, make changes to `deploy/helm-chart/values.yaml` according to your environment. refer to [metrics exporter configuration](#metrics-exporter-configuration)
+
+- Run `helm install -n ${namespace} exascaler-csi-file-driver deploy/helm-chart/`
+
+#### Uninstall
+`helm uninstall -n ${namespace} exascaler-csi-file-driver`
+
+#### Upgrade
+
+- Make any necessary changes to the chart, for example new driver version: `tag: "v2.3.4"` in `deploy/helm-chart/values.yaml`.
+- If metrics exporter is required, make changes to `deploy/helm-chart/values.yaml` according to your environment. refer to [metrics exporter configuration](#metrics-exporter-configuration)
+- Run `helm upgrade -n ${namespace} exascaler-csi-file-driver deploy/helm-chart/`
+
+
+### Metrics exporter configuration.
+
+List of exported metrics:
+- exa_csi_pvc_capacity_bytes - Total capacity in bytes of PVs with provisioner exa.csi.ddn.com
+- exa_csi_pvc_used_bytes - Used bytes of PVs with provisioner exa.csi.ddn.com
+- exa_csi_pvc_available_bytes - Available bytes of PVs with provisioner exa.csi.ddn.com
+- exa_csi_pvc_pod_count - Number of pods using a PVC with provisioner exa.csi.ddn.com
+
+Each of those metrics reports with 3 labels: "pvc", "storage_class", "exported_namespace".
+These labels can be used to group the metrics by kubernetes storage class and namespace as shown below in [Add Prometheus Rules for StorageClass and Namespace Aggregation](#add-prometheus-rules-for-storageclass-and-namespace-aggregation)
+
+```yaml
+metrics:
+  enabled: true      # Enable metrics exporter 
+  exporter:
+    repository: quay.io/ddn/exa-csi-metrics-exporter
+    tag: master # use latest version same as driver version e.g - tag: "2.3.4"
+    pullPolicy: Always
+    containerPort: "9200" # Metrics Exporter port
+    servicePort: "9200" # This port will be used to create service for exporter
+    namespace: default # Namespace where metrics exporter will be deployed
+    logLevel: info     # Log level. Debug is not recommended for production, default is info.
+    collectTimeout: 10 # Metrics refresh timeout in seconds. Default is 10.
+  serviceMonitor:
+    enabled: false # set to true if using prometheus operator
+    interval: 30s
+    namespace: monitoring # Namespace where prometheus operator is deployed
+  prometheus:                                                 
+    releaseLabel: prometheus      # release label for prometheus operator                 
+    createClusterRole: false   # set to false if using existing ClusterRole
+    createClustrerRoleName: exa-prometheus-cluster-role  # Name of ClusterRole to create, this name will be used to bind cluster role to prometheus service account
+    clusterRoleName: cluster-admin  # Name of ClusterRole to bind to prometheus service account if createClusterRole is set to false otherwise `createClustrerRoleName` will be used
+    serviceAccountName: prometheus-kube-prometheus-prometheus # Service account name of prometheus
+
+```
+
+## Prometheus Server Integration
+
+This section describes how to:
+- Add the EXAScaler CSI metrics exporter to Prometheus' scrape configuration.
+- Apply PrometheusRule definitions for metrics aggregation by StorageClass and Namespace.
+
+### 1. Add EXAScaler CSI Metrics Exporter to Prometheus
+
+#### Step 1.1 – Create a Secret with Scrape Config
+
+Get the hostPort number of the EXAScaler CSI metrics exporter:
+```bash
+kubectl get daemonset exa-csi-metrics-exporter -o jsonpath='{.spec.template.spec.containers[*].ports[*].hostPort}{"\n"}'
+
+32666
+```
+
+Create a Kubernetes secret that holds your additional Prometheus scrape configuration:
+```bash
+PORT=32666
+cat <<EOF | envsubst > additional-scrape-configs.yaml
+- job_name: 'exa-csi-metrics-exporter-remote'
+  static_configs:
+    - targets:
+        - 10.20.30.1:$PORT
+        - 10.20.30.2:$PORT
+        - 10.20.30.3:$PORT
+EOF
+kubectl create secret generic additional-scrape-configs \
+  --from-file=additional-scrape-configs.yaml -n monitoring
+```
+Note: Replace the targets value with the actual IP and port of your EXAScaler CSI metrics exporter.
+
+#### Step 1.2 – Patch the Prometheus Custom Resource
+
+Edit the Prometheus CR (custom resource) and add the additionalScrapeConfigs reference:
+```bash
+kubectl edit prometheus prometheus-kube-prometheus-prometheus -n monitoring
+```
+Add the following under spec:
+```yaml
+  additionalScrapeConfigs:
+    name: additional-scrape-configs
+    key: additional-scrape-configs.yaml
+```
+
+#### Step 1.3 – Restart Prometheus
+
+After editing the CR, restart Prometheus so it reloads the configuration:
+```bash
+kubectl delete pod -l app.kubernetes.io/name=prometheus -n monitoring
+```
+
+### 2. Add Prometheus Rules for StorageClass and Namespace Aggregation
+
+Apply a PrometheusRule resource to aggregate PVC metrics by StorageClass and Namespace:
+```yaml
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: exa-csi-rules
+  namespace: monitoring
+  labels:
+    release: prometheus
+spec:
+  groups:
+    - name: exa-storageclass-rules
+      interval: 15s
+      rules:
+        - record: exa_csi_sc_capacity_bytes
+          expr: sum(exa_csi_pvc_capacity_bytes) by (storage_class)
+        - record: exa_csi_sc_used_bytes
+          expr: sum(exa_csi_pvc_used_bytes) by (storage_class)
+        - record: exa_csi_sc_available_bytes
+          expr: sum(exa_csi_pvc_available_bytes) by (storage_class)
+        - record: exa_csi_sc_pvc_count
+          expr: count(exa_csi_pvc_capacity_bytes) by (storage_class)
+
+    - name: exa-namespace-rules
+      interval: 15s
+      rules:
+        - record: exa_csi_namespace_capacity_bytes
+          expr: sum(exa_csi_pvc_capacity_bytes) by (exported_namespace)
+        - record: exa_csi_namespace_used_bytes
+          expr: sum(exa_csi_pvc_used_bytes) by (exported_namespace)
+        - record: exa_csi_namespace_available_bytes
+          expr: sum(exa_csi_pvc_available_bytes) by (exported_namespace)
+        - record: exa_csi_namespace_pvc_count
+          expr: count(exa_csi_pvc_capacity_bytes) by (exported_namespace)
+```
+Apply the rule:
+```bash
+kubectl apply -f exa-csi-rules.yaml
+```
+Ensure that Prometheus is watching the monitoring namespace for PrometheusRule CRDs.
+
+
+
+### Using docker load and kubectl commands
+   ```bash
+   docker load -i /opt/exascaler-csi-file-driver/bin/exascaler-csi-file-driver.tar
+   ```
+#### Verify version
+kubectl get deploy/exascaler-csi-controller -o jsonpath="{..image}"
+
+2. Copy /opt/exascaler-csi-file-driver/deploy/kubernetes/exascaler-csi-file-driver-config.yaml to /etc/exascaler-csi-file-driver-v1.0/exascaler-csi-file-driver-config.yaml
+   ```
+   cp /opt/exascaler-csi-file-driver/deploy/kubernetes/exascaler-csi-file-driver-config.yaml /etc/exascaler-csi-file-driver-v1.0/exascaler-csi-file-driver-config.yaml 
+   ```
+Edit `/etc/exascaler-csi-file-driver-v1.0/exascaler-csi-file-driver-config.yaml` file. Driver configuration example:
+   ```yaml
+   exascaler_map:
+     exa1:
+       mountPoint: /exaFS                                          # mountpoint on the host where the exaFS will be mounted
+       exaFS: 192.168.88.114@tcp2:192.168.98.114@tcp2:/testfs                            # default path to exa filesystem where the PVCs will be stored
+       managementIp: 10.204.86.114@tcp                           # network for management operations, such as create/delete volume
+       zone: zone-1
+
+     exa2:
+       mountPoint: /exaFS-zone-2                                          # mountpoint on the host where the exaFS will be mounted
+       exaFS: 192.168.78.112@tcp2:/testfs/zone-2                            # default path to exa filesystem where the PVCs will be stored
+       managementIp: 10.204.86.114@tcp                           # network for management operations, such as create/delete volume
+       zone: zone-2
+
+     exa3:
+       mountPoint: /exaFS-zone-3                                          # mountpoint on the host where the exaFS will be mounted
+       exaFS: 192.168.98.113@tcp2:192.168.88.113@tcp2:/testfs/zone-3                            # default path to exa filesystem where the PVCs will be stored
+       managementIp: 10.204.86.114@tcp                           # network for management operations, such as create/delete volume
+       zone: zone-3
+
+   debug: true
+   ```
+
+3. Create Kubernetes secret from the file:
+   ```bash
+   kubectl create secret generic exascaler-csi-file-driver-config --from-file=/etc/exascaler-csi-file-driver-v1.0/exascaler-csi-file-driver-config.yaml
+   ```
+
+4. Register driver to Kubernetes:
+   ```bash
+   kubectl apply -f /opt/exascaler-csi-file-driver/deploy/kubernetes/exascaler-csi-file-driver.yaml
+   ```
+
+## Usage
+
+### Dynamically provisioned volumes
+
+For dynamic volume provisioning, the administrator needs to set up a _StorageClass_ in the PV yaml (/opt/exascaler-csi-file-driver/examples/exa-dynamic-nginx.yaml for this example) pointing to the driver.
+For dynamically provisioned volumes, Kubernetes generates volume name automatically (for example `pvc-ns-cfc67950-fe3c-11e8-a3ca-005056b857f8-projectId-1001`).
+
+Basic storage class example, uses config to access Exascaler
+```yaml
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: exascaler-csi-file-driver-sc-nginx-dynamic
+provisioner: exa.csi.ddn.com
+configName: exa1
+```
+
+The following example shows how to use storage class to override config values
+```yaml
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: exascaler-csi-file-driver-sc-nginx-dynamic
+provisioner: exa.csi.ddn.com
+allowedTopologies:
+- matchLabelExpressions:
+  - key: topology.exa.csi.ddn.com/zone
+    values:
+    - zone-1
+mountOptions:                        # list of options for `mount -o ...` command
+#  - noatime                         #
+parameters:
+  exaMountUid: "1001"      # Uid which will be used to access the volume in pod. Should be synced between EXA server and clients.
+  exaMountGid: "1002"      # Gid which will be used to access the volume in pod. Should be synced between EXA server and clients.
+  bindMount: "true"       # Determines, whether volume will bind mounted or as a separate lustre mount.
+  exaFS: "10.204.86.114@tcp:/testfs"   # Overrides exaFS value from config. Use this to support multiple EXA filesystems.
+  mountPoint: /exaFS       # Overrides mountPoint value from config. Use this to support multiple EXA filesystems.
+  mountOptions: ro,noflock
+  minProjectId: 10001      # project id range for this storage class
+  maxProjectId: 20000
+```
+
+#### Example
+
+Run Nginx pod with dynamically provisioned volume:
+
+```bash
+kubectl apply -f /opt/exascaler-csi-file-driver/examples/exa-dynamic-nginx.yaml
+
+# to delete this pod:
+kubectl delete -f /opt/exascaler-csi-file-driver/examples/exa-dynamic-nginx.yaml
+```
+
+### Static (pre-provisioned) volumes
+
+The driver can use already existing Exasaler filesystem,
+in this case, _StorageClass_, _PersistentVolume_ and _PersistentVolumeClaim_ should be configured.
+
+Quota can be manually assigned for static volumes. First we need to associate a project with the volume folder
+```bash
+lfs project -p 1000001 -s /mountPoint-csi/nginx-persistent
+```
+
+Then a quota can be set for that project
+```bash
+lfs setquota -p 1000001 -B 2G /mountPoint-csi/
+```
+
+#### _StorageClass_ configuration
+
+```yaml
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: exascaler-csi-driver-sc-nginx-persistent
+provisioner:  exa.csi.ddn.com
+allowedTopologies:
+- matchLabelExpressions:
+  - key: topology.exa.csi.ddn.com/zone
+    values:
+    - zone-1
+mountOptions:                         # list of options for `mount -o ...` command
+#  - noatime                         #
+```
+
+#### _PersistentVolume_ configuration
+
+```yaml
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: exascaler-csi-driver-pv-nginx-persistent
+  labels:
+    name: exascaler-csi-driver-pv-nginx-persistent
+spec:
+  storageClassName: exascaler-csi-driver-sc-nginx-persistent
+  accessModes:
+    - ReadWriteMany
+  capacity:
+    storage: 1Gi
+  csi:
+    driver: exa.csi.ddn.com
+    volumeHandle: exa1:10.3.3.200@tcp;/exaFS:/mountPoint-csi:/nginx-persistent
+    volumeAttributes:         # volumeAttributes are the alternative of storageClass params for static (precreated) volumes.
+      exaMountUid: "1001"     # Uid which will be used to access the volume in pod.
+      #mountOptions: ro, flock # list of options for `mount` command
+      projectId: "1000001"
+```
+
+### Topology configuration
+In order to configure CSI driver with kubernetes topology, use the `zone` parameter in driver config or storageClass parameters. Example config file with zones:
+  ```bash
+
+  exascaler_map:
+    exa1:
+      exaFS: 10.3.196.24@tcp:/csi
+      mountPoint: /exaFS
+      zone: us-west
+    exa2:
+      exaFS: 10.3.196.24@tcp:/csi-2
+      mountPoint: /exaFS-zone2
+      zone: us-east
+  ```
+
+This will assign volumes to be created on Exascaler cluster that correspond with the zones requested by allowedTopologies values.
+
+For topology-aware scheduling:
+
+Each node must have the `topology.exa.csi.ddn.com/zone` label with the zone configuration.
+
+Label node with topology zone:
+```bash
+kubectl label node <node-name> topology.exa.csi.ddn.com/zone=us-west
+```
+For removing label:
+```bash
+kubectl label node <node-name> topology.exa.csi.ddn.com/zone-
+```
+
+Important: If topology labels are added after driver installation, the node driver must be restarted on affected nodes.
+
+Restart node driver pod on specific node:
+```bash
+kubectl delete pod -l app=exascaler-csi-node -n <namespace> --field-selector spec.nodeName=<node-name>
+```
+
+or restart all node driver pods:
+```bash
+kubectl rollout restart daemonset exascaler-csi-node -n <namespace>
+```
+
+For using `volumeBindingMode: WaitForFirstConsumer`, the topology must be configured.
+
+```yaml
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: exascaler-csi-file-driver-sc-nginx-dynamic
+provisioner: exa.csi.ddn.com
+volumeBindingMode: WaitForFirstConsumer
+parameters:
+  configName: exa1
+```
+
+### CSI Parameters
+If a parameter is available for both config and storage class, storage class parameter will override config values
+
+|Config|Storage class| Description                                                       | Example                              |
+|--------|------------|------------------------------------------------------------------|--------------------------------------|
+| `exaFS` | `exaFS` | [required] Full path to EXAScaler filesystem | `10.3.3.200@tcp:/csi-fs` |
+| `mountPoint`  |`mountPoint` | [required] Mountpoint on Kubernetes host where the exaFS will be mounted | `/exa-csi-mnt` |
+| - | `driver`    [required]  |  Installed driver name " exa.csi.ddn.com"        | `exa.csi.ddn.com` |
+| - | `volumeHandle` | [required for static volumes] The format is &lt;configName&gt;:&lt;NID&gt;;&lt;Exascaler filesystem path&gt;:&lt;mountPoint&gt;:&lt;volumeHandle&gt;. **Note:** The NID and Exascaler filesystem path are separated by a semicolon ( ; ), all other fields are delimited by a colon ( : ). | `exa1:10.3.3.200@tcp;/exaFS:/mountPoint-csi:/nginx-persistent`      |
+| - | `exaMountUid`  | Uid which will be used to access the volume from the pod. | `1015` |
+| - | `exaMountGid`  | Gid which will be used to access the volume from the pod. | `1015` |
+| - | `projectId`    | Points to EXA project id to be used to set volume quota. Automatically generated by the driver if not provided. | `100001` |
+| `managementIp` | `managementIp` | Should be used if there is a separate network configured for management operations, such as create/delete volumes. This network should have access to all Exa filesystems in a isolated zones environment | `192.168.10.20@tcp2` |
+| `bindMount` | `bindMount`    | Determines, whether volume will bind mounted or as a separate lustre mount. Default is `true` | `true` |
+| `defaultMountOptions` | `mountOptions` | Options that will be passed to mount command (-o <opt1,opt2,opt3>)  | `ro,flock` |
+| - | `minProjectId` | Minimum project ID number for automatic generation. Only used when projectId is not provided. | 10000 |
+| - | `maxProjectId` | Maximum project ID number for automatic generation. Only used when projectId is not provided. | 4294967295 |
+| - | `generateProjectIdRetries` | Maximum retry count for generating random project ID. Only used when projectId is not provided. | `5` |
+| `zone` | `zone`        | Topology zone to control where the volume should be created. Should match topology.exa.csi.ddn.com/zone label on node(s). | `us-west` |
+| `v1xCompatible` | - | [Optional] Only used when upgrading the driver from v1.x.x to v2.x.x. Provides compatibility for volumes that were created beore the upgrade. Set it to `true` to point to the Exa cluster that was configured before the upgrade | `false` |
+|`tempMountPoint` | `tempMountPoint` | [Optional] Used when `exaFS` points to a subdirectory that does not exist on Exascaler and will be automatically created by the driver. This parameter sets the directory where Exascaler filesystem will be temporarily mounted to create the subdirectory. | `/tmp/exafs-mnt` |
+| `volumeDirPermissions` | `volumeDirPermissions` | [Optional] Defines file permissions for mounted volumes. | `0777` |
+| `hotNodes` | `hotNodes` | Determines whether `HotNodes` feature should be used. This feature can only be used by the driver when Hot Nodes (PCC) service is disabled and not used manually on the kubernetes workers. | `false` |
+| `pccCache` | `pccCache` | Directory for cached files of the file system. Note that lpcc does not recognize directories with a
+trailing slash (“/” at the end). | `/csi-pcc` |
+| `pccAutocache` | `pccAutocache` | Condition for automatic file attachment (caching) | `projid={500}` |
+| `pccPurgeHighUsage` | `pccPurgeHighUsage` | If the disk usage of cache device is higher than high_usage, start detaching the files. Defaults to 90 (90% disk/inode usage). | `90` |
+| `pccPurgeLowUsage` | `pccPurgeLowUsage` | If the disk usage of cache device is lower than low_usage, stop detaching the files. Defaults to 75 (75% disk/inode usage). | `70` |
+| `pccPurgeScanThreads` | `pccPurgeScanThreads` | Threads to use for scanning cache device in parallel. Defaults to 1. | `1` |
+| `pccPurgeInterval` | `pccPurgeInterval` | Interval for lpcc_purge to check cache device usage, in seconds. Defaults to 30. | `30` |
+| `pccPurgeLogLevel` | `pccPurgeLogLevel` | Log level for lpcc_purge: either “fatal”, “error”, “warn”, “normal”, “info” (default), or “debug”. | `info` |
+| `pccPurgeForceScanInterval` | `pccPurgeForceScanInterval` | Scan PCC backends forcefully after this number of seconds to refresh statistic data. | `30` |
+| - | `compression` | Algorithm ["lz4", "gzip", "lzo"] to use for data compression. default is "false" | `false` | 
+| - | `configName` | Config entry name to use from the config map | `exa1` |
+
+#### _PersistentVolumeClaim_ (pointing to created _PersistentVolume_)
+
+```yaml
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: exascaler-csi-driver-pvc-nginx-persistent
+spec:
+  storageClassName: exascaler-csi-driver-cs-nginx-persistent
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 1Gi
+  selector:
+    matchLabels:
+      # to create 1-1 relationship for pod - persistent volume use unique labels
+      name: exascaler-csi-file-driver-pv-nginx-persistent
+```
+
+#### Example
+
+Run nginx server using PersistentVolume.
+
+**Note:** Pre-configured filesystem should exist on the EXAScaler:
+`/exaFS/nginx-persistent`.
+
+```bash
+kubectl apply -f /opt/exascaler-csi-file-driver/examples/nginx-persistent-volume.yaml
+
+# to delete this pod:
+kubectl delete -f /opt/exascaler-csi-file-driver/examples/nginx-persistent-volume.yaml
+```
+
+### Snapshots
+To use CSI snapshots, the snapshot CRDs along with the csi-snapshotter must be installed.
+```bash
+kubectl apply -f deploy/kubernetes/snapshots/
+```
+
+After that the snapshot class for EXA CSI must be created
+Snapshot parameters can be passed through snapshot class as can be seen is `examples/snapshot-class.yaml`
+List of available snapshot parameters:
+
+| Name           | Description                                                       | Example                              |
+|----------------|-------------------------------------------------------------------|--------------------------------------|
+| `snapshotFolder` | [Optional] Folder on ExaScaler filesystem where the snapshots will be created. | `csi-snapshots` |
+| `snapshotUtility` | [Optional] Either `tar` or `dtar`.  `dtar` is faster but requires `dtar` to be installed on all k8s nodes. Default is `tar`| `dtar` |
+| `dtarPath` | [Optional] If `snapshotUtility` is `dtar` points to where the `dtar` utility is installed | `/opt/ddn/mpifileutils/bin/dtar` |
+| `snapshotMd5Verify` | [Optional] Defines whether the driver should do md5sum check on the snapshot. Ensures that the snapshot is not corrupt but reduces performance. Default is `false` | `true` |
+| `exaFS` | Same parameter as in storage class. If the original volume was created using storage class parameter for `exaFS` this MUST match the value of storage class. | `10.3.3.200@tcp:/csi-fs` |
+| `mountPoint` | Same parameter as in storage class. If the original volume was created using storage class parameter for `mountPoint` this MUST match the value of storage class. | `/exa-csi-mnt` |
+
+```bash
+kubectl apply -f examples/snapshot-class.yaml
+```
+
+Now a snapshot can be created, for example
+```bash
+kubectl apply -f examples/snapshot-from-dynamic.yaml
+```
+
+We can create a volume using the snapshot
+```bash
+kubectl apply -f examples/nginx-from-snapshot.yaml
+```
+
+Exascaler CSI driver supports 2 snapshot modes: `tar` or `dtar`.
+Default mode is `tar`.
+To enable `dtar` set `snapshotUtility: dtar` in config.
+Dtar is much faster but requires mpifileutils `dtar` installed on all the nodes as a prerequisite.
+Installing `dtar` might differ depending on your OS. Here is an example for Ubuntu 22
+
+```bash
+# Install dependencies mpifileutils
+cd
+mkdir -p /opt/ddn/mpifileutils
+cd /opt/ddn/mpifileutils
+sudo apt-get install -y cmake libarchive-dev libbz2-dev libcap-dev libssl-dev openmpi-bin libopenmpi-dev libattr1-dev
+
+export INSTALL_DIR=/opt/ddn/mpifileutils
+
+git clone https://github.com/LLNL/lwgrp.git
+cd lwgrp
+./autogen.sh
+./configure --prefix=$INSTALL_DIR
+make -j 16
+make install
+cd ..
+
+git clone https://github.com/LLNL/dtcmp.git
+cd dtcmp
+./autogen.sh
+./configure --prefix=$INSTALL_DIR --with-lwgrp=$INSTALL_DIR
+make -j 16
+make install
+cd ..
+
+git clone https://github.com/hpc/libcircle.git
+cd libcircle
+sh ./autogen.sh
+./configure --prefix=$INSTALL_DIR
+make -j 16
+make install
+cd ..
+
+git clone https://github.com/hpc/mpifileutils.git mpifileutils-clone
+cd mpifileutils-clone
+cmake ./ -DWITH_DTCMP_PREFIX=$INSTALL_DIR -DWITH_LibCircle_PREFIX=$INSTALL_DIR -DCMAKE_INSTALL_PREFIX=$INSTALL_DIR -DENABLE_LUSTRE=ON -DENABLE_XATTRS=ON
+make -j 16
+make install
+cd ..
+```
+
+If you use a different `INSTALL_DIR` path, pass it in config using `DtarPath` parameter.
+
+
+## Updating the driver version
+To update to a new driver version, you need to follow the following steps:
+
+1. Remove the old driver version
+```bash
+kubectl delete -f /opt/exascaler-csi-file-driver/deploy/kubernetes/exascaler-csi-file-driver.yaml
+kubectl delete secrets exascaler-csi-file-driver-config
+rpm -evh exa-csi-driver
+```
+2. Download the new driver version (git clone or new ISO)
+3. Copy and edit config file
+  ```bash
+  cp /opt/exascaler-csi-file-driver/deploy/kubernetes/exascaler-csi-file-driver-config.yaml /etc/exascaler-csi-file-driver-v1.1/exascaler-csi-file-driver-config.yaml
+  ```
+
+  If you are upgrading from v1.x.x to v2.x.x, config file structure will change to a map of Exascaler clusters instead of a flat structure. To support old volume that were created using v1.x.x, old config should be put in the config map with `v1xCompatible: true`. For example:
+
+  v1.x.x config
+  ```bash
+  exaFS: 10.3.196.24@tcp:/csi
+  mountPoint: /exaFS
+  debug: true
+  ```
+  v2.x.x config with support of previously created volumes
+  ```bash
+
+  exascaler_map:
+    exa1:
+      exaFS: 10.3.196.24@tcp:10.3.199.24@tcp:/csi
+      mountPoint: /exaFS
+      v1xCompatible: true
+
+    exa2:
+      exaFS: 10.3.1.200@tcp:10.3.2.200@tcp:10.3.3.200@tcp:/csi-fs
+      mountPoint: /mnt2
+
+  debug: true
+```
+
+Only one of the Exascaler clusters can have `v1xCompatible: true` since old config supported only 1 cluster.
+
+4. Update version in /opt/exascaler-csi-file-driver/deploy/kubernetes/exascaler-csi-file-driver.yaml
+```
+          image: exascaler-csi-file-driver:v2.2.5
+```
+5. Load new image
+```bash
+docker load -i /opt/exascaler-csi-file-driver/bin/exascaler-csi-file-driver.tar
+```
+6. Apply new driver
+```bash
+kubectl create secret generic exascaler-csi-file-driver-config --from-file=/etc/exascaler-csi-file-driver-v1.1/exascaler-csi-file-driver-config.yaml
+kubectl apply -f /opt/exascaler-csi-file-driver/deploy/kubernetes/exascaler-csi-file-driver.yaml
+```
+7. Verify version
+```bash
+kubectl get deploy/exascaler-csi-controller -o jsonpath="{..image}"
+```
+
+## Troubleshooting
+### Driver logs
+To collect all driver related logs, you can use the `kubectl logs` command.
+All in on command:
+```bash
+mkdir exa-csi-logs
+for name in $(kubectl get pod -owide | grep exascaler | awk '{print $1}'); do kubectl logs $name --all-containers > exa-csi-logs/$name; done
+```
+
+To get logs from all containers of a single pod 
+```bash
+kubectl logs <pod_name> --all-containers
+```
+
+Logs from a single container of a pod
+```bash
+kubectl logs <pod_name> -c driver
+```
+
+#### Driver secret
+```bash
+kubectl get secret exascaler-csi-file-driver-config -o json | jq '.data | map_values(@base64d)' > exa-csi-logs/exascaler-csi-file-driver-config
+```
+
+#### PV/PVC/Pod data
+```bash
+kubectl get pvc > exa-csi-logs/pvcs
+kubectl get pv > exa-csi-logs/pvs
+kubectl get pod > exa-csi-logs/pods
+```
+
+#### Extended info about a PV/PVC/Pod:
+```bash
+kubectl describe pvc <pvc_name> > exa-csi-logs/<pvc_name>
+kubectl describe pv <pv_name> > exa-csi-logs/<pv_name>
+kubectl describe pod <pod_name> > exa-csi-logs/<pod_name>
+```

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/helm-chart/Chart.yaml

@@ -0,0 +1,24 @@
+apiVersion: v2
+name: exa-csi
+description: A Helm chart for ExaScaler CSI file driver.
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+# Versions are expected to follow Semantic Versioning (https://semver.org/)
+version: 2.3.5
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application. Versions are not expected to
+# follow Semantic Versioning. They should reflect the version the application is using.
+# It is recommended to use it with quotes.
+appVersion: 2.3.5

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/helm-chart/templates/config.yaml

@@ -0,0 +1,9 @@
+# This secret is used to set the initial credentials of the node container.
+apiVersion: v1
+kind: Secret
+metadata:
+  name: {{ .Values.secretName }}
+  namespace: {{ .Release.Namespace }}
+type: "Opaque"
+data:
+  exascaler-csi-file-driver-config.yaml: {{ .Values.config | toYaml | b64enc }}

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/helm-chart/templates/controller-driver.yaml

@@ -0,0 +1,349 @@
+# ----------------------
+# Exascaler CSI Driver
+# ----------------------
+
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: exa.csi.ddn.com
+spec:
+  attachRequired: false
+  podInfoOnMount: false
+  fsGroupPolicy: File
+---
+
+# ---------------------------------
+# Exascaler CSI Controller Server
+# ---------------------------------
+#
+# Runs single driver controller server (driver + provisioner + attacher + snapshotter) on one of the nodes
+#
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: exascaler-csi-controller-service-account
+  namespace: {{ .Release.Namespace }}
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: exascaler-csi-controller-cluster-role
+rules:
+  - apiGroups: ['']
+    resources: ['secrets']
+    verbs: ['get', 'list', "watch"]
+  - apiGroups: ['']
+    resources: ['persistentvolumes']
+    verbs: ['get', 'list', 'watch', 'create', 'update', 'delete'] # "update" for attacher
+  - apiGroups: ['']
+    resources: ['persistentvolumeclaims']
+    verbs: ['get', 'list', 'watch', 'update']
+  - apiGroups: ['storage.k8s.io']
+    resources: ['storageclasses']
+    verbs: ['get', 'list', 'watch']
+  - apiGroups: ['']
+    resources: ['events']
+    verbs: ['list', 'watch', 'create', 'update', 'patch']
+  - apiGroups: ['snapshot.storage.k8s.io']
+    resources: ['volumesnapshots']
+    verbs: ['get', 'list']
+  - apiGroups: ['snapshot.storage.k8s.io']
+    resources: ['volumesnapshotcontents']
+    verbs: ['get', 'list']
+  # attacher specific
+  - apiGroups: ['']
+    resources: ['nodes', 'pods']
+    verbs: ['get', 'list', 'watch']
+  - apiGroups: ['csi.storage.k8s.io']
+    resources: ['csinodeinfos']
+    verbs: ['get', 'list', 'watch']
+  - apiGroups: ['storage.k8s.io']
+    resources: ['volumeattachments']
+    verbs: ['get', 'list', 'watch', 'update']
+  # snapshotter specific
+  - apiGroups: ['snapshot.storage.k8s.io']
+    resources: ['volumesnapshotclasses']
+    verbs: ['get', 'list', 'watch']
+  - apiGroups: ['snapshot.storage.k8s.io']
+    resources: ['volumesnapshotcontents']
+    verbs: ['create', 'get', 'list', 'watch', 'update', 'delete', 'patch']
+  - apiGroups: ['snapshot.storage.k8s.io']
+    resources: ['volumesnapshots']
+    verbs: ['get', 'list', 'watch', 'update']
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["update"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
+  - apiGroups: ['apiextensions.k8s.io']
+    resources: ['customresourcedefinitions']
+    verbs: ['create', 'list', 'watch', 'delete']
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]
+  # CSINode specific
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["watch", "list", "get"]
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: exascaler-csi-controller-cluster-role-binding
+subjects:
+  - kind: ServiceAccount
+    name: exascaler-csi-controller-service-account
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: exascaler-csi-controller-cluster-role
+  apiGroup: rbac.authorization.k8s.io
+---
+
+# External Resizer
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-resizer-role
+rules:
+  # The following rule should be uncommented for plugins that require secrets
+  # for provisioning.
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-resizer-binding
+subjects:
+  - kind: ServiceAccount
+    name: exascaler-csi-controller-service-account
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: csi-resizer-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: {{ .Release.Namespace }}
+  name: external-resizer-cfg
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-resizer-role-cfg
+  namespace: {{ .Release.Namespace }}
+subjects:
+  - kind: ServiceAccount
+    name: exascaler-csi-controller-service-account
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: Role
+  name: external-resizer-cfg
+  apiGroup: rbac.authorization.k8s.io
+---
+
+kind: Service
+apiVersion: v1
+metadata:
+  name: exascaler-csi-controller-service
+  labels:
+    app: exascaler-csi-controller
+spec:
+  selector:
+    app: exascaler-csi-controller
+  ports:
+    - name: dummy
+      port: 12345
+---
+
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: exascaler-csi-controller
+  namespace: {{ .Release.Namespace }}
+spec:
+  # serviceName: exascaler-csi-controller-service
+  selector:
+    matchLabels:
+      app: exascaler-csi-controller # has to match .spec.template.metadata.labels
+  template:
+    metadata:
+      labels:
+        app: exascaler-csi-controller
+    spec:
+      serviceAccount: exascaler-csi-controller-service-account
+      priorityClassName: {{ .Values.priorityClassName }}
+      containers:
+        # csi-provisioner: sidecar container that watches Kubernetes PersistentVolumeClaim objects
+        # and triggers CreateVolume/DeleteVolume against a CSI endpoint
+        - name: csi-provisioner
+          resources: {{ .Values.resources | default .Values.provisioner.resources | toYaml | nindent 12 }}
+          image: {{ .Values.provisioner.repository }}:{{ .Values.provisioner.tag }}
+          imagePullPolicy: {{ .Values.provisioner.pullPolicy }}
+          args:
+            - --csi-address=/var/lib/csi/sockets/pluginproxy/csi.sock
+            - --volume-name-prefix={{ .Values.volumeNamePrefix }}
+            - --strict-topology
+            - --immediate-topology=false
+            - --feature-gates=Topology=true
+            - --timeout={{ .Values.provisioner.timeout }}
+          env:
+            - name: GOMEMLIMIT
+              valueFrom:
+                resourceFieldRef:
+                  divisor: "0"
+                  resource: limits.memory
+            - name: GOMAXPROCS
+              valueFrom:
+                resourceFieldRef:
+                  divisor: "0"
+                  resource: limits.cpu
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy
+        - name: csi-attacher
+          resources: {{ .Values.resources | default .Values.attacher.resources | toYaml | nindent 12 }}
+          image: {{ .Values.attacher.repository }}:{{ .Values.attacher.tag }}
+          imagePullPolicy: {{ .Values.attacher.pullPolicy }}
+          args:
+            - --csi-address=$(ADDRESS)
+            - --v=2
+            - --leader-election=true
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: GOMEMLIMIT
+              valueFrom:
+                resourceFieldRef:
+                  divisor: "0"
+                  resource: limits.memory
+            - name: GOMAXPROCS
+              valueFrom:
+                resourceFieldRef:
+                  divisor: "0"
+                  resource: limits.cpu
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-snapshotter
+          resources: {{ .Values.resources | default .Values.csi_snapshotter.resources | toYaml | nindent 12 }}
+          image: {{ .Values.csi_snapshotter.repository }}:{{ .Values.csi_snapshotter.tag }}
+          imagePullPolicy: {{ .Values.csi_snapshotter.pullPolicy }}
+          args:
+            - -v=3
+            - --csi-address=/var/lib/csi/sockets/pluginproxy/csi.sock
+            - --timeout={{ .Values.csi_snapshotter.timeout }}
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy
+          env:
+            - name: GOMEMLIMIT
+              valueFrom:
+                resourceFieldRef:
+                  divisor: "0"
+                  resource: limits.memory
+            - name: GOMAXPROCS
+              valueFrom:
+                resourceFieldRef:
+                  divisor: "0"
+                  resource: limits.cpu
+        - name: csi-resizer
+          resources: {{ .Values.resources | default .Values.resizer.resources | toYaml | nindent 12 }}
+          image: {{ .Values.resizer.repository }}:{{ .Values.resizer.tag }}
+          imagePullPolicy: {{ .Values.resizer.pullPolicy }}
+          args:
+            - "--csi-address=$(ADDRESS)"
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+            - name: GOMEMLIMIT
+              valueFrom:
+                resourceFieldRef:
+                  divisor: "0"
+                  resource: limits.memory
+            - name: GOMAXPROCS
+              valueFrom:
+                resourceFieldRef:
+                  divisor: "0"
+                  resource: limits.cpu
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: driver
+          resources: {{ .Values.resources | default .Values.image.resources | toYaml | nindent 12 }}
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ['SYS_ADMIN']
+            allowPrivilegeEscalation: true
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: "{{ .Values.image.pullPolicy }}"
+          args:
+            - --nodeid=$(KUBE_NODE_NAME)
+            - --endpoint=unix://csi/csi.sock
+            - --role=controller
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: GOMEMLIMIT
+              valueFrom:
+                resourceFieldRef:
+                  divisor: "0"
+                  resource: limits.memory
+            - name: GOMAXPROCS
+              valueFrom:
+                resourceFieldRef:
+                  divisor: "0"
+                  resource: limits.cpu
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: secret
+              mountPath: /config
+              readOnly: true
+            - name: host
+              mountPath: /host
+              mountPropagation: Bidirectional
+      volumes:
+        - name: socket-dir
+          emptyDir:
+        - name: secret
+          secret:
+            secretName: {{ .Values.secretName }}
+        - name: host
+          hostPath:
+            path: /
+            type: Directory
+---

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/helm-chart/templates/metrics/crb.yaml

@@ -0,0 +1,70 @@
+{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }}
+
+{{- if .Values.metrics.prometheus.createClusterRole }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ .Values.metrics.prometheus.createClustrerRoleName }}
+rules:
+- apiGroups: [""]
+  resources:
+  - pods
+  - pods/metrics
+  - pods/status
+  - services
+  - endpoints
+  - nodes
+  - nodes/metrics
+  - nodes/proxy
+  - nodes/stats
+  - namespaces
+  - configmaps
+  - secrets
+  - serviceaccounts
+  - events
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["apps"]
+  resources:
+  - deployments
+  - daemonsets
+  - replicasets
+  - statefulsets
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["monitoring.coreos.com"]
+  resources: 
+  - servicemonitors
+  - prometheusrules
+  - podmonitors
+  - alerts
+  - alertmanagers
+  - prometheuses
+  - thanosrulers
+  - probes
+  verbs: ["get", "list", "watch"]
+- apiGroups: ["networking.k8s.io"] 
+  resources:
+  - ingresses
+  - networkpolicies
+  verbs: ["get", "list", "watch"]
+
+{{- end }}
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: exa-prometheus-clustre-role-binding
+subjects:
+- kind: ServiceAccount
+  name: {{ .Values.metrics.prometheus.serviceAccountName }}
+  namespace: {{ .Values.metrics.serviceMonitor.namespace }}
+roleRef:
+  kind: ClusterRole
+  {{- if .Values.metrics.prometheus.createClusterRole }}
+  name: {{ .Values.metrics.prometheus.createClustrerRoleName }}
+  {{- else }}
+  name: {{ .Values.metrics.prometheus.clusterRoleName }}
+  {{- end }}
+  apiGroup: rbac.authorization.k8s.io
+{{- end }}
+

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/helm-chart/templates/metrics/daemonset.yaml

@@ -0,0 +1,80 @@
+{{- if .Values.metrics.enabled }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: exa-csi-metrics-exporter
+  namespace: {{ .Values.metrics.exporter.namespace }}
+spec:
+  selector:
+    matchLabels:
+      app: exa-csi-metrics-exporter
+  template:
+    metadata:
+      labels:
+        app: exa-csi-metrics-exporter
+    spec:
+      serviceAccountName: exa-csi-metrics-sa
+      automountServiceAccountToken: true    # Ensure token is mounted
+      containers:
+      - name: exporter
+        imagePullPolicy: {{ .Values.metrics.exporter.pullPolicy }}
+        image: {{ .Values.metrics.exporter.repository }}:{{ .Values.metrics.exporter.tag }}
+        securityContext:
+          privileged: true
+          capabilities:
+            add: ['SYS_ADMIN']
+          allowPrivilegeEscalation: true
+        ports:
+        - containerPort: {{ .Values.metrics.exporter.containerPort }}
+          name: http
+          hostPort: {{ .Values.metrics.exporter.servicePort }}
+        env:
+        - name: COLLECT_INTERVAL
+          value: "{{ .Values.metrics.exporter.collectInterval }}"
+        - name: LOG_LEVEL
+          value: "{{ .Values.metrics.exporter.logLevel }}"
+        - name: PORT
+          value: "{{ .Values.metrics.exporter.containerPort }}"
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        volumeMounts:
+        - name: secret
+          mountPath: /config
+          readOnly: true
+        - name: ca-certificates
+          mountPath: /var/run/secrets/kubernetes.io/serviceaccount
+          readOnly: true
+        - name: host
+          mountPath: /host
+          mountPropagation: Bidirectional
+      volumes:
+      - name: secret
+        secret:
+          secretName: {{ .Values.secretName }}
+      - name: host
+        hostPath:
+          path: /
+          type: Directory
+      - name: ca-certificates
+        projected:
+          sources:
+          - serviceAccountToken:
+              path: token
+              expirationSeconds: 3600
+          - configMap:
+              name: kube-root-ca.crt   # Mount CA cert
+              items:
+              - key: ca.crt
+                path: ca.crt
+
+{{- end }}

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/helm-chart/templates/metrics/lease.yaml

@@ -0,0 +1,13 @@
+{{- if .Values.metrics.enabled }}
+apiVersion: coordination.k8s.io/v1
+kind: Lease
+metadata:
+  name: exa-csi-metrics-exporter-leader-election
+  namespace: {{ .Values.metrics.exporter.namespace }}
+spec:
+  holderIdentity: ""
+  leaseDurationSeconds: 15
+  renewTime: null
+  acquireTime: null
+  leaseTransitions: 0
+{{- end }}

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/helm-chart/templates/metrics/prometheus-rule.yaml

@@ -0,0 +1,36 @@
+{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }}
+
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: exa-csi-rules
+  namespace: {{ .Values.metrics.serviceMonitor.namespace }}  # Ensure this is the correct namespace where Prometheus is running
+  labels:
+    release: {{ .Values.metrics.prometheus.releaseLabel }}
+spec:
+  groups:
+    - name: exa-storageclass-rules
+      interval: 15s
+      rules:
+        - record: exa_csi_sc_capacity_bytes
+          expr: sum(exa_csi_pvc_capacity_bytes) by (storage_class)
+        - record: exa_csi_sc_used_bytes
+          expr: sum(exa_csi_pvc_used_bytes) by (storage_class)
+        - record: exa_csi_sc_available_bytes
+          expr: sum(exa_csi_pvc_available_bytes) by (storage_class)
+        - record: exa_csi_sc_pvc_count
+          expr: count(exa_csi_pvc_capacity_bytes) by (storage_class)
+
+    - name: exa-namespace-rules
+      interval: 15s
+      rules:
+        - record: exa_csi_namespace_capacity_bytes
+          expr: sum(exa_csi_pvc_capacity_bytes) by (exported_namespace)
+        - record: exa_csi_namespace_used_bytes
+          expr: sum(exa_csi_pvc_used_bytes) by (exported_namespace)
+        - record: exa_csi_namespace_available_bytes
+          expr: sum(exa_csi_pvc_available_bytes) by (exported_namespace)
+        - record: exa_csi_namespace_pvc_count
+          expr: count(exa_csi_pvc_capacity_bytes) by (exported_namespace)
+
+{{- end }}

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/helm-chart/templates/metrics/rbac.yaml

@@ -0,0 +1,56 @@
+{{- if .Values.metrics.enabled }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: exa-csi-metrics-sa
+  namespace: {{ .Values.metrics.exporter.namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: exa-csi-metrics-clusterrole
+rules:
+- apiGroups: [""]
+  resources: ["persistentvolumes", "persistentvolumeclaims", "nodes", "pods"]
+  verbs: ["get", "list"]
+- apiGroups: ["storage.k8s.io"]
+  resources: ["storageclasses"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: exa-csi-metrics-clusterrole-binding
+subjects:
+- kind: ServiceAccount
+  name: exa-csi-metrics-sa
+  namespace: {{ .Values.metrics.exporter.namespace }}
+roleRef:
+  kind: ClusterRole
+  name: exa-csi-metrics-clusterrole
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: exa-csi-leader-election
+  namespace: {{ .Values.metrics.exporter.namespace }}
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "create", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: exa-csi-leader-election
+  namespace: {{ .Values.metrics.exporter.namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: exa-csi-leader-election
+subjects:
+- kind: ServiceAccount
+  name: exa-csi-metrics-sa
+  namespace: {{ .Values.metrics.exporter.namespace }}
+{{- end }}

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/helm-chart/templates/metrics/service.yaml

@@ -0,0 +1,18 @@
+{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }}
+apiVersion: v1
+kind: Service
+metadata:
+  name: exa-csi-metrics-exporter
+  namespace: {{ .Values.metrics.exporter.namespace }}
+  labels:
+    app: exa-csi-metrics-exporter
+spec:
+  type: ClusterIP
+  ports:
+    - port: {{ .Values.metrics.exporter.servicePort }}
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app: exa-csi-metrics-exporter
+{{- end }}

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/helm-chart/templates/metrics/servicemonitor.yaml

@@ -0,0 +1,21 @@
+{{- if and .Values.metrics.enabled .Values.metrics.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: exa-csi-metrics-exporter
+  namespace: {{ .Values.metrics.serviceMonitor.namespace }}
+  labels:
+    release: {{ .Values.metrics.prometheus.releaseLabel }}
+spec:
+  endpoints:
+  - interval: {{ .Values.metrics.serviceMonitor.interval }}
+    path: /metrics
+    port: http  # Ensure this matches exporter port http 
+    scheme: http
+  namespaceSelector:
+    matchNames:
+    - {{ .Values.metrics.exporter.namespace }}
+  selector:
+    matchLabels:
+      app: exa-csi-metrics-exporter
+{{- end }}

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/helm-chart/templates/node-driver.yaml

@@ -0,0 +1,161 @@
+
+# ---------------------------
+# exascaler CSI Node Server
+# ---------------------------
+#
+# Runs driver node server (driver + registrar) on each node
+#
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: exascaler-csi-node-service-account
+  namespace: {{ .Release.Namespace }}
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: exascaler-csi-node-cluster-role
+rules:
+  - apiGroups: ['']
+    resources: ['events']
+    verbs: ['get', 'list', 'watch', 'create', 'update', 'patch']
+  - apiGroups: ['']
+    resources: ['nodes']
+    verbs: ['get']
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: exascaler-csi-node-cluster-role-binding
+subjects:
+  - kind: ServiceAccount
+    name: exascaler-csi-node-service-account
+    namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: exascaler-csi-node-cluster-role
+  apiGroup: rbac.authorization.k8s.io
+---
+
+# exascaler Node Server as a daemon
+
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: exascaler-csi-node
+  namespace: {{ .Release.Namespace }}
+spec:
+  selector:
+    matchLabels:
+      app: exascaler-csi-node
+  template:
+    metadata:
+      labels:
+        app: exascaler-csi-node
+    spec:
+      serviceAccount: exascaler-csi-node-service-account
+      priorityClassName: {{ .Values.priorityClassName }}
+      hostNetwork: true
+      containers:
+        # driver-registrar: sidecar container that:
+        # 1) registers the CSI driver with kubelet
+        # 2) adds the drivers custom NodeId to a label on the Kubernetes Node API Object
+        - name: driver-registrar
+          resources: {{ .Values.resources | default .Values.registrar.resources | toYaml | nindent 12 }}
+          image: {{ .Values.registrar.repository }}:{{ .Values.registrar.tag }}
+          imagePullPolicy: {{ .Values.registrar.pullPolicy }}
+          args:
+            - --v=5
+            - --csi-address=/csi/csi.sock
+            - --kubelet-registration-path=/var/lib/kubelet/plugins/exa.csi.ddn.com/csi.sock
+          livenessProbe:
+            exec:
+              command:
+              - /csi-node-driver-registrar
+              - --kubelet-registration-path=/var/lib/kubelet/plugins/exa.csi.ddn.com/csi.sock
+              - --mode=kubelet-registration-probe
+            initialDelaySeconds: 30
+            timeoutSeconds: 15
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: GOMEMLIMIT
+              valueFrom:
+                resourceFieldRef:
+                  divisor: "0"
+                  resource: limits.memory
+            - name: GOMAXPROCS
+              valueFrom:
+                resourceFieldRef:
+                  divisor: "0"
+                  resource: limits.cpu
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+        - name: driver
+          resources: {{ .Values.resources | default .Values.image.resources | toYaml | nindent 12 }}
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ['SYS_ADMIN']
+            allowPrivilegeEscalation: true
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          imagePullPolicy: "{{ .Values.image.pullPolicy }}"
+          args:
+            - --nodeid=$(KUBE_NODE_NAME)
+            - --endpoint=unix://csi/csi.sock
+            - --role=node
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: GOMEMLIMIT
+              valueFrom:
+                resourceFieldRef:
+                  divisor: "0"
+                  resource: limits.memory
+            - name: GOMAXPROCS
+              valueFrom:
+                resourceFieldRef:
+                  divisor: "0"
+                  resource: limits.cpu
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: secret
+              mountPath: /config
+            - name: host
+              mountPath: /host
+              mountPropagation: Bidirectional
+            - name: pods-mount-dir
+              mountPath: /var/lib/kubelet/pods
+              mountPropagation: Bidirectional
+      volumes:
+        - name: socket-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/exa.csi.ddn.com
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: Directory
+        - name: pods-mount-dir
+          hostPath:
+            path: /var/lib/kubelet/pods
+            type: Directory
+        - name: host
+          hostPath:
+            path: /
+            type: Directory
+        - name: secret
+          secret:
+            secretName: {{ .Values.secretName }}
+---

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/helm-chart/values.yaml

@@ -0,0 +1,97 @@
+# Default values for exa-csi.
+# This is a YAML-formatted file.
+# Declare variables to be passed into your templates.
+
+volumeNamePrefix: "pvc-exa"
+
+image:
+  repository: "quay.io/ddn/exascaler-csi-file-driver"
+  tag: v2.3.5
+  pullPolicy: "Always"
+  # Overrides the image tag whose default is the chart appVersion.
+  resources:
+
+registrar:
+  repository: registry.k8s.io/sig-storage/csi-node-driver-registrar
+  tag: v2.10.1
+  pullPolicy: IfNotPresent
+  resources:
+
+provisioner:
+  repository: registry.k8s.io/sig-storage/csi-provisioner
+  tag: v4.0.1
+  pullPolicy: IfNotPresent
+  timeout: 120m
+  resources:
+
+attacher:
+  repository: registry.k8s.io/sig-storage/csi-attacher
+  tag: v4.5.1
+  pullPolicy: IfNotPresent
+  resources:
+
+resizer:
+  repository: registry.k8s.io/sig-storage/csi-resizer
+  tag: v1.10.1
+  pullPolicy: IfNotPresent
+  resources:
+
+csi_snapshotter:
+  repository: registry.k8s.io/sig-storage/csi-snapshotter
+  tag: v5.0.1
+  pullPolicy: IfNotPresent
+  timeout: 120m
+  resources:
+
+resources:
+  limits:
+    cpu: 8
+    memory: 512Mi
+  requests:
+    cpu: 2
+    memory: 256Mi
+
+priorityClassName: system-cluster-critical
+
+secretName: exascaler-csi-file-driver-config
+
+# exascaler driver specific config
+config:
+  exascaler_map:
+    exa1:
+      mountPoint: /exaFS                                          # mountpoint on the host where the exaFS will be mounted
+      exaFS: 10.204.86.217@tcp:/testfs                            # default path to exa filesystem
+      zone: zone-1
+  #   exa2:
+  #     mountPoint: /exaFS-zone-2                                          # mountpoint on the host where the exaFS will be mounted
+  #     exaFS: 10.204.86.114@tcp:/testfs/zone-2                            # default path to exa filesystem
+  #     zone: zone-2
+  #   exa3:
+  #     mountPoint: /exaFS-zone-3                                          # mountpoint on the host where the exaFS will be mounted
+  #     exaFS: 10.204.86.114@tcp:/testfs/zone-3                            # default path to exa filesystem
+  #     zone: zone-3
+  debug: true
+  openshift: false
+
+# metrics exporter
+metrics:
+  enabled: false
+  exporter:
+    repository: quay.io/ddn/exa-csi-metrics-exporter
+    tag: v2.3.5
+    pullPolicy: Always
+    containerPort: "9200"
+    servicePort: "32666"
+    namespace: default
+    logLevel: info
+    collectInterval: 30
+  serviceMonitor:
+    enabled: false # set to true if using local prometheus operator
+    interval: 30s
+    namespace: monitoring
+  prometheus:                                                 # prometheus configuration if using prometheus operator
+    releaseLabel: prometheus                                  # release label for prometheus operator                 
+    createClusterRole: false                                  # set to false if using existing ClusterRole
+    createClustrerRoleName: exa-prometheus-cluster-role       # Name of ClusterRole to create, this name will be used to bind cluster role to prometheus service account
+    clusterRoleName: cluster-admin                            # Name of ClusterRole to bind to prometheus service account if createClusterRole is set to false otherwise `createClustrerRoleName` will be used
+    serviceAccountName: prometheus-kube-prometheus-prometheus # Service account name of prometheus

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/kubernetes/exa-csi-driver-config.yaml

@@ -0,0 +1,15 @@
+# exascaler-csi-file-driver config file to create k8s secret
+#
+# $ kubectl create secret generic exascaler-csi-file-driver-config \
+#   --from-file=deploy/kubernetes/exascaler-csi-file-driver-config.yaml
+#
+
+exascaler_map:
+  exa1:
+    mountPoint: /data                      # mountpoint on the host where the exaFS will be mounted
+    exaFS: 10.140.224.175@tcp:10.140.224.176@tcp:10.140.224.177@tcp:10.140.224.178@tcp:/GSDATA1            # default path to exa filesystem
+    zone: zone-1
+    managementIp: 10.140.224.175@tcp
+    v1xCompatible: true                         # Optional. Can only be true for one of the Exa clusters
+    volumeDirPermissions: 0777                  # Optional. Defines file permissions for mounted volumes
+debug: true

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/kubernetes/exascaler-csi-file-driver-config.yaml

@@ -0,0 +1,25 @@
+# exascaler-csi-file-driver config file to create k8s secret
+#
+# $ kubectl create secret generic exascaler-csi-file-driver-config \
+#   --from-file=deploy/kubernetes/exascaler-csi-file-driver-config.yaml
+#
+
+exascaler_map:
+  exa1:
+    mountPoint: /exaFS                          # mountpoint on the host where the exaFS will be mounted
+    exaFS: 10.204.86.114@tcp:/testfs            # default path to exa filesystem
+    zone: zone-1
+    v1xCompatible: true                         # Optional. Can only be true for one of the Exa clusters
+    volumeDirPermissions: 0777                  # Optional. Defines file permissions for mounted volumes.
+
+  exa2:
+    mountPoint: /exaFS-zone-2                   # mountpoint on the host where the exaFS will be mounted
+    exaFS: 10.204.86.114@tcp:/testfs/zone-2     # default path to exa filesystem
+    zone: zone-2
+
+  exa3:
+    mountPoint: /exaFS-zone-3                   # mountpoint on the host where the exaFS will be mounted
+    exaFS: 10.204.86.114@tcp:/testfs/zone-3     # default path to exa filesystem
+    zone: zone-3
+
+debug: true                                     # more logs

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/kubernetes/exascaler-csi-file-driver.yaml

@@ -0,0 +1,442 @@
+# Exascaler CSI Driver (2.2.4)
+#
+# This driver version works with Kubernetes version >=1.14
+#
+# In production, each CSI driver deployment has to be customized to avoid conflicts,
+# use non-default namespace and different names for non-namespaced entities like the ClusterRole
+#
+# Install to Kubernetes:
+# $ kubectl apply -f deploy/kubernetes/exascaler-csi-file-driver.yaml
+#
+
+
+# ----------------------
+# Exascaler CSI Driver
+# ----------------------
+
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: exa.csi.ddn.com
+spec:
+  attachRequired: false
+  podInfoOnMount: false
+  fsGroupPolicy: File
+---
+
+
+# ----
+# RBAC
+# ----
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: exascaler-csi-controller-service-account
+  namespace: default # replace with non-default namespace name if needed
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: exascaler-csi-controller-cluster-role
+rules:
+  - apiGroups: ['']
+    resources: ['secrets']
+    verbs: ['get', 'list', "watch"]
+  - apiGroups: ['']
+    resources: ['persistentvolumes']
+    verbs: ['get', 'list', 'watch', 'create', 'update', 'delete'] # "update" for attacher
+  - apiGroups: ['']
+    resources: ['persistentvolumeclaims']
+    verbs: ['get', 'list', 'watch', 'update']
+  - apiGroups: ['storage.k8s.io']
+    resources: ['storageclasses']
+    verbs: ['get', 'list', 'watch']
+  - apiGroups: ['']
+    resources: ['events']
+    verbs: ['list', 'watch', 'create', 'update', 'patch']
+  # attacher specific
+  - apiGroups: ['']
+    resources: ['nodes', 'pods']
+    verbs: ['get', 'list', 'watch']
+  - apiGroups: ['csi.storage.k8s.io']
+    resources: ['csinodeinfos']
+    verbs: ['get', 'list', 'watch']
+  - apiGroups: ['storage.k8s.io']
+    resources: ['volumeattachments']
+    verbs: ['get', 'list', 'watch', 'update']
+  - apiGroups: ['storage.k8s.io']
+    resources: ['volumeattachments/status']
+    verbs: ['get', 'list', 'watch', 'update', 'patch']
+  # snapshotter specific
+  - apiGroups: ['snapshot.storage.k8s.io']
+    resources: ['volumesnapshotclasses']
+    verbs: ['get', 'list', 'watch']
+  - apiGroups: ['snapshot.storage.k8s.io']
+    resources: ['volumesnapshotcontents']
+    verbs: ['create', 'get', 'list', 'watch', 'update', 'delete', 'patch']
+  - apiGroups: ['snapshot.storage.k8s.io']
+    resources: ['volumesnapshots']
+    verbs: ['get', 'list', 'watch', 'update']
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["update"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update"]
+  - apiGroups: ['apiextensions.k8s.io']
+    resources: ['customresourcedefinitions']
+    verbs: ['create', 'list', 'watch', 'delete']
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]
+  # CSINode specific
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["watch", "list", "get"]
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: exascaler-csi-controller-cluster-role-binding
+subjects:
+  - kind: ServiceAccount
+    name: exascaler-csi-controller-service-account
+    namespace: default # replace with non-default namespace name if needed
+roleRef:
+  kind: ClusterRole
+  name: exascaler-csi-controller-cluster-role
+  apiGroup: rbac.authorization.k8s.io
+---
+
+# External Resizer
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-resizer-role
+rules:
+  # The following rule should be uncommented for plugins that require secrets
+  # for provisioning.
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-resizer-binding
+subjects:
+  - kind: ServiceAccount
+    name: exascaler-csi-controller-service-account
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: csi-resizer-role
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: default
+  name: external-resizer-cfg
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: csi-resizer-role-cfg
+  namespace: default
+subjects:
+  - kind: ServiceAccount
+    name: exascaler-csi-controller-service-account
+    namespace: default
+roleRef:
+  kind: Role
+  name: external-resizer-cfg
+  apiGroup: rbac.authorization.k8s.io
+---
+
+kind: Service
+apiVersion: v1
+metadata:
+  name: exascaler-csi-controller-service
+  labels:
+    app: exascaler-csi-controller
+spec:
+  selector:
+    app: exascaler-csi-controller
+  ports:
+    - name: dummy
+      port: 12345
+---
+
+# ---------------------------------
+# Exascaler CSI Controller Server
+# ---------------------------------
+#
+# Runs single driver controller server (driver + provisioner + attacher + snapshotter) on one of the nodes.
+# Controller driver deployment does not support running multiple replicas.
+#
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: exascaler-csi-controller
+spec:
+  # serviceName: exascaler-csi-controller-service
+  selector:
+    matchLabels:
+      app: exascaler-csi-controller # has to match .spec.template.metadata.labels
+  template:
+    metadata:
+      labels:
+        app: exascaler-csi-controller
+    spec:
+      serviceAccount: exascaler-csi-controller-service-account
+      containers:
+        # csi-provisioner: sidecar container that watches Kubernetes PersistentVolumeClaim objects
+        # and triggers CreateVolume/DeleteVolume against a CSI endpoint
+        - name: csi-provisioner
+          image: k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0
+          imagePullPolicy: IfNotPresent
+          args:
+            - --csi-address=/var/lib/csi/sockets/pluginproxy/csi.sock
+            - --volume-name-prefix=pvc-exa
+            - --strict-topology
+            - --immediate-topology=false
+            - --feature-gates=Topology=true
+            - --timeout=120m
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy
+        - name: csi-attacher
+          image: k8s.gcr.io/sig-storage/csi-attacher:v3.5.0
+          imagePullPolicy: IfNotPresent
+          args:
+            - --csi-address=$(ADDRESS)
+            - --v=2
+            - --leader-election=true
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-snapshotter
+          image: registry.k8s.io/sig-storage/csi-snapshotter:v5.0.1
+          imagePullPolicy: IfNotPresent
+          args:
+            - -v=3
+            - --csi-address=/var/lib/csi/sockets/pluginproxy/csi.sock
+            - --timeout=120m
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy
+        - name: csi-resizer
+          image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0
+          args:
+            - "--csi-address=$(ADDRESS)"
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: driver
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ['SYS_ADMIN']
+            allowPrivilegeEscalation: true
+          image: quay.io/ddn/exascaler-csi-file-driver:v2.3.5
+          imagePullPolicy: Always
+          args:
+            - --nodeid=$(KUBE_NODE_NAME)
+            - --endpoint=unix://csi/csi.sock
+            - --role=controller
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: secret
+              mountPath: /config
+              readOnly: true
+            - name: host
+              mountPath: /host
+              mountPropagation: Bidirectional
+      volumes:
+        - name: socket-dir
+          emptyDir:
+        - name: secret
+          secret:
+            secretName: exascaler-csi-file-driver-config
+        - name: host
+          hostPath:
+            path: /
+            type: Directory
+---
+
+
+# ---------------------------
+# exascaler CSI Node Server
+# ---------------------------
+#
+# Runs driver node server (driver + registrar) on each node
+#
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: exascaler-csi-node-service-account
+  namespace: default # replace with non-default namespace name if needed
+---
+
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: exascaler-csi-node-cluster-role
+rules:
+  - apiGroups: ['']
+    resources: ['events']
+    verbs: ['get', 'list', 'watch', 'create', 'update', 'patch']
+  - apiGroups: ['']
+    resources: ['nodes']
+    verbs: ['get']
+---
+
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: exascaler-csi-node-cluster-role-binding
+subjects:
+  - kind: ServiceAccount
+    name: exascaler-csi-node-service-account
+    namespace: default # replace with non-default namespace name if needed
+roleRef:
+  kind: ClusterRole
+  name: exascaler-csi-node-cluster-role
+  apiGroup: rbac.authorization.k8s.io
+---
+
+# exascaler Node Server as a daemon
+
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: exascaler-csi-node
+spec:
+  selector:
+    matchLabels:
+      app: exascaler-csi-node
+  template:
+    metadata:
+      labels:
+        app: exascaler-csi-node
+    spec:
+      serviceAccount: exascaler-csi-node-service-account
+      hostNetwork: true
+      containers:
+        # driver-registrar: sidecar container that:
+        # 1) registers the CSI driver with kubelet
+        # 2) adds the drivers custom NodeId to a label on the Kubernetes Node API Object
+        - name: driver-registrar
+          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0
+          imagePullPolicy: IfNotPresent
+          args:
+            - --v=5
+            - --csi-address=/csi/csi.sock
+            - --kubelet-registration-path=/var/lib/kubelet/plugins/exa.csi.ddn.com/csi.sock
+          livenessProbe:
+            exec:
+              command:
+              - /csi-node-driver-registrar
+              - --kubelet-registration-path=/var/lib/kubelet/plugins/exa.csi.ddn.com/csi.sock
+              - --mode=kubelet-registration-probe
+            initialDelaySeconds: 30
+            timeoutSeconds: 15
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+        - name: driver
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ['SYS_ADMIN']
+            allowPrivilegeEscalation: true
+          image: quay.io/ddn/exascaler-csi-file-driver:v2.3.5
+          imagePullPolicy: Always
+          args:
+            - --nodeid=$(KUBE_NODE_NAME)
+            - --endpoint=unix://csi/csi.sock
+            - --role=node
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: secret
+              mountPath: /config
+            - name: host
+              mountPath: /host
+              mountPropagation: Bidirectional
+            - name: pods-mount-dir
+              mountPath: /var/lib/kubelet/pods
+              mountPropagation: Bidirectional
+      volumes:
+        - name: socket-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/exa.csi.ddn.com
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: Directory
+        - name: pods-mount-dir
+          hostPath:
+            path: /var/lib/kubelet/pods
+            type: Directory
+        - name: host
+          hostPath:
+            path: /
+            type: Directory
+        - name: secret
+          secret:
+            secretName: exascaler-csi-file-driver-config
+---

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/kubernetes/metrics/crb.yaml

@@ -0,0 +1,13 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: prometheus-endpoints-access
+subjects:
+- kind: ServiceAccount
+  name: monitoring-kube-prometheus-prometheus
+  namespace: monitoring
+roleRef:
+  kind: ClusterRole
+  name: cluster-admin
+  apiGroup: rbac.authorization.k8s.io
+

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/kubernetes/metrics/daemonset.yaml

@@ -0,0 +1,77 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: exa-csi-metrics-exporter
+  namespace: default
+spec:
+  selector:
+    matchLabels:
+      app: exa-csi-metrics-exporter
+  template:
+    metadata:
+      labels:
+        app: exa-csi-metrics-exporter
+    spec:
+      serviceAccountName: exa-csi-metrics-sa
+      automountServiceAccountToken: true    # Ensure token is mounted
+      containers:
+      - name: exporter
+        imagePullPolicy: Always
+        image: quay.io/ddn/exa-csi-metrics-exporter:v2.3.5
+        securityContext:
+          privileged: true
+          capabilities:
+            add: ['SYS_ADMIN']
+          allowPrivilegeEscalation: true
+        ports:
+        - containerPort: 9200
+          name: http
+          hostPort: 32666
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        - name: PORT
+          value: "9200"
+        - name: POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        - name: COLLECT_INTERVAL
+          value: "30"
+        - name: LOG_LEVEL
+          value: "info"
+        volumeMounts:
+        - name: secret
+          mountPath: /config
+          readOnly: true
+        - name: ca-certificates
+          mountPath: /var/run/secrets/kubernetes.io/serviceaccount
+          readOnly: true
+        - name: host
+          mountPath: /host
+          mountPropagation: Bidirectional
+      volumes:
+      - name: secret
+        secret:
+          secretName: exascaler-csi-file-driver-config
+      - name: host
+        hostPath:
+          path: /
+          type: Directory
+      - name: ca-certificates
+        projected:
+          sources:
+          - serviceAccountToken:
+              path: token
+              expirationSeconds: 3600
+          - configMap:
+              name: kube-root-ca.crt   # Mount CA cert
+              items:
+              - key: ca.crt
+                path: ca.crt

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/kubernetes/metrics/lease.yaml

@@ -0,0 +1,11 @@
+apiVersion: coordination.k8s.io/v1
+kind: Lease
+metadata:
+  name: exa-csi-metrics-exporter-leader-election
+  namespace: default
+spec:
+  holderIdentity: ""
+  leaseDurationSeconds: 15
+  renewTime: null
+  acquireTime: null
+  leaseTransitions: 0

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/kubernetes/metrics/prometheus/prometheus-config.yaml

@@ -0,0 +1,55 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: prometheus-config
+  namespace: monitoring
+data:
+  prometheus.yml: |
+    global:
+      scrape_interval: 15s
+
+    rule_files:
+      - "exa.rules.yaml"
+
+    scrape_configs:
+      - job_name: "kubernetes-nodes"
+        kubernetes_sd_configs:
+          - role: node
+
+      - job_name: "exa-csi-metrics-exporter"
+        kubernetes_sd_configs:
+          - role: pod
+        relabel_configs:
+          - source_labels: [__meta_kubernetes_pod_label_app]
+            action: keep
+            regex: exa-csi-metrics-exporter
+        metrics_path: /metrics
+        scheme: http
+        static_configs:
+          - targets: ["exa-csi-metrics-exporter.default.svc.cluster.local:9200"]
+
+  exa.rules.yaml: |
+    groups:
+    - name: exa-storageclass-rules
+      interval: 15s
+      rules:
+      - record: exa_csi_sc_capacity_bytes
+        expr: sum (exa_csi_pvc_capacity_bytes) by (storage_class)
+      - record: exa_csi_sc_used_bytes
+        expr: sum (exa_csi_pvc_used_bytes) by (storage_class)
+      - record: exa_csi_sc_available_bytes
+        expr: sum (exa_csi_pvc_available_bytes) by (storage_class)
+      - record: exa_csi_sc_pvc_count
+        expr: count(exa_csi_pvc_capacity_bytes) by (storage_class)
+
+    - name: exa-namespace-rules
+      interval: 15s
+      rules:
+      - record: exa_csi_namespace_capacity_bytes
+        expr: sum(exa_csi_pvc_capacity_bytes) by (exported_namespace)
+      - record: exa_csi_namespace_used_bytes
+        expr: sum(exa_csi_pvc_used_bytes) by (exported_namespace)
+      - record: exa_csi_namespace_available_bytes
+        expr: sum(exa_csi_pvc_available_bytes) by (exported_namespace)
+      - record: exa_csi_namespace_pvc_count
+        expr: count(exa_csi_pvc_capacity_bytes) by (exported_namespace)

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/kubernetes/metrics/rbac.yaml

@@ -0,0 +1,54 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: exa-csi-metrics-sa
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metrics-reader
+rules:
+- apiGroups: [""]
+  resources: ["persistentvolumes", "persistentvolumeclaims", "nodes", "pods"]
+  verbs: ["get", "list"]
+- apiGroups: ["storage.k8s.io"]
+  resources: ["storageclasses"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-reader-binding
+subjects:
+- kind: ServiceAccount
+  name: exa-csi-metrics-sa
+  namespace: default
+roleRef:
+  kind: ClusterRole
+  name: metrics-reader
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: exa-csi-leader-election
+  namespace: default
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "create", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: exa-csi-leader-election
+  namespace: default
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: exa-csi-leader-election
+subjects:
+- kind: ServiceAccount
+  name: exa-csi-metrics-sa
+  namespace: default

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/kubernetes/metrics/service.yaml

@@ -0,0 +1,16 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: exa-csi-metrics-exporter
+  namespace: default
+  labels:
+    app: exa-csi-metrics-exporter
+spec:
+  type: ClusterIP
+  ports:
+    - port: 32666
+      targetPort: http
+      protocol: TCP
+      name: http
+  selector:
+    app: exa-csi-metrics-exporter

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/kubernetes/metrics/servicemonitor.yaml

@@ -0,0 +1,18 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: exa-csi-metrics-exporter
+  namespace: monitoring
+  labels:
+    release: prometheus
+spec:
+  endpoints:
+  - interval: 30s
+    path: /metrics
+    port: "9200" # Ensure this matches exporter port
+  namespaceSelector:
+    matchNames:
+    - default
+  selector:
+    matchLabels:
+      app: exa-csi-metrics-exporter

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/kubernetes/snapshots/crds.yaml

@@ -0,0 +1,631 @@
+# kubectl create -f deploy/kubernetes/snapshots/crds.yaml
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.0
+    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419"
+  creationTimestamp: null
+  name: volumesnapshotclasses.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotClass
+    listKind: VolumeSnapshotClassList
+    plural: volumesnapshotclasses
+    singular: volumesnapshotclass
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - jsonPath: .driver
+      name: Driver
+      type: string
+    - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted.
+      jsonPath: .deletionPolicy
+      name: DeletionPolicy
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1
+    schema:
+      openAPIV3Schema:
+        description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          deletionPolicy:
+            description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required.
+            enum:
+            - Delete
+            - Retain
+            type: string
+          driver:
+            description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required.
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          parameters:
+            additionalProperties:
+              type: string
+            description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes.
+            type: object
+        required:
+        - deletionPolicy
+        - driver
+        type: object
+    served: true
+    storage: false
+    subresources: {}
+  - additionalPrinterColumns:
+    - jsonPath: .driver
+      name: Driver
+      type: string
+    - description: Determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted.
+      jsonPath: .deletionPolicy
+      name: DeletionPolicy
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: VolumeSnapshotClass specifies parameters that a underlying storage system uses when creating a volume snapshot. A specific VolumeSnapshotClass is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses are non-namespaced
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          deletionPolicy:
+            description: deletionPolicy determines whether a VolumeSnapshotContent created through the VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. Required.
+            enum:
+            - Delete
+            - Retain
+            type: string
+          driver:
+            description: driver is the name of the storage driver that handles this VolumeSnapshotClass. Required.
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          parameters:
+            additionalProperties:
+              type: string
+            description: parameters is a key-value map with storage driver specific parameters for creating snapshots. These values are opaque to Kubernetes.
+            type: object
+        required:
+        - deletionPolicy
+        - driver
+        type: object
+    served: true
+    storage: true
+    subresources: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.0
+    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419"
+  creationTimestamp: null
+  name: volumesnapshotcontents.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshotContent
+    listKind: VolumeSnapshotContentList
+    plural: volumesnapshotcontents
+    singular: volumesnapshotcontent
+  scope: Cluster
+  versions:
+  - additionalPrinterColumns:
+    - description: Indicates if the snapshot is ready to be used to restore a volume.
+      jsonPath: .status.readyToUse
+      name: ReadyToUse
+      type: boolean
+    - description: Represents the complete size of the snapshot in bytes
+      jsonPath: .status.restoreSize
+      name: RestoreSize
+      type: integer
+    - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted.
+      jsonPath: .spec.deletionPolicy
+      name: DeletionPolicy
+      type: string
+    - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system.
+      jsonPath: .spec.driver
+      name: Driver
+      type: string
+    - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+      jsonPath: .spec.volumeSnapshotClassName
+      name: VolumeSnapshotClass
+      type: string
+    - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound.
+      jsonPath: .spec.volumeSnapshotRef.name
+      name: VolumeSnapshot
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1
+    schema:
+      openAPIV3Schema:
+        description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          spec:
+            description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required.
+            properties:
+              deletionPolicy:
+                description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the  VolumeSnapshotContent object. Required.
+                enum:
+                - Delete
+                - Retain
+                type: string
+              driver:
+                description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required.
+                type: string
+              source:
+                description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required.
+                properties:
+                  snapshotHandle:
+                    description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable.
+                    type: string
+                  volumeHandle:
+                    description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable.
+                    type: string
+                type: object
+                oneOf:
+                - required: ["snapshotHandle"]
+                - required: ["volumeHandle"]
+              volumeSnapshotClassName:
+                description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation.
+                type: string
+              volumeSnapshotRef:
+                description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required.
+                properties:
+                  apiVersion:
+                    description: API version of the referent.
+                    type: string
+                  fieldPath:
+                    description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.'
+                    type: string
+                  kind:
+                    description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                    type: string
+                  name:
+                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                    type: string
+                  namespace:
+                    description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                    type: string
+                  resourceVersion:
+                    description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                    type: string
+                  uid:
+                    description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                    type: string
+                type: object
+            required:
+            - deletionPolicy
+            - driver
+            - source
+            - volumeSnapshotRef
+            type: object
+          status:
+            description: status represents the current information of a snapshot.
+            properties:
+              creationTime:
+                description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC.
+                format: int64
+                type: integer
+              error:
+                description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared.
+                properties:
+                  message:
+                    description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.'
+                    type: string
+                  time:
+                    description: time is the timestamp when the error was encountered.
+                    format: date-time
+                    type: string
+                type: object
+              readyToUse:
+                description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown.
+                type: boolean
+              restoreSize:
+                description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown.
+                format: int64
+                minimum: 0
+                type: integer
+              snapshotHandle:
+                description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress.
+                type: string
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: Indicates if the snapshot is ready to be used to restore a volume.
+      jsonPath: .status.readyToUse
+      name: ReadyToUse
+      type: boolean
+    - description: Represents the complete size of the snapshot in bytes
+      jsonPath: .status.restoreSize
+      name: RestoreSize
+      type: integer
+    - description: Determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted.
+      jsonPath: .spec.deletionPolicy
+      name: DeletionPolicy
+      type: string
+    - description: Name of the CSI driver used to create the physical snapshot on the underlying storage system.
+      jsonPath: .spec.driver
+      name: Driver
+      type: string
+    - description: Name of the VolumeSnapshotClass to which this snapshot belongs.
+      jsonPath: .spec.volumeSnapshotClassName
+      name: VolumeSnapshotClass
+      type: string
+    - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound.
+      jsonPath: .spec.volumeSnapshotRef.name
+      name: VolumeSnapshot
+      type: string
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: VolumeSnapshotContent represents the actual "on-disk" snapshot object in the underlying storage system
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          spec:
+            description: spec defines properties of a VolumeSnapshotContent created by the underlying storage system. Required.
+            properties:
+              deletionPolicy:
+                description: deletionPolicy determines whether this VolumeSnapshotContent and its physical snapshot on the underlying storage system should be deleted when its bound VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent and its physical snapshot on underlying storage system are deleted. For dynamically provisioned snapshots, this field will automatically be filled in by the CSI snapshotter sidecar with the "DeletionPolicy" field defined in the corresponding VolumeSnapshotClass. For pre-existing snapshots, users MUST specify this field when creating the  VolumeSnapshotContent object. Required.
+                enum:
+                - Delete
+                - Retain
+                type: string
+              driver:
+                description: driver is the name of the CSI driver used to create the physical snapshot on the underlying storage system. This MUST be the same as the name returned by the CSI GetPluginName() call for that driver. Required.
+                type: string
+              source:
+                description: source specifies whether the snapshot is (or should be) dynamically provisioned or already exists, and just requires a Kubernetes object representation. This field is immutable after creation. Required.
+                properties:
+                  snapshotHandle:
+                    description: snapshotHandle specifies the CSI "snapshot_id" of a pre-existing snapshot on the underlying storage system for which a Kubernetes object representation was (or should be) created. This field is immutable.
+                    type: string
+                  volumeHandle:
+                    description: volumeHandle specifies the CSI "volume_id" of the volume from which a snapshot should be dynamically taken from. This field is immutable.
+                    type: string
+                type: object
+              volumeSnapshotClassName:
+                description: name of the VolumeSnapshotClass from which this snapshot was (or will be) created. Note that after provisioning, the VolumeSnapshotClass may be deleted or recreated with different set of values, and as such, should not be referenced post-snapshot creation.
+                type: string
+              volumeSnapshotRef:
+                description: volumeSnapshotRef specifies the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName field must reference to this VolumeSnapshotContent's name for the bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent object, name and namespace of the VolumeSnapshot object MUST be provided for binding to happen. This field is immutable after creation. Required.
+                properties:
+                  apiVersion:
+                    description: API version of the referent.
+                    type: string
+                  fieldPath:
+                    description: 'If referring to a piece of an object instead of an entire object, this string should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2]. For example, if the object reference is to a container within a pod, this would take on a value like: "spec.containers{name}" (where "name" refers to the name of the container that triggered the event) or if no container name is specified "spec.containers[2]" (container with index 2 in this pod). This syntax is chosen only to have some well-defined way of referencing a part of an object. TODO: this design is not final and this field is subject to change in the future.'
+                    type: string
+                  kind:
+                    description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                    type: string
+                  name:
+                    description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                    type: string
+                  namespace:
+                    description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                    type: string
+                  resourceVersion:
+                    description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                    type: string
+                  uid:
+                    description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                    type: string
+                type: object
+            required:
+            - deletionPolicy
+            - driver
+            - source
+            - volumeSnapshotRef
+            type: object
+          status:
+            description: status represents the current information of a snapshot.
+            properties:
+              creationTime:
+                description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it indicates the creation time is unknown. The format of this field is a Unix nanoseconds time encoded as an int64. On Unix, the command `date +%s%N` returns the current time in nanoseconds since 1970-01-01 00:00:00 UTC.
+                format: int64
+                type: integer
+              error:
+                description: error is the last observed error during snapshot creation, if any. Upon success after retry, this error field will be cleared.
+                properties:
+                  message:
+                    description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.'
+                    type: string
+                  time:
+                    description: time is the timestamp when the error was encountered.
+                    format: date-time
+                    type: string
+                type: object
+              readyToUse:
+                description: readyToUse indicates if a snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown.
+                type: boolean
+              restoreSize:
+                description: restoreSize represents the complete size of the snapshot in bytes. In dynamic snapshot creation case, this field will be filled in by the CSI snapshotter sidecar with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown.
+                format: int64
+                minimum: 0
+                type: integer
+              snapshotHandle:
+                description: snapshotHandle is the CSI "snapshot_id" of a snapshot on the underlying storage system. If not specified, it indicates that dynamic snapshot creation has either failed or it is still in progress.
+                type: string
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.4.0
+    api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/419"
+  creationTimestamp: null
+  name: volumesnapshots.snapshot.storage.k8s.io
+spec:
+  group: snapshot.storage.k8s.io
+  names:
+    kind: VolumeSnapshot
+    listKind: VolumeSnapshotList
+    plural: volumesnapshots
+    singular: volumesnapshot
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: Indicates if the snapshot is ready to be used to restore a volume.
+      jsonPath: .status.readyToUse
+      name: ReadyToUse
+      type: boolean
+    - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created.
+      jsonPath: .spec.source.persistentVolumeClaimName
+      name: SourcePVC
+      type: string
+    - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot.
+      jsonPath: .spec.source.volumeSnapshotContentName
+      name: SourceSnapshotContent
+      type: string
+    - description: Represents the minimum size of volume required to rehydrate from this snapshot.
+      jsonPath: .status.restoreSize
+      name: RestoreSize
+      type: string
+    - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+      jsonPath: .spec.volumeSnapshotClassName
+      name: SnapshotClass
+      type: string
+    - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object.
+      jsonPath: .status.boundVolumeSnapshotContentName
+      name: SnapshotContent
+      type: string
+    - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system.
+      jsonPath: .status.creationTime
+      name: CreationTime
+      type: date
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1
+    schema:
+      openAPIV3Schema:
+        description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          spec:
+            description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.'
+            properties:
+              source:
+                description: source specifies where a snapshot will be created from. This field is immutable after creation. Required.
+                properties:
+                  persistentVolumeClaimName:
+                    description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable.
+                    type: string
+                  volumeSnapshotContentName:
+                    description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable.
+                    type: string
+                type: object
+                oneOf:
+                - required: ["persistentVolumeClaimName"]
+                - required: ["volumeSnapshotContentName"]
+              volumeSnapshotClassName:
+                description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.'
+                type: string
+            required:
+            - source
+            type: object
+          status:
+            description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.
+            properties:
+              boundVolumeSnapshotContentName:
+                description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.'
+                type: string
+              creationTime:
+                description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown.
+                format: date-time
+                type: string
+              error:
+                description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared.
+                properties:
+                  message:
+                    description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.'
+                    type: string
+                  time:
+                    description: time is the timestamp when the error was encountered.
+                    format: date-time
+                    type: string
+                type: object
+              readyToUse:
+                description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown.
+                type: boolean
+              restoreSize:
+                type: string
+                description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown.
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: false
+    subresources:
+      status: {}
+  - additionalPrinterColumns:
+    - description: Indicates if the snapshot is ready to be used to restore a volume.
+      jsonPath: .status.readyToUse
+      name: ReadyToUse
+      type: boolean
+    - description: If a new snapshot needs to be created, this contains the name of the source PVC from which this snapshot was (or will be) created.
+      jsonPath: .spec.source.persistentVolumeClaimName
+      name: SourcePVC
+      type: string
+    - description: If a snapshot already exists, this contains the name of the existing VolumeSnapshotContent object representing the existing snapshot.
+      jsonPath: .spec.source.volumeSnapshotContentName
+      name: SourceSnapshotContent
+      type: string
+    - description: Represents the minimum size of volume required to rehydrate from this snapshot.
+      jsonPath: .status.restoreSize
+      name: RestoreSize
+      type: string
+    - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot.
+      jsonPath: .spec.volumeSnapshotClassName
+      name: SnapshotClass
+      type: string
+    - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot object intends to bind to. Please note that verification of binding actually requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure both are pointing at each other. Binding MUST be verified prior to usage of this object.
+      jsonPath: .status.boundVolumeSnapshotContentName
+      name: SnapshotContent
+      type: string
+    - description: Timestamp when the point-in-time snapshot was taken by the underlying storage system.
+      jsonPath: .status.creationTime
+      name: CreationTime
+      type: date
+    - jsonPath: .metadata.creationTimestamp
+      name: Age
+      type: date
+    name: v1beta1
+    schema:
+      openAPIV3Schema:
+        description: VolumeSnapshot is a user's request for either creating a point-in-time snapshot of a persistent volume, or binding to a pre-existing snapshot.
+        properties:
+          apiVersion:
+            description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            type: string
+          kind:
+            description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            type: string
+          spec:
+            description: 'spec defines the desired characteristics of a snapshot requested by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots Required.'
+            properties:
+              source:
+                description: source specifies where a snapshot will be created from. This field is immutable after creation. Required.
+                properties:
+                  persistentVolumeClaimName:
+                    description: persistentVolumeClaimName specifies the name of the PersistentVolumeClaim object representing the volume from which a snapshot should be created. This PVC is assumed to be in the same namespace as the VolumeSnapshot object. This field should be set if the snapshot does not exists, and needs to be created. This field is immutable.
+                    type: string
+                  volumeSnapshotContentName:
+                    description: volumeSnapshotContentName specifies the name of a pre-existing VolumeSnapshotContent object representing an existing volume snapshot. This field should be set if the snapshot already exists and only needs a representation in Kubernetes. This field is immutable.
+                    type: string
+                type: object
+              volumeSnapshotClassName:
+                description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass requested by the VolumeSnapshot. VolumeSnapshotClassName may be left nil to indicate that the default SnapshotClass should be used. A given cluster may have multiple default Volume SnapshotClasses: one default per CSI Driver. If a VolumeSnapshot does not specify a SnapshotClass, VolumeSnapshotSource will be checked to figure out what the associated CSI Driver is, and the default VolumeSnapshotClass associated with that CSI Driver will be used. If more than one VolumeSnapshotClass exist for a given CSI Driver and more than one have been marked as default, CreateSnapshot will fail and generate an event. Empty string is not allowed for this field.'
+                type: string
+            required:
+            - source
+            type: object
+          status:
+            description: status represents the current information of a snapshot. Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.
+            properties:
+              boundVolumeSnapshotContentName:
+                description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent object to which this VolumeSnapshot object intends to bind to. If not specified, it indicates that the VolumeSnapshot object has not been successfully bound to a VolumeSnapshotContent object yet. NOTE: To avoid possible security issues, consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent point at each other) before using this object.'
+                type: string
+              creationTime:
+                description: creationTime is the timestamp when the point-in-time snapshot is taken by the underlying storage system. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "creation_time" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "creation_time" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. If not specified, it may indicate that the creation time of the snapshot is unknown.
+                format: date-time
+                type: string
+              error:
+                description: error is the last observed error during snapshot creation, if any. This field could be helpful to upper level controllers(i.e., application controller) to decide whether they should continue on waiting for the snapshot to be created based on the type of error reported. The snapshot controller will keep retrying when an error occurrs during the snapshot creation. Upon success, this error field will be cleared.
+                properties:
+                  message:
+                    description: 'message is a string detailing the encountered error during snapshot creation if specified. NOTE: message may be logged, and it should not contain sensitive information.'
+                    type: string
+                  time:
+                    description: time is the timestamp when the error was encountered.
+                    format: date-time
+                    type: string
+                type: object
+              readyToUse:
+                description: readyToUse indicates if the snapshot is ready to be used to restore a volume. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "ready_to_use" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "ready_to_use" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it, otherwise, this field will be set to "True". If not specified, it means the readiness of a snapshot is unknown.
+                type: boolean
+              restoreSize:
+                type: string
+                description: restoreSize represents the minimum size of volume required to create a volume from this snapshot. In dynamic snapshot creation case, this field will be filled in by the snapshot controller with the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing snapshot, this field will be filled with the "size_bytes" value returned from the CSI "ListSnapshots" gRPC call if the driver supports it. When restoring a volume from this snapshot, the size of the volume MUST NOT be smaller than the restoreSize if it is specified, otherwise the restoration will fail. If not specified, it indicates that the size is unknown.
+                pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
+                x-kubernetes-int-or-string: true
+            type: object
+        required:
+        - spec
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}
+status:
+  acceptedNames:
+    kind: ""
+    plural: ""
+  conditions: []
+  storedVersions: []
+

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/kubernetes/snapshots/snapshotter.yaml

@@ -0,0 +1,118 @@
+# RBAC file for the snapshot controller.
+#
+# The snapshot controller implements the control loop for CSI snapshot functionality.
+# It should be installed as part of the base Kubernetes distribution in an appropriate
+# namespace for components implementing base system functionality. For installing with
+# Vanilla Kubernetes, kube-system makes sense for the namespace.
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: snapshot-controller
+  namespace: default # TODO: replace with the namespace you want for your controller, e.g. kube-system
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  # rename if there are conflicts
+  name: snapshot-controller-runner
+rules:
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["update"]
+
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-role
+subjects:
+  - kind: ServiceAccount
+    name: snapshot-controller
+    namespace: default # TODO: replace with the namespace you want for your controller, e.g. kube-system
+roleRef:
+  kind: ClusterRole
+  # change the name also here if the ClusterRole gets renamed
+  name: snapshot-controller-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: default # TODO: replace with the namespace you want for your controller, e.g. kube-system
+  name: snapshot-controller-leaderelection
+rules:
+- apiGroups: ["coordination.k8s.io"]
+  resources: ["leases"]
+  verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: snapshot-controller-leaderelection
+  namespace: default # TODO: replace with the namespace you want for your controller, e.g. kube-system
+subjects:
+  - kind: ServiceAccount
+    name: snapshot-controller
+    namespace: default # TODO: replace with the namespace you want for your controller, e.g. kube-system
+roleRef:
+  kind: Role
+  name: snapshot-controller-leaderelection
+  apiGroup: rbac.authorization.k8s.io
+
+# This YAML file shows how to deploy the snapshot controller
+
+# The snapshot controller implements the control loop for CSI snapshot functionality.
+# It should be installed as part of the base Kubernetes distribution in an appropriate
+# namespace for components implementing base system functionality. For installing with
+# Vanilla Kubernetes, kube-system makes sense for the namespace.
+
+---
+kind: StatefulSet
+apiVersion: apps/v1
+metadata:
+  name: snapshot-controller
+  namespace: default # TODO: replace with the namespace you want for your controller, e.g. kube-system
+spec:
+  serviceName: "snapshot-controller"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: snapshot-controller
+  template:
+    metadata:
+      labels:
+        app: snapshot-controller
+    spec:
+      serviceAccount: snapshot-controller
+      containers:
+        - name: snapshot-controller
+          image: k8s.gcr.io/sig-storage/snapshot-controller:v4.0.0
+          args:
+            - "--v=5"
+            - "--leader-election=false"
+          imagePullPolicy: IfNotPresent

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/openshift/exascaler-csi-file-driver-config.yaml

@@ -0,0 +1,27 @@
+# exascaler-csi-file-driver config file to create k8s secret
+#
+# $ oc create secret generic exascaler-csi-file-driver-config \
+#   --from-file=deploy/openshift/exascaler-csi-file-driver-config.yaml
+#
+
+exascaler_map:
+  exa1:
+    bindMount: true
+    mountPoint: /exaFS                          # mountpoint on the host where the exaFS will be mounted
+    exaFS: 10.204.86.217@tcp:/testfs            # default path to exa filesystem
+    zone: zone-1
+    v1xCompatible: true                         # Optional. Can only be true for one of the Exa clusters  
+    volumeDirPermissions: 0777                  # Optional. Defines file permissions for mounted volumes.
+
+  exa2:
+    mountPoint: /exaFS-zone-2                   # mountpoint on the host where the exaFS will be mounted
+    exaFS: 10.204.86.217@tcp:/testfs/zone-2     # default path to exa filesystem
+    zone: zone-2
+
+  exa3:
+    mountPoint: /exaFS-zone-3                   # mountpoint on the host where the exaFS will be mounted
+    exaFS: 10.204.86.217@tcp:/testfs/zone-3     # default path to exa filesystem
+    zone: zone-3
+
+debug: true                                     # more logs
+openshift: true

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/openshift/exascaler-csi-file-driver.yaml

@@ -0,0 +1,492 @@
+# Exascaler CSI Driver (1.0.0)
+#
+# In production, each CSI driver deployment has to be customized to avoid conflicts,
+# use non-default namespace and different names for non-namespaced entities like the ClusterRole
+#
+# Install to Openshift:
+# $ oc apply -f deploy/openshift/exascaler-csi-file-driver.yaml
+#
+
+# ----------------------
+# Exascaler CSI Driver
+# ----------------------
+
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  name: exa.csi.ddn.com
+spec:
+  attachRequired: false
+  podInfoOnMount: false
+---
+
+
+# ----
+# RBAC
+# ----
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: sa-driver
+  namespace: openshift-kmm
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: sa-driver
+rules:
+- apiGroups:
+  - security.openshift.io
+  resources:
+  - securitycontextconstraints
+  verbs:
+  - use
+  resourceNames:
+  - sa-driver
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: sa-driver
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: sa-driver
+subjects:
+- kind: ServiceAccount
+  name: sa-driver
+userNames:
+- system:serviceaccount:openshift-kmm:sa-driver
+---
+
+allowHostDirVolumePlugin: true
+allowHostIPC: true
+allowHostNetwork: true
+allowHostPID: true
+allowHostPorts: true
+allowPrivilegeEscalation: true
+allowPrivilegedContainer: true
+allowedCapabilities: null
+apiVersion: security.openshift.io/v1
+defaultAddCapabilities: null
+fsGroup:
+  type: RunAsAny
+groups: []
+kind: SecurityContextConstraints
+metadata:
+  annotations:
+    kubernetes.io/description: this is a custom SCC which combines the allowances of hostnetwork and hostmount-anyuid
+  name: sa-driver
+priority: null
+readOnlyRootFilesystem: false
+runAsUser:
+  type: RunAsAny
+seLinuxContext:
+  type: MustRunAs
+supplementalGroups:
+  type: RunAsAny
+users:
+- system:serviceaccount:openshift-kmm:sa-driver
+volumes:
+- configMap
+- downwardAPI
+- emptyDir
+- hostPath
+- nfs
+- persistentVolumeClaim
+- projected
+- secret
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: exascaler-driver-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumes
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - "storage.k8s.io"
+  resources:
+  - volumeattachments
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+  - update
+  - patch
+- apiGroups:
+  - "storage.k8s.io"
+  resources:
+  - volumeattachments/status
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims/status
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - events
+  verbs:
+  - list
+  - watch
+  - create
+  - update
+  - patch
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - csinodes
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - coordination.k8s.io
+  resources:
+  - leases
+  verbs:
+  - get
+  - watch
+  - list
+  - delete
+  - update
+  - create
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - delete
+  - patch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshots
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshots/status
+  verbs:
+  - update
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
+  - volumesnapshotcontents/status
+  verbs:
+  - update
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+---
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: exascaler-driver-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: exascaler-driver-role
+subjects:
+- kind: ServiceAccount
+  name: sa-driver
+  namespace: openshift-kmm
+---
+
+# ---------------------------------
+# Exascaler CSI Controller Server
+# ---------------------------------
+#
+# Runs single driver controller server (driver + provisioner + attacher + snapshotter) on one of the nodes.
+# Controller driver deployment does not support running multiple replicas.
+#
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: exascaler-csi-controller
+  namespace: openshift-kmm
+spec:
+  selector:
+    matchLabels:
+      app: exascaler-csi-controller
+  template:
+    metadata:
+      labels:
+        app: exascaler-csi-controller
+    spec:
+      serviceAccount: sa-driver
+      hostNetwork: true
+      securityContext:
+        runAsUser: 0
+      containers:
+      - name: csi-provisioner
+        image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.3
+        imagePullPolicy: IfNotPresent
+        args:
+          - --csi-address=/csi/csi.sock
+          - --volume-name-prefix=pvc-exa
+          - --strict-topology
+          - --immediate-topology=false
+          - --feature-gates=Topology=true
+          - --timeout=120m
+        volumeMounts:
+          - name: socket-dir
+            mountPath: /csi
+      - name: csi-attacher
+        image: registry.k8s.io/sig-storage/csi-attacher:v4.4.0
+        imagePullPolicy: IfNotPresent
+        args:
+          - --csi-address=$(ADDRESS)
+          - --v=2
+          - --leader-election=true
+        env:
+          - name: ADDRESS
+            value: /csi/csi.sock
+        volumeMounts:
+          - name: socket-dir
+            mountPath: /csi/
+      - name: csi-snapshotter
+        image: registry.k8s.io/sig-storage/csi-snapshotter:v5.0.1
+        imagePullPolicy: IfNotPresent
+        args:
+          - -v=3
+          - --csi-address=/var/lib/csi/sockets/pluginproxy/csi.sock
+          - --timeout=120m
+        volumeMounts:
+          - name: socket-dir
+            mountPath: /var/lib/csi/sockets/pluginproxy
+      - name: csi-resizer
+        image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0
+        args:
+          - "--csi-address=$(ADDRESS)"
+        env:
+          - name: ADDRESS
+            value: /csi/csi.sock
+        imagePullPolicy: IfNotPresent
+        volumeMounts:
+          - name: socket-dir
+            mountPath: /csi/
+      - image: quay.io/ddn/exascaler-openshift-file-driver:v2.3.5
+        imagePullPolicy: Always
+        args:
+          - --nodeid=$(KUBE_NODE_NAME)
+          - --endpoint=unix:///csi/csi.sock
+          - --role=controller
+        env:
+          - name: KUBE_NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+        securityContext:
+          allowPrivilegeEscalation: true
+          privileged: true
+        name: driver
+        volumeMounts:
+        - mountPath: /csi
+          name: socket-dir
+        - name: secret
+          mountPath: /config
+          readOnly: true
+        - name: dev
+          mountPath: /dev
+
+      volumes:
+      - name: dev
+        hostPath:
+          path: /dev
+          type: Directory
+      - name: socket-dir
+        hostPath:
+          path: /var/lib/kubelet/plugins/exa.csi.ddn.com/
+          type: DirectoryOrCreate
+      - name: secret
+        secret:
+          secretName: exascaler-csi-file-driver-config
+---
+
+# ---------------------------
+# Exascaler CSI Node Server
+# ---------------------------
+#
+# Runs driver node server (driver + registrar) on each node
+#
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: exascaler-csi-node
+spec:
+  selector:
+    matchLabels:
+      app: exascaler-csi-node
+  template:
+    metadata:
+      labels:
+        app: exascaler-csi-node
+    spec:
+      serviceAccount: sa-driver
+      hostNetwork: true
+      containers:
+        # driver-registrar: sidecar container that:
+        # 1) registers the CSI driver with kubelet
+        # 2) adds the drivers custom NodeId to a label on the Kubernetes Node API Object
+        - name: driver-registrar
+          image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0
+          imagePullPolicy: IfNotPresent
+          args:
+            - --v=5
+            - --csi-address=/csi/csi.sock
+            - --kubelet-registration-path=/var/lib/kubelet/plugins_registry/exa.csi.ddn.com/csi.sock
+          livenessProbe:
+            exec:
+              command:
+              - /csi-node-driver-registrar
+              - --kubelet-registration-path=/var/lib/kubelet/plugins_registry/exa.csi.ddn.com/csi.sock
+              - --mode=kubelet-registration-probe
+            initialDelaySeconds: 30
+            timeoutSeconds: 15
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+        - name: driver
+          securityContext:
+            privileged: true
+            allowPrivilegeEscalation: true
+          image: quay.io/ddn/exascaler-openshift-file-driver:v2.3.5
+          imagePullPolicy: Always
+          args:
+            - --nodeid=$(KUBE_NODE_NAME)
+            - --endpoint=unix:///csi/csi.sock
+            - --role=node
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: kubelet-dir
+              mountPath: /var/lib/kubelet
+              mountPropagation: "Bidirectional"
+            - name: dev
+              mountPath: /dev
+            - name: socket-dir
+              mountPath: /csi
+            - name: secret
+              mountPath: /config
+      volumes:
+        - name: kubelet-dir
+          hostPath:
+            path: /var/lib/kubelet
+            type: Directory
+        - name: dev
+          hostPath:
+            path: /dev
+            type: Directory
+        - name: socket-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/exa.csi.ddn.com/
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: Directory
+        - name: secret
+          secret:
+            secretName: exascaler-csi-file-driver-config
+---

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/openshift/lustre-module/ko2iblnd-mod.yaml

@@ -0,0 +1,35 @@
+apiVersion: kmm.sigs.x-k8s.io/v1beta1
+kind: Module
+metadata:
+  name: ko2iblnd
+spec:
+  moduleLoader:
+    container:
+      modprobe:
+        moduleName: ko2iblnd
+        dirName: /opt
+
+      kernelMappings:  # At least one item is required
+        # For any other kernel, build the image using the Dockerfile in the my-kmod ConfigMap.
+        - regexp: '^.+$'
+          containerImage: "image-registry.openshift-image-registry.svc:5000/openshift-kmm/lustre-client-moduleloader:5.14.0-284.25.1.el9_2.x86_64"
+          build:
+            baseImageRegistryTLS:
+              # Optional and not recommended! If true, the build will be allowed to pull the image in the Dockerfile's
+              # FROM instruction using plain HTTP.
+              insecure: false
+              # Optional and not recommended! If true, the build will skip any TLS server certificate validation when
+              # pulling the image in the Dockerfile's FROM instruction using plain HTTP.
+              insecureSkipTLSVerify: false
+            dockerfileConfigMap:  # Required
+              name: lustre-ci-dockerfile
+          registryTLS:
+            # Optional and not recommended! If true, KMM will be allowed to check if the container image already exists
+            # using plain HTTP.
+            insecure: false
+            # Optional and not recommended! If true, KMM will skip any TLS server certificate validation when checking if
+            # the container image already exists.
+            insecureSkipTLSVerify: false
+
+  selector:
+    node-role.kubernetes.io/worker: ""

template/stg/ddn/ddn-install-running/exa-csi-driver-2.3.5/deploy/openshift/lustre-module/ksocklnd-mod.yaml

@@ -0,0 +1,35 @@
+apiVersion: kmm.sigs.x-k8s.io/v1beta1
+kind: Module
+metadata:
+  name: ksocklnd
+spec:
+  moduleLoader:
+    container:
+      modprobe:
+        moduleName: ksocklnd
+        dirName: /opt
+
+      kernelMappings:  # At least one item is required
+        # For any other kernel, build the image using the Dockerfile in the my-kmod ConfigMap.
+        - regexp: '^.+$'
+          containerImage: "image-registry.openshift-image-registry.svc:5000/openshift-kmm/lustre-client-moduleloader:5.14.0-284.25.1.el9_2.x86_64"
+          build:
+            baseImageRegistryTLS:
+              # Optional and not recommended! If true, the build will be allowed to pull the image in the Dockerfile's
+              # FROM instruction using plain HTTP.
+              insecure: false
+              # Optional and not recommended! If true, the build will skip any TLS server certificate validation when
+              # pulling the image in the Dockerfile's FROM instruction using plain HTTP.
+              insecureSkipTLSVerify: false
+            dockerfileConfigMap:  # Required
+              name: lustre-ci-dockerfile
+          registryTLS:
+            # Optional and not recommended! If true, KMM will be allowed to check if the container image already exists
+            # using plain HTTP.
+            insecure: false
+            # Optional and not recommended! If true, KMM will skip any TLS server certificate validation when checking if
+            # the container image already exists.
+            insecureSkipTLSVerify: false
+
+  selector:
+    node-role.kubernetes.io/worker: ""