From b416dccc0490ffcc021af09968b4b1b2ccc66601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BE=90=E6=B3=BD=E6=84=8F?= Date: Thu, 8 Sep 2022 14:11:25 +0800 Subject: [PATCH] up --- 1 | 0 Chart.yaml | 12 ++++ README.md | 39 +++++++++++ manifest.yaml | 113 ++++++++++++++++++++++++++++++ templates/attacher.yaml | 99 +++++++++++++++++++++++++++ templates/csi-s3.yaml | 133 ++++++++++++++++++++++++++++++++++++ templates/provisioner.yaml | 115 +++++++++++++++++++++++++++++++ templates/secret.yaml | 11 +++ templates/storageclass.yaml | 26 +++++++ values-overrides.yaml | 44 ++++++++++++ values.yaml | 44 ++++++++++++ 11 files changed, 636 insertions(+) delete mode 100644 1 create mode 100644 Chart.yaml create mode 100644 README.md create mode 100644 manifest.yaml create mode 100644 templates/attacher.yaml create mode 100644 templates/csi-s3.yaml create mode 100644 templates/provisioner.yaml create mode 100644 templates/secret.yaml create mode 100644 templates/storageclass.yaml create mode 100644 values-overrides.yaml create mode 100644 values.yaml diff --git a/1 b/1 deleted file mode 100644 index e69de29..0000000 diff --git a/Chart.yaml b/Chart.yaml new file mode 100644 index 0000000..0e48f61 --- /dev/null +++ b/Chart.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +appVersion: 0.31.6 +description: "Container Storage Interface (CSI) driver for S3 volumes" +name: csi-s3 +version: 0.31.6 +keywords: + - s3 +home: https://github.com/yandex-cloud/k8s-csi-s3 +sources: + - https://github.com/yandex-cloud/k8s-csi-s3/deploy/helm +icon: https://raw.githubusercontent.com/yandex-cloud/geesefs/master/doc/geesefs.png diff --git a/README.md b/README.md new file mode 100644 index 0000000..2c32ede --- /dev/null +++ b/README.md @@ -0,0 +1,39 @@ +# Helm chart for csi-s3 + +This chart adds S3 volume support to your cluster. + +## Install chart + +- Helm 2.x: `helm install [--set secret.accessKey=... --set secret.secretKey=... ...] --namespace kube-system --name csi-s3 .` +- Helm 3.x: `helm install [--set secret.accessKey=... --set secret.secretKey=... ...] --namespace kube-system csi-s3 .` + +After installation succeeds, you can get a status of Chart: `helm status csi-s3`. + +## Delete Chart + +- Helm 2.x: `helm delete --purge csi-s3` +- Helm 3.x: `helm uninstall csi-s3 --namespace kube-system` + +## Configuration + +By default, this chart creates a secret and a storage class. You should at least set `secret.accessKey` and `secret.secretKey` +to your [Yandex Object Storage](https://cloud.yandex.com/en-ru/services/storage) keys for it to work. + +The following table lists all configuration parameters and their default values. + +| Parameter | Description | Default | +| ---------------------------- | ---------------------------------------------------------------------- | ------------------------------------------------------ | +| `storageClass.create` | Specifies whether the storage class should be created | true | +| `storageClass.name` | Storage class name | csi-s3 | +| `storageClass.singleBucket` | Use a single bucket for all dynamically provisioned persistent volumes | | +| `storageClass.mountOptions` | GeeseFS mount options | `--memory-limit 1000 --dir-mode 0777 --file-mode 0666` | +| `storageClass.reclaimPolicy` | Volume reclaim policy | Delete | +| `storageClass.annotations` | Annotations for the storage class | | +| `secret.create` | Specifies whether the secret should be created | true | +| `secret.name` | Name of the secret | csi-s3-secret | +| `secret.accessKey` | S3 Access Key | | +| `secret.secretKey` | S3 Secret Key | | +| `secret.endpoint` | Endpoint | https://storage.yandexcloud.net | +| `tolerations.all` | Tolerate all taints by the CSI-S3 node driver (mounter) | false | +| `tolerations.node` | Custom tolerations for the CSI-S3 node driver (mounter) | [] | +| `tolerations.controller` | Custom tolerations for the CSI-S3 controller (provisioner) | [] | diff --git a/manifest.yaml b/manifest.yaml new file mode 100644 index 0000000..f0f0250 --- /dev/null +++ b/manifest.yaml @@ -0,0 +1,113 @@ +helm_chart: + name: cr.yandex/crp9ftr22d26age3hulg/yandex-cloud/csi-s3/csi-s3 + tag: 0.31.6 +requirements: + k8s_version: ">=1.13" +images: + - full: images.attacher + - full: images.registrar + - full: images.provisioner + - full: images.csi +user_values: + - name: storageClass.create + title: + en: Create storage class + ru: Создать класс хранения + description: + en: Specifies whether the storage class should be created + ru: 'Если "да", при установке будет создан класс хранения S3' + boolean_value: + default_value: true + - name: secret.create + title: + en: Create secret + ru: Создать секрет + description: + en: Specifies whether the secret should be created + ru: 'Если "да", при установке будет создан секрет, иначе для класса хранения будет использован существующий' + boolean_value: + default_value: true + - name: secret.accessKey + title: + en: S3 Access Key ID + ru: Идентификатор ключа S3 + description: + en: S3 Access Key ID + ru: Идентификатор ключа S3 + string_value: + default_value: "" + - name: secret.secretKey + title: + en: S3 Secret Key + ru: Секретный ключ S3 + description: + en: S3 Secret Key + ru: Секретный ключ S3 + string_value: + default_value: "" + - name: storageClass.singleBucket + title: + en: Single S3 bucket for volumes + ru: Общий S3 бакет для томов + description: + en: Single S3 bucket to use for all dynamically provisioned persistent volumes + ru: Общий бакет S3, в котором будут создаваться все динамически распределяемые тома. Если пусто, под каждый том будет создаваться новый бакет + string_value: + default_value: "" + - name: secret.endpoint + title: + en: S3 endpoint + ru: Адрес S3-сервиса + description: + en: S3 service endpoint to use + ru: Адрес S3-сервиса, который будет использоваться + string_value: + default_value: "https://storage.yandexcloud.net" + - name: storageClass.mountOptions + title: + en: GeeseFS mount options + ru: Опции монтирования GeeseFS + description: + en: GeeseFS mount options to use. Consult GeeseFS (https://github.com/yandex-cloud/geesefs) help for the full option list + ru: Опции монтирования GeeseFS. Смотрите справку GeeseFS (https://github.com/yandex-cloud/geesefs) для полного перечня опций + string_value: + default_value: "--memory-limit 1000 --dir-mode 0777 --file-mode 0666" + - name: storageClass.reclaimPolicy + title: + en: Volume reclaim policy + ru: Политика очистки томов + description: + en: Volume reclaim policy for the storage class (Retain or Delete) + ru: Политика очистки PV, связанных с PVC (Retain - сохранять при удалении PVC, Delete - удалять при удалении PVC) + string_selector_value: + default_value: Delete + values: + - Delete + - Retain + - name: storageClass.name + title: + en: Storage class name + ru: Название класса хранения + description: + en: Name of the storage class that will be created + ru: Название класса хранения, который будет создан при установке + string_value: + default_value: csi-s3 + - name: secret.name + title: + en: Name of the secret + ru: Название секрета + description: + en: Name of the secret to create or use for the storage class + ru: Название секрета, который будет создан или использован для класса хранения + string_value: + default_value: csi-s3-secret + - name: tolerations.all + title: + en: Tolerate all taints + ru: Игнорировать все taint-ы + description: + en: Tolerate all taints by the CSI-S3 node driver (mounter) + ru: Игнорировать все taint-ы узлов кластера драйвером CSI-S3, монтирующим ФС на узлах + boolean_value: + default_value: false diff --git a/templates/attacher.yaml b/templates/attacher.yaml new file mode 100644 index 0000000..c825406 --- /dev/null +++ b/templates/attacher.yaml @@ -0,0 +1,99 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-attacher-sa + namespace: {{ .Release.Namespace }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-attacher-runner +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-attacher-role +subjects: + - kind: ServiceAccount + name: csi-attacher-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: external-attacher-runner + apiGroup: rbac.authorization.k8s.io +--- +# needed for StatefulSet +kind: Service +apiVersion: v1 +metadata: + name: csi-attacher-s3 + namespace: {{ .Release.Namespace }} + labels: + app: csi-attacher-s3 +spec: + selector: + app: csi-attacher-s3 + ports: + - name: csi-s3-dummy + port: 65535 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-attacher-s3 + namespace: {{ .Release.Namespace }} +spec: + serviceName: "csi-attacher-s3" + replicas: 1 + selector: + matchLabels: + app: csi-attacher-s3 + template: + metadata: + labels: + app: csi-attacher-s3 + spec: + serviceAccount: csi-attacher-sa + tolerations: + - key: node-role.kubernetes.io/master + operator: "Exists" + containers: + - name: csi-attacher + image: {{ .Values.images.attacher }} + args: + - "--v=4" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /var/lib/kubelet/plugins/ru.yandex.s3.csi/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/ru.yandex.s3.csi + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/ru.yandex.s3.csi + type: DirectoryOrCreate diff --git a/templates/csi-s3.yaml b/templates/csi-s3.yaml new file mode 100644 index 0000000..9cf046f --- /dev/null +++ b/templates/csi-s3.yaml @@ -0,0 +1,133 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-s3 + namespace: {{ .Release.Namespace }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-s3 +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "update"] + - apiGroups: [""] + resources: ["namespaces"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "update"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-s3 +subjects: + - kind: ServiceAccount + name: csi-s3 + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: csi-s3 + apiGroup: rbac.authorization.k8s.io +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-s3 + namespace: {{ .Release.Namespace }} +spec: + selector: + matchLabels: + app: csi-s3 + template: + metadata: + labels: + app: csi-s3 + spec: + tolerations: + {{- if .Values.tolerations.all }} + - operator: Exists + {{- else }} + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + effect: NoExecute + tolerationSeconds: 300 + {{- end }} + {{- with .Values.tolerations.node }} + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccount: csi-s3 + hostNetwork: true + containers: + - name: driver-registrar + image: {{ .Values.images.registrar }} + args: + - "--kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)" + - "--v=4" + - "--csi-address=$(ADDRESS)" + env: + - name: ADDRESS + value: /csi/csi.sock + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/ru.yandex.s3.csi/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration/ + - name: csi-s3 + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: {{ .Values.images.csi }} + imagePullPolicy: IfNotPresent + args: + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(NODE_ID)" + - "--v=4" + env: + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: plugin-dir + mountPath: /csi + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - name: fuse-device + mountPath: /dev/fuse + volumes: + - name: registration-dir + hostPath: + path: /var/lib/kubelet/plugins_registry/ + type: DirectoryOrCreate + - name: plugin-dir + hostPath: + path: /var/lib/kubelet/plugins/ru.yandex.s3.csi + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - name: fuse-device + hostPath: + path: /dev/fuse diff --git a/templates/provisioner.yaml b/templates/provisioner.yaml new file mode 100644 index 0000000..46d0bb8 --- /dev/null +++ b/templates/provisioner.yaml @@ -0,0 +1,115 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-provisioner-sa + namespace: {{ .Release.Namespace }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: external-provisioner-runner +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: csi-provisioner-role +subjects: + - kind: ServiceAccount + name: csi-provisioner-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: external-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- +kind: Service +apiVersion: v1 +metadata: + name: csi-provisioner-s3 + namespace: {{ .Release.Namespace }} + labels: + app: csi-provisioner-s3 +spec: + selector: + app: csi-provisioner-s3 + ports: + - name: csi-s3-dummy + port: 65535 +--- +kind: StatefulSet +apiVersion: apps/v1 +metadata: + name: csi-provisioner-s3 + namespace: {{ .Release.Namespace }} +spec: + serviceName: "csi-provisioner-s3" + replicas: 1 + selector: + matchLabels: + app: csi-provisioner-s3 + template: + metadata: + labels: + app: csi-provisioner-s3 + spec: + serviceAccount: csi-provisioner-sa + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - operator: Exists + effect: NoExecute + tolerationSeconds: 300 + {{- with .Values.tolerations.controller }} + {{- toYaml . | nindent 8 }} + {{- end }} + containers: + - name: csi-provisioner + image: {{ .Values.images.provisioner }} + args: + - "--csi-address=$(ADDRESS)" + - "--v=4" + env: + - name: ADDRESS + value: /var/lib/kubelet/plugins/ru.yandex.s3.csi/csi.sock + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/ru.yandex.s3.csi + - name: csi-s3 + image: {{ .Values.images.csi }} + imagePullPolicy: IfNotPresent + args: + - "--endpoint=$(CSI_ENDPOINT)" + - "--nodeid=$(NODE_ID)" + - "--v=4" + env: + - name: CSI_ENDPOINT + value: unix:///var/lib/kubelet/plugins/ru.yandex.s3.csi/csi.sock + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /var/lib/kubelet/plugins/ru.yandex.s3.csi + volumes: + - name: socket-dir + emptyDir: {} diff --git a/templates/secret.yaml b/templates/secret.yaml new file mode 100644 index 0000000..dd6a748 --- /dev/null +++ b/templates/secret.yaml @@ -0,0 +1,11 @@ +{{- if .Values.secret.create -}} +apiVersion: v1 +kind: Secret +metadata: + namespace: {{ .Release.Namespace }} + name: {{ .Values.secret.name }} +stringData: + accessKeyID: {{ .Values.secret.accessKey }} + secretAccessKey: {{ .Values.secret.secretKey }} + endpoint: {{ .Values.secret.endpoint }} +{{- end -}} diff --git a/templates/storageclass.yaml b/templates/storageclass.yaml new file mode 100644 index 0000000..57758a3 --- /dev/null +++ b/templates/storageclass.yaml @@ -0,0 +1,26 @@ +{{- if .Values.storageClass.create -}} +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: {{ .Values.storageClass.name }} +{{- if .Values.storageClass.annotations }} + annotations: +{{ toYaml .Values.storageClass.annotations | indent 4 }} +{{- end }} +provisioner: ru.yandex.s3.csi +parameters: + mounter: geesefs + options: "{{ .Values.storageClass.mountOptions }}" +{{- if .Values.storageClass.singleBucket }} + bucket: "{{ .Values.storageClass.singleBucket }}" +{{- end }} + csi.storage.k8s.io/provisioner-secret-name: {{ .Values.secret.name }} + csi.storage.k8s.io/provisioner-secret-namespace: {{ .Release.Namespace }} + csi.storage.k8s.io/controller-publish-secret-name: {{ .Values.secret.name }} + csi.storage.k8s.io/controller-publish-secret-namespace: {{ .Release.Namespace }} + csi.storage.k8s.io/node-stage-secret-name: {{ .Values.secret.name }} + csi.storage.k8s.io/node-stage-secret-namespace: {{ .Release.Namespace }} + csi.storage.k8s.io/node-publish-secret-name: {{ .Values.secret.name }} + csi.storage.k8s.io/node-publish-secret-namespace: {{ .Release.Namespace }} +reclaimPolicy: {{ .Values.storageClass.reclaimPolicy }} +{{- end -}} diff --git a/values-overrides.yaml b/values-overrides.yaml new file mode 100644 index 0000000..7559db8 --- /dev/null +++ b/values-overrides.yaml @@ -0,0 +1,44 @@ +--- +images: + # Source: quay.io/k8scsi/csi-attacher:v3.0.1 + attacher: hub.wodcloud.com/wod/csi-attacher:v3.0.1 + # Source: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0 + registrar: hub.wodcloud.com/wod/csi-node-driver-registrar:v1.2.0 + # Source: quay.io/k8scsi/csi-provisioner:v2.1.0 + provisioner: hub.wodcloud.com/wod/csi-provisioner:v2.1.0 + # Main image + csi: hub.wodcloud.com/wod/csi-s3-driver:0.31.6 + +storageClass: + # Specifies whether the storage class should be created + create: true + # Name + name: csi-s3 + # Use a single bucket for all dynamically provisioned persistent volumes + singleBucket: "test" + # GeeseFS mount options + mountOptions: "--memory-limit 1000 --dir-mode 0777 --file-mode 0666" + # Volume reclaim policy + reclaimPolicy: Delete + # Annotations for the storage class + # Example: + # annotations: + # storageclass.kubernetes.io/is-default-class: "true" + annotations: {} + +secret: + # Specifies whether the secret should be created + create: true + # Name of the secret + name: csi-s3-secret + # S3 Access Key + accessKey: "console" + # S3 Secret Key + secretKey: "console123" + # Endpoint + endpoint: http://103.81.5.56:33070 + +tolerations: + all: false + node: [] + controller: [] diff --git a/values.yaml b/values.yaml new file mode 100644 index 0000000..018fa13 --- /dev/null +++ b/values.yaml @@ -0,0 +1,44 @@ +--- +images: + # Source: quay.io/k8scsi/csi-attacher:v3.0.1 + attacher: cr.yandex/crp9ftr22d26age3hulg/yandex-cloud/csi-s3/csi-attacher:v3.0.1 + # Source: quay.io/k8scsi/csi-node-driver-registrar:v1.2.0 + registrar: cr.yandex/crp9ftr22d26age3hulg/yandex-cloud/csi-s3/csi-node-driver-registrar:v1.2.0 + # Source: quay.io/k8scsi/csi-provisioner:v2.1.0 + provisioner: cr.yandex/crp9ftr22d26age3hulg/yandex-cloud/csi-s3/csi-provisioner:v2.1.0 + # Main image + csi: cr.yandex/crp9ftr22d26age3hulg/yandex-cloud/csi-s3/csi-s3-driver:0.31.6 + +storageClass: + # Specifies whether the storage class should be created + create: true + # Name + name: csi-s3 + # Use a single bucket for all dynamically provisioned persistent volumes + singleBucket: "" + # GeeseFS mount options + mountOptions: "--memory-limit 1000 --dir-mode 0777 --file-mode 0666" + # Volume reclaim policy + reclaimPolicy: Delete + # Annotations for the storage class + # Example: + # annotations: + # storageclass.kubernetes.io/is-default-class: "true" + annotations: {} + +secret: + # Specifies whether the secret should be created + create: true + # Name of the secret + name: csi-s3-secret + # S3 Access Key + accessKey: "" + # S3 Secret Key + secretKey: "" + # Endpoint + endpoint: https://storage.yandexcloud.net + +tolerations: + all: false + node: [] + controller: [] -- 2.26.0