小炒肉
文章 标签 分类 关于
小炒肉

目录

NFS - StorageClass

小炒肉 收录于 kubernetes
   约 1392 字   预计阅读 3 分钟    次阅读  

基于 StorageClass 的 NFS 动态卷

基本信息

1
2
3
4
5
6

172.16.1.64  K8s-Master and node

172.16.1.65  K8s-Master and node

172.16.1.66  K8s-Master and node and NFS Server

NFS 服务

1
2
3

# 安装 NFS

yum -y install nfs-utils rpcbind

1
2
3
4

# k8s 所有节点 安装 NFS 客户端

yum -y install nfs-utils

配置 NFS 目录与权限

1
2
3
4
5
6

vi /etc/exports

增加

/opt/nfsdata   172.16.1.0/24(rw,sync,no_root_squash)

启动 NFS 服务

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13

systemctl enable rpcbind.service    
systemctl enable nfs-server.service

systemctl start rpcbind.service    
systemctl start nfs-server.service


# 查看信息

showmount -e 172.16.1.66

Export list for 172.16.1.66:
/opt/nfsdata 172.16.1.0/24

配置 NFS Client Provisioner

1
2
3
4
5
6
7

# 官网镜像地址
quay.io/external_storage/nfs-client-provisioner:latest


# 个人镜像地址

jicki/nfs-client-provisioner:latest

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42

# 配置一个 rbac.yaml

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-client-provisioner
  
---

kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["list", "watch", "create", "update", "patch"]

---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-client-provisioner
subjects:
  - kind: ServiceAccount
    name: nfs-client-provisioner
    namespace: default
roleRef:
  kind: ClusterRole
  name: nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io


 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

# 配置一个 deployment 服务


kind: Deployment
apiVersion: extensions/v1beta1
metadata:
  name: nfs-client-provisioner
spec:
  replicas: 1
  strategy:
    type: Recreate
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccount: nfs-client-provisioner
      containers:
        - name: nfs-client-provisioner
          image: jicki/nfs-client-provisioner:latest
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes
          env:
            - name: PROVISIONER_NAME
              value: fuseim.pri/ifs
            - name: NFS_SERVER
              value: 172.16.1.66
            - name: NFS_PATH
              value: /opt/nfsdata
      volumes:
        - name: nfs-client-root
          nfs:
            server: 172.16.1.66
            path: /opt/nfsdata

创建 服务

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12

kubectl apply -f .
serviceaccount "nfs-client-provisioner" created
clusterrole "nfs-client-provisioner-runner" created
clusterrolebinding "run-nfs-client-provisioner" created
deployment "nfs-client-provisioner" created


# 查看服务

kubectl get pods |grep nfs
nfs-client-provisioner-8cdb56f4d-l8vmr   1/1       Running   0          26s

创建 StorageClass

1
2
3
4
5
6
7
8

# nfs-storageclass

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: nfs-storage 
provisioner: fuseim.pri/ifs  # fuseim.pri/ifs 是 nfs-client-provisioner 服务中的一个 env


 1
 2
 3
 4
 5
 6
 7
 8
 9
10

# 导入文件
kubectl apply -f nfs-storageclass.yaml 
storageclass "nfs-storage" created


#  查看服务
kubectl get storageclass
NAME          PROVISIONER
nfs-storage   fuseim.pri/ifs

测试

创建一个 nginx StatefulSet

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31

# nginx-statefulset

apiVersion: apps/v1beta1
kind: StatefulSet
metadata:
  name: web
spec:
  serviceName: "nginx"
  replicas: 2
  volumeClaimTemplates:
  - metadata:
      name: html 
      annotations:
        volume.beta.kubernetes.io/storage-class: "nfs-storage" # 这里配置 上面创建的 storageclass 的名称
    spec:
      accessModes: [ "ReadWriteOnce" ]
      resources:
        requests:
          storage: 2Gi 
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: nginx:alpine
        volumeMounts:
        - mountPath: "/usr/share/nginx/html/"
          name: html


 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19

# 导入nginx-statefulset
kubectl apply -f nginx-statefulset.yaml 
statefulset "web" created


# 查看服务
kubectl get pods|grep web
web-0                                    1/1       Running   0          1m
web-1                                    1/1       Running   0          1m



# 查看 pvc

kubectl get pvc
NAME         STATUS    VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
html-web-0   Bound     pvc-bb0c0ada-b3aa-11e7-b194-80d4a5d413e2   2Gi        RWO            nfs-storage    1m
html-web-1   Bound     pvc-bc3478ac-b3aa-11e7-b194-80d4a5d413e2   2Gi        RWO            nfs-storage    1m

验证 测试 挂载

缩容 测试

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82

# 查看服务

kubectl get pods|grep web
web-0                                    1/1       Running   0          1m
web-1                                    1/1       Running   0          1m



# 查看 nfs-server 中的目录

ll
总用量 8
drwxrwxrwx 2 root root 4096 10月 18 10:19 default-html-web-0-pvc-bb0c0ada-b3aa-11e7-b194-80d4a5d413e2
drwxrwxrwx 2 root root 4096 10月 18 10:19 default-html-web-1-pvc-bc3478ac-b3aa-11e7-b194-80d4a5d413e2



# 写入文件到挂载目录中

kubectl exec -it web-1 -- touch /usr/share/nginx/html/jicki.txt


kubectl exec -it web-1 -- ls -lt /usr/share/nginx/html
total 0
-rw-r--r--    1 root     root             0 Oct 18 02:24 jicki.txt


# nfs server 中的文件

ls default-html-web-1-pvc-bc3478ac-b3aa-11e7-b194-80d4a5d413e2/
jicki.txt


# 收缩  pod = 1个 既 缩掉 web-1

kubectl scale statefulset web --replicas=1
statefulset "web" scaled



# 查看 pod 
kubectl get pods |grep web
web-0                                    1/1       Running   0          7m


# 查看 pvc (发现 web-1 的 pvc 仍然在, 而且是 Bound 状态)

kubectl get pvc
NAME         STATUS    VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
html-web-0   Bound     pvc-bb0c0ada-b3aa-11e7-b194-80d4a5d413e2   2Gi        RWO            nfs-storage    8m
html-web-1   Bound     pvc-bc3478ac-b3aa-11e7-b194-80d4a5d413e2   2Gi        RWO            nfs-storage    8m



# 恢复 web-1

kubectl scale statefulset web --replicas=2
statefulset "web" scaled


# 查看服务

kubectl get pods |grep web
web-0                                    1/1       Running   0          13m
web-1                                    1/1       Running   0          22s


# 查看pvc 

kubectl get pvc
NAME         STATUS    VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
html-web-0   Bound     pvc-bb0c0ada-b3aa-11e7-b194-80d4a5d413e2   2Gi        RWO            nfs-storage    13m
html-web-1   Bound     pvc-bc3478ac-b3aa-11e7-b194-80d4a5d413e2   2Gi        RWO            nfs-storage    13m


# 查看之前创建的文件 ( 发现恢复了 )


[root@k8s-master-64 yaml]# kubectl exec -it web-1 -- ls -lt /usr/share/nginx/html
total 0
-rw-r--r--    1 root     root             0 Oct 18 02:24 jicki.txt

删除 statefulset 测试

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40

# delete 掉

 kubectl delete -f nginx-statefulset.yaml 
statefulset "web" deleted


# 查看 statefulset

kubectl get statefulset
No resources found.


# 查看 pvc (发现 仍然 存在)

kubectl get pvc
NAME         STATUS    VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
html-web-0   Bound     pvc-bb0c0ada-b3aa-11e7-b194-80d4a5d413e2   2Gi        RWO            nfs-storage    16m
html-web-1   Bound     pvc-bc3478ac-b3aa-11e7-b194-80d4a5d413e2   2Gi        RWO            nfs-storage    16m


# nfs server 中的文件 (数据仍然存在)

ls default-html-web-1-pvc-bc3478ac-b3aa-11e7-b194-80d4a5d413e2/
jicki.txt


# 重新创建 相同名称的 statefulset

kubectl get pods |grep web
web-0                                    1/1       Running   0          3m
web-1                                    1/1       Running   0          3m


# 查看 文件，可以看到 文件仍然存在

kubectl exec -it web-1 -- ls -lt /usr/share/nginx/html
total 0
-rw-r--r--    1 root     root             0 Oct 18 02:24 jicki.txt

删除 pvc 测试

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36

# 收缩 replicas = 1 

kubectl scale statefulsets web --replicas=1

# 查看 pvc

 kubectl get pvc
NAME         STATUS    VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS   AGE
html-web-0   Bound     pvc-bb0c0ada-b3aa-11e7-b194-80d4a5d413e2   2Gi        RWO            nfs-storage    22h
html-web-1   Bound     pvc-bc3478ac-b3aa-11e7-b194-80d4a5d413e2   2Gi        RWO            nfs-storage    22h


# 删除 pvc 

kubectl delete pvc/html-web-1
persistentvolumeclaim "html-web-1" deleted


# nfs server (原目录更改为 archived 开头)

archived-default-html-web-1-pvc-bc3478ac-b3aa-11e7-b194-80d4a5d413e2
default-html-web-0-pvc-bb0c0ada-b3aa-11e7-b194-80d4a5d413e2


# 重新扩容 replicas = 2

kubectl scale statefulsets web --replicas=2
statefulset "web" scaled


# nfs server (生成新的目录)


archived-default-html-web-1-pvc-bc3478ac-b3aa-11e7-b194-80d4a5d413e2  default-html-web-1-pvc-4a64a8dc-b469-11e7-b194-80d4a5d413e2
default-html-web-0-pvc-bb0c0ada-b3aa-11e7-b194-80d4a5d413e2
更新于 2017-10-18
阅读原始文档
返回 | 主页