目录
一.storageclass产生背景
二.storageClass的具体使用
1.创建NFS共享目录和服务
2.创建Service Account来管控NFS provisioner在k8s集群中运行的权限
3.创建StorageClass来建立PVC并调用NFS provisioner进行预定的工作
4..创建NFS provisioner来共享NFS并建立PV 将PV与NFS的挂载点产生关联
5.创建PVC看看是否自动创建PV实现两者绑定并创建Pod测试结果
6.statefulset+headless service+volumeClaimTemplates自动建立PV使用
一.storageclass产生背景
1.主要是为了根据用户提出的资源申请去找更为合适的存储资源。通过Dynamic Provisioning这个机制可以实现自动创建PV资源，他的的核心在于StorageClass， StorageClass会定义PV的存储类型,Volume大小等。Kubernetes根据用户提交的PVC请求,找到一个对应的StorageClass, Kubernetes就会调用该StorageClass声明的存储插件,创建出需要的PV供存储使用。
2.在此过程中，需要用到nfs-client自动配置程序，Provisioner，通过他来自动创建PV。
创建出来的PV时以namespace-pvcname-pvname的格式存储于NFS共享目录中，PV被回收时会被标记为archived-namespace-pvcname-pvname。
二.storageClass的具体使用
1.创建NFS共享目录和服务
[root@k8s-master storageclass]# ll
total 20
-rw-r--r-- 1 root root 1025 Mar 8 21:40 provisioner.yaml
-rw-r--r-- 1 root root 1524 Mar 8 21:28 rbac.yaml
drwxr-xr-x 3 root root 71 Mar 9 11:31 share #将此目录共享出去
-rw-r--r-- 1 root root 151 Mar 8 21:31 storageclass-nfs.yaml
-rw-r--r-- 1 root root 354 Mar 9 11:41 test-pod.yaml
-rw-r--r-- 1 root root 261 Mar 9 11:31 test-pvc.yaml
[root@k8s-master storageclass]# pwd
/root/storageclass
[root@k8s-master storageclass]# cat /etc/exports
#/root/pv/pv1 192.168.2.0/24(rw,no_root_squash)
#/root/pv/pv2 192.168.2.0/24(rw,no_root_squash)
/root/storageclass/share 192.168.2.0/24(rw,no_root_squash) #如此配置2.创建Service Account来管控NFS provisioner在k8s集群中运行的权限
链接：https://pan.baidu.com/s/1uBc3Stcc6Gq9cMC3Rt90tw
提取码：v2np
[root@k8s-master storageclass]# cat rbac.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
name: nfs-client-provisioner
namespace: myns #名称空间需要自己创建，后面的都更改为自己实际的namespace，其余基本不需要修改
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: nfs-client-provisioner-runner
rules:
- apiGroups: [""]
resources: ["persistentvolumes"]
verbs: ["get", "list", "watch", "create", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "update"]
- apiGroups: ["storage.k8s.io"]
resources: ["storageclasses"]
verbs: ["get", "list", "watch"]
- apiGroups: [""]
resources: ["events"]
verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: run-nfs-client-provisioner
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
namespace: myns
roleRef:
kind: ClusterRole
name: nfs-client-provisioner-runner
apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
namespace: myns
rules:
- apiGroups: [""]
resources: ["endpoints"]
verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: leader-locking-nfs-client-provisioner
namespace: myns
subjects:
- kind: ServiceAccount
name: nfs-client-provisioner
namespace: myns
roleRef:
kind: Role
name: leader-locking-nfs-client-provisioner
apiGroup: rbac.authorization.k8s.io
[root@k8s-master storageclass]# kubectl apply -f rbac.yaml
[root@k8s-master storageclass]# kubectl get role,rolebinding -n myns
NAME CREATED AT
role.rbac.authorization.k8s.io/leader-locking-nfs-client-provisioner 2024-03-09T03:26:18Z
NAME ROLE AGE
rolebinding.rbac.authorization.k8s.io/leader-locking-nfs-client-provisioner Role/leader-locking-nfs-client-provisioner 26m3.创建StorageClass来建立PVC并调用NFS provisioner进行预定的工作
链接：https://pan.baidu.com/s/13dVlD5Dyb3B0Ef1cJDZHTQ
提取码：wwwj
[root@k8s-master storageclass]# cat storageclass-nfs.yaml
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
name: managed-nfs-storage #此处的值和下面一行的值要明确，后面provisioner文件需要用到
provisioner: nfs-storage
parameters:
archiveOnDelete: "false"
[root@k8s-master storageclass]# kubectl apply -f storageclass-nfs.yaml
[root@k8s-master storageclass]# kubectl get sc -n myns
NAME PROVISIONER RECLAIMPOLICY VOLUMEBINDINGMODE ALLOWVOLUMEEXPANSION AGE
managed-nfs-storage nfs-storage Delete Immediate false 27m4..创建NFS provisioner来共享NFS并建立PV 将PV与NFS的挂载点产生关联
链接：https://pan.baidu.com/s/1u9zxreO-qHniowfkptV4QQ
提取码：w4wt
[root@k8s-master storageclass]# cat provisioner.yaml
apiVersion: apps/v1
kind: Deployment
metadata:
name: nfs-client-provisioner
namespace: myns #名称空间也要一致
labels:
app: nfs-client-provisioner
spec:
replicas: 1
selector:
matchLabels:
app: nfs-client-provisioner
strategy:
type: Recreate
selector:
matchLabels:
app: nfs-client-provisioner
template:
metadata:
labels:
app: nfs-client-provisioner
spec:
serviceAccountName: nfs-client-provisioner
containers:
- name: nfs-client-provisioner
image: registry.cn-beijing.aliyuncs.com/mydlq/nfs-subdir-external-provisioner:v4.0.0
volumeMounts:
- name: nfs-client-root
mountPath: /persistentvolumes
env:
- name: PROVISIONER_NAME
value: nfs-storage #和storageclass-nfs中的provisioner一致
- name: NFS_SERVER
value: 192.168.2.150 #此处和后面的值都要修改为自己共享出来的nfs目录的真实情况
- name: NFS_PATH
value: /root/storageclass/share
volumes:
- name: nfs-client-root
nfs:
server: 192.168.2.150
path: /root/storageclass/share
[root@k8s-master storageclass]# kubectl apply -f provisioner.yaml
[root@k8s-master storageclass]# kubectl get deploy,pod -n myns
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/nfs-client-provisioner 1/1 1 1 29m
NAME READY STATUS RESTARTS AGE
pod/nfs-client-provisioner-7c5cccbf84-6q4n8 1/1 Running 0 29m5.创建PVC看看是否自动创建PV实现两者绑定并创建Pod测试结果
（1）pvc
[root@k8s-master storageclass]# cat test-pvc.yaml
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
name: my-test-pvc
namespace: myns
#annotations: #老版本用annotations
#volume.beta.kubernetes.io/storage-class: "managed-nfs-storage"
spec:
storageClassName: "managed-nfs-storage"
#高一点的版本应该都用了storageClassName来写，与storageclass-nfs的name一致，产生关联，与PV相匹配
accessModes:
- ReadWriteMany
resources:
requests:
storage: 10M
[root@k8s-master storageclass]# kubectl apply -f test-pvc.yaml
[root@k8s-master storageclass]# kubectl get pv,pvc -n myns #如下，pv和pvc已经完成申请并绑定
NAME CAPACITY ACCESS MODES RECLAIM POLICY STATUS CLAIM STORAGECLASS REASON AGE
persistentvolume/pvc-a3d9105a-c3aa-4bd9-a9ea-8596ccd537cf 10M RWX Delete Bound myns/my-test-pvc managed-nfs-storage 25m
NAME STATUS VOLUME CAPACITY ACCESS MODES STORAGECLASS AGE
persistentvolumeclaim/my-test-pvc Bound pvc-a3d9105a-c3aa-4bd9-a9ea-8596ccd537cf 10M RWX managed-nfs-storage 25m（2）Pod创建和删除测试
[root@k8s-master storageclass]# cat test-pod.yaml #接下来就创建一个pod进行测试是否将文件存储出来了
apiVersion: v1
kind: Pod
metadata:
name: my-test-pod
namespace: myns
spec:
containers:
- name: my-test-busybox
image: busybox
command: ["/bin/sh","-c","touch /root/a.txt && exit"] #创建好/root/a.txt就退出
volumeMounts:
- name: test-nfs-pvc
mountPath: "/root" #挂载到镜像中的/root
volumes:
- name: test-nfs-pvc
persistentVolumeClaim:
claimName: my-test-pvc #使用刚才创建的PVC
[root@k8s-master storageclass]# kubectl get pods -n myns
NAME READY STATUS RESTARTS AGE
my-test-pod 0/1 CrashLoopBackOff 8 (82s ago) 17m #创建完成即结束了任务
nfs-client-provisioner-7c5cccbf84-6q4n8 1/1 Running 0 32m
[root@k8s-master storageclass]# tree share/
share/
└── myns-my-test-pvc-pvc-a3d9105a-c3aa-4bd9-a9ea-8596ccd537cf
└── a.txt #成功存储出来了
1 directory, 1 file
[root@k8s-master storageclass]# kubectl delete pod my-test-pod -n myns
#文件不受Pod删除影响
pod "my-test-pod" deleted
share/
└── myns-my-test-pvc-pvc-a3d9105a-c3aa-4bd9-a9ea-8596ccd537cf
└── a.txt
1 directory, 1 file6.statefulset+headless service+volumeClaimTemplates自动建立PV使用
（1）创建无头服务和statefulset
[root@k8s-master storageclass]# cat test-headless.yaml #创建无头服务
apiVersion: v1
kind: Service
metadata:
name: test-headless
namespace: myns
labels:
app: nginx
spec:
ports:
- port: 80
name: nginx-web
clusterIP: None #无头服务关键
selector:
app: nginx
---
apiVersion: apps/v1
kind: StatefulSet #创建statefulset
metadata:
name: nginx-web
namespace: myns
spec:
selector:
matchLabels:
app: nginx
serviceName: "test-headless"
replicas: 2
template:
metadata:
labels:
app: nginx
spec:
containers:
- name: nginx
image: nginx
ports:
- containerPort: 80
name: nginx-web
volumeMounts:
- name: www
mountPath: /usr/share/nginx/html #将nginx的html目录挂载给storageClass的NFS共享目录
volumeClaimTemplates:
- metadata:
name: www
annotations:
volume.beta.kubernetes.io/storage-class: "managed-nfs-storage"
spec:
accessModes: [ "ReadWriteOnce" ]
resources:
requests:
storage: 1G
[root@k8s-master storageclass]# kubectl get svc,pod -n myns -o wide
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
service/test-headless ClusterIP None <none> 80/TCP 124m app=nginx
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
pod/nfs-client-provisioner-7c5cccbf84-6q4n8 1/1 Running 0 170m 10.244.107.193 k8s-node3 <none> <none>
pod/nginx-web-0 1/1 Running 0 117m 10.244.36.66 k8s-node1 <none> <none>
pod/nginx-web-1 1/1 Running 0 117m 10.244.169.131 k8s-node2 <none> <none>（2）测试
#写入内容测试
[root@k8s-master storageclass]# echo hello web0 > share/myns-www-nginx-web-0-pvc-d2efdacb-2333-4306-a83d-df9b9b5e5dc5/index.html
[root@k8s-master storageclass]# echo hello web1 > share/myns-www-nginx-web-1-pvc-386882a8-dcf8-497e-943d-14e769406eab/index.html
[root@k8s-master storageclass]# tree share/
share/
├── myns-my-test-pvc-pvc-a3d9105a-c3aa-4bd9-a9ea-8596ccd537cf
│ └── a.txt
├── myns-www-nginx-web-0-pvc-d2efdacb-2333-4306-a83d-df9b9b5e5dc5
│ └── index.html
└── myns-www-nginx-web-1-pvc-386882a8-dcf8-497e-943d-14e769406eab
└── index.html
[root@k8s-master storageclass]# kubectl exec -it nginx-web-0 -n myns -- /bin/sh -c "cat /etc/resolv.conf"
search myns.svc.cluster.local svc.cluster.local cluster.local #记录的DNS
nameserver 10.96.0.10
options ndots:5
[root@k8s-master storageclass]# nslookup test-headless.myns.svc.cluster.local 10.96.0.10 #通过记录的DNS就可以解析出来两个Pod对应的IP地址并进行访问了
Server: 10.96.0.10
Address: 10.96.0.10#53
Name: test-headless.myns.svc.cluster.local
Address: 10.244.169.131
Name: test-headless.myns.svc.cluster.local
Address: 10.244.36.66
[root@k8s-master storageclass]# dig @10.96.0.10 test-headless.myns.svc.cluster.local
#dig也可以解析
; <<>> DiG 9.11.4-P2-RedHat-9.11.4-26.P2.el7_9.15 <<>> @10.96.0.10 test-headless.myns.svc.cluster.local
; (1 server found)
;; global options: +cmd
;; Got answer:
;; WARNING: .local is reserved for Multicast DNS
;; You are currently testing what happens when an mDNS query is leaked to DNS
;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 366
;; flags: qr aa rd; QUERY: 1, ANSWER: 2, AUTHORITY: 0, ADDITIONAL: 1
;; WARNING: recursion requested but not available
;; OPT PSEUDOSECTION:
; EDNS: version: 0, flags:; udp: 4096
;; QUESTION SECTION:
;test-headless.myns.svc.cluster.local. IN A
;; ANSWER SECTION:
test-headless.myns.svc.cluster.local. 30 IN A 10.244.169.131
test-headless.myns.svc.cluster.local. 30 IN A 10.244.36.66
;; Query time: 0 msec
;; SERVER: 10.96.0.10#53(10.96.0.10)
;; WHEN: Sat Mar 09 14:21:32 CST 2024
;; MSG SIZE rcvd: 169
[root@k8s-node1 ~]# curl 10.244.36.66
hello web0
[root@k8s-node2 ~]# curl 10.244.169.131
hello web1