集群节点伸缩管理
添加Master
- kubeasz部署的集群每个node上都有一个用nginx作为LB的组件
[root@K8s-noded01 ~]#cat /etc/kube-lb/conf/kube-lb.conf
user root;
worker_processes 1;
error_log /etc/kube-lb/logs/error.log warn;
events {
worker_connections 3000;
}
stream {
upstream backend {
server 192.168.11.211:6443 max_fails=2 fail_timeout=3s;
server 192.168.11.212:6443 max_fails=2 fail_timeout=3s;
}
server {
listen 127.0.0.1:6443;
proxy_connect_timeout 1s;
proxy_pass backend;
}
}- 进行Master伸缩变更时，nginx配置文件也会自动更新
#查看变更前的md5值
[root@K8s-noded01 ~]#ls -l /etc/kube-lb/conf/kube-lb.conf
-rw-r--r-- 1 root root 407 Mar 26 13:45 /etc/kube-lb/conf/kube-lb.conf
[root@K8s-noded01 ~]#md5sum /etc/kube-lb/conf/kube-lb.conf
547a406d41656f940f37f3546420087f /etc/kube-lb/conf/kube-lb.conf
#在部署节点进行Master节点的横向扩展
[root@K8s-ansible kubeasz]#./ezctl add-master k8s-cluster1 192.168.11.213
2023-03-27 11:50:27 INFO add 192.168.11.213 into 'kube_master' group
2023-03-27 11:50:27 INFO start to add a master node:192.168.11.213 into cluster:k8s-cluster1
PLAY [192.168.11.213] ************************************************************************************************************************************************************
TASK [Gathering Facts] ***********************************************************************************************************************************************************
ok: [192.168.11.213]
TASK [prepare : apt更新缓存刷新] *******************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [prepare : 删除ubuntu默认安装] ****************************************************************************************************************************************************
changed: [192.168.11.213] => (item=ufw)
changed: [192.168.11.213] => (item=lxd)
changed: [192.168.11.213] => (item=lxcfs)
changed: [192.168.11.213] => (item=lxc-common)
TASK [prepare : 安装 ubuntu/debian基础软件] ********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [prepare : 准备 journal 日志相关目录] ***********************************************************************************************************************************************
changed: [192.168.11.213] => (item=/etc/systemd/journald.conf.d)
ok: [192.168.11.213] => (item=/var/log/journal)
TASK [prepare : 优化设置 journal 日志] *************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [prepare : 重启 journald 服务] **************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [prepare : 禁用系统 swap] *******************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [prepare : 删除fstab swap 相关配置] ***********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [prepare : 转换内核版本为浮点数] ******************************************************************************************************************************************************
ok: [192.168.11.213]
TASK [prepare : 加载内核模块] **********************************************************************************************************************************************************
changed: [192.168.11.213] => (item=br_netfilter)
changed: [192.168.11.213] => (item=ip_vs)
changed: [192.168.11.213] => (item=ip_vs_rr)
changed: [192.168.11.213] => (item=ip_vs_wrr)
changed: [192.168.11.213] => (item=ip_vs_sh)
ok: [192.168.11.213] => (item=nf_conntrack)
TASK [prepare : 尝试加载nf_conntrack_ipv4] *******************************************************************************************************************************************
changed: [192.168.11.213]
TASK [prepare : 启用systemd自动加载模块服务] ***********************************************************************************************************************************************
ok: [192.168.11.213]
TASK [prepare : 增加内核模块开机加载配置] ****************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [prepare : 设置系统参数] **********************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [prepare : 生效系统参数] **********************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [prepare : 创建 systemd 配置目录] *************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [prepare : 设置系统 ulimits] ****************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [prepare : 把SCTP列入内核模块黑名单] **************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [prepare : prepare some dirs] ***********************************************************************************************************************************************
ok: [192.168.11.213] => (item=/usr/local/bin)
changed: [192.168.11.213] => (item=/etc/kubernetes/ssl)
changed: [192.168.11.213] => (item=/root/.kube)
changed: [192.168.11.213] => (item=/etc/cni/net.d)
TASK [prepare : symlink /usr/bin/python -> /usr/bin/python3] *********************************************************************************************************************
changed: [192.168.11.213]
TASK [prepare : 写入环境变量$PATH] *****************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [prepare : 添加 local registry hosts 解析] **************************************************************************************************************************************
changed: [192.168.11.213]
TASK [containerd : 获取是否已经安装containerd] *******************************************************************************************************************************************
changed: [192.168.11.213]
TASK [containerd : 准备containerd相关目录] *********************************************************************************************************************************************
ok: [192.168.11.213] => (item=/usr/local/bin)
changed: [192.168.11.213] => (item=/etc/containerd)
TASK [containerd : 加载内核模块 overlay] ***********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [containerd : 下载 containerd 二进制文件] ******************************************************************************************************************************************
changed: [192.168.11.213] => (item=containerd)
changed: [192.168.11.213] => (item=containerd-shim)
changed: [192.168.11.213] => (item=containerd-shim-runc-v1)
changed: [192.168.11.213] => (item=containerd-shim-runc-v2)
changed: [192.168.11.213] => (item=crictl)
changed: [192.168.11.213] => (item=ctr)
changed: [192.168.11.213] => (item=runc)
TASK [containerd : 添加 crictl 自动补全] ***********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [containerd : 创建 containerd 配置文件] *******************************************************************************************************************************************
changed: [192.168.11.213]
TASK [containerd : 创建systemd unit文件] *********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [containerd : 创建 crictl 配置] *************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [containerd : 开机启用 containerd 服务] *******************************************************************************************************************************************
changed: [192.168.11.213]
TASK [containerd : 开启 containerd 服务] *********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [containerd : 轮询等待containerd服务运行] *******************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-lb : prepare some dirs] ***********************************************************************************************************************************************
changed: [192.168.11.213] => (item=/etc/kube-lb/sbin)
changed: [192.168.11.213] => (item=/etc/kube-lb/logs)
changed: [192.168.11.213] => (item=/etc/kube-lb/conf)
TASK [kube-lb : 下载二进制文件kube-lb(nginx)] *******************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-lb : 创建kube-lb的配置文件] **************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-lb : 创建kube-lb的systemd unit文件] ****************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-lb : 开机启用kube-lb服务] ***************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-lb : 开启kube-lb服务] *****************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-lb : 以轮询的方式等待kube-lb服务启动] *********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-master : 下载 kube_master 二进制] ******************************************************************************************************************************************
changed: [192.168.11.213] => (item=kube-apiserver)
changed: [192.168.11.213] => (item=kube-controller-manager)
changed: [192.168.11.213] => (item=kube-scheduler)
changed: [192.168.11.213] => (item=kubectl)
TASK [kube-master : 分发controller/scheduler kubeconfig配置文件] ***********************************************************************************************************************
changed: [192.168.11.213] => (item=kube-controller-manager.kubeconfig)
changed: [192.168.11.213] => (item=kube-scheduler.kubeconfig)
TASK [kube-master : 创建 kubernetes 证书签名请求] ****************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-master : 创建 kubernetes 证书和私钥] *****************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-master : 创建 aggregator proxy证书签名请求] ***********************************************************************************************************************************
ok: [192.168.11.213]
TASK [kube-master : 创建 aggregator-proxy证书和私钥] ************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-master : 分发 kubernetes证书] *********************************************************************************************************************************************
changed: [192.168.11.213] => (item=ca.pem)
changed: [192.168.11.213] => (item=ca-key.pem)
changed: [192.168.11.213] => (item=kubernetes.pem)
changed: [192.168.11.213] => (item=kubernetes-key.pem)
changed: [192.168.11.213] => (item=aggregator-proxy.pem)
changed: [192.168.11.213] => (item=aggregator-proxy-key.pem)
TASK [kube-master : 替换 kubeconfig 的 apiserver 地址] ********************************************************************************************************************************
changed: [192.168.11.213] => (item=/etc/kubernetes/kube-controller-manager.kubeconfig)
changed: [192.168.11.213] => (item=/etc/kubernetes/kube-scheduler.kubeconfig)
TASK [kube-master : 创建 master 服务的 systemd unit 文件] *******************************************************************************************************************************
changed: [192.168.11.213] => (item=kube-apiserver.service)
changed: [192.168.11.213] => (item=kube-controller-manager.service)
changed: [192.168.11.213] => (item=kube-scheduler.service)
TASK [kube-master : enable master 服务] ********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-master : 启动 master 服务] ************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-master : 轮询等待kube-apiserver启动] ****************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-master : 轮询等待kube-controller-manager启动] *******************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-master : 轮询等待kube-scheduler启动] ****************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-master : 复制kubectl.kubeconfig] ****************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-master : 替换 kubeconfig 的 apiserver 地址] ********************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-master : 轮询等待master服务启动完成] ********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-master : 获取user:kubernetes是否已经绑定对应角色] *********************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 创建kube_node 相关目录] **********************************************************************************************************************************************
changed: [192.168.11.213] => (item=/var/lib/kubelet)
changed: [192.168.11.213] => (item=/var/lib/kube-proxy)
TASK [kube-node : 下载 kubelet,kube-proxy 二进制和基础 cni plugins] **********************************************************************************************************************
ok: [192.168.11.213] => (item=kubectl)
changed: [192.168.11.213] => (item=kubelet)
changed: [192.168.11.213] => (item=kube-proxy)
changed: [192.168.11.213] => (item=bridge)
changed: [192.168.11.213] => (item=host-local)
changed: [192.168.11.213] => (item=loopback)
TASK [kube-node : 添加 kubectl 自动补全] ***********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 准备kubelet 证书签名请求] **********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 创建 kubelet 证书与私钥] **********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 设置集群参数] ********************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 设置客户端认证参数] *****************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 设置上下文参数] *******************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 选择默认上下文] *******************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 分发ca 证书] *******************************************************************************************************************************************************
ok: [192.168.11.213]
TASK [kube-node : 分发kubelet 证书] **************************************************************************************************************************************************
changed: [192.168.11.213] => (item=kubelet.pem)
changed: [192.168.11.213] => (item=kubelet-key.pem)
TASK [kube-node : 分发kubeconfig] **************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 准备 cni配置文件] ****************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 创建kubelet的配置文件] ************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 创建kubelet的systemd unit文件] **************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 开机启用kubelet 服务] ************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 开启kubelet 服务] **************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 分发 kube-proxy.kubeconfig配置文件] **********************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 替换 kube-proxy.kubeconfig 的 apiserver 地址] ***********************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 创建kube-proxy 配置] ***********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 创建kube-proxy 服务文件] *********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 开机启用kube-proxy 服务] *********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 开启kube-proxy 服务] ***********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 轮询等待kube-proxy启动] **********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : 轮询等待kubelet启动] *************************************************************************************************************************************************
changed: [192.168.11.213]
FAILED - RETRYING: 轮询等待node达到Ready状态 (8 retries left).
TASK [kube-node : 轮询等待node达到Ready状态] *********************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : Setting worker role name] **************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : Setting master role name] **************************************************************************************************************************************
changed: [192.168.11.213]
TASK [kube-node : Making master nodes SchedulingDisabled] ************************************************************************************************************************
changed: [192.168.11.213]
TASK [calico : 创建calico 证书请求] ****************************************************************************************************************************************************
ok: [192.168.11.213]
TASK [calico : 创建 calico证书和私钥] ***************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [calico : 删除旧 calico-etcd-secrets] ******************************************************************************************************************************************
changed: [192.168.11.213]
TASK [calico : 创建 calico-etcd-secrets] *******************************************************************************************************************************************
changed: [192.168.11.213]
TASK [calico : 配置 calico DaemonSet yaml文件] ***************************************************************************************************************************************
changed: [192.168.11.213]
TASK [calico : 运行 calico网络] ******************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [calico : 在节点创建相关目录] ********************************************************************************************************************************************************
changed: [192.168.11.213] => (item=/etc/calico/ssl)
TASK [calico : 分发calico证书相关] *****************************************************************************************************************************************************
changed: [192.168.11.213] => (item=ca.pem)
changed: [192.168.11.213] => (item=calico.pem)
changed: [192.168.11.213] => (item=calico-key.pem)
TASK [calico : 删除默认cni配置] ********************************************************************************************************************************************************
changed: [192.168.11.213]
TASK [calico : 下载calicoctl 客户端] **************************************************************************************************************************************************
changed: [192.168.11.213] => (item=calicoctl)
TASK [calico : 准备 calicoctl配置文件] *************************************************************************************************************************************************
changed: [192.168.11.213]
FAILED - RETRYING: 轮询等待calico-node 运行 (15 retries left).
TASK [calico : 轮询等待calico-node 运行] ***********************************************************************************************************************************************
changed: [192.168.11.213]
PLAY RECAP ***********************************************************************************************************************************************************************
192.168.11.213 : ok=100 changed=94 unreachable=0 failed=0 skipped=156 rescued=0 ignored=0
2023-03-27 11:54:07 INFO reconfigure and restart 'kube-lb' service
PLAY [kube_master,kube_node,etcd,ex_lb,chrony] ***********************************************************************************************************************************
TASK [Gathering Facts] ***********************************************************************************************************************************************************
ok: [192.168.11.213]
ok: [192.168.11.211]
ok: [192.168.11.214]
ok: [192.168.11.212]
ok: [192.168.11.215]
ok: [192.168.11.217]
ok: [192.168.11.218]
ok: [192.168.11.219]
PLAY [localhost] *****************************************************************************************************************************************************************
TASK [Gathering Facts] ***********************************************************************************************************************************************************
ok: [localhost]
PLAY [kube_master,kube_node,etcd] ************************************************************************************************************************************************
PLAY [etcd] **********************************************************************************************************************************************************************
PLAY [kube_master,kube_node] *****************************************************************************************************************************************************
PLAY [kube_master] ***************************************************************************************************************************************************************
TASK [kube-lb : 创建kube-lb的配置文件] **************************************************************************************************************************************************
ok: [192.168.11.213]
changed: [192.168.11.211]
changed: [192.168.11.212]
TASK [kube-lb : 创建kube-lb的systemd unit文件] ****************************************************************************************************************************************
ok: [192.168.11.213]
ok: [192.168.11.211]
ok: [192.168.11.212]
TASK [kube-lb : 开启kube-lb服务] *****************************************************************************************************************************************************
changed: [192.168.11.213]
changed: [192.168.11.212]
changed: [192.168.11.211]
TASK [kube-lb : 以轮询的方式等待kube-lb服务启动] *********************************************************************************************************************************************
changed: [192.168.11.213]
changed: [192.168.11.212]
changed: [192.168.11.211]
PLAY [kube_node] *****************************************************************************************************************************************************************
TASK [kube-lb : 创建kube-lb的配置文件] **************************************************************************************************************************************************
changed: [192.168.11.214]
changed: [192.168.11.215]
TASK [kube-lb : 创建kube-lb的systemd unit文件] ****************************************************************************************************************************************
ok: [192.168.11.215]
ok: [192.168.11.214]
TASK [kube-lb : 开启kube-lb服务] *****************************************************************************************************************************************************
changed: [192.168.11.215]
changed: [192.168.11.214]
TASK [kube-lb : 以轮询的方式等待kube-lb服务启动] *********************************************************************************************************************************************
changed: [192.168.11.215]
changed: [192.168.11.214]
PLAY [kube_master,kube_node] *****************************************************************************************************************************************************
PLAY [localhost] *****************************************************************************************************************************************************************
PLAY RECAP ***********************************************************************************************************************************************************************
192.168.11.211 : ok=5 changed=3 unreachable=0 failed=0 skipped=2 rescued=0 ignored=0
192.168.11.212 : ok=5 changed=3 unreachable=0 failed=0 skipped=2 rescued=0 ignored=0
192.168.11.213 : ok=5 changed=2 unreachable=0 failed=0 skipped=2 rescued=0 ignored=0
192.168.11.214 : ok=5 changed=3 unreachable=0 failed=0 skipped=2 rescued=0 ignored=0
192.168.11.215 : ok=5 changed=3 unreachable=0 failed=0 skipped=2 rescued=0 ignored=0
192.168.11.217 : ok=1 changed=0 unreachable=0 failed=0 skipped=2 rescued=0 ignored=0
192.168.11.218 : ok=1 changed=0 unreachable=0 failed=0 skipped=2 rescued=0 ignored=0
192.168.11.219 : ok=1 changed=0 unreachable=0 failed=0 skipped=2 rescued=0 ignored=0
localhost : ok=1 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
2023-03-27 11:54:29 INFO reconfigure and restart 'ex-lb' service
PLAY [ex_lb] *********************************************************************************************************************************************************************
skipping: no hosts matched
PLAY RECAP ***********************************************************************************************************************************************************************
#确认master已被添加到集群中
[root@K8s-ansible kubeasz]#kubectl get node
NAME STATUS ROLES AGE VERSION
192.168.11.211 Ready,SchedulingDisabled master 22h v1.24.10
192.168.11.212 Ready,SchedulingDisabled master 22h v1.24.10
192.168.11.213 Ready,SchedulingDisabled master 3m24s v1.24.10 --> 新建master
192.168.11.214 Ready node 22h v1.24.10
192.168.11.215 Ready node 22h v1.24.10
#此时kubeasz的hosts配置文件中master地址被添加
[root@K8s-ansible kubeasz]#cat clusters/k8s-cluster1/hosts
....
# master node(s), set unique 'k8s_nodename' for each node
# CAUTION: 'k8s_nodename' must consist of lower case alphanumeric characters, '-' or '.',
# and must start and end with an alphanumeric character
[kube_master]
192.168.11.213
192.168.11.211
192.168.11.212
#node节点上LB配置也添加了新的IP
[root@K8s-noded01 ~]#cat /etc/kube-lb/conf/kube-lb.conf
user root;
worker_processes 1;
error_log /etc/kube-lb/logs/error.log warn;
events {
worker_connections 3000;
}
stream {
upstream backend {
server 192.168.11.213:6443 max_fails=2 fail_timeout=3s;
server 192.168.11.211:6443 max_fails=2 fail_timeout=3s;
server 192.168.11.212:6443 max_fails=2 fail_timeout=3s;
}
server {
listen 127.0.0.1:6443;
proxy_connect_timeout 1s;
proxy_pass backend;
}
}
[root@K8s-noded01 ~]#ls -l /etc/kube-lb/conf/kube-lb.conf
-rw-r--r-- 1 root root 474 Mar 27 11:54 /etc/kube-lb/conf/kube-lb.conf
[root@K8s-noded01 ~]#md5sum /etc/kube-lb/conf/kube-lb.conf
9a9749a8eb227729ff9db1d7d5368d07 /etc/kube-lb/conf/kube-lb.conf添加Node
- 横向扩展Node节点
[root@K8s-ansible kubeasz]#./ezctl add-node k8s-cluster1 192.168.11.216
2023-03-27 12:02:15 INFO add 192.168.11.216 into 'kube_node' group
2023-03-27 12:02:15 INFO start to add a work node:192.168.11.216 into cluster:k8s-cluster1
PLAY [192.168.11.216] ************************************************************************************************************************************************************
TASK [Gathering Facts] ***********************************************************************************************************************************************************
ok: [192.168.11.216]
TASK [prepare : apt更新缓存刷新] *******************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [prepare : 删除ubuntu默认安装] ****************************************************************************************************************************************************
changed: [192.168.11.216] => (item=ufw)
changed: [192.168.11.216] => (item=lxd)
changed: [192.168.11.216] => (item=lxcfs)
changed: [192.168.11.216] => (item=lxc-common)
TASK [prepare : 安装 ubuntu/debian基础软件] ********************************************************************************************************************************************
changed: [192.168.11.216]
TASK [prepare : 准备 journal 日志相关目录] ***********************************************************************************************************************************************
changed: [192.168.11.216] => (item=/etc/systemd/journald.conf.d)
ok: [192.168.11.216] => (item=/var/log/journal)
TASK [prepare : 优化设置 journal 日志] *************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [prepare : 重启 journald 服务] **************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [prepare : 禁用系统 swap] *******************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [prepare : 删除fstab swap 相关配置] ***********************************************************************************************************************************************
changed: [192.168.11.216]
TASK [prepare : 转换内核版本为浮点数] ******************************************************************************************************************************************************
ok: [192.168.11.216]
TASK [prepare : 加载内核模块] **********************************************************************************************************************************************************
changed: [192.168.11.216] => (item=br_netfilter)
changed: [192.168.11.216] => (item=ip_vs)
changed: [192.168.11.216] => (item=ip_vs_rr)
changed: [192.168.11.216] => (item=ip_vs_wrr)
changed: [192.168.11.216] => (item=ip_vs_sh)
ok: [192.168.11.216] => (item=nf_conntrack)
TASK [prepare : 尝试加载nf_conntrack_ipv4] *******************************************************************************************************************************************
changed: [192.168.11.216]
TASK [prepare : 启用systemd自动加载模块服务] ***********************************************************************************************************************************************
ok: [192.168.11.216]
TASK [prepare : 增加内核模块开机加载配置] ****************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [prepare : 设置系统参数] **********************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [prepare : 生效系统参数] **********************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [prepare : 创建 systemd 配置目录] *************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [prepare : 设置系统 ulimits] ****************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [prepare : 把SCTP列入内核模块黑名单] **************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [prepare : prepare some dirs] ***********************************************************************************************************************************************
ok: [192.168.11.216] => (item=/usr/local/bin)
changed: [192.168.11.216] => (item=/etc/kubernetes/ssl)
changed: [192.168.11.216] => (item=/root/.kube)
changed: [192.168.11.216] => (item=/etc/cni/net.d)
TASK [prepare : symlink /usr/bin/python -> /usr/bin/python3] *********************************************************************************************************************
changed: [192.168.11.216]
TASK [prepare : 写入环境变量$PATH] *****************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [prepare : 添加 local registry hosts 解析] **************************************************************************************************************************************
changed: [192.168.11.216]
TASK [containerd : 获取是否已经安装containerd] *******************************************************************************************************************************************
changed: [192.168.11.216]
TASK [containerd : 准备containerd相关目录] *********************************************************************************************************************************************
ok: [192.168.11.216] => (item=/usr/local/bin)
changed: [192.168.11.216] => (item=/etc/containerd)
TASK [containerd : 加载内核模块 overlay] ***********************************************************************************************************************************************
changed: [192.168.11.216]
TASK [containerd : 下载 containerd 二进制文件] ******************************************************************************************************************************************
changed: [192.168.11.216] => (item=containerd)
changed: [192.168.11.216] => (item=containerd-shim)
changed: [192.168.11.216] => (item=containerd-shim-runc-v1)
changed: [192.168.11.216] => (item=containerd-shim-runc-v2)
changed: [192.168.11.216] => (item=crictl)
changed: [192.168.11.216] => (item=ctr)
changed: [192.168.11.216] => (item=runc)
TASK [containerd : 添加 crictl 自动补全] ***********************************************************************************************************************************************
changed: [192.168.11.216]
TASK [containerd : 创建 containerd 配置文件] *******************************************************************************************************************************************
changed: [192.168.11.216]
TASK [containerd : 创建systemd unit文件] *********************************************************************************************************************************************
changed: [192.168.11.216]
TASK [containerd : 创建 crictl 配置] *************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [containerd : 开机启用 containerd 服务] *******************************************************************************************************************************************
changed: [192.168.11.216]
TASK [containerd : 开启 containerd 服务] *********************************************************************************************************************************************
changed: [192.168.11.216]
TASK [containerd : 轮询等待containerd服务运行] *******************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-lb : prepare some dirs] ***********************************************************************************************************************************************
changed: [192.168.11.216] => (item=/etc/kube-lb/sbin)
changed: [192.168.11.216] => (item=/etc/kube-lb/logs)
changed: [192.168.11.216] => (item=/etc/kube-lb/conf)
TASK [kube-lb : 下载二进制文件kube-lb(nginx)] *******************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-lb : 创建kube-lb的配置文件] **************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-lb : 创建kube-lb的systemd unit文件] ****************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-lb : 开机启用kube-lb服务] ***************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-lb : 开启kube-lb服务] *****************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-lb : 以轮询的方式等待kube-lb服务启动] *********************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 创建kube_node 相关目录] **********************************************************************************************************************************************
changed: [192.168.11.216] => (item=/var/lib/kubelet)
changed: [192.168.11.216] => (item=/var/lib/kube-proxy)
TASK [kube-node : 下载 kubelet,kube-proxy 二进制和基础 cni plugins] **********************************************************************************************************************
changed: [192.168.11.216] => (item=kubectl)
changed: [192.168.11.216] => (item=kubelet)
changed: [192.168.11.216] => (item=kube-proxy)
changed: [192.168.11.216] => (item=bridge)
changed: [192.168.11.216] => (item=host-local)
changed: [192.168.11.216] => (item=loopback)
TASK [kube-node : 添加 kubectl 自动补全] ***********************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 准备kubelet 证书签名请求] **********************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 创建 kubelet 证书与私钥] **********************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 设置集群参数] ********************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 设置客户端认证参数] *****************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 设置上下文参数] *******************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 选择默认上下文] *******************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 分发ca 证书] *******************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 分发kubelet 证书] **************************************************************************************************************************************************
changed: [192.168.11.216] => (item=kubelet.pem)
changed: [192.168.11.216] => (item=kubelet-key.pem)
TASK [kube-node : 分发kubeconfig] **************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 准备 cni配置文件] ****************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 创建kubelet的配置文件] ************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 创建kubelet的systemd unit文件] **************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 开机启用kubelet 服务] ************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 开启kubelet 服务] **************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 分发 kube-proxy.kubeconfig配置文件] **********************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 替换 kube-proxy.kubeconfig 的 apiserver 地址] ***********************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 创建kube-proxy 配置] ***********************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 创建kube-proxy 服务文件] *********************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 开机启用kube-proxy 服务] *********************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 开启kube-proxy 服务] ***********************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 轮询等待kube-proxy启动] **********************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 轮询等待kubelet启动] *************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : 轮询等待node达到Ready状态] *********************************************************************************************************************************************
changed: [192.168.11.216]
TASK [kube-node : Setting worker role name] **************************************************************************************************************************************
changed: [192.168.11.216]
TASK [calico : 创建calico 证书请求] ****************************************************************************************************************************************************
ok: [192.168.11.216]
TASK [calico : 创建 calico证书和私钥] ***************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [calico : 删除旧 calico-etcd-secrets] ******************************************************************************************************************************************
changed: [192.168.11.216]
TASK [calico : 创建 calico-etcd-secrets] *******************************************************************************************************************************************
changed: [192.168.11.216]
TASK [calico : 配置 calico DaemonSet yaml文件] ***************************************************************************************************************************************
ok: [192.168.11.216]
TASK [calico : 运行 calico网络] ******************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [calico : 在节点创建相关目录] ********************************************************************************************************************************************************
changed: [192.168.11.216] => (item=/etc/calico/ssl)
TASK [calico : 分发calico证书相关] *****************************************************************************************************************************************************
changed: [192.168.11.216] => (item=ca.pem)
changed: [192.168.11.216] => (item=calico.pem)
changed: [192.168.11.216] => (item=calico-key.pem)
TASK [calico : 删除默认cni配置] ********************************************************************************************************************************************************
changed: [192.168.11.216]
TASK [calico : 下载calicoctl 客户端] **************************************************************************************************************************************************
changed: [192.168.11.216] => (item=calicoctl)
TASK [calico : 准备 calicoctl配置文件] *************************************************************************************************************************************************
changed: [192.168.11.216]
FAILED - RETRYING: 轮询等待calico-node 运行 (15 retries left).
FAILED - RETRYING: 轮询等待calico-node 运行 (14 retries left).
TASK [calico : 轮询等待calico-node 运行] ***********************************************************************************************************************************************
changed: [192.168.11.216]
PLAY RECAP ***********************************************************************************************************************************************************************
192.168.11.216 : ok=80 changed=75 unreachable=0 failed=0 skipped=157 rescued=0 ignored=0
#确认Node节点添加成功
[root@K8s-ansible kubeasz]#kubectl get node
NAME STATUS ROLES AGE VERSION
192.168.11.211 Ready,SchedulingDisabled master 22h v1.24.10
192.168.11.212 Ready,SchedulingDisabled master 22h v1.24.10
192.168.11.213 Ready,SchedulingDisabled master 13m v1.24.10
192.168.11.214 Ready node 22h v1.24.10
192.168.11.215 Ready node 22h v1.24.10
192.168.11.216 Ready node 2m30s v1.24.10 --> 新的Node集群版本升级
下载指定版本的二进制包
#下载变更的二进制文件
[root@K8s-ansible data]#wget https://dl.k8s.io/v1.26.3/kubernetes.tar.gz
[root@K8s-ansible data]#wget https://dl.k8s.io/v1.26.3/kubernetes-client-linux-amd64.tar.gz
[root@K8s-ansible data]#wget https://dl.k8s.io/v1.26.3/kubernetes-server-linux-amd64.tar.gz
[root@K8s-ansible data]#wget https://dl.k8s.io/v1.26.3/kubernetes-node-linux-amd64.tar.gz
[root@K8s-ansible data]#ls
kubernetes-client-linux-amd64.tar.gz kubernetes-node-linux-amd64.tar.gz kubernetes-server-linux-amd64.tar.gz kubernetes.tar.gz
[root@K8s-ansible data]#mv kubernetes-client-linux-amd64.tar.gz kubernetes-node-linux-amd64.tar.gz kubernetes-server-linux-amd64.tar.gz kubernetes.tar.gz /usr/local/src
[root@K8s-ansible data]#cd /usr/local/src/
[root@K8s-ansible src]#ls
kubernetes-client-linux-amd64.tar.gz kubernetes-node-linux-amd64.tar.gz kubernetes-server-linux-amd64.tar.gz kubernetes.tar.gz
#解压
[root@K8s-ansible src]#tar xf kubernetes-client-linux-amd64.tar.gz
[root@K8s-ansible src]#tar xf kubernetes-node-linux-amd64.tar.gz
[root@K8s-ansible src]#tar xf kubernetes-server-linux-amd64.tar.gz
[root@K8s-ansible src]#tar xf kubernetes.tar.gz
[root@K8s-ansible src]#ll
total 501192
drwxr-xr-x 3 root root 4096 Mar 27 12:36 ./
drwxr-xr-x 10 root root 4096 Aug 9 2022 ../
drwxr-xr-x 10 root root 4096 Mar 15 14:01 kubernetes/
-rw-r--r-- 1 root root 33016823 Mar 17 19:05 kubernetes-client-linux-amd64.tar.gz
-rw-r--r-- 1 root root 130544073 Mar 17 19:05 kubernetes-node-linux-amd64.tar.gz
-rw-r--r-- 1 root root 349110643 Mar 17 19:05 kubernetes-server-linux-amd64.tar.gz
-rw-r--r-- 1 root root 528218 Mar 17 19:05 kubernetes.tar.gz
#进入kubernetes目录确认版本
[root@K8s-ansible kubernetes]#ll
total 37372
drwxr-xr-x 10 root root 4096 Mar 15 14:01 ./
drwxr-xr-x 3 root root 4096 Mar 27 12:36 ../
drwxr-xr-x 4 root root 4096 Mar 15 14:01 LICENSES/
-rw-r--r-- 1 root root 4269 Mar 15 14:01 README.md
drwxr-xr-x 2 root root 4096 Mar 15 13:55 addons/
drwxr-xr-x 3 root root 4096 Mar 15 14:01 client/
drwxr-xr-x 9 root root 4096 Mar 15 14:01 cluster/
drwxr-xr-x 2 root root 4096 Mar 15 14:01 docs/
drwxr-xr-x 3 root root 4096 Mar 15 14:01 hack/
-rw-r--r-- 1 root root 38208750 Mar 15 13:55 kubernetes-src.tar.gz
drwxr-xr-x 3 root root 4096 Mar 15 13:54 node/
drwxr-xr-x 3 root root 4096 Mar 15 14:01 server/
-rw-r--r-- 1 root root 8 Mar 15 14:01 version
[root@K8s-ansible kubernetes]#./server/bin/kube-apiserver --version
Kubernetes v1.26.3节点逐个进行版本升级
- 升级Master节点
#Kubernetes的组件默认都在/usr/local/bin下面，升级就是将其中的主要文件替换成为新版，然后运行
[root@K8s-master01 ~]#ll /usr/local/bin/
total 897540
drwxr-xr-x 2 root root 4096 Mar 27 11:55 ./
drwxr-xr-x 10 root root 4096 Aug 9 2022 ../
-rwxr-xr-x 1 root root 3780654 Mar 26 14:15 bandwidth*
-rwxr-xr-x 1 root root 4221977 Mar 26 13:21 bridge*
-rwsr-xr-x 1 root root 58076620 Mar 26 14:15 calico*
-rwsr-xr-x 1 root root 58076620 Mar 26 14:15 calico-ipam*
-rwxr-xr-x 1 root root 59499254 Mar 26 14:14 calicoctl*
-rwxr-xr-x 1 root root 51529720 Mar 26 12:37 containerd*
-rwxr-xr-x 1 root root 7254016 Mar 26 12:37 containerd-shim*
-rwxr-xr-x 1 root root 9359360 Mar 26 12:37 containerd-shim-runc-v1*
-rwxr-xr-x 1 root root 9375744 Mar 26 12:37 containerd-shim-runc-v2*
-rwxr-xr-x 1 root root 52586151 Mar 26 12:37 crictl*
-rwxr-xr-x 1 root root 26712216 Mar 26 12:37 ctr*
-rwxr-xr-x 1 root root 2474798 Mar 26 14:15 flannel*
-rwxr-xr-x 1 root root 3241605 Mar 26 13:21 host-local*
-rwsr-xr-x 1 root root 58076620 Mar 26 14:15 install*
-rwxr-xr-x 1 root root 126513152 Mar 26 13:20 kube-apiserver*
-rwxr-xr-x 1 root root 116121600 Mar 26 13:20 kube-controller-manager*
-rwxr-xr-x 1 root root 42123264 Mar 26 13:21 kube-proxy*
-rwxr-xr-x 1 root root 47529984 Mar 26 13:21 kube-scheduler*
-rwxr-xr-x 1 root root 46067712 Mar 26 13:21 kubectl*
-rwxr-xr-x 1 root root 116628888 Mar 26 13:21 kubelet*
-rwxr-xr-x 1 root root 3295519 Mar 26 13:21 loopback*
-rwxr-xr-x 1 root root 3679140 Mar 26 14:15 portmap*
-rwxr-xr-x 1 root root 9431456 Mar 26 12:37 runc*
-rwxr-xr-x 1 root root 3379564 Mar 26 14:15 tuning*
#关闭Node节点上向Master节点的调度，所有Node节点都需要设置
[root@K8s-noded01 ~]#vim /etc/kube-lb/conf/kube-lb.conf
[root@K8s-noded01 ~]#cat /etc/kube-lb/conf/kube-lb.conf
user root;
worker_processes 1;
error_log /etc/kube-lb/logs/error.log warn;
events {
worker_connections 3000;
}
stream {
upstream backend {
server 192.168.11.213:6443 max_fails=2 fail_timeout=3s;
# server 192.168.11.211:6443 max_fails=2 fail_timeout=3s; #注释掉对Master节点的调度
server 192.168.11.212:6443 max_fails=2 fail_timeout=3s;
}
server {
listen 127.0.0.1:6443;
proxy_connect_timeout 1s;
proxy_pass backend;
}
}
[root@K8s-noded01 ~]#systemctl reload kube-lb.service
#关闭Master节点上的kubernetes服务
[root@K8s-master01 ~]#systemctl stop kube-apiserver.service kube-controller-manager.service kube-scheduler.service kube-proxy.service kubelet.service
#将提前下载的二进制升级包复制到Master节点的组件目录下替换旧版
[root@K8s-ansible bin]#pwd
/usr/local/src/kubernetes/server/bin
[root@K8s-ansible bin]#scp kube-apiserver kube-controller-manager kube-scheduler kube-proxy kubelet kubectl 192.168.11.211:/usr/local/bin/
kube-apiserver 100% 124MB 173.8MB/s 00:00
kube-controller-manager 100% 114MB 252.7MB/s 00:00
kube-scheduler 100% 50MB 237.5MB/s 00:00
kube-proxy 100% 43MB 267.3MB/s 00:00
kubelet 100% 116MB 249.5MB/s 00:00
kubectl 100% 46MB 199.5MB/s 00:00
#确认替换为新版
[root@K8s-master01 ~]#/usr/local/bin/kube-apiserver --version
Kubernetes v1.26.3
#重启服务
[root@K8s-master01 ~]#systemctl start kube-apiserver.service kube-controller-manager.service kube-scheduler.service kube-proxy.service kubelet.service
#确认版本升级
[root@K8s-ansible bin]#kubectl get node
NAME STATUS ROLES AGE VERSION
192.168.11.211 Ready,SchedulingDisabled master 24h v1.26.3 #版本升级成功
192.168.11.212 Ready,SchedulingDisabled master 24h v1.24.10
192.168.11.213 Ready,SchedulingDisabled master 111m v1.24.10
192.168.11.214 Ready node 23h v1.24.10
192.168.11.215 Ready node 23h v1.24.10
192.168.11.216 Ready node 100m v1.24.10
#Master升级完成后放开Node对节点的调度
[root@K8s-noded01 ~]#cat /etc/kube-lb/conf/kube-lb.conf
user root;
worker_processes 1;
error_log /etc/kube-lb/logs/error.log warn;
events {
worker_connections 3000;
}
stream {
upstream backend {
server 192.168.11.213:6443 max_fails=2 fail_timeout=3s;
server 192.168.11.211:6443 max_fails=2 fail_timeout=3s;
server 192.168.11.212:6443 max_fails=2 fail_timeout=3s;
}
server {
listen 127.0.0.1:6443;
proxy_connect_timeout 1s;
proxy_pass backend;
}
}
#之后以相同的方式对其他Master节点进行全部升级即可
[root@K8s-ansible bin]#kubectl get node
NAME STATUS ROLES AGE VERSION
192.168.11.211 Ready,SchedulingDisabled master 24h v1.26.3
192.168.11.212 Ready,SchedulingDisabled master 24h v1.26.3
192.168.11.213 Ready,SchedulingDisabled master 120m v1.26.3
192.168.11.214 Ready node 24h v1.24.10
192.168.11.215 Ready node 24h v1.24.10
192.168.11.216 Ready node 109m v1.24.10- 升级Node节点
注意：Node节点有正在运行的Pod服务，此时对Node升级需要对需要升级节点的Pod进行驱逐，Pod会在其他Node节点重新创建并运行，确保Node节点Pod驱逐干净了再进行升级
#驱逐业务容器Pod，注意要保证有Node节点运行，以免升级时业务容器的出现丢失问题
[root@K8s-ansible bin]#kubectl drain 192.168.11.214 --ignore-daemonsets --force
node/192.168.11.214 already cordoned
WARNING: ignoring DaemonSet-managed Pods: kube-system/calico-node-s6hp4; deleting Pods that declare no controller: myserver/net-test1, myserver/net-test3, myserver/net-test4
evicting pod myserver/net-test1
evicting pod default/demoapp-78b49597cf-4ql4c
evicting pod myserver/net-test3
evicting pod myserver/net-test4
pod/demoapp-78b49597cf-4ql4c evicted
pod/net-test4 evicted
pod/net-test1 evicted
pod/net-test3 evicted
node/192.168.11.214 drained
#停止Node节点的Kubernetes服务
[root@K8s-noded01 ~]#systemctl stop kubelet.service kube-proxy.service
#替换新版服务文件
[root@K8s-ansible bin]#scp kube-proxy kubelet kubectl 192.168.11.214:/usr/local/bin/
kube-proxy 100% 43MB 104.6MB/s 00:00
kubelet 100% 116MB 243.2MB/s 00:00
kubectl 100% 46MB 242.5MB/s 00:00
#重启Node节点Kubernetes服务
[root@K8s-noded01 ~]#systemctl start kubelet.service kube-proxy.service
#确认Node节点升级成功
[root@K8s-ansible bin]#kubectl get node
NAME STATUS ROLES AGE VERSION
...
192.168.11.214 Ready,SchedulingDisabled node 24h v1.26.3 #升级成功
192.168.11.215 Ready node 24h v1.24.10
192.168.11.216 Ready node 130m v1.24.10
#恢复Node节点的调度服务【取消SchedulingDisabled】
[root@K8s-ansible bin]#kubectl uncordon 192.168.11.214
node/192.168.11.214 uncordoned
[root@K8s-ansible bin]#kubectl get node
NAME STATUS ROLES AGE VERSION
...
192.168.11.214 Ready node 24h v1.26.3
192.168.11.215 Ready node 24h v1.24.10
192.168.11.216 Ready node 130m v1.24.10
#其他Node节点同样升级即可
[root@K8s-ansible bin]#kubectl get node
NAME STATUS ROLES AGE VERSION
192.168.11.211 Ready,SchedulingDisabled master 25h v1.26.3
192.168.11.212 Ready,SchedulingDisabled master 25h v1.26.3
192.168.11.213 Ready,SchedulingDisabled master 153m v1.26.3
192.168.11.214 Ready node 24h v1.26.3
192.168.11.215 Ready node 24h v1.26.3
192.168.11.216 Ready node 142m v1.26.3- 对kubeasz的服务文件更新
#新版本文件同样需要对kubeasz中的bin文件做替换，以免部署节点对后续的节点管理使用旧版
[root@K8s-ansible bin]#\cp kube-apiserver kube-controller-manager kube-scheduler kube-proxy kubelet kubectl /etc/kubeasz/bin/
#确认版本
[root@K8s-ansible bin]#/etc/kubeasz/bin/kube-apiserver --version
Kubernetes v1.26.3- 升级Containerd及runc
注意：升级containerd-正常情况下，先驱逐pod，然后将服务停止或重启服务器，然后替换二进制再启动服务
#查看当前Containerd版本 - 1.6.14
[root@K8s-ansible bin]#kubectl get node -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
192.168.11.211 Ready,SchedulingDisabled master 25h v1.26.3 192.168.11.211 <none> Ubuntu 22.04.1 LTS 5.15.0-60-generic containerd://1.6.14
192.168.11.212 Ready,SchedulingDisabled master 25h v1.26.3 192.168.11.212 <none> Ubuntu 22.04.1 LTS 5.15.0-60-generic containerd://1.6.14
192.168.11.213 Ready,SchedulingDisabled master 154m v1.26.3 192.168.11.213 <none> Ubuntu 22.04.1 LTS 5.15.0-60-generic containerd://1.6.14
192.168.11.214 Ready node 24h v1.26.3 192.168.11.214 <none> Ubuntu 22.04.1 LTS 5.15.0-60-generic containerd://1.6.14
192.168.11.215 Ready node 24h v1.26.3 192.168.11.215 <none> Ubuntu 22.04.1 LTS 5.15.0-60-generic containerd://1.6.14
192.168.11.216 Ready node 143m v1.26.3 192.168.11.216 <none> Ubuntu 22.04.1 LTS 5.15.0-67-generic containerd://1.6.14
#查看当前runc版本
[root@K8s-ansible bin]#/etc/kubeasz/bin/runc --version
runc version 1.1.4
commit: v1.1.4-0-g5fd4c4d1
spec: 1.0.2-dev
go: go1.18.9
libseccomp: 2.5.1下载指定版本的containerd二进制安装包到本地
[root@K8s-ansible src]#wget https://github.com/containerd/containerd/releases/download/v1.7.0/containerd-1.7.0-linux-amd64.tar.gz
[root@K8s-ansible src]#tar xvf containerd-1.7.0-linux-amd64.tar.gz
bin/
bin/containerd-shim-runc-v1
bin/containerd
bin/containerd-shim-runc-v2
bin/containerd-shim
bin/ctr
bin/containerd-stress下载指定版本的containerd二进制安装包到本地
[root@K8s-ansible bin]#wget https://github.com/opencontainers/runc/releases/download/v1.1.4/runc.amd64
[root@K8s-ansible bin]#ls
containerd containerd-shim containerd-shim-runc-v1 containerd-shim-runc-v2 containerd-stress ctr runc.amd64
[root@K8s-ansible bin]#mv runc.amd64 runc
[root@K8s-ansible bin]#chmod a+x runc
[root@K8s-ansible bin]#ls
containerd containerd-shim containerd-shim-runc-v1 containerd-shim-runc-v2 containerd-stress ctr runc将上述文件全部复制到containerd服务文件目录下
#可以参考kubeasz中对containerd配置文件
[root@K8s-ansible src]#cat /etc/kubeasz/roles/containerd/tasks/main.yml
- name: 获取是否已经安装containerd
shell: 'systemctl is-active containerd || echo "NoFound"'
register: containerd_svc
- block:
- name: 准备containerd相关目录
file: name={{ item }} state=directory
with_items:
- "{{ bin_dir }}"
- "/etc/containerd"
- name: 加载内核模块 overlay
modprobe: name=overlay state=present
- name: 下载 containerd 二进制文件
copy: src={{ base_dir }}/bin/containerd-bin/{{ item }} dest={{ bin_dir }}/{{ item }} mode=0755
with_items:
- containerd
- containerd-shim
- containerd-shim-runc-v1
- containerd-shim-runc-v2
- crictl
- ctr
- runc
tags: upgrade
#containerd-bin目录下可以升级其他客户端工具包括crictl,nerdctl，升级方式相同，此处省略
[root@K8s-ansible bin]#ll /etc/kubeasz/bin/containerd-bin/
total 184576
drwxr-xr-x 2 root root 4096 Jan 26 01:51 ./
drwxr-xr-x 3 root root 4096 Mar 26 03:49 ../
-rwxr-xr-x 1 root root 51529720 Dec 19 16:53 containerd*
-rwxr-xr-x 1 root root 7254016 Dec 19 16:53 containerd-shim*
-rwxr-xr-x 1 root root 9359360 Dec 19 16:53 containerd-shim-runc-v1*
-rwxr-xr-x 1 root root 9375744 Dec 19 16:53 containerd-shim-runc-v2*
-rwxr-xr-x 1 root root 22735256 Dec 19 16:53 containerd-stress*
-rwxr-xr-x 1 root root 52586151 Dec 14 07:20 crictl*
-rwxr-xr-x 1 root root 26712216 Dec 19 16:53 ctr*
-rwxr-xr-x 1 root root 9431456 Aug 25 2022 runc*
#复制到kubeasz的配置文件目录下，保证部署节点的文件为最新
[root@K8s-ansible bin]#\cp ./* /etc/kubeasz/bin/containerd-bin/
#升级目标主机的containerd和runc
#有些网络组件会处于持续运行中，shim处于占用状态无法更换
[root@K8s-ansible bin]#scp ./* 192.168.11.214:/usr/local/bin/
scp: /usr/local/bin//containerd: Text file busy
containerd-shim 100% 6404KB 21.3MB/s 00:00
containerd-shim-runc-v1 100% 8072KB 115.1MB/s 00:00
scp: /usr/local/bin//containerd-shim-runc-v2: Text file busy
containerd-stress 100% 24MB 131.4MB/s 00:00
ctr 100% 26MB 111.9MB/s 00:00
runc 100% 9210KB 135.2MB/s 00:00
[root@K8s-noded01 ~]#ps -ef |grep shim
root 10336 1 0 11:53 ? 00:00:30 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id 40d15251108e419658a2bd03b0bb037da819a21ba6fdbfa512c05819cb291f6f -address /run/containerd/containerd.sock
root 118347 1 0 14:25 ? 00:00:08 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id 53c56ddd73945287d1b41f40a07d5129e49f344a005bc1c1fa9092f060f82095 -address /run/containerd/containerd.sock
root 118428 1 0 14:25 ? 00:00:02 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id f203f805388e906a12cf7d18cc165d86a550ce6df11edd6286c1aa1efed5e471 -address /run/containerd/containerd.sock
root 118489 1 0 14:25 ? 00:00:02 /usr/local/bin/containerd-shim-runc-v2 -namespace k8s.io -id 22a63c331d8d5effb6d62aba6fa8edb4b73eb964f7991cf300cd46c22af5dd30 -address /run/containerd/containerd.sock
root 165217 86752 0 15:27 pts/1 00:00:00 grep --color=auto shim
#停止使用shim的服务(会导致业务容器无法对外提供访问)
[root@K8s-noded01 ~]#systemctl disable kubelet.service kube-proxy.service containerd.service
Removed /etc/systemd/system/multi-user.target.wants/kube-proxy.service.
Removed /etc/systemd/system/multi-user.target.wants/kubelet.service.
Removed /etc/systemd/system/multi-user.target.wants/containerd.service.
[root@K8s-noded01 ~]#reboot
#再次替换为新版
[root@K8s-ansible bin]#scp ./* 192.168.11.214:/usr/local/bin/
containerd 100% 52MB 98.4MB/s 00:00
containerd-shim 100% 6404KB 233.7MB/s 00:00
containerd-shim-runc-v1 100% 8072KB 244.1MB/s 00:00
containerd-shim-runc-v2 100% 11MB 249.4MB/s 00:00
containerd-stress 100% 24MB 247.6MB/s 00:00
ctr 100% 26MB 257.7MB/s 00:00
runc 100% 9210KB 243.6MB/s 00:00
#重启目标主机的kubelet，kube-proxy，containerd
[root@K8s-noded01 ~]#systemctl enable --now kubelet.service kube-proxy.service containerd.service
Created symlink /etc/systemd/system/multi-user.target.wants/kubelet.service → /etc/systemd/system/kubelet.service.
Created symlink /etc/systemd/system/multi-user.target.wants/kube-proxy.service → /etc/systemd/system/kube-proxy.service.
Created symlink /etc/systemd/system/multi-user.target.wants/containerd.service → /etc/systemd/system/containerd.service.
#确认升级 - Node节点都升级1.7.0
[root@K8s-ansible bin]#kubectl get node -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
192.168.11.211 Ready,SchedulingDisabled master 26h v1.26.3 192.168.11.211 <none> Ubuntu 22.04.1 LTS 5.15.0-60-generic containerd://1.6.14
192.168.11.212 Ready,SchedulingDisabled master 26h v1.26.3 192.168.11.212 <none> Ubuntu 22.04.1 LTS 5.15.0-60-generic containerd://1.6.14
192.168.11.213 Ready,SchedulingDisabled master 3h48m v1.26.3 192.168.11.213 <none> Ubuntu 22.04.1 LTS 5.15.0-60-generic containerd://1.6.14
192.168.11.214 Ready node 25h v1.26.3 192.168.11.214 <none> Ubuntu 22.04.1 LTS 5.15.0-67-generic containerd://1.7.0
192.168.11.215 Ready node 25h v1.26.3 192.168.11.215 <none> Ubuntu 22.04.1 LTS 5.15.0-67-generic containerd://1.7.0
192.168.11.216 Ready node 3h37m v1.26.3 192.168.11.216 <none> Ubuntu 22.04.1 LTS 5.15.0-67-generic containerd://1.7.0
#其他节点同样配置即可
[root@K8s-ansible bin]#kubectl get node -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
192.168.11.211 Ready,SchedulingDisabled master 26h v1.26.3 192.168.11.211 <none> Ubuntu 22.04.1 LTS 5.15.0-67-generic containerd://1.7.0
192.168.11.212 Ready,SchedulingDisabled master 26h v1.26.3 192.168.11.212 <none> Ubuntu 22.04.1 LTS 5.15.0-67-generic containerd://1.7.0
192.168.11.213 Ready,SchedulingDisabled master 3h54m v1.26.3 192.168.11.213 <none> Ubuntu 22.04.1 LTS 5.15.0-60-generic containerd://1.7.0
192.168.11.214 Ready node 26h v1.26.3 192.168.11.214 <none> Ubuntu 22.04.1 LTS 5.15.0-67-generic containerd://1.7.0
192.168.11.215 Ready node 26h v1.26.3 192.168.11.215 <none> Ubuntu 22.04.1 LTS 5.15.0-67-generic containerd://1.7.0
192.168.11.216 Ready node 3h43m v1.26.3 192.168.11.216 <none> Ubuntu 22.04.1 LTS 5.15.0-67-generic containerd://1.7.0我是moore,大家一起加油！！！