kubeadm单集群部署k8s1.15.1&flannel网络

说明

本次实验在Windows下的VMware进行
系统配置及初始化配置在所有的主机执行
容器镜像全部替换为国内可拉取的
pod网络采用flannel

实验环境

主机名	IP地址	角色	OS	CPU/MEM	网卡/模式	平台
k8s-master01	192.168.181.158	master	CentOS7.6	2C/2G	x1/NAT	VMware
k8s-node1	192.168.181.159	node1	CentOS7.6	2C/2G	x1/NAT	VMware
k8s-node2	192.168.181.160	node2	CentOS7.6	2C/2G	x1/NAT	VMware

初始配置

基本配置为三个主机都需要的操作

history格式设置

cat >> /etc/bashrc << "EOF"
# history actions record，include action time, user, login ip
HISTFILESIZE=4000
HISTSIZE=4000
USER_IP=`who -u am i 2>/dev/null| awk '{print $NF}'|sed -e 's/[()]//g'`
if [ -z $USER_IP ]
then
  USER_IP=`hostname`
fi
HISTTIMEFORMAT="%F %T $USER_IP:`whoami` "
export HISTTIMEFORMAT
EOF

安装常用软件

yum install -y net-tools iproute lrzsz vim bash-completion wget tree bridge-utils unzip bind-utils git gcc

主机名设置

hostnamectl set-hostname k8s-master01
hostnamectl set-hostname k8s-node01
hostnamectl set-hostname k8s-node02

静态IP设置

设置静态IP，进行calico网络方案时，发现配置之后，ip有变化

cat /etc/sysconfig/network-scripts/ifcfg-ens33
TYPE="Ethernet"
PROXY_METHOD="none"
BROWSER_ONLY="no"
BOOTPROTO="static"
DEFROUTE="yes"
IPV4_FAILURE_FATAL="no"
IPV6INIT="yes"
IPV6_AUTOCONF="yes"
IPV6_DEFROUTE="yes"
IPV6_FAILURE_FATAL="no"
IPV6_ADDR_GEN_MODE="stable-privacy"
NAME="ens33"
UUID="41e83853-95e3-4b09-861b-e36dd3ead61b"
DEVICE="ens33"
ONBOOT="yes"
# 根据主机ip设置
IPADDR="192.168.181.158"
PREFIX="24"
GATEWAY="192.168.181.2"
DNS1="202.96.128.166"
IPV6_PRIVACY="no"

重启网络

systemctl restart network

修改/etc/hosts

cat >> /etc/hosts << EOF
192.168.181.158 k8s-master01
192.168.181.159 k8s-node01
192.168.181.160 k8s-node02
EOF

关闭selinux

sed -i 's/^SELINUX=enforcing$/SELINUX=disabled/' /etc/selinux/config && setenforce 0

时间同步

# 安装 chrony 服务，centos7.6默认自带了，没有的按如下安装
yum install -y chrony
systemctl start chronyd
systemctl enable chronyd

关闭防火墙

systemctl stop firewalld
systemctl disable firewalld

关闭swap分区

sed -i '11s/\/dev/# \/dev/g' /etc/fstab
swapoff -a

yum源设置

mkdir /etc/yum.repos.d/ori
mv /etc/yum.repos.d/CentOS-* /etc/yum.repos.d/ori/
cat > /etc/yum.repos.d/CentOS-Base.repo << "EOF"
# CentOS-Base.repo
#
# The mirror system uses the connecting IP address of the client and the
# update status of each mirror to pick mirrors that are updated to and
# geographically close to the client.  You should use this for CentOS updates
# unless you are manually picking other mirrors.
#
# If the mirrorlist= does not work for you, as a fall back you can try the
# remarked out baseurl= line instead.
#
#

[base]
name=CentOS-$releasever - Base
baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos/$releasever/os/$basearch/
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7

#released updates
[updates]
name=CentOS-$releasever - Updates
baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos/$releasever/updates/$basearch/
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=updates
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7

#additional packages that may be useful
[extras]
name=CentOS-$releasever - Extras
baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos/$releasever/extras/$basearch/
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=extras
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7

#additional packages that extend functionality of existing packages
[centosplus]
name=CentOS-$releasever - Plus
baseurl=https://mirrors.tuna.tsinghua.edu.cn/centos/$releasever/centosplus/$basearch/
#mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=centosplus
gpgcheck=1
enabled=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7
EOF

安装epel并配置epel源

yum install -y epel-release
cat > /etc/yum.repos.d/epel.repo <<"EOF"
[epel]
name=Extra Packages for Enterprise Linux 7 - $basearch
baseurl=https://mirrors.tuna.tsinghua.edu.cn/epel/7/$basearch
#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=$basearch
failovermethod=priority
enabled=1
gpgcheck=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7

[epel-debuginfo]
name=Extra Packages for Enterprise Linux 7 - $basearch - Debug
baseurl=https://mirrors.tuna.tsinghua.edu.cn/epel/7/$basearch/debug
#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-debug-7&arch=$basearch
failovermethod=priority
enabled=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7
gpgcheck=1

[epel-source]
name=Extra Packages for Enterprise Linux 7 - $basearch - Source
baseurl=https://mirrors.tuna.tsinghua.edu.cn/epel/7/SRPMS
#mirrorlist=https://mirrors.fedoraproject.org/metalink?repo=epel-source-7&arch=$basearch
failovermethod=priority
enabled=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-7
gpgcheck=1
EOF

yum clean all
yum makecache

升级内核

查看当前发行版和内核
​```
[root@k8s-master01 ~]# cat /etc/redhat-release
CentOS Linux release 7.6.1810 (Core)
[root@k8s-master01 ~]# uname -r
3.10.0-957.el7.x86_64
​```
启用 ELRepo 仓库
​```
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
​```
查看可用内核包
​```
yum --disablerepo="*" --enablerepo="elrepo-kernel" list available
​```
安装最新内核
​```
yum --enablerepo=elrepo-kernel install -y kernel-ml kernel-ml-devel kernel-ml-headers
​```
查看已安装的内核
​```
[root@k8s-master01 ~]# awk -F\' '$1=="menuentry " {print i++ " : " $2}' /etc/grub2.cfg
0 : CentOS Linux (5.1.14-1.el7.elrepo.x86_64) 7 (Core)
1 : CentOS Linux (3.10.0-957.el7.x86_64) 7 (Core)
2 : CentOS Linux (0-rescue-8d615a05e5de49a08ca0e56b285958f7) 7 (Core)
​```
设置启动内核，即就是编号为0的那个
​```
grub2-set-default 0
sed -i 's/saved/0/g'  /etc/default/grub
​```
关闭NUMA
​```
sed -i 's/quiet/quiet numa=off/g' /etc/default/grub
​```
重新生成grub2配置文件
​```
grub2-mkconfig -o /boot/grub2/grub.cfg
reboot
​```

配置IPVS内核

默认情况下，Kube-proxy将在kubeadm部署的集群中以iptables模式运行

需要注意的是，当内核版本大于4.19时，移除了nf_conntrack_ipv4模块，kubernetes官方建议使用nf_conntrack代替，否则报错无法找到nf_conntrack_ipv4模块

yum install -y ipset ipvsadm
cat > /etc/sysconfig/modules/ipvs.modules <<EOF
#!/bin/bash
modprobe -- ip_vs
modprobe -- ip_vs_rr
modprobe -- ip_vs_wrr
modprobe -- ip_vs_sh
modprobe -- nf_conntrack
EOF

chmod +x /etc/sysconfig/modules/ipvs.modules
bash /etc/sysconfig/modules/ipvs.modules

配置内核参数

cat > /etc/sysctl.d/k8s.conf <<EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_nonlocal_bind = 1
net.ipv4.ip_forward = 1
vm.swappiness=0
EOF

modprobe br_netfilter
sysctl -p /etc/sysctl.d/k8s.conf

打开文件数

echo "* soft nofile 65536" >> /etc/security/limits.conf
echo "* hard nofile 65536" >> /etc/security/limits.conf

安装docker

wget -P /etc/yum.repos.d/ http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
yum install -y docker-ce

docker配置修改和镜像加速

[ ! -d /etc/docker ] && mkdir /etc/docker
cat > /etc/docker/daemon.json <<EOF
{
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2",
  "storage-opts": [
    "overlay2.override_kernel_check=true"
  ],
  "registry-mirrors": ["https://uyah70su.mirror.aliyuncs.com"]
}
EOF
# 启动docker
systemctl daemon-reload && systemctl restart docker && systemctl enable docker

到这一步完成之后可以打虚拟机快照保存状态了

安装 kubelet、kubeadm 和 kubectl

kubelet 运行在 Cluster 所有节点上，负责启动 Pod 和容器。
kubeadm 用于初始化 Cluster。
kubectl 是 Kubernetes 命令行工具。通过 kubectl 可以部署和管理应用，查看各种资源，创建、删除和更新各种组件。

# 添加阿里云yum源
cat <<EOF > /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=1
repo_gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
# 默认安装最新版本，此处为1.15.1
yum install -y kubeadm kubelet kubectl
systemctl enable kubelet && systemctl start kubelet

启用kubectl命令的自动补全功能

# 安装并配置bash-completion
yum install -y bash-completion
echo 'source /usr/share/bash-completion/bash_completion' >> /etc/profile
source /etc/profile
echo "source <(kubectl completion bash)" >> ~/.bashrc
source ~/.bashrc

到这一步可以打一个快照，方便后续进行flannel网络测试

初始化Master

使用kubeadm config print init-defaults可以打印集群初始化默认的使用的配置

这里采用命令行方式初始化，注意默认镜像仓库由于在国外，不能访问，这里指定为阿里云镜像仓库

需要注意这里使用的网络方案是flannel，注意CIDR

# kubernetes-version版本和前面安装的kubelet和kubectl一致
[root@k8s-master01 ~]# kubeadm init --apiserver-advertise-address 192.168.181.158 --kubernetes-version="v1.15.1" --pod-network-cidr=10.244.0.0/16 --image-repository=registry.aliyuncs.com/google_containers | tee kubeadm-init.log

初始化完成之后，底部会有节点加入master方法提示，其他两个节点复制执行即可加入master节点

配置kubectl命令

无论在master节点或node节点，要能够执行kubectl命令必须进行以下配置
root用户配置

cat << EOF >> ~/.bashrc
export KUBECONFIG=/etc/kubernetes/admin.conf
EOF
source ~/.bashrc

普通用户配置

 mkdir -p $HOME/.kube
 sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
 sudo chown $(id -u):$(id -g) $HOME/.kube/config

等集群配置完成后，可以在master节点和node节点进行以上配置，以支持kubectl命令。针对node节点复制master节点/etc/kubernetes/admin.conf到本地。
查看集群状态
配置完成后在任意主机上查看

kubectl get nodes
kubectl get pod -n kube-system
kubectl get cs

由于未安装网络插件，coredns处于pending状态，node处于notready状态。

安装flannel网络

Kubernetes 支持多种网络方案，这里我们先使用 flannel。

这里要注意，默认的flannel配置文件拉取镜像在国外，国内拉取失败，很多网上文章没注意这一步，导致flannel部署失败

# master安装flannel
[root@k8s-master ~]# mkdir k8s
wget -P k8s/ https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml
sed -i 's#quay.io#quay-mirror.qiniu.com#g' k8s/kube-flannel.yml
kubectl apply -f k8s/kube-flannel.yml

加入node节点

节点加入master，从初始化输出或kubeadm-init.log中获取命令

kubeadm join 192.168.181.158:6443 --token l3ofhh.ebsctxgnlub8mwei \
    --discovery-token-ca-cert-hash sha256:c9bbe567f213051ebed76b0ac217f231356a4a6078245b01498f83ce8b9a73c1

移除node节点

# 需要移除的k8s-node2节点执行
kubectl drain k8s-node2 --delete-local-data --force --ignore-daemonsets
kubectl delete node k8s-node2
kubeadm reset
ifconfig cni0 down
ip link delete cni0
ifconfig flannel.1 down
ip link delete flannel.1
rm -rf /var/lib/cni/
# k8s-master01 执行
kubectl delete node k8s-node2
# 执行完之后，要重新加入可以按前面的步骤执行添加node和配置kubectl命令
# 集群初始化如果遇到问题（例如CNI问题），k8s-node2可以使用下面的命令进行清理，执行之后还未解决，那么在k8s-master01节点继续执行如下语句
kubeadm reset
systemctl stop kubelet
systemctl stop docker
rm -rf /var/lib/cni/
rm -rf /var/lib/kubelet/*
rm -rf /etc/cni/
ifconfig cni0 down
ifconfig flannel.1 down
ifconfig docker0 down
ip link delete cni0
ip link delete flannel.1
##重启kubelet
systemctl restart kubelet
##重启docker
systemctl restart docker

信息查看

kubectl get nodes
kubectl get pods -n kube-system
kubectl get pods --all-namespaces
# 查看日志
journalctl --since 12:00:00 -u kubelet

测试DNS

kubectl run curl --image=radial/busyboxplus:curl -it
# 进入应用后，解析DNS，这里一定是可以解析出默认DNS，否则后续pod启动无法分配ip
nslookup kubernetes.default

kube-proxy开启ipvs

kubectl get configmap kube-proxy -n kube-system -o yaml > kube-proxy-configmap.yaml
sed -i 's/mode: ""/mode: "ipvs"/' kube-proxy-configmap.yaml
kubectl apply -f kube-proxy-configmap.yaml
rm -f kube-proxy-configmap.yaml
kubectl get pod -n kube-system | grep kube-proxy | awk '{system("kubectl delete pod "$1" -n kube-system")}'

或者用以下方法也可以修改，修改ConfigMap的kube-system/kube-proxy中的config.conf，mode: "ipvs"

kubectl edit configmap kube-proxy -n kube-system
kubectl get pod -n kube-system | grep kube-proxy | awk '{system("kubectl delete pod "$1" -n kube-system")}'

查看IPVS配置

yum install -y ipvsadm
ipvsadm -ln