kubeadm 安装kubernetes

清除集群

# 在全部集群节点执行
kubeadm reset

## ls /sys/class/net/
ip add |grep "cali"|awk '{print $2}'|tr -d ":"| xargs  -i ifconfig {} down
ip add |grep "cali"|awk -F@ '{print $1}'|awk '{print $NF}'| xargs -i ip link delete {}
modprobe -r ipip # 删除Tunl0
ifconfig cni0 down && ip link delete cni0
ifconfig flannel.1 down && ip link delete flannel.1
rm -rf /run/flannel/subnet.env
rm -rf /var/lib/cni/
mv /etc/kubernetes/ /tmp
mv /var/lib/etcd /tmp
mv ~/.kube /tmp
iptables -F
iptables -t nat -F
ipvsadm -C
ip link del kube-ipvs0
ip link del dummy0


rm -rf /etc/containerd/config.toml
systemctl restart containerd

rm  /etc/cni/net.d/* -f # 删除flannel,calico网络配置
# yum remove  -y kubelet kubeadm kubectl

docker配置

{
"exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
    "log-opts": {
        "max-size": "100m"
}              
}
# 调整docker Cgroup Driver为systemd和日志格式设定
# docker log日志大小

containerd

yum install containerd

kubeadm方式部署集群

环境准备

• 已安装docker

配置环境

## 设置每个机器自己的hostname
hostnamectl set-hostname k8s-1

## 将 SELinux 设置为 permissive 模式（相当于将其禁用）
sudo setenforce 0
sudo sed -i 's/^SELINUX=enforcing$/SELINUX=permissive/' /etc/selinux/config

## 关闭swap
swapoff -a  
sed -ri 's/.*swap.*/#&/' /etc/fstab

## 允许 iptables 检查桥接流量
cat <<EOF | sudo tee /etc/modules-load.d/k8s.conf
br_netfilter
EOF
 
cat <<EOF | sudo tee /etc/sysctl.d/k8s.conf
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF


sudo sysctl --system

hosts 配置

192.168.66.161 k8s-1
192.168.66.162 k8s-2
192.168.66.163 k8s-3

安装kubelet、kubeadm、kubectl

## 检查Docker配置 
{
"exec-opts": ["native.cgroupdriver=systemd"],
"graph": "/var/lib",
"log-driver": "json-file",
"log-opts": {
"max-size": "100m"
  },
"registry-mirrors": [
"https://1lcdq5an.mirror.aliyuncs.com",
"https://mirror.ccs.tencentyun.com",
"http://hub-mirror.c.163.com"
  ]
}

## 配置yum源
cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
   http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF
 
 
## 所有节点都需要安装 kubelet，kubeadm，kubectl 组件; 版本查看: https://github.com/kubernetes/kubernetes/tags
## yum install -y kubelet-1.22.13 kubeadm-1.22.13 kubectl-1.22.13



## 启动kubelet
systemctl enable --now kubelet

init : 初始化master节点

• 方式1: 加载配置初始化

kubeadm config print init-defaults > kubeadm.yaml

## 修改内容
localAPIEndpoint:
  advertiseAddress: 192.168.66.161  // 广播地址
nodeRegistration:
  name: k8s-1 // master节点名
imageRepository: registry.aliyuncs.com/google_containers  // 使用国内源

networking:
  podSubnet: 10.244.0.0/16   // 添加pod网段

## 查看需要使用的镜像列表,若无问题，将得到如下列表
kubeadm config images list --config kubeadm.yaml
kubeadm config images pull  ## 各节点可以提交拉去镜像

## master节点通过配置文件初始化集群
kubeadm init --config kubeadm.yaml

• 方式2: 初始化集群命令

## master节点通过命令行参数初始化集群 
kubeadm init --kubernetes-version=1.23.12 \
--apiserver-advertise-address=66.94.121.23 \
--image-repository registry.aliyuncs.com/google_containers \
--service-cidr=10.96.0.0/16 \
--pod-network-cidr=10.244.0.0/16
# 172.16.0.0/16

## v1.22.13 版本
kubeadm init \  
--apiserver-advertise-address=192.168.66.161 \
--control-plane-endpoint=k8s-master \
--image-repository registry.aliyuncs.com/google_containers \
--kubernetes-version v1.22.13 \
--service-cidr=10.96.0.0/16 \
--pod-network-cidr=10.244.0.0/16

# --image-repository registry.aliyuncs.com/google_containers \
# --image-repository registry.cn-hangzhou.aliyuncs.com/lfy_k8s_images \

参数说明：

-–apiserver-advertise-address：集群通告地址

-–image-repository：由于默认拉取镜像地址k8s.gcr.io国内无法访问，这里指定阿里云镜像仓库地址

-–kubernetes-version： K8s版本，与上面安装的一致

-–service-cidr ：集群内部虚拟网络，Pod统一访问入口

-–pod-network-cidr Pod：网络，需要与接下来部署的CNI网络组件yaml中保持一致

kube config添加文件

mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config

## 检查是否有报错
$ systemctl status kubelet -l
$ systemctl status containerd -l
## 查看组件运行情况
$ kubectl  -n kube-system get po
NAMESPACE     NAME                            READY   STATUS              RESTARTS   AGE
kube-system   coredns-c676cc86f-6w5cx         0/1     ContainerCreating   0          3m2s
kube-system   coredns-c676cc86f-gb4n6         0/1     ContainerCreating   0          3m2s
kube-system   etcd-k8s-1                      1/1     Running             0          3m15s
kube-system   kube-apiserver-k8s-1            1/1     Running             0          3m15s
kube-system   kube-controller-manager-k8s-1   1/1     Running             0          3m15s
kube-system   kube-proxy-5b2kp                1/1     Running             0          3m2s
kube-system   kube-scheduler-k8s-1            1/1     Running             0          3m16s

配置跨主机通信网络

参考文档：​​https://kubernetes.io/zh-cn/docs/concepts/cluster-administration/addons/​​

• flannel

## 配置flannel,如果自定义podCIDR（不是10.244.0.0/16）修改进行修改
kubectl apply -f https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml

• calico

kubectl apply -f calico.yaml https://docs.projectcalico.org/manifests/calico.yaml

## 自定义
- name: CALICO_IPV4POOL_CIDR
  value: "10.xx.xx.xx/16"
- name: IP_AUTODETECTION_METHOD
  value: "interface=eth*"

join: slave节点加入集群

## 其他节点执行join加入集群

kubeadm join 192.168.66.161:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:25b2f78741215axxx

## 遗忘token可在主节点重新生成
kubeadm token create --print-join-command

集群状态部署完毕

## 查看状态
$ kubectl get nodes
NAME    STATUS   ROLES                  AGE     VERSION
k8s-1   Ready    control-plane,master   58m     v1.22.13
k8s-2   Ready    <none>                 7m30s   v1.22.13
k8s-3   Ready    <none>                 6m24s   v1.22.13

## 查看系统组件就绪状态
$ kubectl get po -w  -n kube-system

## 集群信息
$ kubectl cluster-info

集群设置

master节点可调度

默认master节点不可调度，pod不会分配到master节点

## 设置master可调度pod
kubectl taint node k8s-1 node-role.kubernetes.io/master:NoSchedule-
## 禁止master节点调度pod
kubectl taint node k8s-1 node-role.kubernetes.io/master:NoSchedule

设置kubectl自动补全

yum install bash-completion -y
source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)
echo "source <(kubectl completion bash)" >> ~/.bashrc

kubeadm证书更新

kubeadm安装默认证书，证书默认有效期为1年，可以通过如下方式修改为10年

cd /etc/kubernetes/pki

# 查看当前证书有效期
for i in $(ls *.crt); do echo "===== $i ====="; openssl x509 -in $i -text -noout | grep -A 3 'Validity' ; done

mkdir backup_key; cp -rp ./* backup_key/
git clone https://github.com/yuyicai/update-kube-cert.git
cd update-kube-cert/ 
bash update-kubeadm-cert.sh all

# 查看服务
kubectl get nodes
kubectl get pods -n kube-system -o wide|grep k8s-1

重建管理服务

kubectl -n kube-system delete po \

kube-apiserver-k8s-1 \

kube-controller-manager-k8s-1 \

kube-scheduler-k8s-1

scheduler Unhealthy问题

## 查看Master健康状态
kubectl get componentstatuses
kubectl get cs

## /etc/kubernetes/manifests/kube-scheduler.yaml 
 containers:
- command:
- kube-scheduler
    ... ...
#- --port=0
# –port=0表示禁用http访问，apiserver默认向127.0.0.1发送请求。

dashboard web

版本地址

​​https://github.com/kubernetes/dashboard/releases​​

## 根据对应版本安装dashboard
kubectl apply -f https://raw.githubusercontent.com/kubernetes/dashboard/v2.7.0/aio/deploy/recommended.yaml

• 授权文件 dashboard-admin.yml

kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: admin
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
roleRef:
  kind: ClusterRole
  name: cluster-admin
  apiGroup: rbac.authorization.k8s.io
subjects:
- kind: ServiceAccount
  name: admin
  namespace: kubernetes-dashboard

---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin
  namespace: kubernetes-dashboarddashboard-admin.yml

## 授权
$ kubectl  apply -f  dashboard-admin.yml
## 查看状态
$ kubectl -n kubernetes-dashboard get svc,pod

$ kubectl -n kubernetes-dashboard get secret |grep admin-token
admin-token-7b9cm                  kubernetes.io/service-account-token   3      2m5s

# 使用该命令拿到admin-token-xxxx，然后粘贴生产token
$ kubectl -n kubernetes-dashboard get secret admin-token-7b9cm   -o jsonpath={.data.token}|base64 -d
eyJhbGciOiJSUzI1NiIsImtpZCI6Ik1rb2xHWHMwbWFPMjJaRzhleGRqaExnVi1BLVNRc2txaEhETmVpRzlDeDQifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlcm5ldGVzLWRhc2hib2FyZCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJhZG1pbi10b2tlbi1mcWRwZiIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50Lm5hbWUiOiJhZG1pbiIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjYy.......

系统组件和资源

• 查看kube系统组件

kubectl get pod -owide -n kube-system

• kube系统静态pod目录

# ls /etc/kubernetes/manifests/
-rw------- 1 root root 2144 1月  24 18:57 etcd.yaml
-rw------- 1 root root 3296 1月  24 18:57 kube-apiserver.yaml
-rw------- 1 root root 2798 1月  24 18:57 kube-controller-manager.yaml
-rw------- 1 root root 1384 1月  24 18:57 kube-scheduler.yaml

systemctl status kubelet
# kubelet组件会实时拉取静态目录下配置进行pod更新