Kubernetes集群部署之三ETCD集群部署

时间：2022-01-23 06:56:58

kuberntes 系统使用 etcd 存储所有数据，本文档介绍部署一个三节点高可用 etcd 集群的步骤,这三个节点复用 kubernetes 集群机器k8s-master、k8s-node-1、k8s-node-2.

1.准备etcd软件包并分发etcd文件:

[root@k8s-master ~]# cd /usr/local/src

[root@k8s-master src]#wget https://github.com/coreos/etcd/releases/download/v3.2.18/etcd-v3.2.18-linux-amd64.tar.gz

[root@k8s-master src]# tar zxf etcd-v3.2.18-linux-amd64.tar.gz

[root@k8s-master src]# cd etcd-v3.2.18-linux-amd64

[root@k8s-master etcd-v3.2.18-linux-amd64]# cp etcd etcdctl /opt/kubernetes/bin/

[root@k8s-master etcd-v3.2.18-linux-amd64]# scp etcd etcdctl 10.200.3.106:/opt/kubernetes/bin/

[root@k8s-master etcd-v3.2.18-linux-amd64]# scp etcd etcdctl 10.200.3.107:/opt/kubernetes/bin/

2.创建 etcd 证书签名请求：

[root@k8s-master ~]# cd /usr/local/src/ssl
[root@k8s-master ssl]# cat > etcd-csr.json <<EOF

{

  "CN": "etcd",

  "hosts": [

  "127.0.0.1",

  "10.200.3.105",

  "10.200.3.106",

  "10.200.3.107"

  ],

  "key": {

    "algo": "rsa",

    "size":

  },

  "names": [

    {

      "C": "CN",

      "ST": "BeiJing",

      "L": "BeiJing",

      "O": "k8s",

      "OU": "System"

    }

  ]

}

EOF

3.生成 etcd 证书和私钥：

[root@k8s-master ssl]# cfssl gencert -ca=/opt/kubernetes/ssl/ca.pem \

  -ca-key=/opt/kubernetes/ssl/ca-key.pem \

  -config=/opt/kubernetes/ssl/ca-config.json \

  -profile=kubernetes etcd-csr.json | cfssljson -bare etcd

会生成以下证书文件

[root@k8s-master ssl]# ls -l etcd*

-rw-r--r--  root root  Mar   : etcd.csr

-rw-r--r--  root root   Mar   : etcd-csr.json

-rw-------  root root  Mar   : etcd-key.pem

-rw-r--r--  root root  Mar   : etcd.pem

4.将证书移动到/opt/kubernetes/ssl目录下:

[root@k8s-master ssl]# cp etcd*.pem /opt/kubernetes/ssl

[root@k8s-master ssl]# scp etcd*.pem 10.200.3.106:/opt/kubernetes/ssl

[root@k8s-master ssl]# scp etcd*.pem 10.200.3.107:/opt/kubernetes/ssl

[root@k8s-master ssl]# rm -f etcd.csr etcd-csr.json

5.设置ETCD 配置文件

root@k8s-master ssl]#cat > /opt/kubernetes/cfg/etcd.conf <<EOF

#[member]

ETCD_NAME="k8s-master"

ETCD_DATA_DIR="/var/lib/etcd/default.etcd"

#ETCD_SNAPSHOT_COUNTER=""

#ETCD_HEARTBEAT_INTERVAL=""

#ETCD_ELECTION_TIMEOUT=""

ETCD_LISTEN_PEER_URLS="https://10.200.3.105:2380"

ETCD_LISTEN_CLIENT_URLS="https://10.200.3.105:2379,https://127.0.0.1:2379"

#ETCD_MAX_SNAPSHOTS=""

#ETCD_MAX_WALS=""

#ETCD_CORS=""

#[cluster]

ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.200.3.105:2380"

# if you use different ETCD_NAME (e.g. test),

# set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."

ETCD_INITIAL_CLUSTER="k8s-master=https://10.200.3.105:2380,k8s-node-1=https://10.200.3.106:2380,k8s-node-2=https://10.200.3.107:2380"

ETCD_INITIAL_CLUSTER_STATE="new"

ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"

ETCD_ADVERTISE_CLIENT_URLS="https://10.200.3.105:2379"

#[security]

CLIENT_CERT_AUTH="true"

ETCD_CA_FILE="/opt/kubernetes/ssl/ca.pem"

ETCD_CERT_FILE="/opt/kubernetes/ssl/etcd.pem"

ETCD_KEY_FILE="/opt/kubernetes/ssl/etcd-key.pem"

PEER_CLIENT_CERT_AUTH="true"

ETCD_PEER_CA_FILE="/opt/kubernetes/ssl/ca.pem"

ETCD_PEER_CERT_FILE="/opt/kubernetes/ssl/etcd.pem"

ETCD_PEER_KEY_FILE="/opt/kubernetes/ssl/etcd-key.pem"

EOF

6.创建ETCD系统服务

[root@k8s-master ssl]# cat > /etc/systemd/system/etcd.service <<EOF

[Unit]

Description=Etcd Server

After=network.target

[Service]

Type=simple

WorkingDirectory=/var/lib/etcd

EnvironmentFile=-/opt/kubernetes/cfg/etcd.conf

# set GOMAXPROCS to number of processors

ExecStart=/bin/bash -c "GOMAXPROCS=$(nproc) /opt/kubernetes/bin/etcd"

Type=notify

[Install]

WantedBy=multi-user.target

EOF

7.文件分发到两个node节点中:

[root@k8s-master ~]# scp /opt/kubernetes/cfg/etcd.conf 10.200.3.106:/opt/kubernetes/cfg/

[root@k8s-master ~]# scp /etc/systemd/system/etcd.service 10.200.3.106:/etc/systemd/system/

[root@k8s-master ~]# scp /opt/kubernetes/cfg/etcd.conf 10.200.3.107:/opt/kubernetes/cfg/

[root@k8s-master ~]# scp /etc/systemd/system/etcd.service 10.200.3.107:/etc/systemd/system/

8.修改node节点etcd.conf文件,ETCD_NAME改为本机的hostname.ETCD...URLS改为本机的ip地址

在k8s-node-1 修改etcd.conf文件

[root@k8s-node- ~]# cat >/opt/kubernetes/cfg/etcd.conf <<EOF

#[member]

ETCD_NAME="k8s-node-1"

ETCD_DATA_DIR="/var/lib/etcd/default.etcd"

#ETCD_SNAPSHOT_COUNTER=""

#ETCD_HEARTBEAT_INTERVAL=""

#ETCD_ELECTION_TIMEOUT=""

ETCD_LISTEN_PEER_URLS="https://10.200.3.106:2380"

ETCD_LISTEN_CLIENT_URLS="https://10.200.3.106:2379,https://127.0.0.1:2379"

#ETCD_MAX_SNAPSHOTS=""

#ETCD_MAX_WALS=""

#ETCD_CORS=""

#[cluster]

ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.200.3.106:2380"

# if you use different ETCD_NAME (e.g. test),

# set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."

ETCD_INITIAL_CLUSTER="k8s-master=https://10.200.3.105:2380,k8s-node-1=https://10.200.3.106:2380,k8s-node-2=https://10.200.3.107:2380"

ETCD_INITIAL_CLUSTER_STATE="new"

ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"

ETCD_ADVERTISE_CLIENT_URLS="https://10.200.3.106:2379"

#[security]

CLIENT_CERT_AUTH="true"

ETCD_CA_FILE="/opt/kubernetes/ssl/ca.pem"

ETCD_CERT_FILE="/opt/kubernetes/ssl/etcd.pem"

ETCD_KEY_FILE="/opt/kubernetes/ssl/etcd-key.pem"

PEER_CLIENT_CERT_AUTH="true"

ETCD_PEER_CA_FILE="/opt/kubernetes/ssl/ca.pem"

ETCD_PEER_CERT_FILE="/opt/kubernetes/ssl/etcd.pem"

ETCD_PEER_KEY_FILE="/opt/kubernetes/ssl/etcd-key.pem"

EOF

在k8s-node-2节点 修改etcd.conf文件

[root@k8s-node- ~]# cat >/opt/kubernetes/cfg/etcd.conf  <<EOF

#[member]

ETCD_NAME="k8s-node-2"

ETCD_DATA_DIR="/var/lib/etcd/default.etcd"

#ETCD_SNAPSHOT_COUNTER=""

#ETCD_HEARTBEAT_INTERVAL=""

#ETCD_ELECTION_TIMEOUT=""

ETCD_LISTEN_PEER_URLS="https://10.200.3.107:2380"

ETCD_LISTEN_CLIENT_URLS="https://10.200.3.107:2379,https://127.0.0.1:2379"

#ETCD_MAX_SNAPSHOTS=""

#ETCD_MAX_WALS=""

#ETCD_CORS=""

#[cluster]

ETCD_INITIAL_ADVERTISE_PEER_URLS="https://10.200.3.107:2380"

# if you use different ETCD_NAME (e.g. test),

# set ETCD_INITIAL_CLUSTER value for this name, i.e. "test=http://..."

ETCD_INITIAL_CLUSTER="k8s-master=https://10.200.3.105:2380,k8s-node-1=https://10.200.3.106:2380,k8s-node-2=https://10.200.3.107:2380"

ETCD_INITIAL_CLUSTER_STATE="new"

ETCD_INITIAL_CLUSTER_TOKEN="k8s-etcd-cluster"

ETCD_ADVERTISE_CLIENT_URLS="https://10.200.3.107:2379"

#[security]

CLIENT_CERT_AUTH="true"

ETCD_CA_FILE="/opt/kubernetes/ssl/ca.pem"

ETCD_CERT_FILE="/opt/kubernetes/ssl/etcd.pem"

ETCD_KEY_FILE="/opt/kubernetes/ssl/etcd-key.pem"

PEER_CLIENT_CERT_AUTH="true"

ETCD_PEER_CA_FILE="/opt/kubernetes/ssl/ca.pem"

ETCD_PEER_CERT_FILE="/opt/kubernetes/ssl/etcd.pem"

ETCD_PEER_KEY_FILE="/opt/kubernetes/ssl/etcd-key.pem"

EOF

9.加载并启动系统服务(先启动node节点的ectd服务，然后在启动master端的etcd服务，避免timeout.):

[root@k8s-master ~]# systemctl daemon-reload

[root@k8s-master ~]# systemctl enable etcd

[root@k8s-master ~]# systemctl start etcd

[root@k8s-master ~]# systemctl status etcd

10.验证集群

[root@k8s-master ~]#etcdctl --endpoints=https://10.200.3.105:2379 \

  --ca-file=/opt/kubernetes/ssl/ca.pem \

  --cert-file=/opt/kubernetes/ssl/etcd.pem \

  --key-file=/opt/kubernetes/ssl/etcd-key.pem cluster-health

输出结果：

member ccbb1e9d1fcf5b1 is healthy: got healthy result from https://10.200.3.105:2379

member 69e75d4dd1a9a289 is healthy: got healthy result from https://10.200.3.106:2379

member fe4515ae4c34c4e2 is healthy: got healthy result from https://10.200.3.107:2379

cluster is healthy

使用etcdctl member list 来查看集群中的成员列表

#etcdctl --endpoints=https://10.200.3.105:2379 \

  --ca-file=/opt/kubernetes/ssl/ca.pem \

  --cert-file=/opt/kubernetes/ssl/etcd.pem \

  --key-file=/opt/kubernetes/ssl/etcd-key.pem member list

输出结果：

4c650c413eacdd52: name=k8s-master peerURLs=https://10.200.3.105:2380 clientURLs=https://10.200.3.105:2379 isLeader=false

bc2b7558b8439c74: name=k8s-node- peerURLs=https://10.200.3.107:2380 clientURLs=https://10.200.3.107:2379 isLeader=false

d71aa8654024396a: name=k8s-node- peerURLs=https://10.200.3.106:2380 clientURLs=https://10.200.3.106:2379 isLeader=true

至此etcd集群搭建完成!!!

相关文章