Ceph luminous版本 Centos7.4 安装配置

时间：2022-07-24 12:37:48
#环境准备
#升级内核
#官网说明 http://docs.ceph.com/docs/master/start/os-recommendations/#linux-kernel
#CentOS7 升级内核

rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
#安装kernel最新稳定版
yum --enablerepo=elrepo-kernel install kernel-lt-devel kernel-lt -y
awk -F\' '$1=="menuentry " {print $2}' /etc/grub2.cfg #查看启动项
grub2-set-default 0 #设置第一行为默认启动
rpm -e elrepo-release #删除源
reboot
#uname -r #重启后查看内核
#4.4.98-1.el7.elrepo.x86_64
###########################

#使用阿里源
rm -f /etc/yum.repos.d/*
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
sed -i '/aliyuncs.com/d' /etc/yum.repos.d/*.repo #删除阿里内网地址
#创建ceph源
echo '#阿里ceph源
[ceph]
name=ceph
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/x86_64/
gpgcheck=0
[ceph-noarch]
name=cephnoarch
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/noarch/
gpgcheck=0
[ceph-source]
name=ceph-source
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/SRPMS/
gpgcheck=0
#'>/etc/yum.repos.d/ceph.repo
yum clean all && yum makecache #生成缓存
#关闭selinux、防火墙
systemctl stop firewalld.service
systemctl disable firewalld.service
firewall-cmd --state
sed -i '/^SELINUX=.*/c SELINUX=disabled' /etc/selinux/config
sed -i 's/^SELINUXTYPE=.*/SELINUXTYPE=disabled/g' /etc/selinux/config
grep --color=auto '^SELINUX' /etc/selinux/config
setenforce 0

#时间同步
[[ -f /usr/sbin/ntpdate ]] || { echo "install ntp";yum install ntp -y &> /dev/null; } #若没NTP则安装
/usr/sbin/ntpdate ntp6.aliyun.com 
echo "*/3 * * * * /usr/sbin/ntpdate ntp6.aliyun.com  &> /dev/null" > /tmp/crontab
crontab /tmp/crontab

###########################
#网络

# #外网规划
# 192.168.0.51 ceph1

#内网规划，写入hosts
echo '#ceph
192.168.1.101 ceph1
192.168.1.102 ceph2
192.168.1.103 ceph3
192.168.1.104 ceph4
'>>/etc/hosts

#配置hostname、固定ip
Nu=1
hostnamectl set-hostname ceph$Nu
NetName=em1 #网卡名称
rm -f /etc/sysconfig/network-scripts/ifcfg-$NetName
nmcli con add con-name $NetName ifname $NetName autoconnect yes type ethernet ip4 192.168.1.10$Nu/24 \
 ipv4.dns "8.8.8.8" #ipv4.gateway "192.168.1.1"
nmcli connection reload #重新加载网络


#磁盘(添加2快磁盘)
# [root@ceph01 ~]# lsblk
# NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT

# sdb      8:16   0  100G  0 disk 
# sdc      8:32   0  100G  0 disk

###########################
#ceph安装配置
###########################
#全部在ceph1操作

# #Ceph节点简介
# Mon(Montior)监控节点
# OSD(Object Storage Device)存储节点
# MDS(Meta Data Server)，Ceph作文件系统用到，本实例不配置

#ceph01作为部署节点
#ceph1 ceph2 ceph3作为Mon
#ceph2 ceph3 ceph4作为OSD

#ssh免密验证
所有节点执行：
yum install -y sshpass
rm -rf ~/.ssh
ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa
sshpass -p '123456' ssh-copy-id -o StrictHostKeyChecking=no -i /root/.ssh/id_rsa -p 22 root@ceph1
sshpass -p '123456' ssh-copy-id -o StrictHostKeyChecking=no -i /root/.ssh/id_rsa -p 22 root@ceph2
sshpass -p '123456' ssh-copy-id -o StrictHostKeyChecking=no -i /root/.ssh/id_rsa -p 22 root@ceph3
sshpass -p '123456' ssh-copy-id -o StrictHostKeyChecking=no -i /root/.ssh/id_rsa -p 22 root@ceph4
#以下在 ceph1节点执行
#ceph1 安装ceph-deploy配置工具
yum install -y ceph-deploy

#创建配置目录
mkdir /etc/ceph
cd /etc/ceph/

#初始化Mon配置
ceph-deploy new ceph{1,2,3}
#ceph-deploy new --cluster-network 192.168.1.0/24 --public-network 192.168.0.0/24 ceph{1,2,3}

# #修改冗余份数为2，日志大小2G
# #配置网络,单网卡忽略
echo '
public network = 192.168.0.0/24
cluster network = 192.168.1.0/24

mon_clock_drift_allowed = 2    
osd_journal_size = 4086
osd_pool_default_pg_num = 128
osd_pool_default_pgp_num = 128
osd pool default size = 2
osd pool default min size = 1
rbd_default_features = 1
client_quota = true
'>>./ceph.conf

#安装Ceph
ceph-deploy install ceph1 ceph2 ceph3 ceph4
#yum install -y ceph ceph-radosgw #实际上是安装这2个rpm

#初始化monitor和key
cd /etc/ceph/
ceph-deploy --overwrite-conf mon create-initial

#创建存储节点(使用sdb,sdc磁盘)
#lsblk #查看磁盘

#创建osd
ceph-deploy osd create --data /dev/sdb ceph2
ceph-deploy osd create --data /dev/sdb ceph3
ceph-deploy osd create --data /dev/sdb ceph4
ceph-deploy osd create --data /dev/sdc ceph2
ceph-deploy osd create --data /dev/sdc ceph3
ceph-deploy osd create --data /dev/sdc ceph4

#拷贝配置及密钥
ceph-deploy admin ceph1 ceph2 ceph3 ceph4
chmod 644 /etc/ceph/ceph.client.admin.keyring

#创建mon
ceph-deploy --overwrite-conf mon create ceph1
ceph-deploy --overwrite-conf admin ceph1
ceph-deploy --overwrite-conf mon create ceph2
ceph-deploy --overwrite-conf admin ceph2
ceph-deploy --overwrite-conf mon create ceph3
ceph-deploy --overwrite-conf admin ceph3

#添加mgr
#ceph 12开始，monitor必须添加mgr
ceph-deploy mgr create ceph{1,2,3}:mon_mgr
#ceph-deploy mgr create ceph1:mon_mgr ceph2:mon_mgr ceph3:mon_mgr

#启用dashboard (在mon节点)
ceph mgr module enable dashboard
#http://ip:7000 访问dashboard
netstat -antp|grep 7000

# 设置dashboard的ip和端口
# ceph config-key put mgr/dashboard/server_addr 192.168.0.196  #vip地址
# ceph config-key put mgr/dashboard/server_port 7000
# systemctl restart ceph-mgr@ceph1

###########################
#查看相关命令
ceph health
ceph -s
ceph osd tree

ceph df
ceph mon stat
ceph osd stat
ceph pg stat

ceph osd lspools
ceph auth list

###########################
#参考
https://segmentfault.com/a/1190000011581513
http://blog.csdn.net/wylfengyujiancheng/article/details/78461801

https://segmentfault.com/a/1190000011589519
https://www.cnblogs.com/netmouser/p/6876846.html

#中文 文档
http://docs.ceph.org.cn/man/8/ceph-deploy/

#ceph工作原理和安装
http://www.jianshu.com/p/25163032f57f

###########################
###########################
#简单ceph性能测试

#创建测试池mytest
ceph osd pool create mytest 128
rados lspools
# ceph osd pool set mytest size 2 #副本为2
# ceph osd pool delete mytest #删除


#Rados性能测试(关注 bandwidth带宽,latency延迟)
rados bench -p mytest 10 write --no-cleanup #写测试10秒
rados bench -p mytest 10 seq  #顺序读
rados bench -p mytest 10 rand #随机读
rados -p mytest cleanup #清理测试数据

#rbd块设备测试
rbd create --size 2G mytest/test1 #创建块设备映像test1
rbd ls mytest
rbd info mytest/test1

rbd map mytest/test1   #映射块设备
#/dev/rbd0
#rbd showmapped         #查看已映射块设备
#挂载
mkfs.xfs /dev/rbd0
mkdir -p /mnt/ceph
mount /dev/rbd0 /mnt/ceph/
df -h /mnt/ceph

#测试
rbd bench-write mytest/test1
#默认参数io 4k,线程数16,总写入1024M, seq顺序写


rbd unmap mytest/test1 #取消块设备映射
rbd rm mytest/test1    #删除块设备映像

#参考
https://www.cnblogs.com/sammyliu/p/5557666.html
###########################

 

标签：

相关文章