#环境准备

###########################

#升级内核(可选,)

#官方说明 http://docs.ceph.com/docs/master/start/os-recommendations/#linux-kernel

#CentOS7 升级内核到4. LTS

rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org

rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm

#安装kernel最新稳定版

yum --enablerepo=elrepo-kernel install kernel-lt-devel kernel-lt -y

awk -F\' '$=="menuentry " {print $}' /etc/grub2.cfg #查看启动项

grub2-set-default  #设置第一行为默认启动

rpm -e elrepo-release #删除源

reboot

#uname -r #重启后查看内核

#4.4.-.el7.elrepo.x86_64

###########################

###########################

#使用阿里源
rm -f /etc/yum.repos.d/*
wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo
sed -i '/aliyuncs.com/d' /etc/yum.repos.d/*.repo #删除阿里内网地址
#创建ceph源
echo '#阿里ceph源
[ceph]
name=ceph
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/x86_64/
gpgcheck=0
[ceph-noarch]
name=cephnoarch
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/noarch/
gpgcheck=0
[ceph-source]
name=ceph-source
baseurl=http://mirrors.aliyun.com/ceph/rpm-luminous/el7/SRPMS/
gpgcheck=0
#'>/etc/yum.repos.d/ceph.repo
yum clean all && yum makecache #生成缓存

#关闭selinux、防火墙
systemctl stop firewalld.service
systemctl disable firewalld.service
firewall-cmd --state
sed -i '/^SELINUX=.*/c SELINUX=disabled' /etc/selinux/config
sed -i 's/^SELINUXTYPE=.*/SELINUXTYPE=disabled/g' /etc/selinux/config
grep --color=auto '^SELINUX' /etc/selinux/config
setenforce 0

#时间同步

[[ -f /usr/sbin/ntpdate ]] || { echo "install ntp";yum install ntp -y &> /dev/null; } #若没NTP则安装

/usr/sbin/ntpdate ntp6.aliyun.com

echo "*/3 * * * * /usr/sbin/ntpdate ntp6.aliyun.com  &> /dev/null" > /tmp/crontab

crontab /tmp/crontab

###########################

#网络规划

# #外网规划

# 10.2.1.71 ceph01

#内网规划，写入hosts

echo '#ceph

192.168.58.71 ceph01

192.168.58.72 ceph02

192.168.58.73 ceph03

192.168.58.74 ceph04

'>>/etc/hosts

#配置hostname、固定ip

Nu=

hostnamectl set-hostname ceph0$Nu

NetName=ens37 #网卡名称

rm -f /etc/sysconfig/network-scripts/ifcfg-$NetName

nmcli con add con-name $NetName ifname $NetName autoconnect yes type ethernet ip4 192.168.58.7$Nu/ \

 ipv4.dns "172.16.11.14 119.29.29.29" #ipv4.gateway "192.168.58.1"

nmcli connection reload #重新加载网络

#磁盘(添加2快磁盘)

# [root@ceph01 ~]# lsblk

# NAME   MAJ:MIN RM  SIZE RO TYPE MOUNTPOINT

# sdb      :     100G   disk

# sdc      :     100G   disk

###########################

#ceph安装配置

###########################

#全部在ceph01操作

# #Ceph节点简介

# Mon(Montior)监控节点

# OSD(Object Storage Device)存储节点

# MDS(Meta Data Server)，Ceph作文件系统用到，本实例不配置

#ceph01作为部署节点

#ceph01 ceph02 ceph03作为Mon

#ceph02 ceph03 ceph04作为OSD

#ssh免密验证

#下载shell #

curl http://elven.vip/ks/sh/sshkey.me.sh >sshkey.me.sh

#认证用户及密码#

echo "

USER=root

PASS=

">my.sh.conf

sh ./ssh.sh ceph01 ceph02 ceph03 ceph04

#安装ceph-deploy配置工具

yum install -y ceph-deploy

#创建配置目录

mkdir /etc/ceph

cd /etc/ceph/

#初始化Mon配置

ceph-deploy new ceph{,,}

#ceph-deploy new --cluster-network 192.168.58.0/ --public-network 10.2.1.0/ ceph{,,}

# #修改冗余份数为2，日志大小2G

# #配置网络,单网卡忽略

echo '

public network = 10.2.1.0/

cluster network = 192.168.58.0/

mon_clock_drift_allowed =

osd_journal_size =

osd_pool_default_pg_num =

osd_pool_default_pgp_num =

osd pool default size =

osd pool default min size =

rbd_default_features =

client_quota = true

'>>./ceph.conf

#安装Ceph

ceph-deploy install ceph01 ceph02 ceph03 ceph04

#yum install -y ceph ceph-radosgw #实际上是安装这2个rpm

#初始化monitor和key

cd /etc/ceph/

ceph-deploy --overwrite-conf mon create-initial

#创建存储节点(使用sdb,sdc磁盘)

#lsblk #查看磁盘

#清空磁盘

ceph-deploy disk zap ceph02:sdb ceph02:sdc

ceph-deploy disk zap ceph03:sdb ceph03:sdc

ceph-deploy disk zap ceph04:sdb ceph04:sdc

#创建osd

ceph-deploy --overwrite-conf osd create ceph02:sdb ceph02:sdc

ceph-deploy --overwrite-conf osd create ceph03:sdb ceph03:sdc

ceph-deploy --overwrite-conf osd create ceph04:sdb ceph04:sdc

#拷贝配置及密钥

ceph-deploy admin ceph01 ceph02 ceph03 ceph04

chmod  /etc/ceph/ceph.client.admin.keyring

#创建mon

ceph-deploy --overwrite-conf mon create ceph01

ceph-deploy --overwrite-conf admin ceph01

ceph-deploy --overwrite-conf mon create ceph02

ceph-deploy --overwrite-conf admin ceph02

ceph-deploy --overwrite-conf mon create ceph03

ceph-deploy --overwrite-conf admin ceph03

#添加mgr

#ceph 12开始，monitor必须添加mgr

ceph-deploy mgr create ceph{,,}:mon_mgr

#ceph-deploy mgr create ceph01:mon_mgr ceph02:mon_mgr ceph03:mon_mgr

#启用dashboard (在mon节点)

ceph mgr module enable dashboard

#http://ip:7000 访问dashboard

netstat -antp|grep 

# 设置dashboard的ip和端口

# ceph config-key put mgr/dashboard/server_addr 192.168.68.71

# ceph config-key put mgr/dashboard/server_port

# systemctl restart ceph-mgr@ceph01

###########################

#查看相关命令

ceph health

ceph -s

ceph osd tree

ceph df

ceph mon stat

ceph osd stat

ceph pg stat

ceph osd lspools

ceph auth list

###########################

#参考

https://segmentfault.com/a/1190000011581513

http://blog.csdn.net/wylfengyujiancheng/article/details/78461801

https://segmentfault.com/a/1190000011589519

https://www.cnblogs.com/netmouser/p/6876846.html

#中文 文档

http://docs.ceph.org.cn/man/8/ceph-deploy/

#ceph工作原理和安装

http://www.jianshu.com/p/25163032f57f

###########################

###########################

#简单ceph性能测试

#创建测试池mytest

ceph osd pool create mytest

rados lspools

# ceph osd pool set mytest size  #副本为2

# ceph osd pool delete mytest #删除

#Rados性能测试(关注 bandwidth带宽,latency延迟)

rados bench -p mytest  write --no-cleanup #写测试10秒

rados bench -p mytest  seq  #顺序读

rados bench -p mytest  rand #随机读

rados -p mytest cleanup #清理测试数据

#rbd块设备测试

rbd create --size 2G mytest/test1 #创建块设备映像test1

rbd ls mytest

rbd info mytest/test1

rbd map mytest/test1   #映射块设备

#/dev/rbd0

#rbd showmapped         #查看已映射块设备

#挂载

mkfs.xfs /dev/rbd0

mkdir -p /mnt/ceph

mount /dev/rbd0 /mnt/ceph/

df -h /mnt/ceph

#测试

rbd bench-write mytest/test1

#默认参数io 4k,线程数16,总写入1024M, seq顺序写

rbd unmap mytest/test1 #取消块设备映射

rbd rm mytest/test1    #删除块设备映像

#参考

https://www.cnblogs.com/sammyliu/p/5557666.html

###########################

###########################

#未解决问题：

##、

#dashboard开启后

#所有Mon开了7000端口，只能随机1个Mon能访问web

##、

#rbd map映射块设备失败，暂没解决

#环境 centos  内核升级了的 4.4.-.el7.elrepo.x86_64

#centos 7自带内核3.10.0-693.2..el7.x86_64 正常

###########################