ceph扩展+rgw部署

#新增3台服务器
root@ceph-deploy:~# cat /etc/hosts 127.0.0.1 localhost 127.0.1.1 ubuntu.example.local ubuntu # The following lines are desirable for IPv6 capable hosts ::1 localhost ip6-localhost ip6-loopback ff02::1 ip6-allnodes ff02::2 ip6-allrouters 192.168.192.171 ceph-deploy 192.168.192.172 ceph-mon01-mgr01 ceph-mds01 192.168.192.173 ceph-node01 192.168.192.174 ceph-node02 192.168.192.175 ceph-node03 192.168.192.176 ceph-mon02-mgr02 ceph-mds02 192.168.192.177 ceph-mon03 ceph-mds03 192.168.192.178 ceph-mds04
root@ceph-mon02-mgr02_ceph-mds02:~# groupadd -r -g 2021 cephadmin && useradd -r -m -s /bin/bash -u 2021 -g 2021 cephadmin && echo cephadmin:123456 | chpasswd
root@ceph-mon02-mgr02_ceph-mds02:~# echo "cephadmin ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers
root@ceph-mon02-mgr02_ceph-mds02:~# cat /etc/apt/sources.list
# 默认注释了源码镜像以提高 apt update 速度,如有需要可自行取消注释
deb http://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic main restricted universe multiverse
# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic main restricted universe multiverse
deb http://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-updates main restricted universe multiverse
# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-updates main restricted universe multiverse
deb http://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-backports main restricted universe multiverse
# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-backports main restricted universe multiverse
deb http://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-security main restricted universe multiverse
# deb-src https://mirrors.tuna.tsinghua.edu.cn/ubuntu/ bionic-security main restricted universe multiverse
deb http://mirrors.tuna.tsinghua.edu.cn/ceph/debian-pacific bionic main
wget -q -O- 'http://mirrors.tuna.tsinghua.edu.cn/ceph/keys/release.asc' | sudo apt-key add -
apt update
apt install iproute2  ntpdate  tcpdump telnet traceroute nfs-kernel-server nfs-common  lrzsz tree  openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev ntpdate tcpdump telnet traceroute  gcc openssh-server lrzsz tree  openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev ntpdate tcpdump telnet traceroute iotop unzip zip openjdk-8-jdk -y
cat >/etc/sysctl.conf <<EOF
# Controls source route verification 
net.ipv4.conf.default.rp_filter = 1 
net.ipv4.ip_nonlocal_bind = 1 
net.ipv4.ip_forward = 1 
  
# Do not accept source routing 
net.ipv4.conf.default.accept_source_route = 0 
  
# Controls the System Request debugging functionality of the kernel 
kernel.sysrq = 0 
  
# Controls whether core dumps will append the PID to the core filename. 
# Useful for debugging multi-threaded 
applications. kernel.core_uses_pid = 1 
  
# Controls the use of TCP syncookies 
net.ipv4.tcp_syncookies = 1 
  
# Disable netfilter on bridges. 
net.bridge.bridge-nf-call-ip6tables = 0 
net.bridge.bridge-nf-call-iptables = 0 
net.bridge.bridge-nf-call-arptables = 0 
  
# Controls the default maxmimum size of a mesage queue 
kernel.msgmnb = 65536 
  
# # Controls the maximum size of a message, in bytes 
kernel.msgmax = 65536 
  
# Controls the maximum shared segment size, in bytes 
kernel.shmmax = 68719476736 
  
# # Controls the maximum number of shared memory segments, in pages 
kernel.shmall = 4294967296 
  
# TCP kernel paramater 
net.ipv4.tcp_mem = 786432 1048576 1572864 
net.ipv4.tcp_rmem = 4096        87380   4194304 
net.ipv4.tcp_wmem = 4096        16384   4194304 n
et.ipv4.tcp_window_scaling = 1 
net.ipv4.tcp_sack = 1 
  
# socket buffer 
net.core.wmem_default = 8388608 
net.core.rmem_default = 8388608 
net.core.rmem_max = 16777216 
net.core.wmem_max = 16777216 
net.core.netdev_max_backlog = 262144 
net.core.somaxconn = 20480 
net.core.optmem_max = 81920 
  
# TCP conn 
net.ipv4.tcp_max_syn_backlog = 262144 
net.ipv4.tcp_syn_retries = 3 
net.ipv4.tcp_retries1 = 3 
net.ipv4.tcp_retries2 = 15 
  
# tcp conn reuse 
net.ipv4.tcp_timestamps = 0 
net.ipv4.tcp_tw_reuse = 0 
net.ipv4.tcp_tw_recycle = 0 
net.ipv4.tcp_fin_timeout = 1 
  
  
net.ipv4.tcp_max_tw_buckets = 20000 
net.ipv4.tcp_max_orphans = 3276800 
net.ipv4.tcp_synack_retries = 1 
net.ipv4.tcp_syncookies = 1 
  
# keepalive conn 
net.ipv4.tcp_keepalive_time = 300 
net.ipv4.tcp_keepalive_intvl = 30 
net.ipv4.tcp_keepalive_probes = 3 
net.ipv4.ip_local_port_range = 10001    65000 
  
# swap 
vm.overcommit_memory = 0 
vm.swappiness = 10 
  
#net.ipv4.conf.eth1.rp_filter = 0 
#net.ipv4.conf.lo.arp_ignore = 1 
#net.ipv4.conf.lo.arp_announce = 2 
#net.ipv4.conf.all.arp_ignore = 1 
#net.ipv4.conf.all.arp_announce = 2 
EOF
cat > /etc/security/limits.conf <<EOF
root                soft    core            unlimited 
root                hard    core            unlimited 
root                soft    nproc           1000000 
root                hard    nproc           1000000 
root                soft    nofile          1000000 
root                hard    nofile          1000000 
root                soft    memlock         32000 
root                hard    memlock         32000 
root                soft    msgqueue        8192000 
root                hard    msgqueue        8192000 
  
 
*                soft    core            unlimited 
*                hard    core            unlimited 
*                soft    nproc           1000000 
*                hard    nproc           1000000 
*                soft    nofile          1000000 
*                hard    nofile          1000000 
*                soft    memlock         32000 
*                hard    memlock         32000 
*                soft    msgqueue        8192000 
*                hard    msgqueue        8192000 
EOF
apt install python2.7 -y
ln -sv /usr/bin/python2.7 /usr/bin/python2

拓展ceph集群高可用

1,扩展mon

Ceph-mon 是原生具备自选举以实现高可用机制的ceph 服务,节点数量通常是奇数。

root@ceph-mon02-mgr02_ceph-mds02:~# apt install ceph-mon -y
root@ceph-mon03_ceph-mds03:~# apt install ceph-mon -y
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy mon add ceph-mon02-mgr02
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy mon add ceph-mon03
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
  cluster:
    id:     d2cca32b-57dc-409f-9605-b19a373ce759
    health: HEALTH_OK
 
  services:
    mon: 3 daemons, quorum ceph-mon01-mgr01,ceph-mon02-mgr02_ceph-mds02,ceph-mon03_ceph-mds03 (age 2m)
    mgr: ceph-mon01-mgr01(active, since 54m)
    osd: 9 osds: 9 up (since 53m), 9 in (since 2d)
 
  data:
    pools:   3 pools, 384 pgs
    objects: 19 objects, 9.4 MiB
    usage:   122 MiB used, 450 GiB / 450 GiB avail
    pgs:     384 active+clean
cephadmin@ceph-deploy:~/ceph-cluster$ ceph quorum_status --format json-pretty

{
    "election_epoch": 16,
    "quorum": [
        0,
        1,
        2
    ],
    "quorum_names": [
        "ceph-mon01-mgr01",
        "ceph-mon02-mgr02_ceph-mds02",
        "ceph-mon03_ceph-mds03"
    ],
    "quorum_leader_name": "ceph-mon01-mgr01", #当前mon的lead
    "quorum_age": 206,
    "features": {
        "quorum_con": "4540138297136906239",
        "quorum_mon": [
            "kraken",
            "luminous",
            "mimic",
            "osdmap-prune",
            "nautilus",
            "octopus",
            "pacific",
            "elector-pinging"
        ]
    },
    "monmap": {
        "epoch": 3,
        "fsid": "d2cca32b-57dc-409f-9605-b19a373ce759",
        "modified": "2021-11-08T14:37:40.323082Z",
        "created": "2021-11-06T06:47:54.550256Z",
        "min_mon_release": 16,
        "min_mon_release_name": "pacific",
        "election_strategy": 1,
        "disallowed_leaders: ": "",
        "stretch_mode": false,
        "features": {
            "persistent": [
                "kraken",
                "luminous",
                "mimic",
                "osdmap-prune",
                "nautilus",
                "octopus",
                "pacific",
                "elector-pinging"
            ],
            "optional": []
        },
        "mons": [
            {
                "rank": 0,  ##ceph-mon01的等级
                "name": "ceph-mon01-mgr01", ##mon的节点名称
                "public_addrs": {
                    "addrvec": [
                        {
                            "type": "v2",
                            "addr": "192.168.192.172:3300", ##监控地址
                            "nonce": 0
                        },
                        {
                            "type": "v1",
                            "addr": "192.168.192.172:6789", ##监控地址
                            "nonce": 0
                        }
                    ]
                },
                "addr": "192.168.192.172:6789/0",
                "public_addr": "192.168.192.172:6789/0",
                "priority": 0,
                "weight": 0,
                "crush_location": "{}"
            },
            {
                "rank": 1,
                "name": "ceph-mon02-mgr02_ceph-mds02",
                "public_addrs": {
                    "addrvec": [
                        {
                            "type": "v2",
                            "addr": "192.168.192.176:3300",
                            "nonce": 0
                        },
                        {
                            "type": "v1",
                            "addr": "192.168.192.176:6789",
                            "nonce": 0
                        }
                    ]
                },
                "addr": "192.168.192.176:6789/0",
                "public_addr": "192.168.192.176:6789/0",
                "priority": 0,
                "weight": 0,
                "crush_location": "{}"
            },
            {
                "rank": 2,
                "name": "ceph-mon03_ceph-mds03",
                "public_addrs": {
                    "addrvec": [
                        {
                            "type": "v2",
                            "addr": "192.168.192.177:3300",
                            "nonce": 0
                        },
                        {
                            "type": "v1",
                            "addr": "192.168.192.177:6789",
                            "nonce": 0
                        }
                    ]
                },
                "addr": "192.168.192.177:6789/0",
                "public_addr": "192.168.192.177:6789/0",
                "priority": 0,
                "weight": 0,
                "crush_location": "{}"
            }
        ]
    }
}

2,扩展mgr节点

root@ceph-mon02-mgr02_ceph-mds02:~# apt install -y ceph-mgr
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy mgr create ceph-mon02-mgr02
cephadmin@ceph-deploy:~/ceph-cluster$ ceph -s
  cluster:
    id:     d2cca32b-57dc-409f-9605-b19a373ce759
    health: HEALTH_OK
 
  services:
    mon: 3 daemons, quorum ceph-mon01-mgr01,ceph-mon02-mgr02_ceph-mds02,ceph-mon03_ceph-mds03 (age 10m)
    mgr: ceph-mon01-mgr01(active, since 62m), standbys: ceph-mon02-mgr02 #备用的ceph-mon02-mgr02已经添加完成
    osd: 9 osds: 9 up (since 61m), 9 in (since 2d)

3,部署cephfs

如果要使用cephFS,需要部署cephfs 服务。MDS服务与mon服务部署在一起(最好分开部署)。

在所有ceph-mon上和ceph-mds04 上安装ceph-mds

 

root@ceph-mds04:/etc/netplan# apt install ceph-mds -y
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy mds create ceph-mds01
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy mds create ceph-mds02
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy mds create ceph-mds03
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy mds create ceph-mds04
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool create cephfs-metadata 32 32
pool 'cephfs-metadata' created
cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool create cephfs-data 64 64
pool 'cephfs-data' created
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs new mycephfs cephfs-metadata cephfs-data
new fs with metadata pool 4 and data pool 5
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs ls
name: mycephfs, metadata pool: cephfs-metadata, data pools: [cephfs-data ]
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status mycephfs
mycephfs - 0 clients
========
RANK  STATE      MDS         ACTIVITY     DNS    INOS   DIRS   CAPS  
 0    active  ceph-mds04  Reqs:    0 /s    10     13     12      0   
      POOL         TYPE     USED  AVAIL  
cephfs-metadata  metadata  96.0k   142G  
  cephfs-data      data       0    142G  
STANDBY MDS  
 ceph-mds01  
 ceph-mds02  
 ceph-mds03  
MDS version: ceph version 16.2.6 (ee28fb57e47e9f88813e24bbf4c14496ca299d31) pacific (stable)
cephadmin@ceph-deploy:~/ceph-cluster$ ceph mds stat
mycephfs:1 {0=ceph-mds04=up:active} 3 up:standby

ceph扩展+rgw部署

 

 

 

设置每个Rank 的备份MDS,也就是如果此Rank 当前的MDS 出现问题马上切换到另个MDS。设置备份的方法有很多,常用选项如下。

设置每个Rank 的备份MDS,也就是如果此Rank 当前的MDS 出现问题马上切换到另个MDS。设置备份的方法有很多,常用选项如下。

mds_standby_replay:值为true 或false,true 表示开启replay 模式,这种模式下主MDS 内的数量将实时与从MDS 同步,如果主宕机,从可以快速的切换。如果为false 只有宕机的时候才去同步数据,这样会有一段时间的中断。
 
mds_standby_for_name:设置当前MDS 进程只用于备份于指定名称的MDS。
 
mds_standby_for_rank:设置当前MDS 进程只用于备份于哪个Rank,通常为Rank 编号。另外在存在之个CephFS 文件系统中,还可以使用mds_standby_for_fscid 参数来为指定不同的文件系统。
 
mds_standby_for_fscid:指定CephFS 文件系统ID,需要联合mds_standby_for_rank 生效,如果设置mds_standby_for_rank,那么就是用于指定文件系统的指定Rank,如果没有设置,就是指定文件系统的所有Rank。
ceph fs get mycephfs
max_mds    1
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs set mycephfs max_mds 2
cephadmin@ceph-deploy:~/ceph-cluster$ ceph fs status
mycephfs - 0 clients
========
RANK  STATE      MDS         ACTIVITY     DNS    INOS   DIRS   CAPS  
 0    active  ceph-mds04  Reqs:    0 /s    10     13     12      0   
 1    active  ceph-mds03  Reqs:    0 /s    10     13     11      0   
      POOL         TYPE     USED  AVAIL  
cephfs-metadata  metadata   168k   142G  
  cephfs-data      data       0    142G  
STANDBY MDS  
 ceph-mds01  
 ceph-mds02  
MDS version: ceph version 16.2.6 (ee28fb57e47e9f88813e24bbf4c14496ca299d31) pacific (stable)

目前的状态是ceph-mds03 和ceph-mgr04 分别是active 状态,ceph-mds01和ceph-mds02 分别处于standby 状态,现在可以将ceph-mds03设置为ceph-mds01 的standby,将ceph-mds04 设置为ceph-mgr02的standby,以实现每个主都有一个固定备份角色的结构,则修改配置文件如.

cephadmin@ceph-deploy:~/ceph-cluster$ cat ceph.conf 
[global]
fsid = d2cca32b-57dc-409f-9605-b19a373ce759
public_network = 192.168.192.0/24
cluster_network = 192.168.227.0/24
mon_initial_members = ceph-mon01-mgr01
mon_host = 192.168.192.172
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx

[mds.ceph-mds01]
#mds_standby_for_fscid = mycephfs
mds_standby_for_name = ceph-mds03
mds_standby_replay = true
[mds.ceph-mds02]
mds_standby_for_name = ceph-mgr04
mds_standby_replay = true
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy --overwrite-conf config push ceph-mds01-04
root@ceph-mds04:/etc/netplan# systemctl restart ceph-mds@ceph-mds01-04.service  #先重启1 2
root@ceph-deploy:/etc# ceph fs status
mycephfs - 0 clients
========
RANK  STATE      MDS         ACTIVITY     DNS    INOS   DIRS   CAPS  
 0    active  ceph-mds01  Reqs:    0 /s    10     13     12      0   
 1    active  ceph-mds02  Reqs:    0 /s    10     13     11      0   
      POOL         TYPE     USED  AVAIL  
cephfs-metadata  metadata   180k   142G  
  cephfs-data      data       0    142G  
STANDBY MDS  
 ceph-mds03  
 ceph-mds04  
MDS version: ceph version 16.2.6 (ee28fb57e47e9f88813e24bbf4c14496ca299d31) pacific (stable)

3 部署radosgw服务,不要启用CephFS

RGW提供的事REST接口,客户端通过http与其交互,完成数据的增删改查等管理操作。radosgw用在需要使用RESTful API接口访问ceph数据的场合,因此在使用RBD或者CephFS的场合可以不启用该功能。

cephadmin@ceph-deploy:~/ceph-cluster$ ceph osd pool ls
device_health_metrics
mypool
rbdpool
.rgw.root
default.rgw.log
default.rgw.control
default.rgw.meta
root@ceph-mon01-mgr01:~# apt install radosgw -y
cephadmin@ceph-deploy:~/ceph-cluster$ sudo ceph-deploy rgw create ceph-mon01-mgr01
cephadmin@ceph-deploy:~/ceph-cluster$ curl http://192.168.192.172:7480
<?xml version="1.0" encoding="UTF-8"?><ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Owner><ID>anonymous</ID><DisplayName></DisplayName></Owner><Buckets></Buckets></ListAllMyBucketsResult>

 

 

 
上一篇:Redis 集群


下一篇:redis集群搭建