环境说明：

1、双机互信需要设置好，hosts文件需要解析好，时间要同步，配置好yum源

双击互信及Hosts文件配置，参考：http://blog.csdn.net/reblue520/article/details/51213030
最好使用centos6.5自带光盘，关于将光盘制作成本地httpd yum源可参考：http://blog.csdn.net/reblue520/article/details/51164294

2、准备好自动化安装包

定义ansible的管理组：
/etc/ansible/hosts

[corosync]
node2.chinasoft.com
node4.chinasoft.com

安装apache软件
# ansible corosync -m yum -a "name=httpd state=present"

在8.20上安装nfs文件服务器，并分别挂载在Node2和node4的/web/htdocs下
# yum -y install nfs-utils rpcbind
# mkdir /sharestore
# vi /etc/exports

/sharestore  *(rw,sync,no_root_squash)

启动服务
# service rpcbind restart 
# service nfs restart  
让服务随机启动
# chkconfig nfs on
# chkconfig rpcbind on

在node2和node4上分别执行挂载
# mount -t nfs 192.168.8.20:/sharestore /web/htdocs

# echo "<h1> nfs system server </h1>" /web/htdocs/index.html 
自动安装集群服务的剧本目录：
# cd /ansible/
# ls
corosync
# cd corosync/
# ls
conf  corosync.yaml  packages

剧本内容：
# vim corosync.yaml

- hosts: corosync
  remote_user: root
  vars:
    crmsh: crmsh-1.2.6-4.el6.x86_64.rpm
    pssh: pssh-2.3.1-2.el6.x86_64.rpm
  tasks:
    - name: corosync installing
      yum: name=corosync state=present
    - name: pacemaker installing
      yum: name=pacemaker state=present
    - name: crmsh rpm packages 
      copy: src=/ansible/corosync/packages/{{ crmsh }} dest=/tmp/{{ crmsh }}
    - name: pssh rpm packages 
      copy: src=/ansible/corosync/packages/{{ pssh }} dest=/tmp/{{ pssh }}
    - name: crmsh installing
      command: yum -y install /tmp/{{ crmsh }} /tmp/{{ pssh }}
    - name: authkey configure file
      copy: src=/ansible/corosync/conf/authkey dest=/etc/corosync/authkey
    - name: authkey mode 400
      file: path=/etc/corosync/authkey mode=400
      notify:
        - restart corosync 
    - name: corosync.conf configure file
      copy: src=/ansible/corosync/conf/corosync.conf dest=/etc/corosync/corosync.conf
      tags:
        - conf
      notify:
        - restart corosync 
    - name: ensure the corosync service startup on boot
      service: name=corosync state=started enabled=yes
  handlers:
    - name: restart corosync

service: name=corosync state=restarted

- hosts: corosync

  remote_user: root

  vars:

    crmsh: crmsh-1.2.6-4.el6.x86_64.rpm

    pssh: pssh-2.3.1-2.el6.x86_64.rpm

  tasks:

    - name: corosync installing

      yum: name=corosync state=present

    - name: pacemaker installing

      yum: name=pacemaker state=present

    - name: crmsh rpm packages

      copy: src=/ansible/corosync/packages/{{ crmsh }} dest=/tmp/{{ crmsh }}

    - name: pssh rpm packages

      copy: src=/ansible/corosync/packages/{{ pssh }} dest=/tmp/{{ pssh }}

    - name: crmsh installing

      command: yum -y install /tmp/{{ crmsh }} /tmp/{{ pssh }}

    - name: authkey configure file

      copy: src=/ansible/corosync/conf/authkey dest=/etc/corosync/authkey

    - name: authkey mode 400

      file: path=/etc/corosync/authkey mode=400

      notify:

        - restart corosync

    - name: corosync.conf configure file

      copy: src=/ansible/corosync/conf/corosync.conf dest=/etc/corosync/corosync.conf

      tags:

        - conf

      notify:

        - restart corosync

    - name: ensure the corosync service startup on boot

      service: name=corosync state=started enabled=yes

  handlers:

    - name: restart corosync

      service: name=corosync state=restarted

执行剧本
# ansible-playbook corosync.yaml

报错：
TASK: [crmsh rpm packages] **************************************************** 
failed: [node2.chinasoft.com] => {"checksum": "45b4cbe3bd441d9a02cbcbc3d11091d2ad4388fa", "failed": true}
msg: Aborting, target uses selinux but python bindings (libselinux-python) aren't installed!
failed: [node4.chinasoft.com] => {"checksum": "45b4cbe3bd441d9a02cbcbc3d11091d2ad4388fa", "failed": true}
msg: Aborting, target uses selinux but python bindings (libselinux-python) aren't installed!

FATAL: all hosts have already failed -- aborting

PLAY RECAP ******************************************************************** 
           to retry, use: --limit @/root/corosync.yaml.retry

node2.chinasoft.com        : ok=3    changed=0    unreachable=0    failed=1   
node4.chinasoft.com        : ok=3    changed=0    unreachable=0    failed=1

处理：在node2和node4上安装libselinux-python包
# ansible corosync -a "yum install -y libselinux-python"

在其中的Node2上执行crm status查看集群是否配置成功
# crm status
Last updated: Mon Apr 25 17:51:10 2016
Last change: Mon Apr 25 17:43:35 2016 via crmd on node4.chinasoft.com
Stack: classic openais (with plugin)
Current DC: node4.chinasoft.com - partition with quorum
Version: 1.1.10-14.el6-368c726
2 Nodes configured, 2 expected votes
0 Resources configured

Online: [ node2.chinasoft.com node4.chinasoft.com ]

3、常用配置
# crm configure
crm(live)configure# show
node node2.chinasoft.com
node node4.chinasoft.com
property $id="cib-bootstrap-options" \
dc-version="1.1.10-14.el6-368c726" \
cluster-infrastructure="classic openais (with plugin)" \
expected-quorum-votes="2"
# 禁用stonith-enabled  
crm(live)configure# property stonith-enabled=false
crm(live)configure# verify
# 忽略法定票数限制
crm(live)configure# property no-quorum-policy=ignore
crm(live)configure# verify
crm(live)configure# commit
# 配置虚拟IP为8.77

crm(live)configure# primitive webip ocf:heartbeat:IPaddr params ip=192.168.8.77

crm(live)configure# verify

crm(live)configure# commit

crm(live)configure# show

node node2.chinasoft.com

node node4.chinasoft.com

primitive webip ocf:heartbeat:IPaddr \

params ip="192.168.8.77"

property $id="cib-bootstrap-options" \

dc-version="1.1.10-14.el6-368c726" \

cluster-infrastructure="classic openais (with plugin)" \

expected-quorum-votes="2" \

stonith-enabled="false" \

no-quorum-policy="ignore"

定义web资源

crm(live)configure# primitive webserver lsb:httpd

crm(live)configure# verify

crm(live)configure# commit

crm(live)configure# show

node node2.chinasoft.com

node node4.chinasoft.com

primitive webip ocf:heartbeat:IPaddr \

params ip="192.168.8.77"

primitive webserver lsb:httpd

property $id="cib-bootstrap-options" \

dc-version="1.1.10-14.el6-368c726" \

cluster-infrastructure="classic openais (with plugin)" \

expected-quorum-votes="2" \

stonith-enabled="false" \

no-quorum-policy="ignore"

#定义组

crm(live)configure# group webservice webip webserver

crm(live)configure# verify

crm(live)configure# commit

crm(live)configure# show

node node2.chinasoft.com

node node4.chinasoft.com

primitive webip ocf:heartbeat:IPaddr \

params ip="192.168.8.77"

primitive webserver lsb:httpd

group webservice webip webserver

property $id="cib-bootstrap-options" \

dc-version="1.1.10-14.el6-368c726" \

cluster-infrastructure="classic openais (with plugin)" \

expected-quorum-votes="2" \

stonith-enabled="false" \

no-quorum-policy="ignore"

将node2配置为备用节点

# standby node2.chinasoft.com

crm(live)node# cd

crm(live)# status

Last updated: Mon Apr 25 19:11:07 2016

Last change: Mon Apr 25 19:11:04 2016 via crm_attribute on node2.chinasoft.com

Stack: classic openais (with plugin)

Current DC: node4.chinasoft.com - partition with quorum

Version: 1.1.10-14.el6-368c726

2 Nodes configured, 2 expected votes

2 Resources configured

Node node2.chinasoft.com: standby

Online: [ node4.chinasoft.com ]

 Resource Group: webservice

     webip (ocf::heartbeat:IPaddr):Started node4.chinasoft.com

     webserver (lsb:httpd):Started node4.chinasoft.com

让node2重新上线，主节点仍然是node4

# crm node online node2.chinasoft.com

# crm status

Last updated: Mon Apr 25 19:13:09 2016

Last change: Mon Apr 25 19:13:06 2016 via crm_attribute on node4.chinasoft.com

Stack: classic openais (with plugin)

Current DC: node4.chinasoft.com - partition with quorum

Version: 1.1.10-14.el6-368c726

2 Nodes configured, 2 expected votes

2 Resources configured

Online: [ node2.chinasoft.com node4.chinasoft.com ]

 Resource Group: webservice

     webip (ocf::heartbeat:IPaddr):Started node4.chinasoft.com

     webserver (lsb:httpd):Started node4.chinasoft.com

删除webservice组

# crm configure

crm(live)configure# stop webservice

ERROR: syntax: stop webservice

crm(live)configure# cd

crm(live)# resource

crm(live)resource# stop webservice

crm(live)resource# status webservice

resource webservice is NOT running

crm(live)resource# cd

crm(live)# configure

crm(live)configure# delete webservice

crm(live)configure# show

node node2.chinasoft.com \

attributes standby="off"

node node4.chinasoft.com

primitive webip ocf:heartbeat:IPaddr \

params ip="192.168.8.77"

primitive webserver lsb:httpd

property $id="cib-bootstrap-options" \

dc-version="1.1.10-14.el6-368c726" \

cluster-infrastructure="classic openais (with plugin)" \

expected-quorum-votes="2" \

stonith-enabled="false" \

no-quorum-policy="ignore"

crm(live)configure# commit

定义排序约束关系，即服务绑定在同一资源上运行

crm(live)# configure

crm(live)configure# colocation webserver_with_webip inf: webserver webip

crm(live)configure# show

node node2.chinasoft.com \

attributes standby="off"

node node4.chinasoft.com

primitive webip ocf:heartbeat:IPaddr \

params ip="192.168.8.77"

primitive webserver lsb:httpd

colocation webserver_with_webip inf: webserver webip

property $id="cib-bootstrap-options" \

dc-version="1.1.10-14.el6-368c726" \

cluster-infrastructure="classic openais (with plugin)" \

expected-quorum-votes="2" \

stonith-enabled="false" \

no-quorum-policy="ignore"

crm(live)configure# show xml

<?xml version="1.0" ?>

<cib num_updates="9" dc-uuid="node4.chinasoft.com" update-origin="node2.chinasoft.com" crm_feature_set="3.0.7" validate-with="pacemaker-1.2" update-client="cibadmin" epoch="13" admin_epoch="0" cib-last-written="Mon Apr 25 19:17:30 2016" have-quorum="1">

  <configuration>

    <crm_config>

      <cluster_property_set id="cib-bootstrap-options">

        <nvpair id="cib-bootstrap-options-dc-version" name="dc-version" value="1.1.10-14.el6-368c726"/>

        <nvpair id="cib-bootstrap-options-cluster-infrastructure" name="cluster-infrastructure" value="classic openais (with plugin)"/>

        <nvpair id="cib-bootstrap-options-expected-quorum-votes" name="expected-quorum-votes" value="2"/>

        <nvpair name="stonith-enabled" value="false" id="cib-bootstrap-options-stonith-enabled"/>

        <nvpair name="no-quorum-policy" value="ignore" id="cib-bootstrap-options-no-quorum-policy"/>

      </cluster_property_set>

    </crm_config>

    <nodes>

      <node id="node2.chinasoft.com" uname="node2.chinasoft.com">

        <instance_attributes id="nodes-node2.chinasoft.com">

          <nvpair id="nodes-node2.chinasoft.com-standby" name="standby" value="off"/>

        </instance_attributes>

      </node>

      <node id="node4.chinasoft.com" uname="node4.chinasoft.com"/>

    </nodes>

    <resources>

      <primitive id="webip" class="ocf" provider="heartbeat" type="IPaddr">

        <instance_attributes id="webip-instance_attributes">

          <nvpair name="ip" value="192.168.8.77" id="webip-instance_attributes-ip"/>

        </instance_attributes>

      </primitive>

      <primitive id="webserver" class="lsb" type="httpd"/>

    </resources>

    <constraints>

      <rsc_colocation id="webserver_with_webip" score="INFINITY" rsc="webserver" with-rsc="webip"/>

    </constraints>

  </configuration>

</cib>

定义顺序关系
crm(live)configure# order webip_before_webserver mandatory: webip webserver
crm(live)configure# show xml
crm(live)configure# verify
crm(live)configure# commit
crm(live)configure# show
node node2.chinasoft.com \
attributes standby="off"
node node4.chinasoft.com
primitive webip ocf:heartbeat:IPaddr \
params ip="192.168.8.77"
primitive webserver lsb:httpd
colocation webserver_with_webip inf: webserver webip
order webip_before_webserver inf: webip webserver
property $id="cib-bootstrap-options" \
dc-version="1.1.10-14.el6-368c726" \
cluster-infrastructure="classic openais (with plugin)" \
expected-quorum-votes="2" \
stonith-enabled="false" \
no-quorum-policy="ignore"

定义资源粘性，更倾向于运行在node4.chinasoft.com节点上

<pre name="code" class="plain">crm(live)configure# location webip_on_node4 webip 200: node4.chinasoft.com

crm(live)configure# verify

crm(live)configure# commit

crm(live)configure# cd

crm(live)# status

Last updated: Mon Apr 25 19:32:59 2016

Last change: Mon Apr 25 19:32:57 2016 via cibadmin on node2.chinasoft.com

Stack: classic openais (with plugin)

Current DC: node4.chinasoft.com - partition with quorum

Version: 1.1.10-14.el6-368c726

2 Nodes configured, 2 expected votes

2 Resources configured

Online: [ node2.chinasoft.com node4.chinasoft.com ]

 webip (ocf::heartbeat:IPaddr):Started node4.chinasoft.com

 webserver (lsb:httpd):Started node4.chinasoft.com

将node4调整为备用节点，可以看到node2变为主节点，重新上线node4发现主节点又回到了node4上
crm(live)# node
crm(live)node# standby node4.chinasoft.com
crm(live)node# cd
crm(live)# show
ERROR: syntax: show
crm(live)# status
Last updated: Mon Apr 25 19:38:45 2016
Last change: Mon Apr 25 19:38:37 2016 via crm_attribute on node2.chinasoft.com
Stack: classic openais (with plugin)
Current DC: node4.chinasoft.com - partition with quorum
Version: 1.1.10-14.el6-368c726
2 Nodes configured, 2 expected votes
2 Resources configured

Node node4.chinasoft.com: standby
Online: [ node2.chinasoft.com ]

webip(ocf::heartbeat:IPaddr):Started node2.chinasoft.com 
 webserver(lsb:httpd):Started node2.chinasoft.com
crm(live)# node
crm(live)node# online node4.chinasoft.com
crm(live)node# cd
crm(live)# status
Last updated: Mon Apr 25 19:40:20 2016
Last change: Mon Apr 25 19:39:02 2016 via crm_attribute on node2.chinasoft.com
Stack: classic openais (with plugin)
Current DC: node4.chinasoft.com - partition with quorum
Version: 1.1.10-14.el6-368c726
2 Nodes configured, 2 expected votes
2 Resources configured

Online: [ node2.chinasoft.com node4.chinasoft.com ]

webip(ocf::heartbeat:IPaddr):Started node4.chinasoft.com

webserver(lsb:httpd):Started node4.chinasoft.com

删除资源的方法
# crm resource
crm(live)resource# stop vip
crm(live)resource# stop webip
crm(live)resource# stop webserver
crm(live)resource# cd
crm(live)# configure 
crm(live)configure# delete 
cib-bootstrap-options    node4.chinasoft.com      webip                    webip_on_node4           webserver_with_webip     
node2.chinasoft.com      vip                      webip_before_webserver   webserver                
crm(live)configure# delete vip
crm(live)configure# delete webserver 
crm(live)configure# delete webip
crm(live)configure# verify
crm(live)configure# commit
crm(live)configure# cd
crm(live)# status
Last updated: Tue Apr 26 15:56:37 2016
Last change: Tue Apr 26 15:56:33 2016 via cibadmin on node4.chinasoft.com
Stack: classic openais (with plugin)
Current DC: node4.chinasoft.com - partition with quorum
Version: 1.1.10-14.el6-368c726
2 Nodes configured, 2 expected votes
0 Resources configured

Online: [ node2.chinasoft.com node4.chinasoft.com ]
crm(live)# configure
crm(live)configure# show
node node2.chinasoft.com \
attributes standby="off"
node node4.chinasoft.com \
attributes standby="off"
property $id="cib-bootstrap-options" \
dc-version="1.1.10-14.el6-368c726" \
cluster-infrastructure="classic openais (with plugin)" \
expected-quorum-votes="2" \
stonith-enabled="false" \
no-quorum-policy="ignore" \
last-lrm-refresh="1461657233"

这样定义代表集群中每一个资源的默认权重为100
crm(live)configure# rsc_defaults resource-stickiness=100
crm(live)configure# verify
crm(live)configure# commit
定义webstore、webip并添加到webservice组中
crm(live)configure# primitive webstore ocf:heartbeat:Filesystem params device="192.168.8.20:/sharestore" directory="/web/htdocs" fstype="nfs" op monitor interval=20s timeout=40s op start timeout=60s op stop timeout=60s on-fail=restart
crm(live)configure# verify
crm(live)configure# primitive webserver lsb:httpd op monitor interval=30s timeout=20s on-fail=restart
crm(live)configure# verify
crm(live)configure# primitive webip ocf:heartbeat:IPaddr params ip=192.168.8.77 op monitor interval=30s timeout=20s on-fail=restart
crm(live)configure# verify
crm(live)configure# group webservice webip webstore webserver
crm(live)configure# verify
crm(live)configure# commit
crm(live)configure# show
node node2.chinasoft.com \
attributes standby="off"
node node4.chinasoft.com \
attributes standby="off"
primitive webip ocf:heartbeat:IPaddr \
params ip="192.168.8.77" \
op monitor interval="30s" timeout="20s" on-fail="restart"
primitive webserver lsb:httpd \
op monitor interval="30s" timeout="20s" on-fail="restart"
primitive webstore ocf:heartbeat:Filesystem \
params device="192.168.8.20:/sharestore" directory="/web/htdocs" fstype="nfs" \
op monitor interval="20s" timeout="40s" \
op start timeout="60s" interval="0" \
op stop timeout="60s" on-fail="restart" interval="0"
group webservice webip webstore webserver
property $id="cib-bootstrap-options" \
dc-version="1.1.10-14.el6-368c726" \
cluster-infrastructure="classic openais (with plugin)" \
expected-quorum-votes="2" \
stonith-enabled="false" \
no-quorum-policy="ignore" \
last-lrm-refresh="1461657233"
rsc_defaults $id="rsc-options" \
resource-stickiness="100"

定义先后顺序
crm(live)configure# order webip_before_webstore_before_webserver mandatory: webip webstore webserver
crm(live)configure# show xml

crm(live)# status
Last updated: Tue Apr 26 16:48:20 2016
Last change: Tue Apr 26 16:47:17 2016 via cibadmin on node2.chinasoft.com
Stack: classic openais (with plugin)
Current DC: node4.chinasoft.com - partition with quorum
Version: 1.1.10-14.el6-368c726
2 Nodes configured, 2 expected votes
3 Resources configured

Online: [ node2.chinasoft.com node4.chinasoft.com ]

Resource Group: webservice
     webip(ocf::heartbeat:IPaddr):Started node2.chinasoft.com 
     webstore(ocf::heartbeat:Filesystem):Started node2.chinasoft.com 
     webserver(lsb:httpd):Started node2.chinasoft.com

[root@node4 ~]# crm node online node4.chinasoft.com
[root@node4 ~]# crm status
Last updated: Tue Apr 26 16:53:15 2016
Last change: Tue Apr 26 16:53:13 2016 via crm_attribute on node4.chinasoft.com
Stack: classic openais (with plugin)
Current DC: node4.chinasoft.com - partition with quorum
Version: 1.1.10-14.el6-368c726
2 Nodes configured, 2 expected votes
3 Resources configured

Online: [ node2.chinasoft.com node4.chinasoft.com ]

Resource Group: webservice
     webip(ocf::heartbeat:IPaddr):Started node2.chinasoft.com 
     webstore(ocf::heartbeat:Filesystem):Started node2.chinasoft.com 
     webserver(lsb:httpd):Started node2.chinasoft.com

将node2置为standby则node4上线顶替
[root@node4 ~]# crm status
Last updated: Tue Apr 26 16:53:20 2016
Last change: Tue Apr 26 16:53:13 2016 via crm_attribute on node4.chinasoft.com
Stack: classic openais (with plugin)
Current DC: node4.chinasoft.com - partition with quorum
Version: 1.1.10-14.el6-368c726
2 Nodes configured, 2 expected votes
3 Resources configured

Online: [ node2.chinasoft.com node4.chinasoft.com ]

Resource Group: webservice
     webip(ocf::heartbeat:IPaddr):Started node2.chinasoft.com 
     webstore(ocf::heartbeat:Filesystem):Started node2.chinasoft.com 
     webserver(lsb:httpd):Started node2.chinasoft.com
[root@node4 ~]# crm node standby node2.chinasoft.com
[root@node4 ~]# crm status
Last updated: Tue Apr 26 16:53:50 2016
Last change: Tue Apr 26 16:53:47 2016 via crm_attribute on node4.chinasoft.com
Stack: classic openais (with plugin)
Current DC: node4.chinasoft.com - partition with quorum
Version: 1.1.10-14.el6-368c726
2 Nodes configured, 2 expected votes
3 Resources configured

Node node2.chinasoft.com: standby
Online: [ node4.chinasoft.com ]

Resource Group: webservice
     webip(ocf::heartbeat:IPaddr):Started node4.chinasoft.com 
     webstore(ocf::heartbeat:Filesystem):Started node4.chinasoft.com 
     webserver(lsb:httpd):Started node4.chinasoft.com

pcs命令的使用：

环境准备：
删除之前建立的webip、webstore、webservice
crm(live)resource# stop webip
crm(live)resource# stop webstore 
crm(live)resource# stop webservice
crm(live)resource# status
 Resource Group: webservice
     webip(ocf::heartbeat:IPaddr):Stopped 
     webstore(ocf::heartbeat:Filesystem):Stopped 
     webserver(lsb:httpd):Stopped 
crm(live)resource# cd
crm(live)# configure                                
crm(live)configure# delete webservice
crm(live)configure# delete webip
INFO: constraint order:webip_before_webstore_before_webserver updated
crm(live)configure# delete webstore
crm(live)configure# show
node node2.chinasoft.com \
attributes standby="on"
node node4.chinasoft.com \
attributes standby="off"
property $id="cib-bootstrap-options" \
dc-version="1.1.10-14.el6-368c726" \
cluster-infrastructure="classic openais (with plugin)" \
expected-quorum-votes="2" \
stonith-enabled="false" \
no-quorum-policy="ignore" \
last-lrm-refresh="1461664392"
rsc_defaults $id="rsc-options" \
resource-stickiness="100"
crm(live)configure# verify
crm(live)configure# commit

安装
# ansible corosync -m yum -a "name=pcs state=present"

设置vip
# pcs resource create webip ocf:heartbeat:IPaddr ip=192.168.8.77 op monitor interval=30s timeout=20s
[root@node2 ~]# pcs status
Cluster name: 
Last updated: Tue Apr 26 17:58:13 2016
Last change: Tue Apr 26 17:58:16 2016 via cibadmin on node2.chinasoft.com
Stack: classic openais (with plugin)
Current DC: node4.chinasoft.com - partition with quorum
Version: 1.1.10-14.el6-368c726
2 Nodes configured, 2 expected votes
1 Resources configured

Node node2.chinasoft.com: standby
Online: [ node4.chinasoft.com ]

Full list of resources:

webip(ocf::heartbeat:IPaddr):Started node4.chinasoft.com

在node4上可以看到vip已经生成
 [root@node4 ~]# ip addr show
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN 
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP qlen 1000
    link/ether 00:0c:29:eb:42:3e brd ff:ff:ff:ff:ff:ff
    inet 192.168.8.42/24 brd 192.168.8.255 scope global eth0
    inet 192.168.8.77/24 brd 192.168.8.255 scope global secondary eth0
    inet6 fe80::20c:29ff:feeb:423e/64 scope link 
       valid_lft forever preferred_lft forever

定义nfs存储
# pcs resource create webstore ocf:heartbeat:Filesystem device="192.168.8.20:/sharestore" directory="/web/htdocs" fstype="nfs" op monitor interval=20s timeout=40s op start timeout=60s op stop timeout=60s
[root@node4 ~]# pcs status
Cluster name: 
Last updated: Tue Apr 26 18:05:21 2016
Last change: Tue Apr 26 18:05:18 2016 via cibadmin on node4.chinasoft.com
Stack: classic openais (with plugin)
Current DC: node4.chinasoft.com - partition with quorum
Version: 1.1.10-14.el6-368c726
2 Nodes configured, 2 expected votes
2 Resources configured

Node node2.chinasoft.com: standby
Online: [ node4.chinasoft.com ]

Full list of resources:

webip(ocf::heartbeat:IPaddr):Started node4.chinasoft.com 
 webstore(ocf::heartbeat:Filesystem):Started node4.chinasoft.com

可以看到已经成功挂载
[root@node4 ~]# mount
/dev/mapper/vg_node4-root on / type ext4 (rw)
proc on /proc type proc (rw)
sysfs on /sys type sysfs (rw)
devpts on /dev/pts type devpts (rw,gid=5,mode=620)
tmpfs on /dev/shm type tmpfs (rw,rootcontext="system_u:object_r:tmpfs_t:s0")
/dev/sda1 on /boot type ext4 (rw)
/dev/mapper/vg_node4-data on /data type ext4 (rw)
/dev/mapper/vg_node4-usr on /usr/local type ext4 (rw)
/dev/mapper/vg_node4-web on /web type ext4 (rw)
none on /proc/sys/fs/binfmt_misc type binfmt_misc (rw)
/dev/sr0 on /mnt type iso9660 (ro)
sunrpc on /var/lib/nfs/rpc_pipefs type rpc_pipefs (rw)
192.168.8.20:/sharestore on /web/htdocs type nfs (rw,vers=4,addr=192.168.8.20,clientaddr=192.168.8.42)

创建webserver
[root@node4 ~]# pcs resource create webserver lsb:httpd op monitor interval=30s timeout=20s on-fail=restart
[root@node4 ~]# psc status
-bash: psc: command not found
[root@node4 ~]# pcs status
Cluster name: 
Last updated: Tue Apr 26 18:12:56 2016
Last change: Tue Apr 26 18:12:49 2016 via cibadmin on node4.chinasoft.com
Stack: classic openais (with plugin)
Current DC: node4.chinasoft.com - partition with quorum
Version: 1.1.10-14.el6-368c726
2 Nodes configured, 2 expected votes
3 Resources configured

Node node2.chinasoft.com: standby
Online: [ node4.chinasoft.com ]

Full list of resources:

webip(ocf::heartbeat:IPaddr):Started node4.chinasoft.com 
 webstore(ocf::heartbeat:Filesystem):Started node4.chinasoft.com 
 webserver(lsb:httpd):Started node4.chinasoft.com

将webip、webstore、webserver定义到webservice组中
[root@node4 ~]# pcs resource group add webservice webip webstore webserver
[root@node4 ~]# pcs status
Cluster name: 
Last updated: Tue Apr 26 18:13:37 2016
Last change: Tue Apr 26 18:13:35 2016 via cibadmin on node4.chinasoft.com
Stack: classic openais (with plugin)
Current DC: node4.chinasoft.com - partition with quorum
Version: 1.1.10-14.el6-368c726
2 Nodes configured, 2 expected votes
3 Resources configured

Node node2.chinasoft.com: standby
Online: [ node4.chinasoft.com ]

Full list of resources:

Resource Group: webservice
     webip(ocf::heartbeat:IPaddr):Started node4.chinasoft.com 
     webstore(ocf::heartbeat:Filesystem):Started node4.chinasoft.com 
     webserver(lsb:httpd):Started node4.chinasoft.com

此时可以通过访问http://192.168.8.77 看到挂载的nfs文件

配置node4为备用节点
# crm node standby node4.chinasoft.com

定义约束关系，pcs只支持一次两种资源之间关系的定义
[root@node4 ~]# pcs constraint order webip then webstore then webserver 
Adding webip webstore (kind: Mandatory) (Options: then webserver first-action=start then-action=start)
[root@node4 ~]# pcs constraint order show
Ordering Constraints:
  start webip then start webstore

[root@node4 ~]# pcs constraint order remove webstore webip
[root@node4 ~]# pcs constraint order webip then webserver
Adding webip webserver (kind: Mandatory) (Options: first-action=start then-action=start)
[root@node4 ~]# pcs constraint order webstore then webserver
Adding webstore webserver (kind: Mandatory) (Options: first-action=start then-action=start)
[root@node4 ~]# pcs constraint order show
Ordering Constraints:
  start webip then start webserver
  start webstore then start webserver

定义资源之间绑定关系
# pcs constraint colocation add webstore with webip

删除组资源，注意：delete命令会将所有的节点都删除
[root@node4 ~]# pcs resource delete webservice
Removing group: webservice (and all resources within group)
Stopping all resources in group: webservice...
Removing Constraint - colocation-webstore-webip-INFINITY
Removing Constraint - order-webip-webserver-mandatory
Deleting Resource - webip
Removing Constraint - order-webstore-webserver-mandatory
Deleting Resource - webstore
Deleting Resource (and group) - webserver
[root@node4 ~]# pcs status
Cluster name: 
Last updated: Tue Apr 26 19:17:21 2016
Last change: Tue Apr 26 19:17:11 2016 via cibadmin on node4.chinasoft.com
Stack: classic openais (with plugin)
Current DC: node4.chinasoft.com - partition with quorum
Version: 1.1.10-14.el6-368c726
2 Nodes configured, 2 expected votes
0 Resources configured

Node node2.chinasoft.com: standby
Online: [ node4.chinasoft.com ]

Full list of resources:

重新添加资源
# pcs resource create webip ocf:heartbeat:IPaddr ip=192.168.8.77 op monitor interval=30s timeout=20s on-fail=restart
移除webip
# pcs resource move webip
# pcs status