HADOOP HA 搭建

2024-02-26 11:15:22

1. 准备三台虚拟机

修改host文件

[root@localhost ~]# vi /etc/hosts

192.168.2.151 bigdata1
192.168.2.152 bigdata2
192.168.2.153 bigdata3

查看防火墙状态

[root@bigdata1 ~]# systemctl status firewalld

关闭防火墙

[root@bigdata1 ~]# systemctl stop firewalld

开机防火墙不启动

[root@bigdata1 ~]# systemctl disable firewalld

2. 设置静态ip/主机名/重启

[root@localhost ~]# vi /etc/sysconfig/network-scripts/ifcfg-ens33

TYPE="Ethernet"
PROXY_METHOD="none"
BROWSER_ONLY="no"
BOOTPROTO="static"
DEFROUTE="yes"
IPV4_FAILURE_FATAL="no"
IPV6INIT="yes"
IPV6_AUTOCONF="yes"
IPV6_DEFROUTE="yes"
IPV6_FAILURE_FATAL="no"
IPV6_ADDR_GEN_MODE="stable-privacy"
NAME="ens33"
UUID="f543d854-9cf6-43e5-9746-74658a75e808"
DEVICE="ens33"
ONBOOT="yes"
IPADDR=192.168.2.151   #对应的ip
NETMASK=255.255.255.0
GATEWAY=192.168.2.2
DNS1=8.8.8.8
DNS2=114.114.114.114

[root@localhost ~]# hostnamectl set-hostname <newhostname>
[root@localhost ~]# reboot

3. 编写群发脚本，便于发送

[root@bigdata1 ~]# vi /bin/xsync

#!/bin/bash
#获取输入参数个数，如果没有参数，直接退出
pcount=$#
if((pcount==0)); then
echo no arg;
exit;
fi

#获取文件名称
p1=$1
fname=`basename $p1`
echo fname=$fname

#获取上级目录的绝对路径
pdir=`cd -P $(dirname $p1); pwd`

echo pdir=$pdir

#4获取当前用户名称
user=`whoami`

#5循环
for((host=1;host<4;host++)); do
        echo ------------------- bigdata$host -------------------
        rsync -av $pdir/$fname $user@bigdata$host:$pdir
done

修改脚本使其具有执行权限

[root@bigdata1 ~]# chmod +x /bin/xsync

3. 创建安装目录

[root@bigdata1 ~]# mkdir -p /opt/module/
[root@bigdata1 ~]# mkdir -p /opt/software/

4. 角色分配

bigdata1	bigdata2	bigdata3
NameNode	NameNode
JournalNode	JournalNode	JournalNode
DataNode	DataNode	DataNode
Zookeeper	Zookeeper	Zookeeper
	ResourceManeger	ResourceManeger
NodeManager	NodeManager	NodeManager

5. 下载

jdk下载地址：https://www.oracle.com/java/technologies/javase-jdk15-downloads.html

zookeeper下载地址：https://www.apache.org/dyn/closer.lua/zookeeper/zookeeper-3.6.1/apache-zookeeper-3.6.1-bin.tar.gz

hadoop下载地址：https://www.apache.org/dyn/closer.cgi/hadoop/common/hadoop-2.10.1/hadoop-2.10.1.tar.gz

下载完毕全部上传到 /opt/software/ 目录

6. JDK安装

查看已安装JDK

[root@bigdata1 ~]# rpm -qa | grep java
javapackages-tools-3.4.1-11.el7.noarch
java-1.8.0-openjdk-headless-1.8.0.262.b10-0.el7_8.x86_64
java-1.8.0-openjdk-devel-1.8.0.262.b10-0.el7_8.x86_64
python-javapackages-3.4.1-11.el7.noarch
tzdata-java-2020a-1.el7.noarch
java-1.8.0-openjdk-1.8.0.262.b10-0.el7_8.x86_64

卸载已安装JDK

[root@bigdata1 ~]# rpm -qa | grep java | xargs rpm -e --nodeps

解压安装

[root@bigdata1 ~]# tar -zxvf /opt/software/jdk-15_linux-x64_bin.tar.gz -C /opt/module/

配置JDK环境变量

[root@bigdata1 ~]# vi /etc/profile

#JAVA_HOME
export JAVA_HOME=/opt/module/jdk-15
export PATH=$PATH:$JAVA_HOME/bin

远程复制JDK、环境变量到另外两台主机 bigdata2、bigdata3 上并刷新环境变量

[root@bigdata1 ~]# xsync /opt/module/jdk-15/
[root@bigdata1 ~]# xsync /etc/profile

[root@bigdata1 ~]# source /etc/profile
[root@bigdata2 ~]# source /etc/profile
[root@bigdata3 ~]# source /etc/profile

验证是否安装成功

[root@bigdata1 jdk-15]# java -version
java version "15" 2020-09-15
Java(TM) SE Runtime Environment (build 15+36-1562)
Java HotSpot(TM) 64-Bit Server VM (build 15+36-1562, mixed mode, sharing)

7. 安装zookeeper

解压安装

[root@bigdata1 ~]# tar -zxvf /opt/software/apache-zookeeper-3.6.1-bin.tar.gz /opt/module/

切换到conf目录

[root@bigdata1 ~]# cd /opt/module/apache-zookeeper-3.6.1-bin/conf

拷贝zoo_sample.cfg目录命名为zoo.cfg

[root@bigdata1 conf]# cp zoo_sample.cfg zoo.cfg

修改zoo.cfg配置文件

tickTime=2000
initLimit=10
syncLimit=5
dataDir=/opt/module/apache-zookeeper-3.6.1-bin/data
clientPort=2181
server.1=bigdata1:2888:3888
server.2=bigdata2:2888:3888
server.3=bigdata3:2888:3888

在 bigdata1 上创建data目录和myid文件

[root@bigdata1 ~]# mkdir /opt/module/apache-zookeeper-3.6.1-bin/data
[root@bigdata1 ~]# echo 1 > /opt/module/apache-zookeeper-3.6.1-bin/myid

远程复制zookeeper到另外两台主机 bigdata2、bigdata3 上并修改myid文件

[root@bigdata1 ~]# xsync /opt/module/apache-zookeeper-3.6.1-bin/

[root@bigdata2 ~]# echo 2 > /opt/module/apache-zookeeper-3.6.1-bin/data/myid
[root@bigdata3 ~]# echo 3 > /opt/module/apache-zookeeper-3.6.1-bin/data/myid

配置三台主机zookeeper环境变量

[root@bigdata1 ~]# vi /etc/profile

#ZOOKEEPER_HOME
export ZOOKEEPER_HOME=/opt/module/apache-zookeeper-3.6.1-bin
export PATH=$PATH:$ZOOKEEPER_HOME/bin

[root@bigdata1 ~]# xsync /etc/profile

[root@bigdata1 ~]# source /etc/profile
[root@bigdata2 ~]# source /etc/profile
[root@bigdata3 ~]# source /etc/profile

启动

在三台主机上分别执行如下命令

[root@bigdata1 ~]# zkServer.sh start
ZooKeeper JMX enabled by default
Using config: /opt/module/apache-zookeeper-3.6.1-bin/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
[root@bigdata2 ~]# zkServer.sh start
ZooKeeper JMX enabled by default
Using config: /opt/module/apache-zookeeper-3.6.1-bin/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED
[root@bigdata3 ~]# zkServer.sh start
ZooKeeper JMX enabled by default
Using config: /opt/module/apache-zookeeper-3.6.1-bin/bin/../conf/zoo.cfg
Starting zookeeper ... STARTED

查询运行状态

[root@bigdata1 ~]# zkServer.sh status
ZooKeeper JMX enabled by default
Using config: /opt/module/apache-zookeeper-3.6.1-bin/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: follower
[root@bigdata2 ~]# zkServer.sh status
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/module/apache-zookeeper-3.6.1-bin/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: leader
[root@bigdata3 ~]# zkServer.sh status
/usr/bin/java
ZooKeeper JMX enabled by default
Using config: /opt/module/apache-zookeeper-3.6.1-bin/bin/../conf/zoo.cfg
Client port found: 2181. Client address: localhost.
Mode: follower

码农公寓