cdh 大数据平台搭建

2024-02-29 08:13:40

安装包下载

链接: https://pan.baidu.com/s/1dyvSej5tSrUC4ja8-usqvA 提取码: a9hx

上传压缩包

上传cdh6.3.2.tar安装包到linux服务器，并解压

tar -xvf cdh6.3.2.tar -C /tmp/

组件安装

安装 expect 命令，实现脚本交互

#所有节点
yum install expect -y

rhel 7 默认没有pstree命令，使用以下命令进行安装

#所有节点
yum install psmisc -y

rhel 7 如果为最小安装时，全报客户端部署错误，此时一般是由于缺程序引起的，常见的是缺perl

#所有节点
yum install perl -y

hue 需要使用到 httpd

#所有节点
yum install httpd mod_ssl -y

kudu 需要以下组件

#所有节点
yum install gcc python-devel -y
yum install cyrus-sasl* -y

调整host文件

#所有节点
cat >> /etc/hosts <<EOF
192.168.88.220 worker-1
192.168.88.221 worker-2
192.168.88.222 worker-3
EOF

关闭selinux

#所有节点
grep ^SELINUX= /etc/selinux/config
sed -i 's/^SELINUX=.*/SELINUX=disabled/g' /etc/selinux/config
setenforce 0
getenforce

关闭iptables

#所有节点
systemctl disable firewalld
systemctl stop firewalld

ssh互信

#所有节点
ssh-keygen -t rsa

在每台机器械上执行

#逐条执行
ssh-copy-id root@worker-1
ssh-copy-id root@worker-2
ssh-copy-id root@worker-3

主节点执行

scp ~/.ssh/authorized_keys root@worker-1:/root/.ssh/
scp ~/.ssh/authorized_keys root@worker-2:/root/.ssh/
scp ~/.ssh/authorized_keys root@worker-3:/root/.ssh/

验证ssh效果

调整系统使用swap的策略
调整系统使用swap的策略，设置为1，即最大限度使用物理内存，少使用swap空间

#所有节点执行

#检查当前设置值
sysctl -a | grep vm.swappiness
echo 1 > /proc/sys/vm/swappiness
sysctl -a | grep vm.swappiness
#更改默认配置
cat >> /etc/sysctl.conf <<EOF
vm.swappiness=1
EOF
#设置立即生效
sysctl vm.swappiness=1
#设置下次启动生效
cat >> /etc/rc.d/rc.local <<EOF
if test -f /sys/kernel/mm/transparent_hugepage/enabled; then
    echo never > /sys/kernel/mm/transparent_hugepage/enabled
fi
if test -f /sys/kernel/mm/transparent_hugepage/defrag; then
    echo never > /sys/kernel/mm/transparent_hugepage/defrag
fi
EOF
#配置即刻生效
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo never > /sys/kernel/mm/transparent_hugepage/defrag
#检查配置是否生效
cat /sys/kernel/mm/transparent_hugepage/enabled
cat /sys/kernel/mm/transparent_hugepage/defrag
#赋予执行权限
chmod 755 /etc/rc.d/rc.local

修改sudoers

#所有节点执行
unalias cp
cp -n /etc/sudoers /etc/sudoers.bak && \
cp -f /etc/sudoers.bak /etc/sudoers && \
chmod 740 /etc/sudoers && \
sed -i 's/^\(Defaults requiretty\)/# \1/g' /etc/sudoers && \
chmod 440 /etc/sudoers
grep requiretty /etc/sudoers

配置ntp时间同步

#所有节点执行
yum install ntp -y
#主节点执行
mv -n /etc/ntp.conf /etc/ntp.conf.bak
cat > /etc/ntp.conf << EOF
server 127.127.1.0
fudge 127.127.1.0 stratum 10
EOF
#从节点执行，且只能填写ip不能使用hostname，对应的ip为主节点的ip地址
mv -n /etc/ntp.conf /etc/ntp.conf.bak
cat > /etc/ntp.conf << EOF
server 192.168.88.220 iburst
EOF
#禁用chronyd服务，RHEL 7中默认启用了，这会导致ntpd服务无法开机启动
systemctl disable chronyd
#所有节点启动ntp服务
chmod 644 /etc/ntp.conf
systemctl restart ntpd.service
systemctl status ntpd.service
systemctl enable ntpd.service
#检查ntp是否生效
ntpdate -u 192.168.88.220
ntpq -p

ntp安装

主节点配置

从节点配置

禁用chronyd

服务启动

服务检查

调整后

安装jdk

# 所有节点执行

#所有的都删除掉
rpm -qa | grep java
#安装jdk
cd /tmp/
#包分发到其他节点上
scp oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm root@worker-2:/tmp/
scp oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm root@worker-3:/tmp/
yum install oracle-j2sdk1.8-1.8.0+update181-1.x86_64.rpm -y
ln -s /usr/java/jdk1.8.0_181-cloudera /usr/java/latest
#配置环境变量
cat >> /etc/profile <<EOF
export JAVA_HOME=/usr/java/latest
export JRE_HOME=\$JAVA_HOME/jre
export CLASSPATH=.:\$CLASSPATH:\$JAVA_HOME/lib:\$JRE_HOME/lib
export PATH=\$PATH:\$JAVA_HOME/bin:\$JRE_HOME/bin
EOF
#source环境变量及验证是否生效
source /etc/profile
java -version

安装、配置mysql

#主节点执行即可

#安装mariadb软件
#所有的都删除掉
rpm -qa | grep mysql
#安装
yum install -y mariadb
yum install -y mariadb-server
#启动
systemctl enable mariadb
systemctl start mariadb
#初始化配置
mysql_secure_installation

创建cm元数据库

mysql -uroot -p

create database metastore default character set utf8;
create user 'hive'@'%' identified by 'hivedemima';
grant all privileges on metastore.* to 'hive'@'%';

create database cm default character set utf8;
create user 'cm'@'%' identified by 'cmdemima';
grant all privileges on cm.* to 'cm'@'%';

create database am default character set utf8;
create user 'am'@'%' identified by 'amdemima';
grant all privileges on am.* to 'am'@'%';

create database rm default character set utf8;
create user 'rm'@'%' identified by 'rmdemima';
grant all privileges on rm.* to 'rm'@'%';

create database hue default character set utf8;
create user 'hue'@'%' identified by 'huedemima';
grant all privileges on hue.* to 'hue'@'%';

create database oozie default character set utf8;
create user 'oozie'@'%' identified by 'ooziedemima';
grant all privileges on oozie.* to 'oozie'@'%';

create database sentry default character set utf8;
create user 'sentry'@'%' identified by 'sentrydemima';
grant all privileges on sentry.* to 'sentry'@'%';

create database nas default character set utf8;
create user 'nas'@'%' identified by 'nasdemima';
grant all privileges on nas.* to 'nas'@'%';

create database nms default character set utf8;
create user 'nms'@'%' identified by 'nmsdemima';
grant all privileges on nms.* to 'nms'@'%';

flush privileges;
exit;

配置http服务

#所有节点安装

#安装及开启http服务
yum install httpd -y
systemctl start httpd
systemctl enable httpd
systemctl status httpd

#主节点执行
#配置http服务
#移动两个文件夹到html目录下
mkdir -p /var/www/html/cm6.3.1
mkdir -p /var/www/html/cdh6.3.2
cd /tmp
mv CDH-6.3.2-1.cdh6.3.2.p0.1605554-el7.parcel* manifest.json /var/www/html/cdh6.3.2/
mv allkeys.asc cloudera-manager-* enterprise-debuginfo-6.3.1-1466458.el7.x86_64.rpm /var/www/html/cm6.3.1/

使用浏览器测试访问

http://worker-1/cm6.3.1/

http://worker-1/cdh6.3.2/

创建cm的repo

#主节点执行
yum install createrepo -y
cd /var/www/html/cm6.3.1
createrepo .
cat > /etc/yum.repos.d/cloudera.repo <<EOF
[Cloudera_Manager]
name=Cloudera Manager 6.3.1
baseurl=http://worker-1/cm6.3.1/
enabled=1
gpgcheck=0
EOF
yum clean all
yum repolist

安装CDH6

#所有节点

#复制MySQL的JDBC包
mkdir /usr/share/java
#主节点执行
scp /tmp/mysql-connector-java-5.1.35.jar root@worker-2:/tmp/
scp /tmp/mysql-connector-java-5.1.35.jar root@worker-3:/tmp/
#所有节点执行
mv /tmp/mysql-connector-java-5.1.35.jar /usr/share/java/
ln -s /usr/share/java/mysql-connector-java-5.1.35.jar /usr/share/java/mysql-connector-java.jar

安装CM

#只在主节点执行
yum install cloudera-manager-server -y

初始化CM元数据库

/opt/cloudera/cm/schema/scm_prepare_database.sh mysql -h 192.168.88.220 cm cm cmdemima

启动cm服务

#主节点执行
systemctl start cloudera-scm-server
systemctl status cloudera-scm-server
#检查端口状态
yum install net-tools -y
netstat -apn | grep 7180

网页登陆

登录cloudera manager管理页面，默认的账号、密码为：admin/admin

http://worker-1:7180

“同意协议” -> “选择Cloudera Express” -> “选择节点” -> …

集群安装时库选择

parcel的远程安装库为：http://worker-1/cdh6.3.2/

agent的远程安装库为：http://worker-1/cm6.3.1/

Kudu在安装时需要填写以下几项内容：

Kudu Master WAL Directory: /kudu_master/fs_wal_dir

Kudu Master Data Directories: /kudu_master/fs_data_dirs

Kudu Tablet Server WAL Directory: /kudu_tablet/fs_wal_dir

Kudu Tablet Server Data Directories: /kudu_tablet/fs_data_dirs

单机版问题解决

单点部署时HDFS 副本不足告警：Under-Replicated Blocks

修改 dfs.replication 参数为1，之后在HDFS下执行以下命令将所有文件副本数设置为1：

hadoop fs -setrep -R 1 /

海汼部落原创文章，原文链接：http://www.hainiubl.com/topics/75615

码农公寓