hadoop.create.0.1

2021-06-22 20:45:45

#!/bin/sh
exit 0

linux单机分布式实验环境数据策略
#数据:不同源 -v ,link 各自独立的文件夹
#配置:同源,写时复制存于docker images
#程序体,基本不改变.或者写时复制存于docker images
#源码,部分熟悉的程序体,确定它们没有严重的相对于主机的运行时变化,适合单程序体,多实例的.
# 用 -v 所有节点,包括物理机,共享一个真实物理数据源.
#原则上能够共用的,要共用.除了节省磁盘空间外,更重要的是:它们可以共享同一份文件的内存缓存,节省内存,高速!

#data 物理机内data
ids=`seq 1 9`
for id in $ids ;do
#rm -fr /data/dk/c7$id/data/hadoop2.6.5
#!注意该文件为备份,导出文件夹,不能直接挂该文件夹.写时复制;
mkdir -p /win.d/data/dk/c7$id/data/hadoop-2.6.5/{fullhdfs,vhdfs,hahdfs}/tmp /win.d/data/dk/c7$id/data/zookeeper-3.4.12/tmp/
echo $id>/win.d/data/dk/c7$id/data/zookeeper-3.4.12/tmp/myid
done

#初始配置文件都是一样的,但是后来可能会改变.所以配置文件,直接放在克隆源中!而不是放在-v;初始安装文件也是如此,不使用win.d/opt/
#confs/vhdfs/###########################################################################################
core-site.xml
    <property>
        <name>fs.defaultFS</name>
        <value>hdfs://namenode:9000</value>
    </property>
    <property>
        <name>hadoop.tmp.dir</name>
        <value>/data/hadoop-2.6.5/fullhdfs/tmp</value>
    </property>

hdfs-site.xml
    <property>
        <name>dfs.replication</name>
        <value>3</value>
    </property>
    <property>
        <name>dfs.namenode.secondary.http-address</name>
        <value>namenode2:50090</value>
    </property>

#配置java的绝对路径;配置文件中最好不要用${},因为这种写法其实还是依赖解析环境的,解偶合失效;
slaves
datanode01
datanode02
datanode03

hdfs namenode -format
start-dfs.sh
#confs/hahdfs/#######################up is full or v hdfs###############################################

#confs/hahdfs/#######################down is ha hdfs#########################################################
#hahdfs
#hdfs-site.xml
<property>
<name>dfs.replication</name>
<value>3</value>
</property>
<property>
<name>dfs.nameservices</name>
<value>dkc7</value>
</property>
<property>
<name>dfs.ha.namenodes.dkc7</name>
<value>namenode,namenode2</value>
</property>
<property>
<name>dfs.namenode.rpc-address.dkc7.namenode</name>
<value>namenode:8020</value>
</property>
<property>
<name>dfs.namenode.rpc-address.dkc7.namenode2</name>
<value>namenode2:8020</value>
</property>
<property>
<name>dfs.namenode.http-address.dkc7.namenode</name>
<value>namenode:50070</value>
</property>
<property>
<name>dfs.namenode.http-address.dkc7.namenode2</name>
<value>namenode2:50070</value>
</property>

<property>
   <name>dfs.ha.automatic-failover.enabled</name>
   <value>true</value>
</property>
<property>
<name>dfs.namenode.shared.edits.dir</name>
<value>qjournal://datanode01:8485;datanode02:8485;datanode03:8485/dkc7</value>
</property>
<property>
<name>dfs.journalnode.edits.dir</name>
<value>/data/hadoop2.6.5/hahdfs/jn</value>
</property>
<property>
<name>dfs.client.failover.proxy.provider.dkc7</name>
<value>org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider</value>
</property>
<property>
<name>dfs.ha.fencing.methods</name>
<value>sshfence</value>
</property>
<property>
<name>dfs.ha.fencing.ssh.private-key-files</name>
<value>/root/.ssh/id_rsa</value>
    
</property>

#core-site.xml

<property>
<name>hadoop.tmp.dir</name>
<value>/data/hadoop-2.6.5/confs/hahdfs/tmp</value>
</property>

<property>
<name>fs.defaultFS</name>
<value>hdfs://dkc7</value>
</property>

<property>
<name>ha.zookeeper.quorum</name>
<value>zk01:2181,zk02:2181,zk03:2181</value>
</property>

#zookeeper-3.4.12
ids=`seq 1 9`
for id in $ids ;do
echo $id>/win.d/data/dk/c7$id/data/zookeeper-3.4.12/tmp/myid
done
#zoo.cfg
cp $ZOOKEEPER_HOME/conf/zoo.cfg
sed -i s#dataDir=.*#dataDir=/data/zookeeper-3.4.12/tmp#g $ZOOKEEPER_HOME/conf/zoo.cfg
#dataDir=/data/zookeeper-3.4.12/tmp

# run in clone-source c70,then to image
for i in 4 5 6; do
echo "server.$i=1.1.1.7$i:2888:3888">>$ZOOKEEPER_HOME/conf/zoo.cfg
echo $i>/data/zookeeper-3.4.12/tmp/myid
done
echo "rm -fr /data;ln -sf /data/dk/\$HOSTNAME/data /data">>/dockerstartup/rc.local
#zookeeper to images

excp c70 "pushd;cd /opt/hadoop-2.6.5; rm -fr etc;ln -sf confs/hahdfs/etc etc ;popd"
###########################ha hdfs启动顺序##################
zk-->zkfc-->jn-->namenode-->standby-->start-dfs

for id in 4 5 6; do
excp c78 "zkServer.sh start"
done

$ZOOKEEPER_HOME/bin/zkCli.sh
#run in namenode,it then up zk
excp c78 "hdfs zkfc -formatZK "

#id in 1 2 3 8 9 7 do:
#zkServer.sh status
#1
excp c78 "hadoop-daemon.sh start journalnode"
#2.
excp c78 "hadoop-deamon.sh start namenode"
#第一台NN第一次：
#hdfs namenode -format
#3.
#另一台NN：
excp c79 "hdfs namenode -bootstrapStandby"
excp c78 "start-dfs.sh"

excp rm1 "start-yarn.sh"
excp rm2 "start-yarn.sh"

excp c78 "stop-dfs.sh && start-dfs.sh"
#|| hadoop-daemon.sh start zkfc
#confs/hahdfs/#######################up is ha hdfs#########################################################

###############################################down is yarn#################################################
#yarn-site.xml
<property>
   <name>yarn.resourcemanager.ha.enabled</name>
   <value>true</value>
</property>
<property>
   <name>yarn.resourcemanager.cluster-id</name>
   <value>yarnrmcluster</value>
</property>
<property>
   <name>yarn.resourcemanager.ha.rm-ids</name>
   <value>rm1,rm2</value>
</property>
<property>
   <name>yarn.resourcemanager.hostname.rm1</name>
   <value>rm2</value>
</property>
<property>
   <name>yarn.resourcemanager.hostname.rm2</name>
   <value>rm2</value>
</property>
<property>
   <name>yarn.resourcemanager.zk-address</name>
   <value>zk01:2181,zk02:2181,zk03:2181</value>
</property>

#map
    <property>
           <name>mapreduce.framework.name</name>
           <value>yarn</value>
   </property>

#$HADOOP_HOME/sbin/
excp rm1 "start-yarn.sh"
excp rm2 "start-yarn.sh"

############################################up is ha yarn #################################################
# set rm1=zk01 rm2=zk02 to hosts

码农公寓

相关文章