hadoop编译

1.hadoop源代码下载 [root@testdb sourcecode]# pwd /opt/sourcecode [root@testdb sourcecode]# ll total 33756 -rw-r--r--. 1 root root 34523353 Aug 20 12:14 hadoop-2.8.1-src.tar.gz [root@testdb sourcecode]# tar -xzvf hadoop-2.8.1-src.tar.gz [root@testdb sourcecode]# ll total 33760 drwxr-xr-x. 17 root root 4096 Jun 2 14:13 hadoop-2.8.1-src -rw-r--r--. 1 root root 34523353 Aug 20 12:14 hadoop-2.8.1-src.tar.gz [root@testdb sourcecode]# cd hadoop-2.8.1-src [root@testdb hadoop-2.8.1-src]# ll total 224 -rw-rw-r--. 1 root root 15623 May 24 07:14 BUILDING.txt drwxr-xr-x. 4 root root 4096 Aug 20 12:44 dev-support drwxr-xr-x. 3 root root 4096 Aug 20 12:44 hadoop-assemblies drwxr-xr-x. 3 root root 4096 Aug 20 12:44 hadoop-build-tools drwxrwxr-x. 2 root root 4096 Aug 20 12:44 hadoop-client drwxr-xr-x. 10 root root 4096 Aug 20 12:44 hadoop-common-project drwxr-xr-x. 2 root root 4096 Aug 20 12:44 hadoop-dist drwxr-xr-x. 8 root root 4096 Aug 20 12:44 hadoop-hdfs-project drwxr-xr-x. 9 root root 4096 Aug 20 12:44 hadoop-mapreduce-project drwxr-xr-x. 3 root root 4096 Aug 20 12:44 hadoop-maven-plugins drwxr-xr-x. 2 root root 4096 Aug 20 12:44 hadoop-minicluster drwxr-xr-x. 3 root root 4096 Aug 20 12:44 hadoop-project drwxr-xr-x. 2 root root 4096 Aug 20 12:44 hadoop-project-dist drwxr-xr-x. 18 root root 4096 Aug 20 12:44 hadoop-tools drwxr-xr-x. 3 root root 4096 Aug 20 12:44 hadoop-yarn-project -rw-rw-r--. 1 root root 99253 May 24 07:14 LICENSE.txt -rw-rw-r--. 1 root root 15915 May 24 07:14 NOTICE.txt drwxrwxr-x. 2 root root 4096 Jun 2 14:24 patchprocess -rw-rw-r--. 1 root root 20477 May 29 06:36 pom.xml -rw-r--r--. 1 root root 1366 May 20 13:30 README.txt -rwxrwxr-x. 1 root root 1841 May 24 07:14 start-build-env.sh [root@testdb hadoop-2.8.1-src]#  [root@testdb hadoop-2.8.1-src]# cat BUILDING.txt Build instructions for Hadoop   ---------------------------------------------------------------------------------- Requirements:   * Unix System * JDK 1.7+ * Maven 3.0 or later * Findbugs 1.3.9 (if running findbugs) * ProtocolBuffer 2.5.0 * CMake 2.6 or newer (if compiling native code), must be 3.0 or newer on Mac * Zlib devel (if compiling native code) * openssl devel (if compiling native hadoop-pipes and to get the best HDFS encryption performance) * Linux FUSE (Filesystem in Userspace) version 2.6 or above (if compiling fuse_dfs) * Internet connection for first build (to fetch all Maven and Hadoop dependencies)   2.JAVA安装 [root@testdb ~]# rz #上传jdk-8u45-linux-x64.gz [root@testdb ~]# mkdir -p /usr/java [root@testdb ~]# mv jdk-8u45-linux-x64.gz /usr/java [root@testdb ~]# cd /usr/java [root@testdb ~]# tar -xzvf jdk-8u45-linux-x64.gz   #修改用户和用户组 [root@hadoop002 java]# ll total 169388 drwxr-xr-x. 8 uucp 143 4096 Apr 11 2015 jdk1.8.0_45 -rw-r--r--. 1 root root 173271626 Mar 16 15:25 jdk-8u45-linux-x64.gz [root@hadoop002 java]# chown -R root:root jdk1.8.0_45 [root@hadoop002 java]# ll total 169388 drwxr-xr-x. 8 root root 4096 Apr 11 2015 jdk1.8.0_45 -rw-r--r--. 1 root root 173271626 Mar 16 15:25 jdk-8u45-linux-x64.gz [root@hadoop002 java]#      [root@testdb java]# vi /etc/profile export JAVA_HOME=/usr/java/jdk1.8.0_45 export PATH=$JAVA_HOME/bin:$PATH   [root@testdb java]# source /etc/profile [root@testdb java]# which java /usr/java/jdk1.8.0_45/bin/java [root@testdb java]# java -version java version "1.8.0_45" Java(TM) SE Runtime Environment (build 1.8.0_45-b14) Java HotSpot(TM) 64-Bit Server VM (build 25.45-b02, mixed mode) [root@testdb java]#    3.Maven安装 [root@testdb ~]# mkdir -p /opt/sourcecode /opt/software [root@testdb ~]# cd /opt/software/ [root@testdb software]# rz rz waiting to receive. Starting zmodem transfer. Press Ctrl+C to cancel. Transferring apache-maven-3.3.9-bin.zip...   100% 8415 KB 8415 KB/sec 00:00:01 0 Errors     [root@testdb software]#  [root@testdb software]# ll total 8432 -rw-r--r--. 1 root root 8617253 Aug 20 12:35 apache-maven-3.3.9-bin.zip [root@testdb software]# unzip apache-maven-3.3.9-bin.zip     [root@testdb java]# vi /etc/profile export MAVEN_HOME=/opt/software/apache-maven-3.3.9 export MAVEN_OPTS="-Xms256m -Xmx512m" export PATH=$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATH   [root@testdb ~]# mvn -version Apache Maven 3.3.9 (bb52d8502b132ec0a5a3f4c09453c07478323dc5; 2015-11-11T00:41:47+08:00) Maven home: /opt/software/apache-maven-3.3.9 Java version: 1.8.0_45, vendor: Oracle Corporation Java home: /usr/java/jdk1.8.0_45/jre Default locale: en_US, platform encoding: UTF-8 OS name: "linux", version: "2.6.32-431.el6.x86_64", arch: "amd64", family: "unix" [root@testdb ~]#      4.protobuf安装 [root@testdb software]# rz rz waiting to receive. Starting zmodem transfer. Press Ctrl+C to cancel. Transferring protobuf-2.5.0.tar.gz...   100% 2345 KB 2345 KB/sec 00:00:01 0 Errors     [root@testdb software]#  [root@testdb software]# ll total 10792 drwxr-xr-x. 6 root root 4096 Nov 10 2015 apache-maven-3.3.9 -rw-r--r--. 1 root root 8617253 Aug 20 12:35 apache-maven-3.3.9-bin.zip drwxr-xr-x. 10 109965 5000 4096 Feb 27 2013 protobuf-2.5.0 -rw-r--r--. 1 root root 2401901 Aug 20 13:03 protobuf-2.5.0.tar.gz [root@testdb software]# cd protobuf-2.5.0 [root@testdb protobuf-2.5.0]# yum install -y gcc gcc-c++ make cmake #已经安装cmake [root@testdb protobuf-2.5.0]# ./configure --prefix=/usr/local/protobuf [root@testdb protobuf-2.5.0]# make && make install   [root@testdb java]# vi /etc/profile export PROTOC_HOME=/usr/local/protobuf export PATH=$PROTOC_HOME/bin:$FINDBUGS_HOME/bin:$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATH [root@testdb protobuf-2.5.0]# source /etc/profile [root@testdb protobuf-2.5.0]# protoc --version libprotoc 2.5.0 [root@testdb protobuf-2.5.0]#      5.Findbugs安装 [root@testdb software]# rz rz waiting to receive. Starting zmodem transfer. Press Ctrl+C to cancel. Transferring findbugs-1.3.9.zip...   100% 7369 KB 7369 KB/sec 00:00:01 0 Errors     [root@testdb software]# unzip findbugs-1.3.9.zip   [root@hadoop002 software]# vi /etc/profile export FINDBUGS_HOME=/opt/software/findbugs-1.3.9 export PATH=$FINDBUGS_HOME/bin:$MAVEN_HOME/bin:$JAVA_HOME/bin:$PATH [root@testdb software]#  [root@testdb software]# source /etc/profile [root@testdb software]# findbugs -version 1.3.9 [root@testdb software]#    6.其他依赖 yum install -y openssl openssl-devel svn ncurses-devel zlib-devel libtool yum install -y snappy snappy-devel bzip2 bzip2-devel lzo lzo-devel lzop autoconf automake   7.编译 [root@testdb sourcecode]# cd hadoop-2.8.1-src [root@testdb sourcecode]# mvn clean package -Pdist,native -DskipTests -Dtar   8.tar  /opt/sourcecode/hadoop-2.8.1-src/hadoop-dist/target/hadoop-2.8.1.tar.gz       提醒:  1、有时候编译过程中会出现下载某个包的时间太久,这是由于连接网站的过程中会出现假死, 此时按ctrl+c,重新运行编译命令。  2、如果出现缺少了某个文件的情况,则要先清理maven(使用命令 mvn clean) 再重新编译。
上一篇:如何基于Avalon总线完成QSYS IP定制


下一篇:Docker入门 安装Tomcat以及报404解决方案