hadoop 自编译(Linux)

  1. 准备JDK,maven,ant,protobuf, hadoop 源码包

  2. 下载包链接: https://pan.baidu.com/s/1a16hxPheMdm99g01yZvjSw?pwd=rzm9 提取码: rzm9

  3. 上传包到Linux 中

  4. 解压安装 JDK,Maven,ant

    
    tar -zxvf jdk-8u301-linux-x64.tar.gz 
    mv jdk1.8.0_301/ /usr/local/
    tar -zxvf apache-maven-3.6.3-bin.tar.gz 
    mv apache-maven-3.6.3/ /usr/local/
    
    tar -zvf apache-ant-1.10.7-bin.tar.gz     
    mv apache-ant-1.10.7/ /usr/local/
    
    
  5. 配置环境变量(vi /etc/profile)

    #JAVA_HOME 
    export JAVA_HOME=/usr/local/jdk1.8.0_301
    export PATH=$PATH:$JAVA_HOME/bin
    #MAVEN_HOM 
    export MAVEN_HOME=/usr/local/apache-maven-3.6.3
    export PATH=$PATH:$MAVEN_HOME/bin
    #ANT_HOME 
    export ANT_HOME=/usr/local/apache-ant-1.10.7
    export PATH=$PATH:$ANT_HOME/bin ```
    
  6. source /etc/profile (使配置生效)

  7. 配置maven源下载(vi /usr/local/apache-maven-3.6.3/conf/settings.xml)

    <mirrors>
     <!-- mirror
      | Specifies a repository mirror site to use instead of a given repository. The repository that
      | this mirror serves has an ID that matches the mirrorOf element of this mirror. IDs are used
      | for inheritance and direct lookup purposes, and must be unique across the set of mirrors.
      |
     
         <mirror>
         <id>alimaven</id>
         <mirrorOf>central</mirrorOf>
          <name>aliyun maven</name>
         <url>http://maven.aliyun.com/nexus/content/groups/public/</url>
         </mirror>
         <mirror>
         <id>mirrorId</id>
         <mirrorOf>repositoryId</mirrorOf>
         <name>Human Readable Name for this Mirror.</name>
         <url>http://my.repository.com/repo/path</url>
         </mirror>
      -->
      </mirrors>
    
  8. 配置protobuf

    yum -y install gcc c++
    yum  -y install  gcc-c++
    yum -y install  make
    tar -zxvf protobuf-2.5.0.tar.gz 
    mv protobuf-2.5.0/ /usr/local/
    cd /usr/local/protobuf-2.5.0/
    ./configure 
    
    #配置环境变量(vi /etc/profile)
    #protobuf
     export LD_LIBRARY_PATH=/usr/local/protobuf-2.5.0
     export PATH=$PATH:$LD_LIBRARY_PATH
     ##配置生效
     source /etc/profile
    
  9. 各种依赖命令

    yum -y install svn ncurses-devel gcc*  
    yum -y install lzo-devel zlib-devel autoconf automake libtool cmake openssl-devel
    
  10. 编译hadoop

tar -zxvf hadoop-3.3.1-src.tar.gz 
mv hadoop-3.3.1-src /usr/local/ 
cd /usr/local/hadoop-3.3.1-src/
mvn package -Pdist -DskipTests -Dtar -Dmaven.javadoc.skip=true   (约30分钟左右耐心等待)
  1. 成功后在 hadoop-dist/target/ 下找生成的包
上一篇:ES6模块化


下一篇:Spark入门之环境搭建