<?xml version="1.0" encoding="UTF-8"?>

 <project xmlns="http://maven.apache.org/POM/4.0.0"

          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">

     <parent>

         <artifactId>HADOOP</artifactId>

         <groupId>yinzhengjie.org.cn</groupId>

         <version>1.0-SNAPSHOT</version>

     </parent>

     <modelVersion>4.0.0</modelVersion>

     <artifactId>MapReduce</artifactId>

     <dependencies>

         <dependency>

             <groupId>org.apache.hadoop</groupId>

             <artifactId>hadoop-common</artifactId>

             <version>2.6.0</version>

         </dependency>

         <dependency>

             <groupId>org.apache.hadoop</groupId>

             <artifactId>hadoop-client</artifactId>

             <version>2.6.0</version>

         </dependency>

         <dependency>

             <groupId>junit</groupId>

             <artifactId>junit</artifactId>

             <version>4.11</version>

             <scope>test</scope>

         </dependency>

     </dependencies>

 </project>

 /*

 @author :yinzhengjie

 Blog:http://www.cnblogs.com/yinzhengjie/tag/Hadoop%E7%94%9F%E6%80%81%E5%9C%88/

 EMAIL:y1053419035@qq.com

 */

 package mapreduce.yinzhengjie.org.cn;

 import org.apache.hadoop.io.IntWritable;

 import org.apache.hadoop.io.LongWritable;

 import org.apache.hadoop.io.Text;

 import org.apache.hadoop.mapreduce.Mapper;

 import java.io.IOException;

 public class WordcountMapper extends Mapper<LongWritable, Text, Text, IntWritable>{

     Text k = new Text();

     IntWritable v = new IntWritable(1);

     @Override

     protected void map(LongWritable key, Text value, Context context)

             throws IOException, InterruptedException {

         // 1 获取一行

         String line = value.toString();

         // 2 切割

         String[] words = line.split(" ");

         // 3 输出

         for (String word : words) {

             k.set(word);

             context.write(k, v);

         }

     }

 }

 /*

 @author :yinzhengjie

 Blog:http://www.cnblogs.com/yinzhengjie/tag/Hadoop%E7%94%9F%E6%80%81%E5%9C%88/

 EMAIL:y1053419035@qq.com

 */

 package mapreduce.yinzhengjie.org.cn;

 import java.io.IOException;

 import org.apache.hadoop.io.IntWritable;

 import org.apache.hadoop.io.Text;

 import org.apache.hadoop.mapreduce.Reducer;

 public class WordcountReducer extends Reducer<Text, IntWritable, Text, IntWritable>{

     @Override

     protected void reduce(Text key, Iterable<IntWritable> value,

                           Context context) throws IOException, InterruptedException {

         // 1 累加求和

         int sum = 0;

         for (IntWritable count : value) {

             sum += count.get();

         }

         // 2 输出

         context.write(key, new IntWritable(sum));

     }

 }

 /*

 @author :yinzhengjie

 Blog:http://www.cnblogs.com/yinzhengjie/tag/Hadoop%E7%94%9F%E6%80%81%E5%9C%88/

 EMAIL:y1053419035@qq.com

 */

 package mapreduce.yinzhengjie.org.cn;

 import java.io.IOException;

 import org.apache.hadoop.conf.Configuration;

 import org.apache.hadoop.fs.Path;

 import org.apache.hadoop.io.IntWritable;

 import org.apache.hadoop.io.Text;

 import org.apache.hadoop.mapreduce.Job;

 import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

 import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

 public class WordcountDriver {

     public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {

         /**

          *   配合Hadoop的环境变量，如果没有配置可能会抛异常：“ERROR util.Shell: Failed to locate the winutils binary in the hadoop binary path”，

          * 还有一件事就是你的HADOOP_HOME的bin目录下必须得有winutils.exe

          *

          */

             System.setProperty("hadoop.home.dir", "D:\\yinzhengjie\\softwares\\hadoop-2.7.3");

         //获取配置信息

         Configuration conf = new Configuration();

         Job job = Job.getInstance(conf);

         //设置jar加载路径

         job.setJarByClass(WordcountDriver.class);

         //设置map和Reduce类

         job.setMapperClass(WordcountMapper.class);

         job.setReducerClass(WordcountReducer.class);

         //设置map输出

         job.setMapOutputKeyClass(Text.class);

         job.setMapOutputValueClass(IntWritable.class);

         //设置Reduce输出

         job.setOutputKeyClass(Text.class);

         job.setOutputValueClass(IntWritable.class);

         //设置输入和输出路径

         FileInputFormat.setInputPaths(job, new Path(args[0]));

         FileOutputFormat.setOutputPath(job, new Path(args[1]));

         //等待job提交完毕

         boolean result = job.waitForCompletion(true);

         System.exit(result ? 0 : 1);

     }

 }

  System.setProperty("hadoop.home.dir", "D:\\yinzhengjie\\softwares\\hadoop-2.7.3");

log4j.rootLogger=${hadoop.root.logger}

hadoop.root.logger=INFO,console

log4j.appender.console=org.apache.log4j.ConsoleAppender

log4j.appender.console.target=System.err

log4j.appender.console.layout=org.apache.log4j.PatternLayout

log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{}: %m%n

Apache Hive is an open source project run by volunteers at the Apache Software Foundation.

The Apache Hadoop software library is a framework that allows for the distributed processing of large data sets across clusters of computers using simple programming models. It is designed to scale up from single servers to thousands of machines, each offering local computation and storage. Rather than rely on hardware to deliver high-availability, the library itself is designed to detect and handle failures at the application layer, so delivering a highly-available service on top of a cluster of computers, each of which may be prone to failures.

Use Apache HBase when you need random, realtime read/write access to your Big Data. This project's goal is the hosting of very

large tables -- billions of rows X millions of columns -- atop clusters of commodity hardware. Apache HBase is an open-source, distributed, versioned, non-relational database modeled after Google's Bigtable: A Distributed Storage System for Structured Data by Chang et al. Just as Bigtable leverages the distributed data storage provided by the Google File System, Apache HBase provides Bigtable-like capabilities on top of Hadoop and HDFS.

Flume is a distributed, reliable, and available service for efficiently collecting, aggregating, and moving large amounts of log data. It has a simple and flexible architecture based on streaming data flows. It is robust and fault tolerant with tunable reliability mechanisms and many failover and recovery mechanisms. It uses a simple extensible data model that allows for online analytic application.

Apache Spark is a unified analytics engine for large-scale data processing.

D:\yinzhengjie\data\hello.txt                                                        文件内容戳我！！！！

[root@node105 ~]# hdfs dfs -put hello.txt /yinzhengjie/data/

[root@node105 ~]# hadoop jar MapReduce.jar  mapreduce.yinzhengjie.org.cn.WordcountDriver /yinzhengjie/data/hello.txt /yinzhengjie/data/output