package com.zjc.hadoop;

import java.io.IOException;

import java.util.StringTokenizer;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.Mapper;

import org.apache.hadoop.mapreduce.Reducer;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

public class WordCount {

    public static class TokenizerMapper extends Mapper<Object, Text, Text, IntWritable> {

        // 下面的IntWritable 跟 Text 类是hadoop内部类，相当于 java 中的 int 与 String

        // MapReduce 程序中互相传递的是这种类型的参数

        private final static IntWritable one = new IntWritable(1);

        private Text word = new Text();

        public void map(Object key, Text value, Context context) throws IOException, InterruptedException {

            StringTokenizer itr = new StringTokenizer(value.toString());//java 自带的字符串分割函数

            while (itr.hasMoreTokens()) {

                word.set(itr.nextToken());

                context.write(word, one);

                /*

                 *eg map output:

                 *         hello    1

                 *         word    1

                 *         hello    1

                 *         hadoop    1

                 */

            }

        }

    }

    /*

     * Reduce 输入：

     * key:        hello

     * value:    [1,1]

     *

     *     Hadoop负责将Map产生的<key,value>处理成{具有相同key的value集合}，传给Reducer

        输入：<key,(listof values)>

        输出：<key,value>

        reduce函数（必须是这个名字）的参数，（输入key,输入具有相同key的value集合，Context）其中，

        输入的key,value必须类型与map的输出<key,value>相同，这一点适用于map，reduce类及函数

     *

     */

    public static class IntSumReducer extends Reducer<Text, IntWritable, Text, IntWritable> {

        private IntWritable result = new IntWritable();

        public void reduce(Text key, Iterable<IntWritable> values, Context context)

                throws IOException, InterruptedException {

            int sum = 0;

            System.out.println("-----------------------------------------");

            System.out.println("key: "+key);

            for (IntWritable val : values) {

                System.out.println("val: "+val);

                sum += val.get();

            }

            result.set(sum);

            System.out.println("result: "+result.toString());

            context.write(key, result);

        }

    }

    public static void main(String[] args) throws Exception {

        Configuration conf = new Configuration();

        Job job = Job.getInstance(conf, "word count"); // 任务名

        job.setJarByClass(WordCount.class); // 指定Class

        job.setMapperClass(TokenizerMapper.class); // 指定 Mapper Class

        job.setCombinerClass(IntSumReducer.class); // 指定 Combiner Class

        job.setReducerClass(IntSumReducer.class); // 指定Reucer Class

        job.setOutputKeyClass(Text.class); // 指定输出的KEY的格式

        job.setOutputValueClass(IntWritable.class); // 指定输出的VALUE的格式

        job.setNumReduceTasks(2); //设置Reducer 个数默认1

        // Mapper<Object, Text, Text, IntWritable> 输出格式必须与继承类的后两个输出类型一致

        String args_0 = "hdfs://master:9000/test/input";

        String args_1 = "hdfs://master:9000/test/output";

        FileInputFormat.addInputPath(job, new Path(args_0)); // 输入路径

        FileOutputFormat.setOutputPath(job, new Path(args_1)); // 输出路径

        System.exit(job.waitForCompletion(true) ? 0 : 1);

    }

}

FileInputFormat.addInputPath(job, new Path(otherArgs[0]));

FileOutputFormat.setOutputPath(job, new Path(otherArgs[1]));