Hadoop 二次排序

2024-04-06 19:35:32

需求

求每年的最高气温，年份升序，温度求最高

数据源内容如下

temperature.txt

思路

需要排序2次，先比较年份，再比较相同年份下，温度最高的值，也就是说需要对2个纬度的值做排序。但是hadoop只能够在key上进行排序，所以气温和年份的值都得放在key里面，也就是需要创建自定义组合key

pom依赖

<dependency>
    <groupId>org.apache.hadoop</groupId>
    <artifactId>hadoop-client</artifactId>
    <version>2.7.3</version>
</dependency>

代码

1.自定义key

import org.apache.hadoop.io.WritableComparable;

import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;

/**
 所有自定义的组合key应该实现接口WritableComparable，WritableComparable接口继承自writable和comparable这两个接
 因为writable接口是可序列化的并且可比较的。WritableComparable。组合key按照年份升序按照气温降序
 */

public class ComboKey implements WritableComparable<ComboKey> {
    private int year ;
    private int temp ;

    public int getYear() {
        return year;
    }

    public void setYear(int year) {
        this.year = year;
    }

    public int getTemp() {
        return temp;
    }

    public void setTemp(int temp) {
        this.temp = temp;
    }

    /**
     * 对key进行比较实现
     */
    public int compareTo(ComboKey o) {
        int y0 = o.getYear();
        int t0 = o.getTemp();

        if(year == y0){
            //气温降序
            return -(temp - t0) ;
        }
        //年份升序
        else{
            return year - y0 ;
        }
    }



    /**
     * 串行化过程
     */
    public void write(DataOutput out) throws IOException {
        //年份
        out.writeInt(year);
        //气温
        out.writeInt(temp);
    }

    ////反串行化的过程
    public void readFields(DataInput in) throws IOException {
        year = in.readInt();
        temp = in.readInt();
    }
}

2.自定义分区

import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapreduce.Partitioner;

//该分区类按照年份进行分区，相同的年份会进入到同一个分区中去
public class YearPartitioner extends Partitioner<ComboKey,NullWritable> {
    public int getPartition(ComboKey key, NullWritable nullWritable, int numPartitions) {
        int year = key.getYear();
        return year % numPartitions;
    }
}

3.CombokeyComparator

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;

/**
 同时完成Combokey中的first和second排序。
 */
public class ComboKeyComparator extends WritableComparator {

    protected ComboKeyComparator() {
        super(ComboKey.class, true);
    }

    public int compare(WritableComparable a, WritableComparable b) {
        ComboKey k1 = (ComboKey) a;
        ComboKey k2 = (ComboKey) b;
        return k1.compareTo(k2);
    }
}

4.分组函数

import org.apache.hadoop.io.WritableComparable;
import org.apache.hadoop.io.WritableComparator;

/**
 分组在Reduce阶段，构造一个与 Key 相对应的 Value 迭代器的时候，只要year相同就属于同一个组，放在一个Value迭代器,不同的year按照年份升序进行排序。
 最后，ComboKey的year相同，但是temp不同的数据会进入一组，并且按temp降序排列。如：
 2018 40
 2018 38
 2018 37
 分组后的第一条数据就是我们需要的（也就是reduce阶段的key）
 */
public class YearGroupComparator extends WritableComparator {
    protected YearGroupComparator() {
        super(ComboKey.class, true);
    }

    public int compare(WritableComparable a, WritableComparable b) {
        ComboKey k1 = (ComboKey)a ;
        ComboKey k2 = (ComboKey)b ;
        return k1.getYear() - k2.getYear() ;
    }
}

5.Map

import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

/**
 * Map端，输入的(key,value)缩进长度和文本文档，输出的key是组合key,value值是空值
 */
public class MaxTempMapper extends Mapper<LongWritable,Text,ComboKey,NullWritable>{
    @Override
    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {
        System.out.println("MaxTempMapper.map");
        String[] arr = value.toString().split(" ");
        ComboKey keyout = new ComboKey();
        keyout.setYear(Integer.parseInt(arr[0]));
        keyout.setTemp(Integer.parseInt(arr[1]));
        context.write(keyout,NullWritable.get());
    }
}

6.reduce

/**
 * Reduce端，将组合key切割成key为year，value为气温的一个列表
 */
public class MaxTempReducer extends Reducer<ComboKey, NullWritable, IntWritable, IntWritable>{

    protected void reduce(ComboKey key, Iterable<NullWritable> values, Context context) throws IOException, InterruptedException {
        int year = key.getYear();
        int temp = key.getTemp();
        for(NullWritable v : values){
            System.out.println(key.getYear() + " : " + key.getTemp());
        }
        context.write(new IntWritable(year),new IntWritable(temp));
    }
}

7.APP

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import java.io.File;


/**
 *二次排序
 *求每年最高的温度，年份升序，温度求最高
 *
 * 数据格式
 2004 49
 1981 -22
 1981 -31
 1965 -47
 2017 -2
 */
public class APP {
    public static String run_mode = "local";
    //public static String run_mode = "cluster";
    public static void main(String[] args) throws Exception {
        Configuration conf = new Configuration();
        Job job = Job.getInstance(conf);

        //本地
        if(run_mode.equals("local")) {
            File dir = new File("c:\\out");
            if (dir.exists()) {
                APP.delFile(dir);
            }
            conf.set("fs.defaultFS", "file:///");
            //添加输入路径
            FileInputFormat.addInputPath(job,new Path("C://temperature.txt"));
            //设置输出路径
            FileOutputFormat.setOutputPath(job,new Path("C://out"));
            //linux集群
        } else if(run_mode.equals("cluster")) {
            conf.set("fs.defaultFS", "hdfs://master:9000");
            //添加输入路径
            FileInputFormat.addInputPath(job,new Path("/temperature.txt"));
            //设置输出路径
            FileOutputFormat.setOutputPath(job,new Path("/out"));
        }

        //设置job的各种属性
        job.setJobName("WCApp");                        //作业名称
        job.setJarByClass(APP.class);                 //搜索类
        //job.setInputFormatClass(FileInputFormat.class); //设置输入格式


        //添加输入路径
        //FileInputFormat.addInputPath(job,new Path(args[0]));
        //设置输出路径
        //FileOutputFormat.setOutputPath(job,new Path(args[1]));


        job.setMapperClass(MaxTempMapper.class);             //mapper类
        job.setReducerClass(MaxTempReducer.class);           //reducer类

        job.setNumReduceTasks(1);                       //reduce个数

        job.setMapOutputKeyClass(ComboKey.class);
        job.setMapOutputValueClass(NullWritable.class);

        job.setOutputKeyClass(IntWritable.class);
        job.setOutputValueClass(IntWritable.class);

        //设置分区类
        job.setPartitionerClass(YearPartitioner.class);
        //设置分组对比器
        job.setGroupingComparatorClass(YearGroupComparator.class);
        //设置排序对比器(好像不写也行,因为不写的话，WritableComparator的compare方法底层还是会调用自定义key ComboKey.class里的compareTo方法)
        job.setSortComparatorClass(ComboKeyComparator.class);

        job.waitForCompletion(true);
    }

    static boolean delFile(File file) {
        if (!file.exists()) {
            return false;
        }

        if (file.isDirectory()) {
            File[] files = file.listFiles();
            for (File f : files) {
                delFile(f);
            }
        }
        return file.delete();
    }
}

码农公寓

需求

思路

pom依赖

代码

相关文章