【Hadoop离线基础总结】MapReduce 社交粉丝数据分析求出哪些人两两之间有共同好友，及他俩的共同好友都有谁？

2023-08-13 22:20:52

MapReduce 社交粉丝数据分析

求出哪些人两两之间有共同好友，及他俩的共同好友都有谁？

用户及好友数据

A:B,C,D,F,E,O

B:A,C,E,K

C:F,A,D,I

D:A,E,F,L

E:B,C,D,M,L

F:A,B,C,D,E,O,M

G:A,C,D,E,F

H:A,C,D,E,O

I:A,O

J:B,O

K:A,C,D

L:D,E,F

M:E,F,G

O:A,H,I,J

java代码

需要两步完成需求

首先先创建第一步的package

在package中定义main、Mapper、Reducer三个类

定义一个Mapper类

package cn.itcast.demo1.step1;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

public class Step1Mapper extends Mapper<LongWritable, Text, Text, Text> {

    @Override

    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

        //输入数据如下格式  A:B,C,D,E,O

        //将用户和好友列表分开

        String[] split = value.toString().split(":");

        //将好友列表分开，放到一个数组中去

        String[] friendList = split[1].split(",");

        //循环遍历，输出的k2,v2格式为 B [A,E]

        for (String friend : friendList) {

            context.write(new Text(friend), new Text(split[0]));

        }

    }

}

定义一个Reducer类

package cn.itcast.demo1.step1;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class Step1Reducer extends Reducer<Text,Text,Text,Text> {

    /*

    reduce接收到数据是 B [A,E]

    B是好友，集合里面装的是多个用户

    将数据最终转换成这样的形式进行输出 A-B-E-F-G-H-K-  C

     */

    @Override

    protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {

        //创建StringBuffer对象

        StringBuffer sb = new StringBuffer();

        //循环遍历得到v2并拼接成字符串

        for (Text value : values) {

            sb.append(value.toString()).append("-");

        }

        context.write(new Text(sb.toString()),key);

    }

}

程序main函数入口

package cn.itcast.demo1.step1;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.conf.Configured;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

import org.apache.hadoop.util.Tool;

import org.apache.hadoop.util.ToolRunner;

public class Step1Main extends Configured implements Tool {

    @Override

    public int run(String[] args) throws Exception {

        //创建job对象

        Job job = Job.getInstance(super.getConf(), "step1");

        //输入数据，设置输入路径

        job.setInputFormatClass(TextInputFormat.class);

        TextInputFormat.setInputPaths(job, new Path("file:////Volumes/赵壮备份/大数据离线课程资料/5.大数据离线第五天/共同好友/input/friends.txt"));

        //自定义map逻辑

        job.setMapperClass(Step1Mapper.class);

        //设置k2,v2输出类型

        job.setMapOutputKeyClass(Text.class);

        job.setMapOutputValueClass(Text.class);

        //自定义reduce逻辑

        job.setReducerClass(Step1Reducer.class);

        //设置k3,v3输出类型

        job.setOutputKeyClass(Text.class);

        job.setOutputValueClass(Text.class);

        //输出数据，设置输出路径

        job.setOutputFormatClass(TextOutputFormat.class);

        TextOutputFormat.setOutputPath(job, new Path("file:////Volumes/赵壮备份/大数据离线课程资料/5.大数据离线第五天/共同好友/step1_output"));

        //将任务提交至集群

        boolean b = job.waitForCompletion(true);

        return b ? 0 : 1;

    }

    public static void main(String[] args) throws Exception {

        int run = ToolRunner.run(new Configuration(), new Step1Main(), args);

        System.exit(run);

    }

}

运行完成后，得到第一步的数据

F-D-O-I-H-B-K-G-C-	A

E-A-J-F-	B

K-A-B-E-F-G-H-	C

G-K-C-A-E-L-F-H-	D

G-F-M-B-H-A-L-D-	E

M-D-L-A-C-G-	F

M-	G

O-	H

C-O-	I

O-	J

B-	K

E-D-	L

F-E-	M

J-I-H-A-F-	O

创建第二步的package

在package中定义main、Mapper、Reducer三个类

定义一个Mapper类

package cn.itcast.demo1.step2;

import org.apache.hadoop.io.LongWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Mapper;

import java.io.IOException;

import java.util.Arrays;

public class Step2Mapper extends Mapper<LongWritable, Text, Text, Text> {

    @Override

    protected void map(LongWritable key, Text value, Context context) throws IOException, InterruptedException {

        //对拿到的数据进行分割，得到用户列表和好友

        String[] split = value.toString().split("\t");

        //再对用户列表进行分割，得到用户列表数组

        String[] userList = split[0].split("-");

        //因为文件中的数据并不是按照字典顺序进行排序，所以有可能会出来A-E E-A的情况，reduceTask是无法将这种情况视为key相同的

        //所以需要进行排序

        Arrays.sort(userList);

        for (int i = 0; i < userList.length - 1; i++) {

            for (int j = i + 1; j < userList.length; j++) {

                String userTwo = userList[i] + "-" + userList[j];

                context.write(new Text(userTwo), new Text(split[1]));

            }

        }

    }

}

定义一个reducer类

package cn.itcast.demo1.step2;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Reducer;

import java.io.IOException;

public class Step2Reducer extends Reducer<Text, Text, Text, Text> {

    @Override

    protected void reduce(Text key, Iterable<Text> values, Context context) throws IOException, InterruptedException {

        //创建StringBuffer对象

        StringBuffer sb = new StringBuffer();

        for (Text value : values) {

            //获取共同好友列表

            sb.append(value.toString()).append("-");

        }

        context.write(key, new Text(sb.toString()));

    }

}

程序main函数入口

package cn.itcast.demo1.step2;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.conf.Configured;

import org.apache.hadoop.fs.Path;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;

import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;

import org.apache.hadoop.util.Tool;

import org.apache.hadoop.util.ToolRunner;

public class Step2Main extends Configured implements Tool {

    @Override

    public int run(String[] args) throws Exception {

        //创建job对象

        Job job = Job.getInstance(super.getConf(), "step2");

        //输入数据，设置输入路径

        job.setInputFormatClass(TextInputFormat.class);

        TextInputFormat.setInputPaths(job, new Path("file:////Volumes/赵壮备份/大数据离线课程资料/5.大数据离线第五天/共同好友/step1_output"));

        //自定义map逻辑

        job.setMapperClass(Step2Mapper.class);

        //设置k2,v2输出类型

        job.setMapOutputKeyClass(Text.class);

        job.setMapOutputValueClass(Text.class);

        //自定义reduce逻辑

        job.setReducerClass(Step2Reducer.class);

        //设置k3,v3输出类型

        job.setOutputKeyClass(Text.class);

        job.setOutputValueClass(Text.class);

        //输出数据，设置输出路径

        job.setOutputFormatClass(TextOutputFormat.class);

        TextOutputFormat.setOutputPath(job, new Path("file:////Volumes/赵壮备份/大数据离线课程资料/5.大数据离线第五天/共同好友/step2_output"));

        //提交任务至集群

        boolean b = job.waitForCompletion(true);

        return b ? 0 : 1;

    }

    public static void main(String[] args) throws Exception {

        int run = ToolRunner.run(new Configuration(), new Step2Main(), args);

        System.exit(run);

    }

}

运行结果为

A-B	C-E-

A-C	D-F-

A-D	F-E-

A-E	C-B-D-

A-F	D-O-E-B-C-

A-G	C-D-F-E-

A-H	E-C-O-D-

A-I	O-

A-J	O-B-

A-K	C-D-

A-L	E-D-F-

A-M	F-E-

B-C	A-

B-D	E-A-

B-E	C-

B-F	E-A-C-

B-G	A-E-C-

B-H	E-C-A-

B-I	A-

B-K	A-C-

B-L	E-

B-M	E-

B-O	A-

C-D	F-A-

C-E	D-

C-F	A-D-

C-G	F-D-A-

C-H	D-A-

C-I	A-

C-K	A-D-

C-L	D-F-

C-M	F-

C-O	I-A-

D-E	L-

D-F	A-E-

D-G	F-A-E-

D-H	A-E-

D-I	A-

D-K	A-

D-L	F-E-

D-M	F-E-

D-O	A-

E-F	M-C-B-D-

E-G	C-D-

E-H	C-D-

E-J	B-

E-K	C-D-

E-L	D-

F-G	A-D-E-C-

F-H	D-O-C-E-A-

F-I	O-A-

F-J	B-O-

F-K	A-D-C-

F-L	D-E-

F-M	E-

F-O	A-

G-H	E-A-C-D-

G-I	A-

G-K	C-D-A-

G-L	D-E-F-

G-M	E-F-

G-O	A-

H-I	O-A-

H-J	O-

H-K	D-A-C-

H-L	E-D-

H-M	E-

H-O	A-

I-J	O-

I-K	A-

I-O	A-

K-L	D-

K-O	A-

L-M	F-E-

码农公寓

MapReduce 社交粉丝数据分析

求出哪些人两两之间有共同好友，及他俩的共同好友都有谁？

相关文章