1、pom文件依赖
<!--storm相关jar -->
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-core</artifactId>
<version>${storm.version}</version>
<!--排除相关依赖 -->
<exclusions>
<!--<exclusion>-->
<!--<groupId>io.dropwizard.metrics</groupId>-->
<!--<artifactId>metrics-core</artifactId>-->
<!--</exclusion>-->
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-web</artifactId>
</exclusion>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
<exclusion>
<artifactId>ring-cors</artifactId>
<groupId>ring-cors</groupId>
</exclusion>
</exclusions>
<!--<scope>provided</scope>--><!--注意本地调试和集群部署-->
</dependency>
<dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-kafka-client</artifactId>
<version>1.2.2</version>
<!--<scope>provided</scope>--><!--注意本地调试和集群部署-->
</dependency>
<!--注:老版本使用的storm-kafka依赖已经被废弃,建议在以后使用storm-kafka-client依赖进行开发,老版本的storm-kafka依赖为:-->
<!-- <dependency> -->
<!-- <groupId>org.apache.storm</groupId> -->
<!-- <artifactId>storm-kafka</artifactId> -->
<!-- <version>1.2.2</version> -->
<!-- </dependency> -->
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>2.1.0</version>
</dependency>
2、Topology(拓扑中配置Spout,简单的配置可以不用单独的写KafkaSpout)
本打算用spring-kafka的监听器去完成kafka的消费,实现KafkaSpout感觉有点绕,后面再研究,有相关经验的请告知更好的方式
@Component
public class KafkaStormSpoutWordCountTopology {
public static void main(String[] args) {
KafkaSpoutConfig.Builder<String,String> builder =
KafkaSpoutConfig.builder(
"192.168.8.101:9092,192.168.8.102:9092,192.168.8.103:9092",
"topic");
builder.setGroupId("storm_group");
KafkaSpoutConfig<String, String> kafkaSpoutConfig= builder.build();
TopologyBuilder topologyBuilder = new TopologyBuilder();
topologyBuilder.setSpout("WordCountKafkaSpout",new KafkaSpout<String,String>(kafkaSpoutConfig), 1);
topologyBuilder.setBolt("ReadKafkaSpoutBolt",new ReadKafkaSpoutBolt()).shuffleGrouping("WordCountKafkaSpout");
Config config = new Config();
System.out.println("准备启动kafkaStromTopo");
LocalCluster cluster= new LocalCluster();
cluster.submitTopology("kafkaStromTopo", config, topologyBuilder.createTopology());
// //启动topology的配置信息
// Config conf = new Config();
// //TOPOLOGY_DEBUG(setDebug),当他被设置成true的话,storm会记录下每个组件所发射的每条消息
// //这在本地环境调试topology很有用。但是在线上这么做的话,会影响性能
// conf.setDebug(false);
//
// //storm的运行模式有两种:本地模式和分布式模式
// if(args != null || args.length>0){
// conf.setNumWorkers(3);
// //向集群提交topology
// try {
// StormSubmitter.submitTopologyWithProgressBar(args[0],conf,topologyBuilder.createTopology());
// } catch (AlreadyAliveException e) {
// e.printStackTrace();
// } catch (InvalidTopologyException e) {
// e.printStackTrace();
// } catch (AuthorizationException e) {
// e.printStackTrace();
// }
// }
// else{
//
//
// conf.setMaxTaskParallelism(3);
//
// LocalCluster cluster = new LocalCluster();
// cluster.submitTopology("word-count",conf,builder.createTopology());
// }
}
}
3、Bolt, 负责拓扑请跟根据自己的业务
public class ReadKafkaSpoutBolt extends BaseBasicBolt {
@Override
public void execute(Tuple input, BasicOutputCollector basicOutputCollector) {
System.out.println(input.getValues().get(4)+"消息接受bolt");
/*
input 获取到的值
0索引代表kafka的topic
1索引代表kafka的分区
2索引代表kafka的偏移量
3索引代表kafka的key值
4索引代表kafka的value值
*/
}
@Override
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
}
}