Flume抽取数据(包含自定义拦截器和时间戳拦截器)

#为各组件命名

a1.sources = r1

a1.channels = c1

a1.sinks = k1

#描述source

a1.sources.r1.type = TAILDIR

a1.sources.r1.filegroups = f1

a1.sources.r1.filegroups.f1 = /home/behavior/.*

a1.sources.r1.positionFile = /root/taildir_position.json

a1.sources.r1.interceptors =  i1

a1.sources.r1.interceptors.i1.type = cn.wolfcode.flume.interceptor.ETLInterceptor$Builder

a1.sources.r1.interceptors =  i2

a1.sources.r1.interceptors.i2.type = cn.wolfcode.flume.interceptor.TimeStampInterceptor$Builder

## channel1

a1.channels.c1.type = memory

a1.channels.c1.capacity = 1000

a1.channels.c1.transactionCapacity = 100

## sink1

a1.sinks.k1.type = hdfs

a1.sinks.k1.hdfs.path = /behavior/origin/log/%Y-%m-%d

a1.sinks.k1.hdfs.filePrefix = log-

a1.sinks.k1.hdfs.round = false

a1.sinks.k1.hdfs.rollInterval = 10

a1.sinks.k1.hdfs.rollSize = 134217728

a1.sinks.k1.hdfs.rollCount = 0

## 控制输出文件是原生文件。

a1.sinks.k1.hdfs.fileType = DataStream

a1.sinks.k1.hdfs.writeFormat= Text

## 拼装

a1.sources.r1.channels = c1

a1.sinks.k1.channel= c1

上一篇:使用多IP香港站群服务器对网站管理seo优化提升排名有益处


下一篇:鸿蒙Flutter实战:01-搭建开发环境