记一次Flume-KAFKA、kafka-hdfs操作
TAILDIR-KAFKA
a1.sources = r1
a1.channels = c1 c2
a1.sources.r1.type = TAILDIR
a1.sources.r1.filegroups = f1
a1.sources.r1.filegroups.f1 = /tmp/logs/app-.*log
a1.sources.r1.positionFile = /opt/installs/flume1.9/logs/taildir_position.json
a1.sources.r1.interceptors = i1
a1.sources.r1.interceptors.i1.type = demo.MyInterceptors$Bulider
a1.sources.r1.selector.type = multiplexing
a1.sources.r1.selector.header = type
a1.sources.r1.selector.mapping.start = c1
a1.sources.r1.selector.mapping.event = c2
a1.channels.c1.type = org.apache.flume.channel.kafka.KafkaChannel
a1.channels.c1.kafka.bootstrap.servers = hadoop11:9092,hadoop12:9092,hadoop13:9092
a1.channels.c1.kafka.topic = topic1
a1.channels.c1.parseAsFlumeEvent = false
a1.channels.c2.type = org.apache.flume.channel.kafka.KafkaChannel
a1.channels.c2.kafka.bootstrap.servers = hadoop11:9092,hadoop12:9092,hadoop13:9092
a1.channels.c2.kafka.topic = topic2
a1.channels.c2.parseAsFlumeEvent = false
a1.sources.r1.channels = c1 c2
kafka-mem-hdfs
a1.sources = r1 r2
a1.channels = c1 c2
a1.sinks = k1 k2
a1.sources.r1.type = org.apache.flume.source.kafka.KafkaSource
a1.sources.r1.kafka.bootstrap.servers = hadoop11:9092,hadoop12:9092,hadoop13:9092
a1.sources.r1.kafka.topics = topic1
a1.sources.r1.kafka.consumer.group.id = g1
a1.sources.r2.type = org.apache.flume.source.kafka.KafkaSource
a1.sources.r2.kafka.bootstrap.servers = hadoop11:9092,hadoop12:9092,hadoop13:9092
a1.sources.r2.kafka.topics = topic2
a1.sources.r2.kafka.consumer.group.id = g1
a1.channels.c1.type = memory
a1.channels.c1.capacity = 10000
a1.channels.c1.transactionCapacity = 10000
a1.channels.c2.type = memory
a1.channels.c2.capacity = 10000
a1.channels.c2.transactionCapacity = 10000
a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.path = hdfs://hdfs-cluster/origin_data/gmall/log/topic1/%Y-%m-%d
a1.sinks.k1.hdfs.useLocalTimeStamp = true
a1.sinks.k1.hdfs.fileType=DataStream
a1.sinks.k1.hdfs.rollInterval = 10
a1.sinks.k1.hdfs.rollSize = 104857600
a1.sinks.k1.hdfs.rollCount = 0
a1.sinks.k1.hdfs.filePrefix = logstart-
a1.sinks.k2.type = hdfs
a1.sinks.k2.hdfs.path = hdfs://hdfs-cluster/origin_data/gmall/log/topic2/%Y-%m-%d
a1.sinks.k2.hdfs.useLocalTimeStamp = true
a1.sinks.k2.hdfs.fileType=DataStream
a1.sinks.k2.hdfs.rollInterval = 10
a1.sinks.k2.hdfs.rollSize = 104857600
a1.sinks.k2.hdfs.rollCount = 0
a1.sinks.k2.hdfs.filePrefix = logevent-
a1.sources.r1.channels = c1
a1.sinks.k1.channel = c1
a1.sources.r2.channels = c2
a1.sinks.k2.channel = c2
拦截器代码
public class MyInterceptors implements Interceptor {
@Override
public void initialize() {
}
@Override
public Event intercept(Event event) {
byte[] body = event.getBody();
String str = new String(body);
Map<String, String> map = event.getHeaders();
if(str.contains("\"en\":\"start\"")){
map.put("type","start");
}else{
map.put("type","event");
}
return event;
}
@Override
public List<Event> intercept(List<Event> list) {
for(Event event:list){
intercept(event);//循环,使集合中的每个event调用intercept方法,改变每个event中map的值为type-N/L
}
return list;
}
@Override
public void close() {
}
public static class Bulider implements Interceptor.Builder{
@Override
public Interceptor build() {
return new MyInterceptors();
}
@Override
public void configure(Context context) {
}
}
}
这篇好文章是转载于:学新通技术网
- 版权申明: 本站部分内容来自互联网,仅供学习及演示用,请勿用于商业和其他非法用途。如果侵犯了您的权益请与我们联系,请提供相关证据及您的身份证明,我们将在收到邮件后48小时内删除。
- 本站站名: 学新通技术网
- 本文地址: /boutique/detail/tanhgikfbk
-
photoshop保存的图片太大微信发不了怎么办
PHP中文网 06-15 -
《学习通》视频自动暂停处理方法
HelloWorld317 07-05 -
word里面弄一个表格后上面的标题会跑到下面怎么办
PHP中文网 06-20 -
Android 11 保存文件到外部存储,并分享文件
Luke 10-12 -
photoshop扩展功能面板显示灰色怎么办
PHP中文网 06-14 -
微信公众号没有声音提示怎么办
PHP中文网 03-31 -
excel下划线不显示怎么办
PHP中文网 06-23 -
excel打印预览压线压字怎么办
PHP中文网 06-22 -
TikTok加速器哪个好免费的TK加速器推荐
TK小达人 10-01 -
怎样阻止微信小程序自动打开
PHP中文网 06-13