Commit 8b62325d by 吴延飞

优化按方向计算的任务

parent 315034c9
...@@ -37,39 +37,24 @@ public class DirectionEvalJob { ...@@ -37,39 +37,24 @@ public class DirectionEvalJob {
// 获取信控数据-周期结束实时流 Kafka // 获取信控数据-周期结束实时流 Kafka
DataStream<Cycle> cycleStream = DataStream<Cycle> cycleStream =
FlinkUtils.createKafkaStream(args, env, Constant.CYCLE_TOPIC_NAME, "DirectionEvalJob") FlinkUtils.createKafkaStream(args, env, Constant.CYCLE_TOPIC_NAME, "DirectionEvalJob")
.map(record -> { .map(record -> JSON.parseObject(record, Cycle.class));
return JSON.parseObject(record, Cycle.class);
});
// 获取旅行事件数据实时流 kafka // 获取旅行事件数据实时流 kafka
DataStream<TravelEvent> travelEventStream = DataStream<TravelEvent> travelEventStream =
FlinkUtils.createKafkaStream(args, env, Constant.TRAVEL_EVENT_TOPIC_NAME, "DirectionEvalJob") FlinkUtils.createKafkaStream(args, env, Constant.TRAVEL_EVENT_TOPIC_NAME, "DirectionEvalJob")
.map(record -> { .map(record -> JSON.parseObject(record, TravelEvent.class)).filter(event -> StringUtils.isNotEmpty(event.getEventLineId()));
return JSON.parseObject(record, TravelEvent.class);
}).filter(event -> {
return StringUtils.isNotEmpty(event.getEventLineId());
});
WatermarkStrategy<Cycle> cycleWatermarkStrategy = WatermarkStrategy
.<Cycle>forBoundedOutOfOrderness((Duration.ofSeconds(3)))
.withTimestampAssigner((cycle, timestamp) -> Long.parseLong(cycle.getEndDateTime()))
.withIdleness(Duration.ofSeconds(600));
WatermarkStrategy<TravelEvent> travelEventWatermarkStrategy = WatermarkStrategy
.<TravelEvent>forBoundedOutOfOrderness(Duration.ofSeconds(3))
.withTimestampAssigner((xx, timestamp) -> Long.parseLong(xx.getEventTime()));
// 周期数据 interval join 事件流数据,计算 交通流量和排队长度两个指标 // 周期数据 interval join 事件流数据,计算 交通流量和排队长度两个指标
// 旅行事件数据流 // 旅行事件数据流
KeyedStream<TravelEvent, String> travelEventKeyedStream = travelEventStream.assignTimestampsAndWatermarks(travelEventWatermarkStrategy) KeyedStream<TravelEvent, String> travelEventKeyedStream = travelEventStream
.keyBy(record -> record.getCrossId()); .keyBy(record -> record.getCrossId());
// 定义好doris sink builder // 定义好doris sink builder
DorisSink.Builder<String> builder = DorisUtils.getBuilder("doris.properties", args, "app_direction_eval_second_metric"); DorisSink.Builder<String> builder = DorisUtils.getBuilder("doris.properties", args, "app_direction_eval_second_metric");
// 周期数据join 旅行事件 // 周期数据join 旅行事件
cycleStream.assignTimestampsAndWatermarks(cycleWatermarkStrategy) cycleStream.keyBy(record -> record.getCrossCode())
.keyBy(record -> record.getCrossCode())
.connect(travelEventKeyedStream) .connect(travelEventKeyedStream)
.process(new CalDirectionSecondMetricFunction()) .process(new CalDirectionSecondMetricFunction())
.map((MapFunction<DirectionEvalSecondMetric, String>) desm -> { .map((MapFunction<DirectionEvalSecondMetric, String>) desm -> {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment