Commit 8435a187 by 吴延飞

修改按方向的任务的计算策略,改为connected stream来计算

parent ad30d1f1
...@@ -4,15 +4,20 @@ import com.alibaba.fastjson.JSON; ...@@ -4,15 +4,20 @@ import com.alibaba.fastjson.JSON;
import com.zhht.irn.consts.Constant; import com.zhht.irn.consts.Constant;
import com.zhht.irn.entity.Cycle; import com.zhht.irn.entity.Cycle;
import com.zhht.irn.entity.TravelEvent; import com.zhht.irn.entity.TravelEvent;
import com.zhht.irn.entity.metric.DirectionEvalSecondMetric;
import com.zhht.irn.functions.CalDirectionSecondMetricFunction; import com.zhht.irn.functions.CalDirectionSecondMetricFunction;
import com.zhht.irn.sink.DirectionEvalSecondSink; import com.zhht.irn.sink.DirectionEvalSecondSink;
import com.zhht.irn.source.CycleMockSource;
import com.zhht.irn.source.TravelEventMockSource;
import com.zhht.irn.utils.FlinkUtils; import com.zhht.irn.utils.FlinkUtils;
import com.zhht.irn.utils.StringUtils; import com.zhht.irn.utils.StringUtils;
import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream; import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.co.KeyedCoProcessFunction;
import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.util.Collector;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -38,6 +43,7 @@ public class DirectionEvalJob { ...@@ -38,6 +43,7 @@ public class DirectionEvalJob {
DataStream<Cycle> cycleStream = DataStream<Cycle> cycleStream =
FlinkUtils.createKafkaStream(args, env, Constant.CYCLE_TOPIC_NAME, "DirectionEvalJob") FlinkUtils.createKafkaStream(args, env, Constant.CYCLE_TOPIC_NAME, "DirectionEvalJob")
.map(record -> { .map(record -> {
System.out.println(record);
return JSON.parseObject(record, Cycle.class); return JSON.parseObject(record, Cycle.class);
}); });
...@@ -45,6 +51,7 @@ public class DirectionEvalJob { ...@@ -45,6 +51,7 @@ public class DirectionEvalJob {
DataStream<TravelEvent> travelEventStream = DataStream<TravelEvent> travelEventStream =
FlinkUtils.createKafkaStream(args, env, Constant.TRAVEL_EVENT_TOPIC_NAME, "DirectionEvalJob") FlinkUtils.createKafkaStream(args, env, Constant.TRAVEL_EVENT_TOPIC_NAME, "DirectionEvalJob")
.map(record -> { .map(record -> {
System.out.println(record);
return JSON.parseObject(record, TravelEvent.class); return JSON.parseObject(record, TravelEvent.class);
}).filter(event -> { }).filter(event -> {
return StringUtils.isNotEmpty(event.getEventLineId()); return StringUtils.isNotEmpty(event.getEventLineId());
...@@ -67,8 +74,7 @@ public class DirectionEvalJob { ...@@ -67,8 +74,7 @@ public class DirectionEvalJob {
// 周期数据join 旅行事件 // 周期数据join 旅行事件
cycleStream.assignTimestampsAndWatermarks(cycleWatermarkStrategy) cycleStream.assignTimestampsAndWatermarks(cycleWatermarkStrategy)
.keyBy(record -> record.getCrossCode()) .keyBy(record -> record.getCrossCode())
.intervalJoin(travelEventKeyedStream) .connect(travelEventKeyedStream)
.between(Time.minutes(-9), Time.minutes(1))
.process(new CalDirectionSecondMetricFunction()) .process(new CalDirectionSecondMetricFunction())
.addSink(new DirectionEvalSecondSink()).setParallelism(4); .addSink(new DirectionEvalSecondSink()).setParallelism(4);
......
...@@ -43,7 +43,6 @@ public class DirectionEvalSecondSink extends RichSinkFunction<DirectionEvalSecon ...@@ -43,7 +43,6 @@ public class DirectionEvalSecondSink extends RichSinkFunction<DirectionEvalSecon
@Override @Override
public void invoke(DirectionEvalSecondMetric metric, Context context) throws Exception { public void invoke(DirectionEvalSecondMetric metric, Context context) throws Exception {
System.out.println("===== insert metric {" + metric + "}");
insertPstmt.setString(1, metric.getCrossId()); insertPstmt.setString(1, metric.getCrossId());
insertPstmt.setString(2, metric.getLineId()); insertPstmt.setString(2, metric.getLineId());
insertPstmt.setLong(3, metric.getCycleOrder()); insertPstmt.setLong(3, metric.getCycleOrder());
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment