Skip to content

Z
ZHHT-IRN-BD-ANALYSIS
Commit 8435a187 by 吴延飞
Options

修改按方向的任务的计算策略，改为connected stream来计算

parent ad30d1f1
Showing with 132 additions and 107 deletions
realtime/hologram-streaming/src/main/java/com/zhht/irn/functions/CalDirectionSecondMetricFunction.java
...@@ -10,7 +10,7 @@ import org.apache.flink.api.common.state.ListStateDescriptor; ...@@ -10,7 +10,7 @@ import org.apache.flink.api.common.state.ListStateDescriptor;
import org.apache.flink.api.common.state.MapState; import org.apache.flink.api.common.state.MapState;
import org.apache.flink.api.common.state.MapStateDescriptor; import org.apache.flink.api.common.state.MapStateDescriptor;
import org.apache.flink.configuration.Configuration; import org.apache.flink.configuration.Configuration;
import org.apache.flink.streaming.api.functions.co.ProcessJoinFunction; import org.apache.flink.streaming.api.functions.co.KeyedCoProcessFunction;
import org.apache.flink.util.Collector; import org.apache.flink.util.Collector;
import java.util.*; import java.util.*;
...@@ -24,7 +24,7 @@ import java.util.stream.Collectors; ...@@ -24,7 +24,7 @@ import java.util.stream.Collectors;
* @author 吴延飞 * @author 吴延飞
* @date 2022-11-15 11:23:00 * @date 2022-11-15 11:23:00
*/ */
public class CalDirectionSecondMetricFunction extends ProcessJoinFunction<Cycle, TravelEvent, DirectionEvalSecondMetric> { public class CalDirectionSecondMetricFunction extends KeyedCoProcessFunction<String, Cycle, TravelEvent, DirectionEvalSecondMetric> {
private MapState<String, List<TravelEvent>> mapState; private MapState<String, List<TravelEvent>> mapState;
...@@ -38,126 +38,146 @@ public class CalDirectionSecondMetricFunction extends ProcessJoinFunction<Cycle, ...@@ -38,126 +38,146 @@ public class CalDirectionSecondMetricFunction extends ProcessJoinFunction<Cycle,
} }
@Override @Override
public void processElement(Cycle cycle, TravelEvent travelEvent, ProcessJoinFunction<Cycle, TravelEvent, DirectionEvalSecondMetric>.Context ctx, Collector<DirectionEvalSecondMetric> out) throws Exception { public void processElement1(Cycle cycle, KeyedCoProcessFunction<String, Cycle, TravelEvent, DirectionEvalSecondMetric>.Context context, Collector<DirectionEvalSecondMetric> collector) throws Exception {
// 进入该方法表明，周期已经到了，而且周期已经向过车记录interval join了[-9,1]分钟的旅行事件数据 // 周期数据进入，开始计算
long cycleStart = Long.parseLong(cycle.getBeginDateTime());
// 先累计旅行事件数据到 mapState 中, 只累计到达和进入事件，其他类型计算用不上 long cycleEnd = Long.parseLong(cycle.getEndDateTime());
List<TravelEvent> cachedTravelEvent = mapState.get(cycle.getCrossCode()); List<TravelEvent> cachedTravelEvent = mapState.get(cycle.getCrossCode());
if(cachedTravelEvent == null) { Map<String, Boolean> globalRecordStopStatusMap = new HashMap<>();
cachedTravelEvent = new ArrayList<>(); // 计算每个车道的到达流量, 按秒计算
mapState.put(cycle.getCrossCode(), cachedTravelEvent); for(long start = cycleStart; start <= cycleEnd; start += 1000) {
} // 首先过滤出到当前周期的事件数据
if(travelEvent.getEventType() == EventType.ARRIVED || travelEvent.getEventType() == EventType.INCROSS long finalStart = start;
|| travelEvent.getEventType() == EventType.STOP || travelEvent.getEventType() == EventType.STARTUP) if (cachedTravelEvent != null) {
cachedTravelEvent.add(travelEvent); List<TravelEvent> currentEventList = cachedTravelEvent.stream().filter(event -> {
return Long.parseLong(event.getEventTime()) >= cycleStart && Long.parseLong(event.getEventTime()) <= finalStart;
// 确定触发条件 }).collect(Collectors.toList());
if(Long.parseLong(travelEvent.getEventTime()) - Constant.CYCLE_CALCULATE_LATENESS > Long.parseLong(cycle.getEndDateTime())) {
long cycleStart = Long.parseLong(cycle.getBeginDateTime()); // 根据车道分组
long cycleEnd = Long.parseLong(cycle.getEndDateTime()); Map<String, List<TravelEvent>> lineMap = new HashMap<>(); // key： 车道ID， value：对应到车道的截止到当前时刻的旅行事件记录集合
// 计算每个车道的到达流量, 按秒计算 currentEventList.stream().forEach(record -> {
for(long start = cycleStart; start <= cycleEnd; start += 1000) { List<TravelEvent> lineList = lineMap.get(record.getEventLineId());
// 首先过滤出到当前秒的事件数据 if(lineList == null) {
long finalStart = start; lineList = new ArrayList<>();
if (cachedTravelEvent != null) { lineMap.put(record.getEventLineId(), lineList);
List<TravelEvent> currentEventList = cachedTravelEvent.stream().filter(event -> { }
return Long.parseLong(event.getEventTime()) >= cycleStart && Long.parseLong(event.getEventTime()) <= finalStart; lineList.add(record);
}).collect(Collectors.toList()); });
// 根据车道分组
Map<String, List<TravelEvent>> lineMap = new HashMap<>(); // key： 车道ID， value：对应到车道的截止到当前时刻的旅行事件记录集合
currentEventList.stream().forEach(record -> {
List<TravelEvent> lineList = lineMap.get(record.getEventLineId());
if(lineList == null) {
lineList = new ArrayList<>();
lineMap.put(record.getEventLineId(), lineList);
}
lineList.add(record);
});
// 计算两个指标，一个 “发生到达，未进入的车辆数”， 一个“发生到达数”，粒度计算到车道级 // 计算两个指标，一个 “发生到达，未进入的车辆数”， 一个“发生到达数”，粒度计算到车道级
for(Map.Entry<String, List<TravelEvent>> entry : lineMap.entrySet()) { for(Map.Entry<String, List<TravelEvent>> entry : lineMap.entrySet()) {
List<TravelEvent> travelEventList = entry.getValue(); List<TravelEvent> travelEventList = entry.getValue();
// 发生到达的旅行记录集合Set // 发生到达的旅行记录集合Set
Set<String> recordIdSet = new HashSet<>(); Set<String> recordIdSet = new HashSet<>();
// 每个旅行对应的形成状态，key: 记录ID， value: 停车状态，进入该map表示停过车。 // 每个旅行对应的形成状态，key: 记录ID， value: 停车状态，进入该map表示停过车。
Map<String, Boolean> recordStopStatusMap = new HashMap<>(); Map<String, Boolean> recordStopStatusMap = new HashMap<>();
// 计算停车状态和停车次数（同一个周期，只算一次） // 计算停车状态和停车次数（同一个周期，只算一次）
// 先获取上一个周期所对应的处于停车状态的记录 // 先获取上一个周期所对应的处于停车状态的记录
for(String recordId : listState.get()) { for(String recordId : listState.get()) {
recordStopStatusMap.put(recordId, true); recordStopStatusMap.put(recordId, true);
} }
// 再处理本周期内发生的事件 // 严格一点，这里要对事件进行时间的排序, 因为乱序会导致车辆的状态不正确
for(int i = 0; i < travelEventList.size(); i ++) { Collections.sort(travelEventList, Comparator.comparing(TravelEvent::getEventTime));
// 严格一点，这里要对事件进行时间的排序，暂且不做
TravelEvent e = travelEventList.get(i); // 再处理本周期内发生的事件
if(e.getEventType() == EventType.STOP) { for(int i = 0; i < travelEventList.size(); i ++) {
recordStopStatusMap.put(e.getRecordId(), true); TravelEvent e = travelEventList.get(i);
} else if(e.getEventType() == EventType.STARTUP) { if(e.getEventType() == EventType.STOP) {
recordStopStatusMap.put(e.getRecordId(), false); recordStopStatusMap.put(e.getRecordId(), true);
} globalRecordStopStatusMap.put(e.getRecordId(), true);
System.out.println("you che 停了" + e.getRecordId());
} else if(e.getEventType() == EventType.STARTUP) {
recordStopStatusMap.put(e.getRecordId(), false);
globalRecordStopStatusMap.put(e.getRecordId(), false);
System.out.println("you che 启动了" + e.getRecordId());
} }
}
List<TravelEvent> lineArrivedList = travelEventList.stream().filter(record -> { List<TravelEvent> lineArrivedList = travelEventList.stream().filter(record -> {
if (record.getEventType() == EventType.ARRIVED) { if (record.getEventType() == EventType.ARRIVED) {
recordIdSet.add(record.getRecordId()); recordIdSet.add(record.getRecordId());
return true; return true;
} else { } else {
return false; return false;
} }
}).collect(Collectors.toList()); }).collect(Collectors.toList());
List<TravelEvent> lineArrivedAndInCrossList = travelEventList.stream().filter(record -> { List<TravelEvent> lineArrivedAndInCrossList = travelEventList.stream().filter(record -> {
return record.getEventType() == EventType.INCROSS && recordIdSet.contains(record.getRecordId()); return record.getEventType() == EventType.INCROSS && recordIdSet.contains(record.getRecordId());
}).collect(Collectors.toList()); }).collect(Collectors.toList());
Set<String> arrivedRecordIdSet = new HashSet<>(); Set<String> arrivedRecordIdSet = new HashSet<>();
Set<String> arrivedAndInCrossRecordIdSet = new HashSet<>(); Set<String> arrivedAndInCrossRecordIdSet = new HashSet<>();
for(TravelEvent travel: lineArrivedList) { for(TravelEvent travel: lineArrivedList) {
arrivedRecordIdSet.add(travel.getRecordId()); arrivedRecordIdSet.add(travel.getRecordId());
} }
for(TravelEvent travel: lineArrivedAndInCrossList) { for(TravelEvent travel: lineArrivedAndInCrossList) {
arrivedAndInCrossRecordIdSet.add(travel.getRecordId()); arrivedAndInCrossRecordIdSet.add(travel.getRecordId());
} }
int lineArrivedCount = arrivedRecordIdSet.size(); int lineArrivedCount = arrivedRecordIdSet.size();
int lineArrivedAndInCrossCount = arrivedAndInCrossRecordIdSet.size(); int lineArrivedAndInCrossCount = arrivedAndInCrossRecordIdSet.size();
int stopFlagCount = 0; int stopFlagCount = 0;
int stopCount = 0; int stopCount = 0;
for(Map.Entry<String, Boolean> en : recordStopStatusMap.entrySet()) { for(Map.Entry<String, Boolean> en : recordStopStatusMap.entrySet()) {
if(en.getValue() == true) stopFlagCount ++; if(en.getValue() == true) stopFlagCount ++;
stopCount ++; stopCount ++;
} }
DirectionEvalSecondMetric metric = new DirectionEvalSecondMetric();
metric.setCycleStart(cycleStart);
metric.setCycleEnd(cycleEnd);
metric.setCrossId(cycle.getCrossCode());
metric.setLineId(entry.getKey());
metric.setCurrentSecond(start);
metric.setCycleOrder(cycle.getCycleOrder());
metric.setAccArrivedFlow(lineArrivedCount);
metric.setAccQueueLength(lineArrivedCount - lineArrivedAndInCrossCount);
metric.setStopFlagCount(stopFlagCount);
metric.setAccStopCount(stopCount);
collector.collect(metric);
}
}
}
DirectionEvalSecondMetric metric = new DirectionEvalSecondMetric(); // 将本周期结束后还处于停车状态的车辆放到listState中
metric.setCycleStart(cycleStart); listState.clear();
metric.setCycleEnd(cycleEnd); for(Map.Entry<String, Boolean> en : globalRecordStopStatusMap.entrySet()) {
metric.setCrossId(cycle.getCrossCode()); System.out.println("当前周期的不同记录数：" + globalRecordStopStatusMap.size());
metric.setLineId(entry.getKey()); if(en.getValue() == true) {
metric.setCurrentSecond(start); System.out.println("stay stop status car record id:" + en.getKey());
metric.setCycleOrder(cycle.getCycleOrder()); listState.add(en.getKey());
metric.setAccArrivedFlow(lineArrivedCount); }
metric.setAccQueueLength(lineArrivedCount - lineArrivedAndInCrossCount); }
metric.setStopFlagCount(stopFlagCount);
metric.setAccStopCount(stopCount);
out.collect(metric); if(cachedTravelEvent != null) {
} List<TravelEvent> newOne = new ArrayList<>();
for(TravelEvent t : cachedTravelEvent) {
// 删除掉这个周期结束时间之间的数据
if(Long.parseLong(t.getEventTime()) >= Long.parseLong(cycle.getEndDateTime())) {
newOne.add(t);
} }
} }
cachedTravelEvent = newOne;
mapState.put(cycle.getCrossCode(), cachedTravelEvent);
}
}
// 计算后要清空 @Override
cachedTravelEvent.clear(); public void processElement2(TravelEvent travelEvent, KeyedCoProcessFunction<String, Cycle, TravelEvent, DirectionEvalSecondMetric>.Context context, Collector<DirectionEvalSecondMetric> collector) throws Exception {
List<TravelEvent> cachedTravelEvent = mapState.get(travelEvent.getCrossId());
if(cachedTravelEvent == null) {
cachedTravelEvent = new ArrayList<>();
mapState.put(travelEvent.getCrossId(), cachedTravelEvent);
} }
if(travelEvent.getEventType() == EventType.ARRIVED || travelEvent.getEventType() == EventType.INCROSS
|| travelEvent.getEventType() == EventType.STOP || travelEvent.getEventType() == EventType.STARTUP)
cachedTravelEvent.add(travelEvent);
} }
} }
realtime/hologram-streaming/src/main/java/com/zhht/irn/job/DirectionEvalJob.java
...@@ -4,15 +4,20 @@ import com.alibaba.fastjson.JSON; ...@@ -4,15 +4,20 @@ import com.alibaba.fastjson.JSON;
import com.zhht.irn.consts.Constant; import com.zhht.irn.consts.Constant;
import com.zhht.irn.entity.Cycle; import com.zhht.irn.entity.Cycle;
import com.zhht.irn.entity.TravelEvent; import com.zhht.irn.entity.TravelEvent;
import com.zhht.irn.entity.metric.DirectionEvalSecondMetric;
import com.zhht.irn.functions.CalDirectionSecondMetricFunction; import com.zhht.irn.functions.CalDirectionSecondMetricFunction;
import com.zhht.irn.sink.DirectionEvalSecondSink; import com.zhht.irn.sink.DirectionEvalSecondSink;
import com.zhht.irn.source.CycleMockSource;
import com.zhht.irn.source.TravelEventMockSource;
import com.zhht.irn.utils.FlinkUtils; import com.zhht.irn.utils.FlinkUtils;
import com.zhht.irn.utils.StringUtils; import com.zhht.irn.utils.StringUtils;
import org.apache.flink.api.common.eventtime.WatermarkStrategy; import org.apache.flink.api.common.eventtime.WatermarkStrategy;
import org.apache.flink.streaming.api.datastream.DataStream; import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.datastream.KeyedStream; import org.apache.flink.streaming.api.datastream.KeyedStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.api.functions.co.KeyedCoProcessFunction;
import org.apache.flink.streaming.api.windowing.time.Time; import org.apache.flink.streaming.api.windowing.time.Time;
import org.apache.flink.util.Collector;
import org.slf4j.Logger; import org.slf4j.Logger;
import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory;
...@@ -38,6 +43,7 @@ public class DirectionEvalJob { ...@@ -38,6 +43,7 @@ public class DirectionEvalJob {
DataStream<Cycle> cycleStream = DataStream<Cycle> cycleStream =
FlinkUtils.createKafkaStream(args, env, Constant.CYCLE_TOPIC_NAME, "DirectionEvalJob") FlinkUtils.createKafkaStream(args, env, Constant.CYCLE_TOPIC_NAME, "DirectionEvalJob")
.map(record -> { .map(record -> {
System.out.println(record);
return JSON.parseObject(record, Cycle.class); return JSON.parseObject(record, Cycle.class);
}); });
...@@ -45,6 +51,7 @@ public class DirectionEvalJob { ...@@ -45,6 +51,7 @@ public class DirectionEvalJob {
DataStream<TravelEvent> travelEventStream = DataStream<TravelEvent> travelEventStream =
FlinkUtils.createKafkaStream(args, env, Constant.TRAVEL_EVENT_TOPIC_NAME, "DirectionEvalJob") FlinkUtils.createKafkaStream(args, env, Constant.TRAVEL_EVENT_TOPIC_NAME, "DirectionEvalJob")
.map(record -> { .map(record -> {
System.out.println(record);
return JSON.parseObject(record, TravelEvent.class); return JSON.parseObject(record, TravelEvent.class);
}).filter(event -> { }).filter(event -> {
return StringUtils.isNotEmpty(event.getEventLineId()); return StringUtils.isNotEmpty(event.getEventLineId());
...@@ -67,8 +74,7 @@ public class DirectionEvalJob { ...@@ -67,8 +74,7 @@ public class DirectionEvalJob {
// 周期数据join 旅行事件 // 周期数据join 旅行事件
cycleStream.assignTimestampsAndWatermarks(cycleWatermarkStrategy) cycleStream.assignTimestampsAndWatermarks(cycleWatermarkStrategy)
.keyBy(record -> record.getCrossCode()) .keyBy(record -> record.getCrossCode())
.intervalJoin(travelEventKeyedStream) .connect(travelEventKeyedStream)
.between(Time.minutes(-9), Time.minutes(1))
.process(new CalDirectionSecondMetricFunction()) .process(new CalDirectionSecondMetricFunction())
.addSink(new DirectionEvalSecondSink()).setParallelism(4); .addSink(new DirectionEvalSecondSink()).setParallelism(4);
......
realtime/hologram-streaming/src/main/java/com/zhht/irn/sink/DirectionEvalSecondSink.java
...@@ -43,7 +43,6 @@ public class DirectionEvalSecondSink extends RichSinkFunction<DirectionEvalSecon ...@@ -43,7 +43,6 @@ public class DirectionEvalSecondSink extends RichSinkFunction<DirectionEvalSecon
@Override @Override
public void invoke(DirectionEvalSecondMetric metric, Context context) throws Exception { public void invoke(DirectionEvalSecondMetric metric, Context context) throws Exception {
System.out.println("===== insert metric {" + metric + "}");
insertPstmt.setString(1, metric.getCrossId()); insertPstmt.setString(1, metric.getCrossId());
insertPstmt.setString(2, metric.getLineId()); insertPstmt.setString(2, metric.getLineId());
insertPstmt.setLong(3, metric.getCycleOrder()); insertPstmt.setLong(3, metric.getCycleOrder());
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment